diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 8d03ad9..0000000 --- a/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -__pycache__ -*.egg-info -.sesskey -examples/fasthtml/adv_app/data diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 03dad77..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Pytest", - "type": "debugpy", - "request": "launch", - "cwd": "${workspaceFolder}", - "module": "pytest", - "console": "integratedTerminal" - } - ] -} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 73cf8e4..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "python.linting.ruffEnabled": true, - "python.linting.enabled": true, - - // Enable Pylance - "python.languageServer": "Pylance", - - // Enable type checking mode and set it to strict - "python.analysis.typeCheckingMode": "strict", - - // Configure specific diagnostic settings - "python.analysis.diagnosticSeverityOverrides": { - // My desired configuration is for Pylance to treat unknown symbols - // as an error only when it can be determined they are not imported - // anywhere, even through wildcard imports. - // - // But alas, Pyright cannot track definitions with wildcard imports. - // - // Also, I don't know how to configure Pylance to report unknown - // symbols as errors ONLY when all imports are known. - // - // so atm one must choose: - // 1. treat wildcard imports AND unknown symbols as an error - // 2 treat neither as an error. - - // "reportWildcardImportFromLibrary": "none", // Ignore wildcard imports - "reportUnknownParameterType": "none", // Ignore unknown parameter types - "reportUnknownVariableType": "none", // Ignore unknown variable types - "reportUnknownMemberType": "none", // Ignore unknown member types - "reportUnknownArgumentType": "none", // Ignore unknown argument types - "reportUnknownLambdaType": "none", // Ignore unknown lambda types - "reportUnknownType": "none", // Ignore unknown types - // - // TRULY HARMLESS - // - // unused symbols - "reportUnusedImport": "none", - "reportUnusedVariable": "none", - "reportUnusedFunction": "none", - // issing type annotations - "reportMissingTypeStubs": "none", // Ignore missing type stubs - "reportMissingImports": "none", // Ignore missing imports - "reportMissingParameterType": "none", - "reportMissingTypeArgument": "none", // Ignore missing type arguments - "reportUntypedFunctionDecorator": "none", - }, - - // Exclude specific files or directories from Pylance analysis - "python.analysis.exclude": [ - "examples/**", - ], - "python.testing.pytestArgs": [ - "tests" - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, -} \ No newline at end of file diff --git a/CNAME b/CNAME new file mode 100644 index 0000000..fd12ded --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +www.pythonrunscript.org diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 7088f60..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2023 Alexis Gallagher - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index 80d8a88..0000000 --- a/README.md +++ /dev/null @@ -1,205 +0,0 @@ -# ./pythonrunscript - -`pythonrunscript` lets you - -- define and run **single-file python scripts** -- which **automatically install needed dependencies into isolated environments** -- while **using normal external dependencies** from pip or conda - -## How to use pythonrunscript - -pythonrunscript lets you declare your script’s dependencies in the script itself. - -This is in order to let you build and use scripts that *just run*, with no environment setup. - -To declare your script’s pip dependencies, add a *pip requirements block*. This is a block of comments which embeds a pip requirements.txt file into your script, using Python's official PEP723 syntax for [inline script metadata](https://packaging.python.org/en/latest/specifications/inline-script-metadata/#inline-script-metadata). For example, the script below includes a pip requirement block which defines a one-line requirements.txt file, declaring a dependency on version 4.66.4 of the pip package `tqdm`. - -``` python -#!/usr/bin/env pythonrunscript -# -# /// pythonrunscript-requirements-txt -# tqdm==4.66.4 -# /// -# -from tqdm import tqdm -import sys - -print("Hello, I depend on the tqdm package!") -for i in tqdm(range(10000)): - pass -print("Phew. That was fun!") -``` - -The first time you run this script, pythonrunscript will extract the embedded requirements.txt file, create an isolated environment, install the dependencies listed in the requirements.txt, and run the script from that environment. In later runs, it will just re-use that environment. To create the environment it uses venv, which is built into python, so the only requirement to use pythonrunscript is that python itself is already installed. - -To run your script, call it with `pythonrunscript hello.py`. Alternatively, in order to execute your script directly with `./hello.py`, change your script’s first line (the "shebang" line) to the following: `#!/usr/bin/env pythonrunscript`. In both cases, if you later remove the inline metadata comment block, pythonrunscript will run the script normally since it just passes the script to `python3`. In other words, pythonrunscript does not depend on finding metadata. - -Conversely, if you use the normal interpreter to call a script with inline metadata, by doing `python3 hello.py`, then python3 will ignore the special metadata comments just like other comments, and run the script as usual. - -The upshot is, you can use pythonrunscript to run a normal python script, but if you add pythonrunscript metadata to a script, it can still be run normally by someone who has never heard of pythonrunscript. - -## Installation and Requirements - -Here are two ways to install pythonrunscript: - -1. Do `python3 -m pip install pythonrunscript`, which will install `pythonrunscript` into your PATH. -2. Or, manually copy the file `pythonrunscript` from the `bin/` directory of this repo into a directory named in your `PATH` variable. - -`pythonrunscript` runs on Linux and macOS, and needs Python 3.9.6 or higher (which means it can run using the system Python built into macOS Sonoma). It requires conda only if you want to run scripts which themselves require conda. - -## How can I check what it will do? If it worked? What happened exactly? - -Run pythonrunscript in `--dry-run` mode to get a preview of how it parses your file and the actions it *would* take. - -Run with `--verbose` to hear copious commentary, and to see all output of subprocess commands. Normally, pythonrunscript only prints logging in case of an installation error. It always saves all generated output in a logs directory, as well as saving an `environment-resolved.yml` and a `pip-list.txt` which reflect the exact state of the environment after creation. These files are saved in the script’s project directory, in the cache, which is revealed by running with the `--show-cache` command. Running with `--clean-cache` moves all cached directories to the OS’s temporary directory for automatic disposal. - -## What, why would I want this? - -(For a longer explanation, check out [dev chat with me (Alexis), Jeremy Howard, and Johno Whitaker](https://www.youtube.com/watch?v=IECcEbXlIl8).) - -If none of the above appeals to you, perhaps you never had problems with python dependency management before? (Congrats!) But if the dilemma is totally alien to you and you’re curious, here is the issue. - -Suppose you’re writing a single-file Python module, i.e., a script, `hello.py`, like this: - -``` python -#!/usr/bin/env python3 -print("Hello, world") -``` - -Running it is easy. You can run the script by running `./hello.py` and it will just work. Hurray! - -But then you need some functionality outside of the standard library, like yaml-parsing from an external dependency like PyYAML. So now you need to install that dependency. This is where pain starts. - -Where do you install it? In your system’s python? Should you use sudo? Or in a managed environment? Managed how? With venv? With virtualenv? With conda? With nix, docker, something else? So should you install that manager first? Where? Do you need requirements.txt? Pipenv? Poetry? pyproject.toml? Something else? These questions all have answers but they are *boring and painful*. Once you distribute your script to someone else, the pain falls on them too. Can you just give them the file? No, since now you need to give them your script and also, perhaps, a mini-essay on how to answer those questions. If you are lucky enough (?) to be a seasoned expert in python dependency management and to associate only with other such experts, then you may not see the problem here. But it is real outside such orbits. - -(soapbox mode…) - -There is a reason that the Go and rust communities excel at shipping so many charming and effective command line applications. Why? Because their language toolchains have good support for shipping a static linked executable – that is to say, a single damn file which Just Works. Python would be truly great for scripting, except for the fact that it lacks this power, which makes it impractical to give your script to someone else who just wants to solve their problem with the minimum possible fuss. Sad. - -Hopefully, this tool helps. The dependency syntaxes which it uses are exactly the syntaxes already supported by pip and conda. There is nothing exotic here. So you can do everything you can normally do with pip, such as specify dependencies not only as public PyPI packages but also as [URLs to GitHub repos](https://pip.pypa.io/en/stable/topics/vcs-support/). And you can do everything you can normally do with conda, such as specify the version of Python itself, or install vast wads of awkward binary code from dedicated channels like nvidia’s. - -(soapbox off…) - -## Wait, but I *enjoy* manually keeping environments nice and tidy, even for tiny little scripts! - -Great, then, this is not for you. 😇 Cultivate your garden. - -## What about conda dependencies? - -Some popular dependencies, like [cudatoolkit](https://developer.nvidia.com/cuda-toolkit), cannot be installed by pip but need conda. - -To specify conda dependencies, you must add a *conda environment.yml block* or a *conda_install_specs_.txt block*. You may use this instead of, or in addition to, a pip requirements.txt block. They use two other types of inline metadata. The environment block is intoduced by ```` /// pythonrunscript-environment-yml ```` and it should contain an environment.yml file verbatim. - -The install spec block is introduced by ```` /// pythonrunscript-conda-install-specs-txt ```` block, and it should introduce conda install specs. A conda install spec is simply the string passed to the `conda install` command. Conda documents the [exact syntax for a conda install spec](https://conda.io/projects/conda/en/latest/user-guide/concepts/pkg-search.html), which only requires naming the conda package, but also allows specifying the version, the channel, or specific builds. - -For instance, this script uses a conda environment.yml block: - -``` python -#!/usr/bin/env pythonrunscript -# -# /// pythonrunscript-environment-yml -# dependencies: -# - python=3.10 -# - numpy -# /// -# -import numpy -import sys - -print("I depend on numpy") -``` - -And this script uses a conda install specs block, for the same dependencies: - -``` python -#!/usr/bin/env pythonrunscript -# -# /// pythonrunscript-conda-install-specs-txt -# python=3.10 -# numpy -# /// -# -from tqdm import tqdm -import sys - -print("Hello, I depend on the tqdm package!") -for i in tqdm(range(10000)): - pass -print("Phew. That was fun!") -``` - -Do you really need conda? Maybe not! If you don’t specify conda dependencies, pythonrunscript won’t use conda. - -But you might need conda if you want to use conda-only dependencies, to specify the version of python itself, or to use packages outside of the Python ecosystem. This [weights & biases blog post](https://wandb.ai/wandb_fc/pytorch-image-models/reports/A-faster-way-to-get-working-and-up-to-date-conda-environments-using-fastchan---Vmlldzo2ODIzNzA) explains the situation well, and [this script will install conda](https://github.com/fastai/fastsetup/blob/master/setup-conda.sh) on all the platforms.) - -## So what is PEP-723? - -Here is the situation. - -[PEP-723](https://packaging.python.org/en/latest/specifications/inline-script-metadata/#inline-script-metadata) defines the official syntax for embedding any kind of metadata in a Python script. It also shows _one_ particular implementation of that standard, with the "script" type of metadata. This is a metadata block introduced by the opening delimiter ```` /// script ````. Here's an example of a script with such a block: - -``` python -# /// script -# requires-python = ">=3.11" -# dependencies = [ -# "numpy", -# ] -# /// -# -import numpy -import sys - -print("I depend on numpy") -``` - -As you can see, the "script" metadata type requires a TOML-like syntax, and it lets you declare pip dependencies (like `numpy`) as well as a required version of python (like `3.11`). When a Python script has this "script" type metadata, a compliant tool can use it to run the Python script as a single-file script, by reading the metadata, fetching the dependencies, and running the script in an isolated environment containing those dependencies. - -`pythonrunscript` does this. It recognizes the "script" metadata type. If the "script" metadata only declares pip dependencies, pythonrunscript just uses pip to install them. If it also declares a required python version, then pythonrunscript uses conda to manage the python version itself. - -But in addition to the "script" type metadata, PEP-723 also specifies how tools can define _other_ metadata types. pythonrunscript also defines the following types: - -- `pythonrunscript-requirements-txt`, which lets you embed a standard `requirements.txt` file. -- `pythonrunscript-conda-install-specs-txt`, which lets you embed a list of normal conda install specs. -- `pythonrunscript-environment-yml`, which lets you embed a normal `environment.yml` file. - -In other words, pythonrunscript implements the PEP723 syntax for embedding inline metadata, implements the recommended "script" metadata type, and also implements PEP723's mechanism for defining particular types of inline metadata. It defines three types which represent embedding the popular dependency file types which you might be using already. - -(In addition, pythonrunscript also supports a legacy syntax for embedding metadata in terms of markdown-style code fences. I designed it to use this syntax before I realized that PEP723 existed! It is now deprecated.) - -## Comparable solutions - -What else is out there, to solve this problem of "I just want a single-file I can deploy easily?" - -### `uv` - -Another tool which supports PEP732 is [uv](https://docs.astral.sh/uv/guides/scripts/#declaring-script-dependencies). It is is excellent and I would recommend it. But it only implements the "script" metadata type. - -Compared to pythonrunscript, I'd say that `uv` is better choice if: - -- You don't mind rewriting your dependencies as TOML fragments. -- You don't need conda-only dependencies. -- You value `uv`'s superior speed. - -On the other hand, pythonrunscript might be a better choice if: - -- You want to describe your dependencies using formats you already use, like requirements.txt, conda install specs, or environment.yml files. -- You need certain dependencies available only on conda, such as executables like `ffmpeg` for video transcoding, `poppler` for processing PDFs, or various heavier ML components. -- You prefer a tool which requires only python, and which is small enough to understand and hack on. (pythonrunscript itself is less than 600 lines of Python.) - -### pip.wtf - -I haven't tried this but it seems like an inspired solution if you want your scripts to have zero install-time dependencies, not even pythonrunscript itself. - -It's just eight lines of code which defines a function which you then add directly to your script. In your script, calling this function will install the script's dependencies and run it using them. Beautiful. - -However, it might not be approriate when you want your script to run like a normal script by default, e.g., not to install dependencies when you just run it with the normal interpreter, or when you need conda because your project depends on conda-only dependencies. - -### Inspiration and related projects - -- [pip.wtf](https://pip.wtf), as mentioned above. -- [uv](https://docs.astral.sh/uv/guides/scripts/#declaring-script-dependencies), as mentioned above. -- [PEP-723](https://peps.python.org/pep-0723/) defines a standard, general syntax for different types of *inline script metadata*. This lets you embed metadata into a script's comments (just like pythonrunscript does). -- [swift-sh](https://github.com/mxcl/swift-sh) does the same trick for Swift -- [rust-script](https://rust-script.org) does the same trick for rust -- [scriptisto](https://github.com/igor-petruk/scriptisto) does this for any language, but at the cost of shifting the complexity into the comment block. - diff --git a/bin/pythonrunscript b/bin/pythonrunscript deleted file mode 100755 index ab6eb54..0000000 --- a/bin/pythonrunscript +++ /dev/null @@ -1,585 +0,0 @@ -#!/usr/bin/env python3 -# python>=3.9.6 -import sys, re, os, subprocess, hashlib, logging, platform, argparse, tempfile, shutil, uuid, textwrap, shlex -from abc import ABC -from enum import Enum -from typing import NoReturn, Union - -logging.basicConfig(level=logging.WARNING) -Log = Enum('Log', ['SILENT','ERRORS','VERBOSE']) - -version_str = "0.2.0 (2024-10-07)" - -def main(): - parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - parser.description='Runs a script, installing its dependencies in a cached, isolated environment' - parser.add_argument('--dry-run', action='store_true', help='report what pythonrunscript would do, without writing any files') - parser.add_argument('--version', action='store_true', help='prints current version') - parser.add_argument('--verbose', action='store_true', help='comments on actions and prints all outputs and errors') - parser.add_argument('--show-cache', action='store_true', help='print the cache directory of script environments') - parser.add_argument('--clean-cache', action='store_true', help='purges all pythonrunscript environments') - parser.add_argument('script', nargs='?', default=None, help='path to the script to run') - parser.add_argument('arguments', nargs=argparse.REMAINDER, help='optional arguments to be passed to that script') - parser.epilog=''' pythonrunscript runs Python scripts, installing their dependencies. - - That is, it will automatically install all dependencies in a cached, isolated - environment dedicated to your script, and run it in that environment. - - To do this it looks in your script for a comment declaring dependencies - using the inline metadata syntax defined in PEP723. This syntax uses a "type" - tag to indicate the type of dependency meatadata. - - You use the type tag pythonrunscript-requirements-txt in order to embed an - ordinary requirements.txt file, like so: - - # /// pythonrunscript-requirements-txt - # tqdm==4.66.4 - # /// - - You can also use the type pythonrunscript-environment-yml to embed an - environment.yml file, or pythonrunscript-conda-install-specs-txt to embed a list - of conda install specs. A conda install spec is just the syntax for arguments - passed to `conda install`. It is documented here: - https://conda.io/projects/conda/en/latest/user-guide/concepts/pkg-search.html - - To run a script with conda dependencies or which specifies the python version, - you must already have conda installed. - - Finally, pythonrunscript also supports the "script" type, which is the TOML-like - syntax given as an initial example in PEP723. It looks like so: - - # /// script - # dependencies = [ - # "tqdm==4.66.4", - # ] - # /// - - You can explicitly call pythonrunscript to run your script by doing - `pythonrunscript myscript.py`. Or you can change your script's first line to use - pythonrunscript as an interpreter (setting its first line to - "#!/usr/bin/env pythonrunscript"), and then execute your script directly. - - pythonrunscript requires Python 3.9.6 and later, which ships with macOS Sonoma. - Since it creates isolated environments, you can run it using the system's - Python without corrupting the system. It also works on Linux. Untested on Windows. - ''' - args = parser.parse_args() - - if sys.version_info < (3,9,6): - print(f"I am being interpreted by Python version:\n{sys.version}") # pyright: ignore - print("But I need python version 3.9.6 or higher.\nAborting.") # pyright: ignore - exit(1) # pyright: ignore - elif args.version: - print(f"pythonrunscript {version_str}") - exit(0) - elif are_dependencies_missing(): - print(f"I am being run on the platform {platform.system()} and") - print("I cannot find the required external commands bash and tee, so") - print("I probably will not work. Aborting.") - exit(1) - elif args.show_cache: - print_base_dirs() - exit(0) - elif args.clean_cache: - pseudo_erase_dir(cache_base()) - exit(0) - elif args.script is None: - print(f"Error: pythonrunscript must be called with either the path to a script, --show-cache, --clean-cache, or --help.") - exit(1) - else: - script = args.script - - if not os.path.exists(script): - print(f"Error: did not find the script {script}. Nothing to do.") - exit(1) - - if args.dry_run: - print("## This is a dry run. No files will be written.\n") - - if args.verbose: - logging.info("Running in verbose") - - proj = Project.make_project(script,args.verbose,args.dry_run) - - if args.dry_run: - perform_dry_run(proj) - exit(0) - - if isinstance(proj, ProjectNoDeps): - logging.info("No pip block and no conda block detected. Running directly") - if args.verbose: - print("## No dependencies needed. Running the script directly") - proj.run(args.arguments) - elif not proj.exists(): - logging.info("Needs an environment but none exists. Creating it") - creation_success = proj.create() - if not creation_success: - trashed_env = pseudo_erase_dir(proj.project_path) - print(f"## Creating a managed environment failed. Moved the broken environment to {trashed_env}",file=sys.stderr) - exit(1) - else: - logging.info(f"Found pre-existing project dir: {proj.project_path}") - if args.verbose: - print(f"## Found pre-existing project dir: {proj.project_path}") - # assert: proj exists - if args.verbose: - print("## Running the script using the project directory environment") - proj.run(args.arguments) - -def perform_dry_run(proj): - "Describes actions for exists(), creates(), runs()" - print("## After parsing, I would take these actions.\n") - if isinstance(proj, ProjectNoDeps): - print(f"## No project directory is needed since parsing found no dependencies in the file {proj.script}\n") - print(f"## In a live run, I would run the script using the first python3 in your PATH.\n") - print_python3_path() - return - elif not proj.exists(): - print(f"## The needed project directory does not exist so I would create this project directory:\n{proj.project_path}\n") - print(f"## Inside, I would create this environment directory:\n{proj.envdir}\n") - if proj.conda_envyml: - print(f"## I found an environment.yml dependency block, so I'd use that.") - print(f"## To install conda dependencies, I'd execute this conda environment creation command:\n") - install_env_f = os.path.join(proj.project_path,'environment.yml') - print(f"\t{make_conda_install_yml_command(proj.project_path,install_env_f)}\n") - elif proj.conda_specs: - print(f"## I found a conda_install_specs.txt block, so I'd use that.") - print(f"## To install conda dependencies, I'd execute this conda install command:") - install_spec_f = os.path.join(proj.project_path,'conda_install_specs.txt') - print(f"{make_conda_install_spec_command(proj.project_path, install_spec_f)}\n") - if proj.pip_requirements: - print(f"## To install pip dependencies, I'd execute the following pip command:") - print(f"python3 -m pip install -r {os.path.join(proj.envdir,'requirements.txt')}\n") - print_python3_path() - print(f"## At this point, this project directory would exist:\n{proj.project_path}\n") - print(f"## I'd run using this env dir:\n{proj.envdir}\n") - return - -def parse_dependencies(script, verbose=False) -> tuple[str,str,str,str]: - "Parses script and returns any conda or pip dep blocks" - LT = Enum('LT', [ - 'BEG_SCRIPT_YML', - 'BEG_CONDA_SPEC_YML','BEG_CONDA_ENV_YML','BEG_PIP_YML','END_YML', - 'BEG_CONDA_SPEC','BEG_CONDA_ENV','BEG_PIP','END', - 'TEXT']) - - p = { - LT.BEG_SCRIPT_YML : r"^# /// script$", - LT.BEG_CONDA_SPEC_YML : r"^# /// pythonrunscript-conda-install-specs-txt$", - LT.BEG_CONDA_ENV_YML : r"^# /// pythonrunscript-environment-yml$", - LT.BEG_PIP_YML : r"^# /// pythonrunscript-requirements-txt$", - LT.END_YML : r"^# ///$", - LT.BEG_CONDA_SPEC : r"^# ```conda_install_specs.txt$", - LT.BEG_CONDA_ENV : r"^# ```environment.yml$", - LT.BEG_PIP : r"^# ```requirements.txt$", - LT.END : r"^# ```$", - LT.TEXT : r"^#(| .*)$", - } - - boxed_pip_block = [''] - boxed_conda_spec_block = [''] - boxed_conda_env_block = [''] - - block_type_content_delimiters = [ - ('script',[], [(LT.BEG_SCRIPT_YML,LT.END_YML)]), - ('requirements.txt', boxed_pip_block, [(LT.BEG_PIP_YML,LT.END_YML), - (LT.BEG_PIP,LT.END),]), - ('conda_install_specs.txt', boxed_conda_spec_block, [(LT.BEG_CONDA_SPEC_YML,LT.END_YML), - (LT.BEG_CONDA_SPEC,LT.END),]), - ('environment.yml', boxed_conda_env_block, [(LT.BEG_CONDA_ENV_YML,LT.END_YML), - (LT.BEG_CONDA_ENV,LT.END),]), - ] - - def make_block_pattern(begend:tuple[LT,LT]) -> str: - (beg,end) = begend - return rf"(?m:{p[beg]}\s(?P({p[LT.TEXT]}\s)+?){p[end]}(?:\s)?)" - - def extract_content(match): - return ''.join( - line[2:] if line.startswith('# ') else line[1:] - for line in match.group('content').splitlines(keepends=True) - ) - - # collect all comment lines starting with "# " or equalling "#" - # transforming to strip # prefix - comments = open(script,'r').read() - if verbose: - print(f"## Parsing this script for dependencies:\n{script}") - print() - - for (block_type, boxed_content, begend_pairs) in block_type_content_delimiters: - for begend in begend_pairs: - block_pattern = make_block_pattern(begend) - match = re.compile(block_pattern).search(comments) - if match: - if verbose: - print(f"### Extracted this {block_type} comment block:\n") - s = '\n'.join([(line[2:] if len(line)>1 else "") - for line in match.group('content').split('\n')]) - print(textwrap.indent(s,'\t')) - print() - if block_type == 'script': - (pip_env, conda_env) = parse_script_toml(extract_content(match)) - boxed_pip_block[0] = pip_env - boxed_conda_spec_block[0] = conda_env - break - else: - boxed_content[0] = extract_content(match) - break - - hash = hashlib.md5() - hash.update(boxed_pip_block[0].encode('utf-8')) - hash.update(boxed_conda_env_block[0].encode('utf-8')) - hash.update(boxed_conda_spec_block[0].encode('utf-8')) - return (hash.hexdigest(), boxed_pip_block[0], boxed_conda_env_block[0], boxed_conda_spec_block[0]) - -def tomlconfig_to_pip_conda(toml_config) -> tuple[str,str]: - "From a TOML dict, to (pip reqs, conda python spec)" - if 'requires-python' in toml_config: - conda_python_install_spec = f"python{toml_config['requires-python']}" - else: - conda_python_install_spec = '' - if 'dependencies' in toml_config: - pip_reqs = '\n'.join(toml_config['dependencies']) + '\n' - else: - pip_reqs = '' - return (pip_reqs,conda_python_install_spec) - -def parse_script_toml(toml_str) -> tuple[str,str]: - """ - From script TOML text, to (pip_reqs,conda python spec). - - This parses the TOML fragment in a PEP723 metadata block where TYPE=script. - Uses a limited custom parser to neeed only Python 3.9.6 and zero deps. - """ - toml_str = re.sub(r'#.*$', '', toml_str, flags=re.MULTILINE) - config = {} - requires_python_match = re.search(r'requires-python\s*=\s*"([^"]*)"', toml_str) - if requires_python_match: - config['requires-python'] = requires_python_match.group(1) - dependencies_match = re.search(r'dependencies\s*=\s*\[(.*?)\]', toml_str, re.DOTALL) - if dependencies_match: - dependencies_str = dependencies_match.group(1) - dependencies = re.findall(r'"([^"]*)"', dependencies_str) - config['dependencies'] = dependencies - return tomlconfig_to_pip_conda(config) - -class Project(ABC): - @staticmethod - def make_project(script:str, verbose:bool, dry_run:bool): - (dep_hash, pip_requirements, conda_envyml, conda_specs ) = parse_dependencies(script,verbose or dry_run) - if conda_envyml or conda_specs: - logging.info("dep block implies script will need conda for an environment.yml or conda_specs installation") - return ProjectConda(script, dep_hash, pip_requirements, conda_specs, conda_envyml, verbose) - elif pip_requirements: - logging.info("dep block implies script will need only venv + pip") - return ProjectPip(script, dep_hash, pip_requirements,conda_specs, conda_envyml, verbose) - else: - logging.info("no valid dep block found. no environment needed") - return ProjectNoDeps(script,dep_hash, pip_requirements,conda_specs, conda_envyml, verbose) - - def __init__(self, script:str, dep_hash, pip_requirements:str, conda_specs:str, conda_envyml:str, verbose:bool): - assert isinstance(conda_specs,str), "Bad input" - self.script = script - self.dep_hash = dep_hash - self.pip_requirements = pip_requirements - self.conda_specs = conda_specs - self.conda_envyml = conda_envyml - self.verbose = verbose - - @property - def project_path(self): - "path to the project dir" - return os.path.join( cache_base(), self.dep_hash ) - @property - def envdir(self) -> str: - "for pip projects, the venv dir. for conda, the prefix dir" - return "" - @property - def interpreter(self) -> str: - return os.path.join( self.envdir, 'bin','python3') - def exists(self) -> bool: - return False - def create(self) -> bool: - "False if creation failed, maybe leaving self.project_path in a non-runnable state" - return True - def run(self, args) -> NoReturn: - run_script(self.interpreter,self.script,args) - -def log_level_for_verbose(v:bool) -> Log: - return Log.VERBOSE if v else Log.ERRORS - -class ProjectPip(Project): - @property - def envdir(self): return os.path.join( self.project_path, 'venv' ) - def exists(self): return os.path.exists( self.project_path ) - def create(self): - return create_venv(self.project_path, self.envdir, - self.pip_requirements, - log_level_for_verbose(self.verbose)) - - -class ProjectConda(Project): - @property - def envdir(self): return os.path.join( self.project_path, 'condaenv' ) - def exists(self): return os.path.exists( self.project_path ) - def create(self): - return setup_conda_prefix(self.project_path, self.envdir, - self.conda_envyml, - self.conda_specs, - self.pip_requirements, - log_level_for_verbose(self.verbose)) - def run(self, args) -> NoReturn: - conda_run_script(self.interpreter,self.script,args,self.envdir) - -class ProjectNoDeps(Project): - def exists(self): return True - def create(self): return True - @property - def interpreter(self): - return sys.executable - -def run_with_logging(command:Union[str,list],proj_dir,out_f,err_f,verbosity): - ''' - Runs command. Logs and maybe streams stdout and stderr. - - verbosity=Log.SILENT: log out and err. Report errors later - verbosity=Log.VERBOSE: log and stream out and err. - ''' - log_dir = os.path.join(proj_dir,"logs") - os.makedirs(log_dir,exist_ok=True) - out_f = os.path.join(log_dir, os.path.basename(out_f)) - err_f = os.path.join(log_dir, os.path.basename(err_f)) - if isinstance(command,list): - command = shlex.join(command) - if verbosity == Log.SILENT: - command += f' 2>> "{err_f}"' - command += f' 1>> "{out_f}"' - elif verbosity == Log.ERRORS: - command += f' 2> >(tee -a "{err_f}")' - command += f' 1>> {out_f}' - elif verbosity == Log.VERBOSE: - command += f' 2> >(tee -a "{err_f}")' - command += f' 1> >(tee -a "{out_f}")' - else: - assert True, "unreachable" - - cp = subprocess.run(command, - shell=True, - executable=shutil.which('bash')) - did_succeed = (cp.returncode == 0) - - if (verbosity, did_succeed) == (Log.VERBOSE,True): - print(f"## This command completed successfully:\n\t{command}") - elif (verbosity, did_succeed) == (Log.VERBOSE,False): - print(f"## This command failed:\n\t{command}\n", file=sys.stderr) - print(f"## Standard error output was printed above\n", file=sys.stderr) - print(f"## Logs may be found in:\n\t{proj_dir}/logs", file=sys.stderr) - elif (verbosity, did_succeed) == (Log.ERRORS,True): - pass - elif (verbosity, did_succeed) == (Log.ERRORS,False): - print(f"## Error encountered trying to run this command:\n\t{command}", file=sys.stderr) - print(f"## Logs may be found in:\n\t{proj_dir}/logs\n", file=sys.stderr) - print(f"## This is the contents of the stderr:\n") - with open(err_f,"r") as f: - print(f.read()) - else: - pass - - return did_succeed - - -def create_conda_prefix(proj_dir,condaprefix_dir:str,log_level:Log): - success = run_with_logging(f'conda create --quiet --yes --prefix "{condaprefix_dir}"', - proj_dir, - "conda_create.out","conda_create.err", - log_level) - if success: - return True - else: - print("## Errors trying to create conda prefix directory",file=sys.stderr) - return False - - -def pseudo_erase_dir(path): - "Pseudo-erases a project dir by moving it to the temporary dir" - logging.info(f"Moving {path} to {trash_base()}") - dst = os.path.join( trash_base(), os.path.basename(path), str(uuid.uuid4()) ) - return shutil.move(path, dst ) - - -def install_pip_requirements(proj_dir, pip_requirements, interpreter, log_level:Log) -> bool: - reqs_path = os.path.join(proj_dir,'requirements.txt') - with open(reqs_path, 'w') as f: - f.write(pip_requirements) - - success = run_with_logging([interpreter, "-m", "pip", "install", "-r", reqs_path], - proj_dir, - "pip_install.out","pip_install.err", - log_level) - if success: - with open(os.path.join(proj_dir,"piplist.txt"),"w") as f: - subprocess.run(shlex.join([interpreter, "-m", "pip", "list"]), - stdout=f, stderr=f, - shell=True,executable=shutil.which('bash')) - return True - else: - print("## Errors trying to install pip requirements",file=sys.stderr) - return False - - -def make_conda_install_yml_command(condaprefix_dir, env_yml_file) -> str: - return f'conda env create --quiet --yes --file "{env_yml_file}" --prefix "{condaprefix_dir}"' - -def make_conda_install_spec_command(condaprefix_dir, install_spec_file) -> str: - return f'conda install --quiet --yes --file "{install_spec_file}" --prefix "{condaprefix_dir}"' - -def setup_conda_prefix(proj_dir:str, condaprefix_dir:str, - conda_envyml:str, - conda_specs:str, - pip_requirements, - log_level:Log) -> bool: - logging.info(f"creating conda prefix {condaprefix_dir}") - create_conda_prefix(proj_dir, condaprefix_dir, log_level) - - success = False - if conda_envyml: - install_env_f = os.path.join(proj_dir,'environment.yml') - with open(install_env_f, 'w') as f: - f.write(conda_envyml) - command_to_run = make_conda_install_yml_command(condaprefix_dir, install_env_f) - success = run_with_logging(command_to_run, - proj_dir, - "conda_env_create_f.out","conda_env_create_f.err", - log_level) - elif conda_specs: - install_spec_f = os.path.join(proj_dir,'conda_install_specs.txt') - with open(install_spec_f, 'w') as f: - f.write(conda_specs) - command_to_run = make_conda_install_spec_command(condaprefix_dir, install_spec_f) - success = run_with_logging(command_to_run, - proj_dir, - "conda_install.out","conda_install.err", - log_level) - else: - assert True, "unreachable. " - if success: - with open(os.path.join(proj_dir,"exported-environment.yml"),"w") as f: - cmd = ["conda","env","export","--quiet", "--prefix",condaprefix_dir] - logging.info(f"exporting env with {cmd}") - subprocess.run(shlex.join(cmd), - stdout=f, stderr=f, - shell=True,executable=shutil.which('bash')) - else: - print("## Errors trying to install conda dependencies",file=sys.stderr) - return False - - if pip_requirements: - interpreter = os.path.join(condaprefix_dir, 'bin','python3') - return install_pip_requirements(proj_dir,pip_requirements, - interpreter, - log_level) - else: - return True - - -def run_script(interpreter, script, args) -> NoReturn: - logging.info( - f"running {script} using {interpreter} with args: {args}" - ) - sys.stdout.flush() - logging.info(f'os.execvp({interpreter}, [{interpreter},{script}] + {args})') - os.execvp(interpreter, [interpreter,script] + args) - -def conda_run_script(interpreter, script, args, conda_env_dir) -> NoReturn: - logging.info( - f"using conda run to run {script} using {interpreter} with args: {args}" - ) - logging.info(f'os.execvp({interpreter}, [{interpreter},{script}] + {args})') - # to workaround the conda bug https://github.com/conda/conda/issues/13639 - should_use_wrapper = True - if should_use_wrapper: - workaround_path = os.path.join(conda_env_dir,'exec_script') - with open(workaround_path,'w') as f: - workaround_script = f"exec {interpreter} {script}" - for arg in args: - workaround_script += f" {arg}" - workaround_script += "\n" - logging.info(f'builiding script with contents: {workaround_script}') - logging.info(f'writing script to path: {workaround_path}') - f.write(workaround_script) - os.chmod(workaround_path, 0o755) - cmd = ["conda","run","-p", conda_env_dir, "--no-capture-output", workaround_path] - else: - cmd = ["conda","run","-p", conda_env_dir, "--no-capture-output", interpreter,script] + args - sys.stdout.flush() - os.execvp(cmd[0],cmd) - -# -# venv operations -# - -def create_venv(proj_dir, venv_dir, pip_requirements, log_level:Log) -> bool: - "Creates a script project dir for script at script_path" - logging.info(f"Creating venv at {venv_dir}") - - success = run_with_logging(["python3", "-m", "venv", venv_dir], - proj_dir, - "create_venv.out","creat_evenv.err", - log_level) - if not success: - print(f"## Error trying to create venv",file=sys.stderr) - return False - - if pip_requirements: - interpreter = os.path.join(venv_dir, 'bin','python3') - return install_pip_requirements(proj_dir, - pip_requirements, - interpreter,log_level) - else: - return True - - -# -# helpers -# - -def clean_name_from_path(p): - return re.sub(r'[^A-Za-z0-9-]', '', os.path.basename(p)) - - -def print_base_dirs(): - print(f"Cached project directores are in:\n{cache_base()}\n\n") - print(f"Each directory's contains logs and other build artifacts.\n\n") - print(f"Trashed and cleaned projects are here, waiting for disposal by the OS:\n{trash_base()}") - - -def trash_base() -> str: - "Directory to use for trashing broken project dirs" - return os.path.join( tempfile.gettempdir(), "pythonrunscript" ) - -def print_python3_path(): - if p := shutil.which('python3'): - print(f"## The first python3 in your PATH: {p}") - else: - print("## There is no python3 in your PATH!") - -def cache_base(): - cache_base = None - if "XDG_CACHE_HOME" in os.environ: - cache_base = os.environ["XDG_CACHE_HOME"] - elif platform.system() == "Darwin": - cache_base = os.path.join(os.path.expanduser("~"), "Library", "Caches") - else: - cache_base = os.path.join(os.path.expanduser("~"), ".cache") - cache_base = os.path.join(cache_base, "pythonrunscript") - return cache_base - -def are_dependencies_missing() -> bool: - return (platform.system() not in ['Linux','Darwin'] - and (shutil.which('bash') is None - or shutil.which('tee') is None)) - -if __name__ == "__main__": - main() - diff --git a/examples/clicker.py b/examples/clicker.py deleted file mode 100755 index 5d8663e..0000000 --- a/examples/clicker.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env pythonrunscript -# -# Defines and serves an idleclicker game -# -# Just run me with pythonrunscript, and I will auto-install -# my dependencies and serve on port 8090 -# -# /// pythonrunscript-requirements-txt -# python-fasthtml>=0.6.9,<0.7 -# uvicorn>=0.29 -# python-multipart -# sqlite-utils -# requests -# replicate -# pillow -# /// - -from dataclasses import dataclass -from enum import Enum -from threading import Timer -import webbrowser, sys, copy - -import uvicorn -from fasthtml.common import FastHTML, Div, P, Title, Main, Button, H1 - -#### -#### define game logic -#### - -@dataclass -class GameState: - money:int - factories:int - materials:int - factory_price:int - material_price:int - material_needed_per_factory:int - units_per_second:int - -initial_state = GameState(90, 4, 5, 100, 10, 8, 4) -ACTIONS = Enum('ACTIONS',['PASS','BUY_FACTORY','BUY_MATERIAL']) - -def print_game(state:GameState): - state_dict = state.__dict__ - max_key_length = max(len(key) for key in state_dict) - print("") - for (key,value) in state_dict.items(): - print(f" {key:<{max_key_length}}\t{value}") - print("") - -def evolve_state(starting_state, action): - new_state = copy.copy(starting_state) - if action == ACTIONS.PASS: - new_state.money = new_state.money + new_state.factories - elif action == ACTIONS.BUY_FACTORY: - new_state.factories = new_state.factories + 1 - new_state.money = new_state.money - new_state.factory_price - new_state.materials = new_state.materials - new_state.material_needed_per_factory - elif action == ACTIONS.BUY_MATERIAL: - new_state.money = new_state.money - new_state.material_price - new_state.materials = new_state.materials + 1 - else: - print("Error: unknown action") - return new_state - -#### -#### define app -#### - -app = FastHTML() - -# state -state = initial_state - -def make_state_div(): - ''' - Returns a fresh state DIV reflecting game state - ''' - return Div(P(f'Money: {state.money} '), - P(f"Factories: {state.factories}"), - P(f'Factory Price: {state.factory_price}'), - P(f'Materials: {state.materials}'), - P(f'Material Price: {state.material_price}'), - P(f'Units Per Second: {state.units_per_second}'), - id='state_div') - -@app.get("/") -async def _(): - 'Renders all UI' - return ( - Title("Clicker Game"), - Main( - H1("hello world"), - # Specifies that a click here will: - # 1. will update #state_div element (hx_target='#state_div') - # 2. by REPLACING it (hx_swap='outerHTML') - # 3. with value returned by the function - # associated with /buyfactory route (hx_post='/buyfactory') - Button('Buy Factory', - hx_post="/buyfactory", - hx_target='#state_div', - hx_swap="outerHTML"), - Button('Buy Materials', - hx_post="/buymaterials", - hx_target='#state_div', - hx_swap="outerHTML"), - make_state_div())) - - -@app.post("/buyfactory") -def _(): - 'Handles a buy factory action, returns a fresh state div' - global state - action = ACTIONS.BUY_FACTORY - state = evolve_state(state, action) - return make_state_div() - -@app.post("/buymaterials") -def _(): - global state - action = ACTIONS.BUY_MATERIAL - state = evolve_state(state, action) - return make_state_div() - -port=8090 - -def open_browser(): - url = f"http://localhost:{port}" - print(f"Browsing to {url}") - webbrowser.open(url) - -if __name__ == "__main__": - if len(sys.argv) > 1 and sys.argv[1].startswith("--b"): - Timer(1.5,open_browser).start() - else: - pass - uvicorn.run(app, host="0.0.0.0", port=port) - diff --git a/examples/example-requirements.py b/examples/example-requirements.py deleted file mode 100644 index f5b0f67..0000000 --- a/examples/example-requirements.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env pythonrunscript -# -# /// pythonrunscript-requirements-txt -# tqdm==4.66.4 -# /// -# -from tqdm import tqdm -import sys - -print("Hello, I depend on the tqdm package!") -for i in tqdm(range(10000)): - pass -print("Phew. That was fun!") diff --git a/examples/example-requirements2.py b/examples/example-requirements2.py deleted file mode 100644 index 5103300..0000000 --- a/examples/example-requirements2.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env pythonrunscript -# -# /// pythonrunscript-conda-install-specs-txt -# python>=3.11 -# /// -# -# /// pythonrunscript-requirements-txt -# tqdm==4.66.4 -# /// -# -from tqdm import tqdm -import sys - -print("Hello, I depend on the tqdm package!") -for i in tqdm(range(10000)): - pass -print("Phew. That was fun!") diff --git a/examples/example-script.py b/examples/example-script.py deleted file mode 100644 index 876ab0f..0000000 --- a/examples/example-script.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env pythonrunscript -# -# /// script -# dependencies = [ -# "tqdm==4.66.4", -# ] -# /// -# -from tqdm import tqdm -import sys - -print("Hello, I depend on the tqdm package!") -for i in tqdm(range(10000)): - pass -print("Phew. That was fun!") diff --git a/examples/example-script2.py b/examples/example-script2.py deleted file mode 100644 index 506a307..0000000 --- a/examples/example-script2.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env pythonrunscript -# -# /// script -# requires-python = ">=3.11" -# dependencies = [ -# "tqdm==4.66.4", -# ] -# /// -# -from tqdm import tqdm -import sys - -print("Hello, I depend on the tqdm package!") -for i in tqdm(range(10000)): - pass -print("Phew. That was fun!") diff --git a/examples/fasthtml-examples/chess/chess_app.py b/examples/fasthtml-examples/chess/chess_app.py deleted file mode 100644 index efc4f41..0000000 --- a/examples/fasthtml-examples/chess/chess_app.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env pythonrunscript -# /// pythonrunscript-requirements-txt -# python-fasthtml>=0.6.9,<0.7 -# chess -# /// - - -# AUTOGENERATED! DO NOT EDIT! File to edit: chess_app.ipynb. - -# %% auto 0 -__all__ = ['cboard', 'css', 'gridlink', 'htmx_ws', 'app', 'rt', 'player_queue', 'ROWS', 'COLS', 'WS', 'Board', 'Home', 'get', - 'post', 'put'] - -# %% chess_app.ipynb 4 -#from fasthtml import * -from fasthtml.common import * -from fastcore.utils import * -from fastcore.xml import to_xml -from starlette.endpoints import WebSocketEndpoint -from starlette.routing import WebSocketRoute - -import chess -import chess.svg - -cboard = chess.Board() -# move e2e4 -cboard.push_san('e4') -cboard.push_san('e5') -css = Style( - '''\ - #chess-board { display: grid; grid-template-columns: repeat(8, 64px); grid-template-rows: repeat(8, 64px);gap: 1px; } - .board-cell { width: 64px; height: 64px; border: 1px solid black; } - .black { background-color: grey; } - .white { background-color: white; } - .active { background-color: green; } - ''' -) -# Flexbox CSS (http://flexboxgrid.com/) -gridlink = Link(rel="stylesheet", href="https://cdnjs.cloudflare.com/ajax/libs/flexboxgrid/6.3.1/flexboxgrid.min.css", type="text/css") -htmx_ws = Script(src="https://unpkg.com/htmx-ext-ws@2.0.0/ws.js") - -app = FastHTML(hdrs=(gridlink, css, htmx_ws,)) -rt = app.route - -# %% chess_app.ipynb 5 -player_queue = [] -class WS(WebSocketEndpoint): - encoding = 'text' - - async def on_connect(self, websocket): - global player_queue - player_queue.append(websocket) - await websocket.accept() - print(f'There are {len(player_queue)} players in the queue') - await websocket.send_text("
Hello, you connected!
") - if len(player_queue) == 2: - await player_queue[0].send_text("
Opponent joined! Let the game begin!
") - await player_queue[1].send_text("
You joined! Let the game begin!
") - - async def on_receive(self, websocket, data): - await websocket.send_text("hi") - - async def on_disconnect(self, websocket, close_code): - global player_queue - player_queue.remove(websocket) - for player in player_queue: - await player.send_text("
Opponent disconnected!
") - -app.routes.append(WebSocketRoute('/chess', WS)) - -# %% chess_app.ipynb 6 -ROWS = '87654321' -COLS = 'abcdefgh' -def Board(lmoves: list[str] = [], selected: str = ''): - board = [] - for row in ROWS: - board_row = [] - for col in COLS: - pos = f"{col}{row}" - cell_color = "black" if (ROWS.index(row) + COLS.index(col)) % 2 == 0 else "white" - cell_color = 'active' if pos in lmoves else cell_color - cell_cls = f'board-cell {cell_color}' - if pos == selected: - cell_cls += ' selected' - piece = cboard.piece_at(chess.parse_square(pos)) - if piece: - piece = NotStr(chess.svg.piece(piece)) - board_row.append( - Div( - piece, id=pos, cls=cell_cls, hx_post="/select", hx_vals={'col': col, 'row': row}, - hx_swap='outerHTML', hx_target='#chess-board', hx_trigger='click' - ) - ) - else: - cell = Div(id=pos, cls=cell_cls) - if selected != '': - move = f'{selected}{pos}' - print(move) - if chess.Move.from_uci(move) in cboard.legal_moves: - cell = Div(id=pos, cls=cell_cls, hx_put="/move", hx_vals={'move': move}, - hx_swap='outerHTML', hx_target='#chess-board', hx_trigger='click' - ) - board_row.append(cell) - board.append(Div(*board_row, cls="board-row")) - return Div(*board, id="chess-board") - -# %% chess_app.ipynb 7 -def Home(): - return Div( - Div('Hello, still waiting on an opponent!', id='user-message'), - Board(), - hx_ext="ws", ws_connect="/chess" - ) - -# %% chess_app.ipynb 8 -@rt("/") -def get(): - return Home() - -# %% chess_app.ipynb 9 -@rt('/select') -async def post(col: str, row: str): - global cboards - lmoves = [] - for m in cboard.legal_moves: - if str(m).startswith(f'{col}{row}'): - lmoves.append(str(m)[2:]) - return Board(lmoves=lmoves, selected=f'{col}{row}') - -# %% chess_app.ipynb 10 -@rt('/move') -async def put(move: str): - global cboards - cboard.push_san(move) - for player in player_queue: - await player.send_text(to_xml(Board())) - - -from fasthtml.common import serve - -serve() diff --git a/examples/fasthtml-examples/h2f/main.py b/examples/fasthtml-examples/h2f/main.py deleted file mode 100644 index 09fd3c3..0000000 --- a/examples/fasthtml-examples/h2f/main.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env pythonrunscript -# /// pythonrunscript-requirements-txt -# python-fasthtml>=0.4.5,<0.7 -# /// -from fasthtml.common import * - -app,rt = fast_app(hdrs=[HighlightJS()]) - -@rt("/convert") -def post(html:str, attr1st:bool): return Pre(Code(html2ft(html, attr1st=str2bool(attr1st)))) if html else '' - -@rt("/") -def get(): - return Titled( - "Convert HTML to FT", - Form(hx_post='/convert', target_id="ft", hx_trigger="change from:#attr1st, keyup delay:500ms from:#html")( - Select(style="width: auto", id="attr1st")( - Option("Children 1st", value="0", selected=True), Option("Attrs 1st", value="1")), - Textarea(placeholder='Paste HTML here', id="html", rows=10)), - Div(id="ft")) - -serve() diff --git a/examples/fasthtml-examples/note.txt b/examples/fasthtml-examples/note.txt deleted file mode 100644 index 0025e5e..0000000 --- a/examples/fasthtml-examples/note.txt +++ /dev/null @@ -1 +0,0 @@ -Some examples adapted from: https://github.com/AnswerDotAI/fasthtml-example diff --git a/examples/fasthtml-examples/todo2/.gitignore b/examples/fasthtml-examples/todo2/.gitignore deleted file mode 100644 index 1269488..0000000 --- a/examples/fasthtml-examples/todo2/.gitignore +++ /dev/null @@ -1 +0,0 @@ -data diff --git a/examples/fasthtml-examples/todo2/favicon.ico b/examples/fasthtml-examples/todo2/favicon.ico deleted file mode 100644 index 8d9f7f3..0000000 Binary files a/examples/fasthtml-examples/todo2/favicon.ico and /dev/null differ diff --git a/examples/fasthtml-examples/todo2/main.py b/examples/fasthtml-examples/todo2/main.py deleted file mode 100755 index a706a46..0000000 --- a/examples/fasthtml-examples/todo2/main.py +++ /dev/null @@ -1,140 +0,0 @@ -# !/usr/bin/env pythonrunscript -# -# /// pythonrunscript-conda-install-specs-txt -# python>=3.11 -# /// -# -# /// pythonrunscript-requirements-txt -# python-fasthtml>=0.0.8,<0.7 -# uvicorn>=0.29 -# python-multipart -# /// -from fasthtml.common import * -from hmac import compare_digest -# required for sqlalchemy: -# from fastsql import * - -db = database('data/utodos.db') -# for sqlalchemy: -# url = 'postgresql://' -# db = Database(url) -class User: name:str; pwd:str -class Todo: - id:int; title:str; done:bool; name:str; details:str; priority:int - def __ft__(self): - ashow = A(self.title, hx_post=retr.rt(id=self.id), target_id='current-todo') - aedit = A('edit', hx_post=edit.rt(id=self.id), target_id='current-todo') - dt = '✅ ' if self.done else '' - cts = (dt, ashow, ' | ', aedit, Hidden(id="id", value=self.id), Hidden(id="priority", value="0")) - return Li(*cts, id=f'todo-{self.id}') - -users = db.create(User, pk='name') -todos = db.create(Todo) - -login_redir = RedirectResponse('/login', status_code=303) - -def before(req, sess): - auth = req.scope['auth'] = sess.get('auth', None) - if not auth: return login_redir - todos.xtra(name=auth) - -def _not_found(req, exc): return Titled('Oh no!', Div('We could not find that page :(')) - -hdrs=( - SortableJS('.sortable'), - MarkdownJS('.markdown'), -) - -bware = Beforeware(before, skip=[r'/favicon\.ico', r'/static/.*', r'.*\.js', r'.*\.css', '/login']) -app,rt = fast_app(before=bware, live=True, - exception_handlers={404: _not_found}, - hdrs=hdrs) - -@app.get -def login(): - frm = Form(action='/login', method='post')( - Input(id='name', placeholder='Name'), - Input(id='pwd', type='password', placeholder='Password'), - Button('login')) - return Titled("Login", frm) - -@dataclass -class Login: name:str; pwd:str - -@rt("/login") -def post(login:Login, sess): - if not login.name or not login.pwd: return login_redir - try: u = users[login.name] - except NotFoundError: u = users.insert(login) - if not compare_digest(u.pwd.encode("utf-8"), login.pwd.encode("utf-8")): return login_redir - sess['auth'] = u.name - return RedirectResponse('/', status_code=303) - -@app.get("/logout") -def logout(sess): - del sess['auth'] - return login_redir - -@rt("/") -def get(auth): - title = f"{auth}'s Todo list" - cts = Container( - Grid(H1(title), - Div(style='text-align: right')( - A('logout', href='/logout') - ) - ), - Card( - Ul( - Form(id='todo-list', cls='sortable', hx_post=reorder, hx_trigger="end")( - *todos(order_by='priority') - ) - ), - header=Form(hx_post=create, target_id='todo-list', hx_swap="afterbegin")( - Group( - Input(id="new-title", name="title", placeholder="New Todo"), - Button("Add") - ) - ), - footer=Div(id='current-todo') - ) - ) - return Title(title), cts - -@rt -def reorder(id:list[int]): - for i,id_ in enumerate(id): todos.update(priority=i, id=id_) - return tuple(todos(order_by='priority')) - -@rt -def create(todo:Todo): - new_inp = Input(id="new-title", name="title", placeholder="New Todo", hx_swap_oob='true') - return todos.insert(todo), new_inp - -@rt -def remove(id:int): - todos.delete(id) - return clear('current-todo') - -@rt -def edit(id:int): - res = Form(hx_post=replace, target_id=f'todo-{id}', id="edit")( - Group(Input(id="title"), Button("Save")), - Hidden(id="id"), Hidden(priority="priority"), - Hidden(name="done"), CheckboxX(id="done", label='Done'), - Textarea(id="details", name="details", rows=10)) - return fill_form(res, todos[id]) - -@rt -def replace(todo: Todo): return todos.update(todo), clear('current-todo') - -@rt -def retr(id:int): - todo = todos[id] - btn = Button('delete', - name='id', value=id, target_id=f'todo-{todo.id}', - hx_post=remove, hx_swap="outerHTML") - return Div(H2(todo.title), Div(todo.details, cls="markdown"), btn) - -serve() - diff --git a/examples/fasthtml-examples/todo2/picovars.css b/examples/fasthtml-examples/todo2/picovars.css deleted file mode 100644 index fbaa2db..0000000 --- a/examples/fasthtml-examples/todo2/picovars.css +++ /dev/null @@ -1,2 +0,0 @@ -:root { --pico-font-size: 100%; } - diff --git a/index.html b/index.html new file mode 100644 index 0000000..b7722fd --- /dev/null +++ b/index.html @@ -0,0 +1,6 @@ + + + + + + diff --git a/notes/README.org b/notes/README.org deleted file mode 100644 index 1adb4cd..0000000 --- a/notes/README.org +++ /dev/null @@ -1,153 +0,0 @@ -* ./pythonrunscript - -=pythonrunscript= lets you - -- define and run *single-file python scripts* -- which *can use external dependencies* from pip or conda -- while *automatically installing dependencies into isolated environments* - -** Installation - -There are two possible ways to install pythonrunscript: - -1. Either, do =python3 -m pip install pythonrunscript=. -1. Or, just copy the file =pythonrunscript= into a directory named in your =PATH= variable. - -** How to use pythonrunscript - -pythonrunscript lets you declare your script's dependencies in the script itself. - -That is, the goal is to let you build and use scripts that /just run/, with no setup. - -To declare your script's pip dependencies, add a /pip requirements.txt block/ to your script's comments. This is a comment block which specifies the script's external requirements, delimited by markdown code fencing, using the exact syntax of a pip requirements.txt file. Here below, the block declares a dependency on the pip package tqdm. - -#+begin_src python - #!/usr/bin/env pythonrunscript - # - # ```requirements.txt: - # tqdm==4.66.4 - # ``` - # - from tqdm import tqdm - import sys - - print("Hello, I depend on the tqdm package!") - for i in tqdm(range(10000)): - pass - print("Phew. That was fun!") -#+end_src - -The first time you run this script, pythonrunscript will parse its requirements.txt block, create an isolated environment in a cache directory, install its dependencies, and run the script from that environment. In later runs, it will just re-use that environment. To create the environment it uses venv, which is built into python, so the only requirement to use pythonrunscript is that python itself is already installed. - -To run your script, call it with =pythonrunscript hello.py=. - -To run your script directly (e.g, =./hello.py=), change your script's first line, the shebang line, to the following: =#!/usr/bin/env pythonrunscript=. - -How will this affect normal execution? It won't. If you script does not have a pip requirements.txt block, pythonrunscript will simply pass it to python3 for execution as usual. - -** How can I check what it will do? If it worked? What happened exactly? - -Run pythonrunscript in =--dry-run= mode to get a preview of how it parses your file and the actions it /would/ take. - -Run with =--verbose= to hear copious commentary, and to see all output of subprocess commands. Normally, pythonrunscript only prints in case of an installation error. Under all circumstances it saves all generated output in a logs directory, as well as saving an =environment-resolved.yml= and a =pip-list.txt= that reflect the exact state of the environment after creation. These files are in a script's project directory, in the cache, which is revealed by running with the =--show-cache= command. Running with =--clean-cache= erases all cached directories (actually, it moves them to the OS's temporary directory for automatic disposal). - -** What about conda dependencies? - -Some popular dependencies, like [[https://developer.nvidia.com/cuda-toolkit][cudatoolkit]], cannot be installed by pip but need conda. - -To specify conda dependencies, you must add a /conda environment.yml block/ or a /conda_install_specs.txt block/. You may use this instead of, or in addition to, a pip requirements.txt block. They use two other types of fenced comment blocks. The environment block is intoduced by =```environment.yml= and it should contain an environment.yml file verbatim. - -The install spec block is introduced by =```conda_install_specs.txt= block, and it should introduce conda install specs. which should specify a "conda install specification". A conda install spec is simply the string passed to the =conda install= command. Conda documents [[https://conda.io/projects/conda/en/latest/user-guide/concepts/pkg-search.html][exact syntax for an install specification]], which only requires naming the conda package, but also allows specifying the version, the channel, or specific builds. - -For instance, this script uses a conda environment.yml block: - -#+begin_src python - #!/usr/bin/env pythonrunscript - # - # ```environment.yml - # dependencies: - # - python=3.10 - # - numpy - # ``` - # - import numpy - import sys - - print("I depend on numpy") -#+end_src - -And this script uses a conda install specs block, for the same dependencies: - -#+begin_src python - #!/usr/bin/env pythonrunscript - # - # ```conda_install_specs.txt - # python=3.10 - # numpy - # ``` - # - from tqdm import tqdm - import sys - - print("Hello, I depend on the tqdm package!") - for i in tqdm(range(10000)): - pass - print("Phew. That was fun!") -#+end_src - - -Do you really need conda? Maybe not! If you don't specify conda dependencies, pythonrunscript won't try to use it. - -But you might need conda if you need conda-only dependencies, if you want to specify the version of python itself or to use packages outside of the Python ecosystem. This [[https://wandb.ai/wandb_fc/pytorch-image-models/reports/A-faster-way-to-get-working-and-up-to-date-conda-environments-using-fastchan---Vmlldzo2ODIzNzA][weights & biases blog post]] explains the situation well, and [[https://github.com/fastai/fastsetup/blob/master/setup-conda.sh][this script will install conda]] on all the platforms.) - -** What, why would I want this? - -If none of the above appeals to you, perhaps you never had problems with python dependency management before? (Congrats!) But if the dilemma is totally alien to you and you're curious, here is the issue. - -Suppose you're writing a single-file Python module, i.e., a script, =hello.py=, like this: - -#+begin_src python - #!/usr/bin/env python3 - print("Hello, world") -#+end_src - -Running it is easy. You can run the script by running =./hello.py= and it will just work. Hurray! - -But then you need some functionality outside of the standard library, like yaml-parsing from an external dependency like PyYAML. So now you need to install that dependency. This is where pain starts. - -Where do you install it? In your system's python? Should you use sudo? Or in a managed environment? Managed how? With venv? With virtualenv? With conda? With nix, docker, something else? So should you install that manager first? Where? Do you need requirements.txt? Pipenv? Poetry? pyproject.toml? Something else? These questions all have answers but they are /boring and painful/. Once you distribute your script to someone else, the pain falls on them too. Can you just give them the file? No, since now you need to give them your script and also, perhaps, a mini-essay on how to answer those questions. If you are lucky enough (?) to be a seasoned expert in python dependency management and to associate only with other such experts, then you may not see the problem here. But it is real outside such orbits. - -(soapbox mode...) - -There is a reason that the Go and rust communities excel at shipping so many charming and effective command line applications. Why? Because their language toolchains have good support for shipping a static linked executable -- that is to say, a single damn file which Just Works. Python would be truly great for scripting, except for the fact that it lacks this power, which makes it impractical to give your script to someone else who just wants to solve their problem with the minimum possible fuss. Sad. - -Hopefully, this tool helps. The dependency syntaxes which it uses are exactly the syntaxes already supported by pip and conda. There is nothing exotic here. So you can do everything you can normally do with pip, such as specify dependencies not only as public PyPI packages but also as [[https://pip.pypa.io/en/stable/topics/vcs-support/][URLs to GitHub repos]]. And you can do everything you can normally do with conda, such as specify the version of Python itself, or install vast wads of awkward binary code from dedicated channels like nvidia's. - -(soapbox off...) - -** Wait, but I /enjoy/ manually keeping environments nice and tidy, even for tiny little scripts! - -Great, then, this is not for you. 😇 Cultivate your garden. - -** Inspiration - -- [[https://github.com/mxcl/swift-sh][swift-sh]] does the same trick for Swift -- [[https://rust-script.org][rust-script]] does the same trick for rust -- [[https://github.com/igor-petruk/scriptisto][scriptisto]] does this for any language, but at the cost of shifting the complexity into the comment block. - -#+DATE: [2024-06-05] -#+AUTHOR: Alexis Gallagher -#+OPTIONS: toc:nil -#+HTML_DOCTYPE: html5 -#+OPTIONS: html-style:nil -#+OPTIONS: html5-fancy -#+OPTIONS: num:nil - - - -#+BEGIN_SRC emacs-lisp :exports none - (progn - (defalias 'openhtml - (kmacro "C-c C-e h o")) - (local-set-key (kbd "C-c u") #'openhtml)) -#+END_SRC diff --git a/notes/log-python-script.org b/notes/log-python-script.org deleted file mode 100644 index 09e7f9a..0000000 --- a/notes/log-python-script.org +++ /dev/null @@ -1,374 +0,0 @@ -* Tasks -** DONE make this pip-installable -https://python-packaging.readthedocs.io/en/latest/command-line-scripts.html -** TODO make it work on windows -use asyncio instead of bash and tee? -** TODO raise error if conda is needed and missing -** TODO readme: show cooler examples -examples that show what you can do: - -- single-file notebook server! -- demo with axolotl -- simple webserver -- use pandas to convert CSV to excel -- use language model -- use for PDF extraction -** TODO ? readme: show example of a requirements.txt that references a dependency by URL -** TODO ? add flag --install-miniconda flag -** TODO ? consider making it installable as a brew package -** TODO develop example with test with advanced conda dependencies -Try to convert this to a single comment block: - -#+begin_src sh -# create conda env -conda create -n multistep python=3.10 # higher versions might break vllm -conda activate multistep - -# Linux install cuda and cuda toolkit and pytorch. Good as of 2024-05-14 Tue 15:45 -conda install cuda -c nvidia/label/cuda-12.1.0 -conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia - -# conda-install conda-packaged dependencies -conda install openai networkx numpy pandas regex - -# pip-install pip-only dependencies -python3 -m pip install claudette -python3 -m pip install vllm -python3 -m pip install datasets -python3 -m pip install nltk -#+end_src - -** nice-to-have -** TODO ? add --force-rebuild flag -** TODO ? provide stats regarding total space used cache -** TODO ? create a custom GPT readme -** TODO ? design way to add persistent link in env to originating script -- could name directory with first script name used - - con. hides later scripts -- could add symlink in the dir back to the script? - - but what if different scripts have the same name? add junk to uniqiify? -** TODO ? add command line option to read Python expressions -** TODO ? implement inline (workflow for pure-venv case) -design goal: -- instead of putting this script in the path, it should also be sufficiently literally to append it to any python script -issues: -- overriding existing defintiions of main -- collisions on globals namespace -** done -** DONE update macOS cache dir based on [[https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html#//apple_ref/doc/uid/TP40010672-CH10-SW1][apple file system programming guide]] -page defining location: [[https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html#//apple_ref/doc/uid/TP40010672-CH10-SW1][macOS Library Directory Details]] - -options: -- $HOME/Library/Caches/python-script/ -- $HOME/Library/Developer/python-script/ - -* Resources - -- https://rust-script.org/ -- https://github.com/mxcl/swift-sh/tree/master/Examples -- scriptisto (more general, but more opaque) - -* Log - -** [2024-06-03 Mon 09:28] what format for conda? - -Options: - -1. embed environment.yml - - works with -n if environment.yml does not specify the env name! - - works with -prefix if environment.yml does not specify the env name! - - -3. embed whatever format =conda install --file= expects - - that format is also what =conda install= expectsspecification?apparently yes. - - that format is the [[https://conda.io/projects/conda/en/latest/user-guide/concepts/pkg-search.html][install specification]] - -[[https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html][conda install docs]] note that with --file conda intsll will "Read package versions from the given file." - -** [2024-06-06 Thu 19:03] - -packaging - -https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#writing-pyproject-toml - -[[https://packaging.python.org/en/latest/guides/single-sourcing-package-version/#single-sourcing-the-version][single-sourcing version]] - -** [2024-06-07 Fri 11:44] - -macOS Sonoma ships with Python 3.9.6 and with setuptools 58.04 - -setuptools can be configured with pyproject.toml only as setuptools 61.0.0 - -So: -- must not rely on pyproject.toml -- can use a setup.cfg - -twine upload to PyPi - -** [2024-06-07 Fri 14:07] two configurations: - -*** config 1 - -#+begin_quote -. - ├── pyproject.toml - ├── python_script - │ ├── __init__.py - │ ├── __main__.py - │ └── python_script.py -#+end_quote - -properties: - -- pro: $ python3 -m python_script -- pro: $ python-script -- con: then python_script.py can't just go into the path! - -Conflict: -- copy-file installability is good, and requires files be named like CL tools -- pip-installability is good -- but pip requires modules to have bad names for a CL tool - -Options: -1. duplicate the file, python-script AND python_script?? -2. use =exec= to load a file without the py extension? - -One might think there was an option 3, to just use a Python-compliant name also for the CLI, by calling it python_script.py or pythonscript.py. But this seems not to work. The =python3 -m= builds a shim. When that shim runs, it does not find the package named python_script. Not clear why. In other words, there is a defect in the =python3 -m= run mechanism such that you cannot name one of your project entry point scripts "foo.py" if the import name of your package is "foo", since then - - -*** config 2? - . - ├── pyproject.toml - ├── python_script - │ ├── __init__.py - │ ├── __main__.py - │ └── python_script.py - -** [2024-06-07 Fri 14:16] naming - -names: -- pythonscript -- python-script -- pythonscript.py -- python_script.py - -- pyrun -- pyrunscript -- python-script.py -- pyscript: - - no used -- pydepscript -- depscript - - no. no mention of python - - -** [2024-06-07 Fri 16:59] name collision - -The project name "pythonscript" was rejected. - -presumably because of the existing project PythonScript1, a [[https://pypi.org/project/PythonScript1/#history][dead stub of a project]]. [[https://pypi.org/project/PythonScript1/#history][collision policy]] - -will try pythonrunscript - -** [2024-08-20 Tue 22:32] noted - -An official spec for embedded deps: https://packaging.python.org/en/latest/specifications/inline-script-metadata/#inline-script-metadata - -PEP: https://peps.python.org/pep-0723/ - - -Noted re: inline script metadata: - -one example for a tool called "some-toml" - -#+begin_quote -# /// some-toml -# embedded-csharp = """ -# /// -# /// text -# /// -# /// -# public class MyClass { } -# """ -# /// -#+end_quote - -ANother example for the tool called "script": - -#+begin_quote -# /// script -# requires-python = ">=3.11" -# dependencies = [ -# "requests<3", -# "rich", -# ] -# /// -#+end_quote - -Possible use of this syntax: add support for doc comments with these tools: -- pythonrunscript-requirements.txt -- pythonrunscript-conda_install_specs.txt - -current parser works by: -- defining 3 begin patterns, 1 end pattern, and 1 data pattern -- defining 4 parser patterns (3 in-block, 1 out-of-block) -- elif-based pattern matching on (line_type, match obj) - -notable: https://pip.wtf/ - -** [2024-09-02 Mon 23:29] notes on wtf.pip - -- PRO. lightweight, comprehensible. Very good. <10 LOC. -- CON. semi-nonstandard syntax for specifying deps -- CON. Only pip, no conda -- CON. installs and uses dedicated env, unconditionally, and even if current env suffices - - inefficient. - - interferes with other workflows, like developing the script in an IDE-managed env - -Evolution?: -- only install deps if needed and confirmed? - - q: does pip itself already providing UI prompting? - - q: how to check deps? - - also with pip - - -** [2024-09-06] - -- [ ] Q: how to convert all version specs valid in script TOML, to specs valid in conda python specs and requirementx.txt dep specs? - -- exact version format required in PEP723 TOML? - - for deps, Uses https://packaging.python.org/en/latest/specifications/dependency-specifiers/#dependency-specifiers - - for python, uses https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers - -- exact version format required in requirements.txt? - - requirements.txt ALLOWs use of requirements specifiers: https://pip.pypa.io/en/stable/reference/requirements-file-format/ - - https://pip.pypa.io/en/stable/reference/requirement-specifiers/#requirement-specifiers - - "Generally speaking, a requirement specifier is composed of a project name followed by optional version specifiers." - -- exact version format required in a conda spec? - -** [2024-09-12] next steps - -- [X] update README. -- [X] update CLI docs -- [X] update dry run mode -- [ ] verify that TOML syntax for python version and pip versions matches conda install spec and reqs.txt syntax -- [X] enhance fork into environment with components installed through pip and conda - -** [2024-09-20 Fri 20:33] - -run_ffmpeg: -- run as executable, triggering pythonrunscript.py: - - BROKEN - - incorrectly tries to interpret the file not as python? - - does not interpreter just as if called by the python3 in the env - - but: calling =conda run -p /Users/alexis/Library/Caches/pythonrunscript/60b45f611c8f43cc4df61e1bd5157fd8/condaenv python3 run_ffmpeg.py= works - -- run from python3: - - works: runs as desired, as if =ffmpeg -version= was called directly -- run from pythonrunscript.py: - - works: runs as desired, as if =ffmpeg -version= was called directly - -run_conda.py: -- run from python3: - - -- run as executable: - - -** [2024-09-23 Mon 22:29] status - -- what behavior is desired? - - -- does workaround improve behavior? - -NO WRAPPER: - -test of =ffmpeg=: -- =ffmpeg=: - - exits, with code 1 -- =./run_ffmpeg=: - - exits with code 1, and a message emitted by conda saying: - - BAD: discrepancy. conda inserts a message -- =pythonrunscript.py run_ffmpeg.py=: - - exits with code 1 - - BAD: and a message emitted by conda saying. (same) - -test of =ffmpeg -version=: -- =ffmpeg -version=: - - exits, with code 0 -- =./run_ffmpeg -version=: - - exits, with code 0 - - BAD: conda intercepts the command line argument. -- =pythonrunscript.py run_ffmpeg.py -version=: - - BAD: conda intercepts the command line argument. - -test of =python3=: -- =./python3=: - - opens interactive prompt -- =./run_python3.py=: - - VERY VERY BAD: exits immediately -- =pythonrunscript ./run_python3.py=: - - without wrapper: opens interactive prompt - - with wrapper: -- =python3 ./run_python3.py=: - - opens interactive prompt - -test of =python3 -V=: -- =./python3 -V=: - - works -- =./run_python3.py -V=: - - works -- =pythonrunscript ./run_python3.py -V=: - - works - -WITH WRAPPER: - -- Finding: wrapper SUCCEEDS in fixing the problem that with -version. -- wrapper does not prevent interpreter for quitting rather than opening a terminal -- but wrapper did not cause that problem - - - -summary of non-transparencies: -- =conda run= inserts a message when exit code == 1 -- conda intercepts command line arguments like "-version" - - fixed with wrapper -- conda fails to run an interactive prommpt - - fixed with command line arg to conda --no-capture-output - - -#+begin_src -ERROR conda.cli.main_run:execute(125): `conda run /Users/alexis/Library/Caches/pythonrunscript/60b45f611c8f43cc4df61e1bd5157fd8/condaenv/bin/python3 ./run_ffmpeg.py` failed. (See above for error) -#+end_src - -** summary of results of work to be perfectly transparent to conda - -- scripts can now call conda-installed executable -- worked around a known conda bug - -- one non-transparency, is that if pythonrunscript is used to run a - script in a conda environment, which exits with a nonzero exit code, - then conda will print some additional logging at exit time. - - -** [2024-10-07 Mon 15:43] - -next steps: - -- [X] re-run examples as tests -- [ ] add meatier examples? -- [X] update versioning metadata -- [x] add versioning to CLT -- [X] copy script to binary -- [ ] deploy to pypi - -** [2024-10-07 Mon 19:02] plan to use twine to upload - -poetry build -poetry publish - diff --git a/notes/processtest/printer.py b/notes/processtest/printer.py deleted file mode 100755 index 60c18ed..0000000 --- a/notes/processtest/printer.py +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python3 -import sys -print("I'm talking to stdout", file=sys.stdout) -print("I'm talking to stderr", file=sys.stderr) diff --git a/notes/processtest/printerfail.py b/notes/processtest/printerfail.py deleted file mode 100755 index e7f1618..0000000 --- a/notes/processtest/printerfail.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python3 -import sys -print("I'm talking to stdout", file=sys.stdout) -print("I'm talking to stderr", file=sys.stderr) -exit(1) - diff --git a/notes/processtest/runner.py b/notes/processtest/runner.py deleted file mode 100755 index 4fe6849..0000000 --- a/notes/processtest/runner.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -import subprocess,os,shutil - -def run_with_logging(command:str,out_f,err_f,verbosity:str): - ''' - Runs command. Logs and maybe streams stdout and stderr. - ''' - assert (verbosity in {'silent','errors','verbose'}), "unexpected verbosity" - if verbosity == 'silent': - command += f' 2> "{err_f}"' - command += f' 1> "{out_f}"' - elif verbosity == 'errors': - command += f' 2> >(tee "{err_f}")' - command += f' 1> "{out_f}"' - elif verbosity == 'verbose': - command += f' 2> >(tee "{err_f}")' - command += f' 1> >(tee "{out_f}")' - else: - assert True, "unreachable" - - cp = subprocess.run(command, - shell=True, - executable=shutil.which('bash')) - return cp - -cp = run_with_logging("./printer.py","printed.out","printed.err", verbosity='errors') - -success = cp.returncode - -if success: - print("run succeeded") -else: - print("run failed") - diff --git a/notes/test-scripts/mdfence/testscript-Conda b/notes/test-scripts/mdfence/testscript-Conda deleted file mode 100755 index 5e78f88..0000000 --- a/notes/test-scripts/mdfence/testscript-Conda +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env pythonrunscript -# -# ```conda_install_specs.txt -# python=3.10 -# numpy -# ``` -# -import numpy -import sys - -print("I'm the test script3") -print("I'm the test script's second line") -print(f"test script args: {sys.argv}") - - - diff --git a/notes/test-scripts/mdfence/testscript-CondaYML b/notes/test-scripts/mdfence/testscript-CondaYML deleted file mode 100755 index c66d013..0000000 --- a/notes/test-scripts/mdfence/testscript-CondaYML +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env pythonrunscript -# -# ```environment.yml -# dependencies: -# - python=3.10 -# - numpy -# ``` -# -import numpy -import sys - -print("I'm the test script3") -print("I'm the test script's second line") -print(f"test script args: {sys.argv}") - - - diff --git a/notes/test-scripts/mdfence/testscript-Nodeps b/notes/test-scripts/mdfence/testscript-Nodeps deleted file mode 100755 index c2f4f02..0000000 --- a/notes/test-scripts/mdfence/testscript-Nodeps +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env pythonrunscript -import sys - -print("I'm the test script") -print("I'm the test script's second line") -print(f"len(sys.argv) = {len(sys.argv)}") -print(f"sys.argv = {sys.argv}") - - diff --git a/notes/test-scripts/mdfence/testscript-Pip b/notes/test-scripts/mdfence/testscript-Pip deleted file mode 100755 index c05d1cd..0000000 --- a/notes/test-scripts/mdfence/testscript-Pip +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env pythonrunscript -# ```requirements.txt -# tqdm==4.66.4 -# ``` -from tqdm import tqdm -import sys - -print("I'm the test script") -print("I'm the test script's second line") - -print(f"test script args: {sys.argv}") -for i in tqdm(range(10000)): - pass - - - diff --git a/notes/test-scripts/mdfence/testscript-Pip-broken b/notes/test-scripts/mdfence/testscript-Pip-broken deleted file mode 100755 index f25bc88..0000000 --- a/notes/test-scripts/mdfence/testscript-Pip-broken +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env pythonrunscript -# -# ```requirements.txt -# I_am_impossible_to_install! -# ``` -from tqdm import tqdm -import sys - -print("I'm the test script") -print("I'm the test script's second line") - -print(f"test script args: {sys.argv}") -for i in tqdm(range(10000)): - pass - - - diff --git a/notes/test-scripts/mdfence/testscript-Pip2 b/notes/test-scripts/mdfence/testscript-Pip2 deleted file mode 100755 index 932bfc8..0000000 --- a/notes/test-scripts/mdfence/testscript-Pip2 +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env pythonrunscript -# -# ```requirements.txt -# tqdm==4.66.4 -# ``` -# -# ```requirements.txt -# ```requirements.txt -# pyyaml==6 -# ``` - -from tqdm import tqdm -import sys - -print("I'm the test script") -print("I'm the test script's second line") - -print(f"test script args: {sys.argv}") -for i in tqdm(range(10000)): - pass - - - diff --git a/notes/test-scripts/mdfence/testscript-PipConda b/notes/test-scripts/mdfence/testscript-PipConda deleted file mode 100755 index c6df387..0000000 --- a/notes/test-scripts/mdfence/testscript-PipConda +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env pythonrunscript -# -# ```conda_install_specs.txt -# python=3.10 -# numpy -# ``` -# -# ```requirements.txt -# tqdm==4.66.4 -# ``` -# -import numpy -import sys - -print("I'm the test script3") -print("I'm the test script's second line") -print(f"test script argv: {sys.argv}") - - - diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 41617c4..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,43 +0,0 @@ -[build-system] -requires = ["setuptools>=61.0.0"] -build-backend = "setuptools.build_meta" - -[project] -name = "pythonrunscript" -version = "0.2.0" -authors = [ - { name="Alexis Gallagher", email="alexis@alexisgallagher.com"}, -] -description = "pythonrunscript runs scripts installing their dependencies in cached, isolated environments." -readme = { file = "README.md", content-type = "text/markdown" } -requires-python = ">= 3.9.6" -keywords = ["scripting"] -license = { file = "LICENSE" } -classifiers = [ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: MacOS", - "Operating System :: POSIX :: Linux", -] -dependencies = [] - -[tool.setuptools] -packages = ["pythonrunscript"] - -[project.urls] -Homepage = "https://github.com/AnswerDotAI/pythonrunscript" -Issues = "https://github.com/AnswerDotAI/pythonrunscript/issues" - -[project.scripts] -"pythonrunscript" = "pythonrunscript.pythonrunscript:main" - -[project.optional-dependencies] -dev = ["pytest","tomlkit"] - -[tool.pytest.ini_options] -testpaths = ["tests"] -python_files = "test_*.py" -addopts = "-v --tb=short --import-mode=importlib" -pythonpath = "." diff --git a/pythonrunscript/__init__.py b/pythonrunscript/__init__.py deleted file mode 100644 index 9d380b8..0000000 --- a/pythonrunscript/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .pythonrunscript import main - -__all__ = ["main"] diff --git a/pythonrunscript/__main__.py b/pythonrunscript/__main__.py deleted file mode 100644 index 75f8735..0000000 --- a/pythonrunscript/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -from . import main - -if __name__ == '__main__': - main() - diff --git a/pythonrunscript/pythonrunscript.py b/pythonrunscript/pythonrunscript.py deleted file mode 100755 index ab6eb54..0000000 --- a/pythonrunscript/pythonrunscript.py +++ /dev/null @@ -1,585 +0,0 @@ -#!/usr/bin/env python3 -# python>=3.9.6 -import sys, re, os, subprocess, hashlib, logging, platform, argparse, tempfile, shutil, uuid, textwrap, shlex -from abc import ABC -from enum import Enum -from typing import NoReturn, Union - -logging.basicConfig(level=logging.WARNING) -Log = Enum('Log', ['SILENT','ERRORS','VERBOSE']) - -version_str = "0.2.0 (2024-10-07)" - -def main(): - parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - parser.description='Runs a script, installing its dependencies in a cached, isolated environment' - parser.add_argument('--dry-run', action='store_true', help='report what pythonrunscript would do, without writing any files') - parser.add_argument('--version', action='store_true', help='prints current version') - parser.add_argument('--verbose', action='store_true', help='comments on actions and prints all outputs and errors') - parser.add_argument('--show-cache', action='store_true', help='print the cache directory of script environments') - parser.add_argument('--clean-cache', action='store_true', help='purges all pythonrunscript environments') - parser.add_argument('script', nargs='?', default=None, help='path to the script to run') - parser.add_argument('arguments', nargs=argparse.REMAINDER, help='optional arguments to be passed to that script') - parser.epilog=''' pythonrunscript runs Python scripts, installing their dependencies. - - That is, it will automatically install all dependencies in a cached, isolated - environment dedicated to your script, and run it in that environment. - - To do this it looks in your script for a comment declaring dependencies - using the inline metadata syntax defined in PEP723. This syntax uses a "type" - tag to indicate the type of dependency meatadata. - - You use the type tag pythonrunscript-requirements-txt in order to embed an - ordinary requirements.txt file, like so: - - # /// pythonrunscript-requirements-txt - # tqdm==4.66.4 - # /// - - You can also use the type pythonrunscript-environment-yml to embed an - environment.yml file, or pythonrunscript-conda-install-specs-txt to embed a list - of conda install specs. A conda install spec is just the syntax for arguments - passed to `conda install`. It is documented here: - https://conda.io/projects/conda/en/latest/user-guide/concepts/pkg-search.html - - To run a script with conda dependencies or which specifies the python version, - you must already have conda installed. - - Finally, pythonrunscript also supports the "script" type, which is the TOML-like - syntax given as an initial example in PEP723. It looks like so: - - # /// script - # dependencies = [ - # "tqdm==4.66.4", - # ] - # /// - - You can explicitly call pythonrunscript to run your script by doing - `pythonrunscript myscript.py`. Or you can change your script's first line to use - pythonrunscript as an interpreter (setting its first line to - "#!/usr/bin/env pythonrunscript"), and then execute your script directly. - - pythonrunscript requires Python 3.9.6 and later, which ships with macOS Sonoma. - Since it creates isolated environments, you can run it using the system's - Python without corrupting the system. It also works on Linux. Untested on Windows. - ''' - args = parser.parse_args() - - if sys.version_info < (3,9,6): - print(f"I am being interpreted by Python version:\n{sys.version}") # pyright: ignore - print("But I need python version 3.9.6 or higher.\nAborting.") # pyright: ignore - exit(1) # pyright: ignore - elif args.version: - print(f"pythonrunscript {version_str}") - exit(0) - elif are_dependencies_missing(): - print(f"I am being run on the platform {platform.system()} and") - print("I cannot find the required external commands bash and tee, so") - print("I probably will not work. Aborting.") - exit(1) - elif args.show_cache: - print_base_dirs() - exit(0) - elif args.clean_cache: - pseudo_erase_dir(cache_base()) - exit(0) - elif args.script is None: - print(f"Error: pythonrunscript must be called with either the path to a script, --show-cache, --clean-cache, or --help.") - exit(1) - else: - script = args.script - - if not os.path.exists(script): - print(f"Error: did not find the script {script}. Nothing to do.") - exit(1) - - if args.dry_run: - print("## This is a dry run. No files will be written.\n") - - if args.verbose: - logging.info("Running in verbose") - - proj = Project.make_project(script,args.verbose,args.dry_run) - - if args.dry_run: - perform_dry_run(proj) - exit(0) - - if isinstance(proj, ProjectNoDeps): - logging.info("No pip block and no conda block detected. Running directly") - if args.verbose: - print("## No dependencies needed. Running the script directly") - proj.run(args.arguments) - elif not proj.exists(): - logging.info("Needs an environment but none exists. Creating it") - creation_success = proj.create() - if not creation_success: - trashed_env = pseudo_erase_dir(proj.project_path) - print(f"## Creating a managed environment failed. Moved the broken environment to {trashed_env}",file=sys.stderr) - exit(1) - else: - logging.info(f"Found pre-existing project dir: {proj.project_path}") - if args.verbose: - print(f"## Found pre-existing project dir: {proj.project_path}") - # assert: proj exists - if args.verbose: - print("## Running the script using the project directory environment") - proj.run(args.arguments) - -def perform_dry_run(proj): - "Describes actions for exists(), creates(), runs()" - print("## After parsing, I would take these actions.\n") - if isinstance(proj, ProjectNoDeps): - print(f"## No project directory is needed since parsing found no dependencies in the file {proj.script}\n") - print(f"## In a live run, I would run the script using the first python3 in your PATH.\n") - print_python3_path() - return - elif not proj.exists(): - print(f"## The needed project directory does not exist so I would create this project directory:\n{proj.project_path}\n") - print(f"## Inside, I would create this environment directory:\n{proj.envdir}\n") - if proj.conda_envyml: - print(f"## I found an environment.yml dependency block, so I'd use that.") - print(f"## To install conda dependencies, I'd execute this conda environment creation command:\n") - install_env_f = os.path.join(proj.project_path,'environment.yml') - print(f"\t{make_conda_install_yml_command(proj.project_path,install_env_f)}\n") - elif proj.conda_specs: - print(f"## I found a conda_install_specs.txt block, so I'd use that.") - print(f"## To install conda dependencies, I'd execute this conda install command:") - install_spec_f = os.path.join(proj.project_path,'conda_install_specs.txt') - print(f"{make_conda_install_spec_command(proj.project_path, install_spec_f)}\n") - if proj.pip_requirements: - print(f"## To install pip dependencies, I'd execute the following pip command:") - print(f"python3 -m pip install -r {os.path.join(proj.envdir,'requirements.txt')}\n") - print_python3_path() - print(f"## At this point, this project directory would exist:\n{proj.project_path}\n") - print(f"## I'd run using this env dir:\n{proj.envdir}\n") - return - -def parse_dependencies(script, verbose=False) -> tuple[str,str,str,str]: - "Parses script and returns any conda or pip dep blocks" - LT = Enum('LT', [ - 'BEG_SCRIPT_YML', - 'BEG_CONDA_SPEC_YML','BEG_CONDA_ENV_YML','BEG_PIP_YML','END_YML', - 'BEG_CONDA_SPEC','BEG_CONDA_ENV','BEG_PIP','END', - 'TEXT']) - - p = { - LT.BEG_SCRIPT_YML : r"^# /// script$", - LT.BEG_CONDA_SPEC_YML : r"^# /// pythonrunscript-conda-install-specs-txt$", - LT.BEG_CONDA_ENV_YML : r"^# /// pythonrunscript-environment-yml$", - LT.BEG_PIP_YML : r"^# /// pythonrunscript-requirements-txt$", - LT.END_YML : r"^# ///$", - LT.BEG_CONDA_SPEC : r"^# ```conda_install_specs.txt$", - LT.BEG_CONDA_ENV : r"^# ```environment.yml$", - LT.BEG_PIP : r"^# ```requirements.txt$", - LT.END : r"^# ```$", - LT.TEXT : r"^#(| .*)$", - } - - boxed_pip_block = [''] - boxed_conda_spec_block = [''] - boxed_conda_env_block = [''] - - block_type_content_delimiters = [ - ('script',[], [(LT.BEG_SCRIPT_YML,LT.END_YML)]), - ('requirements.txt', boxed_pip_block, [(LT.BEG_PIP_YML,LT.END_YML), - (LT.BEG_PIP,LT.END),]), - ('conda_install_specs.txt', boxed_conda_spec_block, [(LT.BEG_CONDA_SPEC_YML,LT.END_YML), - (LT.BEG_CONDA_SPEC,LT.END),]), - ('environment.yml', boxed_conda_env_block, [(LT.BEG_CONDA_ENV_YML,LT.END_YML), - (LT.BEG_CONDA_ENV,LT.END),]), - ] - - def make_block_pattern(begend:tuple[LT,LT]) -> str: - (beg,end) = begend - return rf"(?m:{p[beg]}\s(?P({p[LT.TEXT]}\s)+?){p[end]}(?:\s)?)" - - def extract_content(match): - return ''.join( - line[2:] if line.startswith('# ') else line[1:] - for line in match.group('content').splitlines(keepends=True) - ) - - # collect all comment lines starting with "# " or equalling "#" - # transforming to strip # prefix - comments = open(script,'r').read() - if verbose: - print(f"## Parsing this script for dependencies:\n{script}") - print() - - for (block_type, boxed_content, begend_pairs) in block_type_content_delimiters: - for begend in begend_pairs: - block_pattern = make_block_pattern(begend) - match = re.compile(block_pattern).search(comments) - if match: - if verbose: - print(f"### Extracted this {block_type} comment block:\n") - s = '\n'.join([(line[2:] if len(line)>1 else "") - for line in match.group('content').split('\n')]) - print(textwrap.indent(s,'\t')) - print() - if block_type == 'script': - (pip_env, conda_env) = parse_script_toml(extract_content(match)) - boxed_pip_block[0] = pip_env - boxed_conda_spec_block[0] = conda_env - break - else: - boxed_content[0] = extract_content(match) - break - - hash = hashlib.md5() - hash.update(boxed_pip_block[0].encode('utf-8')) - hash.update(boxed_conda_env_block[0].encode('utf-8')) - hash.update(boxed_conda_spec_block[0].encode('utf-8')) - return (hash.hexdigest(), boxed_pip_block[0], boxed_conda_env_block[0], boxed_conda_spec_block[0]) - -def tomlconfig_to_pip_conda(toml_config) -> tuple[str,str]: - "From a TOML dict, to (pip reqs, conda python spec)" - if 'requires-python' in toml_config: - conda_python_install_spec = f"python{toml_config['requires-python']}" - else: - conda_python_install_spec = '' - if 'dependencies' in toml_config: - pip_reqs = '\n'.join(toml_config['dependencies']) + '\n' - else: - pip_reqs = '' - return (pip_reqs,conda_python_install_spec) - -def parse_script_toml(toml_str) -> tuple[str,str]: - """ - From script TOML text, to (pip_reqs,conda python spec). - - This parses the TOML fragment in a PEP723 metadata block where TYPE=script. - Uses a limited custom parser to neeed only Python 3.9.6 and zero deps. - """ - toml_str = re.sub(r'#.*$', '', toml_str, flags=re.MULTILINE) - config = {} - requires_python_match = re.search(r'requires-python\s*=\s*"([^"]*)"', toml_str) - if requires_python_match: - config['requires-python'] = requires_python_match.group(1) - dependencies_match = re.search(r'dependencies\s*=\s*\[(.*?)\]', toml_str, re.DOTALL) - if dependencies_match: - dependencies_str = dependencies_match.group(1) - dependencies = re.findall(r'"([^"]*)"', dependencies_str) - config['dependencies'] = dependencies - return tomlconfig_to_pip_conda(config) - -class Project(ABC): - @staticmethod - def make_project(script:str, verbose:bool, dry_run:bool): - (dep_hash, pip_requirements, conda_envyml, conda_specs ) = parse_dependencies(script,verbose or dry_run) - if conda_envyml or conda_specs: - logging.info("dep block implies script will need conda for an environment.yml or conda_specs installation") - return ProjectConda(script, dep_hash, pip_requirements, conda_specs, conda_envyml, verbose) - elif pip_requirements: - logging.info("dep block implies script will need only venv + pip") - return ProjectPip(script, dep_hash, pip_requirements,conda_specs, conda_envyml, verbose) - else: - logging.info("no valid dep block found. no environment needed") - return ProjectNoDeps(script,dep_hash, pip_requirements,conda_specs, conda_envyml, verbose) - - def __init__(self, script:str, dep_hash, pip_requirements:str, conda_specs:str, conda_envyml:str, verbose:bool): - assert isinstance(conda_specs,str), "Bad input" - self.script = script - self.dep_hash = dep_hash - self.pip_requirements = pip_requirements - self.conda_specs = conda_specs - self.conda_envyml = conda_envyml - self.verbose = verbose - - @property - def project_path(self): - "path to the project dir" - return os.path.join( cache_base(), self.dep_hash ) - @property - def envdir(self) -> str: - "for pip projects, the venv dir. for conda, the prefix dir" - return "" - @property - def interpreter(self) -> str: - return os.path.join( self.envdir, 'bin','python3') - def exists(self) -> bool: - return False - def create(self) -> bool: - "False if creation failed, maybe leaving self.project_path in a non-runnable state" - return True - def run(self, args) -> NoReturn: - run_script(self.interpreter,self.script,args) - -def log_level_for_verbose(v:bool) -> Log: - return Log.VERBOSE if v else Log.ERRORS - -class ProjectPip(Project): - @property - def envdir(self): return os.path.join( self.project_path, 'venv' ) - def exists(self): return os.path.exists( self.project_path ) - def create(self): - return create_venv(self.project_path, self.envdir, - self.pip_requirements, - log_level_for_verbose(self.verbose)) - - -class ProjectConda(Project): - @property - def envdir(self): return os.path.join( self.project_path, 'condaenv' ) - def exists(self): return os.path.exists( self.project_path ) - def create(self): - return setup_conda_prefix(self.project_path, self.envdir, - self.conda_envyml, - self.conda_specs, - self.pip_requirements, - log_level_for_verbose(self.verbose)) - def run(self, args) -> NoReturn: - conda_run_script(self.interpreter,self.script,args,self.envdir) - -class ProjectNoDeps(Project): - def exists(self): return True - def create(self): return True - @property - def interpreter(self): - return sys.executable - -def run_with_logging(command:Union[str,list],proj_dir,out_f,err_f,verbosity): - ''' - Runs command. Logs and maybe streams stdout and stderr. - - verbosity=Log.SILENT: log out and err. Report errors later - verbosity=Log.VERBOSE: log and stream out and err. - ''' - log_dir = os.path.join(proj_dir,"logs") - os.makedirs(log_dir,exist_ok=True) - out_f = os.path.join(log_dir, os.path.basename(out_f)) - err_f = os.path.join(log_dir, os.path.basename(err_f)) - if isinstance(command,list): - command = shlex.join(command) - if verbosity == Log.SILENT: - command += f' 2>> "{err_f}"' - command += f' 1>> "{out_f}"' - elif verbosity == Log.ERRORS: - command += f' 2> >(tee -a "{err_f}")' - command += f' 1>> {out_f}' - elif verbosity == Log.VERBOSE: - command += f' 2> >(tee -a "{err_f}")' - command += f' 1> >(tee -a "{out_f}")' - else: - assert True, "unreachable" - - cp = subprocess.run(command, - shell=True, - executable=shutil.which('bash')) - did_succeed = (cp.returncode == 0) - - if (verbosity, did_succeed) == (Log.VERBOSE,True): - print(f"## This command completed successfully:\n\t{command}") - elif (verbosity, did_succeed) == (Log.VERBOSE,False): - print(f"## This command failed:\n\t{command}\n", file=sys.stderr) - print(f"## Standard error output was printed above\n", file=sys.stderr) - print(f"## Logs may be found in:\n\t{proj_dir}/logs", file=sys.stderr) - elif (verbosity, did_succeed) == (Log.ERRORS,True): - pass - elif (verbosity, did_succeed) == (Log.ERRORS,False): - print(f"## Error encountered trying to run this command:\n\t{command}", file=sys.stderr) - print(f"## Logs may be found in:\n\t{proj_dir}/logs\n", file=sys.stderr) - print(f"## This is the contents of the stderr:\n") - with open(err_f,"r") as f: - print(f.read()) - else: - pass - - return did_succeed - - -def create_conda_prefix(proj_dir,condaprefix_dir:str,log_level:Log): - success = run_with_logging(f'conda create --quiet --yes --prefix "{condaprefix_dir}"', - proj_dir, - "conda_create.out","conda_create.err", - log_level) - if success: - return True - else: - print("## Errors trying to create conda prefix directory",file=sys.stderr) - return False - - -def pseudo_erase_dir(path): - "Pseudo-erases a project dir by moving it to the temporary dir" - logging.info(f"Moving {path} to {trash_base()}") - dst = os.path.join( trash_base(), os.path.basename(path), str(uuid.uuid4()) ) - return shutil.move(path, dst ) - - -def install_pip_requirements(proj_dir, pip_requirements, interpreter, log_level:Log) -> bool: - reqs_path = os.path.join(proj_dir,'requirements.txt') - with open(reqs_path, 'w') as f: - f.write(pip_requirements) - - success = run_with_logging([interpreter, "-m", "pip", "install", "-r", reqs_path], - proj_dir, - "pip_install.out","pip_install.err", - log_level) - if success: - with open(os.path.join(proj_dir,"piplist.txt"),"w") as f: - subprocess.run(shlex.join([interpreter, "-m", "pip", "list"]), - stdout=f, stderr=f, - shell=True,executable=shutil.which('bash')) - return True - else: - print("## Errors trying to install pip requirements",file=sys.stderr) - return False - - -def make_conda_install_yml_command(condaprefix_dir, env_yml_file) -> str: - return f'conda env create --quiet --yes --file "{env_yml_file}" --prefix "{condaprefix_dir}"' - -def make_conda_install_spec_command(condaprefix_dir, install_spec_file) -> str: - return f'conda install --quiet --yes --file "{install_spec_file}" --prefix "{condaprefix_dir}"' - -def setup_conda_prefix(proj_dir:str, condaprefix_dir:str, - conda_envyml:str, - conda_specs:str, - pip_requirements, - log_level:Log) -> bool: - logging.info(f"creating conda prefix {condaprefix_dir}") - create_conda_prefix(proj_dir, condaprefix_dir, log_level) - - success = False - if conda_envyml: - install_env_f = os.path.join(proj_dir,'environment.yml') - with open(install_env_f, 'w') as f: - f.write(conda_envyml) - command_to_run = make_conda_install_yml_command(condaprefix_dir, install_env_f) - success = run_with_logging(command_to_run, - proj_dir, - "conda_env_create_f.out","conda_env_create_f.err", - log_level) - elif conda_specs: - install_spec_f = os.path.join(proj_dir,'conda_install_specs.txt') - with open(install_spec_f, 'w') as f: - f.write(conda_specs) - command_to_run = make_conda_install_spec_command(condaprefix_dir, install_spec_f) - success = run_with_logging(command_to_run, - proj_dir, - "conda_install.out","conda_install.err", - log_level) - else: - assert True, "unreachable. " - if success: - with open(os.path.join(proj_dir,"exported-environment.yml"),"w") as f: - cmd = ["conda","env","export","--quiet", "--prefix",condaprefix_dir] - logging.info(f"exporting env with {cmd}") - subprocess.run(shlex.join(cmd), - stdout=f, stderr=f, - shell=True,executable=shutil.which('bash')) - else: - print("## Errors trying to install conda dependencies",file=sys.stderr) - return False - - if pip_requirements: - interpreter = os.path.join(condaprefix_dir, 'bin','python3') - return install_pip_requirements(proj_dir,pip_requirements, - interpreter, - log_level) - else: - return True - - -def run_script(interpreter, script, args) -> NoReturn: - logging.info( - f"running {script} using {interpreter} with args: {args}" - ) - sys.stdout.flush() - logging.info(f'os.execvp({interpreter}, [{interpreter},{script}] + {args})') - os.execvp(interpreter, [interpreter,script] + args) - -def conda_run_script(interpreter, script, args, conda_env_dir) -> NoReturn: - logging.info( - f"using conda run to run {script} using {interpreter} with args: {args}" - ) - logging.info(f'os.execvp({interpreter}, [{interpreter},{script}] + {args})') - # to workaround the conda bug https://github.com/conda/conda/issues/13639 - should_use_wrapper = True - if should_use_wrapper: - workaround_path = os.path.join(conda_env_dir,'exec_script') - with open(workaround_path,'w') as f: - workaround_script = f"exec {interpreter} {script}" - for arg in args: - workaround_script += f" {arg}" - workaround_script += "\n" - logging.info(f'builiding script with contents: {workaround_script}') - logging.info(f'writing script to path: {workaround_path}') - f.write(workaround_script) - os.chmod(workaround_path, 0o755) - cmd = ["conda","run","-p", conda_env_dir, "--no-capture-output", workaround_path] - else: - cmd = ["conda","run","-p", conda_env_dir, "--no-capture-output", interpreter,script] + args - sys.stdout.flush() - os.execvp(cmd[0],cmd) - -# -# venv operations -# - -def create_venv(proj_dir, venv_dir, pip_requirements, log_level:Log) -> bool: - "Creates a script project dir for script at script_path" - logging.info(f"Creating venv at {venv_dir}") - - success = run_with_logging(["python3", "-m", "venv", venv_dir], - proj_dir, - "create_venv.out","creat_evenv.err", - log_level) - if not success: - print(f"## Error trying to create venv",file=sys.stderr) - return False - - if pip_requirements: - interpreter = os.path.join(venv_dir, 'bin','python3') - return install_pip_requirements(proj_dir, - pip_requirements, - interpreter,log_level) - else: - return True - - -# -# helpers -# - -def clean_name_from_path(p): - return re.sub(r'[^A-Za-z0-9-]', '', os.path.basename(p)) - - -def print_base_dirs(): - print(f"Cached project directores are in:\n{cache_base()}\n\n") - print(f"Each directory's contains logs and other build artifacts.\n\n") - print(f"Trashed and cleaned projects are here, waiting for disposal by the OS:\n{trash_base()}") - - -def trash_base() -> str: - "Directory to use for trashing broken project dirs" - return os.path.join( tempfile.gettempdir(), "pythonrunscript" ) - -def print_python3_path(): - if p := shutil.which('python3'): - print(f"## The first python3 in your PATH: {p}") - else: - print("## There is no python3 in your PATH!") - -def cache_base(): - cache_base = None - if "XDG_CACHE_HOME" in os.environ: - cache_base = os.environ["XDG_CACHE_HOME"] - elif platform.system() == "Darwin": - cache_base = os.path.join(os.path.expanduser("~"), "Library", "Caches") - else: - cache_base = os.path.join(os.path.expanduser("~"), ".cache") - cache_base = os.path.join(cache_base, "pythonrunscript") - return cache_base - -def are_dependencies_missing() -> bool: - return (platform.system() not in ['Linux','Darwin'] - and (shutil.which('bash') is None - or shutil.which('tee') is None)) - -if __name__ == "__main__": - main() - diff --git a/ruff.toml b/ruff.toml deleted file mode 100644 index 687d73f..0000000 --- a/ruff.toml +++ /dev/null @@ -1 +0,0 @@ -ignore = ["E401", "F401", "E701", "E722", "F541"] diff --git a/tests/run_conda.py b/tests/run_conda.py deleted file mode 100755 index d1720bd..0000000 --- a/tests/run_conda.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -import os, logging, sys -from typing import NoReturn - -# The purpose of this script -# is to verify how to call an executable in a specified conda env -# -# conda run seems to have a bug, where it grabs arguments intended to be arguments for the executable -# -# findings: -# - `./run_conda.py test ffmpeg -version`: -# - fails, because conda tries to parse -version rather than letting fmpeg use it -# - this is a known bug: https://github.com/conda/conda/issues/13639 -# - workaround: auto-generate an exec script with the args, and call that from conda??? -# - `./run_conda.py test ffmpeg --version`: -# - works, because -# - ./run_conda test ffmpeg --help: -# - works -# -def conda_run_script(interpreter, script, args,conda_env_dir) -> NoReturn: - logging.info( - f"using conda run to run {script} using {interpreter} with args: {args}" - ) - sys.stdout.flush() - logging.info(f'os.execvp({interpreter}, [{interpreter},{script}] + {args})') - cmd = ["conda","run","--name",conda_env_dir,interpreter,script] + args - print(f"os.execvp: {cmd=}") - sys.stdout.flush() - os.execvp(cmd[0],cmd) - -if __name__ == "__main__": - if len(sys.argv) < 4: - print(f"usage: {sys.argv[0]} conda_env_path executable_path executable_args...") - sys.exit(1) - print(f"{sys.argv=}") - conda_run_script(interpreter=sys.argv[2], - script=sys.argv[3], - args=sys.argv[4:], - conda_env_dir=sys.argv[1]) diff --git a/tests/run_ffmpeg.py b/tests/run_ffmpeg.py deleted file mode 100755 index 9227f0b..0000000 --- a/tests/run_ffmpeg.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env pythonrunscript -# ```conda_install_specs.txt -# python=3.10 -# ffmpeg -# ``` - -# if pythonrunscript works properly, -# then ffmpeg should be run properly -# -# this shows pythonrunscript is not only running scripts with the python -# interpreter associated with the ad-hoc conda environment, but that it -# is reproducing the PATH variable and other parts of the environment, -# which is needed to allow the full benefit of conda deps. - -import os, logging, sys -from typing import NoReturn - -print("top-line") -sys.stdout.flush() - -if __name__ == "__main__": - print("main: ENTRY") - print(f"{sys.argv=}") - cmd = sys.argv - cmd[0] = "ffmpeg" - print(f"{cmd=}") - sys.stdout.flush() - os.execvp(cmd[0],cmd) - - diff --git a/tests/run_python3.py b/tests/run_python3.py deleted file mode 100755 index 0c8f019..0000000 --- a/tests/run_python3.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env pythonrunscript -# ```conda_install_specs.txt -# python=3.10 -# ffmpeg -# ``` - -# if pythonrunscript works properly, -# then it should be possible to run -# an interactive executable such -# as python itself, from a script, -# as usual - -import os, logging, sys -from typing import NoReturn - -print("top-line") -sys.stdout.flush() - -if __name__ == "__main__": - print("main: ENTRY") - print(f"{sys.argv=}") - cmd = sys.argv - cmd[0] = "python3" - print(f"{cmd=}") - sys.stdout.flush() - os.execvp(cmd[0],cmd) - - diff --git a/tests/test_parse_dependencies.py b/tests/test_parse_dependencies.py deleted file mode 100644 index 60a1987..0000000 --- a/tests/test_parse_dependencies.py +++ /dev/null @@ -1,160 +0,0 @@ -import pytest, os, tempfile, textwrap, tomlkit # noqa: E401 -from pythonrunscript.pythonrunscript import parse_dependencies, parse_script_toml, tomlconfig_to_pip_conda - -from typing import NamedTuple - -class Expected(NamedTuple): - reqs: str - cis: str - envyml: str - -tests: list[tuple[str, Expected]] = [ - ## requirements, 1-line - ( - """ - # ```requirements.txt - # requests==2.26.0 - # ``` - """, - Expected( - reqs=""" - requests==2.26.0 - """, - cis="", - envyml="") - ), - ## conda_install_sepcs, 1-line - ( - """ - # ```conda_install_specs.txt - # requests==2.27.0 - # ``` - """, - Expected( - reqs=""" - """, - cis=""" - requests==2.27.0 - """, - envyml="") - ), - ## one-line requirements - ( - """ - # ```requirements.txt - # requests==2.26.0 - # ``` - # - # ```conda_install_specs.txt - # python=<3.11 - # ``` - """, - Expected( - reqs=""" - requests==2.26.0 - """, - cis="python=<3.11\n", - envyml="") - ), - ## 2-line conda - ( - """ - # ```conda_install_specs.txt - # python=3.11 - # nbclassic - # ``` - """, - Expected( - reqs="", - cis=""" - python=3.11 - nbclassic - """, - envyml="") - ), - ## script - ( - """ - # /// script - # requires-python = ">=3.11" - # dependencies = ["requests<3", "rich", ] - # /// - """, - Expected( - reqs=""" - requests<3 - rich - """, - cis="python>=3.11", - envyml="" - ) - ) -] -def f(s): - return textwrap.dedent(s).lstrip() -tests = [(f(a),(f(b),f(c),f(d))) for (a,(b,c,d)) in tests] # type: ignore -del f - -@pytest.mark.parametrize("test_index", list(range(len(tests)))) -def test_parse_dependencies_basic(test_index:int): - to_run = tests[test_index:test_index+1] - print(f"{to_run=}") - for (input,(expected_pip_val,expected_conda_specs,expected_conda_env)) in to_run: - (_, out_pip, out_conda_env, out_conda_specs) = (None,None,None,None) - p = os.path.join( tempfile.gettempdir(), "test_script.py" ) - with open(p, 'w') as f: - f.write(input) - try: - # Call the function - result = parse_dependencies(p) - - # Assert the expected results - assert isinstance(result, tuple) - assert len(result) == 4 - (_, out_pip, out_conda_env, out_conda_specs) = result - assert out_pip == expected_pip_val - assert out_conda_env == expected_conda_env - assert out_conda_specs == expected_conda_specs - finally: - # Clean up the temporary file - os.remove(p) - -toml_inputs:list[str] = [ -""" -requires-python = ">=3.11" -dependencies = ["requests<3", "rich", ] -""", -""" -requires-python = ">=3.11" -dependencies = [ - "requests<3", - "rich", -] -""", -""" -requires-python = ">=3.11" -dependencies = [ - "requests<3", #comment - # comment -"rich", -] -""", -] - -@pytest.mark.parametrize("toml_test_index", list(range(len(toml_inputs)))) -def test_parse_toml(toml_test_index): - s = toml_inputs[toml_test_index] - (out_pip,out_conda) = parse_script_toml(s) - - # test if we're parsing like tomlkit - config = tomlkit.parse(s) - (kit_pip_deps,kit_conda_python_spec) = tomlconfig_to_pip_conda(config) - assert kit_pip_deps == out_pip - assert kit_conda_python_spec == out_conda - - exp_conda_specs = "python>=3.11" - exp_pip = """requests<3 -rich -""" - assert exp_pip == out_pip, "unexpected requirements generated from script TOML" - assert exp_conda_specs == out_conda, "unexpected conda install specs generated from script TOML"