diff --git a/pkgs/modules/python-base/default.nix b/pkgs/modules/python-base/default.nix index b324b80c..d1d7ba98 100644 --- a/pkgs/modules/python-base/default.nix +++ b/pkgs/modules/python-base/default.nix @@ -16,6 +16,10 @@ let sitecustomize = pkgs.callPackage ../python/sitecustomize.nix { }; + poetry = pkgs.callPackage ../../poetry { + inherit python; + }; + binary-wrapped-python = pkgs.callPackage ../../python-wrapped { inherit pkgs python python-ld-library-path; }; @@ -34,7 +38,7 @@ in replit.packages = [ binary-wrapped-python pypkgs.pip - pkgs.poetry + poetry pkgs.uv ]; diff --git a/pkgs/modules/python/default.nix b/pkgs/modules/python/default.nix index efceb8fd..25592f4e 100644 --- a/pkgs/modules/python/default.nix +++ b/pkgs/modules/python/default.nix @@ -29,9 +29,8 @@ let name = "pip"; }; - poetry = pkgs.callPackage (../../poetry/poetry-py + "${pythonVersion}.nix") { + poetry = pkgs.callPackage ../../poetry { inherit python; - inherit pypkgs; }; poetry-config = pkgs.writeTextFile { @@ -118,13 +117,6 @@ in POETRY_CONFIG_DIR = poetry-config.outPath; POETRY_CACHE_DIR = "$REPL_HOME/.cache/pypoetry"; POETRY_VIRTUALENVS_CREATE = "0"; - POETRY_INSTALLER_MODERN_INSTALLATION = "1"; - POETRY_DOWNLOAD_WITH_CURL = "1"; - POETRY_PIP_USE_PIP_CACHE = "1"; - POETRY_PIP_NO_ISOLATE = "1"; - POETRY_PIP_NO_PREFIX = "1"; - POETRY_PIP_FROM_PATH = "1"; - POETRY_USE_USER_SITE = "1"; PIP_CONFIG_FILE = pip.config.outPath; PYTHONUSERBASE = userbase; PYTHONPATH = "${sitecustomize}:${pip.pip}/${python.sitePackages}"; diff --git a/pkgs/poetry/README.md b/pkgs/poetry/README.md index 1b290725..25ffd638 100644 --- a/pkgs/poetry/README.md +++ b/pkgs/poetry/README.md @@ -1,68 +1,21 @@ -# Replit Custom Poetry Nix package +# Poetry packaging -## Why do we need this? +We now use upstream nixpkgs Poetry directly instead of the old Replit fork and +prebuilt bundle tarballs. -We have some custom changes to: +## How it works -* [Poetry](https://github.com/replit/poetry) - for some performance optimizations when -resolving what packages to fetch (avoid downloading .tar.gz source distributions and building them) -* [Pip](https://github.com/replit/pip) - for integration with our Python package cacache +`pkgs/poetry/default.nix` keeps a thin wrapper around nixpkgs `poetry` for two +reasons: -Previously/currently we had an inelegent way of pre-installing these packages in the virtual environment -of the Repl in replspace. See https://replit.com/@util/Pythonify for details. Instead of this, we want to have one install path via Nix that gets these two custom programs into a Repl without -having to install them into replspace. The python Nix module will bring it all together. +* build Poetry against the same Python minor version as the module's project + Python via `pkgs.poetry.override { python3 = python; }` +* keep the low-vCPU installer parallelism guard we already use in Repls -## Add A new Python Version +The Python module still sets `POETRY_VIRTUALENVS_CREATE=0`, `PYTHONUSERBASE`, +and `PATH` so Poetry installs into `.pythonlibs` in the same place as pip. -If you want to add a new version of Python to Nix modules, you'll need to create a Poetry bundle specifically -for it. To do so, follow the instructions at https://github.com/replit/poetry#bundle and then -Added a new file named `poetry-py.nix` to this directory using the existing ones as -examples. Tip: empty the `sha256` field before attempting to build it, and then update it with -the sha value reported by Nix. +## Add a new Python version -## Problems - -Things that make this difficult are: - -* in order to prevent poetry from including its own dependencies (requests, in particular) during its operation -we need to install it inside its own virtual environment the way their [official installer](https://python-poetry.org/docs/#installing-with-the-official-installer) does. Using this workaround: https://github.com/replit/poetry/blob/replit-1.1/poetry/utils/env.py#L885 poetry can use the environment of the project for its operations, ignoring its own environment -* creating a virtual environment in replspace on behave of the user has a few downsides: - 1. somewhat slow to initialize the env ~2 second - 2. when poetry creates a virtual environment, it automatically installs a stock version - of pip which is not our own. We'd have to add customization to poetry to override the pip version - 3. the generated environment contains a config file `pyvenv.cfg` that has a reference to the path of the - python executable, which in our case would be coming from the `/nix/store` directory. It breaks if we use - a different version of python with this env - -## How does it work? - -1. For pip (`pkgs/pip/default.nix`), we'll install it using the buildPythonPackage helper -2. For poetry: - * we download a poetry bundle tarball from gcs which contains our version of poetry + its dependencies and then - use pip's [offline install scheme](https://stackoverflow.com/questions/36725843/installing-python-packages-without-internet-and-using-source-code-as-tar-gz-and) to install it into a virtual env: this is how we isolate its deps away - from the user's project deps, and how the official poetry installer does it. - The tarball is built in the https://github.com/replit/poetry repo. See https://github.com/replit/poetry/pull/4 - for details of how it is built. -3. Inside a repl - a. the custom pip and poetry will be made available to the user via the `PATH` variable. - b. pip will be instructed to install packages via -[user install mode](https://pip.pypa.io/en/stable/user_guide/#user-installs) via the `PIP_USER` variable. This is commonly used for shared Linux systems where the user does not have write access -to the system Python's site-packages directory. - c. The `PYTHONUSERBASE` variable will tell pip where to install packages -in user mode, and we'll point it to a directory in replspace `$HOME/$REPL_SLUG/.pythonlibs`. - d. The site package directory -within `PYTHONUSERBASE`: `$HOME/$REPL_SLUG/.pythonlibs/lib/python3.10/site-packages` is added to the `PYTHONPATH` -variable so it gets into python's module search path. - e. `POETRY_VIRTUALENVS_CREATE` is set to false to instruct poetry not to create a virtual environment - -## Known Issue - -The python module way of calling poetry: `python -m poetry add requests` will no longer work, this means -UPM will need to be updated to call poetry via its binary. While we'd like this to work because there are -existing users using this technique, I don't think it's feasible if we want to isolate poetry's dependencies -away from the user project. - -To allow the above would mean allowing the user to be able to `import poetry` from python. That would mean -adding poetry and its dependencies to `PYTHONPATH` or `sys.path`, which would in turn mean its resolver -would access those dependencies and treat them as belonging to the user project. Well, unless we add -further customizations to poetry... +No Poetry bundle needs to be generated anymore. Adding a new Python version only +requires wiring the new `python` / `pypkgs` pair into the relevant module. diff --git a/pkgs/poetry/poetry-in-venv.nix b/pkgs/poetry/default.nix similarity index 50% rename from pkgs/poetry/poetry-in-venv.nix rename to pkgs/poetry/default.nix index c668a064..3a768ef3 100644 --- a/pkgs/poetry/poetry-in-venv.nix +++ b/pkgs/poetry/default.nix @@ -1,24 +1,7 @@ -{ pkgs, python, pypkgs, version, url, sha256 }: +{ pkgs, python }: let - poetry = pkgs.stdenv.mkDerivation { - name = "poetry-in-venv"; - inherit version; - - src = builtins.fetchTarball { - inherit url sha256; - }; - - installPhase = '' - mkdir -p $out/bin - ${python}/bin/python3 -m venv $out/env - touch $out/env/poetry_env # This allows poetry to recognize it - # https://github.com/replit/poetry/blob/replit-1.5/src/poetry/utils/env.py#L1154 - # invoking the workaround so that poetry - # does not use its own venv for the project - # env - $out/env/bin/pip install poetry --find-links ./ --no-index - ln -s $out/env/bin/poetry $out/bin/poetry - ''; + upstreamPoetry = pkgs.poetry.override { + python3 = python; }; in pkgs.writeShellApplication { @@ -26,7 +9,7 @@ pkgs.writeShellApplication { text = '' # Determine how many vcpus we have, first in the standard location, with a fallback # to the cgroup info, since resources.json does not exist during Deployments. - if [ -e /repl/stats/resources.json ]; then # Common resources location + if [ -e /repl/stats/resources.json ]; then numVCpu="$(${pkgs.jq}/bin/jq .numVCpu