From e3c28dd0131e68ef5a8194c54b7d55bd2ccb45d1 Mon Sep 17 00:00:00 2001 From: Sergey Yedrikov Date: Thu, 27 Aug 2026 20:49:41 -0400 Subject: [PATCH] LCORE-3121: [Cleanup] scripts/konflux_requirements.sh vs scripts konflux_resolve.py LCORE-3367: Bug-free variant of scripts/konfux_resolve.py konfux_resolve.py now calls the variant of uv with support for the "prefer-index" package resolution policy from a container image. --- Makefile | 4 +- README.md | 2 +- scripts/konflux_requirements.sh | 93 --------------------------------- scripts/konflux_resolve.py | 21 ++++---- 4 files changed, 15 insertions(+), 105 deletions(-) delete mode 100755 scripts/konflux_requirements.sh mode change 100644 => 100755 scripts/konflux_resolve.py diff --git a/Makefile b/Makefile index 2fa421ada..8c9b9cde4 100644 --- a/Makefile +++ b/Makefile @@ -362,8 +362,8 @@ distribution-archives: ## Generate distribution archives to be uploaded into Pyt upload-distribution-archives: ## Upload distribution archives into Python registry uv run python -m twine upload --repository ${PYTHON_REGISTRY} dist/* -konflux-requirements: ## Generate hermetic requirements.*.txt file for Konflux build - ./scripts/konflux_requirements.sh +konflux-requirements: ## Generate .konflux/requirements.*.txt files for Konflux hermetic build + scripts/konflux_resolve.py --profile cpu konflux-rpm-lock: ## Generate rpm.lock.yaml file for Konflux build ./scripts/generate-rpm-lock.sh diff --git a/README.md b/README.md index d2bd85afb..f6c5f1a59 100644 --- a/README.md +++ b/README.md @@ -1515,7 +1515,7 @@ Update these files when you: make konflux-requirements ``` -This compiles Python dependencies from `pyproject.toml` using `uv`, splits packages by their source index (PyPI vs Red Hat's internal registry), and generates hermetic requirements files with pinned versions and hashes for Konflux builds. +This compiles Python dependencies from `pyproject.toml` using a fork of `uv` that supports a "prefer-index" package version resolution policy, splits packages by their source index (PyPI vs Red Hat's internal registry), and generates hermetic requirements files with pinned versions and hashes for Konflux builds. **Files produced:** - `.konflux/requirements.hashes.source.txt` – PyPI packages with hashes diff --git a/scripts/konflux_requirements.sh b/scripts/konflux_requirements.sh deleted file mode 100755 index 96e66b5de..000000000 --- a/scripts/konflux_requirements.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash - -# Script to split requirements by index source -# Packages from pypi.org go to requirements.source.txt -# Packages from console.redhat.com go to requirements.wheel.txt - -set -x - -KONFLUX_DIR=".konflux" -RAW_REQ_FILE="requirements.no_hashes.txt" -SOURCE_FILE="${KONFLUX_DIR}/requirements.source.txt" -WHEEL_FILE="${KONFLUX_DIR}/requirements.wheel.txt" -SOURCE_HASH_FILE="${KONFLUX_DIR}/requirements.hashes.source.txt" -WHEEL_HASH_FILE="${KONFLUX_DIR}/requirements.hashes.wheel.txt" -BUILD_FILE="${KONFLUX_DIR}/requirements-build.txt" -RHOAI_INDEX_URL="https://console.redhat.com/api/pypi/public-rhai/rhoai/3.3/cpu-ubi9/simple/" - -# extra wheels to be included in the wheel list, often come from build-time dependencies -EXTRA_WHEELS="uv,pip,maturin" -# packages to exclude from the wheel list -NO_WHEEL_PACKAGES="markupsafe" - -# Generate requirements list from pyproject.toml from both indexes -uv pip compile pyproject.toml -o "$RAW_REQ_FILE" \ - --group ogxlibdev \ - --python-platform x86_64-unknown-linux-gnu \ - --python-version 3.12 \ - --refresh \ - --index ${RHOAI_INDEX_URL} \ - --default-index https://pypi.org/simple/ \ - --index-strategy unsafe-best-match \ - --emit-index-annotation \ - --no-sources \ - --override "${KONFLUX_DIR}/requirements.overrides.txt" - -# Initialize output files -echo "# Packages from pypi.org" > "$SOURCE_FILE" -echo "# This file was autogenerated by split_requirements.sh" >> "$SOURCE_FILE" -echo "# Packages from console.redhat.com" > "$WHEEL_FILE" -echo "# This file was autogenerated by split_requirements.sh" >> "$WHEEL_FILE" -echo "--index-url ${RHOAI_INDEX_URL}" >> "$WHEEL_FILE" - -current_package="" - -while IFS= read -r line || [[ -n "$line" ]]; do - # Check if this is a package line (starts with a letter/digit, not whitespace or #) - if [[ "$line" =~ ^[a-zA-Z0-9] ]]; then - current_package="$line" - package_name="${current_package%%==*}" - # Check if this is a "# from" annotation line - elif [[ "$line" =~ ^[[:space:]]*#[[:space:]]*from[[:space:]]+(.*) ]]; then - index_url="${BASH_REMATCH[1]}" - - if [[ -n "$current_package" ]]; then - if [[ "$index_url" == "https://pypi.org/simple/" ]]; then - echo "$current_package" >> "$SOURCE_FILE" - elif [[ "$NO_WHEEL_PACKAGES" == *"$package_name"* ]]; then - echo "$current_package" >> "$SOURCE_FILE" - elif [[ "$index_url" == "${RHOAI_INDEX_URL}" ]]; then - echo "$current_package" >> "$WHEEL_FILE" - fi - current_package="" - fi - fi -done < "$RAW_REQ_FILE" - -# replace the list of binary packages in konflux pipeline configuration -# only the package names, not the versions, delimited by commas -wheel_packages=$(grep -v "^[#-]" "$WHEEL_FILE" | sed 's/==.*//' | tr '\n' ',' | sed 's/,$//') -# append extra wheels to the list -wheel_packages="$wheel_packages,$EXTRA_WHEELS" -sed -i 's/"packages": "[^"]*"/"packages": "'"$wheel_packages"'"/' .tekton/lightspeed-stack-pull-request.yaml -sed -i 's/"packages": "[^"]*"/"packages": "'"$wheel_packages"'"/' .tekton/lightspeed-stack-push.yaml - -echo "Packages from pypi.org written to: $SOURCE_FILE ($(wc -l < "$SOURCE_FILE") packages)" -echo "Packages from console.redhat.com written to: $WHEEL_FILE ($(wc -l < "$WHEEL_FILE") packages)" - -uv pip compile "$WHEEL_FILE" --refresh --generate-hashes --index-url "${RHOAI_INDEX_URL}" --python-version 3.12 --emit-index-url --no-deps --no-annotate --universal > "$WHEEL_HASH_FILE" -uv pip compile "$SOURCE_FILE" --refresh --generate-hashes --python-version 3.12 --emit-index-url --no-deps --no-annotate > "$SOURCE_HASH_FILE" -uv run pybuild-deps compile --output-file="$BUILD_FILE" "$SOURCE_FILE" - -# pin maturin to the version available in the Red Hat registry -# LCORE-2324: commented out (temporarily?) -# sed -i 's/maturin==[0-9.]*/maturin==1.10.2/' "$BUILD_FILE" - -# remove intermediate files -rm "$RAW_REQ_FILE" "$WHEEL_FILE" "$SOURCE_FILE" - -echo "Done!" -echo "Packages from pypi.org written to: $SOURCE_HASH_FILE ($(wc -l < "$SOURCE_HASH_FILE") packages)" -echo "Packages from console.redhat.com written to: $WHEEL_HASH_FILE ($(wc -l < "$WHEEL_HASH_FILE") packages)" -echo "Build dependencies written to: $BUILD_FILE ($(wc -l < "$BUILD_FILE") packages)" -echo "Remember to commit $SOURCE_HASH_FILE, $WHEEL_HASH_FILE, $BUILD_FILE, pipeline configurations and push the changes" diff --git a/scripts/konflux_resolve.py b/scripts/konflux_resolve.py old mode 100644 new mode 100755 index e3a4836af..92312ec75 --- a/scripts/konflux_resolve.py +++ b/scripts/konflux_resolve.py @@ -985,13 +985,8 @@ def load_wheel_only(path: str) -> set[str]: _UV_COMPILED_RE = re.compile(r"^([a-zA-Z0-9][a-zA-Z0-9._-]*)([=<>!~].*)?$") - -UV_BINARY = os.environ.get( - "UV_BINARY", - os.path.join( - os.path.dirname(__file__), "..", "..", "uv", "target", "release", "uv" - ), -) +UV_IMAGE = os.environ.get("UV_IMAGE", "quay.io/syedriko/uv:prefer-index") +CONTAINER_RUNTIME = os.environ.get("CONTAINER_RUNTIME", "podman") def uv_resolve( @@ -1012,9 +1007,15 @@ def uv_resolve( else "requirements.overrides.txt" ), ) - uv = UV_BINARY if os.path.isfile(UV_BINARY) else "uv" cmd = [ - uv, + CONTAINER_RUNTIME, + "run", + "--rm", + "--volume", + f"{os.getcwd()}:/io:ro", + "--workdir", + "/io", + UV_IMAGE, "pip", "compile", "pyproject.toml", @@ -1027,6 +1028,8 @@ def uv_resolve( rhoai_index_url, "--default-index", "https://pypi.org/simple/", + "--index-strategy", + "prefer-index", "--emit-index-annotation", "--no-sources", "--group",