Python 3.13 support 02 base images

.github/cloud_builder/run_command_on_active_checkout.yaml

@@ -3,7 +3,7 @@ substitutions:
 options:
   logging: CLOUD_LOGGING_ONLY
 steps:
-  - name: us-central1-docker.pkg.dev/external-snap-ci-github-gigl/gigl-base-images/gigl-builder:7d3182eeb6446ce3e35910babba990c8e003879d.109.1
+  - name: us-central1-docker.pkg.dev/external-snap-ci-github-gigl/gigl-base-images/gigl-builder:0c41d2857437453a651888d02978071c5c3ab1e2.113.1
     entrypoint: /bin/bash
     # Route sbt through Google's Maven Central mirror to avoid 429 rate limits from repo1.maven.org.
     # Intentionally set here (CI env) rather than in scala/.sbtopts or scala_spark35/.sbtopts to avoid

Makefile

@@ -173,8 +173,14 @@ format_cpp:
 
 format: format_py format_cpp format_scala format_md
 
+# ty resolves the stdlib against a single Python version per invocation, so each end of
+# the supported range needs its own pass.
 type_check:
+# Floor: 3.11, from [tool.ty.environment] in pyproject.toml. Catches stdlib APIs that do
+# not exist that far back.
 	uv run ty check ${PYTHON_DIRS}
+# Ceiling: catches stdlib modules and signatures 3.13 removed or changed.
+	uv run ty check --python-version 3.13 ${PYTHON_DIRS}
 
 build_cpp_extensions:
 	$(MAKE) -C gigl-core build_cpp_extensions

containers/Dockerfile.cuda.base

@@ -1,15 +1,34 @@
 # syntax=docker/dockerfile:1
 
 
-FROM pytorch/pytorch:2.8.0-cuda12.8-cudnn9-devel
+# The `-devel` variant ships nvcc at /usr/local/cuda/bin/nvcc, which both gigl-core's
+# CMake build and GraphLearn-for-PyTorch's CUDA build need while installing dependencies;
+# a runtime-only CUDA image cannot compile them. Its CUDA 12.8 toolkit has to stay matched
+# to the cu128 torch wheels pinned in uv.lock, because those two builds compile against
+# the toolkit here and link the wheels' runtime.
+# The image's own torch and Python 3.12 go unused: GiGL is `requires-python = "==3.11.*"`,
+# so requirements/install_py_deps.sh builds its own venv at /gigl_deps/.venv on the
+# interpreter named in .python-version, and the ENV PATH below puts that venv's bin ahead
+# of the image's Python.
+# Pinned by digest because the version tag is floating: upstream rebuilds it in place.
+# The tag has a single-platform (linux/amd64) manifest, so this digest is that manifest's,
+# not an index's.
+FROM pytorch/pytorch:2.10.0-cuda12.8-cudnn9-devel@sha256:b574d4ccf6d8856a5d87dcadc667aa4f95dc18d337ef3a28d02b7b01897d7081
 
 SHELL ["/bin/bash", "-c"]
 ENV DEBIAN_FRONTEND=noninteractive
 
-# Already has python 3.11 installed - no need to install it again.
-# We use system python since it has packages pre-installed for us.
-ENV UV_SYSTEM_PYTHON=true
-ENV UV_PROJECT_ENVIRONMENT=/opt/conda/
+# The NVIDIA driver — libcuda.so.1 and libnvidia-ml.so.1 — ships in no image: Vertex AI
+# and GKE bind-mount it into /usr/local/nvidia/lib{,64} at container start, so both
+# directories are empty while this image builds and nothing here can verify them.
+# Losing them fails two ways: fbgemm_gpu names libcuda.so.1 in DT_NEEDED across 19 of its
+# extensions, so `import torchrec` raises outright, while plain torch imports fine and
+# reports no CUDA device, which reads as a CPU machine rather than a broken image.
+# Prepended so the base's own entries survive, and set here rather than inherited because
+# upstream shuffles both variables between releases —
+# pytorch/pytorch:2.11.0-cuda12.8-cudnn9-devel already puts /usr/local/cuda/lib64 first.
+ENV LD_LIBRARY_PATH="/usr/local/nvidia/lib:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}"
+ENV PATH="/usr/local/nvidia/bin:${PATH}"
 
 # Install basic dependencies
 # TODO(mkolodner-sc): iputils-ping temporarily needed to setup inter-job VAI communication for GLT Inference.
@@ -31,6 +50,7 @@ COPY pyproject.toml pyproject.toml
 COPY uv.lock uv.lock
 COPY requirements requirements
 COPY gigl/scripts gigl/scripts
+COPY .python-version .python-version
 # gigl-core is a path dependency in pyproject.toml. uv sync needs its metadata to
 # resolve the lockfile. Copying only the build manifest (no C++ sources) so cmake
 # configures but compiles nothing — the src Dockerfile installs the real wheel later.
@@ -40,8 +60,13 @@ COPY gigl-core/README.md gigl-core/README.md
 
 RUN bash ./requirements/install_py_deps.sh
 
-# Note: Since we are using system python, we dont need to create a virtual environment here unlike the
-# cpu base image.
+# The UV_PROJECT_ENVIRONMENT environment variable can be used to configure the project virtual environment path
+# Since the above command should have created the .venv, we activate by default for any future uv commands.
+# We also need to set VIRTUAL_ENV so pip envocations can find the virtual environment.
+ENV UV_PROJECT_ENVIRONMENT=/gigl_deps/.venv
+ENV VIRTUAL_ENV="${UV_PROJECT_ENVIRONMENT}"
+# We just created a virtual environment, lets add the bin to the path
+ENV PATH="${UV_PROJECT_ENVIRONMENT}/bin:${PATH}"
 # We also need to make UV detectable by the system
 ENV PATH="/root/.local/bin:${PATH}"
 

containers/Dockerfile.dataflow.base

@@ -1,10 +1,15 @@
-FROM apache/beam_python3.11_sdk:2.56.0
+# This reference is coupled twice over, and the two halves live in different places:
+#   - the tag (`2.56.0`) must match the `apache-beam` pin in pyproject.toml, because Beam
+#     requires a worker's installed apache-beam to be the same version as the submitting
+#     environment's, and the boot harness copied out of this stage is that entrypoint.
+#   - the Python minor in the image *name* (`beam_python3.11_sdk`) must match
+#     .python-version, which is the interpreter uv installs into /gigl_deps/.venv below.
+ARG BEAM_SDK_IMAGE=apache/beam_python3.11_sdk:2.56.0
+FROM ${BEAM_SDK_IMAGE} AS beam_sdk
 
-ENV DEBIAN_FRONTEND=noninteractive
+FROM ubuntu:noble-20251001
 
-# We use system python for dataflow images since it has python and apache beam pre-installed.
-ENV UV_SYSTEM_PYTHON=true
-ENV UV_PROJECT_ENVIRONMENT=/usr/local
+ENV DEBIAN_FRONTEND=noninteractive
 
 # TODO(mkolodner-sc): iputils-ping temporarily needed to setup inter-job VAI communication for GLT Inference.
 # Once VAI natively supports this communication, we can remove this requirement.
@@ -25,6 +30,7 @@ COPY pyproject.toml pyproject.toml
 COPY uv.lock uv.lock
 COPY requirements requirements
 COPY gigl/scripts gigl/scripts
+COPY .python-version .python-version
 # gigl-core is a path dependency in pyproject.toml. uv sync needs its metadata to
 # resolve the lockfile. Copying only the build manifest (no C++ sources) so cmake
 # configures but compiles nothing — the src Dockerfile installs the real wheel later.
@@ -34,9 +40,28 @@ COPY gigl-core/README.md gigl-core/README.md
 
 RUN bash ./requirements/install_py_deps.sh --skip-glt-post-install
 
-# Note: Since we are using system python, we dont need to create a virtual environment here unlike the
-# cpu base image.
+# The UV_PROJECT_ENVIRONMENT environment variable can be used to configure the project virtual environment path
+# Since the above command should have created the .venv, we activate by default for any future uv commands.
+# We also need to set VIRTUAL_ENV so pip envocations can find the virtual environment.
+ENV UV_PROJECT_ENVIRONMENT=/gigl_deps/.venv
+ENV VIRTUAL_ENV="${UV_PROJECT_ENVIRONMENT}"
+# We just created a virtual environment, lets add the bin to the path
+ENV PATH="${UV_PROJECT_ENVIRONMENT}/bin:${PATH}"
 # We also need to make UV detectable by the system
 ENV PATH="/root/.local/bin:${PATH}"
 
+# Required, not optional. Unset, boot creates a per-worker venv with
+# `python -m venv --system-site-packages` from the `python` on PATH, then requires
+# apache-beam to import inside it. Seeded from /gigl_deps/.venv/bin/python, that venv
+# inherits the *base* interpreter's site-packages rather than this venv's, so the import
+# fails and every worker dies with "Apache Beam is not installed in the runtime
+# environment". Set, boot runs the SDK directly in the environment on PATH.
+ENV RUN_PYTHON_SDK_IN_DEFAULT_ENVIRONMENT=1
+
 WORKDIR /
+
+# boot is the Dataflow worker harness. It is a statically linked Go binary, so it runs on
+# this rootfs unaltered even though the SDK image is built on a different distro.
+# Dockerfile.dataflow.src declares no ENTRYPOINT, so this one is what workers execute.
+COPY --from=beam_sdk /opt/apache/beam /opt/apache/beam
+ENTRYPOINT ["/opt/apache/beam/boot"]

examples/link_prediction/graph_store/storage_main.py

@@ -74,14 +74,14 @@
 import argparse
 import ast
 import os
-from distutils.util import strtobool
 from typing import Literal, Optional, Union
 
 import torch
 
 from gigl.common import Uri, UriFactory
 from gigl.common.logger import Logger
 from gigl.common.utils.os_utils import import_obj
+from gigl.common.utils.parse import str_to_bool
 from gigl.distributed.graph_store import (
     GraphStoreInfo,
     build_storage_dataset,
@@ -254,7 +254,7 @@ def storage_node_process(
         splitter=splitter,
         ssl_positive_label_percentage=ssl_positive_label_percentage,
         should_load_tf_records_in_parallel=bool(
-            strtobool(args.should_load_tf_records_in_parallel)
+            str_to_bool(args.should_load_tf_records_in_parallel)
         ),
         num_rpc_threads=args.num_rpc_threads,
         rpc_timeout=args.rpc_timeout,

examples/tutorial/KDD_2025/heterogeneous_inference.py

@@ -26,7 +26,6 @@
 import argparse
 import datetime
 from collections.abc import Mapping
-from distutils.util import strtobool
 from pathlib import Path
 
 import fastavro
@@ -38,6 +37,7 @@
 from gigl.common import Uri, UriFactory
 from gigl.common.data.export import EmbeddingExporter
 from gigl.common.logger import Logger
+from gigl.common.utils.parse import str_to_bool
 from gigl.distributed import (
     DistDataset,
     DistNeighborLoader,
@@ -154,7 +154,7 @@ def inference(
         task_config_uri,
         _tfrecord_uri_pattern=".*tfrecord",
     )
-    if strtobool(args.use_local_saved_model):
+    if str_to_bool(args.use_local_saved_model):
         model_uri = LOCAL_SAVED_MODEL_URI
     else:
         model_uri = gbml_config_pb_wrapper.shared_config.trained_model_metadata.trained_model_uri

examples/tutorial/KDD_2025/heterogeneous_training.py

@@ -37,7 +37,6 @@
 
 import argparse
 from collections.abc import Iterable, Mapping
-from distutils.util import strtobool
 from typing import Literal
 
 import torch
@@ -48,6 +47,7 @@
 from examples.tutorial.KDD_2025.utils import LOCAL_SAVED_MODEL_URI, init_model
 from gigl.common import UriFactory
 from gigl.common.logger import Logger
+from gigl.common.utils.parse import str_to_bool
 from gigl.distributed import (
     DistABLPLoader,
     DistDataset,
@@ -269,7 +269,7 @@ def train(
         logger.info(f"Test node type {node_type} has {node_ids.size(0)} nodes.")  # ty: ignore[unresolved-attribute] TODO(ty-torch-keyed-access): fix ty false positives for torch-backed keyed container access.
     training_process_port = get_free_port()
     logger.info(f"Will train for {max_training_batches} batches.")
-    if strtobool(args.use_local_saved_model):
+    if str_to_bool(args.use_local_saved_model):
         model_uri = LOCAL_SAVED_MODEL_URI
     else:
         model_uri = gbml_config_pb_wrapper.shared_config.trained_model_metadata.trained_model_uri

gigl/common/utils/parse.py

@@ -0,0 +1,41 @@
+from typing import Final
+
+_TRUE_VALUES: Final[frozenset[str]] = frozenset({"y", "yes", "t", "true", "on", "1"})
+_FALSE_VALUES: Final[frozenset[str]] = frozenset({"n", "no", "f", "false", "off", "0"})
+
+
+def str_to_bool(value: str) -> bool:
+    """
+    Converts a string representation of truth to a bool.
+
+    Accepts and rejects the same spellings as `distutils.util.strtobool`, which is no
+    longer part of the standard library, though the `ValueError` message keeps the
+    caller's casing instead of lowercasing it:
+
+      - True: "y", "yes", "t", "true", "on", "1"
+      - False: "n", "no", "f", "false", "off", "0"
+
+    Matching is case-insensitive. Whitespace is not stripped, so " true" raises rather
+    than returning True.
+
+    Example:
+        >>> str_to_bool("TRUE")
+        True
+        >>> str_to_bool("off")
+        False
+
+    Args:
+        value (str): The string to interpret as a boolean.
+
+    Returns:
+        bool: The truth value `value` spells.
+
+    Raises:
+        ValueError: If `value` is not one of the accepted spellings.
+    """
+    normalized_value = value.lower()
+    if normalized_value in _TRUE_VALUES:
+        return True
+    if normalized_value in _FALSE_VALUES:
+        return False
+    raise ValueError(f"invalid truth value {value!r}")

gigl/dep_vars.env

@@ -1,7 +1,7 @@
 # Note this file only supports static key value pairs so it can be loaded by make, bash, python, and sbt without any additional parsing.
-DOCKER_LATEST_BASE_CUDA_IMAGE_NAME_WITH_TAG=us-central1-docker.pkg.dev/external-snap-ci-github-gigl/public-gigl/gigl-cuda-base:7d3182eeb6446ce3e35910babba990c8e003879d.109.1
-DOCKER_LATEST_BASE_CPU_IMAGE_NAME_WITH_TAG=us-central1-docker.pkg.dev/external-snap-ci-github-gigl/public-gigl/gigl-cpu-base:7d3182eeb6446ce3e35910babba990c8e003879d.109.1
-DOCKER_LATEST_BASE_DATAFLOW_IMAGE_NAME_WITH_TAG=us-central1-docker.pkg.dev/external-snap-ci-github-gigl/public-gigl/gigl-dataflow-base:7d3182eeb6446ce3e35910babba990c8e003879d.109.1
+DOCKER_LATEST_BASE_CUDA_IMAGE_NAME_WITH_TAG=us-central1-docker.pkg.dev/external-snap-ci-github-gigl/public-gigl/gigl-cuda-base:0c41d2857437453a651888d02978071c5c3ab1e2.113.1
+DOCKER_LATEST_BASE_CPU_IMAGE_NAME_WITH_TAG=us-central1-docker.pkg.dev/external-snap-ci-github-gigl/public-gigl/gigl-cpu-base:0c41d2857437453a651888d02978071c5c3ab1e2.113.1
+DOCKER_LATEST_BASE_DATAFLOW_IMAGE_NAME_WITH_TAG=us-central1-docker.pkg.dev/external-snap-ci-github-gigl/public-gigl/gigl-dataflow-base:0c41d2857437453a651888d02978071c5c3ab1e2.113.1
 
 DEFAULT_GIGL_RELEASE_SRC_IMAGE_CUDA=us-central1-docker.pkg.dev/external-snap-ci-github-gigl/public-gigl/src-cuda:0.3.1
 DEFAULT_GIGL_RELEASE_SRC_IMAGE_CPU=us-central1-docker.pkg.dev/external-snap-ci-github-gigl/public-gigl/src-cpu:0.3.1

gigl/distributed/dataset_factory.py

@@ -5,7 +5,6 @@
 
 import time
 from collections.abc import Mapping
-from distutils.util import strtobool
 from typing import Literal, MutableMapping, Optional, Tuple, Type, Union
 
 import torch
@@ -29,6 +28,7 @@
 )
 from gigl.common.logger import Logger
 from gigl.common.utils.decorator import tf_on_cpu
+from gigl.common.utils.parse import str_to_bool
 from gigl.distributed.constants import DEFAULT_MASTER_DATA_BUILDING_PORT
 from gigl.distributed.dist_context import DistributedContext
 from gigl.distributed.dist_dataset import DistDataset
@@ -629,11 +629,11 @@ def build_dataset_from_task_config_uri(
     )
 
     should_use_range_partitioning = bool(
-        strtobool(args.get("should_use_range_partitioning", "True"))
+        str_to_bool(args.get("should_use_range_partitioning", "True"))
     )
 
     should_load_tensors_in_parallel = bool(
-        strtobool(args.get("should_load_tensors_in_parallel", "True"))
+        str_to_bool(args.get("should_load_tensors_in_parallel", "True"))
     )
 
     logger.info(

gigl/scripts/install_glt.sh

@@ -9,6 +9,17 @@ is_running_on_mac() {
 }
 
 has_cuda_driver() {
+    # Callers use this as an `if` condition, which suspends `set -e` for the whole
+    # function body. A missing `whereis` would therefore leave $cuda_location empty and
+    # report "no CUDA" with the build still exiting 0, installing CPU torch and a
+    # WITH_CUDA=OFF GLT into a CUDA image. `exit` is the only way out of a
+    # suspended-errexit context, so fail the build outright instead of returning.
+    if ! command -v whereis &> /dev/null
+    then
+        echo "whereis is unavailable, so CUDA presence cannot be determined." >&2
+        exit 1
+    fi
+
     # Use the whereis command to locate the CUDA driver
     cuda_location=$(whereis cuda)
 

gigl/scripts/post_install.py

@@ -34,8 +34,17 @@ def run_command_and_stream_stdout(cmd: str) -> Optional[int]:
     return return_code
 
 
-def main():
-    """Main entry point for the post-install script."""
+def main() -> int:
+    """Main entry point for the post-install script.
+
+    Returns:
+        int: 0, when install_glt.sh succeeds.
+
+    Raises:
+        SystemExit: With a non-zero code when install_glt.sh is missing, fails, or
+            reports no exit status. Callers must propagate it: swallowing it lets a
+            build succeed while shipping an environment with no working GLT.
+    """
     print("Running GIGL post-install script...")
 
     # Get the directory where this script is located
@@ -52,9 +61,16 @@ def main():
 
     try:
         print(f"Executing {cmd}...")
-        result = run_command_and_stream_stdout(cmd)
-        print("Post-install script finished running, with return code: ", result)
-        return result
+        return_code = run_command_and_stream_stdout(cmd)
+        print("Post-install script finished running, with return code: ", return_code)
+        # `Popen.poll()` returns None while the child has no recorded status, so an
+        # unknown outcome is not evidence of success.
+        if return_code is None:
+            print("Error: could not determine the exit status of install_glt.sh")
+            sys.exit(1)
+        if return_code != 0:
+            sys.exit(return_code)
+        return return_code
 
     except subprocess.CalledProcessError as e:
         print(f"Error running install_glt.sh: {e}")
@@ -65,4 +81,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    raise SystemExit(main())

gigl/src/common/modeling_task_specs/node_anchor_based_link_prediction_modeling_task_spec.py

@@ -1,6 +1,5 @@
 from collections import OrderedDict
 from contextlib import ExitStack
-from distutils.util import strtobool
 from time import time
 from typing import Any, Optional, Type
 
@@ -12,6 +11,7 @@
 
 from gigl.common.logger import Logger
 from gigl.common.utils import os_utils
+from gigl.common.utils.parse import str_to_bool
 from gigl.common.utils.torch_training import (
     get_rank,
     get_world_size,
@@ -201,7 +201,7 @@ def __init__(self, **kwargs) -> None:
         # Retrieval-specific Task Parameters
         softmax_temp = float(kwargs.get("softmax_temp", 0.07))
         should_remove_accidental_hits = bool(
-            strtobool(kwargs.get("should_remove_accidental_hits", "True"))
+            str_to_bool(kwargs.get("should_remove_accidental_hits", "True"))
         )
         task = base_task(
             temperature=softmax_temp,