From a4f6598d0b3298085f028e50b38f82c9f13948ea Mon Sep 17 00:00:00 2001 From: kerthcet Date: Sun, 26 Jul 2026 21:41:10 +0100 Subject: [PATCH 01/12] support modal Signed-off-by: kerthcet --- .env.example | 24 + .github/workflows/lint.yml | 23 + .github/workflows/test-e2e.yml | 32 + .github/workflows/test.yml | 23 + .gitignore | 7 + .golangci.yml | 52 ++ Dockerfile | 39 ++ Makefile | 270 ++++++++ OWNERS | 4 +- PROJECT | 36 + README.md | 4 +- api/v1alpha1/groupversion_info.go | 95 +++ api/v1alpha1/nodeclaim_types.go | 133 ++++ api/v1alpha1/nodepool_types.go | 175 +++++ api/v1alpha1/zz_generated.deepcopy.go | 287 ++++++++ cmd/main.go | 328 +++++++++ config/catalog/kustomization.yaml | 23 + config/certmanager/certificate-metrics.yaml | 20 + config/certmanager/certificate-webhook.yaml | 20 + config/certmanager/issuer.yaml | 13 + config/certmanager/kustomization.yaml | 7 + config/certmanager/kustomizeconfig.yaml | 8 + .../bases/nebula.inftyai.com_nodeclaims.yaml | 138 ++++ .../bases/nebula.inftyai.com_nodepools.yaml | 222 ++++++ config/crd/kustomization.yaml | 17 + config/crd/kustomizeconfig.yaml | 19 + .../default/cert_metrics_manager_patch.yaml | 30 + config/default/kustomization.yaml | 244 +++++++ config/default/manager_metrics_patch.yaml | 4 + config/default/manager_webhook_patch.yaml | 31 + config/default/metrics_service.yaml | 18 + config/manager/kustomization.yaml | 8 + config/manager/manager.yaml | 134 ++++ config/manager/modal-credentials.example.yaml | 40 ++ .../network-policy/allow-metrics-traffic.yaml | 27 + .../network-policy/allow-webhook-traffic.yaml | 27 + config/network-policy/kustomization.yaml | 3 + config/prometheus/kustomization.yaml | 11 + config/prometheus/monitor.yaml | 27 + config/prometheus/monitor_tls_patch.yaml | 19 + config/rbac/kustomization.yaml | 31 + config/rbac/leader_election_role.yaml | 40 ++ config/rbac/leader_election_role_binding.yaml | 15 + config/rbac/metrics_auth_role.yaml | 17 + config/rbac/metrics_auth_role_binding.yaml | 12 + config/rbac/metrics_reader_role.yaml | 9 + config/rbac/nodeclaim_admin_role.yaml | 27 + config/rbac/nodeclaim_editor_role.yaml | 33 + config/rbac/nodeclaim_viewer_role.yaml | 29 + config/rbac/nodepool_admin_role.yaml | 27 + config/rbac/nodepool_editor_role.yaml | 33 + config/rbac/nodepool_viewer_role.yaml | 29 + config/rbac/role.yaml | 96 +++ config/rbac/role_binding.yaml | 15 + config/rbac/service_account.yaml | 8 + config/samples/deployment.yaml | 74 ++ config/samples/nebula_v1alpha1_nodepool.yaml | 21 + config/webhook/kustomization.yaml | 12 + config/webhook/kustomizeconfig.yaml | 22 + config/webhook/manifests.yaml | 25 + config/webhook/selector_patch.yaml | 22 + config/webhook/service.yaml | 16 + docs/architecture.md | 638 ++++++++++++++++++ docs/deploy.md | 244 +++++++ go.mod | 113 ++++ go.sum | 390 +++++++++++ hack/boilerplate.go.txt | 15 + hack/deploy.sh | 152 +++++ hack/gen-webhook-cert.sh | 116 ++++ internal/controller/nodeclaim_controller.go | 446 ++++++++++++ .../controller/nodeclaim_controller_test.go | 473 +++++++++++++ internal/controller/nodepool_controller.go | 185 +++++ .../controller/nodepool_controller_test.go | 199 ++++++ .../controller/nodepool_validation_test.go | 74 ++ .../controller/pod_placement_controller.go | 184 +++++ .../pod_placement_controller_test.go | 420 ++++++++++++ internal/controller/pod_placement_helpers.go | 268 ++++++++ internal/controller/suite_test.go | 131 ++++ internal/webhook/v1/pod_webhook.go | 135 ++++ internal/webhook/v1/pod_webhook_test.go | 174 +++++ internal/webhook/v1/webhook_suite_test.go | 178 +++++ pkg/provider/catalog/base.go | 91 +++ pkg/provider/catalog/catalog.go | 167 +++++ pkg/provider/catalog/catalog_test.go | 145 ++++ pkg/provider/catalog/data/modal.csv | 30 + pkg/provider/errors.go | 93 +++ pkg/provider/errors_test.go | 60 ++ pkg/provider/modal/client.go | 232 +++++++ pkg/provider/modal/modal.go | 380 +++++++++++ pkg/provider/modal/modal_test.go | 330 +++++++++ pkg/provider/provider.go | 195 ++++++ pkg/provider/registry.go | 77 +++ pkg/util/accelerator.go | 82 +++ pkg/util/accelerator_test.go | 101 +++ pkg/util/claim.go | 30 + pkg/version/version.go | 42 ++ pkg/vnode/doc.go | 29 + pkg/vnode/handler.go | 341 ++++++++++ pkg/vnode/handler_test.go | 251 +++++++ pkg/vnode/node.go | 306 +++++++++ pkg/vnode/status.go | 140 ++++ test/e2e/e2e_suite_test.go | 89 +++ test/e2e/e2e_test.go | 354 ++++++++++ test/utils/utils.go | 254 +++++++ 104 files changed, 11605 insertions(+), 4 deletions(-) create mode 100644 .env.example create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/test-e2e.yml create mode 100644 .github/workflows/test.yml create mode 100644 .golangci.yml create mode 100644 PROJECT create mode 100644 api/v1alpha1/groupversion_info.go create mode 100644 api/v1alpha1/nodeclaim_types.go create mode 100644 api/v1alpha1/nodepool_types.go create mode 100644 api/v1alpha1/zz_generated.deepcopy.go create mode 100644 cmd/main.go create mode 100644 config/catalog/kustomization.yaml create mode 100644 config/certmanager/certificate-metrics.yaml create mode 100644 config/certmanager/certificate-webhook.yaml create mode 100644 config/certmanager/issuer.yaml create mode 100644 config/certmanager/kustomization.yaml create mode 100644 config/certmanager/kustomizeconfig.yaml create mode 100644 config/crd/bases/nebula.inftyai.com_nodeclaims.yaml create mode 100644 config/crd/bases/nebula.inftyai.com_nodepools.yaml create mode 100644 config/crd/kustomization.yaml create mode 100644 config/crd/kustomizeconfig.yaml create mode 100644 config/default/cert_metrics_manager_patch.yaml create mode 100644 config/default/kustomization.yaml create mode 100644 config/default/manager_metrics_patch.yaml create mode 100644 config/default/manager_webhook_patch.yaml create mode 100644 config/default/metrics_service.yaml create mode 100644 config/manager/kustomization.yaml create mode 100644 config/manager/manager.yaml create mode 100644 config/manager/modal-credentials.example.yaml create mode 100644 config/network-policy/allow-metrics-traffic.yaml create mode 100644 config/network-policy/allow-webhook-traffic.yaml create mode 100644 config/network-policy/kustomization.yaml create mode 100644 config/prometheus/kustomization.yaml create mode 100644 config/prometheus/monitor.yaml create mode 100644 config/prometheus/monitor_tls_patch.yaml create mode 100644 config/rbac/kustomization.yaml create mode 100644 config/rbac/leader_election_role.yaml create mode 100644 config/rbac/leader_election_role_binding.yaml create mode 100644 config/rbac/metrics_auth_role.yaml create mode 100644 config/rbac/metrics_auth_role_binding.yaml create mode 100644 config/rbac/metrics_reader_role.yaml create mode 100644 config/rbac/nodeclaim_admin_role.yaml create mode 100644 config/rbac/nodeclaim_editor_role.yaml create mode 100644 config/rbac/nodeclaim_viewer_role.yaml create mode 100644 config/rbac/nodepool_admin_role.yaml create mode 100644 config/rbac/nodepool_editor_role.yaml create mode 100644 config/rbac/nodepool_viewer_role.yaml create mode 100644 config/rbac/role.yaml create mode 100644 config/rbac/role_binding.yaml create mode 100644 config/rbac/service_account.yaml create mode 100644 config/samples/deployment.yaml create mode 100644 config/samples/nebula_v1alpha1_nodepool.yaml create mode 100644 config/webhook/kustomization.yaml create mode 100644 config/webhook/kustomizeconfig.yaml create mode 100644 config/webhook/manifests.yaml create mode 100644 config/webhook/selector_patch.yaml create mode 100644 config/webhook/service.yaml create mode 100644 docs/architecture.md create mode 100644 docs/deploy.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hack/boilerplate.go.txt create mode 100755 hack/deploy.sh create mode 100755 hack/gen-webhook-cert.sh create mode 100644 internal/controller/nodeclaim_controller.go create mode 100644 internal/controller/nodeclaim_controller_test.go create mode 100644 internal/controller/nodepool_controller.go create mode 100644 internal/controller/nodepool_controller_test.go create mode 100644 internal/controller/nodepool_validation_test.go create mode 100644 internal/controller/pod_placement_controller.go create mode 100644 internal/controller/pod_placement_controller_test.go create mode 100644 internal/controller/pod_placement_helpers.go create mode 100644 internal/controller/suite_test.go create mode 100644 internal/webhook/v1/pod_webhook.go create mode 100644 internal/webhook/v1/pod_webhook_test.go create mode 100644 internal/webhook/v1/webhook_suite_test.go create mode 100644 pkg/provider/catalog/base.go create mode 100644 pkg/provider/catalog/catalog.go create mode 100644 pkg/provider/catalog/catalog_test.go create mode 100644 pkg/provider/catalog/data/modal.csv create mode 100644 pkg/provider/errors.go create mode 100644 pkg/provider/errors_test.go create mode 100644 pkg/provider/modal/client.go create mode 100644 pkg/provider/modal/modal.go create mode 100644 pkg/provider/modal/modal_test.go create mode 100644 pkg/provider/provider.go create mode 100644 pkg/provider/registry.go create mode 100644 pkg/util/accelerator.go create mode 100644 pkg/util/accelerator_test.go create mode 100644 pkg/util/claim.go create mode 100644 pkg/version/version.go create mode 100644 pkg/vnode/doc.go create mode 100644 pkg/vnode/handler.go create mode 100644 pkg/vnode/handler_test.go create mode 100644 pkg/vnode/node.go create mode 100644 pkg/vnode/status.go create mode 100644 test/e2e/e2e_suite_test.go create mode 100644 test/e2e/e2e_test.go create mode 100644 test/utils/utils.go diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f817380 --- /dev/null +++ b/.env.example @@ -0,0 +1,24 @@ +# Nebula provider credentials — SECRETS ONLY. +# +# Copy to .env and fill in real values. .env is gitignored — never commit tokens. +# Consumed by hack/deploy.sh (make deploy-all), which turns these into one +# Kubernetes Secret PER PROVIDER. Non-secret config (image, namespace, Kind +# cluster) is NOT here — pass it as make variables, e.g. +# make deploy-all IMG=myrepo/nebula:v1 KIND_CLUSTER=nebula-test-e2e +# +# cp .env.example .env +# # edit .env +# make deploy-all + +# --- Modal provider -------------------------------------------------------- +# From `modal token new`. Leave blank to skip creating the Modal secret (the +# provider is then skipped at registration — not fatal). +MODAL_TOKEN_ID= +MODAL_TOKEN_SECRET= +# Optional: Modal workspace environment and the App all sandboxes run under. +MODAL_ENVIRONMENT= +MODAL_APP_NAME=nebula + +# --- Additional providers (add as adapters land) --------------------------- +# Each provider gets its OWN secret (see hack/deploy.sh PROVIDER_SECRETS), e.g.: +# RUNPOD_API_KEY= diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..67ff2bf --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,23 @@ +name: Lint + +on: + push: + pull_request: + +jobs: + lint: + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Run linter + uses: golangci/golangci-lint-action@v8 + with: + version: v2.1.6 diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml new file mode 100644 index 0000000..68fd1ed --- /dev/null +++ b/.github/workflows/test-e2e.yml @@ -0,0 +1,32 @@ +name: E2E Tests + +on: + push: + pull_request: + +jobs: + test-e2e: + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Install the latest version of kind + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + + - name: Verify kind installation + run: kind version + + - name: Running Test e2e + run: | + go mod tidy + make test-e2e diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..fc2e80d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: Tests + +on: + push: + pull_request: + +jobs: + test: + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Running Tests + run: | + go mod tidy + make test diff --git a/.gitignore b/.gitignore index e2a094c..04b6cc1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,10 @@ Dockerfile.cross !vendor/**/zz_generated.* +# local environment / credentials — never commit real tokens +.env +.env.local + # editor and IDE paraphernalia .idea .vscode @@ -31,3 +35,6 @@ __pycache__ *.pyc .pytest_cache *.tgz + +.devcontainer +bin \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..e5b21b0 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,52 @@ +version: "2" +run: + allow-parallel-runners: true +linters: + default: none + enable: + - copyloopvar + - dupl + - errcheck + - ginkgolinter + - goconst + - gocyclo + - govet + - ineffassign + - lll + - misspell + - nakedret + - prealloc + - revive + - staticcheck + - unconvert + - unparam + - unused + settings: + revive: + rules: + - name: comment-spacings + - name: import-shadowing + exclusions: + generated: lax + rules: + - linters: + - lll + path: api/* + - linters: + - dupl + - lll + path: internal/* + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/Dockerfile b/Dockerfile index e69de29..bbac4a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -0,0 +1,39 @@ +# Build the manager binary +FROM golang:1.24 AS builder +ARG TARGETOS +ARG TARGETARCH +# VERSION is stamped into the binary (pkg/version) and surfaces as the virtual +# node's kubelet VERSION. Passed by `make docker-build` (defaults to git describe). +ARG VERSION=nebula-dev + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY cmd/main.go cmd/main.go +COPY api/ api/ +COPY internal/ internal/ +COPY pkg/ pkg/ + +# Build +# the GOARCH has not a default value to allow the binary be built according to the host where the command +# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO +# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, +# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -a -ldflags "-X github.com/InftyAI/Nebula/pkg/version.gitVersion=${VERSION}" \ + -o manager cmd/main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index e69de29..e51afc0 100644 --- a/Makefile +++ b/Makefile @@ -0,0 +1,270 @@ +# Image URL to use all building/pushing image targets +IMG ?= inftyai/nebula-controller:latest + +# NAMESPACE is where the manager runs (must match config/manager). Consumed by +# hack/deploy.sh via the deploy-all target. +NAMESPACE ?= nebula-system + +# DEPLOY_KIND_CLUSTER, when set, makes deploy-all load the image into that Kind +# cluster instead of pushing it. It is deliberately SEPARATE from the e2e +# KIND_CLUSTER (which defaults to a throwaway test cluster), so a plain +# `make deploy-all` pushes the image rather than loading it into the e2e cluster. +DEPLOY_KIND_CLUSTER ?= + +# VERSION is stamped into the binary (pkg/version) and surfaces as the virtual +# node's kubelet VERSION. Defaults to `git describe` (tag, or short commit for an +# untagged repo, with -dirty when the tree has uncommitted changes). Override +# with `make build VERSION=v1.2.3`. +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo nebula-dev) +LDFLAGS ?= -X github.com/InftyAI/Nebula/pkg/version.gitVersion=$(VERSION) + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# CONTAINER_TOOL defines the container tool to be used for building images. +# Be aware that the target commands are only tested with Docker which is +# scaffolded by default. However, you might want to replace it to use other +# tools. (i.e. podman) +CONTAINER_TOOL ?= docker + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk command is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +# The catalog ConfigMap is generated from CSVs that live canonically under +# pkg/ (so Go can embed them), which is outside config/catalog. Kustomize's +# default load restrictor forbids reaching up the tree, so builds that include +# the catalog pass LoadRestrictionsNone. +KUSTOMIZE_BUILD_FLAGS ?= --load-restrictor=LoadRestrictionsNone + +.PHONY: verify-catalog +verify-catalog: kustomize ## Verify the price catalog CSVs parse and the catalog ConfigMap renders. + go test ./pkg/provider/catalog/... + $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_FLAGS) config/catalog >/dev/null && echo "catalog ConfigMap renders OK" + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +.PHONY: test +test: manifests generate fmt vet setup-envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + +# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. +# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. +# CertManager is installed by default; skip with: +# - CERT_MANAGER_INSTALL_SKIP=true +KIND_CLUSTER ?= nebula-test-e2e + +.PHONY: setup-test-e2e +setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist + @command -v $(KIND) >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @case "$$($(KIND) get clusters)" in \ + *"$(KIND_CLUSTER)"*) \ + echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ + *) \ + echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ + $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ + esac + +.PHONY: test-e2e +test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. + KIND_CLUSTER=$(KIND_CLUSTER) go test ./test/e2e/ -v -ginkgo.v + $(MAKE) cleanup-test-e2e + +.PHONY: cleanup-test-e2e +cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests + @$(KIND) delete cluster --name $(KIND_CLUSTER) + +.PHONY: lint +lint: golangci-lint ## Run golangci-lint linter + $(GOLANGCI_LINT) run + +.PHONY: lint-fix +lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes + $(GOLANGCI_LINT) run --fix + +.PHONY: lint-config +lint-config: golangci-lint ## Verify golangci-lint linter configuration + $(GOLANGCI_LINT) config verify + +##@ Build + +.PHONY: build +build: manifests generate fmt vet ## Build manager binary. + go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run -ldflags "$(LDFLAGS)" ./cmd/main.go + +# If you wish to build the manager image targeting other platforms you can use the --platform flag. +# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. +# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +.PHONY: docker-build +docker-build: ## Build docker image with the manager. + $(CONTAINER_TOOL) build --build-arg VERSION=$(VERSION) -t ${IMG} . + +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + $(CONTAINER_TOOL) push ${IMG} + +# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple +# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: +# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ +# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) +# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. +PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le +.PHONY: docker-buildx +docker-buildx: ## Build and push docker image for the manager for cross-platform support + # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + - $(CONTAINER_TOOL) buildx create --name nebula-builder + $(CONTAINER_TOOL) buildx use nebula-builder + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --build-arg VERSION=$(VERSION) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx rm nebula-builder + rm Dockerfile.cross + +.PHONY: build-installer +build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. + mkdir -p dist + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_FLAGS) config/default > dist/install.yaml + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: deploy +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_FLAGS) config/default | $(KUBECTL) apply -f - + +.PHONY: undeploy +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_FLAGS) config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: deploy-all +deploy-all: ## Build the image, apply per-provider credential Secrets from .env, and deploy. See .env.example. + IMG=$(IMG) NAMESPACE=$(NAMESPACE) KIND_CLUSTER=$(DEPLOY_KIND_CLUSTER) KUBECTL=$(KUBECTL) hack/deploy.sh + +##@ Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Binaries +KUBECTL ?= kubectl +KIND ?= kind +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint + +## Tool Versions +KUSTOMIZE_VERSION ?= v5.6.0 +CONTROLLER_TOOLS_VERSION ?= v0.18.0 +#ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) +ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') +#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) +ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}') +GOLANGCI_LINT_VERSION ?= v2.1.6 + +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) + +.PHONY: setup-envtest +setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory. + @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..." + @$(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path || { \ + echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \ + exit 1; \ + } + +.PHONY: envtest +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. +$(ENVTEST): $(LOCALBIN) + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +$(GOLANGCI_LINT): $(LOCALBIN) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f "$(1)-$(3)" ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +rm -f $(1) || true ;\ +GOBIN=$(LOCALBIN) go install $${package} ;\ +mv $(1) $(1)-$(3) ;\ +} ;\ +ln -sf $(1)-$(3) $(1) +endef diff --git a/OWNERS b/OWNERS index 4ca4a20..515ec16 100644 --- a/OWNERS +++ b/OWNERS @@ -1,5 +1,5 @@ approvers: - - TBD + - kerthcet reviewers: - - TBD + - kerthcet diff --git a/PROJECT b/PROJECT new file mode 100644 index 0000000..4a26e04 --- /dev/null +++ b/PROJECT @@ -0,0 +1,36 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html +cliVersion: 4.7.0 +domain: inftyai.com +layout: +- go.kubebuilder.io/v4 +projectName: nebula +repo: github.com/InftyAI/Nebula +resources: +- api: + crdVersion: v1 + controller: true + domain: inftyai.com + group: nebula + kind: NodePool + path: github.com/InftyAI/Nebula/api/v1alpha1 + version: v1alpha1 +- api: + crdVersion: v1 + controller: true + domain: inftyai.com + group: nebula + kind: NodeClaim + path: github.com/InftyAI/Nebula/api/v1alpha1 + version: v1alpha1 +- external: true + group: core + kind: Pod + path: k8s.io/api/core/v1 + version: v1 + webhooks: + defaulting: true + webhookVersion: v1 +version: "3" diff --git a/README.md b/README.md index 4baad06..e46b075 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ -# template-repo +# Nebula -A template repo. +The control plane for GPUaaS. diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..dab8323 --- /dev/null +++ b/api/v1alpha1/groupversion_info.go @@ -0,0 +1,95 @@ +// Package v1alpha1 contains the Nebula API types. +// +// Nebula is an operator that orchestrates GPU workloads across NeoClouds +// (RunPod, Modal, Kubernetes, ...). It follows a Karpenter-style split: +// +// NodePool - policy: which providers are allowed, how to choose between +// them (cost/availability), failover behaviour, and the GPU shape. +// NodeClaim - one provisioned external instance and its lifecycle. Owns the +// terminate finalizer so a paid instance is never leaked. +// +// +kubebuilder:object:generate=true +// +groupName=nebula.inftyai.com +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is the group/version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "nebula.inftyai.com", Version: "v1alpha1"} + + // SchemeBuilder registers the types with a Scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +// Well-known keys used across the project. Kept here so the webhook, the +// placement controller and the NodeClaim controller share one source of truth. +const ( + // EnabledLabel opts a Pod into Nebula. It doubles as the webhook's + // objectSelector so only opted-in Pods ever hit the mutating webhook. + EnabledLabel = "nebula.inftyai.com/enabled" + + // ProviderSelectionGate is the scheduling gate the webhook injects at Pod + // CREATE. The placement controller removes it once it has chosen a + // provider (by adding a provider nodeSelector), releasing the Pod to the + // scheduler. + ProviderSelectionGate = "nebula.inftyai.com/provider-selection" + + // ProviderLabel is set on each provider's virtual node and added to a Pod's + // nodeSelector by the placement controller to route it to that provider. + ProviderLabel = "nebula.inftyai.com/provider" + + // ManagedByLabel marks every object Nebula creates and owns (starting with + // the virtual nodes). It uses the well-known app.kubernetes.io/managed-by key + // so standard tooling recognizes it; its value is always ManagedByValue. This + // is the stable, management-scoped selector for "everything Nebula manages", + // independent of provider routing — for NetworkPolicies, monitoring scrape + // configs, and operator queries. + ManagedByLabel = "app.kubernetes.io/managed-by" + // ManagedByValue is the sole value of ManagedByLabel. + ManagedByValue = "nebula" + + // PoolLabel records which NodePool a Pod (and its NodeClaim) belongs to. Its + // value is the NodePool name, so the key mirrors the CRD kind. + PoolLabel = "nebula.inftyai.com/nodepool" + + // AcceleratorTypeLabel carries the requested accelerator TYPE only (e.g. + // "a100-40gb" or "h100"). The COUNT is expressed separately as a standard + // resource request/limit on the container (nvidia.com/gpu for the NVIDIA + // accelerators the wired providers serve today) — so scheduling fit and + // provisioning read the same number, and there is no bespoke count grammar. It + // is a label (not an annotation) so Pods can be selected/validated by + // accelerator type; label values forbid ":", which is exactly why the count + // could never live here. The name is provider-neutral (accelerator, not GPU) + // so it also fits non-GPU accelerators (TPUs, etc.) when such a provider lands. + // The type is matched case-insensitively against the provider catalog, so + // "a100", "A100" both resolve; the provider's canonical casing is what actually + // gets provisioned (see catalog.Base.MapAccelerator). Read type+count together + // via util.AcceleratorRequest. + AcceleratorTypeLabel = "nebula.inftyai.com/accelerator-type" + + // CapacityTypeAnnotation carries the optimizer-chosen purchase tier + // (Spot/OnDemand/Reserved). It is the one provisioning input that cannot be + // read off the Pod's own spec, so the placement controller writes it here + // when it ungates the Pod. The virtual kubelet — which provisions solely from + // the Pod — reads it back on CreatePod. Empty means "let the provider use its + // default" (e.g. Modal is OnDemand-only and ignores it). + CapacityTypeAnnotation = "nebula.inftyai.com/capacity-type" + + // TerminateInstanceFinalizer is held by every NodeClaim to guarantee teardown. + // The virtual kubelet owns the happy path (DeletePod → provider.Terminate, + // keyed on the Pod-derived claim name), but its teardown is edge-triggered and + // its instance tracking is in-memory, so a Pod force-deleted during a VK outage + // would leak a paid instance. This finalizer makes teardown level-triggered: + // the cluster-scoped claim outlives the namespaced Pod, so on delete the + // NodeClaim controller resolves the provider, finds the instance by claim name + // via List, and Terminates it before releasing the finalizer — independent of + // VK liveness (see docs/architecture.md §3). + TerminateInstanceFinalizer = "nebula.inftyai.com/terminate-instance" +) diff --git a/api/v1alpha1/nodeclaim_types.go b/api/v1alpha1/nodeclaim_types.go new file mode 100644 index 0000000..902f473 --- /dev/null +++ b/api/v1alpha1/nodeclaim_types.go @@ -0,0 +1,133 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NodeClaimSpec is the durable identity of one external instance: who it serves, +// which provider it lives on, and which policy produced it. It is created and +// owned by the controller (not users), one per placed Pod. The workload shape +// (image, resources, GPU type/count, spot) is NOT duplicated here — that lives +// on the Pod, which the provider controller reads directly. NodeClaim is a +// ledger, not a spec: its reason to exist is to survive the Node so teardown can +// reclaim the external instance and never leak a paid GPU. +type NodeClaimSpec struct { + // PodRef links this claim to the Pod it serves. UID pins the exact Pod so a + // recreated Pod of the same name gets a fresh claim rather than adopting the + // old instance. + PodRef PodReference `json:"podRef"` + + // Provider is the chosen NeoCloud. Immutable: a NodeClaim never migrates. + // Recovery from preemption is delete-and-recreate. Held durably so teardown + // knows which provider API to call even after status is lost. + Provider string `json:"provider"` + + // CapacityType is the purchase tier the placement optimizer selected + // (Spot/OnDemand/Reserved). It is stored durably here because it is the one + // provisioning input that cannot be read off the Pod, and Provision needs it + // to re-issue the request after a controller restart. Immutable, like + // Provider. Empty means "let the provider use its default" (e.g. Modal is + // OnDemand-only and ignores it). + // +optional + CapacityType CapacityType `json:"capacityType,omitempty"` + + // PoolRef is the NodePool whose policy produced this claim, for reporting. + // +optional + PoolRef string `json:"poolRef,omitempty"` +} + +// PodReference identifies the namespaced Pod a cluster-scoped NodeClaim serves. +type PodReference struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + // UID pins the exact Pod object; a recreated Pod of the same name gets a new + // NodeClaim rather than silently adopting the old instance. + UID string `json:"uid"` +} + +// NodeClaimPhase is the coarse, user-facing lifecycle state. +// +// The NodeClaim is a passive teardown ledger, not a status mirror: it does NOT +// track finer workload runtime status (CPU/logs/restarts) — the Pod is the +// source of truth for that (see pkg/vnode/status.go). It tracks only the three +// coarse states that matter to its own job as a ledger, keyed off the served +// Pod's phase: Provisioning (instance coming up), Bound (instance running — the +// guard the teardown backstop trusts), and Terminated (instance gone). Finer +// states (e.g. Preempted) are deliberately absent: preemption cannot be detected +// — the provider contract's InstanceState has no Preempted value, and an absent +// instance only tells us it is gone, not why. Reintroduce a phase only when +// something actually sets it. +type NodeClaimPhase string + +const ( + // NodeClaimProvisioning: the served Pod has been observed but is not yet + // running — the external instance is still being provisioned. The claim does + // NOT earn the Bound teardown guard here: a Pod that vanishes while still + // provisioning is treated as possible cache lag (grace window), not a real + // teardown, because we never confirmed the instance was actually up. + NodeClaimProvisioning NodeClaimPhase = "Provisioning" + // NodeClaimBound: the served Pod has been observed running (present and not in + // a terminal phase). This is the durable guard the backstop trusts — a Bound + // claim whose Pod later disappears is a real teardown, not cache lag. The claim + // does not track finer workload status; the Pod is the source of truth for that. + NodeClaimBound NodeClaimPhase = "Bound" + // NodeClaimTerminated: the external instance is gone. Set when the served Pod + // has reached a terminal phase (Failed/Succeeded) — VK reports that when the + // provider's instance disappears (torn down, reclaimed, or exited). The claim + // stays around as a ledger of the vanished instance until its Pod is deleted; + // it does NOT self-delete on this transition (the instance is already gone, so + // there is nothing left to reclaim). + NodeClaimTerminated NodeClaimPhase = "Terminated" +) + +// NodeClaimStatus is the durable record reconciled against the provider by the +// poll loop. +type NodeClaimStatus struct { + // Phase is the coarse lifecycle state. + // +optional + Phase NodeClaimPhase `json:"phase,omitempty"` + + // InstanceID is the provider's identifier for the external instance (e.g. a + // RunPod pod id). This is the field that must not be lost: the terminate + // finalizer uses it to reclaim the instance. + // +optional + InstanceID string `json:"instanceID,omitempty"` + + // NodeName is the virtual Node created for this instance, once it exists. + // +optional + NodeName string `json:"nodeName,omitempty"` + + // Endpoint is the reachable address (e.g. SSH host:port) once ready. + // +optional + Endpoint string `json:"endpoint,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName=nc +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Provider",type=string,JSONPath=`.spec.provider` +// +kubebuilder:printcolumn:name="Instance",type=string,JSONPath=`.status.instanceID` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// NodeClaim represents one external GPU instance and its lifecycle. +type NodeClaim struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NodeClaimSpec `json:"spec,omitempty"` + Status NodeClaimStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NodeClaimList contains a list of NodeClaim. +type NodeClaimList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NodeClaim `json:"items"` +} + +func init() { + SchemeBuilder.Register(&NodeClaim{}, &NodeClaimList{}) +} diff --git a/api/v1alpha1/nodepool_types.go b/api/v1alpha1/nodepool_types.go new file mode 100644 index 0000000..080d132 --- /dev/null +++ b/api/v1alpha1/nodepool_types.go @@ -0,0 +1,175 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NodePoolSpec is the placement policy for a set of workloads. It is the +// long-lived, user-facing object: editing it changes behaviour for every Pod +// that selects the pool, without touching any workload. +// +// Placement resolves along two orthogonal axes, and the ORDER between them is +// fixed: capacity type first, provider second. +// +// FOR each capacityType in CapacityTypes (in listed order): // outer: hard tier +// candidates = Providers x {this capacityType}, available now, minus blocklist +// IF candidates non-empty: +// pick one via Strategy (LowestPrice | Ordered | Weighted) // inner: rank providers +// DONE +// // else fall through to the next capacity tier +// +// So CapacityTypes is a hard preference: every provider's Spot is tried before +// ANY provider's OnDemand. This is deliberate — "spot everywhere before any +// on-demand" is the least surprising behaviour, even if a different provider's +// on-demand were momentarily cheaper. Strategy only ranks providers *within* +// the active capacity tier; it never crosses tiers. +// +// The Weighted strategy requires a weight on every provider ref. This is a +// static property of the spec, so it is enforced at admission by the CEL rule +// below rather than surfaced as a status condition after the fact. +// +kubebuilder:validation:XValidation:rule="self.strategy != 'Weighted' || self.providers.all(p, has(p.weight))",message="strategy Weighted requires a weight on every provider" +type NodePoolSpec struct { + // Providers is the ordered set of NeoClouds this pool is allowed to use. + // A Pod bound to this pool can only ever be placed on a provider in this + // list. Order is significant only for the Ordered strategy (it is the + // inner, provider-ranking axis). + // +kubebuilder:validation:MinItems=1 + Providers []ProviderRef `json:"providers"` + + // CapacityTypes is the OUTER axis: the purchase models to try, in fallback + // order. e.g. [Spot, OnDemand] means "use spot on any provider first; only + // when spot is exhausted everywhere, drop to on-demand". A single-element + // list pins the pool to that type. This replaces a spot on/off flag and + // extends to Reserved without new fields. + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:default={Reserved,OnDemand,Spot} + CapacityTypes []CapacityType `json:"capacityTypes,omitempty"` + + // Strategy is the INNER axis: how to rank providers within the active + // capacity tier. It never overrides the capacity tier ordering. + // +kubebuilder:validation:Enum=LowestPrice;Ordered;Weighted + // +kubebuilder:default=Ordered + Strategy PlacementStrategy `json:"strategy,omitempty"` + + // Failover controls how a provider that fails at provision time (e.g. + // RunPod reports no capacity) is temporarily excluded and re-tried. + // +optional + Failover *FailoverPolicy `json:"failover,omitempty"` +} + +// ProviderRef names a provider and, for the Weighted strategy, its share. +type ProviderRef struct { + // Name is the provider identifier, matching the ProviderLabel value on that + // provider's virtual node (e.g. "runpod", "modal", "kubernetes"). + Name string `json:"name"` + + // Weight is the relative share of new placements for the Weighted strategy. + // Ignored by other strategies. + // +kubebuilder:validation:Minimum=1 + // +optional + Weight *int32 `json:"weight,omitempty"` +} + +// PlacementStrategy ranks providers WITHIN a capacity tier (the inner axis). +type PlacementStrategy string + +const ( + // StrategyLowestPrice picks the lowest $/hr provider in the active tier. + StrategyLowestPrice PlacementStrategy = "LowestPrice" + // StrategyOrdered uses the Providers list order as strict priority. + StrategyOrdered PlacementStrategy = "Ordered" + // StrategyWeighted spreads placements to match per-provider weights. + StrategyWeighted PlacementStrategy = "Weighted" +) + +// CapacityType is the purchase model (the outer axis). Each provider maps it to +// its own concept — e.g. RunPod Spot -> interruptible/podRentInterruptable. +// +kubebuilder:validation:Enum=Spot;OnDemand;Reserved +type CapacityType string + +const ( + // CapacitySpot is interruptible/preemptible capacity (cheapest, reclaimable). + CapacitySpot CapacityType = "Spot" + // CapacityOnDemand is standard pay-as-you-go capacity. + CapacityOnDemand CapacityType = "OnDemand" + // CapacityReserved is pre-committed/reserved capacity (not all providers + // support it; reserved for future use). + CapacityReserved CapacityType = "Reserved" +) + +// FailoverPolicy tunes capacity-error failover. Failover is always on — backing +// off a placement that just failed is correct behaviour, not a toggle (to avoid +// a whole provider, use a single-element Providers list instead). The blocklist +// itself is derived, high-churn runtime state held in controller memory — NOT in +// the API: when a provision fails, the controller excludes the failing placement +// (keyed at the granularity of the error, e.g. a specific provider+GPU+capacity +// type, so a failed H100 request does not block A100 requests on the same +// provider) for BlocklistTTL, then reconsiders it. This spec only tunes that. +type FailoverPolicy struct { + // BlocklistTTL is how long a failed placement is excluded before the + // provider becomes a candidate for it again. + // +kubebuilder:default="10m" + BlocklistTTL metav1.Duration `json:"blocklistTTL,omitempty"` +} + +// NodePool condition types (standard Kubernetes condition convention). +const ( + // NodePoolConditionReady is True when the pool's policy is valid and usable: + // every referenced provider is registered and the strategy is well-formed. + // It flips to False (with a reason) on a configuration error, so an operator + // sees the problem on the pool rather than as silent placement failures. + NodePoolConditionReady = "Ready" +) + +// NodePool condition reasons. +const ( + // ReasonPoolValid: the pool passed validation. + ReasonPoolValid = "Valid" + // ReasonUnknownProvider: a spec.providers[] entry names a provider with no + // registered adapter, so the pool cannot place onto it. This is an + // environmental check (it depends on the provider registry, populated at + // controller startup), so it lives here as a status condition rather than as + // an admission rule. Static spec validation (e.g. Weighted requires weights) + // is enforced at admission by a CEL rule on NodePoolSpec instead. + ReasonUnknownProvider = "UnknownProvider" +) + +// NodePoolStatus surfaces the current placement picture for observability. +type NodePoolStatus struct { + // Placed counts running instances per provider, for at-a-glance balance. + // +optional + Placed map[string]int32 `json:"placed,omitempty"` + + // Conditions follows the standard Kubernetes condition convention. + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName=np +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Strategy",type=string,JSONPath=`.spec.strategy` +// +kubebuilder:printcolumn:name="Providers",type=string,JSONPath=`.spec.providers[*].name` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// NodePool is the placement policy for GPU workloads across NeoClouds. +type NodePool struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NodePoolSpec `json:"spec,omitempty"` + Status NodePoolStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// NodePoolList contains a list of NodePool. +type NodePoolList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NodePool `json:"items"` +} + +func init() { + SchemeBuilder.Register(&NodePool{}, &NodePoolList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..e619508 --- /dev/null +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,287 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FailoverPolicy) DeepCopyInto(out *FailoverPolicy) { + *out = *in + out.BlocklistTTL = in.BlocklistTTL +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FailoverPolicy. +func (in *FailoverPolicy) DeepCopy() *FailoverPolicy { + if in == nil { + return nil + } + out := new(FailoverPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClaim) DeepCopyInto(out *NodeClaim) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClaim. +func (in *NodeClaim) DeepCopy() *NodeClaim { + if in == nil { + return nil + } + out := new(NodeClaim) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NodeClaim) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClaimList) DeepCopyInto(out *NodeClaimList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NodeClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClaimList. +func (in *NodeClaimList) DeepCopy() *NodeClaimList { + if in == nil { + return nil + } + out := new(NodeClaimList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NodeClaimList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClaimSpec) DeepCopyInto(out *NodeClaimSpec) { + *out = *in + out.PodRef = in.PodRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClaimSpec. +func (in *NodeClaimSpec) DeepCopy() *NodeClaimSpec { + if in == nil { + return nil + } + out := new(NodeClaimSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClaimStatus) DeepCopyInto(out *NodeClaimStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClaimStatus. +func (in *NodeClaimStatus) DeepCopy() *NodeClaimStatus { + if in == nil { + return nil + } + out := new(NodeClaimStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePool) DeepCopyInto(out *NodePool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePool. +func (in *NodePool) DeepCopy() *NodePool { + if in == nil { + return nil + } + out := new(NodePool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NodePool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePoolList) DeepCopyInto(out *NodePoolList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NodePool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePoolList. +func (in *NodePoolList) DeepCopy() *NodePoolList { + if in == nil { + return nil + } + out := new(NodePoolList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NodePoolList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePoolSpec) DeepCopyInto(out *NodePoolSpec) { + *out = *in + if in.Providers != nil { + in, out := &in.Providers, &out.Providers + *out = make([]ProviderRef, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.CapacityTypes != nil { + in, out := &in.CapacityTypes, &out.CapacityTypes + *out = make([]CapacityType, len(*in)) + copy(*out, *in) + } + if in.Failover != nil { + in, out := &in.Failover, &out.Failover + *out = new(FailoverPolicy) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePoolSpec. +func (in *NodePoolSpec) DeepCopy() *NodePoolSpec { + if in == nil { + return nil + } + out := new(NodePoolSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePoolStatus) DeepCopyInto(out *NodePoolStatus) { + *out = *in + if in.Placed != nil { + in, out := &in.Placed, &out.Placed + *out = make(map[string]int32, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePoolStatus. +func (in *NodePoolStatus) DeepCopy() *NodePoolStatus { + if in == nil { + return nil + } + out := new(NodePoolStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodReference) DeepCopyInto(out *PodReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodReference. +func (in *PodReference) DeepCopy() *PodReference { + if in == nil { + return nil + } + out := new(PodReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProviderRef) DeepCopyInto(out *ProviderRef) { + *out = *in + if in.Weight != nil { + in, out := &in.Weight, &out.Weight + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderRef. +func (in *ProviderRef) DeepCopy() *ProviderRef { + if in == nil { + return nil + } + out := new(ProviderRef) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..b505780 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,328 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "crypto/tls" + "flag" + "os" + "path/filepath" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + corev1 "k8s.io/api/core/v1" + + "k8s.io/client-go/kubernetes" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/internal/controller" + webhookv1 "github.com/InftyAI/Nebula/internal/webhook/v1" + "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/provider/modal" + "github.com/InftyAI/Nebula/pkg/vnode" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(nebulav1alpha1.AddToScheme(scheme)) + utilruntime.Must(corev1.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme +} + +// nolint:gocyclo +func main() { + var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string + var webhookCertPath, webhookCertName, webhookCertKey string + var enableLeaderElection bool + var probeAddr string + var secureMetrics bool + var enableHTTP2 bool + var tlsOpts []func(*tls.Config) + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") + flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") + flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") + flag.BoolVar(&enableHTTP2, "enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + // Create watchers for metrics and webhooks certificates + var metricsCertWatcher, webhookCertWatcher *certwatcher.CertWatcher + + // Initial webhook TLS options + webhookTLSOpts := tlsOpts + + if len(webhookCertPath) > 0 { + setupLog.Info("Initializing webhook certificate watcher using provided certificates", + "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey) + + var err error + webhookCertWatcher, err = certwatcher.New( + filepath.Join(webhookCertPath, webhookCertName), + filepath.Join(webhookCertPath, webhookCertKey), + ) + if err != nil { + setupLog.Error(err, "Failed to initialize webhook certificate watcher") + os.Exit(1) + } + + webhookTLSOpts = append(webhookTLSOpts, func(config *tls.Config) { + config.GetCertificate = webhookCertWatcher.GetCertificate + }) + } + + webhookServer := webhook.NewServer(webhook.Options{ + TLSOpts: webhookTLSOpts, + }) + + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(metricsCertPath, metricsCertName), + filepath.Join(metricsCertPath, metricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsServerOptions, + WebhookServer: webhookServer, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "nebula.inftyai.com", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + // Register provider backends into the process-wide registry that both + // reconcilers resolve through (their Providers field defaults to + // provider.Get). Done before SetupWithManager so a pool/claim reconciled at + // startup already sees its provider. + registerProviders(context.Background()) + + if err := (&controller.NodePoolReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NodePool") + os.Exit(1) + } + if err := (&controller.NodeClaimReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NodeClaim") + os.Exit(1) + } + if err := (&controller.PodPlacementReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "PodPlacement") + os.Exit(1) + } + + // Start one virtual node per registered provider. The virtual kubelet owns + // provisioning: its pod controller calls provider.Provision on CreatePod and + // provider.Terminate on DeletePod, so an ungated Pod bound to a provider's + // virtual node materializes an external instance. Each Runner is a + // manager.Runnable, so it shares the manager's lifecycle and leader election. + if err := setupVirtualNodes(mgr); err != nil { + setupLog.Error(err, "unable to set up virtual nodes") + os.Exit(1) + } + // nolint:goconst + if os.Getenv("ENABLE_WEBHOOKS") != "false" { + if err := webhookv1.SetupPodWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Pod") + os.Exit(1) + } + } + // +kubebuilder:scaffold:builder + + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + + if webhookCertWatcher != nil { + setupLog.Info("Adding webhook certificate watcher to manager") + if err := mgr.Add(webhookCertWatcher); err != nil { + setupLog.Error(err, "unable to add webhook certificate watcher to manager") + os.Exit(1) + } + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} + +// setupVirtualNodes adds a vnode.Runner to the manager for every registered +// provider. The Runner needs a typed clientset (the virtual kubelet's node/pod +// controllers use client-go directly, not the controller-runtime client), built +// from the same rest.Config the manager uses. +func setupVirtualNodes(mgr ctrl.Manager) error { + clientset, err := kubernetes.NewForConfig(mgr.GetConfig()) + if err != nil { + return err + } + for _, name := range provider.Names() { + prov, ok := provider.Get(name) + if !ok { + continue + } + if err := mgr.Add(vnode.NewRunner(prov, clientset)); err != nil { + return err + } + setupLog.Info("registered virtual node", "provider", name, "node", vnode.NodeName(name)) + } + return nil +} + +// registerProviders wires the compiled-in provider adapters into the registry. +// A provider whose credentials are absent (e.g. Modal creds not mounted in a +// dev cluster) is logged and skipped rather than fatal: the control plane must +// still run for the providers that ARE configured, and a pool referencing an +// unregistered provider surfaces as a clear NodePool condition rather than a +// crash loop. +func registerProviders(ctx context.Context) { + if p, err := modal.NewSDKClient(ctx, os.Getenv("MODAL_APP_NAME")); err != nil { + setupLog.Info("skipping Modal provider registration", "reason", err.Error()) + } else { + provider.Register(p) + setupLog.Info("registered provider", "provider", p.Name()) + } +} diff --git a/config/catalog/kustomization.yaml b/config/catalog/kustomization.yaml new file mode 100644 index 0000000..efc7dd7 --- /dev/null +++ b/config/catalog/kustomization.yaml @@ -0,0 +1,23 @@ +# Renders the checked-in, git-tracked provider price catalogs +# (pkg/provider/catalog/data/*.csv) into a ConfigMap the manager mounts at +# /etc/nebula/catalog. The CSVs remain the source of truth in-repo (reviewed via +# PR); this ConfigMap is how they reach the running controller, and ops can +# `kubectl edit configmap nebula-catalog` to adjust prices live without a +# rebuild (the manager re-reads on restart). +# +# This component sets NO namespace/namePrefix of its own: it is included by +# config/default, which applies the shared `nebula-` prefix and `nebula-system` +# namespace. That also lets kustomize rewrite the Deployment's `catalog` volume +# reference to the generated `nebula-catalog` name automatically. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +configMapGenerator: +- name: catalog + files: + - modal.csv=../../pkg/provider/catalog/data/modal.csv + +generatorOptions: + # Stable name (no content-hash suffix) so `kubectl edit` and the volume + # reference stay valid across price updates. + disableNameSuffixHash: true diff --git a/config/certmanager/certificate-metrics.yaml b/config/certmanager/certificate-metrics.yaml new file mode 100644 index 0000000..93051d2 --- /dev/null +++ b/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert diff --git a/config/certmanager/certificate-webhook.yaml b/config/certmanager/certificate-webhook.yaml new file mode 100644 index 0000000..d1a4e4e --- /dev/null +++ b/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert diff --git a/config/certmanager/issuer.yaml b/config/certmanager/issuer.yaml new file mode 100644 index 0000000..a9bc7e2 --- /dev/null +++ b/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 0000000..fcb7498 --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 0000000..cf6f89e --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,8 @@ +# This configuration is for teaching kustomize how to update name ref substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name diff --git a/config/crd/bases/nebula.inftyai.com_nodeclaims.yaml b/config/crd/bases/nebula.inftyai.com_nodeclaims.yaml new file mode 100644 index 0000000..7e2943e --- /dev/null +++ b/config/crd/bases/nebula.inftyai.com_nodeclaims.yaml @@ -0,0 +1,138 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: nodeclaims.nebula.inftyai.com +spec: + group: nebula.inftyai.com + names: + kind: NodeClaim + listKind: NodeClaimList + plural: nodeclaims + shortNames: + - nc + singular: nodeclaim + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.provider + name: Provider + type: string + - jsonPath: .status.instanceID + name: Instance + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeClaim represents one external GPU instance and its lifecycle. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodeClaimSpec is the durable identity of one external instance: who it serves, + which provider it lives on, and which policy produced it. It is created and + owned by the controller (not users), one per placed Pod. The workload shape + (image, resources, GPU type/count, spot) is NOT duplicated here — that lives + on the Pod, which the provider controller reads directly. NodeClaim is a + ledger, not a spec: its reason to exist is to survive the Node so teardown can + reclaim the external instance and never leak a paid GPU. + properties: + capacityType: + description: |- + CapacityType is the purchase tier the placement optimizer selected + (Spot/OnDemand/Reserved). It is stored durably here because it is the one + provisioning input that cannot be read off the Pod, and Provision needs it + to re-issue the request after a controller restart. Immutable, like + Provider. Empty means "let the provider use its default" (e.g. Modal is + OnDemand-only and ignores it). + enum: + - Spot + - OnDemand + - Reserved + type: string + podRef: + description: |- + PodRef links this claim to the Pod it serves. UID pins the exact Pod so a + recreated Pod of the same name gets a fresh claim rather than adopting the + old instance. + properties: + name: + type: string + namespace: + type: string + uid: + description: |- + UID pins the exact Pod object; a recreated Pod of the same name gets a new + NodeClaim rather than silently adopting the old instance. + type: string + required: + - name + - namespace + - uid + type: object + poolRef: + description: PoolRef is the NodePool whose policy produced this claim, + for reporting. + type: string + provider: + description: |- + Provider is the chosen NeoCloud. Immutable: a NodeClaim never migrates. + Recovery from preemption is delete-and-recreate. Held durably so teardown + knows which provider API to call even after status is lost. + type: string + required: + - podRef + - provider + type: object + status: + description: |- + NodeClaimStatus is the durable record reconciled against the provider by the + poll loop. + properties: + endpoint: + description: Endpoint is the reachable address (e.g. SSH host:port) + once ready. + type: string + instanceID: + description: |- + InstanceID is the provider's identifier for the external instance (e.g. a + RunPod pod id). This is the field that must not be lost: the terminate + finalizer uses it to reclaim the instance. + type: string + nodeName: + description: NodeName is the virtual Node created for this instance, + once it exists. + type: string + phase: + description: Phase is the coarse lifecycle state. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/nebula.inftyai.com_nodepools.yaml b/config/crd/bases/nebula.inftyai.com_nodepools.yaml new file mode 100644 index 0000000..5599385 --- /dev/null +++ b/config/crd/bases/nebula.inftyai.com_nodepools.yaml @@ -0,0 +1,222 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: nodepools.nebula.inftyai.com +spec: + group: nebula.inftyai.com + names: + kind: NodePool + listKind: NodePoolList + plural: nodepools + shortNames: + - np + singular: nodepool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.strategy + name: Strategy + type: string + - jsonPath: .spec.providers[*].name + name: Providers + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: NodePool is the placement policy for GPU workloads across NeoClouds. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: "NodePoolSpec is the placement policy for a set of workloads. + It is the\nlong-lived, user-facing object: editing it changes behaviour + for every Pod\nthat selects the pool, without touching any workload.\n\nPlacement + resolves along two orthogonal axes, and the ORDER between them is\nfixed: + capacity type first, provider second.\n\n\tFOR each capacityType in + CapacityTypes (in listed order): // outer: hard tier\n\t candidates + = Providers x {this capacityType}, available now, minus blocklist\n\t + \ IF candidates non-empty:\n\t pick one via Strategy (LowestPrice + | Ordered | Weighted) // inner: rank providers\n\t DONE\n\t // + else fall through to the next capacity tier\n\nSo CapacityTypes is a + hard preference: every provider's Spot is tried before\nANY provider's + OnDemand. This is deliberate — \"spot everywhere before any\non-demand\" + is the least surprising behaviour, even if a different provider's\non-demand + were momentarily cheaper. Strategy only ranks providers *within*\nthe + active capacity tier; it never crosses tiers.\n\nThe Weighted strategy + requires a weight on every provider ref. This is a\nstatic property + of the spec, so it is enforced at admission by the CEL rule\nbelow rather + than surfaced as a status condition after the fact." + properties: + capacityTypes: + default: + - Reserved + - OnDemand + - Spot + description: |- + CapacityTypes is the OUTER axis: the purchase models to try, in fallback + order. e.g. [Spot, OnDemand] means "use spot on any provider first; only + when spot is exhausted everywhere, drop to on-demand". A single-element + list pins the pool to that type. This replaces a spot on/off flag and + extends to Reserved without new fields. + items: + description: |- + CapacityType is the purchase model (the outer axis). Each provider maps it to + its own concept — e.g. RunPod Spot -> interruptible/podRentInterruptable. + enum: + - Spot + - OnDemand + - Reserved + type: string + minItems: 1 + type: array + failover: + description: |- + Failover controls how a provider that fails at provision time (e.g. + RunPod reports no capacity) is temporarily excluded and re-tried. + properties: + blocklistTTL: + default: 10m + description: |- + BlocklistTTL is how long a failed placement is excluded before the + provider becomes a candidate for it again. + type: string + type: object + providers: + description: |- + Providers is the ordered set of NeoClouds this pool is allowed to use. + A Pod bound to this pool can only ever be placed on a provider in this + list. Order is significant only for the Ordered strategy (it is the + inner, provider-ranking axis). + items: + description: ProviderRef names a provider and, for the Weighted + strategy, its share. + properties: + name: + description: |- + Name is the provider identifier, matching the ProviderLabel value on that + provider's virtual node (e.g. "runpod", "modal", "kubernetes"). + type: string + weight: + description: |- + Weight is the relative share of new placements for the Weighted strategy. + Ignored by other strategies. + format: int32 + minimum: 1 + type: integer + required: + - name + type: object + minItems: 1 + type: array + strategy: + default: Ordered + description: |- + Strategy is the INNER axis: how to rank providers within the active + capacity tier. It never overrides the capacity tier ordering. + enum: + - LowestPrice + - Ordered + - Weighted + type: string + required: + - providers + type: object + x-kubernetes-validations: + - message: strategy Weighted requires a weight on every provider + rule: self.strategy != 'Weighted' || self.providers.all(p, has(p.weight)) + status: + description: NodePoolStatus surfaces the current placement picture for + observability. + properties: + conditions: + description: Conditions follows the standard Kubernetes condition + convention. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + placed: + additionalProperties: + format: int32 + type: integer + description: Placed counts running instances per provider, for at-a-glance + balance. + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 0000000..b8628ac --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,17 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/nebula.inftyai.com_nodepools.yaml +- bases/nebula.inftyai.com_nodeclaims.yaml +# +kubebuilder:scaffold:crdkustomizeresource + +patches: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +# +kubebuilder:scaffold:crdkustomizewebhookpatch + +# [WEBHOOK] To enable webhook, uncomment the following section +# the following config is for teaching kustomize how to do kustomization for CRDs. +#configurations: +#- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 0000000..ec5c150 --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/default/cert_metrics_manager_patch.yaml b/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 0000000..d975015 --- /dev/null +++ b/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,30 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 0000000..a0e5424 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,244 @@ +# Adds namespace to all resources. +namespace: nebula-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: nebula- + +# Labels to add to all resources and selectors. +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue + +resources: +- ../crd +- ../rbac +- ../manager +# Price/availability catalog ConfigMap, rendered from the checked-in CSVs and +# mounted by the manager at /etc/nebula/catalog. +- ../catalog +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- ../webhook +# [CERTMANAGER] cert-manager is intentionally NOT used. The webhook serving +# cert is generated as a self-signed cert by hack/gen-webhook-cert.sh (run via +# `make deploy-all`), which also injects the CA into the webhook config. To +# switch back to cert-manager, re-add `- ../certmanager` here and re-enable the +# CERTMANAGER replacements blocks below. See docs/deploy.md. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml +# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. +# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. +# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will +# be able to communicate with the Webhook Server. +#- ../network-policy + +# Uncomment the patches line if you enable Metrics +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment + +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- path: manager_webhook_patch.yaml + target: + kind: Deployment + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - select: # Uncomment the following to set the Service name for TLS config in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 0 +# create: true + +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# - select: # Uncomment the following to set the Service namespace for TLS in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 1 +# create: true + +# [CERTMANAGER] Disabled — self-signed cert via hack/gen-webhook-cert.sh instead. +# - source: # Uncomment the following block if you have any webhook +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name # Name of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace # Namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have a ValidatingWebhook (--programmatic-validation) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true + +# [CERTMANAGER] Disabled — the CA is injected into the MutatingWebhookConfiguration +# by hack/gen-webhook-cert.sh (kubectl patch caBundle) instead of this annotation. +# - source: # Uncomment the following block if you have a DefaultingWebhook (--defaulting ) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have a ConversionWebhook (--conversion) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionns +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/config/default/manager_metrics_patch.yaml b/config/default/manager_metrics_patch.yaml new file mode 100644 index 0000000..2aaef65 --- /dev/null +++ b/config/default/manager_metrics_patch.yaml @@ -0,0 +1,4 @@ +# This patch adds the args to allow exposing the metrics endpoint using HTTPS +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 0000000..963c8a4 --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,31 @@ +# This patch ensures the webhook certificates are properly mounted in the manager container. +# It configures the necessary arguments, volumes, volume mounts, and container ports. + +# Add the --webhook-cert-path argument for configuring the webhook certificate path +- op: add + path: /spec/template/spec/containers/0/args/- + value: --webhook-cert-path=/tmp/k8s-webhook-server/serving-certs + +# Add the volumeMount for the webhook certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-webhook-server/serving-certs + name: webhook-certs + readOnly: true + +# Add the port configuration for the webhook server +- op: add + path: /spec/template/spec/containers/0/ports/- + value: + containerPort: 9443 + name: webhook-server + protocol: TCP + +# Add the volume configuration for the webhook certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: webhook-certs + secret: + secretName: webhook-server-cert diff --git a/config/default/metrics_service.yaml b/config/default/metrics_service.yaml new file mode 100644 index 0000000..9cb46a1 --- /dev/null +++ b/config/default/metrics_service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager + app.kubernetes.io/name: nebula diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 0000000..086bc1e --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,8 @@ +resources: +- manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: inftyai/nebula-controller + newTag: latest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 0000000..64fe631 --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,134 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize +spec: + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: nebula + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + app.kubernetes.io/name: nebula + spec: + # TODO(user): Uncomment the following code to configure the nodeAffinity expression + # according to the platforms which are supported by your solution. + # It is considered best practice to support multiple architectures. You can + # build your manager image using the makefile target docker-buildx. + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/arch + # operator: In + # values: + # - amd64 + # - arm64 + # - ppc64le + # - s390x + # - key: kubernetes.io/os + # operator: In + # values: + # - linux + securityContext: + # Projects are configured by default to adhere to the "restricted" Pod Security Standards. + # This ensures that deployments meet the highest security requirements for Kubernetes. + # For more details, see: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - command: + - /manager + args: + - --leader-elect + - --health-probe-bind-address=:8081 + image: controller:latest + name: manager + env: + # Point the price catalog at the mounted ConfigMap so prices can be + # edited live (kubectl edit configmap nebula-catalog) without a + # rebuild. If unset/absent, the manager falls back to the CSVs + # embedded in the binary. + - name: NEBULA_CATALOG_DIR + value: /etc/nebula/catalog + envFrom: + # Provider credentials live in a per-provider Secret, one secretRef per + # provider — NOT a single shared secret. This matches the "creds-absent → + # skip that provider, not fatal" model (see registerProviders): each + # secretRef is independently optional=true, so configuring Modal but not + # RunPod just skips RunPod. It also isolates rotation and RBAC per + # provider. Providers namespace their env vars (MODAL_TOKEN_ID, + # RUNPOD_API_KEY, ...), so the merged environment never collides. + # + # The Modal SDK reads MODAL_TOKEN_ID / MODAL_TOKEN_SECRET (and optionally + # MODAL_ENVIRONMENT / MODAL_APP_NAME) from the environment. + - secretRef: + name: nebula-modal-credentials + optional: true + # Add one secretRef per provider as adapters land, e.g.: + # - secretRef: + # name: nebula-runpod-credentials + # optional: true + ports: [] + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + volumeMounts: + - name: catalog + mountPath: /etc/nebula/catalog + readOnly: true + volumes: + - name: catalog + configMap: + # Unprefixed on purpose: config/default's namePrefix rewrites this + # reference to the generated "nebula-catalog" ConfigMap. optional so + # the manager still starts (embedded-CSV fallback) if it's absent. + name: catalog + optional: true + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/config/manager/modal-credentials.example.yaml b/config/manager/modal-credentials.example.yaml new file mode 100644 index 0000000..c733ab7 --- /dev/null +++ b/config/manager/modal-credentials.example.yaml @@ -0,0 +1,40 @@ +# Modal provider credentials — EXAMPLE, not applied by kustomize. +# +# The manager's Deployment consumes this via an optional `envFrom.secretRef` +# (see manager.yaml). Credentials are kept in a per-provider Secret rather than +# the manifest, and one secretRef per provider — configuring Modal but not +# another provider simply skips the unconfigured one (see registerProviders). +# +# To use: +# 1. Get a Modal token pair: `modal token new` (or from the Modal dashboard), +# which yields a token id (ak-...) and token secret (as-...). +# 2. Create the Secret in the manager's namespace. Prefer `kubectl create +# secret` over committing this file with real values: +# +# kubectl create secret generic nebula-modal-credentials \ +# --namespace nebula-system \ +# --from-literal=MODAL_TOKEN_ID=ak-xxxxxxxxxxxxxxxx \ +# --from-literal=MODAL_TOKEN_SECRET=as-xxxxxxxxxxxxxxxx +# +# Optional keys: MODAL_ENVIRONMENT (Modal workspace environment) and +# MODAL_APP_NAME (the Modal App all Nebula sandboxes are created under; +# defaults to "nebula"). +# +# The key NAMES matter: the Modal SDK reads MODAL_TOKEN_ID / MODAL_TOKEN_SECRET +# straight from the environment, and envFrom projects every key in this Secret +# as an env var of the same name. +# +# This YAML is provided only for teams that manage Secrets declaratively (sealed +# secrets, SOPS, GitOps). Replace the placeholders below — do NOT commit real +# tokens in plaintext. +apiVersion: v1 +kind: Secret +metadata: + name: nebula-modal-credentials + namespace: nebula-system +type: Opaque +stringData: + MODAL_TOKEN_ID: "ak-REPLACE_ME" + MODAL_TOKEN_SECRET: "as-REPLACE_ME" + # MODAL_ENVIRONMENT: "main" + # MODAL_APP_NAME: "nebula" diff --git a/config/network-policy/allow-metrics-traffic.yaml b/config/network-policy/allow-metrics-traffic.yaml new file mode 100644 index 0000000..37d5f48 --- /dev/null +++ b/config/network-policy/allow-metrics-traffic.yaml @@ -0,0 +1,27 @@ +# This NetworkPolicy allows ingress traffic +# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those +# namespaces are able to gather data from the metrics endpoint. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: allow-metrics-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: nebula + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label metrics: enabled + - from: + - namespaceSelector: + matchLabels: + metrics: enabled # Only from namespaces with this label + ports: + - port: 8443 + protocol: TCP diff --git a/config/network-policy/allow-webhook-traffic.yaml b/config/network-policy/allow-webhook-traffic.yaml new file mode 100644 index 0000000..33c839f --- /dev/null +++ b/config/network-policy/allow-webhook-traffic.yaml @@ -0,0 +1,27 @@ +# This NetworkPolicy allows ingress traffic to your webhook server running +# as part of the controller-manager from specific namespaces and pods. CR(s) which uses webhooks +# will only work when applied in namespaces labeled with 'webhook: enabled' +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: allow-webhook-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: nebula + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label webhook: enabled + - from: + - namespaceSelector: + matchLabels: + webhook: enabled # Only from namespaces with this label + ports: + - port: 443 + protocol: TCP diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml new file mode 100644 index 0000000..0872bee --- /dev/null +++ b/config/network-policy/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- allow-webhook-traffic.yaml +- allow-metrics-traffic.yaml diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 0000000..fdc5481 --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,11 @@ +resources: +- monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 0000000..cd43a12 --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,27 @@ +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: nebula diff --git a/config/prometheus/monitor_tls_patch.yaml b/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 0000000..5bf84ce --- /dev/null +++ b/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,19 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +- op: replace + path: /spec/endpoints/0/tlsConfig + value: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + serverName: SERVICE_NAME.SERVICE_NAMESPACE.svc + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 0000000..f15d94b --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,31 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml +# For each CRD, "Admin", "Editor" and "Viewer" roles are scaffolded by +# default, aiding admins in cluster management. Those roles are +# not used by the nebula itself. You can comment the following lines +# if you do not want those helpers be installed with your Project. +- nodeclaim_admin_role.yaml +- nodeclaim_editor_role.yaml +- nodeclaim_viewer_role.yaml +- nodepool_admin_role.yaml +- nodepool_editor_role.yaml +- nodepool_viewer_role.yaml + diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 0000000..bb5b625 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,40 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000..2410860 --- /dev/null +++ b/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/metrics_auth_role.yaml b/config/rbac/metrics_auth_role.yaml new file mode 100644 index 0000000..32d2e4e --- /dev/null +++ b/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/metrics_auth_role_binding.yaml b/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 0000000..e775d67 --- /dev/null +++ b/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/metrics_reader_role.yaml b/config/rbac/metrics_reader_role.yaml new file mode 100644 index 0000000..51a75db --- /dev/null +++ b/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/nodeclaim_admin_role.yaml b/config/rbac/nodeclaim_admin_role.yaml new file mode 100644 index 0000000..a6d6f6e --- /dev/null +++ b/config/rbac/nodeclaim_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project nebula itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over nebula.inftyai.com. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: nodeclaim-admin-role +rules: +- apiGroups: + - nebula.inftyai.com + resources: + - nodeclaims + verbs: + - '*' +- apiGroups: + - nebula.inftyai.com + resources: + - nodeclaims/status + verbs: + - get diff --git a/config/rbac/nodeclaim_editor_role.yaml b/config/rbac/nodeclaim_editor_role.yaml new file mode 100644 index 0000000..9f8e728 --- /dev/null +++ b/config/rbac/nodeclaim_editor_role.yaml @@ -0,0 +1,33 @@ +# This rule is not used by the project nebula itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the nebula.inftyai.com. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: nodeclaim-editor-role +rules: +- apiGroups: + - nebula.inftyai.com + resources: + - nodeclaims + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - nebula.inftyai.com + resources: + - nodeclaims/status + verbs: + - get diff --git a/config/rbac/nodeclaim_viewer_role.yaml b/config/rbac/nodeclaim_viewer_role.yaml new file mode 100644 index 0000000..c0c0e82 --- /dev/null +++ b/config/rbac/nodeclaim_viewer_role.yaml @@ -0,0 +1,29 @@ +# This rule is not used by the project nebula itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to nebula.inftyai.com resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: nodeclaim-viewer-role +rules: +- apiGroups: + - nebula.inftyai.com + resources: + - nodeclaims + verbs: + - get + - list + - watch +- apiGroups: + - nebula.inftyai.com + resources: + - nodeclaims/status + verbs: + - get diff --git a/config/rbac/nodepool_admin_role.yaml b/config/rbac/nodepool_admin_role.yaml new file mode 100644 index 0000000..987dc0a --- /dev/null +++ b/config/rbac/nodepool_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project nebula itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over nebula.inftyai.com. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: nodepool-admin-role +rules: +- apiGroups: + - nebula.inftyai.com + resources: + - nodepools + verbs: + - '*' +- apiGroups: + - nebula.inftyai.com + resources: + - nodepools/status + verbs: + - get diff --git a/config/rbac/nodepool_editor_role.yaml b/config/rbac/nodepool_editor_role.yaml new file mode 100644 index 0000000..bc451ad --- /dev/null +++ b/config/rbac/nodepool_editor_role.yaml @@ -0,0 +1,33 @@ +# This rule is not used by the project nebula itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the nebula.inftyai.com. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: nodepool-editor-role +rules: +- apiGroups: + - nebula.inftyai.com + resources: + - nodepools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - nebula.inftyai.com + resources: + - nodepools/status + verbs: + - get diff --git a/config/rbac/nodepool_viewer_role.yaml b/config/rbac/nodepool_viewer_role.yaml new file mode 100644 index 0000000..803e6c2 --- /dev/null +++ b/config/rbac/nodepool_viewer_role.yaml @@ -0,0 +1,29 @@ +# This rule is not used by the project nebula itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to nebula.inftyai.com resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: nodepool-viewer-role +rules: +- apiGroups: + - nebula.inftyai.com + resources: + - nodepools + verbs: + - get + - list + - watch +- apiGroups: + - nebula.inftyai.com + resources: + - nodepools/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 0000000..201b154 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,96 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manager-role +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + - services + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - nodes/status + - pods/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - nebula.inftyai.com + resources: + - nodeclaims + - nodepools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - nebula.inftyai.com + resources: + - nodeclaims/finalizers + - nodepools/finalizers + verbs: + - update +- apiGroups: + - nebula.inftyai.com + resources: + - nodeclaims/status + - nodepools/status + verbs: + - get + - patch + - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml new file mode 100644 index 0000000..873e53f --- /dev/null +++ b/config/rbac/role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml new file mode 100644 index 0000000..5f2eaa3 --- /dev/null +++ b/config/rbac/service_account.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: controller-manager + namespace: system diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml new file mode 100644 index 0000000..2d9b784 --- /dev/null +++ b/config/samples/deployment.yaml @@ -0,0 +1,74 @@ +# A minimal GPU workload that opts into Nebula placement. +# +# A workload opts in with just THREE fields on the POD TEMPLATE — everything else +# Nebula adds for you (the webhook and placement controller act on Pods, so the +# opt-in lives on spec.template, NOT on the Deployment's own metadata): +# 1. label nebula.inftyai.com/enabled: "true" -> opt in (webhook acts only on this) +# 2. label nebula.inftyai.com/nodepool: -> which NodePool to place against +# 3. label nebula.inftyai.com/accelerator-type: -> requested GPU type +# (count rides on the standard nvidia.com/gpu resource below) +# +# On CREATE, the mutating webhook injects the provider-selection scheduling gate +# and a toleration for the virtual-node taint on each Pod. The Pod sits +# SchedulingGated until the placement controller picks a provider from the pool, +# stamps the provider nodeSelector, and removes the gate — then the Pod binds to +# that provider's virtual node and the VK provisions the real instance. +# +# A Deployment (rather than a bare Pod) means its ReplicaSet recreates a Pod that +# fails or is torn down, so the workload — and its backing GPU instance — is +# self-healing rather than a one-shot that lingers Failed. +# +# Prereqs: the NodePool below must exist (kubectl apply -k config/samples applies +# both), and its providers must be registered (their credential Secrets present). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gpu-workload-sample + namespace: default + labels: + app.kubernetes.io/managed-by: nebula +spec: + replicas: 8 + selector: + matchLabels: + app: gpu-workload-sample + template: + metadata: + labels: + app: gpu-workload-sample + # --- opt in (must be on the Pod template) --- + nebula.inftyai.com/enabled: "true" + # Place against this pool (see nebula_v1alpha1_nodepool.yaml). + nebula.inftyai.com/nodepool: nodepool-sample + # Requested accelerator TYPE. Matched case-insensitively against the + # provider catalog (e.g. Modal offers T4/L4/A10G/A100-40GB/A100-80GB/ + # H100/H200). The COUNT is the nvidia.com/gpu resource below — omit it + # for a single GPU. + nebula.inftyai.com/accelerator-type: l4 + spec: + # Do NOT set nodeName or a provider nodeSelector yourself — the placement + # controller fills the nodeSelector in when it ungates the Pod. Setting + # nodeName would bypass placement entirely (the webhook skips bound Pods). + containers: + - name: workload + # A CUDA base image so nvidia-smi and the GPU userspace are present — a + # plain image (e.g. busybox) has no nvidia-smi. Any CUDA/GPU image works; + # this one is small and only needs the driver from the host. + image: nvidia/cuda:12.4.1-base-ubuntu22.04 + # Print the GPUs the sandbox sees at startup, then idle. + # + # NOTE: `kubectl logs` and `kubectl exec` do NOT work against a Nebula + # virtual node — it does not serve the kubelet API (see pkg/vnode + # handler.go: GetContainerLogs/RunInContainer return NotFound). So to see + # this output, read it on the PROVIDER side (the Modal dashboard or + # `modal app logs`), not via kubectl. + command: ["sh", "-c", "nvidia-smi --query-gpu=index,name,memory.total --format=csv || echo 'no nvidia-smi'; sleep 3600"] + resources: + requests: + cpu: "1" + memory: "1Gi" + limits: + # GPU count. Standard extended resource, so the scheduler's fit check + # and provisioning read the same number. 8 => 8x the accelerator-type + # above. + nvidia.com/gpu: "1" diff --git a/config/samples/nebula_v1alpha1_nodepool.yaml b/config/samples/nebula_v1alpha1_nodepool.yaml new file mode 100644 index 0000000..4eed653 --- /dev/null +++ b/config/samples/nebula_v1alpha1_nodepool.yaml @@ -0,0 +1,21 @@ +apiVersion: nebula.inftyai.com/v1alpha1 +kind: NodePool +metadata: + labels: + app.kubernetes.io/managed-by: nebula + name: nodepool-sample +spec: + # The NeoClouds this pool may place onto. Order matters only for the Ordered + # strategy (the inner, provider-ranking axis). + providers: + - name: modal + # - name: runpod + # Outer axis: try spot on every provider first, fall back to on-demand. + capacityTypes: + - Reserved + - OnDemand + - Spot + # Inner axis: within the active capacity tier. + strategy: Ordered + failover: + blocklistTTL: 10m diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 0000000..051676b --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,12 @@ +resources: +- manifests.yaml +- service.yaml + +# Narrow the generated Pod webhook to opt-in Pods outside system namespaces. +patches: +- path: selector_patch.yaml + target: + kind: MutatingWebhookConfiguration + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 0000000..206316e --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,22 @@ +# the following config is for teaching kustomize where to look at when substituting nameReference. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 0000000..a92f82b --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate--v1-pod + failurePolicy: Fail + name: mpod-v1.nebula.inftyai.com + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None diff --git a/config/webhook/selector_patch.yaml b/config/webhook/selector_patch.yaml new file mode 100644 index 0000000..4233a3c --- /dev/null +++ b/config/webhook/selector_patch.yaml @@ -0,0 +1,22 @@ +# Narrow the Pod mutating webhook so it only ever intercepts Pods that opt into +# Nebula, and never Pods in system namespaces. This is critical: the webhook is +# failurePolicy=Fail and on the Pod-CREATE critical path, so an over-broad scope +# would make every Pod creation in the cluster depend on the Nebula webhook being +# healthy. controller-gen markers cannot express selectors, so this patch layers +# them onto the generated MutatingWebhookConfiguration. +- op: add + path: /webhooks/0/objectSelector + value: + matchLabels: + nebula.inftyai.com/enabled: "true" +- op: add + path: /webhooks/0/namespaceSelector + value: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kube-node-lease + - kube-public + - nebula-system diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 0000000..d241e85 --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: nebula + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager + app.kubernetes.io/name: nebula diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..463c4b4 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,638 @@ +# Nebula Architecture + +Nebula is a Kubernetes control plane for GPU-as-a-Service: it schedules GPU +workloads across heterogeneous compute providers ("NeoClouds" — RunPod, Modal, +CoreWeave, Lambda — and vanilla Kubernetes) as if they were one cluster. A user +submits an ordinary `Deployment`/`Pod`; Nebula decides *which provider* runs it +based on cost, availability, and policy, provisions the external instance, and +reclaims it on teardown so a paid GPU is never leaked. + +This document is the top-level design. It is descriptive of decisions already +made and prescriptive for the parts not yet built (marked **PLANNED**). For +building and deploying the manager (including provider credentials), see +[docs/deploy.md](deploy.md). + +- [Goals and non-goals](#goals-and-non-goals) +- [System overview](#system-overview) +- [The placement flow](#the-placement-flow-end-to-end) + - [Status flow — two independent lanes](#status-flow--two-independent-lanes) +- [Components](#components) + - [Scheduling-gate webhook](#1-scheduling-gate-webhook) + - [Placement controller](#2-placement-controller-done) + - [Virtual Kubelet node](#3-virtual-kubelet-node-done) + - [NodeClaim controller](#4-nodeclaim-controller) + - [NodePool controller](#5-nodepool-controller) + - [Provider abstraction](#6-provider-abstraction) + - [Optimizer & poll loop](#7-optimizer--poll-loop-planned) +- [CRDs](#crds) +- [Failure domains & HA](#failure-domains--ha) +- [Build status](#build-status) + +--- + +## Goals and non-goals + +**Goals** + +- One Kubernetes API surface over many GPU clouds. Users write standard Pods; + they never call a provider SDK. +- Cost/availability-aware placement with failover: try the cheapest viable + {capacity tier, provider} first, fall back on capacity errors. +- Never leak a paid instance: teardown is guaranteed by a finalizer even if the + node/Pod disappears. +- Provider quirks stay behind one narrow seam; the control plane is + provider-agnostic. + +**Non-goals (v1)** + +- Region/zone placement. v1 targets NeoClouds, which are region-simple. Region + is a planned additive field, not modeled yet. +- Bin-packing many pods per instance. One workload → one external instance. +- In-place migration. Recovery from preemption is delete-and-recreate. + +--- + +## System overview + +Nebula borrows two proven patterns and fuses them: + +- **Virtual Kubelet (VK)** provides the *mechanism*: each provider is a single + virtual Node in the cluster. A Pod scheduled to that node is provisioned on + the NeoCloud via that provider's API instead of by a container runtime. +- **Karpenter-style CRDs** provide the *policy + ledger*: `NodePool` (which + providers, how to choose) and `NodeClaim` (a durable, cluster-scoped record of + one external instance. The virtual kubelet owns provisioning and the happy-path + teardown; the claim outlives the Pod and carries a finalizer so it can act as a + level-triggered teardown *backstop* when the Pod is force-deleted during a VK + outage — a paid GPU is never leaked). +- **Scheduling gates** are the *placement hook*: a mutating webhook holds an + opted-in Pod until Nebula's placement controller picks a provider, then + releases it to the native scheduler. + +``` + ┌──────────────────────────────────────────────────────┐ + │ Nebula manager │ + kubectl apply │ ┌────────────┐ ┌─────────────┐ ┌────────────────┐ │ + Deployment ──────┼─▶│ webhook │ │ placement │ │ NodePool / │ │ + (labeled │ │ (gate Pod) │ │ controller │ │NodeClaim ctrls │ │ + enabled=true) │ └────────────┘ └─────────────┘ └────────────────┘ │ + │ │ │ │ │ + └────────┼────────────────┼──────────────────┼──────────┘ + │ │ │ + gate added│ nodeSelector+ungate│ provision/terminate + ▼ ▼ ▼ + ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ + │ SchedulingGated│ │native scheduler│ │ Provider │ + │ Pod │──▶│ binds to vnode │──▶│ seam (Get/ │ + └──────────────┘ └──────────────┘ │ Provision..) │ + │ └──────┬───────┘ + ▼ │ + ┌─────────────────┐ ▼ + │ Virtual Kubelet │ ┌──────────────┐ + │ node (1/provider)│──▶│ NeoCloud API│ + │ CreatePod → │ │ (RunPod/Modal│ + │ provision │ │ /CoreWeave) │ + └─────────────────┘ └──────────────┘ +``` + +Everything runs in one controller-runtime manager binary (`cmd/main.go`) today. +The VK node processes are the one component that may run separately — see +[Virtual Kubelet node](#3-virtual-kubelet-node-done). + +--- + +## The placement flow (end to end) + +Follow a single GPU Pod from `kubectl apply` to a running instance: + +1. **Opt-in.** User creates a Pod (usually via a Deployment) carrying the label + `nebula.inftyai.com/enabled: "true"` and the label + `nebula.inftyai.com/accelerator-type: a100-40gb` (accelerator type), with the + count expressed as a standard `nvidia.com/gpu` resource on the container + (omit it for a single GPU). The Pod is the *source of truth* for the workload + shape — image, command, env, resources, the accelerator type (label) and count + (`nvidia.com/gpu`); keeping the count on the standard resource means the + scheduler's fit check and provisioning read the same number. + +2. **Gate (webhook).** The mutating webhook, scoped by an objectSelector to + opted-in Pods outside system namespaces, injects the scheduling gate + `nebula.inftyai.com/provider-selection` and a key-only `Exists` toleration for + the virtual-node taint `nebula.inftyai.com/provider:NoSchedule`. The Pod is + now `SchedulingGated`; the native scheduler will not bind it. `failurePolicy= + Fail` — a Pod that should be placed by Nebula must never slip past ungated. + +3. **Place (placement controller).** Watches SchedulingGated Pods with our + gate. For each opted-in, gated, not-yet-bound Pod it resolves the Pod's + `NodePool` (via the `nebula.inftyai.com/nodepool` label), selects a provider + (v1 policy: **first matching provider** — the first in the pool's provider list + whose catalog offers the Pod's GPU type; a CPU-only Pod matches any), and then, + **in this order**: + - creates the `NodeClaim` **first** — named deterministically + `-`, back-referencing the Pod by UID, recording the chosen + provider, capacity tier, and pool. The claim is the durable teardown ledger, + so it must exist *before* the Pod can bind and provision. `Create` is + idempotent on the fixed name, but idempotency is **UID-scoped**: if a claim + of that name already exists, placement ungates only when it pins *this* Pod's + UID (a genuine retry after a crash between create and ungate). If it still + pins a prior same-named Pod (recreated faster than the backstop reaped the old + claim), placement does **not** ungate — it requeues until the NodeClaim + backstop sees the old Pod is gone, terminates that instance, and deletes the + stale claim, after which the create succeeds with the correct UID. This is + what makes "the claim exists before bind/provision" a guarantee that the + claim names *this* workload's instance, not a stale one. + - then, in one Pod update: adds `nodeSelector: + {nebula.inftyai.com/provider: }`, writes the chosen tier to + `nebula.inftyai.com/capacity-type` (the one provisioning input not otherwise + on the Pod), and removes the gate. + + If no pool resolves, or no provider in the pool can serve the Pod, the Pod is + **left gated** (no guessing); a later reconcile — a pool edit that adds a + matching provider re-enqueues its gated Pods — can place it. + +4. **Bind.** With the gate gone and a provider nodeSelector set, the native + scheduler binds the Pod to that provider's virtual Node. + +5. **Provision (VK).** Binding writes `spec.nodeName = nebula-`; + that provider's VK node observes the Pod via its node-scoped informer and fires + `CreatePod`. It reads the workload off the Pod, builds a + `ProvisionRequest{ClaimName, CapacityType}` (ClaimName derived deterministically + from the Pod, CapacityType from the annotation), and calls `provider.Provision`. + **The virtual kubelet owns provisioning** — the placement controller never + calls Provision; it only routes the Pod. The observed instance state is + projected onto the Pod's status. + +6. **Observe (poll loop in VK).** The VK handler's poll loop calls + `provider.List()` periodically (no NeoCloud pushes preemption events) and + pushes observed instance state onto Pod status via VK's `NotifyPods`. The + NodeClaim controller does **not** mirror this finer runtime status — the Pod is + the source of truth for it. The claim tracks only a coarse guard: once its + served Pod is observed, the claim is marked `Bound` (see step 7). + +7. **Teardown (VK happy path + NodeClaim backstop — DONE).** Two layers guarantee + the paid instance is reclaimed: + - **Happy path (VK):** when the Pod/Deployment is deleted, VK calls + `DeletePod`, which runs `provider.Terminate(instanceID)`. + - **Backstop (NodeClaim):** the claim is cluster-scoped and *outlives* the Pod, + and carries the `TerminateInstance` finalizer. The NodeClaim controller is + level-triggered: it marks a claim `Bound` once it has seen the served Pod + (the durable guard), and when a **Bound** claim's Pod later vanishes it treats + that as a real teardown and self-deletes; the finalizer then resolves the + provider, finds the instance by **claim name via `List()`** (not + `status.InstanceID`, which VK tracks in-memory and loses on a crash), and + calls `Terminate` before releasing. On the happy path this is a redundant + idempotent no-op (VK already terminated, so `List` finds nothing); its whole + reason to exist is the case where the Pod is force-deleted while that + provider's VK is down, so `DeletePod` never runs. A claim that was **never** + Bound and whose Pod is absent is treated as possible cache lag: it waits a + `placementGracePeriod` (2m) before being deleted, so a fresh claim isn't + reaped in the window between create and the Pod appearing. + +### Status flow — two independent lanes + +The single most subtle part of the design: **the Pod and the NodeClaim are +populated from different sources and never mirror each other.** Runtime status +lives on the Pod; the claim tracks only a coarse teardown guard. + +``` + Pod.status ◀── (pkg/vnode) NodeClaim.status ◀── (NodeClaim controller) + ═══════════════════════════ ═══════════════════════════════════════════ + source: the provider instance source: whether the served Pod EXISTS + (never the instance's runtime state) + + edge-triggered (instant): Reconcile(Pod present/absent): + CreatePod → PodPending Pod present → phase=Bound (one-way latch) + Provision err → PodFailed Pod absent + wasBound → self-delete → backstop + DeletePod → PodSucceeded Pod absent + !Bound + grace → self-delete + Pod absent + !Bound + in grace → requeue + level-triggered (poll, §7): + List() → match by ClaimName + InstanceRunning → PodRunning, Ready=True, PodIP=endpoint + InstancePending → PodPending, Ready=False + InstanceFailed → PodFailed + InstanceTerminated → PodFailed/"Preempted" (also: absent from List) +``` + +- **Pod lane (`pkg/vnode`).** The happy-path transitions are pushed + synchronously — `CreatePod` emits `Pending` (or `Failed`) the moment + `Provision` returns, `DeletePod` emits a terminal phase — so provisioning-start + and teardown do **not** wait on any poll. The poll loop's *only* job is the + out-of-band transitions nothing pushes: an instance becoming reachable + (`Pending→Running`) and, the costly one, a spot reclaim (`Running→Terminated`). + `applyState` (`status.go`) is the sole mapping from `provider.InstanceState` to + a standard Pod phase/condition, so `kubectl get pod` shows Running/Ready/PodIP. + +- **NodeClaim lane (NodeClaim controller).** The claim deliberately does **not** + copy the instance's runtime state. Its phase is only `Pending → Bound`, where + `Bound` is a one-way latch meaning "I have observed the served Pod at least + once." That latch is the *arming signal for the teardown backstop*: a `Bound` + claim whose Pod later vanishes is a real teardown (self-delete → finalizer + terminates the instance), whereas a claim that never reached `Bound` and has no + Pod is treated as possible cache lag and waits out `placementGracePeriod`. + +**Why the split.** An earlier design had the claim mirror the instance state +(the still-present but now-unused `NodeClaimProvisioning/Running/Preempted/ +Terminating` constants are its residue). It was dropped because it put two +pollers — VK writing the Pod, the claim controller writing the claim — on the +same fact from different lag windows, so they disagreed during the poll window +for no benefit: the claim needs only "did the Pod ever exist" + "is it gone now" +to do its one job. `NodeClaimStatus.InstanceID`/`Endpoint` fields exist but are +currently unwritten; the backstop re-derives the instance via `List()` + +`ClaimName` precisely so teardown survives a VK crash that lost the in-memory id. + +**Failover (PLANNED).** If `Provision` fails (e.g. RunPod reports no capacity), +the provider classifies the error into a `BlockScope` (this accelerator+tier, or +the whole provider for auth/quota). The optimizer will exclude that scope for +`BlocklistTTL`; the Deployment recreates the Pod (gated again) and the controller +ungates it toward the next candidate. v1's "first matching provider" selection +does not yet consult a blocklist — a failed Provision surfaces on the Pod and is +retried against the same first match until the optimizer lands. + +--- + +## Components + +### 1. Scheduling-gate webhook + +A mutating (defaulting) webhook on the external `core/v1` Pod type. On CREATE it +injects the `provider-selection` scheduling gate **and** a key-only `Exists` +toleration for the virtual-node taint (`nebula.inftyai.com/provider:NoSchedule`), +when and only when: + +- the Pod carries `nebula.inftyai.com/enabled: "true"`; +- the Pod is not already scheduled (`spec.nodeName == ""` — the API server + rejects adding a gate to a bound Pod). + +Both mutations are idempotent (the gate is not duplicated; a matching provider +toleration is not re-added). The toleration is injected here rather than by the +placement controller because it must be present before the scheduler evaluates +the Pod, and it is provider-agnostic (Exists matches whichever provider the +placement controller later selects). + +**Scoping is critical** and lives in `config/webhook/selector_patch.yaml` (a +kustomize patch, since controller-gen markers cannot express selectors): + +- `objectSelector` on `enabled=true` — only opted-in Pods reach the webhook; +- `namespaceSelector` excluding `kube-system`, `kube-node-lease`, `kube-public`, + `nebula-system`. + +This matters because `failurePolicy=Fail`: an over-broad webhook would make +*every* Pod creation in the cluster depend on the Nebula webhook being healthy. +The narrow selector + verbs=CREATE-only keeps the blast radius to opted-in Pods. + +### 2. Placement controller + +The consumer of the gate — the middle of the critical path. Reconciles Pods that +are opted-in, still hold our gate, and are not yet bound (`needsPlacement` +filters out anything else, so it is a no-op for ordinary Pods). For each: + +- Resolve the target `NodePool` (Pod label `nebula.inftyai.com/nodepool`). Missing or + mislabeled → leave the Pod gated (an operator sees a `SchedulingGated` Pod and a + missing pool, rather than a silent mis-placement). +- **Select a provider.** v1 policy is **first matching provider**: walk the pool's + providers in listed order and pick the first whose registered adapter offers the + Pod's GPU type (`MapAccelerator`); a CPU-only Pod matches any. `selectPlacement` + is the seam the richer optimizer (LowestPrice/Weighted, capacity-tier fallback, + blocklist) swaps in behind later — the surrounding flow does not change. No + match → leave the Pod gated. +- **Create the `NodeClaim` first, then ungate** (ordering is load-bearing — see + the [placement flow](#the-placement-flow-end-to-end) step 3). The claim carries + a fixed, Pod-derived name so `Create` is idempotent across retries; ungating is + a single Pod `Update` that sets the provider `nodeSelector`, the `capacity-type` + annotation, and removes the gate. + +A pool edit re-enqueues that pool's still-gated Pods (`podsForPool`), so adding a +provider that can now serve a stuck Pod retries placement promptly instead of +waiting for the resync. A "gated too long" alert remains a required operational +safeguard (see [HA](#failure-domains--ha)). + +### 3. Virtual Kubelet node + +This is the mechanism that turns "a Pod bound to a fake node" into "an instance +running on a NeoCloud". **Decision: one static virtual Node per provider**, and +**the virtual kubelet owns provisioning** (CreatePod → Provision, DeletePod → +Terminate). Implemented in `pkg/vnode` (`handler.go`, `node.go`, `status.go`). + +#### Why one node per provider + +VK's native model is one node with many pods. The alternative — a node per +instance — means building your own node-fleet/lease manager on top of VK +(registering and deregistering a node per workload, each with its own heartbeat). +We chose the static per-provider node: far simpler, VK-native, and the natural +unit for a provider that is really "a pool of capacity we rent from". The +accepted cost is a single failure domain per provider (if a provider's VK +process is down, that provider can't take new pods) — acceptable because +placement can route to other providers. Note the tradeoff of "VK owns +provisioning": because the happy-path teardown is `DeletePod → +provider.Terminate`, reclaiming an instance on that path depends on that +provider's VK being alive to process the Pod deletion. This gap is closed by the +**NodeClaim teardown backstop** (see [NodeClaim controller](#4-nodeclaim-controller)): +the cluster-scoped claim outlives the Pod and carries a finalizer, so if the Pod +is force-deleted while VK is down, the level-triggered claim controller still +reclaims the instance — found by claim name via `List()` — the next time it runs. + +#### The node object + +Each provider VK registers a Node with: + +- name `nebula-` (e.g. `nebula-runpod`); +- label `nebula.inftyai.com/provider: ` — the key the placement + controller writes into a Pod's nodeSelector to route it here; +- a **taint** `nebula.inftyai.com/provider=:NoSchedule` so that *only* + Pods Nebula has explicitly routed land on it. The mutating webhook adds a + key-only `Exists` toleration for `nebula.inftyai.com/provider:NoSchedule` at + admission (the provider is not chosen yet, so it tolerates any value); the + placement controller then routes the Pod with a provider nodeSelector. Without + the taint, any unscheduled Pod could drift onto a NeoCloud node. +- generous/opaque `capacity` (VK convention) — real capacity is enforced by the + provider API and the optimizer's availability table, not by node allocatable. + +#### The handler: VK events → the provider seam + +VK asks us to implement `node.PodLifecycleHandler`. Each method is a thin +adapter onto the existing `provider.Provider` seam — **no provider-specific +logic lives in the VK layer**; it only translates between Pod events and the +seam: + +VK asks us to implement `node.PodLifecycleHandler` (+ `PodNotifier`). Each method +is a thin adapter onto the existing `provider.Provider` seam — **no +provider-specific logic lives in the VK layer**; it only translates between Pod +events and the seam: + +| `PodLifecycleHandler` method | Nebula action | +|---|---| +| `CreatePod(pod)` | Derive `ClaimName` from the Pod (namespace-name); read `CapacityType` from the `capacity-type` annotation; call `provider.Provision(pod, req)`; track the returned `instanceID` and set Pod status Pending. | +| `UpdatePod(pod)` | No-op for immutable workloads in v1 (a spec change is a new Pod); the tracked copy's metadata is refreshed. | +| `DeletePod(pod)` | Call `provider.Terminate(instanceID)` (idempotent), report a terminal status, and drop the Pod from tracking. This is the happy-path teardown; the NodeClaim finalizer is the backstop when VK never sees the delete. | +| `GetPod` / `GetPodStatus` | Return the tracked Pod / its status. | +| `GetPods` | Return the Pods this node is tracking. | +| `NotifyPods(cb)` | Start the poll loop; `provider.List()` on the provider's cadence (`Capabilities.PollInterval`, default 30s) matches instances to tracked Pods by claim name and pushes state changes (Running/Failed/Preempted) through `cb`. | + +Because Provision is idempotent (keyed on `ClaimName`, derived deterministically +from the Pod), a `CreatePod` retry after a crash re-attaches to the existing +instance instead of double-provisioning. + +The poll cadence is **per-provider** (`Capabilities.PollInterval`, default 30s), +because the trade-off differs by provider: the poll only exists to catch +transitions nothing pushes (`Pending→Running`, and preemption +`Running→Terminated`), so a spot-heavy backend where reclaims are common and +costly wants a short interval to notice them quickly, while an OnDemand-only +backend that never preempts (e.g. Modal) can poll lazily. The happy-path +transitions (provision start, teardown) are emitted synchronously by +`CreatePod`/`DeletePod` and do not wait on the poll. The cost of a faster +cadence is `List()` call volume — one API call per provider per tick regardless +of Pod count — so the real ceiling is the provider's rate limit, not ours. + +The kubelet API surface (logs/exec/attach/stats/port-forward) is intentionally +unsupported — Nebula routes external GPU workloads, it does not proxy their +consoles — so those handler methods return `NotFound`. This also lets us wire the +lower-level `node` package directly rather than the `nodeutil` convenience +wrapper, whose kubelet HTTP/auth stack does not compile against the pinned k8s +0.33 line. + +#### Where VK runs + +VK is a long-running process that registers a node and maintains its lease. Two +options: + +- **A (chosen):** run the per-provider VK nodes *inside* the existing manager, + as `manager.Runnable`s (`vnode.Runner`) the manager starts. One binary, one + deployment, simplest ops. They share the manager's lifecycle and, if enabled, + leader election, so only the leader owns each node's lease. +- **B:** a separate Deployment per provider. Stronger isolation (a crash-looping + Modal adapter can't take down the RunPod node) at the cost of N deployments + and duplicated wiring. + +**Implemented as A.** `setupVirtualNodes` in `cmd/main.go` adds one +`vnode.Runner` per registered provider. Revisit B if provider isolation becomes +a real operational need. + +#### Interaction with capacity types + +A provider VK node exists once, but a Pod's capacity tier (Spot/OnDemand) is a +per-instance decision the optimizer made. Because the VK provisions solely from +the Pod, the tier rides on the `nebula.inftyai.com/capacity-type` annotation +(written by the placement controller) — *not* on the node. `CreatePod` reads it +into the `ProvisionRequest`. This keeps "one node per provider" compatible with +"different pods on the same provider use different tiers". + +### 4. NodeClaim controller + +The VK owns provisioning and the *happy-path* teardown; the NodeClaim controller +is the **level-triggered backstop** that guarantees no paid instance leaks even +when the Pod → VK path never fires (Pod force-deleted while that provider's VK is +down). The claim is cluster-scoped and outlives the Pod, and carries the +`TerminateInstance` finalizer. + +Reconcile logic: + +- **Add the finalizer first.** A claim with no finalizer gets it (and nothing + else) on the first reconcile, so teardown is guaranteed from the moment the + claim exists. +- **Bound guard.** Once the served Pod (matched by UID) is observed, the claim is + marked `Bound`. `Bound` is the durable signal "an instance was really serving + this workload" — the guard the self-delete trusts. +- **Self-delete on real teardown.** A `Bound` claim whose Pod has since vanished + is a real teardown: the claim deletes itself (the finalizer keeps it alive long + enough to run the backstop). +- **Cache-lag guard.** A claim that was **never** `Bound` and whose Pod is absent + might just be racing the Pod's cache appearance. It waits `placementGracePeriod` + (2m) from creation before being reaped, so a freshly-placed claim isn't deleted + in the window between create and the Pod showing up. +- **Deletion path (`reconcileDelete`).** Resolve the provider (unknown adapter → + release the finalizer rather than wedge deletion forever), find the instance — + preferring `status.InstanceID` if set, else matching the **claim name** against + `provider.List()` (VK tracks the id in-memory and loses it on a crash, so `List` + is the reliable recovery path) — call `Terminate` (idempotent), then release the + finalizer. A transient `List` error does **not** release the finalizer: teardown + retries so the instance is never abandoned. + +On the happy path the backstop is a redundant idempotent no-op — VK already +terminated, so `List` finds nothing and `Terminate("")` does nothing. The claim +deliberately does **not** mirror the Pod's finer runtime status; the Pod is the +source of truth for that, and `kubectl get pods` shows it. + +### 5. NodePool controller + +The NodePool is **pure policy** (which providers, capacity-tier order, ranking +strategy, failover TTL) — no workload shape. The controller is its health & +observability loop: + +- **validate** — the one *environmental* check: every referenced provider has a + registered adapter. Surfaced as `Ready=False / UnknownProvider`, not an + admission rejection, so a valid manifest stays appliable while a provider's + creds are temporarily absent (self-heals on the next reconcile). +- **refreshPlaced** — recomputes `status.Placed` (count of `Bound` NodeClaims + per provider for this pool — `Bound` being the claim-level signal that an + instance is live for the workload) each reconcile; watches NodeClaims via + `claimToPool` so the picture tracks instances coming and going. + +*Static* spec validation (e.g. "Weighted strategy requires a weight on every +provider") is enforced at **admission** via a CEL `XValidation` rule on +`NodePoolSpec`, not in the controller — a spec property is knowable at CREATE and +belongs there. The split — static→admission, environmental→status condition — is +deliberate: coupling a fail-closed webhook to the runtime provider registry +would reject valid pools during creds/rollout windows. + +### 6. Provider abstraction + +The single narrow seam between the provider-agnostic control plane and the +heterogeneous cloud APIs. Key rules: + +- **The Pod is the source of truth.** `Provision(pod, req)` reads + image/command/env/ports/resources off the Pod, the accelerator type from its + `accelerator-type` label and the count from the `nvidia.com/gpu` resource + (parsed together by `util.AcceleratorRequest`). `ProvisionRequest` carries only + the two things *not* on the Pod: `ClaimName` (identity) and `CapacityType` (the + optimizer's tier choice). +- **Quirks are data, not branches.** `Capabilities{SupportsStop, SupportsSpot, + NativeTags, PreemptionNotice}` lets the control plane behave generically. +- **Poll-based detection.** `List()` returns all Nebula-owned instances in as + few calls as possible — the engine of the poll loop, since no NeoCloud pushes + preemption. +- **Error classification.** `ClassifyProvisionError(err) → BlockScope` maps a + failure to its blocklist granularity ({accel, tier} vs whole-provider). +- Shared catalog machinery (`catalog.Lookup`, embeddable `catalog.Base` for + Name/Offerings/identity-MapAccelerator) lives in `pkg/provider/catalog`. + +Adapters register into a process-wide registry (`pkg/provider/registry.go`) at +startup via `registerProviders` in `cmd/main.go`. A creds-absent provider is +logged and skipped, not fatal. + +**Modal adapter** (`pkg/provider/modal/`): serverless Sandboxes, create/terminate +only (no stop/spot), native tags (claim id in a sandbox tag), OnDemand-only. SDK +sits behind a `Client` seam so the adapter is unit-testable without network. The +sandbox is built from the Pod as source of truth: image/command/env, accelerator +type+count (from the annotation), CPU/memory (from the container's resource +request), exposed ports (from `containerPorts`, surfaced as the endpoint tunnel), +and lifetime (from `activeDeadlineSeconds`, defaulting to 24h — Modal treats a +zero timeout as its 5-minute default, so the adapter always sets one). Credentials +come from a per-provider Secret (`nebula-modal-credentials`) injected via an +optional `envFrom` — the SDK reads `MODAL_TOKEN_ID`/`MODAL_TOKEN_SECRET` from the +environment; see `config/manager/modal-credentials.example.yaml`. Still open: +`env.valueFrom` (Secret/ConfigMap-backed env) is not yet projected, and there is +no live integration test. + +### 7. Optimizer & poll loop (PLANNED) + +- **Catalog table.** `{provider, accelerator, capacityType} → {price, available}`, + built from provider `Offerings()` (static CSV catalog + live availability + probe), cached and periodically refreshed. +- **Placement resolution — two orthogonal axes, fixed order.** Capacity type is + the *outer* axis (hard tier): try Spot on *all* providers before *any* + OnDemand. Strategy (LowestPrice/Ordered/Weighted) is the *inner* axis: it only + ranks providers *within* the active tier, never across tiers. +- **Blocklist.** Derived, high-churn, in-memory (not in any CRD). Keyed on + {provider, accelerator, capacityType} with wildcard-on-empty, so a failed H100 + request doesn't block A100 on the same provider. Entries expire after + `FailoverPolicy.BlocklistTTL` (default 10m). Failover is always on. +- **Poll loop (DONE, in VK).** Each provider's virtual node calls `List()` on the + provider's cadence (`Capabilities.PollInterval`, default 30s) and projects + observed instance state onto **Pod** status — preemption/termination detected by + an instance disappearing or changing state. It does **not** write NodeClaims (an + earlier mirror-into-claim design was dropped — see + [Status flow](#status-flow--two-independent-lanes)). What remains PLANNED here is + the *optimizer's* offerings/availability refresh, not the pod-status poll. + +--- + +## CRDs + +Both are **cluster-scoped**, under `nebula.inftyai.com/v1alpha1`. Karpenter's +names are kept (not prefixed); disambiguation is via the API group and distinct +shortnames (`np`, `nc`). See `api/v1alpha1/`. + +### NodePool (`np`) — policy + +```yaml +spec: + providers: # ordered; order matters for the Ordered strategy + - name: runpod + weight: 3 # required iff strategy == Weighted (CEL-enforced) + - name: modal + capacityTypes: # OUTER axis, fallback order. default {Reserved,OnDemand,Spot} + - Spot + - OnDemand + strategy: Ordered # INNER axis: LowestPrice | Ordered | Weighted + failover: + blocklistTTL: 10m +status: + placed: {runpod: 2, modal: 1} # running NodeClaims per provider + conditions: [{type: Ready, ...}] +``` + +A pool is pure policy and applies to whatever Pods select it — an H100 and an +A100 Deployment can share one pool. + +### NodeClaim (`nc`) — instance ledger + +```yaml +spec: + podRef: {namespace, name, uid} # UID pins the exact Pod + provider: runpod # immutable; a claim never migrates + capacityType: Spot # tier chosen by the optimizer, recorded for reporting + # (the VK reads it from the Pod's capacity-type annotation) + poolRef: gpu-pool +status: + phase: Bound # Pending (Pod not yet observed) | Bound (Pod observed ≥ once) + instanceID: # provider instance id, when VK recorded it (backstop hint) +``` + +Deliberately does **not** duplicate PodSpec/GPU/tier as workload data, nor mirror +the Pod's finer runtime status — the Pod is the source of truth for both. The +claim's own status is just the coarse `Bound` guard the teardown backstop relies +on. The claim is a durable teardown ledger + backstop, keyed on the Pod-derived +claim name; happy-path teardown is still the VK's `DeletePod → Terminate`. + +--- + +## Failure domains & HA + +Two components are on the fail-closed critical path and must run HA: + +- **Webhook** (`failurePolicy=Fail`): if it's down, opted-in Pod creation is + blocked. Run ≥2 replicas; the narrow objectSelector keeps non-Nebula Pods + unaffected. A gated-Pod path that stays gated is preferable to an ungated Pod + bypassing placement. +- **Placement controller**: if it never ungates, opted-in Pods sit + `SchedulingGated` forever. Required safeguard: a **"gated too long" alert** and + ideally a fallback (e.g. after a timeout, emit an event / optionally place on a + default provider). + +Non-critical by design: + +- **VK node down**: that provider can't take *new* pods, and existing instances + keep running untouched (the external NeoCloud does not care that VK is down). + Placement routes new pods to healthy providers. Happy-path teardown of that + provider's instances is deferred until its VK recovers — but a Pod force-deleted + during the outage is still reclaimed by the NodeClaim finalizer backstop, so no + instance leaks (see [NodeClaim controller](#4-nodeclaim-controller)). +- **Provider creds absent**: the adapter is skipped at registration (logged); + pools referencing it surface `UnknownProvider` and self-heal when creds arrive. + +Leader election (`LeaderElectionID: nebula.inftyai.com`) ensures a single active +manager owns reconciliation and the VK node leases (the VK nodes run inside the +manager as `Runnable`s — option A). + +--- + +## Build status + +| Component | Package | Status | +|---|---|---| +| API types (NodePool, NodeClaim) | `api/v1alpha1` | DONE | +| Provider interface + registry | `pkg/provider` | DONE | +| Price catalog (CSV + ConfigMap) | `pkg/provider/catalog` | DONE | +| Modal adapter | `pkg/provider/modal` | DONE | +| NodeClaim controller (teardown backstop) | `internal/controller` | DONE | +| NodePool controller | `internal/controller` | DONE | +| Scheduling-gate webhook | `internal/webhook/v1` | DONE | +| Provider wiring in manager | `cmd/main.go` | DONE | +| Virtual Kubelet node (VK owns provisioning) | `pkg/vnode` | DONE | +| Placement controller (first-matching-provider) | `internal/controller` | DONE | +| **Optimizer (price/weighted, capacity fallback, blocklist)** | — | **PLANNED** | +| Other adapters (RunPod next) | `pkg/provider/*` | PLANNED | + +The critical path — gated Pod → placed → bound → provisioned → torn down — is now +closed end to end. The next milestone is the **optimizer**: replacing the v1 +"first matching provider" policy behind `selectPlacement` with price/weighted +ranking, capacity-tier fallback, and the failover blocklist. diff --git a/docs/deploy.md b/docs/deploy.md new file mode 100644 index 0000000..deec82a --- /dev/null +++ b/docs/deploy.md @@ -0,0 +1,244 @@ +# Deploying Nebula + +This guide covers building and deploying the Nebula manager to a Kubernetes +cluster, including wiring provider credentials. + +- [Quick start](#quick-start) +- [How credentials are handled](#how-credentials-are-handled) +- [Webhook TLS (no cert-manager)](#webhook-tls-no-cert-manager) +- [What `deploy-all` does](#what-deploy-all-does) +- [Configuration](#configuration) +- [Adding a provider](#adding-a-provider) +- [Manual deployment](#manual-deployment) +- [Verifying the deployment](#verifying-the-deployment) + +--- + +## Quick start + +```bash +# 1. Put provider credentials in .env (secrets only — gitignored). +cp .env.example .env +$EDITOR .env # set MODAL_TOKEN_ID / MODAL_TOKEN_SECRET (from `modal token new`) + +# 2. Build, apply credential Secrets, and deploy in one step. +make deploy-all IMG=/nebula: +``` + +For a local [Kind](https://kind.sigs.k8s.io/) cluster, load the image instead of +pushing it to a registry: + +```bash +make deploy-all IMG=nebula:dev DEPLOY_KIND_CLUSTER= +``` + +That's it. The manager comes up in the `nebula-system` namespace and registers +every provider whose credentials are present. + +--- + +## How credentials are handled + +Nebula keeps provider credentials in **one Kubernetes Secret per provider** — not +a single shared secret. This is deliberate and matches the control plane's +"creds-absent → skip that provider, not fatal" model: + +- Each provider Secret is referenced by the manager Deployment through an + **optional** `envFrom.secretRef` (see `config/manager/manager.yaml`). Optional + means the manager still boots when a Secret is missing. +- A provider whose credentials are absent is **logged and skipped** at + registration (`registerProviders` in `cmd/main.go`), not fatal. A `NodePool` + that references an unregistered provider surfaces `Ready=False / + UnknownProvider`, which self-heals once the creds are applied. +- Splitting per provider isolates rotation and RBAC, and lets you configure one + provider without touching another's Secret. Providers namespace their env vars + (`MODAL_TOKEN_ID`, `RUNPOD_API_KEY`, …), so the merged environment never + collides. + +`.env` holds **only secrets**. Non-secret configuration (image, namespace, Kind +cluster) is passed as `make` variables, never committed to `.env`. + +> **Secret vs. absent looks the same.** Because `envFrom` is `optional: true`, a +> typo in a Secret name silently yields no credentials — the provider is just +> skipped. The `NodePool` `UnknownProvider` condition is your signal that a +> referenced provider did not register. + +--- + +## Webhook TLS (no cert-manager) + +Nebula runs a mutating webhook (it injects the scheduling gate into gated Pods), +and the API server requires TLS to call it. **cert-manager is intentionally not +used** — there is no cert-manager prerequisite to install. Instead, +`hack/gen-webhook-cert.sh` provisions a self-signed certificate. It does the two +things cert-manager would otherwise automate: + +1. **Serving cert** — generates a self-signed cert for the webhook Service DNS + name (`nebula-webhook-service.nebula-system.svc`) and stores it in the + `webhook-server-cert` TLS Secret the manager mounts. +2. **CA trust** — injects that cert's CA into the `MutatingWebhookConfiguration` + `caBundle`, so the API server trusts the webhook. The caBundle is always read + back *from the Secret*, so the served cert and the trusted CA cannot drift. + +**No manager restart is involved.** `deploy-all` orders things so the manager +boots already-correct: the cert Secret is created *before* `make deploy` (it is a +required volume mount, and a running pod would otherwise need remounting), while +the caBundle patch happens *after* (it edits only the webhook config, which only +the API server reads — the manager never sees it). + +Re-running is safe: an existing cert Secret is kept as-is, and the caBundle is +re-derived to match. + +> **No auto-rotation.** This is the one thing cert-manager gives you that the +> self-signed cert does not. The cert is valid 10 years (`CERT_DAYS`, default +> `3650`). To rotate/renew, regenerate the cert and re-inject the CA: +> ```bash +> FORCE_REGEN=true hack/gen-webhook-cert.sh secret +> hack/gen-webhook-cert.sh cabundle +> kubectl rollout restart deployment/nebula-controller-manager -n nebula-system +> ``` +> The restart here is only because you are *replacing* the cert under a +> already-running manager — the initial deploy needs none. + +To switch back to cert-manager, re-add `- ../certmanager` and re-enable the +`CERTMANAGER` replacements blocks in `config/default/kustomization.yaml`, install +cert-manager, and drop the `gen-webhook-cert.sh` calls from `hack/deploy.sh`. + +--- + +## What `deploy-all` does + +`make deploy-all` runs `hack/deploy.sh`, which is idempotent (safe to re-run): + +1. **Builds** the manager image (`make docker-build IMG=…`). +2. **Publishes** it — `docker push`, or `kind load docker-image` when + `DEPLOY_KIND_CLUSTER` is set. +3. **Creates Secrets first**, so the manager boots already-configured: + - the **webhook serving cert** Secret (self-signed — see + [Webhook TLS](#webhook-tls-no-cert-manager)); + - **credential Secrets** from `.env`, one per provider. A Secret is created + only when *all* its required keys are set, so a partial config never + produces a half-populated Secret; a provider with blank keys is skipped. +4. **Deploys** CRDs + the manager (`make deploy IMG=…`). The pod mounts the cert + and reads provider creds on its first and only boot. +5. **Injects the webhook CA bundle** into the `MutatingWebhookConfiguration` + (server-side; the manager is not touched). + +--- + +## Configuration + +`.env` (secrets only, gitignored — see `.env.example`): + +| Key | Provider | Required | Notes | +|---|---|---|---| +| `MODAL_TOKEN_ID` | Modal | yes | From `modal token new` | +| `MODAL_TOKEN_SECRET` | Modal | yes | From `modal token new` | +| `MODAL_ENVIRONMENT` | Modal | no | Modal workspace environment | +| `MODAL_APP_NAME` | Modal | no | Modal App for all sandboxes (default `nebula`) | + +Non-secret config, passed as `make` variables: + +| Variable | Default | Meaning | +|---|---|---| +| `IMG` | `controller:latest` | Manager image to build and deploy | +| `NAMESPACE` | `nebula-system` | Namespace the manager runs in | +| `DEPLOY_KIND_CLUSTER` | *(empty)* | If set, load the image into this Kind cluster instead of pushing. Separate from the e2e `KIND_CLUSTER`, so a plain `make deploy-all` pushes. | +| `KUBECTL` | `kubectl` | kubectl binary to use | + +--- + +## Adding a provider + +When a new provider adapter lands, wire its credentials in two places: + +1. **`hack/deploy.sh`** — add a row to the `PROVIDER_SECRETS` table: + + ```bash + PROVIDER_SECRETS=( + "nebula-modal-credentials|MODAL_TOKEN_ID MODAL_TOKEN_SECRET|MODAL_ENVIRONMENT MODAL_APP_NAME" + "nebula-runpod-credentials|RUNPOD_API_KEY|" + ) + ``` + Format: `||`. + +2. **`config/manager/manager.yaml`** — add an optional `secretRef` under + `envFrom`: + + ```yaml + - secretRef: + name: nebula-runpod-credentials + optional: true + ``` + +Then add the new keys to `.env.example`. Nothing else changes — the script +creates the Secret from `.env` and skips it if the keys are blank. + +--- + +## Manual deployment + +If you don't want the script (e.g. you manage Secrets via sealed-secrets or a +GitOps pipeline), do the same steps by hand. Order matters — create the Secrets +before deploying so the manager boots configured, and inject the CA after: + +```bash +# 1. Namespace + webhook serving cert Secret (before deploy — it's a volume mount). +kubectl create namespace nebula-system --dry-run=client -o yaml | kubectl apply -f - +hack/gen-webhook-cert.sh secret + +# 2. Modal credential Secret (before deploy — read as env at pod startup). +kubectl create secret generic nebula-modal-credentials -n nebula-system \ + --from-literal=MODAL_TOKEN_ID=ak-... \ + --from-literal=MODAL_TOKEN_SECRET=as-... + +# 3. Deploy CRDs + manager. The pod mounts the cert and reads creds on first boot. +make deploy IMG=/nebula: + +# 4. Inject the webhook CA (server-side; needs the webhook config to exist). +hack/gen-webhook-cert.sh cabundle +``` + +No restart is needed — everything the manager consumes exists before it boots. +If you instead created a Secret *after* the manager was already running, restart +it to pick up the change: +`kubectl rollout restart deployment/nebula-controller-manager -n nebula-system`. + +A declarative Secret manifest for GitOps workflows lives at +`config/manager/modal-credentials.example.yaml` (not applied by kustomize — copy +and fill it in, or template it with your secret manager). + +To tear everything down: + +```bash +make undeploy # remove the manager +make uninstall # remove CRDs +``` + +--- + +## Verifying the deployment + +```bash +# Manager is running. +kubectl -n nebula-system get pods + +# Providers registered (look for "registered provider" / "skipping … registration"). +kubectl -n nebula-system logs deploy/nebula-controller-manager | grep -i provider + +# Virtual nodes exist, one per registered provider. +kubectl get nodes -l nebula.inftyai.com/provider + +# Webhook TLS is wired: the caBundle matches the serving cert Secret. +diff <(kubectl get secret webhook-server-cert -n nebula-system -o jsonpath='{.data.tls\.crt}') \ + <(kubectl get mutatingwebhookconfiguration nebula-mutating-webhook-configuration \ + -o jsonpath='{.webhooks[0].clientConfig.caBundle}') \ + && echo "webhook caBundle matches serving cert" +``` + +A pool referencing an unregistered provider shows it plainly: + +```bash +kubectl get nodepool -o jsonpath='{.status.conditions}' +# Ready=False / UnknownProvider means that provider's creds are missing or wrong. +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e4ec53d --- /dev/null +++ b/go.mod @@ -0,0 +1,113 @@ +module github.com/InftyAI/Nebula + +go 1.24.0 + +require ( + github.com/modal-labs/modal-client/go v0.9.0 + github.com/onsi/ginkgo/v2 v2.27.2 + github.com/onsi/gomega v1.38.2 + github.com/prometheus/client_model v0.6.1 + github.com/virtual-kubelet/virtual-kubelet v1.11.0 + k8s.io/api v0.33.3 + k8s.io/apimachinery v0.33.3 + k8s.io/client-go v0.33.3 + sigs.k8s.io/controller-runtime v0.21.0 +) + +require ( + cel.dev/expr v0.24.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/aristanetworks/gomap v0.0.0-20230726210543-f4e41046dced // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/djherbis/buffer v1.2.0 // indirect + github.com/djherbis/nio/v3 v3.0.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.23.2 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kisielk/og-rek v1.3.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/spdystream v0.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect + go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/otel/sdk v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect + golang.org/x/mod v0.33.0 // indirect + golang.org/x/net v0.50.0 // indirect + golang.org/x/oauth2 v0.32.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/term v0.40.0 // indirect + golang.org/x/text v0.34.0 // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.42.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.33.0 // indirect + k8s.io/apiserver v0.33.0 // indirect + k8s.io/component-base v0.33.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c783216 --- /dev/null +++ b/go.sum @@ -0,0 +1,390 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/aristanetworks/gomap v0.0.0-20230726210543-f4e41046dced h1:HxlRMDx/VeRqzj3nvqX9k4tjeBcEIkoNHDJPsS389hs= +github.com/aristanetworks/gomap v0.0.0-20230726210543-f4e41046dced/go.mod h1:p7lmI+ecoe1RTyD11SPXWsSQ3H+pJ4cp5y7vtKW4QdM= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bombsimon/logrusr/v3 v3.1.0 h1:zORbLM943D+hDMGgyjMhSAz/iDz86ZV72qaak/CA0zQ= +github.com/bombsimon/logrusr/v3 v3.1.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= +github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ= +github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= +github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4= +github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= +github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY= +github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kisielk/og-rek v1.3.0 h1:lTXdQXqFETZKA//FWH4RBNAuiJ/dofxIwHAidoUZoMk= +github.com/kisielk/og-rek v1.3.0/go.mod h1:4at7oxyfBTDilURhNCf7irHWtosJlJl9uyqUqAkrP4w= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/modal-labs/modal-client/go v0.9.0 h1:tZ7JFCRzNRrfC3RA6e+4V/KwtGEliEOzEfJ9Xvb+4TM= +github.com/modal-labs/modal-client/go v0.9.0/go.mod h1:dMBMTXVQ6ReY/XFXAB3Tic0IuGtC755UoHbDcqkCgcc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/virtual-kubelet/virtual-kubelet v1.11.0 h1:LOMcZQfP083xmYH9mYtyHAR+ybFbK1uMaRA+EtDcd1I= +github.com/virtual-kubelet/virtual-kubelet v1.11.0/go.mod h1:WQfPHbIlzfhMNYkh6hFXF1ctGfNM8UJCYLYpLa/trxc= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= +golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= +golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= +golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda h1:+2XxjfsAu6vqFxwGBRcHiMaDCuZiqXGDUDVWVtrFAnE= +google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8= +k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE= +k8s.io/apiextensions-apiserver v0.33.0 h1:d2qpYL7Mngbsc1taA4IjJPRJ9ilnsXIrndH+r9IimOs= +k8s.io/apiextensions-apiserver v0.33.0/go.mod h1:VeJ8u9dEEN+tbETo+lFkwaaZPg6uFKLGj5vyNEwwSzc= +k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA= +k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.0 h1:QqcM6c+qEEjkOODHppFXRiw/cE2zP85704YrQ9YaBbc= +k8s.io/apiserver v0.33.0/go.mod h1:EixYOit0YTxt8zrO2kBU7ixAtxFce9gKGq367nFmqI8= +k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA= +k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg= +k8s.io/component-base v0.33.0 h1:Ot4PyJI+0JAD9covDhwLp9UNkUja209OzsJ4FzScBNk= +k8s.io/component-base v0.33.0/go.mod h1:aXYZLbw3kihdkOPMDhWbjGCO6sg+luw554KP51t8qCU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= +sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 0000000..c9914fc --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/hack/deploy.sh b/hack/deploy.sh new file mode 100755 index 0000000..9b358d1 --- /dev/null +++ b/hack/deploy.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# +# deploy.sh — build the manager image, create per-provider credential Secrets +# from a .env file, and deploy Nebula to the cluster in the current kube context. +# +# Usage: +# cp .env.example .env && $EDITOR .env +# hack/deploy.sh # or: make deploy-all +# IMG=myrepo/nebula:v1 KIND_CLUSTER=kind hack/deploy.sh +# +# .env holds ONLY secrets (provider credentials); see .env.example. Non-secret +# config is passed as environment variables / make flags: +# IMG manager image to build+deploy (default example.com/nebula:v0.0.1) +# NAMESPACE namespace the manager runs in (default nebula-system) +# KIND_CLUSTER if set, load the image into this Kind cluster instead of pushing +# +# The script is idempotent: re-running rebuilds the image and re-applies +# Secrets/manifests in place. +# +# Design notes: +# - Credentials live in ONE Secret PER PROVIDER (not a shared one), matching +# Nebula's "creds-absent → skip that provider, not fatal" model: a provider +# whose env vars are blank is skipped here, its Secret is never created, and +# the manager logs+skips it at registration (see registerProviders). +# - A provider Secret is (re)created only when ALL its required keys are set, +# so a partial config never produces a half-populated Secret. +set -euo pipefail + +# --- locate repo root so the script works from anywhere -------------------- +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${REPO_ROOT}" + +log() { printf '\033[36m==>\033[0m %s\n' "$*"; } +warn() { printf '\033[33mWARN:\033[0m %s\n' "$*" >&2; } +die() { printf '\033[31mERROR:\033[0m %s\n' "$*" >&2; exit 1; } + +# --- non-secret config: env/flags only, never from .env -------------------- +IMG="${IMG:-example.com/nebula:v0.0.1}" +NAMESPACE="${NAMESPACE:-nebula-system}" +KIND_CLUSTER="${KIND_CLUSTER:-}" +KUBECTL="${KUBECTL:-kubectl}" + +command -v "${KUBECTL}" >/dev/null 2>&1 || die "kubectl not found on PATH" + +# --- load .env (secrets only) ---------------------------------------------- +ENV_FILE="${ENV_FILE:-.env}" +if [[ -f "${ENV_FILE}" ]]; then + log "loading credentials from ${ENV_FILE}" + set -a # export everything defined while sourcing + # shellcheck disable=SC1090 + source "${ENV_FILE}" + set +a +else + warn "${ENV_FILE} not found; no provider credentials will be applied. Copy .env.example to .env." +fi + +# --- per-provider secret table --------------------------------------------- +# One entry per provider: "||". +# Keys are env var names read from .env; the Secret stores each verbatim (the +# SDKs read them by these exact names). To add a provider, append a row — +# nothing else in this script needs to change. +PROVIDER_SECRETS=( + "nebula-modal-credentials|MODAL_TOKEN_ID MODAL_TOKEN_SECRET|MODAL_ENVIRONMENT MODAL_APP_NAME" + # "nebula-runpod-credentials|RUNPOD_API_KEY|" +) + +# create_provider_secret +# Skips (leaving any existing Secret untouched) when a required key is blank. +create_provider_secret() { + local name="$1" required="$2" optional="$3" + local args=() key val missing=0 + + for key in ${required}; do + val="${!key:-}" + if [[ -z "${val}" ]]; then + missing=1 + continue + fi + args+=(--from-literal="${key}=${val}") + done + + if [[ "${missing}" -eq 1 ]]; then + log "skipping Secret ${name} (required keys unset) — provider will be skipped at registration" + return 0 + fi + + for key in ${optional}; do + val="${!key:-}" + [[ -n "${val}" ]] && args+=(--from-literal="${key}=${val}") + done + + log "applying Secret ${name} in ${NAMESPACE}" + # create --dry-run | apply makes this idempotent (create-or-update in place). + "${KUBECTL}" create secret generic "${name}" \ + --namespace "${NAMESPACE}" \ + "${args[@]}" \ + --dry-run=client -o yaml | "${KUBECTL}" apply -f - +} + +# --- 1. build the image ---------------------------------------------------- +log "building image ${IMG}" +make docker-build IMG="${IMG}" + +# --- 2. make the image reachable by the cluster ---------------------------- +if [[ -n "${KIND_CLUSTER}" ]]; then + command -v kind >/dev/null 2>&1 || die "KIND_CLUSTER=${KIND_CLUSTER} set but 'kind' not found on PATH" + log "loading ${IMG} into Kind cluster ${KIND_CLUSTER}" + kind load docker-image "${IMG}" --name "${KIND_CLUSTER}" +else + log "pushing ${IMG} (set KIND_CLUSTER to load into Kind instead)" + make docker-push IMG="${IMG}" +fi + +# --- 3. Secrets FIRST, so the manager mounts them on its very first boot ---- +# Ordering matters and lets us avoid any manager restart: +# - the webhook cert Secret is a REQUIRED volume mount, so it must exist +# before the pod starts; +# - provider credentials are read from the environment at process start. +# Both are consumed only at pod startup, so creating them before `make deploy` +# means the manager comes up already correct — no restart, no race. +# +# Secrets need the namespace, which `make deploy` would create — so create it +# up front (idempotent; kustomize re-applies it harmlessly during deploy). +log "ensuring namespace ${NAMESPACE}" +"${KUBECTL}" create namespace "${NAMESPACE}" --dry-run=client -o yaml | "${KUBECTL}" apply -f - + +# Webhook serving cert (no cert-manager): generate the self-signed cert into the +# Secret now. The caBundle is injected later, after the webhook config exists. +log "provisioning webhook serving certificate Secret (self-signed)" +NAMESPACE="${NAMESPACE}" KUBECTL="${KUBECTL}" hack/gen-webhook-cert.sh secret + +# Provider credential Secrets, one per provider (blank required keys → skipped). +for row in "${PROVIDER_SECRETS[@]}"; do + IFS='|' read -r name required optional <<<"${row}" + create_provider_secret "${name}" "${required}" "${optional}" +done + +# --- 4. install CRDs + deploy the manager ---------------------------------- +# The pod mounts the cert Secret and reads provider creds at boot — both already +# exist, so the manager comes up fully configured with no restart needed. +log "installing CRDs and deploying the manager" +make deploy IMG="${IMG}" + +# --- 5. inject the webhook CA bundle (server-side, no manager restart) ------ +# This edits only the MutatingWebhookConfiguration, which just got created by +# `make deploy`. Only the API server reads caBundle, so the manager is untouched. +log "injecting webhook CA bundle" +NAMESPACE="${NAMESPACE}" KUBECTL="${KUBECTL}" hack/gen-webhook-cert.sh cabundle + +log "done. Check status with:" +printf ' %s -n %s get pods\n' "${KUBECTL}" "${NAMESPACE}" +printf ' %s -n %s logs deploy/nebula-controller-manager\n' "${KUBECTL}" "${NAMESPACE}" diff --git a/hack/gen-webhook-cert.sh b/hack/gen-webhook-cert.sh new file mode 100755 index 0000000..f5902a4 --- /dev/null +++ b/hack/gen-webhook-cert.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# gen-webhook-cert.sh — provision the webhook serving cert WITHOUT cert-manager. +# +# The webhook needs two things cert-manager would otherwise automate: +# 1. a TLS Secret (webhook-server-cert) the manager mounts and serves from; +# 2. that cert's CA injected into the MutatingWebhookConfiguration caBundle, +# so the API server trusts the webhook when it calls it. +# +# These two steps have DIFFERENT ordering requirements, so this script exposes +# them as separate actions (see usage) and a running manager never needs a +# restart: +# - the Secret must exist BEFORE the manager pod starts (it is a required +# volume mount), so create it before `make deploy`; +# - the caBundle patch is server-side (only the API server reads it), so it +# runs AFTER `make deploy` creates the webhook config. The manager is +# untouched by it. +# +# The caBundle is always derived from the cert already in the Secret, so the +# served cert and the trusted CA can never drift, even across re-runs. +# +# Usage: +# hack/gen-webhook-cert.sh secret # ensure the TLS Secret exists (generate if absent) +# hack/gen-webhook-cert.sh cabundle # inject the Secret's cert into the webhook config +# hack/gen-webhook-cert.sh all # both, in order (default; standalone use) +# +# Config (env / make flags): +# NAMESPACE namespace the manager runs in (default nebula-system) +# SERVICE webhook Service name (default nebula-webhook-service) +# SECRET TLS Secret the manager mounts (default webhook-server-cert) +# WEBHOOK_CONFIG MutatingWebhookConfiguration name (default nebula-mutating-webhook-configuration) +# CERT_DAYS certificate validity in days (default 3650) +# FORCE_REGEN if "true", regenerate even if the Secret exists (default false) +# KUBECTL kubectl binary (default kubectl) +# +# NOTE: the self-signed cert does NOT auto-rotate (that is cert-manager's main +# advantage). It is valid CERT_DAYS days; re-run with FORCE_REGEN=true to renew. +set -euo pipefail + +NAMESPACE="${NAMESPACE:-nebula-system}" +SERVICE="${SERVICE:-nebula-webhook-service}" +SECRET="${SECRET:-webhook-server-cert}" +WEBHOOK_CONFIG="${WEBHOOK_CONFIG:-nebula-mutating-webhook-configuration}" +CERT_DAYS="${CERT_DAYS:-3650}" +FORCE_REGEN="${FORCE_REGEN:-false}" +KUBECTL="${KUBECTL:-kubectl}" +ACTION="${1:-all}" + +log() { printf '\033[36m==>\033[0m %s\n' "$*"; } +die() { printf '\033[31mERROR:\033[0m %s\n' "$*" >&2; exit 1; } + +command -v "${KUBECTL}" >/dev/null 2>&1 || die "kubectl not found on PATH" + +CN="${SERVICE}.${NAMESPACE}.svc" + +# ensure_secret — create the TLS Secret if absent (or if FORCE_REGEN=true). +# Leaves an existing Secret untouched so re-runs don't needlessly rotate the +# cert (which would otherwise require remounting on the manager). +ensure_secret() { + if [[ "${FORCE_REGEN}" != "true" ]] && \ + "${KUBECTL}" get secret "${SECRET}" -n "${NAMESPACE}" >/dev/null 2>&1; then + log "Secret ${SECRET} already exists in ${NAMESPACE}; keeping it (FORCE_REGEN=true to rotate)" + return 0 + fi + + command -v openssl >/dev/null 2>&1 || die "openssl not found on PATH" + + local tmp + tmp="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '${tmp}'" RETURN + + log "generating self-signed cert for ${CN} (valid ${CERT_DAYS}d)" + # The cert is its own CA: it both serves TLS and is trusted via caBundle. + # SANs cover both DNS forms the webhook may be addressed by. + openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "${tmp}/tls.key" -out "${tmp}/tls.crt" \ + -days "${CERT_DAYS}" -subj "/CN=${CN}" \ + -addext "subjectAltName=DNS:${CN},DNS:${CN}.cluster.local" >/dev/null 2>&1 + + log "applying TLS Secret ${SECRET} in ${NAMESPACE}" + "${KUBECTL}" create secret tls "${SECRET}" \ + --cert="${tmp}/tls.crt" --key="${tmp}/tls.key" \ + --namespace "${NAMESPACE}" \ + --dry-run=client -o yaml | "${KUBECTL}" apply -f - +} + +# inject_cabundle — read tls.crt from the Secret and set it as the webhook's +# caBundle. Deriving from the Secret guarantees the trusted CA matches the +# served cert. Requires the MutatingWebhookConfiguration to already exist. +inject_cabundle() { + "${KUBECTL}" get secret "${SECRET}" -n "${NAMESPACE}" >/dev/null 2>&1 \ + || die "Secret ${SECRET} not found in ${NAMESPACE}; run '$0 secret' first" + "${KUBECTL}" get mutatingwebhookconfiguration "${WEBHOOK_CONFIG}" >/dev/null 2>&1 \ + || die "${WEBHOOK_CONFIG} not found; deploy the manager (make deploy) before injecting the CA" + + # tls.crt in the Secret is already base64-encoded, which is exactly the form + # caBundle wants — no re-encoding needed. + local ca_b64 + ca_b64="$("${KUBECTL}" get secret "${SECRET}" -n "${NAMESPACE}" -o jsonpath='{.data.tls\.crt}')" + [[ -n "${ca_b64}" ]] || die "Secret ${SECRET} has no tls.crt" + + log "injecting CA bundle into ${WEBHOOK_CONFIG}" + # JSON Patch "add" on an existing member replaces it, so this is correct on + # both first run (caBundle absent) and re-runs (caBundle present). + "${KUBECTL}" patch mutatingwebhookconfiguration "${WEBHOOK_CONFIG}" \ + --type=json \ + -p="[{\"op\":\"add\",\"path\":\"/webhooks/0/clientConfig/caBundle\",\"value\":\"${ca_b64}\"}]" +} + +case "${ACTION}" in + secret) ensure_secret ;; + cabundle) inject_cabundle ;; + all) ensure_secret; inject_cabundle ;; + *) die "unknown action '${ACTION}' (want: secret | cabundle | all)" ;; +esac diff --git a/internal/controller/nodeclaim_controller.go b/internal/controller/nodeclaim_controller.go new file mode 100644 index 0000000..de8edf8 --- /dev/null +++ b/internal/controller/nodeclaim_controller.go @@ -0,0 +1,446 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/util" +) + +// placementGracePeriod is how long a claim whose served Pod has NEVER been +// observed waits before it is treated as genuinely orphaned and self-deletes. +// It guards against a cache-lag false-negative: the reconciler reads the Pod +// from the shared informer cache, which can briefly lag the API server right +// after the Pod is created, and a stale "absent" must not trigger teardown of a +// live instance. +// +// The window it needs to cover is small: the claim is always created AFTER its +// Pod (placement reads the Pod, then creates the claim), so the Pod already +// exists at the API server by the time this claim reconciles — only cache +// propagation, not Pod creation, is being waited on. A Pod watch re-enqueues the +// claim the moment the cache catches up, so this is just the backstop for a +// missed event; seconds are plenty. Once the Pod has been observed running +// (Phase set to Bound) a later disappearance is trusted immediately — no grace — +// because we KNOW the Pod existed and is now gone. +const placementGracePeriod = 15 * time.Second + +// NodeClaimReconciler reconciles a NodeClaim object. +// +// Ownership model: the virtual kubelet owns the instance lifecycle — its pod +// controller provisions on CreatePod and terminates on DeletePod (see +// pkg/vnode). Workload status is the POD's job; the claim does not mirror it. +// The claim exists for exactly one reason: to be a durable, cluster-scoped +// TEARDOWN BACKSTOP. +// +// Why a backstop is needed: VK's teardown is edge-triggered (it reacts to a Pod +// delete event) and its instance tracking is in-memory, so a Pod force-deleted +// while a provider's VK is down leaks a paid instance — on restart VK sees no +// Pod and no delete event, so it never calls Terminate. The claim closes that +// gap by holding a finalizer: because the claim is cluster-scoped it outlives +// the namespaced Pod, so its lifecycle is level-triggered (reconciled on every +// restart). When the served Pod is gone the claim self-deletes, and the +// finalizer reclaims the instance — resolving the provider, finding the instance +// by its Pod-derived claim name via List, and Terminating it — independent of VK +// liveness. +// +// Self-delete is guarded (see placementGracePeriod) so cache lag never tears +// down a live workload. +type NodeClaimReconciler struct { + client.Client + Scheme *runtime.Scheme + + // Providers resolves a provider name (NodeClaim.spec.provider) to its + // backend, used ONLY on the teardown path. Defaults to the process-wide + // registry; overridable in tests. + Providers func(name string) (provider.Provider, bool) +} + +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=nodeclaims,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=nodeclaims/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=nodeclaims/finalizers,verbs=update +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch + +// Reconcile drives the claim's own lifecycle: hold the terminate finalizer, +// track whether the served Pod exists, self-delete when it is gone (guarded +// against cache lag), and run the teardown backstop on the deletion path. It +// does NOT mirror the Pod's workload status — the Pod is the source of truth for +// that. +func (r *NodeClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + var nc nebulav1alpha1.NodeClaim + if err := r.Get(ctx, req.NamespacedName, &nc); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Deletion path: run the teardown backstop before the finalizer is released. + if !nc.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, &nc) + } + + // Ensure the terminate finalizer is present before the claim is treated as + // placed, so a delete that races provisioning still triggers teardown. + if !controllerutil.ContainsFinalizer(&nc, nebulav1alpha1.TerminateInstanceFinalizer) { + controllerutil.AddFinalizer(&nc, nebulav1alpha1.TerminateInstanceFinalizer) + if err := r.Update(ctx, &nc); err != nil { + return ctrl.Result{}, err + } + // Requeue via the fresh watch event from our own update. + return ctrl.Result{}, nil + } + + pod, err := r.servedPod(ctx, &nc) + if err != nil { + return ctrl.Result{}, err + } + + if pod != nil { + // The workload is present. We do not mirror the Pod's fine-grained runtime + // status, but we DO reflect the one coarse transition that matters to the + // ledger: the external instance going away. VK reports a vanished instance + // as a terminal Pod phase (Failed/Succeeded, see pkg/vnode/status.go). A Pod + // with restartPolicy: Never lingers in that terminal phase rather than being + // deleted, so keying only off Pod existence would wedge the claim at Bound + // forever — reflect Terminated instead. + if isTerminal(pod.Status.Phase) { + return ctrl.Result{}, r.markTerminated(ctx, &nc) + } + // The Pod exists but is not yet running (VK is still provisioning the + // instance). Do NOT mark Bound yet: Bound is the guard that makes a later + // disappearance trustworthy, and we only earn that trust once the instance + // is actually up. Record the instance id early (it exists the moment + // Provision returned) and reflect Provisioning; the Pod watch re-enqueues us + // when it transitions to Running. + if pod.Status.Phase != corev1.PodRunning { + return ctrl.Result{}, r.markProvisioning(ctx, &nc) + } + // Running: the instance is up. Record that we have observed it placed (this + // is the guard that makes a later disappearance trustworthy) and stop. + return ctrl.Result{}, r.markBound(ctx, &nc) + } + + // The served Pod is absent. Decide whether this is a real teardown or a + // transient cache-lag false-negative. + if r.wasBound(&nc) { + // We previously observed the Pod, so its disappearance is real: this is a + // teardown (normal delete, or a force-delete during a VK outage). Delete + // the claim so its finalizer fires the backstop. + log.Info("served Pod is gone after being observed placed; deleting claim to trigger teardown", + "pod", nc.Spec.PodRef.Name) + return ctrl.Result{}, r.deleteSelf(ctx, &nc) + } + + // Never observed the Pod running, and the cached read says it is absent. Since + // the claim is always created after its Pod, the Pod exists at the API server; + // an "absent" this early is almost certainly the informer cache lagging, not a + // real teardown. Wait out the short grace window (a Pod watch re-enqueues us as + // soon as the cache catches up) rather than tearing down a live instance. + if age := time.Since(nc.CreationTimestamp.Time); age < placementGracePeriod { + return ctrl.Result{RequeueAfter: placementGracePeriod - age}, nil + } + + // Grace elapsed and the Pod never appeared: nothing was ever provisioned for + // this claim. Delete it; the backstop List finds no instance and the finalizer + // releases cleanly. + log.Info("served Pod never appeared within grace period; deleting orphaned claim", + "pod", nc.Spec.PodRef.Name) + return ctrl.Result{}, r.deleteSelf(ctx, &nc) +} + +// reconcileDelete is the teardown backstop. It guarantees the external instance +// is reclaimed before the finalizer is released, independent of whether VK ever +// processed the Pod deletion. +// +// In the HAPPY PATH this is redundant work: VK's DeletePod already called +// provider.Terminate on the instance (see pkg/vnode Handler.DeletePod), so by +// the time we get here the instance is usually already gone and the Terminate +// below is a no-op — which is exactly why the provider contract requires +// Terminate to be idempotent. The finalizer earns its keep only when DeletePod +// never ran: the Pod was force-deleted while this provider's VK was down (VK +// misses the delete event), or VK crashed mid-DeletePod. In those cases nothing +// else will ever terminate the instance, and this backstop is the only path that +// reclaims it — which is why teardown lives here and not solely in VK. +func (r *NodeClaimReconciler) reconcileDelete(ctx context.Context, nc *nebulav1alpha1.NodeClaim) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + if !controllerutil.ContainsFinalizer(nc, nebulav1alpha1.TerminateInstanceFinalizer) { + return ctrl.Result{}, nil // already cleaned up + } + + prov, ok := r.provider(nc.Spec.Provider) + if !ok { + // No adapter registered: we cannot call Terminate. Do NOT wedge deletion + // forever on a provider we can't reach — drop the finalizer. A manager + // restart with the adapter registered is the operator's recovery path if + // an instance is later found orphaned. + log.Info("provider not registered while deleting; releasing finalizer to avoid a stuck claim", + "provider", nc.Spec.Provider) + return r.releaseFinalizer(ctx, nc) + } + + // Find the instance by its Pod-derived claim name. We cannot rely on + // status.InstanceID: VK tracks the id in memory, so if VK died (the very case + // this backstop exists for) that id was never persisted anywhere we can read. + // Re-deriving the claim name from PodRef and matching it against List is the + // only way to reclaim an instance VK has forgotten about. If DeletePod already + // ran, List simply won't contain it and findInstanceID returns "" (a no-op + // Terminate). + claim := util.ClaimName(nc.Spec.PodRef.Namespace, nc.Spec.PodRef.Name) + id, err := r.findInstanceID(ctx, prov, claim, nc.Status.InstanceID) + if err != nil { + log.Error(err, "listing provider instances for teardown; will retry") + return ctrl.Result{}, err + } + + // Terminate is idempotent (terminating an already-gone or empty instance + // returns nil per the provider contract). In the happy path VK's DeletePod + // already terminated this instance, so this is a redundant no-op; the call is + // only load-bearing when DeletePod never ran. Idempotency also makes retries + // after a transient error safe. + if err := prov.Terminate(ctx, id); err != nil { + log.Error(err, "terminate failed; will retry", "instanceID", id) + return ctrl.Result{}, err + } + if id != "" { + log.Info("instance terminated by backstop", "instanceID", id, "provider", prov.Name()) + } + + return r.releaseFinalizer(ctx, nc) +} + +// findInstanceID resolves the provider instance id to terminate. It prefers a +// recorded status.InstanceID, then falls back to matching the Pod-derived claim +// name against provider.List(). Returns "" when no instance exists (already gone +// or never provisioned), which Terminate treats as a no-op. +func (r *NodeClaimReconciler) findInstanceID(ctx context.Context, prov provider.Provider, claim, recordedID string) (string, error) { + if recordedID != "" { + return recordedID, nil + } + instances, err := prov.List(ctx) + if err != nil { + return "", err + } + for _, inst := range instances { + if inst.ClaimName == claim { + return inst.ID, nil + } + } + return "", nil // no live instance for this claim +} + +// releaseFinalizer removes the terminate finalizer, allowing the API server to +// delete the NodeClaim. +func (r *NodeClaimReconciler) releaseFinalizer(ctx context.Context, nc *nebulav1alpha1.NodeClaim) (ctrl.Result, error) { + if controllerutil.RemoveFinalizer(nc, nebulav1alpha1.TerminateInstanceFinalizer) { + if err := r.Update(ctx, nc); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil +} + +// provider resolves the backend for name via the injected resolver, defaulting +// to the process-wide registry. +func (r *NodeClaimReconciler) provider(name string) (provider.Provider, bool) { + if r.Providers != nil { + return r.Providers(name) + } + return provider.Get(name) +} + +// markBound records that the served Pod has been observed running. This is the +// persisted guard the self-delete path trusts: a claim in phase Bound whose Pod +// later disappears is a real teardown, whereas a claim that never reached Bound +// is treated as possible cache lag. It does NOT copy the Pod's fine-grained +// runtime status — the Pod is the source of truth for that — but it does record +// status.InstanceID: the provider id is the one datum that must not be lost, so +// the teardown backstop can reclaim the instance even if VK (which otherwise +// holds the id only in memory) has died. The id is resolved once, best-effort; +// if the provider List is unavailable the Bound transition still proceeds and +// the backstop falls back to re-deriving the id by claim name. Idempotent. + +// markProvisioning records that the served Pod exists but is not yet running. +// The claim sits in phase Provisioning (it never earns the Bound teardown guard +// while the instance is still coming up), but we capture status.InstanceID as +// soon as it is resolvable so the backstop can reclaim a half-provisioned +// instance. Idempotent. +func (r *NodeClaimReconciler) markProvisioning(ctx context.Context, nc *nebulav1alpha1.NodeClaim) error { + changed := false + if nc.Status.Phase != nebulav1alpha1.NodeClaimProvisioning { + nc.Status.Phase = nebulav1alpha1.NodeClaimProvisioning + changed = true + } + if r.recordInstanceID(ctx, nc) { + changed = true + } + if !changed { + return nil + } + return r.patchStatus(ctx, nc) +} + +func (r *NodeClaimReconciler) markBound(ctx context.Context, nc *nebulav1alpha1.NodeClaim) error { + changed := false + if nc.Status.Phase != nebulav1alpha1.NodeClaimBound { + nc.Status.Phase = nebulav1alpha1.NodeClaimBound + changed = true + } + if r.recordInstanceID(ctx, nc) { + changed = true + } + if !changed { + return nil + } + return r.patchStatus(ctx, nc) +} + +// markTerminated records that the external instance is gone (the served Pod +// reached a terminal phase). Terminal, so once set it never advances. Like +// markBound it best-effort records status.InstanceID for the backstop; the id is +// still worth capturing here because the instance may already be gone from the +// provider but the finalizer path is independent. Idempotent. +func (r *NodeClaimReconciler) markTerminated(ctx context.Context, nc *nebulav1alpha1.NodeClaim) error { + changed := false + if nc.Status.Phase != nebulav1alpha1.NodeClaimTerminated { + nc.Status.Phase = nebulav1alpha1.NodeClaimTerminated + changed = true + } + if r.recordInstanceID(ctx, nc) { + changed = true + } + if !changed { + return nil + } + return r.patchStatus(ctx, nc) +} + +// recordInstanceID resolves and stores status.InstanceID when it is not already +// set, returning whether it mutated the claim. Best-effort: an unregistered +// provider or a List error leaves the id empty (the backstop re-derives it by +// claim name), so this never fails the caller. +func (r *NodeClaimReconciler) recordInstanceID(ctx context.Context, nc *nebulav1alpha1.NodeClaim) bool { + if nc.Status.InstanceID != "" { + return false + } + prov, ok := r.provider(nc.Spec.Provider) + if !ok { + return false + } + claim := util.ClaimName(nc.Spec.PodRef.Namespace, nc.Spec.PodRef.Name) + id, err := r.findInstanceID(ctx, prov, claim, "") + if err != nil || id == "" { + return false + } + nc.Status.InstanceID = id + return true +} + +// wasBound reports whether the served Pod has ever been observed running for this +// claim. A Terminated claim was necessarily Bound first, so it also counts. +func (r *NodeClaimReconciler) wasBound(nc *nebulav1alpha1.NodeClaim) bool { + return nc.Status.Phase == nebulav1alpha1.NodeClaimBound || + nc.Status.Phase == nebulav1alpha1.NodeClaimTerminated +} + +// isTerminal reports whether a Pod phase is an end state that will not progress. +func isTerminal(phase corev1.PodPhase) bool { + return phase == corev1.PodFailed || phase == corev1.PodSucceeded +} + +// deleteSelf deletes the claim, which triggers reconcileDelete (the backstop) +// via the deletion timestamp + finalizer. Idempotent: a NotFound is treated as +// success. +func (r *NodeClaimReconciler) deleteSelf(ctx context.Context, nc *nebulav1alpha1.NodeClaim) error { + if err := r.Delete(ctx, nc); err != nil { + return client.IgnoreNotFound(err) + } + return nil +} + +// servedPod fetches the Pod named by spec.PodRef, enforcing the UID pin: a +// recreated Pod of the same name (different UID) is treated as gone, so this +// claim is not silently re-attached to a different workload. Returns (nil, nil) +// when the Pod is absent or the UID no longer matches. +func (r *NodeClaimReconciler) servedPod(ctx context.Context, nc *nebulav1alpha1.NodeClaim) (*corev1.Pod, error) { + var pod corev1.Pod + key := types.NamespacedName{Namespace: nc.Spec.PodRef.Namespace, Name: nc.Spec.PodRef.Name} + if err := r.Get(ctx, key, &pod); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + if nc.Spec.PodRef.UID != "" && string(pod.UID) != nc.Spec.PodRef.UID { + return nil, nil // same name, different object => the pinned Pod is gone + } + return &pod, nil +} + +// patchStatus writes the status subresource. +func (r *NodeClaimReconciler) patchStatus(ctx context.Context, nc *nebulav1alpha1.NodeClaim) error { + return r.Status().Update(ctx, nc) +} + +// SetupWithManager wires the controller. It watches NodeClaims and also watches +// Pods (mapping a Pod back to the claim that references it) so that a Pod's +// appearance promptly marks the claim Bound and its deletion promptly triggers +// the self-delete/teardown path — rather than waiting for the grace-period +// requeue. +func (r *NodeClaimReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&nebulav1alpha1.NodeClaim{}). + Watches(&corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(r.claimsForPod)). + Named("nodeclaim"). + Complete(r) +} + +// claimsForPod maps a Pod event to the NodeClaims that serve it (by PodRef). +func (r *NodeClaimReconciler) claimsForPod(ctx context.Context, obj client.Object) []reconcile.Request { + pod, ok := obj.(*corev1.Pod) + if !ok { + return nil + } + var claims nebulav1alpha1.NodeClaimList + if err := r.List(ctx, &claims); err != nil { + return nil + } + var reqs []reconcile.Request + for i := range claims.Items { + ref := claims.Items[i].Spec.PodRef + if ref.Namespace == pod.Namespace && ref.Name == pod.Name { + reqs = append(reqs, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: claims.Items[i].Name}, + }) + } + } + return reqs +} diff --git a/internal/controller/nodeclaim_controller_test.go b/internal/controller/nodeclaim_controller_test.go new file mode 100644 index 0000000..8a1b9c1 --- /dev/null +++ b/internal/controller/nodeclaim_controller_test.go @@ -0,0 +1,473 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" +) + +// fakeProvider is a minimal provider.Provider. On the happy path the NodeClaim +// controller never touches a provider (VK owns provisioning), but the teardown +// backstop calls List/Terminate on the deletion path, so this records those. +type fakeProvider struct { + name string + + list []provider.Instance // what List returns + listErr error // if set, List fails + terminated []string // instance ids passed to Terminate, in order + terminateErr error // if set, Terminate fails + gpus []string // accelerators MapAccelerator offers; nil = offer any +} + +func (f *fakeProvider) Name() string { return f.name } +func (f *fakeProvider) Capabilities() provider.Capabilities { return provider.Capabilities{} } +func (f *fakeProvider) Provision(context.Context, *corev1.Pod, provider.ProvisionRequest) (string, error) { + return "", nil +} +func (f *fakeProvider) Terminate(_ context.Context, id string) error { + f.terminated = append(f.terminated, id) + return f.terminateErr +} +func (f *fakeProvider) Get(context.Context, string) (*provider.Instance, error) { return nil, nil } +func (f *fakeProvider) List(context.Context) ([]provider.Instance, error) { + return f.list, f.listErr +} +func (f *fakeProvider) Offerings(context.Context) ([]provider.Offering, error) { return nil, nil } +func (f *fakeProvider) MapAccelerator(c string) (string, bool) { + if f.gpus == nil { + return c, true // offer any accelerator + } + for _, g := range f.gpus { + if g == c { + return c, true + } + } + return "", false +} +func (f *fakeProvider) ClassifyProvisionError(error) provider.BlockScope { + return provider.BlockScope{} +} + +// resolver returns a Providers func that resolves only the given provider. +func resolver(provs ...*fakeProvider) func(string) (provider.Provider, bool) { + return func(name string) (provider.Provider, bool) { + for _, p := range provs { + if p.name == name { + return p, true + } + } + return nil, false + } +} + +// testScheme builds a scheme with core + Nebula types registered. +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(s); err != nil { + t.Fatalf("add client-go scheme: %v", err) + } + if err := nebulav1alpha1.AddToScheme(s); err != nil { + t.Fatalf("add nebula scheme: %v", err) + } + return s +} + +// newClaimReconciler wires a NodeClaimReconciler over a fake client seeded with +// objs. Any fakeProviders passed are registered as the reconciler's resolver so +// the teardown backstop can reach them. +func newClaimReconciler(t *testing.T, objs []client.Object, provs ...*fakeProvider) (*NodeClaimReconciler, client.Client) { + t.Helper() + s := testScheme(t) + c := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(objs...). + WithStatusSubresource(&nebulav1alpha1.NodeClaim{}). + Build() + r := &NodeClaimReconciler{Client: c, Scheme: s} + if len(provs) > 0 { + r.Providers = resolver(provs...) + } + return r, c +} + +func newPod(name, ns, uid string, phase corev1.PodPhase) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, UID: types.UID(uid)}, + Spec: corev1.PodSpec{ + NodeName: "nebula-fake", + Containers: []corev1.Container{{Name: "main", Image: "img"}}, + }, + Status: corev1.PodStatus{Phase: phase, PodIP: "1.2.3.4"}, + } +} + +// newClaim builds a steady-state claim: it already carries the terminate +// finalizer, since the normal reconcile adds that before doing anything else. +// Use newClaimNoFinalizer to exercise the finalizer-addition path itself. +func newClaim(name, podName, podNS, podUID, prov string) *nebulav1alpha1.NodeClaim { + nc := newClaimNoFinalizer(name, podName, podNS, podUID, prov) + nc.Finalizers = []string{nebulav1alpha1.TerminateInstanceFinalizer} + return nc +} + +func newClaimNoFinalizer(name, podName, podNS, podUID, prov string) *nebulav1alpha1.NodeClaim { + return &nebulav1alpha1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: nebulav1alpha1.NodeClaimSpec{ + PodRef: nebulav1alpha1.PodReference{Namespace: podNS, Name: podName, UID: podUID}, + Provider: prov, + }, + } +} + +func reconcileClaim(t *testing.T, r *NodeClaimReconciler, name string) reconcile.Result { + t.Helper() + res, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Name: name}, + }) + if err != nil { + t.Fatalf("reconcile: %v", err) + } + return res +} + +func getClaim(t *testing.T, c client.Client, name string) *nebulav1alpha1.NodeClaim { + t.Helper() + var nc nebulav1alpha1.NodeClaim + if err := c.Get(context.Background(), types.NamespacedName{Name: name}, &nc); err != nil { + t.Fatalf("get claim %s: %v", name, err) + } + return &nc +} + +func TestReconcile_AddsFinalizerBeforePlacing(t *testing.T) { + // A claim with no finalizer: the first reconcile must add the terminate + // finalizer (and do nothing else) so teardown is guaranteed from the start. + pod := newPod("p1", "default", "uid-1", corev1.PodRunning) + claim := newClaimNoFinalizer("c1", "p1", "default", "uid-1", "fake") + r, c := newClaimReconciler(t, []client.Object{pod, claim}) + + reconcileClaim(t, r, "c1") + + got := getClaim(t, c, "c1") + if !hasFinalizer(got) { + t.Fatal("expected terminate finalizer to be added on first reconcile") + } + // Phase must NOT yet be Bound: the finalizer-add reconcile returns early. + if got.Status.Phase == nebulav1alpha1.NodeClaimBound { + t.Fatal("must not mark Bound in the same reconcile that adds the finalizer") + } +} + +func TestReconcile_MarksBoundWhenPodPresent(t *testing.T) { + // With the finalizer already present and the served Pod alive, the claim is + // marked Bound (the durable guard) and nothing is torn down. + pod := newPod("p1", "default", "uid-1", corev1.PodRunning) + claim := newClaim("c1", "p1", "default", "uid-1", "fake") + r, c := newClaimReconciler(t, []client.Object{pod, claim}) + + reconcileClaim(t, r, "c1") + + got := getClaim(t, c, "c1") + if got.Status.Phase != nebulav1alpha1.NodeClaimBound { + t.Fatalf("expected phase Bound, got %q", got.Status.Phase) + } + if !got.DeletionTimestamp.IsZero() { + t.Fatal("must not delete a claim whose Pod is present") + } +} + +func TestReconcile_PendingPodDoesNotMarkBound(t *testing.T) { + // A served Pod that exists but is still provisioning (PodPending) must NOT + // flip the claim to Bound — Bound is the teardown guard, earned only once the + // instance is actually running. The instance id is still captured early. + pod := newPod("p1", "default", "uid-1", corev1.PodPending) + claim := newClaim("c1", "p1", "default", "uid-1", "fake") + prov := &fakeProvider{ + name: "fake", + list: []provider.Instance{{ID: "inst-1", ClaimName: "default-p1"}}, + } + r, c := newClaimReconciler(t, []client.Object{pod, claim}, prov) + + reconcileClaim(t, r, "c1") + + got := getClaim(t, c, "c1") + if got.Status.Phase == nebulav1alpha1.NodeClaimBound { + t.Fatal("must not mark Bound while the Pod is still provisioning") + } + if got.Status.Phase != nebulav1alpha1.NodeClaimProvisioning { + t.Fatalf("expected phase Provisioning, got %q", got.Status.Phase) + } + if got.Status.InstanceID != "inst-1" { + t.Fatalf("expected instance id captured early, got %q", got.Status.InstanceID) + } +} + +func TestReconcile_RecordsInstanceIDOnBound(t *testing.T) { + // When the served Pod is running, the claim is marked Bound AND its + // status.InstanceID is captured (resolved by claim name via List) so the + // teardown backstop can reclaim the instance even if VK forgets the id. + pod := newPod("p1", "default", "uid-1", corev1.PodRunning) + claim := newClaim("c1", "p1", "default", "uid-1", "fake") + prov := &fakeProvider{ + name: "fake", + list: []provider.Instance{{ID: "inst-1", ClaimName: "default-p1"}}, + } + r, c := newClaimReconciler(t, []client.Object{pod, claim}, prov) + + reconcileClaim(t, r, "c1") + + got := getClaim(t, c, "c1") + if got.Status.Phase != nebulav1alpha1.NodeClaimBound { + t.Fatalf("expected phase Bound, got %q", got.Status.Phase) + } + if got.Status.InstanceID != "inst-1" { + t.Fatalf("expected status.InstanceID recorded as inst-1, got %q", got.Status.InstanceID) + } +} + +func TestReconcile_TerminalPodMarksTerminated(t *testing.T) { + // A served Pod that has reached a terminal phase (VK reports the external + // instance gone) must move the claim to Terminated rather than wedge it at + // Bound. The claim is NOT deleted — its Pod still exists (restartPolicy: + // Never leaves it around), and the instance is already gone. + pod := newPod("p1", "default", "uid-1", corev1.PodFailed) + claim := newClaim("c1", "p1", "default", "uid-1", "fake") + claim.Status.Phase = nebulav1alpha1.NodeClaimBound + prov := &fakeProvider{name: "fake"} // instance already gone from List + r, c := newClaimReconciler(t, []client.Object{pod, claim}, prov) + + reconcileClaim(t, r, "c1") + + got := getClaim(t, c, "c1") + if got.Status.Phase != nebulav1alpha1.NodeClaimTerminated { + t.Fatalf("expected phase Terminated, got %q", got.Status.Phase) + } + if !got.DeletionTimestamp.IsZero() { + t.Fatal("must not delete a Terminated claim whose Pod still exists") + } +} + +func TestReconcile_BoundClaimWithGonePodSelfDeletes(t *testing.T) { + // A claim we previously observed Bound, whose Pod has since vanished, is a + // real teardown: the claim self-deletes (a finalizer keeps it around for the + // backstop rather than dropping it immediately). + claim := newClaim("c1", "gone", "default", "uid-1", "fake") + claim.Status.Phase = nebulav1alpha1.NodeClaimBound + prov := &fakeProvider{name: "fake"} + r, c := newClaimReconciler(t, []client.Object{claim}, prov) + + reconcileClaim(t, r, "c1") + + got := getClaim(t, c, "c1") + if got.DeletionTimestamp.IsZero() { + t.Fatal("expected a Bound claim with a gone Pod to be deleted") + } +} + +func TestReconcile_NeverObservedPodWaitsForGrace(t *testing.T) { + // A claim that never reached Bound and whose Pod is absent is treated as + // possible cache lag within the grace window: requeue, do NOT delete. + claim := newClaim("c1", "pending", "default", "uid-1", "fake") + claim.CreationTimestamp = metav1.NewTime(time.Now()) + r, c := newClaimReconciler(t, []client.Object{claim}) + + res := reconcileClaim(t, r, "c1") + + if res.RequeueAfter <= 0 { + t.Fatalf("expected a requeue within the grace period, got %+v", res) + } + got := getClaim(t, c, "c1") + if !got.DeletionTimestamp.IsZero() { + t.Fatal("must not delete a never-observed claim inside the grace window (cache-lag guard)") + } +} + +func TestReconcile_NeverObservedPodDeletedAfterGrace(t *testing.T) { + // Same claim, but the grace window has elapsed and the Pod never appeared: + // nothing was ever provisioned, so the orphaned claim is deleted. + claim := newClaim("c1", "pending", "default", "uid-1", "fake") + claim.CreationTimestamp = metav1.NewTime(time.Now().Add(-2 * placementGracePeriod)) + prov := &fakeProvider{name: "fake"} + r, c := newClaimReconciler(t, []client.Object{claim}, prov) + + reconcileClaim(t, r, "c1") + + got := getClaim(t, c, "c1") + if got.DeletionTimestamp.IsZero() { + t.Fatal("expected an orphaned claim to be deleted after the grace period") + } +} + +func TestReconcileDelete_TerminatesInstanceByClaimName(t *testing.T) { + // The backstop: on the deletion path, the instance is found by its + // Pod-derived claim name via List and terminated before the finalizer drops. + claim := newClaim("c1", "p1", "default", "uid-1", "fake") + deleteClaim(t, claim) // set deletionTimestamp; finalizer keeps it alive + prov := &fakeProvider{ + name: "fake", + list: []provider.Instance{{ID: "inst-1", ClaimName: "default-p1"}}, + } + r, c := newClaimReconciler(t, []client.Object{claim}, prov) + + reconcileClaim(t, r, "c1") + + if len(prov.terminated) != 1 || prov.terminated[0] != "inst-1" { + t.Fatalf("expected Terminate(inst-1), got %v", prov.terminated) + } + if claimExists(t, c, "c1") { + t.Fatal("expected finalizer released and claim gone after teardown") + } +} + +func TestReconcileDelete_UsesRecordedInstanceID(t *testing.T) { + // When status.InstanceID is set (VK wrote it), the backstop terminates it + // directly without needing a List lookup. + claim := newClaim("c1", "p1", "default", "uid-1", "fake") + claim.Status.InstanceID = "inst-recorded" + deleteClaim(t, claim) + prov := &fakeProvider{name: "fake"} // List would return nothing + r, _ := newClaimReconciler(t, []client.Object{claim}, prov) + + reconcileClaim(t, r, "c1") + + if len(prov.terminated) != 1 || prov.terminated[0] != "inst-recorded" { + t.Fatalf("expected Terminate(inst-recorded) from recorded id, got %v", prov.terminated) + } +} + +func TestReconcileDelete_NoInstanceIsIdempotentNoOp(t *testing.T) { + // The happy path: VK's DeletePod already terminated, so List finds nothing. + // Terminate is called with "" (idempotent no-op) and the finalizer releases. + claim := newClaim("c1", "p1", "default", "uid-1", "fake") + deleteClaim(t, claim) + prov := &fakeProvider{name: "fake"} // empty list + r, c := newClaimReconciler(t, []client.Object{claim}, prov) + + reconcileClaim(t, r, "c1") + + if len(prov.terminated) != 1 || prov.terminated[0] != "" { + t.Fatalf("expected a no-op Terminate(\"\"), got %v", prov.terminated) + } + if claimExists(t, c, "c1") { + t.Fatal("expected claim gone after a no-op teardown") + } +} + +func TestReconcileDelete_UnknownProviderReleasesFinalizer(t *testing.T) { + // If no adapter is registered for the claim's provider, we must not wedge + // deletion forever — the finalizer is released so the claim can be removed. + claim := newClaim("c1", "p1", "default", "uid-1", "ghost") + deleteClaim(t, claim) + prov := &fakeProvider{name: "fake"} // does not match "ghost" + r, c := newClaimReconciler(t, []client.Object{claim}, prov) + + reconcileClaim(t, r, "c1") + + if len(prov.terminated) != 0 { + t.Fatalf("must not call Terminate on an unresolved provider, got %v", prov.terminated) + } + if claimExists(t, c, "c1") { + t.Fatal("expected finalizer released so the claim is not stuck") + } +} + +func TestReconcileDelete_ListErrorRetries(t *testing.T) { + // A transient List error must NOT release the finalizer — teardown retries + // so the instance is never abandoned. + claim := newClaim("c1", "p1", "default", "uid-1", "fake") + deleteClaim(t, claim) + prov := &fakeProvider{name: "fake", listErr: errListFailed} + r, c := newClaimReconciler(t, []client.Object{claim}, prov) + + _, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Name: "c1"}, + }) + if err == nil { + t.Fatal("expected an error so reconcile retries on a List failure") + } + if !claimExists(t, c, "c1") { + t.Fatal("must not release the finalizer while teardown is unresolved") + } +} + +func TestClaimsForPod_MapsByPodRef(t *testing.T) { + pod := newPod("p1", "default", "uid-1", corev1.PodRunning) + claim := newClaim("c1", "p1", "default", "uid-1", "fake") + other := newClaim("c2", "other", "default", "uid-2", "fake") + r, _ := newClaimReconciler(t, []client.Object{pod, claim, other}) + + reqs := r.claimsForPod(context.Background(), pod) + if len(reqs) != 1 || reqs[0].Name != "c1" { + t.Fatalf("expected only c1 enqueued for pod p1, got %+v", reqs) + } +} + +var errListFailed = errTest("list failed") + +type errTest string + +func (e errTest) Error() string { return string(e) } + +func hasFinalizer(nc *nebulav1alpha1.NodeClaim) bool { + for _, f := range nc.Finalizers { + if f == nebulav1alpha1.TerminateInstanceFinalizer { + return true + } + } + return false +} + +// deleteClaim stamps a deletionTimestamp on an in-memory claim so it can be +// seeded into the fake client already on the deletion path. (The fake client +// only accepts a deletionTimestamp on objects that carry a finalizer.) +func deleteClaim(t *testing.T, nc *nebulav1alpha1.NodeClaim) { + t.Helper() + now := metav1.Now() + nc.DeletionTimestamp = &now +} + +// claimExists reports whether the named claim is still present. +func claimExists(t *testing.T, c client.Client, name string) bool { + t.Helper() + var nc nebulav1alpha1.NodeClaim + err := c.Get(context.Background(), types.NamespacedName{Name: name}, &nc) + if err == nil { + return true + } + if apierrors.IsNotFound(err) { + return false + } + t.Fatalf("get claim %s: %v", name, err) + return false +} diff --git a/internal/controller/nodepool_controller.go b/internal/controller/nodepool_controller.go new file mode 100644 index 0000000..4cf788d --- /dev/null +++ b/internal/controller/nodepool_controller.go @@ -0,0 +1,185 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "sort" + "strings" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" +) + +// NodePoolReconciler reconciles a NodePool object. The NodePool is the policy +// object (which providers, which capacity tiers, how to rank); actual placement +// — choosing a provider for a gated Pod — is a separate concern. This controller +// is therefore the pool's health and observability loop: it validates the policy +// against the registered providers and reports the live placement picture +// (running NodeClaims per provider) in status, so a misconfigured pool surfaces +// as a clear condition rather than as silent placement failures. +type NodePoolReconciler struct { + client.Client + Scheme *runtime.Scheme + + // Providers resolves a provider name to its backend; defaults to the + // process-wide registry. Overridable in tests. + Providers func(name string) (provider.Provider, bool) +} + +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=nodepools,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=nodepools/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=nodepools/finalizers,verbs=update +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=nodeclaims,verbs=get;list;watch + +// Reconcile validates the pool's policy and refreshes its status. +func (r *NodePoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + var pool nebulav1alpha1.NodePool + if err := r.Get(ctx, req.NamespacedName, &pool); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Validate the policy against the registered providers. + if reason, msg, ok := r.validate(&pool); !ok { + log.Info("NodePool invalid", "reason", reason, "message", msg) + apimeta.SetStatusCondition(&pool.Status.Conditions, metav1.Condition{ + Type: nebulav1alpha1.NodePoolConditionReady, + Status: metav1.ConditionFalse, + Reason: reason, + Message: msg, + ObservedGeneration: pool.Generation, + }) + // Still refresh Placed so status reflects reality even while invalid. + if err := r.refreshPlaced(ctx, &pool); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, r.Status().Update(ctx, &pool) + } + + if err := r.refreshPlaced(ctx, &pool); err != nil { + return ctrl.Result{}, err + } + apimeta.SetStatusCondition(&pool.Status.Conditions, metav1.Condition{ + Type: nebulav1alpha1.NodePoolConditionReady, + Status: metav1.ConditionTrue, + Reason: nebulav1alpha1.ReasonPoolValid, + Message: "pool policy is valid", + ObservedGeneration: pool.Generation, + }) + return ctrl.Result{}, r.Status().Update(ctx, &pool) +} + +// validate checks the pool against runtime state: every referenced provider has +// a registered adapter. Static, spec-only invariants (e.g. Weighted requires a +// weight on every provider) are enforced at admission by a CEL rule on +// NodePoolSpec, so they never reach here. This check stays in the controller +// because it depends on the provider registry — a webhook's view of registered +// providers can differ from the controller's (rollouts, creds-absent skips), so +// coupling admission to it would be a false-negative footgun on a fail-closed +// path. It returns (reason, message, ok=false) on the first problem. +func (r *NodePoolReconciler) validate(pool *nebulav1alpha1.NodePool) (reason, msg string, ok bool) { + var unknown []string + for _, p := range pool.Spec.Providers { + if _, found := r.provider(p.Name); !found { + unknown = append(unknown, p.Name) + } + } + if len(unknown) > 0 { + sort.Strings(unknown) + return nebulav1alpha1.ReasonUnknownProvider, + fmt.Sprintf("no registered adapter for provider(s): %s", strings.Join(unknown, ", ")), + false + } + return "", "", true +} + +// refreshPlaced recomputes status.Placed: the count of placed NodeClaims per +// provider that belong to this pool. It is derived state — a snapshot of the +// live placement picture — so it is fully recomputed each reconcile rather than +// incremented, which keeps it correct after missed events. +// +// "Placed" is a claim whose served Pod has been observed (phase Bound). The +// claim no longer mirrors the Pod's finer runtime status (the Pod is the source +// of truth for that), so Bound is the claim-level signal that an instance is +// live for the workload. +func (r *NodePoolReconciler) refreshPlaced(ctx context.Context, pool *nebulav1alpha1.NodePool) error { + var claims nebulav1alpha1.NodeClaimList + if err := r.List(ctx, &claims); err != nil { + return err + } + placed := map[string]int32{} + for i := range claims.Items { + nc := &claims.Items[i] + if nc.Spec.PoolRef != pool.Name { + continue + } + if nc.Status.Phase == nebulav1alpha1.NodeClaimBound { + placed[nc.Spec.Provider]++ + } + } + if len(placed) == 0 { + placed = nil // keep status clean when nothing is placed + } + pool.Status.Placed = placed + return nil +} + +// provider resolves the backend for name via the injected resolver, defaulting +// to the process-wide registry. +func (r *NodePoolReconciler) provider(name string) (provider.Provider, bool) { + if r.Providers != nil { + return r.Providers(name) + } + return provider.Get(name) +} + +// SetupWithManager sets up the controller with the Manager. It watches +// NodeClaims too, mapping each back to its pool, so status.Placed tracks +// instances coming and going without waiting for the periodic resync. +func (r *NodePoolReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&nebulav1alpha1.NodePool{}). + Watches( + &nebulav1alpha1.NodeClaim{}, + handler.EnqueueRequestsFromMapFunc(claimToPool), + ). + Named("nodepool"). + Complete(r) +} + +// claimToPool maps a NodeClaim to a reconcile request for its owning pool, so a +// claim changing state re-triggers that pool's status refresh. Cluster-scoped: +// the request carries only the pool name. +func claimToPool(_ context.Context, obj client.Object) []reconcile.Request { + nc, ok := obj.(*nebulav1alpha1.NodeClaim) + if !ok || nc.Spec.PoolRef == "" { + return nil + } + return []reconcile.Request{{NamespacedName: client.ObjectKey{Name: nc.Spec.PoolRef}}} +} diff --git a/internal/controller/nodepool_controller_test.go b/internal/controller/nodepool_controller_test.go new file mode 100644 index 0000000..21684ef --- /dev/null +++ b/internal/controller/nodepool_controller_test.go @@ -0,0 +1,199 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "testing" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" +) + +// newPoolReconciler wires a NodePoolReconciler over a fake client seeded with +// objs. known is the set of provider names treated as registered. +func newPoolReconciler(t *testing.T, known []string, objs ...client.Object) (*NodePoolReconciler, client.Client) { + t.Helper() + s := testScheme(t) + c := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(objs...). + WithStatusSubresource(&nebulav1alpha1.NodePool{}). + Build() + knownSet := map[string]bool{} + for _, n := range known { + knownSet[n] = true + } + r := &NodePoolReconciler{ + Client: c, + Scheme: s, + Providers: func(name string) (provider.Provider, bool) { + if knownSet[name] { + return &fakeProvider{name: name}, true + } + return nil, false + }, + } + return r, c +} + +func newPool(name string, strategy nebulav1alpha1.PlacementStrategy, providers ...nebulav1alpha1.ProviderRef) *nebulav1alpha1.NodePool { + return &nebulav1alpha1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: nebulav1alpha1.NodePoolSpec{ + Providers: providers, + Strategy: strategy, + }, + } +} + +// runningClaim builds a NodeClaim in a given phase for pool/provider accounting. +func poolClaim(name, pool, prov string, phase nebulav1alpha1.NodeClaimPhase) *nebulav1alpha1.NodeClaim { + nc := &nebulav1alpha1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: nebulav1alpha1.NodeClaimSpec{ + PodRef: nebulav1alpha1.PodReference{Namespace: "default", Name: name + "-pod", UID: "u-" + name}, + Provider: prov, + PoolRef: pool, + }, + } + nc.Status.Phase = phase + return nc +} + +func reconcilePool(t *testing.T, r *NodePoolReconciler, name string) { + t.Helper() + if _, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Name: name}, + }); err != nil { + t.Fatalf("reconcile pool: %v", err) + } +} + +func getPool(t *testing.T, c client.Client, name string) *nebulav1alpha1.NodePool { + t.Helper() + var p nebulav1alpha1.NodePool + if err := c.Get(context.Background(), types.NamespacedName{Name: name}, &p); err != nil { + t.Fatalf("get pool %s: %v", name, err) + } + return &p +} + +func poolReadyCond(p *nebulav1alpha1.NodePool) *metav1.Condition { + return apimeta.FindStatusCondition(p.Status.Conditions, nebulav1alpha1.NodePoolConditionReady) +} + +func TestPool_ValidPoolBecomesReady(t *testing.T) { + pool := newPool("gpu", nebulav1alpha1.StrategyOrdered, + nebulav1alpha1.ProviderRef{Name: "modal"}) + r, c := newPoolReconciler(t, []string{"modal"}, pool) + + reconcilePool(t, r, "gpu") + + cond := poolReadyCond(getPool(t, c, "gpu")) + if cond == nil || cond.Status != metav1.ConditionTrue { + t.Fatalf("expected Ready=True, got %+v", cond) + } + if cond.Reason != nebulav1alpha1.ReasonPoolValid { + t.Fatalf("reason = %q, want %q", cond.Reason, nebulav1alpha1.ReasonPoolValid) + } +} + +func TestPool_UnknownProviderNotReady(t *testing.T) { + pool := newPool("gpu", nebulav1alpha1.StrategyOrdered, + nebulav1alpha1.ProviderRef{Name: "modal"}, + nebulav1alpha1.ProviderRef{Name: "ghost"}) + r, c := newPoolReconciler(t, []string{"modal"}, pool) // ghost not registered + + reconcilePool(t, r, "gpu") + + cond := poolReadyCond(getPool(t, c, "gpu")) + if cond == nil || cond.Status != metav1.ConditionFalse { + t.Fatalf("expected Ready=False, got %+v", cond) + } + if cond.Reason != nebulav1alpha1.ReasonUnknownProvider { + t.Fatalf("reason = %q, want %q", cond.Reason, nebulav1alpha1.ReasonUnknownProvider) + } +} + +func TestPool_PlacedCountsBoundClaims(t *testing.T) { + pool := newPool("gpu", nebulav1alpha1.StrategyOrdered, + nebulav1alpha1.ProviderRef{Name: "modal"}, + nebulav1alpha1.ProviderRef{Name: "runpod"}) + // 2 bound on modal, 1 on runpod for this pool; 1 bound for another pool + // (ignored); 1 pending on modal for this pool (ignored — not Bound). + r, c := newPoolReconciler(t, []string{"modal", "runpod"}, + pool, + poolClaim("a", "gpu", "modal", nebulav1alpha1.NodeClaimBound), + poolClaim("b", "gpu", "modal", nebulav1alpha1.NodeClaimBound), + poolClaim("c", "gpu", "runpod", nebulav1alpha1.NodeClaimBound), + poolClaim("x", "otherpool", "modal", nebulav1alpha1.NodeClaimBound), + poolClaim("pend", "gpu", "modal", nebulav1alpha1.NodeClaimProvisioning), + ) + + reconcilePool(t, r, "gpu") + + got := getPool(t, c, "gpu").Status.Placed + if got["modal"] != 2 { + t.Fatalf("modal placed = %d, want 2", got["modal"]) + } + if got["runpod"] != 1 { + t.Fatalf("runpod placed = %d, want 1", got["runpod"]) + } + if _, ok := got["otherpool"]; ok { + t.Fatal("must not count claims from another pool") + } +} + +func TestPool_PlacedNilWhenNothingRunning(t *testing.T) { + pool := newPool("gpu", nebulav1alpha1.StrategyOrdered, + nebulav1alpha1.ProviderRef{Name: "modal"}) + r, c := newPoolReconciler(t, []string{"modal"}, + pool, + poolClaim("pend", "gpu", "modal", nebulav1alpha1.NodeClaimProvisioning), + ) + + reconcilePool(t, r, "gpu") + + if got := getPool(t, c, "gpu").Status.Placed; got != nil { + t.Fatalf("expected nil Placed when nothing running, got %+v", got) + } +} + +func TestClaimToPool(t *testing.T) { + nc := &nebulav1alpha1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "c1"}, + Spec: nebulav1alpha1.NodeClaimSpec{PoolRef: "gpu"}, + } + reqs := claimToPool(context.Background(), nc) + if len(reqs) != 1 || reqs[0].Name != "gpu" { + t.Fatalf("claimToPool = %+v, want one request for 'gpu'", reqs) + } + + // A claim with no pool ref enqueues nothing. + orphan := &nebulav1alpha1.NodeClaim{ObjectMeta: metav1.ObjectMeta{Name: "c2"}} + if reqs := claimToPool(context.Background(), orphan); reqs != nil { + t.Fatalf("expected nil for pool-less claim, got %+v", reqs) + } +} diff --git a/internal/controller/nodepool_validation_test.go b/internal/controller/nodepool_validation_test.go new file mode 100644 index 0000000..85fb718 --- /dev/null +++ b/internal/controller/nodepool_validation_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +// These specs exercise the CEL validation rule on NodePoolSpec, which is only +// enforced by a real apiserver (the fake client used by the unit tests does not +// run CRD x-kubernetes-validations). They require envtest binaries; when those +// are absent the whole suite is skipped in BeforeSuite. +var _ = Describe("NodePool spec validation (CEL)", func() { + newWeightedPool := func(name string, refs ...nebulav1alpha1.ProviderRef) *nebulav1alpha1.NodePool { + return &nebulav1alpha1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: nebulav1alpha1.NodePoolSpec{ + Providers: refs, + Strategy: nebulav1alpha1.StrategyWeighted, + }, + } + } + weight := func(w int32) *int32 { return &w } + + It("rejects a Weighted pool with a provider missing a weight", func() { + pool := newWeightedPool("weighted-missing", + nebulav1alpha1.ProviderRef{Name: "modal", Weight: weight(3)}, + nebulav1alpha1.ProviderRef{Name: "runpod"}, // no weight + ) + err := k8sClient.Create(ctx, pool) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strategy Weighted requires a weight on every provider")) + }) + + It("admits a Weighted pool with a weight on every provider", func() { + pool := newWeightedPool("weighted-ok", + nebulav1alpha1.ProviderRef{Name: "modal", Weight: weight(3)}, + nebulav1alpha1.ProviderRef{Name: "runpod", Weight: weight(1)}, + ) + Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + Expect(k8sClient.Delete(ctx, pool)).To(Succeed()) + }) + + It("admits a non-Weighted pool regardless of weights", func() { + pool := &nebulav1alpha1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: "ordered-noweights"}, + Spec: nebulav1alpha1.NodePoolSpec{ + Providers: []nebulav1alpha1.ProviderRef{{Name: "modal"}}, + Strategy: nebulav1alpha1.StrategyOrdered, + }, + } + Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + Expect(k8sClient.Delete(ctx, pool)).To(Succeed()) + }) +}) diff --git a/internal/controller/pod_placement_controller.go b/internal/controller/pod_placement_controller.go new file mode 100644 index 0000000..e799c69 --- /dev/null +++ b/internal/controller/pod_placement_controller.go @@ -0,0 +1,184 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" +) + +// staleClaimRequeue is how long placement waits before re-checking when a +// same-named NodeClaim from a prior Pod incarnation still exists. It only needs +// to outlast the NodeClaim backstop's reap of the stale claim, so a short, +// bounded retry is enough. +const staleClaimRequeue = 5 * time.Second + +// PodPlacementReconciler is the middle of the placement flow: it turns a gated, +// opted-in Pod into a placed one. The webhook holds the Pod SchedulingGated +// until a provider is chosen; this controller chooses one, records the decision +// on a NodeClaim (the durable teardown ledger), stamps the routing nodeSelector +// and capacity-type annotation onto the Pod, and removes the gate. The scheduler +// then binds the ungated Pod to that provider's virtual node, where the virtual +// kubelet provisions the external instance. +// +// Policy (v1): "first matching provider". Within the pool's provider list, pick +// the first whose catalog offers the requested GPU type. The richer optimizer +// (LowestPrice/Weighted, capacity-tier fallback, blocklist) is a later swap-in +// behind selectPlacement; the surrounding flow (gate, NodeClaim, ungate) is what +// this controller owns and does not change. +type PodPlacementReconciler struct { + client.Client + Scheme *runtime.Scheme + + // Providers resolves a provider name to its backend; defaults to the + // process-wide registry. Overridable in tests. + Providers func(name string) (provider.Provider, bool) +} + +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;update;patch;delete +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=nodepools,verbs=get;list;watch +// +kubebuilder:rbac:groups=nebula.inftyai.com,resources=nodeclaims,verbs=get;list;watch;create + +// Reconcile places one gated Pod. It is a no-op for any Pod that is not an +// opted-in, still-gated, not-yet-scheduled workload. +func (r *PodPlacementReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + var pod corev1.Pod + if err := r.Get(ctx, req.NamespacedName, &pod); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Reap a terminated Nebula workload Pod. When the external instance goes away + // (torn down, reclaimed, or exited) VK reports the Pod in a terminal phase, but + // nothing removes the Pod object: a ReplicaSet leaves a Failed Pod in place + // (creating a replacement beside it), and VK's own provider-eviction path skips + // Pods already terminal. Left alone the tombstone lingers until Kubernetes + // PodGC sweeps it. We hold a Kubernetes client, so delete it here when it is + // controller-owned — the owner (Deployment/Job) then recreates it cleanly and + // the now Pod-less NodeClaim self-deletes via its backstop. A bare, un-owned + // Pod is left intact as an inspectable record (nothing would recreate it). + if reaped, err := r.reapTerminalPod(ctx, &pod); err != nil || reaped { + return ctrl.Result{}, err + } + + // Only act on a Pod still waiting on our gate. A Pod that is unlabeled, + // already ungated, already bound, or being deleted is not ours to place. + if !needsPlacement(&pod) { + return ctrl.Result{}, nil + } + + pool, err := r.poolFor(ctx, &pod) + if err != nil { + return ctrl.Result{}, err + } + if pool == nil { + // No pool to place against. Leave the Pod gated rather than guessing; an + // operator sees a SchedulingGated Pod and a missing/mislabeled pool. + log.Info("no NodePool resolved for Pod; leaving it gated", "pod", pod.Name) + return ctrl.Result{}, nil + } + + placement, ok := r.selectPlacement(&pod, pool) + if !ok { + // No provider in the pool can serve this Pod's GPU type right now. Leave it + // gated; a later reconcile (pool edit, provider registered) can place it. + log.Info("no provider in pool can serve the Pod; leaving it gated", + "pod", pod.Name, "pool", pool.Name) + return ctrl.Result{}, nil + } + + // Create the NodeClaim BEFORE ungating: the claim is the durable teardown + // ledger, so it must exist before the Pod can bind and provision. Idempotent + // on a fixed, Pod-derived name, so a retry after a crash between create and + // ungate adopts the existing claim rather than making a second. + ready, err := r.ensureClaim(ctx, &pod, pool, placement) + if err != nil { + return ctrl.Result{}, err + } + if !ready { + // A stale claim for a prior same-named Pod still exists. Do NOT ungate + // against the wrong ledger; wait for the NodeClaim backstop to reap it, + // then a requeue re-creates the claim with this Pod's UID. + log.Info("waiting for a stale NodeClaim to be reclaimed before placing", + "pod", pod.Name, "claim", claimNameForPod(&pod)) + return ctrl.Result{RequeueAfter: staleClaimRequeue}, nil + } + + // Stamp the routing decision and release the Pod to the scheduler. + if err := r.place(ctx, &pod, placement); err != nil { + return ctrl.Result{}, err + } + log.Info("placed Pod", "pod", pod.Name, "provider", placement.provider, "capacityType", placement.capacityType) + return ctrl.Result{}, nil +} + +// placement is the resolved decision for one Pod. +type placement struct { + provider string + capacityType nebulav1alpha1.CapacityType +} + +// needsPlacement reports whether the Pod is an opted-in workload still held by +// our scheduling gate and not yet bound. +func needsPlacement(pod *corev1.Pod) bool { + if !pod.DeletionTimestamp.IsZero() { + return false + } + if pod.Labels[nebulav1alpha1.EnabledLabel] != "true" { + return false + } + if pod.Spec.NodeName != "" { + return false + } + return hasGateNamed(pod, nebulav1alpha1.ProviderSelectionGate) +} + +// hasGateNamed reports whether the Pod carries the named scheduling gate. +func hasGateNamed(pod *corev1.Pod, name string) bool { + for _, g := range pod.Spec.SchedulingGates { + if g.Name == name { + return true + } + } + return false +} + +// provider resolves the backend for name via the injected resolver, defaulting +// to the process-wide registry. +func (r *PodPlacementReconciler) provider(name string) (provider.Provider, bool) { + if r.Providers != nil { + return r.Providers(name) + } + return provider.Get(name) +} + +// claimName is the fixed, Pod-derived NodeClaim name, so placement is idempotent +// across retries: one Pod always maps to one claim of this name. +func claimNameForPod(pod *corev1.Pod) string { + return fmt.Sprintf("%s-%s", pod.Namespace, pod.Name) +} diff --git a/internal/controller/pod_placement_controller_test.go b/internal/controller/pod_placement_controller_test.go new file mode 100644 index 0000000..726073e --- /dev/null +++ b/internal/controller/pod_placement_controller_test.go @@ -0,0 +1,420 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +// newPlacementReconciler wires a PodPlacementReconciler over a fake client. +func newPlacementReconciler(t *testing.T, objs []client.Object, provs ...*fakeProvider) (*PodPlacementReconciler, client.Client) { + t.Helper() + s := testScheme(t) + _ = clientgoscheme.AddToScheme(s) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build() + r := &PodPlacementReconciler{Client: c, Scheme: s} + if len(provs) > 0 { + r.Providers = resolver(provs...) + } + return r, c +} + +// gatedPod builds an opted-in, gated, unscheduled Pod bound to a pool. +func gatedPod(name, ns, uid, pool, gpu string) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + UID: types.UID(uid), + Labels: map[string]string{ + nebulav1alpha1.EnabledLabel: "true", + nebulav1alpha1.PoolLabel: pool, + }, + }, + Spec: corev1.PodSpec{ + SchedulingGates: []corev1.PodSchedulingGate{ + {Name: nebulav1alpha1.ProviderSelectionGate}, + }, + Containers: []corev1.Container{{Name: "main", Image: "img"}}, + }, + } + if gpu != "" { + // The placement controller matches on the accelerator TYPE only, which + // rides on the accelerator-type label; the count (nvidia.com/gpu) is a + // provisioning detail the adapter reads, not needed here. + pod.Labels[nebulav1alpha1.AcceleratorTypeLabel] = gpu + } + return pod +} + +func poolWith(name string, capTypes []nebulav1alpha1.CapacityType, providers ...string) *nebulav1alpha1.NodePool { + var refs []nebulav1alpha1.ProviderRef + for _, p := range providers { + refs = append(refs, nebulav1alpha1.ProviderRef{Name: p}) + } + return &nebulav1alpha1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: nebulav1alpha1.NodePoolSpec{ + Providers: refs, + CapacityTypes: capTypes, + }, + } +} + +func reconcilePod(t *testing.T, r *PodPlacementReconciler, ns, name string) { + t.Helper() + if _, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Namespace: ns, Name: name}, + }); err != nil { + t.Fatalf("reconcile: %v", err) + } +} + +func getPod(t *testing.T, c client.Client, ns, name string) *corev1.Pod { + t.Helper() + var pod corev1.Pod + if err := c.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, &pod); err != nil { + t.Fatalf("get pod: %v", err) + } + return &pod +} + +func TestPlacement_UngatesAndRoutesAndCreatesClaim(t *testing.T) { + pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") + prov := &fakeProvider{name: "modal", gpus: []string{"H100"}} + r, c := newPlacementReconciler(t, []client.Object{pod, pool}, prov) + + reconcilePod(t, r, "default", "p1") + + got := getPod(t, c, "default", "p1") + // Gate removed. + if hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + t.Fatal("expected the provider-selection gate to be removed") + } + // Routed to the chosen provider. + if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != "modal" { + t.Fatalf("expected nodeSelector provider=modal, got %v", got.Spec.NodeSelector) + } + // Capacity tier stamped for the VK handler. + if got.Annotations[nebulav1alpha1.CapacityTypeAnnotation] != "OnDemand" { + t.Fatalf("expected capacity-type OnDemand, got %q", got.Annotations[nebulav1alpha1.CapacityTypeAnnotation]) + } + // NodeClaim created, pinned to the Pod, on the chosen provider. + var nc nebulav1alpha1.NodeClaim + if err := c.Get(context.Background(), types.NamespacedName{Name: "default-p1"}, &nc); err != nil { + t.Fatalf("expected NodeClaim default-p1: %v", err) + } + if nc.Spec.Provider != "modal" || nc.Spec.PodRef.UID != "uid-1" || nc.Spec.PoolRef != "pool-a" { + t.Fatalf("unexpected claim spec: %+v", nc.Spec) + } +} + +func TestPlacement_FirstMatchingProviderWins(t *testing.T) { + // runpod is listed first but does not offer H100; modal does. First MATCHING + // provider wins, so modal is chosen. + pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "runpod", "modal") + runpod := &fakeProvider{name: "runpod", gpus: []string{"A100"}} + modal := &fakeProvider{name: "modal", gpus: []string{"H100"}} + r, c := newPlacementReconciler(t, []client.Object{pod, pool}, runpod, modal) + + reconcilePod(t, r, "default", "p1") + + got := getPod(t, c, "default", "p1") + if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != "modal" { + t.Fatalf("expected modal (first matching), got %v", got.Spec.NodeSelector) + } +} + +func TestPlacement_OrderedPrefersEarlierProvider(t *testing.T) { + // Both offer H100; the first in the list wins. + pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "runpod", "modal") + runpod := &fakeProvider{name: "runpod", gpus: []string{"H100"}} + modal := &fakeProvider{name: "modal", gpus: []string{"H100"}} + r, c := newPlacementReconciler(t, []client.Object{pod, pool}, runpod, modal) + + reconcilePod(t, r, "default", "p1") + + got := getPod(t, c, "default", "p1") + if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != "runpod" { + t.Fatalf("expected runpod (first in list), got %v", got.Spec.NodeSelector) + } +} + +func TestPlacement_NoMatchingProviderLeavesPodGated(t *testing.T) { + pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "runpod") + runpod := &fakeProvider{name: "runpod", gpus: []string{"A100"}} // no H100 + r, c := newPlacementReconciler(t, []client.Object{pod, pool}, runpod) + + reconcilePod(t, r, "default", "p1") + + got := getPod(t, c, "default", "p1") + if !hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + t.Fatal("expected the Pod to stay gated when no provider matches") + } + if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != "" { + t.Fatal("expected no provider nodeSelector when unplaced") + } +} + +func TestPlacement_CPUOnlyPodMatchesAnyProvider(t *testing.T) { + // No GPU annotation => any provider matches; even one offering nothing. + pod := gatedPod("p1", "default", "uid-1", "pool-a", "") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") + modal := &fakeProvider{name: "modal", gpus: []string{}} // offers no GPUs + r, c := newPlacementReconciler(t, []client.Object{pod, pool}, modal) + + reconcilePod(t, r, "default", "p1") + + got := getPod(t, c, "default", "p1") + if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != "modal" { + t.Fatalf("expected a CPU-only Pod to place on modal, got %v", got.Spec.NodeSelector) + } +} + +func TestPlacement_SkipsPodWithoutOptInLabel(t *testing.T) { + pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") + pod.Labels[nebulav1alpha1.EnabledLabel] = "false" + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") + modal := &fakeProvider{name: "modal"} + r, c := newPlacementReconciler(t, []client.Object{pod, pool}, modal) + + reconcilePod(t, r, "default", "p1") + + got := getPod(t, c, "default", "p1") + if !hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + t.Fatal("expected a non-opted-in Pod to be left untouched") + } +} + +func TestPlacement_SkipsAlreadyScheduledPod(t *testing.T) { + pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") + pod.Spec.NodeName = "some-node" + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") + modal := &fakeProvider{name: "modal"} + r, c := newPlacementReconciler(t, []client.Object{pod, pool}, modal) + + reconcilePod(t, r, "default", "p1") + + // No claim should be created for an already-bound Pod. + var nc nebulav1alpha1.NodeClaim + err := c.Get(context.Background(), types.NamespacedName{Name: "default-p1"}, &nc) + if err == nil { + t.Fatal("expected no claim for an already-scheduled Pod") + } +} + +func TestPlacement_MissingPoolLeavesPodGated(t *testing.T) { + pod := gatedPod("p1", "default", "uid-1", "ghost-pool", "H100") + modal := &fakeProvider{name: "modal"} + r, c := newPlacementReconciler(t, []client.Object{pod}, modal) // no pool seeded + + reconcilePod(t, r, "default", "p1") + + got := getPod(t, c, "default", "p1") + if !hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + t.Fatal("expected the Pod to stay gated when its pool is missing") + } +} + +func TestPlacement_StaleClaimForPriorPodBlocksUngate(t *testing.T) { + // A claim of the Pod's name already exists but pins a PRIOR incarnation + // (different UID). Placement must NOT ungate against the wrong ledger: it + // leaves the Pod gated and requeues until the backstop reaps the stale claim. + pod := gatedPod("p1", "default", "uid-new", "pool-a", "H100") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") + stale := &nebulav1alpha1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "default-p1"}, + Spec: nebulav1alpha1.NodeClaimSpec{ + PodRef: nebulav1alpha1.PodReference{Namespace: "default", Name: "p1", UID: "uid-old"}, + Provider: "modal", + PoolRef: "pool-a", + }, + } + prov := &fakeProvider{name: "modal", gpus: []string{"H100"}} + r, c := newPlacementReconciler(t, []client.Object{pod, pool, stale}, prov) + + res, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Namespace: "default", Name: "p1"}, + }) + if err != nil { + t.Fatalf("reconcile: %v", err) + } + if res.RequeueAfter <= 0 { + t.Fatalf("expected a requeue while the stale claim is reaped, got %+v", res) + } + + got := getPod(t, c, "default", "p1") + if !hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + t.Fatal("expected the Pod to stay gated against a stale claim") + } + // The stale claim must be left untouched (the backstop, not placement, owns it). + var nc nebulav1alpha1.NodeClaim + if err := c.Get(context.Background(), types.NamespacedName{Name: "default-p1"}, &nc); err != nil { + t.Fatalf("stale claim should still exist: %v", err) + } + if nc.Spec.PodRef.UID != "uid-old" { + t.Fatalf("placement must not overwrite the stale claim, got UID %q", nc.Spec.PodRef.UID) + } +} + +func TestPlacement_AdoptsOwnClaimOnRetry(t *testing.T) { + // A claim of this name already exists AND pins this Pod's UID: a genuine retry + // after a crash between create and ungate. Placement adopts it and ungates. + pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") + mine := &nebulav1alpha1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "default-p1"}, + Spec: nebulav1alpha1.NodeClaimSpec{ + PodRef: nebulav1alpha1.PodReference{Namespace: "default", Name: "p1", UID: "uid-1"}, + Provider: "modal", + PoolRef: "pool-a", + }, + } + prov := &fakeProvider{name: "modal", gpus: []string{"H100"}} + r, c := newPlacementReconciler(t, []client.Object{pod, pool, mine}, prov) + + reconcilePod(t, r, "default", "p1") + + got := getPod(t, c, "default", "p1") + if hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + t.Fatal("expected the Pod placed when the existing claim is its own") + } + if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != "modal" { + t.Fatalf("expected routing to modal, got %v", got.Spec.NodeSelector) + } +} + +func TestPlacement_IdempotentOnRetry(t *testing.T) { + // A second reconcile after a successful placement must not error (claim + // AlreadyExists is success) and must leave the Pod placed. + pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") + prov := &fakeProvider{name: "modal", gpus: []string{"H100"}} + r, c := newPlacementReconciler(t, []client.Object{pod, pool}, prov) + + reconcilePod(t, r, "default", "p1") + // Re-gate the in-cluster Pod to simulate a duplicate event racing the claim. + got := getPod(t, c, "default", "p1") + got.Spec.SchedulingGates = []corev1.PodSchedulingGate{{Name: nebulav1alpha1.ProviderSelectionGate}} + if err := c.Update(context.Background(), got); err != nil { + t.Fatalf("re-gate: %v", err) + } + reconcilePod(t, r, "default", "p1") // must not error on AlreadyExists + + final := getPod(t, c, "default", "p1") + if hasGateNamed(final, nebulav1alpha1.ProviderSelectionGate) { + t.Fatal("expected the Pod placed again on retry") + } +} + +// terminalOwnedPod builds an opted-in Pod in a terminal phase. When ownedByRS is +// true it carries a controlling ReplicaSet ownerReference (so a controller would +// recreate it); otherwise it is a bare Pod. +func terminalOwnedPod(name, ns, uid string, phase corev1.PodPhase, ownedByRS bool) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + UID: types.UID(uid), + Labels: map[string]string{nebulav1alpha1.EnabledLabel: "true"}, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "img"}}}, + Status: corev1.PodStatus{Phase: phase}, + } + if ownedByRS { + ctrl := true + pod.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "apps/v1", + Kind: "ReplicaSet", + Name: "rs-1", + UID: types.UID("rs-uid"), + Controller: &ctrl, + }} + } + return pod +} + +func podPresent(c client.Client, ns, name string) bool { + var pod corev1.Pod + return c.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, &pod) == nil +} + +func TestReap_TerminalControllerOwnedPodIsDeleted(t *testing.T) { + // A Failed, controller-owned Nebula Pod is a tombstone: its owner recreates a + // replacement beside it and nothing removes the dead one. Placement reaps it. + pod := terminalOwnedPod("p1", "default", "uid-1", corev1.PodFailed, true) + r, c := newPlacementReconciler(t, []client.Object{pod}) + + reconcilePod(t, r, "default", "p1") + + if podPresent(c, "default", "p1") { + t.Fatal("expected a terminal controller-owned Pod to be deleted") + } +} + +func TestReap_TerminalBarePodIsKept(t *testing.T) { + // A bare (un-owned) terminal Pod is left intact as an inspectable record — + // nothing would recreate it, so deleting it would only lose information. + pod := terminalOwnedPod("p1", "default", "uid-1", corev1.PodFailed, false) + r, c := newPlacementReconciler(t, []client.Object{pod}) + + reconcilePod(t, r, "default", "p1") + + if !podPresent(c, "default", "p1") { + t.Fatal("expected a bare terminal Pod to be kept") + } +} + +func TestReap_RunningPodIsNotReaped(t *testing.T) { + // A live Pod must never be reaped, even when controller-owned. + pod := terminalOwnedPod("p1", "default", "uid-1", corev1.PodRunning, true) + r, c := newPlacementReconciler(t, []client.Object{pod}) + + reconcilePod(t, r, "default", "p1") + + if !podPresent(c, "default", "p1") { + t.Fatal("must not reap a Running Pod") + } +} + +func TestReap_NonNebulaTerminalPodIsIgnored(t *testing.T) { + // A terminal Pod that never opted into Nebula is not ours to reap. + pod := terminalOwnedPod("p1", "default", "uid-1", corev1.PodFailed, true) + pod.Labels[nebulav1alpha1.EnabledLabel] = "false" + r, c := newPlacementReconciler(t, []client.Object{pod}) + + reconcilePod(t, r, "default", "p1") + + if !podPresent(c, "default", "p1") { + t.Fatal("must not reap a non-Nebula Pod") + } +} diff --git a/internal/controller/pod_placement_helpers.go b/internal/controller/pod_placement_helpers.go new file mode 100644 index 0000000..00e0dd3 --- /dev/null +++ b/internal/controller/pod_placement_helpers.go @@ -0,0 +1,268 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/util" +) + +// poolFor resolves the NodePool a Pod is placed against. The Pod names its pool +// via PoolLabel. Returns (nil, nil) when the label is absent or the named pool +// does not exist, so the caller leaves the Pod gated rather than guessing. +func (r *PodPlacementReconciler) poolFor(ctx context.Context, pod *corev1.Pod) (*nebulav1alpha1.NodePool, error) { + name := pod.Labels[nebulav1alpha1.PoolLabel] + if name == "" { + return nil, nil + } + var pool nebulav1alpha1.NodePool + if err := r.Get(ctx, client.ObjectKey{Name: name}, &pool); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return &pool, nil +} + +// selectPlacement implements the v1 "first matching provider" policy: walk the +// pool's providers in listed order and pick the first whose adapter offers the +// Pod's requested GPU type. The capacity tier is the pool's first preference +// (the outer axis); provider quirks (e.g. Modal being OnDemand-only) are handled +// at Provision time, not here. Returns ok=false when nothing matches. +// +// This is the seam the richer optimizer replaces later (price ranking, capacity +// fallback, blocklist); the caller's flow does not depend on how the choice is +// made, only that a (provider, capacityType) comes back. +func (r *PodPlacementReconciler) selectPlacement(pod *corev1.Pod, pool *nebulav1alpha1.NodePool) (placement, bool) { + // Only the accelerator type is needed for matching; the count is a provisioning + // detail the adapter reads. A malformed request is treated as "no accelerator" + // so placement stays a no-op rather than erroring the reconcile — provisioning + // would surface the real error. + accel, _, _ := util.AcceleratorRequest(pod) + + for _, ref := range pool.Spec.Providers { + prov, ok := r.provider(ref.Name) + if !ok { + continue // unregistered; NodePool status surfaces this separately + } + // A CPU-only Pod (no accelerator) matches any provider; an accelerator Pod + // only matches a provider whose catalog maps that accelerator. + if accel != "" { + if _, offered := prov.MapAccelerator(accel); !offered { + continue + } + } + return placement{ + provider: ref.Name, + capacityType: preferredCapacityType(pool), + }, true + } + return placement{}, false +} + +// preferredCapacityType is the pool's first-choice tier (the outer axis's head). +// v1 does not do cross-tier fallback here; empty means "provider default". +func preferredCapacityType(pool *nebulav1alpha1.NodePool) nebulav1alpha1.CapacityType { + if len(pool.Spec.CapacityTypes) == 0 { + return "" + } + return pool.Spec.CapacityTypes[0] +} + +// ensureClaim creates the NodeClaim for this placement if it does not already +// exist. The claim is the durable teardown ledger (see NodeClaimReconciler): it +// pins the Pod by UID, records the chosen provider and capacity tier, and labels +// itself with the pool for status roll-up. +// +// It returns ready=true only when a claim pinned to THIS Pod's UID exists, so the +// caller may ungate. Two cases return ready=false with no error, telling the +// caller to requeue rather than ungate: +// - A pre-existing claim of the same name still pins a PRIOR Pod incarnation +// (same namespace/name, different UID). This happens when a Pod is recreated +// faster than the backstop reaps the old Pod's claim. Ungating now would bind +// the new Pod against a ledger that names the wrong instance, so we wait: the +// NodeClaim controller sees the old Pod is gone (UID mismatch), self-deletes +// the stale claim and terminates its instance, after which our Create succeeds +// with the correct UID. +func (r *PodPlacementReconciler) ensureClaim(ctx context.Context, pod *corev1.Pod, pool *nebulav1alpha1.NodePool, p placement) (ready bool, err error) { + claim := &nebulav1alpha1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimNameForPod(pod), + Labels: map[string]string{ + nebulav1alpha1.ManagedByLabel: nebulav1alpha1.ManagedByValue, + nebulav1alpha1.PoolLabel: pool.Name, + nebulav1alpha1.ProviderLabel: p.provider, + }, + }, + Spec: nebulav1alpha1.NodeClaimSpec{ + PodRef: nebulav1alpha1.PodReference{ + Namespace: pod.Namespace, + Name: pod.Name, + UID: string(pod.UID), + }, + Provider: p.provider, + CapacityType: p.capacityType, + PoolRef: pool.Name, + }, + } + err = r.Create(ctx, claim) + if err == nil { + return true, nil // freshly created for this Pod + } + if !apierrors.IsAlreadyExists(err) { + return false, err + } + + // A claim of this name already exists. Confirm it belongs to THIS Pod before + // treating the create as a successful no-op; otherwise it is a stale claim for + // a prior same-named Pod and we must wait for the backstop to reap it. + var existing nebulav1alpha1.NodeClaim + if err := r.Get(ctx, client.ObjectKeyFromObject(claim), &existing); err != nil { + // A NotFound here means the stale claim was reaped between our Create and + // Get; let the caller requeue so the next pass re-creates it cleanly. + return false, client.IgnoreNotFound(err) + } + if existing.Spec.PodRef.UID == string(pod.UID) { + return true, nil // our own claim (a retry after a crash before ungate) + } + return false, nil // stale claim for a prior Pod; wait for the backstop +} + +// place stamps the routing decision onto the Pod and removes the gate, atomically +// from the Pod's perspective (one Update). After this, the scheduler is free to +// bind the Pod to the chosen provider's virtual node. +func (r *PodPlacementReconciler) place(ctx context.Context, pod *corev1.Pod, p placement) error { + // Route to the provider's virtual node. + if pod.Spec.NodeSelector == nil { + pod.Spec.NodeSelector = map[string]string{} + } + pod.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] = p.provider + + // Carry the capacity tier the VK handler reads on CreatePod (the one input + // that is not otherwise on the Pod). Skip when empty (provider default). + if p.capacityType != "" { + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[nebulav1alpha1.CapacityTypeAnnotation] = string(p.capacityType) + } + + // Remove our gate, releasing the Pod to the scheduler. Preserve any other + // gates a different controller may hold. + pod.Spec.SchedulingGates = removeGate(pod.Spec.SchedulingGates, nebulav1alpha1.ProviderSelectionGate) + + return r.Update(ctx, pod) +} + +// removeGate returns gates with the named gate removed, preserving order. +func removeGate(gates []corev1.PodSchedulingGate, name string) []corev1.PodSchedulingGate { + out := gates[:0] + for _, g := range gates { + if g.Name != name { + out = append(out, g) + } + } + return out +} + +// reapTerminalPod deletes a terminated, Nebula-owned, controller-managed Pod so +// its owner recreates it cleanly instead of leaving a Failed tombstone. It +// returns reaped=true when it issued (or already sees) a delete for this Pod, so +// the caller stops processing it as a placement candidate. +// +// The delete is what actually stamps the Pod's deletionTimestamp — that field is +// server-managed and cannot be written directly, so a real Delete call is the +// only way to remove the object. It is scoped narrowly to avoid touching Pods +// that are not ours to reap: +// - EnabledLabel: only Pods that opted into Nebula (never a plain workload). +// - terminal phase: only Failed/Succeeded (a live Pod is never touched). +// - controller-owned: only Pods a ReplicaSet/Job will recreate; a bare Pod is +// left as an inspectable record. +// +// Delete is UID-pinned so a Pod already replaced by a same-name recreate is not +// clobbered, and a NotFound (already gone) is treated as success. +func (r *PodPlacementReconciler) reapTerminalPod(ctx context.Context, pod *corev1.Pod) (bool, error) { + if pod.Labels[nebulav1alpha1.EnabledLabel] != "true" { + return false, nil + } + if !pod.DeletionTimestamp.IsZero() { + return true, nil // already being deleted; nothing more to place + } + if !isTerminal(pod.Status.Phase) { + return false, nil + } + if !isControllerOwned(pod) { + return false, nil // bare Pod: leave it as a record + } + + preconditions := metav1.Preconditions{UID: &pod.UID} + if err := r.Delete(ctx, pod, &client.DeleteOptions{Preconditions: &preconditions}); err != nil { + return false, client.IgnoreNotFound(err) + } + return true, nil +} + +// isControllerOwned reports whether the Pod has a controlling owner (e.g. a +// ReplicaSet or Job) that will recreate it after deletion. +func isControllerOwned(pod *corev1.Pod) bool { + return metav1.GetControllerOf(pod) != nil +} + +// SetupWithManager wires the controller. It watches Pods; NodePool edits are not +// watched because a Pod already gated will re-reconcile on the periodic resync, +// and a newly-created Pod always triggers its own event. +func (r *PodPlacementReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Pod{}). + Watches(&nebulav1alpha1.NodePool{}, handler.EnqueueRequestsFromMapFunc(r.podsForPool)). + Named("pod-placement"). + Complete(r) +} + +// podsForPool re-enqueues gated Pods that name a pool when that pool changes, so +// a pool edit (e.g. adding a provider that can now serve a stuck Pod) promptly +// retries placement instead of waiting for the resync. +func (r *PodPlacementReconciler) podsForPool(ctx context.Context, obj client.Object) []reconcile.Request { + pool, ok := obj.(*nebulav1alpha1.NodePool) + if !ok { + return nil + } + var pods corev1.PodList + if err := r.List(ctx, &pods, client.MatchingLabels{nebulav1alpha1.PoolLabel: pool.Name}); err != nil { + return nil + } + var reqs []reconcile.Request + for i := range pods.Items { + if needsPlacement(&pods.Items[i]) { + reqs = append(reqs, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&pods.Items[i]), + }) + } + } + return reqs +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go new file mode 100644 index 0000000..b044a45 --- /dev/null +++ b/internal/controller/suite_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "os" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + ctx context.Context + cancel context.CancelFunc + testEnv *envtest.Environment + cfg *rest.Config + k8sClient client.Client +) + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + var err error + err = nebulav1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + // envtest needs the control-plane binaries (etcd/kube-apiserver). They are + // installed by `make setup-envtest`; when absent (e.g. a plain `go test` + // without that setup), skip the envtest specs rather than failing the whole + // package, so the fast fake-client unit tests still run. KUBEBUILDER_ASSETS + // (set by the Makefile target) takes precedence over the local bin lookup. + binDir := getFirstFoundEnvTestBinaryDir() + if os.Getenv("KUBEBUILDER_ASSETS") == "" && binDir == "" { + Skip("envtest binaries not found; run 'make setup-envtest' to enable envtest specs") + } + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + // Retrieve the first found binary directory to allow running tests from IDEs + if binDir != "" { + testEnv.BinaryAssetsDirectory = binDir + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) +}) + +var _ = AfterSuite(func() { + if cancel != nil { + cancel() + } + if testEnv == nil { + return // suite was skipped (no envtest binaries) + } + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. +// ENVTEST-based tests depend on specific binaries, usually located in paths set by +// controller-runtime. When running tests directly (e.g., via an IDE) without using +// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. +// +// This function streamlines the process by finding the required binaries, similar to +// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are +// properly set up, run 'make setup-envtest' beforehand. +func getFirstFoundEnvTestBinaryDir() string { + basePath := filepath.Join("..", "..", "bin", "k8s") + entries, err := os.ReadDir(basePath) + if err != nil { + logf.Log.Error(err, "Failed to read directory", "path", basePath) + return "" + } + for _, entry := range entries { + if entry.IsDir() { + return filepath.Join(basePath, entry.Name()) + } + } + return "" +} diff --git a/internal/webhook/v1/pod_webhook.go b/internal/webhook/v1/pod_webhook.go new file mode 100644 index 0000000..66c8341 --- /dev/null +++ b/internal/webhook/v1/pod_webhook.go @@ -0,0 +1,135 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +var podlog = logf.Log.WithName("pod-webhook") + +// SetupPodWebhookWithManager registers the webhook for Pod in the manager. +func SetupPodWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr).For(&corev1.Pod{}). + WithDefaulter(&PodCustomDefaulter{}). + Complete() +} + +// NOTE: the path is /mutate--v1-pod (double dash), not /mutate-core-v1-pod. +// controller-runtime derives the served path from the GVK, and Pod's API group +// is the empty string "", so the group segment is empty. The generated manifest +// must match that served path or the API server gets a 404 ("could not find the +// requested resource") when calling the webhook. +// +kubebuilder:webhook:path=/mutate--v1-pod,mutating=true,failurePolicy=fail,sideEffects=None,groups="",resources=pods,verbs=create,versions=v1,name=mpod-v1.nebula.inftyai.com,admissionReviewVersions=v1 + +// PodCustomDefaulter injects Nebula's provider-selection scheduling gate onto +// opted-in Pods at CREATE. The gate holds the Pod as SchedulingGated until the +// placement controller has chosen a provider (adding a provider nodeSelector) +// and removes the gate, releasing the Pod to the native scheduler. This is the +// entry point of the whole placement flow: no gate means the Pod is scheduled +// immediately by vanilla Kubernetes with no Nebula involvement. +// +// The webhook is scoped narrowly by an objectSelector on EnabledLabel in the +// generated manifest, so in practice only opted-in Pods reach it; the label +// check here is defence-in-depth for the case the selector is misconfigured. +type PodCustomDefaulter struct{} + +var _ webhook.CustomDefaulter = &PodCustomDefaulter{} + +// Default implements webhook.CustomDefaulter. For an opted-in Pod at CREATE it +// (a) adds the provider-selection scheduling gate and (b) adds a toleration for +// the per-provider virtual-node taint, but only when it is safe and meaningful: +// - the Pod carries the opt-in label (EnabledLabel=="true"); +// - the Pod is not already scheduled (spec.nodeName empty) — the API server +// rejects adding a scheduling gate to an already-bound Pod, and a Pod that +// is already scheduled has bypassed placement anyway; +// - each mutation is idempotent (safe under retries and the +// create;update-vs-create verb narrowing). +func (d *PodCustomDefaulter) Default(_ context.Context, obj runtime.Object) error { + pod, ok := obj.(*corev1.Pod) + if !ok { + return fmt.Errorf("expected a Pod object but got %T", obj) + } + + if pod.Labels[nebulav1alpha1.EnabledLabel] != "true" { + return nil // not opted in; leave the Pod untouched + } + if pod.Spec.NodeName != "" { + return nil // already scheduled; a gate can no longer be added + } + + // Tolerate the virtual node's taint. Every provider's virtual node carries + // nebula.inftyai.com/provider=:NoSchedule so that only Nebula-placed + // Pods land there; without a matching toleration the scheduler would never + // bind this Pod once the placement controller sets the provider nodeSelector. + // The provider is not chosen yet at admission, so the toleration is key-only + // (Operator=Exists), matching the taint for any provider value. + if !hasProviderToleration(pod) { + pod.Spec.Tolerations = append(pod.Spec.Tolerations, corev1.Toleration{ + Key: nebulav1alpha1.ProviderLabel, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }) + } + + if hasGate(pod, nebulav1alpha1.ProviderSelectionGate) { + return nil // already gated; nothing more to do + } + + pod.Spec.SchedulingGates = append(pod.Spec.SchedulingGates, corev1.PodSchedulingGate{ + Name: nebulav1alpha1.ProviderSelectionGate, + }) + podlog.Info("injected provider-selection scheduling gate and virtual-node toleration", + "namespace", pod.Namespace, "name", pod.Name) + return nil +} + +// hasGate reports whether the Pod already carries the named scheduling gate. +func hasGate(pod *corev1.Pod, name string) bool { + for _, g := range pod.Spec.SchedulingGates { + if g.Name == name { + return true + } + } + return false +} + +// hasProviderToleration reports whether the Pod already tolerates the +// provider-node taint. An existing toleration counts if it would match the +// NoSchedule taint on key nebula.inftyai.com/provider — either a key-only +// Exists toleration or the user's own equivalent — so we never add a duplicate. +func hasProviderToleration(pod *corev1.Pod) bool { + for _, t := range pod.Spec.Tolerations { + if t.Key != nebulav1alpha1.ProviderLabel { + continue + } + if t.Effect != "" && t.Effect != corev1.TaintEffectNoSchedule { + continue + } + return true + } + return false +} diff --git a/internal/webhook/v1/pod_webhook_test.go b/internal/webhook/v1/pod_webhook_test.go new file mode 100644 index 0000000..93784e1 --- /dev/null +++ b/internal/webhook/v1/pod_webhook_test.go @@ -0,0 +1,174 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +func gated(pod *corev1.Pod) bool { + return hasGate(pod, nebulav1alpha1.ProviderSelectionGate) +} + +func podWith(labels map[string]string, nodeName string, gates ...string) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "default", Labels: labels}, + Spec: corev1.PodSpec{NodeName: nodeName}, + } + for _, g := range gates { + p.Spec.SchedulingGates = append(p.Spec.SchedulingGates, corev1.PodSchedulingGate{Name: g}) + } + return p +} + +func TestDefault_InjectsGateForOptedInPod(t *testing.T) { + d := &PodCustomDefaulter{} + pod := podWith(map[string]string{nebulav1alpha1.EnabledLabel: "true"}, "") + + if err := d.Default(context.Background(), pod); err != nil { + t.Fatalf("Default: %v", err) + } + if !gated(pod) { + t.Fatal("expected provider-selection gate to be injected") + } + if len(pod.Spec.SchedulingGates) != 1 { + t.Fatalf("expected exactly 1 gate, got %d", len(pod.Spec.SchedulingGates)) + } +} + +func TestDefault_PreservesExistingGates(t *testing.T) { + d := &PodCustomDefaulter{} + pod := podWith(map[string]string{nebulav1alpha1.EnabledLabel: "true"}, "", "example.com/other-gate") + + if err := d.Default(context.Background(), pod); err != nil { + t.Fatalf("Default: %v", err) + } + if len(pod.Spec.SchedulingGates) != 2 { + t.Fatalf("expected the existing gate to be kept alongside ours, got %d", len(pod.Spec.SchedulingGates)) + } + if !gated(pod) { + t.Fatal("expected provider-selection gate present") + } +} + +func TestDefault_SkipsWhenNotOptedIn(t *testing.T) { + d := &PodCustomDefaulter{} + cases := map[string]map[string]string{ + "no labels": nil, + "label absent": {"other": "x"}, + "label not true": {nebulav1alpha1.EnabledLabel: "false"}, + } + for name, labels := range cases { + t.Run(name, func(t *testing.T) { + pod := podWith(labels, "") + if err := d.Default(context.Background(), pod); err != nil { + t.Fatalf("Default: %v", err) + } + if gated(pod) { + t.Fatal("expected no gate for a non-opted-in Pod") + } + }) + } +} + +func TestDefault_SkipsAlreadyScheduledPod(t *testing.T) { + d := &PodCustomDefaulter{} + pod := podWith(map[string]string{nebulav1alpha1.EnabledLabel: "true"}, "node-1") + + if err := d.Default(context.Background(), pod); err != nil { + t.Fatalf("Default: %v", err) + } + if gated(pod) { + t.Fatal("must not add a scheduling gate to an already-scheduled Pod") + } +} + +func TestDefault_IdempotentWhenGateAlreadyPresent(t *testing.T) { + d := &PodCustomDefaulter{} + pod := podWith(map[string]string{nebulav1alpha1.EnabledLabel: "true"}, "", nebulav1alpha1.ProviderSelectionGate) + + if err := d.Default(context.Background(), pod); err != nil { + t.Fatalf("Default: %v", err) + } + if len(pod.Spec.SchedulingGates) != 1 { + t.Fatalf("expected gate not to be duplicated, got %d", len(pod.Spec.SchedulingGates)) + } +} + +func tolerated(pod *corev1.Pod) bool { + return hasProviderToleration(pod) +} + +func TestDefault_InjectsProviderToleration(t *testing.T) { + d := &PodCustomDefaulter{} + pod := podWith(map[string]string{nebulav1alpha1.EnabledLabel: "true"}, "") + + if err := d.Default(context.Background(), pod); err != nil { + t.Fatalf("Default: %v", err) + } + if len(pod.Spec.Tolerations) != 1 { + t.Fatalf("expected exactly 1 toleration, got %d", len(pod.Spec.Tolerations)) + } + tol := pod.Spec.Tolerations[0] + if tol.Key != nebulav1alpha1.ProviderLabel || + tol.Operator != corev1.TolerationOpExists || + tol.Effect != corev1.TaintEffectNoSchedule { + t.Fatalf("unexpected toleration: %+v", tol) + } +} + +func TestDefault_DoesNotDuplicateProviderToleration(t *testing.T) { + d := &PodCustomDefaulter{} + pod := podWith(map[string]string{nebulav1alpha1.EnabledLabel: "true"}, "") + pod.Spec.Tolerations = []corev1.Toleration{{ + Key: nebulav1alpha1.ProviderLabel, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }} + + if err := d.Default(context.Background(), pod); err != nil { + t.Fatalf("Default: %v", err) + } + if len(pod.Spec.Tolerations) != 1 { + t.Fatalf("expected the existing provider toleration to be kept without duplication, got %d", len(pod.Spec.Tolerations)) + } +} + +func TestDefault_NoTolerationForNonOptedInPod(t *testing.T) { + d := &PodCustomDefaulter{} + pod := podWith(map[string]string{nebulav1alpha1.EnabledLabel: "false"}, "") + + if err := d.Default(context.Background(), pod); err != nil { + t.Fatalf("Default: %v", err) + } + if tolerated(pod) { + t.Fatal("expected no provider toleration for a non-opted-in Pod") + } +} + +func TestDefault_RejectsNonPod(t *testing.T) { + d := &PodCustomDefaulter{} + if err := d.Default(context.Background(), &corev1.Service{}); err == nil { + t.Fatal("expected an error for a non-Pod object") + } +} diff --git a/internal/webhook/v1/webhook_suite_test.go b/internal/webhook/v1/webhook_suite_test.go new file mode 100644 index 0000000..8a8b780 --- /dev/null +++ b/internal/webhook/v1/webhook_suite_test.go @@ -0,0 +1,178 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + ctx context.Context + cancel context.CancelFunc + k8sClient client.Client + cfg *rest.Config + testEnv *envtest.Environment +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Webhook Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + var err error + err = corev1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + // envtest needs the control-plane binaries (etcd/kube-apiserver). They are + // installed by `make setup-envtest`; when absent (e.g. a plain `go test` + // without that setup), skip the envtest specs rather than failing the whole + // package, so the fast unit tests in this package still run. KUBEBUILDER_ASSETS + // (set by the Makefile target) takes precedence over the local bin lookup. + binDir := getFirstFoundEnvTestBinaryDir() + if os.Getenv("KUBEBUILDER_ASSETS") == "" && binDir == "" { + Skip("envtest binaries not found; run 'make setup-envtest' to enable envtest specs") + } + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: false, + + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "..", "config", "webhook")}, + }, + } + + // Retrieve the first found binary directory to allow running tests from IDEs + if binDir != "" { + testEnv.BinaryAssetsDirectory = binDir + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // start webhook server using Manager. + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + }), + LeaderElection: false, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = SetupPodWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready. + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + + return conn.Close() + }).Should(Succeed()) +}) + +var _ = AfterSuite(func() { + if cancel != nil { + cancel() + } + if testEnv == nil { + return // suite was skipped (no envtest binaries) + } + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. +// ENVTEST-based tests depend on specific binaries, usually located in paths set by +// controller-runtime. When running tests directly (e.g., via an IDE) without using +// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. +// +// This function streamlines the process by finding the required binaries, similar to +// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are +// properly set up, run 'make setup-envtest' beforehand. +func getFirstFoundEnvTestBinaryDir() string { + basePath := filepath.Join("..", "..", "..", "bin", "k8s") + entries, err := os.ReadDir(basePath) + if err != nil { + logf.Log.Error(err, "Failed to read directory", "path", basePath) + return "" + } + for _, entry := range entries { + if entry.IsDir() { + return filepath.Join(basePath, entry.Name()) + } + } + return "" +} diff --git a/pkg/provider/catalog/base.go b/pkg/provider/catalog/base.go new file mode 100644 index 0000000..046c049 --- /dev/null +++ b/pkg/provider/catalog/base.go @@ -0,0 +1,91 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package catalog + +import ( + "context" + "strings" + + "github.com/InftyAI/Nebula/pkg/provider" +) + +// Lookup is the price/availability seam a provider adapter depends on: given a +// provider name it returns that provider's offering rows. The concrete *Catalog +// satisfies it; tests inject a trivial fake so an adapter can be unit-tested +// without embedding CSVs. It lives in this package (not in provider) so all +// catalog-shaped types share one home and there is no provider.Catalog / +// catalog.Catalog name clash. +type Lookup interface { + // Offerings returns providerName's rows, or nil if the provider has no + // catalog entry. Implementations return a copy the caller may safely annotate. + Offerings(providerName string) []provider.Offering +} + +// Base supplies the parts of provider.Provider that are identical for every +// adapter whose price/availability comes from a catalog: Name, Offerings, and a +// default identity MapAccelerator. Adapters embed it so those three methods are +// not re-implemented per provider: +// +// type Provider struct { +// catalog.Base +// client Client +// } +// +// The split is deliberate: +// - Name and Offerings are FULLY generic — every catalog-backed adapter would +// otherwise write the identical one-liners. +// - MapAccelerator is generic ONLY while a provider names its accelerators +// exactly like Nebula's canonical names (Modal does). A provider whose +// identifiers diverge (e.g. RunPod's "NVIDIA H100 80GB HBM3") overrides just +// this one method — its own method shadows the promoted one — while still +// reusing Name/Offerings. +// +// Lifecycle (Provision/Terminate/Get/List), Capabilities and +// ClassifyProvisionError are genuinely provider-specific and are NOT provided +// here; each adapter implements them itself. +type Base struct { + // ProviderName is this provider's stable identifier (e.g. "modal"), used both + // as Name() and as the key into the catalog. + ProviderName string + // Catalog is the shared price/availability lookup. + Catalog Lookup +} + +// Name returns the provider's stable identifier. +func (b Base) Name() string { return b.ProviderName } + +// Offerings returns this provider's rows from the catalog. The error is always +// nil today (the catalog is in-memory); the signature matches provider.Provider +// so an adapter that later combines the static catalog with a live availability +// probe can return a real error without a signature change. +func (b Base) Offerings(context.Context) ([]provider.Offering, error) { + return b.Catalog.Offerings(b.ProviderName), nil +} + +// MapAccelerator is the default identity translation: it confirms the canonical +// accelerator is in this provider's catalog (case-insensitively) and returns the +// canonical name unchanged. Providers whose identifiers differ from the +// canonical names override this method. Returns ok=false when the provider does +// not offer the accelerator. +func (b Base) MapAccelerator(canonical string) (providerAcceleratorID string, ok bool) { + for _, o := range b.Catalog.Offerings(b.ProviderName) { + if strings.EqualFold(o.AcceleratorType, canonical) { + return o.AcceleratorType, true + } + } + return "", false +} diff --git a/pkg/provider/catalog/catalog.go b/pkg/provider/catalog/catalog.go new file mode 100644 index 0000000..d87dd4b --- /dev/null +++ b/pkg/provider/catalog/catalog.go @@ -0,0 +1,167 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package catalog is the shared price/availability catalog for providers whose +// APIs do not expose a rate card (Modal, and most NeoClouds). It follows +// SkyPilot's community-catalog pattern: prices live in checked-in, git-tracked +// CSV files (pkg/provider/catalog/data/.csv) rather than hardcoded in +// Go, so a price change is a reviewable data diff. +// +// Two-tier load, override-first: +// +// 1. If an override directory is set (NEBULA_CATALOG_DIR, or LoadFrom(dir)), +// CSVs there win. This is how the deployed controller consumes a ConfigMap: +// the CSVs are rendered into a ConfigMap and mounted, so ops can edit prices +// live (kubectl edit configmap) without rebuilding the image. +// 2. Otherwise the CSVs embedded at build time are used as the default, so the +// binary always has a working catalog even with nothing mounted. +// +// A provider's Offerings() becomes a lookup into this catalog instead of a +// hardcoded table. +package catalog + +import ( + "embed" + "encoding/csv" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" +) + +// OverrideDirEnv is the env var pointing at a directory of provider CSVs that +// override the embedded defaults. Set by the manager Deployment to the ConfigMap +// mount path. +const OverrideDirEnv = "NEBULA_CATALOG_DIR" + +//go:embed data/*.csv +var embedded embed.FS + +// Catalog holds parsed offerings keyed by provider name. +type Catalog struct { + byProvider map[string][]provider.Offering +} + +// Load builds a Catalog, preferring CSVs in the override dir named by +// NEBULA_CATALOG_DIR when that env var is set and the dir exists, otherwise +// falling back to the embedded defaults. +func Load() (*Catalog, error) { + if dir := os.Getenv(OverrideDirEnv); dir != "" { + if info, err := os.Stat(dir); err == nil && info.IsDir() { + return LoadFrom(dir) + } + } + return loadFS(embedded, "data") +} + +// LoadFrom builds a Catalog from CSV files in dir (used for the ConfigMap mount +// and in tests). Each file must be named ".csv". +func LoadFrom(dir string) (*Catalog, error) { + return loadFS(os.DirFS(dir), ".") +} + +// loadFS parses every ".csv" under root of fsys into a Catalog. +func loadFS(fsys fs.FS, root string) (*Catalog, error) { + entries, err := fs.ReadDir(fsys, root) + if err != nil { + return nil, fmt.Errorf("catalog: read dir: %w", err) + } + c := &Catalog{byProvider: make(map[string][]provider.Offering)} + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".csv") { + continue + } + providerName := strings.TrimSuffix(name, ".csv") + f, err := fsys.Open(filepath.Join(root, name)) + if err != nil { + return nil, fmt.Errorf("catalog: open %s: %w", name, err) + } + offerings, err := parseCSV(f) + _ = f.Close() + if err != nil { + return nil, fmt.Errorf("catalog: parse %s: %w", name, err) + } + c.byProvider[providerName] = offerings + } + return c, nil +} + +// Offerings returns the catalog rows for providerName, or nil if the provider +// has no catalog file. The returned slice is a copy, safe for the caller to +// annotate (e.g. a live availability probe) without mutating the catalog. +func (c *Catalog) Offerings(providerName string) []provider.Offering { + src := c.byProvider[providerName] + if len(src) == 0 { + return nil + } + out := make([]provider.Offering, len(src)) + copy(out, src) + return out +} + +// parseCSV reads offering rows. Lines beginning with '#' are comments; the first +// non-comment line is the header. Column order is fixed: +// accelerator,capacity_type,price_per_hour,available,updated. The trailing +// updated column is documentation only and ignored here. +// +// There is deliberately no provider_id column yet: every provider we currently +// support (Modal) names its accelerators identically to Nebula's canonical +// names, so MapAccelerator is pure identity. When a provider whose identifiers +// diverge lands (e.g. RunPod's "NVIDIA H100 80GB HBM3"), add a provider_id +// column here and have MapAccelerator return it — no per-provider Go table. +func parseCSV(r io.Reader) ([]provider.Offering, error) { + cr := csv.NewReader(r) + cr.Comment = '#' + cr.FieldsPerRecord = -1 // tolerate a trailing "updated" column we ignore + cr.TrimLeadingSpace = true + + records, err := cr.ReadAll() + if err != nil { + return nil, err + } + + var offerings []provider.Offering + for i, rec := range records { + if i == 0 { + continue // header + } + if len(rec) < 4 { + return nil, fmt.Errorf("row %d: expected >=4 columns, got %d", i, len(rec)) + } + price, err := strconv.ParseFloat(strings.TrimSpace(rec[2]), 64) + if err != nil { + return nil, fmt.Errorf("row %d: price_per_hour %q: %w", i, rec[2], err) + } + available, err := strconv.ParseBool(strings.TrimSpace(rec[3])) + if err != nil { + return nil, fmt.Errorf("row %d: available %q: %w", i, rec[3], err) + } + offerings = append(offerings, provider.Offering{ + AcceleratorType: strings.TrimSpace(rec[0]), + CapacityType: nebulav1alpha1.CapacityType(strings.TrimSpace(rec[1])), + PricePerHour: price, + Available: available, + }) + } + return offerings, nil +} diff --git a/pkg/provider/catalog/catalog_test.go b/pkg/provider/catalog/catalog_test.go new file mode 100644 index 0000000..6234435 --- /dev/null +++ b/pkg/provider/catalog/catalog_test.go @@ -0,0 +1,145 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package catalog + +import ( + "os" + "path/filepath" + "testing" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" +) + +// fakeLookup is a trivial catalog.Lookup for exercising Base's promoted methods +// without loading CSVs. +type fakeLookup struct{ rows []provider.Offering } + +func (l fakeLookup) Offerings(string) []provider.Offering { return l.rows } + +func TestLoad_EmbeddedModal(t *testing.T) { + c, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + offs := c.Offerings("modal") + if len(offs) == 0 { + t.Fatal("expected embedded modal offerings") + } + + // H200 must be present (proto lists it; it was missing from the old map). + var haveH200 bool + for _, o := range offs { + if o.CapacityType != nebulav1alpha1.CapacityOnDemand { + t.Fatalf("modal offering %q must be OnDemand, got %v", o.AcceleratorType, o.CapacityType) + } + if o.PricePerHour <= 0 { + t.Fatalf("offering %q has non-positive price %v", o.AcceleratorType, o.PricePerHour) + } + if o.AcceleratorType == "H200" { + haveH200 = true + } + } + if !haveH200 { + t.Fatal("expected H200 in modal catalog") + } +} + +func TestLoadFrom_OverrideDir(t *testing.T) { + dir := t.TempDir() + csv := "accelerator,capacity_type,price_per_hour,available,updated\n" + + "H100,OnDemand,9.99,true,2026-07-25\n" + if err := os.WriteFile(filepath.Join(dir, "modal.csv"), []byte(csv), 0o644); err != nil { + t.Fatal(err) + } + + c, err := LoadFrom(dir) + if err != nil { + t.Fatalf("LoadFrom: %v", err) + } + offs := c.Offerings("modal") + if len(offs) != 1 || offs[0].AcceleratorType != "H100" || offs[0].PricePerHour != 9.99 { + t.Fatalf("override not applied: %+v", offs) + } +} + +func TestLoad_OverrideEnvWins(t *testing.T) { + dir := t.TempDir() + csv := "accelerator,capacity_type,price_per_hour,available,updated\n" + + "T4,OnDemand,0.11,true,2026-07-25\n" + if err := os.WriteFile(filepath.Join(dir, "modal.csv"), []byte(csv), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv(OverrideDirEnv, dir) + + c, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + offs := c.Offerings("modal") + if len(offs) != 1 || offs[0].PricePerHour != 0.11 { + t.Fatalf("env override not honored: %+v", offs) + } +} + +func TestBaseMapAccelerator_CaseInsensitive(t *testing.T) { + base := Base{ + ProviderName: "modal", + Catalog: fakeLookup{rows: []provider.Offering{ + {AcceleratorType: "H100", CapacityType: nebulav1alpha1.CapacityOnDemand}, + {AcceleratorType: "A100-80GB", CapacityType: nebulav1alpha1.CapacityOnDemand}, + }}, + } + + // A canonical accelerator supplied in any case must resolve, and the returned + // id is always the catalog's canonical casing — so a lowercase + // accelerator-type label ("h100") maps cleanly to the provider's accelerator ("H100"). + cases := map[string]struct { + in string + want string + wantOK bool + }{ + "exact": {"H100", "H100", true}, + "all lower": {"h100", "H100", true}, + "mixed": {"a100-80gb", "A100-80GB", true}, + "unknown": {"tpu-v4", "", false}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + got, ok := base.MapAccelerator(tc.in) + if ok != tc.wantOK || got != tc.want { + t.Fatalf("MapAccelerator(%q) = (%q, %v), want (%q, %v)", tc.in, got, ok, tc.want, tc.wantOK) + } + }) + } +} + +func TestOfferings_CopyIsolation(t *testing.T) { + c, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + a := c.Offerings("modal") + if len(a) == 0 { + t.Skip("no modal rows") + } + a[0].PricePerHour = -1 // mutate the returned copy + b := c.Offerings("modal") + if b[0].PricePerHour == -1 { + t.Fatal("Offerings returned a shared slice; caller mutation leaked into catalog") + } +} diff --git a/pkg/provider/catalog/data/modal.csv b/pkg/provider/catalog/data/modal.csv new file mode 100644 index 0000000..2596277 --- /dev/null +++ b/pkg/provider/catalog/data/modal.csv @@ -0,0 +1,30 @@ +# Modal price/availability catalog — community-maintained. +# +# Modal exposes NO pricing API (SDK and gRPC surface neither a rate card nor +# per-accelerator prices), so these are list prices transcribed from +# https://modal.com/pricing by hand. Treat them as a starting point, not a +# billing source, and refresh via `make update-catalog` (see hack/refresh.go). +# Modal is OnDemand-only: it has no user-facing spot/preemptible tier. +# +# This file is the SINGLE source of truth for Modal's accelerators: the +# accelerator column defines both the supported set + price (for the optimizer) +# and, since Modal names its GPUs identically to Nebula's canonical names, the +# accelerator vocabulary MapAccelerator resolves against. No accelerator table +# lives in Go anymore. (If a future provider's identifiers diverge from the +# canonical names, add a provider_id column here rather than a Go map.) +# +# Columns: +# accelerator canonical Nebula accelerator type +# capacity_type Spot | OnDemand | Reserved +# price_per_hour approximate on-demand USD/GPU-hour +# available whether Modal currently offers it (a live probe may override) +# updated YYYY-MM-DD the row was last verified +accelerator,capacity_type,price_per_hour,available,updated +T4,OnDemand,0.59,true,2026-07-25 +L4,OnDemand,0.80,true,2026-07-25 +A10G,OnDemand,1.10,true,2026-07-25 +L40S,OnDemand,1.95,true,2026-07-25 +A100-40GB,OnDemand,2.10,true,2026-07-25 +A100-80GB,OnDemand,2.50,true,2026-07-25 +H100,OnDemand,3.95,true,2026-07-25 +H200,OnDemand,4.55,true,2026-07-25 diff --git a/pkg/provider/errors.go b/pkg/provider/errors.go new file mode 100644 index 0000000..0f5ee13 --- /dev/null +++ b/pkg/provider/errors.go @@ -0,0 +1,93 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +import ( + "errors" + "strings" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +// Provision failure categories, shared by every adapter. The CATEGORIES are +// universal — no-capacity, unsupported, auth, quota describe outcomes any +// NeoCloud can return — so they live here, not in a single adapter. Each adapter +// recognizes its provider-specific API conditions and wraps the matching +// sentinel (via fmt.Errorf("...: %w", provider.ErrX)); the control plane can then +// errors.Is against these without importing any adapter, and BlockScope is +// derived uniformly by ClassifyError below. +var ( + // ErrNoCapacity: the provider could not allocate the requested accelerator + // right now. Accelerator-scoped and transient. + ErrNoCapacity = errors.New("provider: no capacity for requested accelerator") + // ErrUnsupportedAccelerator: the provider does not offer the accelerator at + // all. Accelerator-scoped, durable until the pool changes. + ErrUnsupportedAccelerator = errors.New("provider: unsupported accelerator") + // ErrAuth: credentials/authorization failed. Whole-provider — nothing will + // succeed until it is fixed. + ErrAuth = errors.New("provider: authentication failed") + // ErrQuota: account quota/limit reached. Whole-provider until quota frees up. + ErrQuota = errors.New("provider: quota exceeded") +) + +// ClassifyError maps a provision error to the BlockScope it should be +// blocklisted at, using the shared sentinels first and a string-heuristic +// fallback for raw API messages that were not wrapped. It encodes the rule that +// a narrow failure (this accelerator has no capacity) must not disqualify other +// accelerators on the same provider, while auth/quota widen to the whole +// provider; an unrecognized error is treated conservatively as whole-provider so +// a failure does not hot-loop (the blocklist TTL bounds the blast radius). +// +// capacityType is the tier the failing request used; it is stamped onto +// accelerator-scoped scopes so the block is precise (a Spot failure does not +// block OnDemand). Adapters call this from ClassifyProvisionError after wrapping +// their own errors, passing the only tier they serve when they are single-tier. +func ClassifyError(err error, capacityType nebulav1alpha1.CapacityType) BlockScope { + if err == nil { + return BlockScope{} + } + + switch { + case errors.Is(err, ErrAuth), errors.Is(err, ErrQuota): + return BlockScope{DenyAll: true} + case errors.Is(err, ErrNoCapacity), errors.Is(err, ErrUnsupportedAccelerator): + return BlockScope{CapacityType: capacityType} + } + + // Fall back to string heuristics for errors not wrapped with a sentinel. + msg := strings.ToLower(err.Error()) + switch { + case containsAny(msg, "unauthorized", "forbidden", "authentication", "invalid token", "api key"): + return BlockScope{DenyAll: true} + case containsAny(msg, "quota", "limit exceeded", "rate limit"): + return BlockScope{DenyAll: true} + case containsAny(msg, "no capacity", "capacity", "unavailable", "out of", "no gpu"): + return BlockScope{CapacityType: capacityType} + default: + return BlockScope{DenyAll: true} + } +} + +// containsAny reports whether s contains any of subs. +func containsAny(s string, subs ...string) bool { + for _, sub := range subs { + if strings.Contains(s, sub) { + return true + } + } + return false +} diff --git a/pkg/provider/errors_test.go b/pkg/provider/errors_test.go new file mode 100644 index 0000000..3c91ceb --- /dev/null +++ b/pkg/provider/errors_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +import ( + "fmt" + "testing" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +func TestClassifyError(t *testing.T) { + const tier = nebulav1alpha1.CapacityOnDemand + tests := []struct { + name string + err error + want BlockScope + }{ + {"nil", nil, BlockScope{}}, + {"auth sentinel", ErrAuth, BlockScope{DenyAll: true}}, + {"quota sentinel", ErrQuota, BlockScope{DenyAll: true}}, + {"no-capacity sentinel", ErrNoCapacity, BlockScope{CapacityType: tier}}, + {"unsupported sentinel", ErrUnsupportedAccelerator, BlockScope{CapacityType: tier}}, + {"wrapped sentinel", fmt.Errorf("provision failed: %w", ErrNoCapacity), BlockScope{CapacityType: tier}}, + {"string unauthorized", fmt.Errorf("HTTP 401 unauthorized"), BlockScope{DenyAll: true}}, + {"string quota", fmt.Errorf("account limit exceeded"), BlockScope{DenyAll: true}}, + {"string capacity", fmt.Errorf("no capacity available"), BlockScope{CapacityType: tier}}, + {"unknown conservative", fmt.Errorf("weird transient blip"), BlockScope{DenyAll: true}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ClassifyError(tt.err, tier); got != tt.want { + t.Fatalf("ClassifyError(%v) = %+v, want %+v", tt.err, got, tt.want) + } + }) + } +} + +// Spot tier must be stamped onto accelerator-scoped blocks so a Spot failure +// does not block OnDemand on the same provider. +func TestClassifyError_TierStamped(t *testing.T) { + got := ClassifyError(ErrNoCapacity, nebulav1alpha1.CapacitySpot) + if got.CapacityType != nebulav1alpha1.CapacitySpot || got.DenyAll { + t.Fatalf("expected Spot accelerator-scoped block, got %+v", got) + } +} diff --git a/pkg/provider/modal/client.go b/pkg/provider/modal/client.go new file mode 100644 index 0000000..2e36f62 --- /dev/null +++ b/pkg/provider/modal/client.go @@ -0,0 +1,232 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modal + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + modal "github.com/modal-labs/modal-client/go" + + "github.com/InftyAI/Nebula/pkg/provider/catalog" +) + +// sdkClient is the real Client, backed by Modal's official Go SDK +// (github.com/modal-labs/modal-client/go, beta). It is the seam between our +// provider-agnostic adapter and Modal's API; all Modal-specific SDK calls live +// here so the adapter and its tests stay SDK-free. +// +// A NodeClaim maps to one Modal Sandbox. Identity is carried in the sandbox's +// native Tags (ClaimTagKey), which are server-side filterable, so List/Get can +// recover the owning claim without a naming hack. +type sdkClient struct { + mc *modal.Client + appName string + + // endpointTimeout bounds the best-effort tunnel lookup used to report an + // instance's reachable address; tunnels only exist once the sandbox is up. + endpointTimeout time.Duration +} + +// compile-time assertion that sdkClient satisfies the adapter's Client seam. +var _ Client = (*sdkClient)(nil) + +// NewSDKClient builds a Modal-backed Client. It reads Modal credentials from the +// environment / ~/.modal.toml via the SDK's default profile. appName is the +// Modal App all Nebula sandboxes are created under (created if missing at first +// use). The returned *Provider is ready to register. +// +// Example wiring: +// +// c, err := modal.NewSDKClient(ctx, "nebula") +// if err != nil { return err } +// provider.Register(modal.New(c)) +func NewSDKClient(ctx context.Context, appName string) (*Provider, error) { + mc, err := modal.NewClient() + if err != nil { + return nil, fmt.Errorf("modal: init SDK client: %w", err) + } + if appName == "" { + appName = "nebula" + } + cat, err := catalog.Load() + if err != nil { + return nil, fmt.Errorf("modal: load price catalog: %w", err) + } + return New(&sdkClient{ + mc: mc, + appName: appName, + endpointTimeout: 5 * time.Second, + }, cat), nil +} + +// app resolves (creating if missing) the Modal App all sandboxes live under. +func (c *sdkClient) app(ctx context.Context) (*modal.App, error) { + return c.mc.Apps.FromName(ctx, c.appName, &modal.AppFromNameParams{CreateIfMissing: true}) +} + +// CreateSandbox implements Client. +func (c *sdkClient) CreateSandbox(ctx context.Context, spec SandboxSpec) (string, error) { + app, err := c.app(ctx) + if err != nil { + return "", fmt.Errorf("modal: resolve app: %w", err) + } + if spec.Image == "" { + return "", fmt.Errorf("modal: empty image in sandbox spec") + } + image := c.mc.Images.FromRegistry(spec.Image, nil) + + sb, err := c.mc.Sandboxes.Create(ctx, app, image, &modal.SandboxCreateParams{ + Command: spec.Command, + Env: spec.Env, + GPU: gpuReservation(spec.GPU, spec.GPUCount), + CPU: spec.CPU, + MemoryMiB: spec.MemoryMiB, + EncryptedPorts: spec.Ports, + Timeout: spec.Timeout, + Tags: spec.Tags, + }) + if err != nil { + return "", err + } + return sb.SandboxID, nil +} + +// TerminateSandbox implements Client. Idempotent: a sandbox that no longer +// exists resolves to a not-found from FromID, which we treat as already gone. +func (c *sdkClient) TerminateSandbox(ctx context.Context, id string) error { + sb, err := c.mc.Sandboxes.FromID(ctx, id, &modal.SandboxFromIDParams{}) + if err != nil { + if isNotFound(err) { + return nil + } + return err + } + if _, err := sb.Terminate(ctx, &modal.SandboxTerminateParams{}); err != nil { + if isNotFound(err) { + return nil + } + return err + } + return nil +} + +// GetSandbox implements Client. Returns (nil, nil) when the sandbox is gone. +func (c *sdkClient) GetSandbox(ctx context.Context, id string) (*Sandbox, error) { + sb, err := c.mc.Sandboxes.FromID(ctx, id, &modal.SandboxFromIDParams{}) + if err != nil { + if isNotFound(err) { + return nil, nil + } + return nil, err + } + out := c.observe(ctx, sb) + return &out, nil +} + +// ListSandboxes implements Client. Filters server-side by the Nebula claim tag +// key so only Nebula-owned sandboxes are returned. +func (c *sdkClient) ListSandboxes(ctx context.Context) ([]Sandbox, error) { + app, err := c.app(ctx) + if err != nil { + return nil, fmt.Errorf("modal: resolve app: %w", err) + } + seq, err := c.mc.Sandboxes.List(ctx, &modal.SandboxListParams{AppID: app.AppID}) + if err != nil { + return nil, err + } + var out []Sandbox + for sb, err := range seq { + if err != nil { + return nil, err + } + out = append(out, c.observe(ctx, sb)) + } + return out, nil +} + +// observe normalizes a live SDK *Sandbox into the adapter-level Sandbox view: +// status (from Poll), tags (from GetTags), and a best-effort endpoint (from +// Tunnels). Tag/tunnel/poll errors are tolerated so a single flaky sandbox +// doesn't fail the whole List — the poll loop will re-observe next tick. +func (c *sdkClient) observe(ctx context.Context, sb *modal.Sandbox) Sandbox { + out := Sandbox{ID: sb.SandboxID} + + // Status. Poll (== sandboxWait(0)) reports whether the sandbox PROCESS HAS + // EXITED: a non-nil exit code means it is gone (terminated), nil means it is + // still live. We treat "still live" as running. This does fold the brief + // startup window (queued, image pull, GPU attach, container boot) into + // "running" rather than "pending", because the SDK at this version exposes no + // lightweight readiness readback: WaitUntilReady only resolves against a + // readiness probe configured at create time (which we do not set) and requires + // a live task-command-router connection, so it can never report ready here. + // Poll is the only reliable signal, and reporting a starting sandbox as running + // a few seconds early is far better than the alternative — never reporting it + // ready at all, which wedges the Pod at Pending and the NodeClaim at Provisioning. + if code, err := sb.Poll(ctx, &modal.SandboxPollParams{}); err == nil { + if code != nil { + out.Status = "terminated" + } else { + out.Status = "running" + } + } + + if tags, err := sb.GetTags(ctx, &modal.SandboxGetTagsParams{}); err == nil { + out.Tags = tags + } + + // Endpoint is only meaningful once running; look it up best-effort. + if out.Status == "running" { + tctx, cancel := context.WithTimeout(ctx, c.endpointTimeout) + if tunnels, err := sb.Tunnels(tctx, c.endpointTimeout, &modal.SandboxTunnelsParams{}); err == nil { + for _, t := range tunnels { + out.Endpoint = t.URL() + break + } + } + cancel() + } + return out +} + +// gpuReservation renders Modal's GPU reservation string. Modal expresses count +// as a "type:count" suffix (e.g. "A100:2"); a count of 0/1 needs no suffix, and +// an empty type means a CPU-only sandbox. +func gpuReservation(gpuType string, count int32) string { + if gpuType == "" { + return "" + } + if count > 1 { + return gpuType + ":" + strconv.FormatInt(int64(count), 10) + } + return gpuType +} + +// isNotFound reports whether err indicates the sandbox no longer exists, so +// Terminate/Get can treat it as already gone (idempotency). The SDK does not +// export a typed not-found error at this beta version, so we match on message. +func isNotFound(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "not found") || strings.Contains(msg, "notfound") || + strings.Contains(msg, "does not exist") +} diff --git a/pkg/provider/modal/modal.go b/pkg/provider/modal/modal.go new file mode 100644 index 0000000..d4ed6d2 --- /dev/null +++ b/pkg/provider/modal/modal.go @@ -0,0 +1,380 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package modal implements the provider.Provider interface for Modal +// (https://modal.com), a serverless GPU compute platform. +// +// Modal's shape drives several adapter decisions: +// - Lifecycle is create/terminate only. A Modal Sandbox is spun up and later +// terminated; there is no stop/resume, so Capabilities.SupportsStop=false. +// - Modal does not expose a user-facing spot/preemptible tier, so +// SupportsSpot=false. The optimizer therefore only ever sends OnDemand +// ProvisionRequests here (the NodePool capacity-tier loop skips Spot for +// providers that don't advertise it). +// - Modal Sandboxes carry native tags, so NativeTags=true and the ClaimName +// is stored as a tag rather than smuggled into the instance name. +// - There is no preemption push; detection is poll-based like every provider. +// +// The concrete Modal API (auth, sandbox create/terminate/list) lives behind the +// Client seam so this package holds only provider-agnostic translation and is +// unit-testable without network access. A real Client wrapping Modal's API is +// wired in separately. +package modal + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/provider/catalog" + "github.com/InftyAI/Nebula/pkg/util" +) + +// defaultSandboxTimeout is the maximum lifetime the adapter sets when a Pod does +// not pin its own activeDeadlineSeconds. Modal requires a bounded timeout (a zero +// value silently means its 5-minute default, which would kill a real workload), +// so we default high — 24h — for a long-running GPU job. A Pod that wants a +// different ceiling sets spec.activeDeadlineSeconds, which maps straight through. +const defaultSandboxTimeout = 24 * time.Hour + +// compile-time assertion that Provider satisfies the interface. +var _ provider.Provider = (*Provider)(nil) + +// Client is the narrow seam over Modal's API. It is intentionally small: only +// the operations the adapter needs, expressed in provider-agnostic terms, so a +// real implementation (Modal SDK/HTTP) and a fake (tests) are interchangeable. +type Client interface { + // CreateSandbox launches one sandbox from spec and returns its Modal id. + CreateSandbox(ctx context.Context, spec SandboxSpec) (id string, err error) + // TerminateSandbox terminates a sandbox by id. Must be idempotent: + // terminating an already-gone sandbox returns nil. + TerminateSandbox(ctx context.Context, id string) error + // GetSandbox returns one sandbox, or (nil, nil) if it no longer exists. + GetSandbox(ctx context.Context, id string) (*Sandbox, error) + // ListSandboxes returns every Nebula-owned sandbox, filtered by the tag the + // adapter sets at create time, in as few calls as possible. + ListSandboxes(ctx context.Context) ([]Sandbox, error) +} + +// SandboxSpec is the resolved, Modal-shaped request the Client turns into a +// sandbox. The adapter builds it from the Pod (source of truth) plus the +// resolved accelerator id. +type SandboxSpec struct { + // Image is the container image, from the Pod's first container. + Image string + // Command is the container command+args, from the Pod. + Command []string + // Env is the environment, flattened from the Pod's container env. + Env map[string]string + // GPU is Modal's accelerator identifier (e.g. "H100", "A100-80GB"), or "" + // for a CPU-only sandbox. + GPU string + // GPUCount is how many accelerators to attach (0 for CPU-only). + GPUCount int32 + // CPU is the requested cores (fractional, physical), from the Pod's first + // container resource request. Zero lets Modal apply its own default. + CPU float64 + // MemoryMiB is the requested memory in MiB, from the Pod's request. Zero lets + // Modal apply its own default. + MemoryMiB int + // Ports are the container ports to expose as encrypted tunnels, from the Pod's + // containerPorts. The reachable endpoint (reported as the Pod's address) is a + // tunnel to one of these, so a Pod that declares no port has no endpoint. + Ports []int + // Timeout is the sandbox's maximum lifetime. It MUST be non-zero: Modal treats + // a zero timeout as its 5-minute default, which would terminate a real + // workload almost immediately. The adapter always sets it (from the Pod's + // activeDeadlineSeconds, else a long default). + Timeout time.Duration + // Tags carry Nebula identity; ClaimTagKey holds the NodeClaim name. + Tags map[string]string +} + +// Sandbox is the adapter-level view of a Modal sandbox as observed. +type Sandbox struct { + ID string + Tags map[string]string + Status string // Modal's own status string, normalized by toState. + Endpoint string +} + +// ClaimTagKey is the sandbox tag under which the NodeClaim name is stored, so +// List/Get can recover Nebula identity. Modal supports native tags, so no +// name-encoding hack is needed. +const ClaimTagKey = "nebula.inftyai.com/claim" + +// Provider is the Modal implementation of provider.Provider. It embeds +// catalog.Base for the generic catalog methods (Name, Offerings, and the +// identity MapAccelerator — Modal names its GPUs exactly like Nebula's canonical +// names) and implements only the Modal-specific lifecycle here. +type Provider struct { + catalog.Base + client Client +} + +// New returns a Modal Provider backed by client and price catalog. Both must be +// non-nil; use catalog.Load() to build the catalog from the CSV/ConfigMap data. +// cat is the catalog.Lookup seam, so tests can inject a fake. +func New(client Client, cat catalog.Lookup) *Provider { + return &Provider{ + Base: catalog.Base{ProviderName: provider.ProviderModal, Catalog: cat}, + client: client, + } +} + +// Capabilities implements provider.Provider. See the package doc for why each +// trait is set the way it is. +func (p *Provider) Capabilities() provider.Capabilities { + return provider.Capabilities{ + SupportsStop: false, // create/terminate only + SupportsSpot: false, // no user-facing preemptible tier + NativeTags: true, // sandbox tags carry identity + PreemptionNotice: 0, // no push; poll-based detection + PollInterval: 0, // OnDemand-only (never preempts) → the default cadence is fine + } +} + +// Provision implements provider.Provider. The Pod is the source of truth for +// the workload; req carries only the claim identity and capacity tier. +func (p *Provider) Provision(ctx context.Context, pod *corev1.Pod, req provider.ProvisionRequest) (string, error) { + if pod == nil { + return "", errors.New("modal: nil pod") + } + if req.ClaimName == "" { + return "", errors.New("modal: empty ClaimName in ProvisionRequest") + } + + // Idempotency: if a sandbox already carries this claim tag, return it rather + // than creating a second (guards against a retry after a partial create). + if existing, err := p.findByClaim(ctx, req.ClaimName); err != nil { + return "", err + } else if existing != nil { + return existing.ID, nil + } + + spec, err := p.sandboxSpecFromPod(pod, req) + if err != nil { + return "", err + } + return p.client.CreateSandbox(ctx, spec) +} + +// Terminate implements provider.Provider. Idempotent by the Client contract. +func (p *Provider) Terminate(ctx context.Context, instanceID string) error { + if instanceID == "" { + return nil // nothing provisioned yet; treat as already gone + } + return p.client.TerminateSandbox(ctx, instanceID) +} + +// Get implements provider.Provider. +func (p *Provider) Get(ctx context.Context, instanceID string) (*provider.Instance, error) { + sb, err := p.client.GetSandbox(ctx, instanceID) + if err != nil { + return nil, err + } + if sb == nil { + return nil, nil // absent => terminated, per interface contract + } + inst := p.toInstance(*sb) + return &inst, nil +} + +// List implements provider.Provider. +func (p *Provider) List(ctx context.Context) ([]provider.Instance, error) { + sandboxes, err := p.client.ListSandboxes(ctx) + if err != nil { + return nil, err + } + out := make([]provider.Instance, 0, len(sandboxes)) + for _, sb := range sandboxes { + out = append(out, p.toInstance(sb)) + } + return out, nil +} + +// ClassifyProvisionError implements provider.Provider. The failure CATEGORIES +// and the scope-derivation rule are shared across all adapters +// (provider.ClassifyError, provider.ErrNoCapacity, ...), so this method only +// supplies what is Modal-specific: the capacity tier to stamp on an +// accelerator-scoped block. Modal is OnDemand-only, so that is always OnDemand. +// +// A real Modal Client that recognizes an SDK error condition should wrap the +// matching shared sentinel (e.g. fmt.Errorf("...: %w", provider.ErrNoCapacity)); +// ClassifyError honours those first and falls back to string heuristics for raw +// API messages, so no Modal-specific matching is duplicated here. +func (p *Provider) ClassifyProvisionError(err error) provider.BlockScope { + return provider.ClassifyError(err, nebulav1alpha1.CapacityOnDemand) +} + +// findByClaim returns the sandbox tagged with claimName, or nil if none. +func (p *Provider) findByClaim(ctx context.Context, claimName string) (*provider.Instance, error) { + sandboxes, err := p.client.ListSandboxes(ctx) + if err != nil { + return nil, err + } + for _, sb := range sandboxes { + if sb.Tags[ClaimTagKey] == claimName { + inst := p.toInstance(sb) + return &inst, nil + } + } + return nil, nil +} + +// sandboxSpecFromPod reads the workload off the Pod (source of truth) and the +// accelerator type (from the AcceleratorTypeLabel) and count (from the +// nvidia.com/gpu resource), then maps the accelerator to Modal's identifier. +func (p *Provider) sandboxSpecFromPod(pod *corev1.Pod, req provider.ProvisionRequest) (SandboxSpec, error) { + if len(pod.Spec.Containers) == 0 { + return SandboxSpec{}, errors.New("modal: pod has no containers") + } + c := pod.Spec.Containers[0] + + env := make(map[string]string, len(c.Env)) + for _, e := range c.Env { + // ValueFrom (secrets/configmaps) is not resolved here; the real Client + // wiring must project those. Plain values are copied through. + if e.ValueFrom == nil { + env[e.Name] = e.Value + } + } + + spec := SandboxSpec{ + Image: c.Image, + Command: append(append([]string{}, c.Command...), c.Args...), + Env: env, + CPU: cpuCores(&c), + MemoryMiB: memoryMiB(&c), + Ports: containerPorts(&c), + Timeout: sandboxTimeout(pod), + Tags: map[string]string{ClaimTagKey: req.ClaimName}, + } + + // Accelerator type comes from the AcceleratorTypeLabel; count from the + // container's nvidia.com/gpu resource (see util.AcceleratorRequest). + canonical, count, err := util.AcceleratorRequest(pod) + if err != nil { + return SandboxSpec{}, fmt.Errorf("modal: %w", err) + } + if canonical != "" { + modalGPU, ok := p.MapAccelerator(canonical) + if !ok { + return SandboxSpec{}, fmt.Errorf("modal: unsupported accelerator %q", canonical) + } + spec.GPU = modalGPU + spec.GPUCount = count + } + // No annotation => CPU-only sandbox (GPU/"" GPUCount 0), handled naturally. + return spec, nil +} + +// cpuCores reads the container's CPU request as fractional physical cores (Modal's +// unit). It prefers requests, falling back to limits, and returns 0 (→ Modal +// default) when neither is set. +func cpuCores(c *corev1.Container) float64 { + q := resourceQty(c, corev1.ResourceCPU) + if q == nil { + return 0 + } + // MilliValue is cores*1000; convert to fractional cores. + return float64(q.MilliValue()) / 1000.0 +} + +// memoryMiB reads the container's memory request in MiB (Modal's unit), preferring +// requests over limits. Returns 0 (→ Modal default) when neither is set. +func memoryMiB(c *corev1.Container) int { + q := resourceQty(c, corev1.ResourceMemory) + if q == nil { + return 0 + } + const miB = 1024 * 1024 + return int(q.Value() / miB) +} + +// resourceQty returns the container's request for name, falling back to its limit, +// or nil when neither is present. +func resourceQty(c *corev1.Container, name corev1.ResourceName) *resource.Quantity { + if q, ok := c.Resources.Requests[name]; ok { + return &q + } + if q, ok := c.Resources.Limits[name]; ok { + return &q + } + return nil +} + +// containerPorts collects the container's declared ports so the Client can open a +// tunnel per port. The observed tunnel URL becomes the Pod's endpoint, so a Pod +// that declares no port is reachable-less by design. +func containerPorts(c *corev1.Container) []int { + if len(c.Ports) == 0 { + return nil + } + ports := make([]int, 0, len(c.Ports)) + for _, p := range c.Ports { + ports = append(ports, int(p.ContainerPort)) + } + return ports +} + +// sandboxTimeout maps the Pod's activeDeadlineSeconds (Kubernetes' own "maximum +// lifetime of the pod") onto Modal's sandbox Timeout, defaulting to +// defaultSandboxTimeout when the Pod does not pin one. It is never zero: a zero +// Timeout is Modal's 5-minute default and would kill a real workload. +func sandboxTimeout(pod *corev1.Pod) time.Duration { + if d := pod.Spec.ActiveDeadlineSeconds; d != nil && *d > 0 { + return time.Duration(*d) * time.Second + } + return defaultSandboxTimeout +} + +// toInstance normalizes a Modal sandbox into the provider-agnostic Instance. +func (p *Provider) toInstance(sb Sandbox) provider.Instance { + return provider.Instance{ + ID: sb.ID, + ClaimName: sb.Tags[ClaimTagKey], + State: toState(sb.Status), + Endpoint: sb.Endpoint, + // Modal is OnDemand-only; reflect that on observed instances. + CapacityType: nebulav1alpha1.CapacityOnDemand, + } +} + +// toState maps Modal's status strings to the provider-agnostic lifecycle state. +// Unknown statuses map to Pending so the poll loop keeps watching rather than +// declaring a premature terminal state. +func toState(modalStatus string) provider.InstanceState { + switch strings.ToLower(modalStatus) { + case "running", "ready": + return provider.InstanceRunning + case "terminated", "stopped", "completed": + return provider.InstanceTerminated + case "error", "failed": + return provider.InstanceFailed + case "pending", "starting", "queued", "": + return provider.InstancePending + default: + return provider.InstancePending + } +} diff --git a/pkg/provider/modal/modal_test.go b/pkg/provider/modal/modal_test.go new file mode 100644 index 0000000..8c4b708 --- /dev/null +++ b/pkg/provider/modal/modal_test.go @@ -0,0 +1,330 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modal + +import ( + "context" + "fmt" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/util" +) + +// fakeClient is an in-memory Client for tests. It records the last CreateSandbox +// spec and lets tests seed existing sandboxes and inject errors. +type fakeClient struct { + sandboxes []Sandbox + lastSpec SandboxSpec + createCnt int + createErr error + createID string + terminated []string +} + +func (f *fakeClient) CreateSandbox(_ context.Context, spec SandboxSpec) (string, error) { + f.createCnt++ + f.lastSpec = spec + if f.createErr != nil { + return "", f.createErr + } + id := f.createID + if id == "" { + id = "sb-new" + } + f.sandboxes = append(f.sandboxes, Sandbox{ID: id, Tags: spec.Tags, Status: "pending"}) + return id, nil +} + +func (f *fakeClient) TerminateSandbox(_ context.Context, id string) error { + f.terminated = append(f.terminated, id) + return nil +} + +func (f *fakeClient) GetSandbox(_ context.Context, id string) (*Sandbox, error) { + for i := range f.sandboxes { + if f.sandboxes[i].ID == id { + s := f.sandboxes[i] + return &s, nil + } + } + return nil, nil +} + +func (f *fakeClient) ListSandboxes(_ context.Context) ([]Sandbox, error) { + return f.sandboxes, nil +} + +// fakeCatalog is a trivial provider.Catalog for tests. +type fakeCatalog struct{ rows []provider.Offering } + +func (c fakeCatalog) Offerings(_ string) []provider.Offering { return c.rows } + +// newTestProvider builds a Provider with a fake client and a small catalog. +func newTestProvider(f *fakeClient) *Provider { + return New(f, fakeCatalog{rows: []provider.Offering{ + {AcceleratorType: "H100", CapacityType: nebulav1alpha1.CapacityOnDemand, PricePerHour: 3.95, Available: true}, + {AcceleratorType: "A100-80GB", CapacityType: nebulav1alpha1.CapacityOnDemand, PricePerHour: 2.50, Available: true}, + }}) +} + +// gpuPod builds a Pod whose accelerator type rides on the accelerator-type label +// and whose count rides on the container's nvidia.com/gpu resource; count<=0 +// means CPU-only (no label, no GPU resource). accel is passed through verbatim +// so tests can also exercise non-canonical casing (e.g. "h100"). +func gpuPod(claim, accel string, count int64) *corev1.Pod { + c := corev1.Container{ + Name: "main", + Image: "myimg:latest", + Command: []string{"run"}, + Args: []string{"--flag"}, + Env: []corev1.EnvVar{{Name: "FOO", Value: "bar"}}, + } + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "default"}, + Spec: corev1.PodSpec{Containers: []corev1.Container{c}}, + } + if accel != "" && count > 0 { + pod.Labels = map[string]string{nebulav1alpha1.AcceleratorTypeLabel: accel} + pod.Spec.Containers[0].Resources.Limits = corev1.ResourceList{ + util.NvidiaGPUResource: *resource.NewQuantity(count, resource.DecimalSI), + } + } + return pod +} + +func TestProvision_GPUPod(t *testing.T) { + f := &fakeClient{createID: "sb-1"} + p := newTestProvider(f) + + id, err := p.Provision(context.Background(), gpuPod("claim-a", "H100", 2), provider.ProvisionRequest{ + ClaimName: "claim-a", + CapacityType: nebulav1alpha1.CapacityOnDemand, + }) + if err != nil { + t.Fatalf("Provision: %v", err) + } + if id != "sb-1" { + t.Fatalf("id = %q, want sb-1", id) + } + if f.lastSpec.GPU != "H100" || f.lastSpec.GPUCount != 2 { + t.Fatalf("spec GPU=%q count=%d, want H100/2", f.lastSpec.GPU, f.lastSpec.GPUCount) + } + if f.lastSpec.Image != "myimg:latest" { + t.Fatalf("image = %q", f.lastSpec.Image) + } + if got := f.lastSpec.Tags[ClaimTagKey]; got != "claim-a" { + t.Fatalf("claim tag = %q, want claim-a", got) + } + if len(f.lastSpec.Command) != 2 || f.lastSpec.Command[0] != "run" { + t.Fatalf("command = %v", f.lastSpec.Command) + } +} + +func TestProvision_LowercaseGPUAnnotation(t *testing.T) { + f := &fakeClient{createID: "sb-lc"} + p := newTestProvider(f) + + // A user may write the accelerator-type label in any case (e.g. "h100"). It must + // resolve to the canonical catalog accelerator ("H100") so the provisioned + // sandbox — and any downstream key (blocklist/catalog) — uses one casing. + _, err := p.Provision(context.Background(), gpuPod("claim-lc", "h100", 1), provider.ProvisionRequest{ + ClaimName: "claim-lc", + CapacityType: nebulav1alpha1.CapacityOnDemand, + }) + if err != nil { + t.Fatalf("Provision: %v", err) + } + if f.lastSpec.GPU != "H100" { + t.Fatalf("spec GPU = %q, want canonical H100 from a lowercase annotation", f.lastSpec.GPU) + } +} + +func TestProvision_MapsResourcesPortsAndTimeout(t *testing.T) { + f := &fakeClient{createID: "sb-res"} + p := newTestProvider(f) + + pod := gpuPod("claim-res", "H100", 1) + c := &pod.Spec.Containers[0] + c.Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2500m"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + }, + } + c.Ports = []corev1.ContainerPort{{ContainerPort: 8000}, {ContainerPort: 9090}} + deadline := int64(3600) + pod.Spec.ActiveDeadlineSeconds = &deadline + + if _, err := p.Provision(context.Background(), pod, provider.ProvisionRequest{ClaimName: "claim-res"}); err != nil { + t.Fatalf("Provision: %v", err) + } + if f.lastSpec.CPU != 2.5 { + t.Fatalf("CPU = %v, want 2.5", f.lastSpec.CPU) + } + if f.lastSpec.MemoryMiB != 4096 { + t.Fatalf("MemoryMiB = %d, want 4096", f.lastSpec.MemoryMiB) + } + if len(f.lastSpec.Ports) != 2 || f.lastSpec.Ports[0] != 8000 || f.lastSpec.Ports[1] != 9090 { + t.Fatalf("Ports = %v, want [8000 9090]", f.lastSpec.Ports) + } + if f.lastSpec.Timeout != time.Hour { + t.Fatalf("Timeout = %v, want 1h (from activeDeadlineSeconds)", f.lastSpec.Timeout) + } +} + +func TestProvision_DefaultsTimeoutWhenNoDeadline(t *testing.T) { + f := &fakeClient{createID: "sb-dt"} + p := newTestProvider(f) + + // No activeDeadlineSeconds: the adapter must still set a non-zero timeout, else + // Modal applies its 5-minute default and the workload dies almost immediately. + if _, err := p.Provision(context.Background(), gpuPod("claim-dt", "H100", 1), provider.ProvisionRequest{ClaimName: "claim-dt"}); err != nil { + t.Fatalf("Provision: %v", err) + } + if f.lastSpec.Timeout != defaultSandboxTimeout { + t.Fatalf("Timeout = %v, want the long default %v", f.lastSpec.Timeout, defaultSandboxTimeout) + } +} + +func TestProvision_CPUOnly(t *testing.T) { + f := &fakeClient{} + p := newTestProvider(f) + + _, err := p.Provision(context.Background(), gpuPod("claim-cpu", "", 0), provider.ProvisionRequest{ + ClaimName: "claim-cpu", + CapacityType: nebulav1alpha1.CapacityOnDemand, + }) + if err != nil { + t.Fatalf("Provision: %v", err) + } + if f.lastSpec.GPU != "" || f.lastSpec.GPUCount != 0 { + t.Fatalf("CPU-only spec should have no GPU, got %q/%d", f.lastSpec.GPU, f.lastSpec.GPUCount) + } +} + +func TestProvision_Idempotent(t *testing.T) { + f := &fakeClient{ + sandboxes: []Sandbox{{ID: "sb-existing", Tags: map[string]string{ClaimTagKey: "claim-a"}, Status: "running"}}, + } + p := newTestProvider(f) + + id, err := p.Provision(context.Background(), gpuPod("claim-a", "H100", 1), provider.ProvisionRequest{ClaimName: "claim-a"}) + if err != nil { + t.Fatalf("Provision: %v", err) + } + if id != "sb-existing" { + t.Fatalf("id = %q, want sb-existing (idempotent reuse)", id) + } + if f.createCnt != 0 { + t.Fatalf("CreateSandbox called %d times, want 0 (idempotent)", f.createCnt) + } +} + +func TestProvision_UnsupportedAccelerator(t *testing.T) { + f := &fakeClient{} + p := newTestProvider(f) + _, err := p.Provision(context.Background(), gpuPod("claim-x", "TPU-v4", 1), provider.ProvisionRequest{ClaimName: "claim-x"}) + if err == nil { + t.Fatal("expected error for unsupported accelerator") + } +} + +func TestClassifyProvisionError(t *testing.T) { + p := newTestProvider(&fakeClient{}) + tests := []struct { + name string + err error + want provider.BlockScope + }{ + {"auth sentinel", provider.ErrAuth, provider.BlockScope{DenyAll: true}}, + {"quota sentinel", provider.ErrQuota, provider.BlockScope{DenyAll: true}}, + {"capacity sentinel", provider.ErrNoCapacity, provider.BlockScope{CapacityType: nebulav1alpha1.CapacityOnDemand}}, + {"wrapped capacity", fmt.Errorf("provision: %w", provider.ErrNoCapacity), provider.BlockScope{CapacityType: nebulav1alpha1.CapacityOnDemand}}, + {"string unauthorized", fmt.Errorf("401 unauthorized"), provider.BlockScope{DenyAll: true}}, + {"string no capacity", fmt.Errorf("no capacity available in region"), provider.BlockScope{CapacityType: nebulav1alpha1.CapacityOnDemand}}, + {"unknown", fmt.Errorf("weird transient blip"), provider.BlockScope{DenyAll: true}}, + {"nil", nil, provider.BlockScope{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := p.ClassifyProvisionError(tt.err) + if got != tt.want { + t.Fatalf("got %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestCapabilities(t *testing.T) { + p := newTestProvider(&fakeClient{}) + caps := p.Capabilities() + if caps.SupportsStop || caps.SupportsSpot || caps.PreemptionNotice != 0 || !caps.NativeTags { + t.Fatalf("unexpected caps: %+v", caps) + } + if p.Name() != provider.ProviderModal { + t.Fatalf("name = %q", p.Name()) + } +} + +func TestOfferings(t *testing.T) { + p := newTestProvider(&fakeClient{}) + offs, err := p.Offerings(context.Background()) + if err != nil { + t.Fatalf("Offerings: %v", err) + } + if len(offs) == 0 { + t.Fatal("expected non-empty catalog") + } + for _, o := range offs { + if o.CapacityType != nebulav1alpha1.CapacityOnDemand { + t.Fatalf("offering %q not OnDemand: %v", o.AcceleratorType, o.CapacityType) + } + } +} + +func TestToState(t *testing.T) { + // The status→state mapping is load-bearing: "pending" must map to + // InstancePending, not InstanceRunning. A still-starting sandbox is reported + // pending by observe (via the readiness probe), and mapping it to Running here + // would resurface the "Pod Ready while the instance is still coming up" bug. + cases := map[string]provider.InstanceState{ + "running": provider.InstanceRunning, + "ready": provider.InstanceRunning, + "pending": provider.InstancePending, + "starting": provider.InstancePending, + "queued": provider.InstancePending, + "": provider.InstancePending, + "terminated": provider.InstanceTerminated, + "stopped": provider.InstanceTerminated, + "completed": provider.InstanceTerminated, + "error": provider.InstanceFailed, + "failed": provider.InstanceFailed, + "weird-new": provider.InstancePending, // unknown => keep watching, not terminal + } + for in, want := range cases { + if got := toState(in); got != want { + t.Fatalf("toState(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go new file mode 100644 index 0000000..3a0261a --- /dev/null +++ b/pkg/provider/provider.go @@ -0,0 +1,195 @@ +// Package provider defines the abstraction every compute provider (RunPod, +// Modal, Kubernetes, ...) implements. It is the single narrow seam between +// Nebula's provider-agnostic control plane (placement controller, NodeClaim +// controller, poll loop) and the heterogeneous cloud APIs underneath. +// +// Scope: v1 targets NeoClouds (RunPod, Modal, CoreWeave, Lambda), which are +// region-simple, so region/zone is intentionally NOT modeled yet. Hyperscalers +// (AWS/GCP/Azure) are a planned near-term expansion; when they land, Region/Zone +// become additive fields on the request/Offering/BlockScope structs and the +// optimizer's candidate key widens to include them — the method signatures here +// are designed not to change. Do not hard-code NeoCloud-only assumptions into +// the control plane; keep provider quirks behind Capabilities. +// +// Design rules learned from SkyPilot / the Nebula design discussion: +// - The Pod is the source of truth for the workload shape. Provision takes the +// Pod and reads image/command/env/ports/resources and the accelerators +// annotation off it; the control plane never re-encodes that into a provider +// call itself. +// - Providers differ in capabilities (RunPod cannot stop, has no native tags, +// no preemption push). Those quirks are declared via Capabilities and +// handled here, not leaked into the control plane. +// - Detection is poll-based everywhere (no provider gives a reliable +// preemption push, and even those with a spot notice are polled uniformly), +// so List must return all of this provider's instances in as few calls as +// possible. +package provider + +import ( + "context" + "time" + + corev1 "k8s.io/api/core/v1" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +// Provider is one compute-provider backend. Implementations are registered by name +// (matching NodePool.spec.providers[].name and the ProviderLabel on the virtual +// node). All methods must be safe for concurrent use. +type Provider interface { + // Name is the stable identifier, e.g. "runpod". + Name() string + + // Capabilities declares provider quirks so the control plane can behave + // generically instead of branching on provider name. + Capabilities() Capabilities + + // --- Lifecycle ------------------------------------------------------- + + // Provision creates exactly one external instance for the given Pod. The Pod + // is the source of truth for the entire workload shape: the implementation + // reads image/command/env/ports/cpu/memory from pod.Spec, the accelerator type + // from the accelerator-type label, and the accelerator count from the + // nvidia.com/gpu resource (via util.AcceleratorRequest; the type is then + // translated via MapAccelerator). The request carries + // only what the Pod cannot express: the optimizer-chosen capacity tier and + // the claim identity. It returns the provider instance id on success. + // Idempotency: if an instance already exists for req.ClaimName (encoded in + // the provider's naming scheme, since most providers lack tags), return that + // id instead of creating a second. + Provision(ctx context.Context, pod *corev1.Pod, req ProvisionRequest) (instanceID string, err error) + + // Terminate destroys the instance by id. Must be idempotent: terminating an + // already-gone instance returns nil (so the NodeClaim finalizer can retry + // safely). This is the call the terminate finalizer relies on to never leak + // a paid instance. + Terminate(ctx context.Context, instanceID string) error + + // Get returns the current state of one instance, or (nil, nil) if it no + // longer exists (treat absence as terminated). + Get(ctx context.Context, instanceID string) (*Instance, error) + + // List returns every instance owned by Nebula on this provider, in as few + // API calls as possible (ideally one — e.g. RunPod's myPods). This is the + // engine of the poll loop: preemption/termination is detected by an instance + // disappearing or changing state here, since no provider pushes such events. + List(ctx context.Context) ([]Instance, error) + + // --- Catalog --------------------------------------------------------- + + // Offerings returns the current price/availability rows this provider can + // serve, feeding the optimizer's {provider,accelerator,capacityType}-> + // {price,avail} table. Cached + periodically refreshed by the caller; + // implementations may combine a static catalog with a live availability probe. + Offerings(ctx context.Context) ([]Offering, error) + + // --- Translation ----------------------------------------------------- + + // MapAccelerator translates a canonical Nebula accelerator type ("H100", + // "A100-80GB", "TPU-v4") into this provider's identifier (e.g. RunPod's + // "NVIDIA H100 80GB HBM3"). Returns ok=false if the provider does not offer + // that accelerator. + MapAccelerator(canonical string) (providerAcceleratorID string, ok bool) + + // ClassifyProvisionError maps a Provision error to the granularity at which + // the failing placement should be blocklisted. This keeps failover precise: + // a "no H100 capacity" error blocks only {provider, H100, capacityType}, + // while an auth/quota error blocks the whole provider. See BlockScope. + ClassifyProvisionError(err error) BlockScope +} + +// ProvisionRequest carries only the decisions the placement controller made +// that are NOT already on the Pod. Everything about the workload — image, +// command, env, ports, cpu/memory, accelerator type (accelerator-type label) and +// count (nvidia.com/gpu resource) — is read from the Pod itself, which +// is the single source of truth; duplicating it here would repeat the mistake +// NodeClaim deliberately avoids. That leaves exactly two fields: the optimizer's +// capacity-tier choice (nowhere on the Pod) and the claim identity. +type ProvisionRequest struct { + // ClaimName is the NodeClaim name; providers without native tags encode it + // into the instance name so List/Terminate can find the instance later. + ClaimName string + // CapacityType is the tier the optimizer selected (Spot/OnDemand/Reserved). + // This is the one workload-independent decision that cannot be expressed on + // the Pod, so it must be passed explicitly. + CapacityType nebulav1alpha1.CapacityType +} + +// Capabilities declares provider quirks as data, so the control plane filters +// and behaves generically rather than branching on provider name. +type Capabilities struct { + // SupportsStop is true if instances can be stopped/resumed (RunPod: false — + // lifecycle is create/terminate only). + SupportsStop bool + // SupportsSpot is true if the provider offers interruptible capacity. + SupportsSpot bool + // NativeTags is true if the provider has real instance tags/labels; when + // false, identity is encoded in the instance name (RunPod: false). + NativeTags bool + // PreemptionNotice is the advance warning before a spot reclaim; zero means + // none (RunPod: 0 — abrupt, detected only by polling). + PreemptionNotice time.Duration + // PollInterval is how often the virtual node re-lists this provider to detect + // out-of-band state changes (Pending→Running, preemption, external teardown); + // see pkg/vnode. It is a per-provider knob because the trade-off differs by + // provider: a spot-heavy backend where preemption is common and costly wants a + // short interval to notice reclaims quickly, while an OnDemand-only backend + // that never preempts can poll lazily. Zero means "use the vnode default" + // (30s). The happy-path transitions (provision start, teardown) do not wait on + // this — they are pushed synchronously by CreatePod/DeletePod — so this only + // bounds detection latency for events no provider pushes. + PollInterval time.Duration +} + +// Instance is the provider-agnostic view of one external instance, as observed. +type Instance struct { + ID string + ClaimName string // recovered from the naming scheme (for tag-less providers) + State InstanceState + // Endpoint is the reachable address once ready (e.g. SSH host:port). + Endpoint string + // CapacityType reflects how the instance was provisioned, when known. + CapacityType nebulav1alpha1.CapacityType +} + +// InstanceState is the provider-agnostic lifecycle state, normalized from each +// provider's own status strings. +type InstanceState string + +const ( + // InstancePending: created but not yet reachable/ready. + InstancePending InstanceState = "Pending" + // InstanceRunning: up and reachable. + InstanceRunning InstanceState = "Running" + // InstanceTerminated: gone (also the mapping for "absent from List"). + InstanceTerminated InstanceState = "Terminated" + // InstanceFailed: entered a terminal error state. + InstanceFailed InstanceState = "Failed" +) + +// Offering is one row of the price/availability catalog. The price/availability +// lookup seam and the embeddable catalog-backed base (Name/Offerings/default +// MapAccelerator) live in the pkg/provider/catalog package, alongside the +// concrete CSV catalog, so all catalog-shaped types share one home. +type Offering struct { + AcceleratorType string + CapacityType nebulav1alpha1.CapacityType + PricePerHour float64 + Available bool +} + +// BlockScope is the granularity at which a failed placement is excluded, matched +// to what actually failed (SkyPilot's blocklist-granularity rule). Empty fields +// act as wildcards in the in-memory blocklist match: a failed H100 request must +// not disqualify A100 requests on the same provider. +type BlockScope struct { + // AcceleratorType empty => blocks all accelerator types on this provider. + AcceleratorType string + // CapacityType empty => blocks all capacity types. + CapacityType nebulav1alpha1.CapacityType + // DenyAll true => block everything on this provider (e.g. auth/quota errors), + // ignoring AcceleratorType/CapacityType. The scope is still this one provider; + // it never spans providers. + DenyAll bool +} diff --git a/pkg/provider/registry.go b/pkg/provider/registry.go new file mode 100644 index 0000000..776720d --- /dev/null +++ b/pkg/provider/registry.go @@ -0,0 +1,77 @@ +package provider + +import ( + "fmt" + "sort" + "sync" +) + +// The canonical provider names for the v1 NeoCloud set. These are the stable +// identifiers used everywhere a provider is referenced by string: a +// Provider.Name() return value, a NodePool.spec.providers[].name entry, and the +// ProviderLabel value on each provider's virtual node. Keeping them as consts in +// one place stops those three call sites from drifting apart on a typo. +// +// Hyperscalers (aws/gcp/azure) are a planned near-term expansion and will be +// added here when their adapters land; nothing else about the registry changes. +const ( + ProviderRunPod = "runpod" + ProviderModal = "modal" + ProviderCoreWeave = "coreweave" + ProviderLambda = "lambda" + ProviderKubernetes = "kubernetes" +) + +// registry is the process-wide map of registered provider backends, keyed by +// Provider.Name(). It is populated at startup (typically each adapter package +// calls Register from an init or a wiring function) and read concurrently by the +// placement controller, the poll loop and the NodeClaim controller, so it is +// guarded by a mutex. +var registry = struct { + sync.RWMutex + m map[string]Provider +}{m: make(map[string]Provider)} + +// Register adds a provider backend under its Name(). It panics on a duplicate +// name or a nil provider, because both are programmer errors that must surface +// at startup rather than as a silently-missing provider at placement time. +func Register(p Provider) { + if p == nil { + panic("provider: Register called with nil provider") + } + name := p.Name() + if name == "" { + panic("provider: Register called with empty provider name") + } + + registry.Lock() + defer registry.Unlock() + if _, dup := registry.m[name]; dup { + panic(fmt.Sprintf("provider: duplicate registration for %q", name)) + } + registry.m[name] = p +} + +// Get returns the registered provider for name, or ok=false if none is +// registered. Callers on the control-plane hot path (e.g. resolving a NodePool +// provider ref) should treat ok=false as a configuration error for that pool, +// not a fatal one — other pools may still be serviceable. +func Get(name string) (p Provider, ok bool) { + registry.RLock() + defer registry.RUnlock() + p, ok = registry.m[name] + return p, ok +} + +// Names returns the sorted names of all registered providers. Sorted so that +// callers (logs, status, tests) get a stable, deterministic order. +func Names() []string { + registry.RLock() + defer registry.RUnlock() + names := make([]string, 0, len(registry.m)) + for name := range registry.m { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/pkg/util/accelerator.go b/pkg/util/accelerator.go new file mode 100644 index 0000000..421bf2f --- /dev/null +++ b/pkg/util/accelerator.go @@ -0,0 +1,82 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +// NvidiaGPUResource is the extended-resource key a GPU count is expressed under. +// It mirrors the ecosystem-standard nvidia.com/gpu so a Pod's accelerator count +// drives both the scheduler's fit check (against the virtual node's advertised +// capacity) and provisioning from a single number. +const NvidiaGPUResource corev1.ResourceName = "nvidia.com/gpu" + +// AcceleratorRequest reads a Pod's accelerator request: the TYPE from the +// AcceleratorTypeLabel and the COUNT from the container's nvidia.com/gpu +// resource. This is the single source of truth for the request grammar, shared +// by the placement controller and the provider adapters. +// +// It returns: +// - ("", 0, nil) no GPU type requested => a CPU-only workload. +// - (type, count, nil) a GPU workload; count defaults to 1 when the type is +// set but no nvidia.com/gpu resource is present, so authors can request a +// single GPU with just the label. +// +// The type is returned verbatim (case is normalized downstream by the provider +// catalog's MapAccelerator). An error is returned only for a genuinely +// contradictory request: an explicit nvidia.com/gpu count with no GPU type. +func AcceleratorRequest(pod *corev1.Pod) (accelerator string, count int32, err error) { + typ := pod.Labels[nebulav1alpha1.AcceleratorTypeLabel] + n := gpuCount(pod) + + if typ == "" { + if n > 0 { + return "", 0, fmt.Errorf("nvidia.com/gpu=%d requested without a %s label", n, nebulav1alpha1.AcceleratorTypeLabel) + } + return "", 0, nil // CPU-only + } + if n == 0 { + n = 1 // a GPU type with no explicit count means one GPU + } + return typ, n, nil +} + +// gpuCount returns the largest nvidia.com/gpu quantity requested across the +// Pod's containers, preferring limits and falling back to requests (Kubernetes +// treats an extended resource's request and limit as equal, but a Pod may set +// only one). Returns 0 when no container asks for a GPU. +func gpuCount(pod *corev1.Pod) int32 { + var max int64 + for i := range pod.Spec.Containers { + c := &pod.Spec.Containers[i] + if q, ok := c.Resources.Limits[NvidiaGPUResource]; ok { + if v := q.Value(); v > max { + max = v + } + } else if q, ok := c.Resources.Requests[NvidiaGPUResource]; ok { + if v := q.Value(); v > max { + max = v + } + } + } + return int32(max) +} diff --git a/pkg/util/accelerator_test.go b/pkg/util/accelerator_test.go new file mode 100644 index 0000000..c7062d4 --- /dev/null +++ b/pkg/util/accelerator_test.go @@ -0,0 +1,101 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/api/resource" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +// acceleratorPod builds a Pod carrying an accelerator-type label (when non-empty) +// and a single container requesting the given nvidia.com/gpu count (when > 0). +func acceleratorPod(typ string, gpuLimit int64) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main"}}}, + } + if typ != "" { + pod.Labels = map[string]string{nebulav1alpha1.AcceleratorTypeLabel: typ} + } + if gpuLimit > 0 { + pod.Spec.Containers[0].Resources.Limits = corev1.ResourceList{ + NvidiaGPUResource: *resource.NewQuantity(gpuLimit, resource.DecimalSI), + } + } + return pod +} + +func TestAcceleratorRequest(t *testing.T) { + cases := []struct { + name string + typ string + gpuLimit int64 + wantAccel string + wantCount int32 + wantErr bool + }{ + {name: "no label is CPU-only", typ: "", gpuLimit: 0, wantAccel: "", wantCount: 0}, + {name: "type and count", typ: "a100-40gb", gpuLimit: 8, wantAccel: "a100-40gb", wantCount: 8}, + {name: "type only defaults to 1", typ: "h100", gpuLimit: 0, wantAccel: "h100", wantCount: 1}, + {name: "casing preserved", typ: "A100-80GB", gpuLimit: 2, wantAccel: "A100-80GB", wantCount: 2}, + {name: "count without type is contradictory", typ: "", gpuLimit: 4, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + accel, count, err := AcceleratorRequest(acceleratorPod(tc.typ, tc.gpuLimit)) + if tc.wantErr { + if err == nil { + t.Fatalf("AcceleratorRequest = (%q, %d, nil), want error", accel, count) + } + return + } + if err != nil { + t.Fatalf("AcceleratorRequest: unexpected error %v", err) + } + if accel != tc.wantAccel || count != tc.wantCount { + t.Fatalf("AcceleratorRequest = (%q, %d), want (%q, %d)", + accel, count, tc.wantAccel, tc.wantCount) + } + }) + } +} + +// TestAcceleratorRequest_CountFromRequestsWhenNoLimit verifies the fallback to +// resource requests when a container sets only requests (not limits). +func TestAcceleratorRequest_CountFromRequestsWhenNoLimit(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{nebulav1alpha1.AcceleratorTypeLabel: "h100"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "main", + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + NvidiaGPUResource: *resource.NewQuantity(3, resource.DecimalSI), + }}, + }}}, + } + accel, count, err := AcceleratorRequest(pod) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if accel != "h100" || count != 3 { + t.Fatalf("AcceleratorRequest = (%q, %d), want (h100, 3)", accel, count) + } +} diff --git a/pkg/util/claim.go b/pkg/util/claim.go new file mode 100644 index 0000000..2359cee --- /dev/null +++ b/pkg/util/claim.go @@ -0,0 +1,30 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package util holds small, dependency-free helpers shared across Nebula's +// control plane and provider adapters. +package util + +// ClaimName is the instance-identity token Nebula encodes into a provider +// instance's name/tag so List/Terminate can find it later without a durable id. +// It is a deterministic function of the served workload's namespace and name, so +// any component that knows the Pod (the virtual kubelet) or the claim's PodRef +// (the teardown backstop) computes the same token. Keep this the single source +// of truth for the convention — the vnode handler and the NodeClaim finalizer +// both depend on producing identical values. +func ClaimName(namespace, name string) string { + return namespace + "-" + name +} diff --git a/pkg/version/version.go b/pkg/version/version.go new file mode 100644 index 0000000..0ad3d02 --- /dev/null +++ b/pkg/version/version.go @@ -0,0 +1,42 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package version exposes the build version of the Nebula manager. The value is +// stamped at build time via -ldflags (see the Makefile LDFLAGS and Dockerfile); +// an unstamped build (e.g. `go run`, `go test`) reports the devFallback below. +package version + +// gitVersion is overridden at link time with: +// +// -ldflags "-X github.com/InftyAI/Nebula/pkg/version.gitVersion=" +// +// Left lowercase/unexported so it can only be set via ldflags or read through +// Get(), never reassigned at runtime. +var gitVersion = devFallback + +// devFallback is what an unstamped build reports. It is deliberately not a +// version-looking string, so a "nebula-dev" in the field is an obvious signal +// the binary was built without the version ldflag. +const devFallback = "nebula-dev" + +// Get returns the build version, e.g. "v0.1.0", "v0.1.0-3-gabc123" (git +// describe of an untagged commit), or "nebula-dev" for an unstamped build. +func Get() string { + if gitVersion == "" { + return devFallback + } + return gitVersion +} diff --git a/pkg/vnode/doc.go b/pkg/vnode/doc.go new file mode 100644 index 0000000..c2e6c65 --- /dev/null +++ b/pkg/vnode/doc.go @@ -0,0 +1,29 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package vnode implements the Virtual Kubelet integration: one static virtual +// Node per provider, whose PodLifecycleHandler provisions/terminates external +// instances through the provider seam. See docs/architecture.md §3. +// +// Ownership model ("VK owns provisioning"): the pod controller's CreatePod calls +// provider.Provision and DeletePod calls provider.Terminate directly. The Pod is +// the single source of truth for the workload; the only provisioning input that +// is not on the Pod — the optimizer's capacity tier — rides on the +// CapacityTypeAnnotation, written by the placement controller when it ungates +// the Pod. Instance identity is derived deterministically from the Pod +// (ClaimName), so a provider whose List reports the claim tag can recover and +// reclaim an instance across a controller restart without a durable ledger. +package vnode diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go new file mode 100644 index 0000000..5347f4e --- /dev/null +++ b/pkg/vnode/handler.go @@ -0,0 +1,341 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vnode + +import ( + "context" + "io" + "sync" + "time" + + dto "github.com/prometheus/client_model/go" + "github.com/virtual-kubelet/virtual-kubelet/errdefs" + vknode "github.com/virtual-kubelet/virtual-kubelet/node" + vkapi "github.com/virtual-kubelet/virtual-kubelet/node/api" + "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/util" +) + +// defaultPollInterval is how often the notifier re-lists the provider to detect +// state changes (ready, preemption, disappearance). Detection is poll-based +// everywhere — no provider pushes preemption/termination — so this is the +// resolution at which the virtual node notices instance state transitions. It is +// only the fallback: a provider may override the cadence via +// Capabilities.PollInterval (e.g. a spot-heavy backend polls faster to notice +// reclaims sooner; an OnDemand-only one can poll lazily). +const defaultPollInterval = 30 * time.Second + +// Handler bridges one provider into the virtual kubelet: it implements the VK +// PodLifecycleHandler (+ PodNotifier) so the pod controller's CreatePod +// provisions an external instance through the provider seam and DeletePod +// terminates it. This is the "VK owns provisioning" model — there is no separate +// controller issuing Provision/Terminate. +// +// Leak-safety: the pod controller only calls DeletePod for a pod it is tracking, +// and CreatePod records the instance id before returning success, so a paid +// instance is always reachable for teardown. Provision is idempotent on +// ClaimName, so a CreatePod retried after a crash between provider-create and +// status-write adopts the existing instance rather than creating a second. +type Handler struct { + prov provider.Provider + + mu sync.Mutex + tracked map[string]*trackedPod // key: namespace/name + notify func(*corev1.Pod) + + // nowFn and pollEvery are seams for tests. + nowFn func() metav1.Time + pollEvery time.Duration +} + +// trackedPod is the virtual node's local record of one pod it provisioned for. +// The Pod object is the source of truth for the workload; we additionally hold +// the provider instance id (for Terminate) and the derived claim name (to match +// this pod against the provider's List during polling). +type trackedPod struct { + pod *corev1.Pod + claimName string + instance string +} + +// NewHandler builds a Handler for the given provider backend. The poll cadence +// comes from the provider's Capabilities.PollInterval, falling back to +// defaultPollInterval when the provider does not set one. +func NewHandler(prov provider.Provider) *Handler { + poll := prov.Capabilities().PollInterval + if poll <= 0 { + poll = defaultPollInterval + } + return &Handler{ + prov: prov, + tracked: make(map[string]*trackedPod), + nowFn: metav1.Now, + pollEvery: poll, + } +} + +// Compile-time proof the Handler satisfies the VK interfaces we rely on. +var ( + _ vknode.PodLifecycleHandler = (*Handler)(nil) + _ vknode.PodNotifier = (*Handler)(nil) +) + +// claimName derives a stable instance identity from the Pod. The Pod is the +// source of truth, so the claim — which providers encode into the instance +// name/tag for tag-less recovery — must be a deterministic function of it. +// namespace/name is stable across reconciles; the instance is always torn down +// on DeletePod, so name reuse after deletion is not a concern. It delegates to +// util.ClaimName so the vnode handler and the NodeClaim teardown backstop +// produce identical tokens. +func claimName(pod *corev1.Pod) string { + return util.ClaimName(pod.Namespace, pod.Name) +} + +func key(namespace, name string) string { return namespace + "/" + name } + +// CreatePod provisions an external instance for the Pod through the provider. +// The Pod carries the whole workload shape; the only out-of-band input, the +// optimizer's capacity tier, rides on CapacityTypeAnnotation. +func (h *Handler) CreatePod(ctx context.Context, pod *corev1.Pod) error { + claim := claimName(pod) + req := provider.ProvisionRequest{ + ClaimName: claim, + CapacityType: nebulav1alpha1.CapacityType(pod.Annotations[nebulav1alpha1.CapacityTypeAnnotation]), + } + + id, err := h.prov.Provision(ctx, pod, req) + if err != nil { + // Surface the failure on the Pod so the placement controller can fail + // over, and return the error so the pod controller retries with backoff. + h.markStatus(pod, corev1.PodFailed, reasonProvisionFailed, err.Error()) + h.store(pod, claim, "") + h.emit(pod) + return err + } + + // Record the instance id before reporting success; teardown relies on it. + h.markStatus(pod, corev1.PodPending, reasonProvisioning, "provisioning external instance") + h.store(pod, claim, id) + h.emit(pod) + return nil +} + +// UpdatePod is a no-op: the external instance's shape is immutable once +// provisioned (recovery from any change is delete-and-recreate, matching the +// NodeClaim ledger's immutability). We still refresh our tracked copy so +// GetPod reflects the latest metadata. +func (h *Handler) UpdatePod(_ context.Context, pod *corev1.Pod) error { + h.mu.Lock() + defer h.mu.Unlock() + if tp, ok := h.tracked[key(pod.Namespace, pod.Name)]; ok { + // Preserve the status we compute from the provider; only adopt spec/meta. + status := tp.pod.Status + tp.pod = pod.DeepCopy() + tp.pod.Status = status + } + return nil +} + +// DeletePod terminates the external instance and drops the pod from tracking. +// Terminate is idempotent, so a repeated DeletePod (VK may call it more than +// once) is safe. +func (h *Handler) DeletePod(ctx context.Context, pod *corev1.Pod) error { + h.mu.Lock() + tp, ok := h.tracked[key(pod.Namespace, pod.Name)] + instance := "" + if ok { + instance = tp.instance + } + h.mu.Unlock() + + if err := h.prov.Terminate(ctx, instance); err != nil { + return err + } + + // Report a terminal status, then forget the pod. VK expects the containers + // and pod to reach a terminal state after DeletePod. + h.markStatus(pod, corev1.PodSucceeded, "Terminated", "external instance terminated") + pod.DeletionTimestamp = ptrNow(h.nowFn()) + h.emit(pod) + + h.mu.Lock() + delete(h.tracked, key(pod.Namespace, pod.Name)) + h.mu.Unlock() + return nil +} + +// GetPod returns the tracked pod, or a NotFound error the pod controller +// understands. +func (h *Handler) GetPod(_ context.Context, namespace, name string) (*corev1.Pod, error) { + h.mu.Lock() + defer h.mu.Unlock() + tp, ok := h.tracked[key(namespace, name)] + if !ok { + return nil, errdefs.NotFoundf("pod %s/%s not found on virtual node", namespace, name) + } + return tp.pod.DeepCopy(), nil +} + +// GetPodStatus returns the tracked pod's status. +func (h *Handler) GetPodStatus(_ context.Context, namespace, name string) (*corev1.PodStatus, error) { + h.mu.Lock() + defer h.mu.Unlock() + tp, ok := h.tracked[key(namespace, name)] + if !ok { + return nil, errdefs.NotFoundf("pod %s/%s not found on virtual node", namespace, name) + } + return tp.pod.Status.DeepCopy(), nil +} + +// GetPods returns every pod this virtual node is tracking. +func (h *Handler) GetPods(_ context.Context) ([]*corev1.Pod, error) { + h.mu.Lock() + defer h.mu.Unlock() + pods := make([]*corev1.Pod, 0, len(h.tracked)) + for _, tp := range h.tracked { + pods = append(pods, tp.pod.DeepCopy()) + } + return pods, nil +} + +// NotifyPods registers the async status callback and starts the poll loop. VK +// calls this once at startup; the loop runs until ctx is cancelled. +func (h *Handler) NotifyPods(ctx context.Context, cb func(*corev1.Pod)) { + h.mu.Lock() + h.notify = cb + h.mu.Unlock() + + go h.pollLoop(ctx) +} + +// pollLoop periodically reconciles tracked pods against the provider's live +// instance list and pushes any status change through the notify callback. +func (h *Handler) pollLoop(ctx context.Context) { + t := time.NewTicker(h.pollEvery) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + h.reconcileOnce(ctx) + } + } +} + +// reconcileOnce lists the provider once and updates every tracked pod's status +// from the matching instance (matched by claim name, since a pod maps 1:1 to a +// claim). A tracked pod whose instance is absent from the list is treated as +// terminated (preempted / externally torn down), per the provider contract. +func (h *Handler) reconcileOnce(ctx context.Context) { + instances, err := h.prov.List(ctx) + if err != nil { + return // transient; retry on the next tick + } + byClaim := make(map[string]provider.Instance, len(instances)) + for _, inst := range instances { + byClaim[inst.ClaimName] = inst + } + + h.mu.Lock() + changed := make([]*corev1.Pod, 0) + for _, tp := range h.tracked { + inst, present := byClaim[tp.claimName] + before := tp.pod.Status.Phase + if !present { + applyState(tp.pod, provider.InstanceTerminated, "", h.nowFn()) + } else { + applyState(tp.pod, inst.State, inst.Endpoint, h.nowFn()) + } + if tp.pod.Status.Phase != before { + changed = append(changed, tp.pod.DeepCopy()) + } + } + notify := h.notify + h.mu.Unlock() + + if notify != nil { + for _, p := range changed { + notify(p) + } + } +} + +// store records/updates the tracked pod under lock. +func (h *Handler) store(pod *corev1.Pod, claim, instance string) { + h.mu.Lock() + defer h.mu.Unlock() + h.tracked[key(pod.Namespace, pod.Name)] = &trackedPod{ + pod: pod.DeepCopy(), + claimName: claim, + instance: instance, + } +} + +// emit pushes a status update through the notify callback if one is registered. +func (h *Handler) emit(pod *corev1.Pod) { + h.mu.Lock() + notify := h.notify + h.mu.Unlock() + if notify != nil { + notify(pod.DeepCopy()) + } +} + +// markStatus sets a coarse pod phase and a single readiness-style condition on +// the passed Pod (which VK then reports to the API server). +func (h *Handler) markStatus(pod *corev1.Pod, phase corev1.PodPhase, reason, msg string) { + setPhase(pod, phase, reason, msg, h.nowFn()) +} + +func ptrNow(t metav1.Time) *metav1.Time { return &t } + +// --- Unused nodeutil.Provider surface -------------------------------------- +// +// The kubelet API (logs/exec/attach/stats/port-forward) is not part of the v1 +// scope: Nebula routes external GPU workloads, it does not proxy their consoles. +// These satisfy the nodeutil.Provider interface required by nodeutil.NewNode and +// return a NotFound so the VK core reports them cleanly rather than panicking. + +func (h *Handler) GetContainerLogs(context.Context, string, string, string, vkapi.ContainerLogOpts) (io.ReadCloser, error) { + return nil, errdefs.NotFound("container logs are not supported by the Nebula virtual node") +} + +func (h *Handler) RunInContainer(context.Context, string, string, string, []string, vkapi.AttachIO) error { + return errdefs.NotFound("exec is not supported by the Nebula virtual node") +} + +func (h *Handler) AttachToContainer(context.Context, string, string, string, vkapi.AttachIO) error { + return errdefs.NotFound("attach is not supported by the Nebula virtual node") +} + +func (h *Handler) GetStatsSummary(context.Context) (*statsv1alpha1.Summary, error) { + return nil, errdefs.NotFound("stats are not supported by the Nebula virtual node") +} + +func (h *Handler) GetMetricsResource(context.Context) ([]*dto.MetricFamily, error) { + return nil, errdefs.NotFound("resource metrics are not supported by the Nebula virtual node") +} + +func (h *Handler) PortForward(context.Context, string, string, int32, io.ReadWriteCloser) error { + return errdefs.NotFound("port-forward is not supported by the Nebula virtual node") +} diff --git a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go new file mode 100644 index 0000000..fe78a50 --- /dev/null +++ b/pkg/vnode/handler_test.go @@ -0,0 +1,251 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vnode + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/virtual-kubelet/virtual-kubelet/errdefs" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" +) + +// fakeProvider records lifecycle calls and returns canned results so each +// Handler branch can be driven deterministically. +type fakeProvider struct { + mu sync.Mutex + provisionID string + provisionErr error + provisionCnt int + lastReq provider.ProvisionRequest + terminateCnt int + terminateID string + terminateErr error + list []provider.Instance + listErr error + capabilities provider.Capabilities +} + +func (f *fakeProvider) Name() string { return "fake" } +func (f *fakeProvider) Capabilities() provider.Capabilities { return f.capabilities } + +func (f *fakeProvider) Provision(_ context.Context, _ *corev1.Pod, req provider.ProvisionRequest) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.provisionCnt++ + f.lastReq = req + return f.provisionID, f.provisionErr +} + +func (f *fakeProvider) Terminate(_ context.Context, id string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.terminateCnt++ + f.terminateID = id + return f.terminateErr +} + +func (f *fakeProvider) Get(context.Context, string) (*provider.Instance, error) { return nil, nil } +func (f *fakeProvider) List(context.Context) ([]provider.Instance, error) { + return f.list, f.listErr +} +func (f *fakeProvider) Offerings(context.Context) ([]provider.Offering, error) { return nil, nil } +func (f *fakeProvider) MapAccelerator(c string) (string, bool) { return c, true } +func (f *fakeProvider) ClassifyProvisionError(error) provider.BlockScope { + return provider.BlockScope{} +} + +func testPod(ns, name string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "img"}}}, + } +} + +func TestCreatePod_ProvisionsAndTracks(t *testing.T) { + fp := &fakeProvider{provisionID: "inst-1"} + h := NewHandler(fp) + pod := testPod("default", "p1") + pod.Annotations = map[string]string{nebulav1alpha1.CapacityTypeAnnotation: string(nebulav1alpha1.CapacitySpot)} + + if err := h.CreatePod(context.Background(), pod); err != nil { + t.Fatalf("CreatePod: %v", err) + } + if fp.provisionCnt != 1 { + t.Fatalf("expected 1 provision, got %d", fp.provisionCnt) + } + if fp.lastReq.ClaimName != "default-p1" { + t.Fatalf("expected claim name default-p1, got %q", fp.lastReq.ClaimName) + } + if fp.lastReq.CapacityType != nebulav1alpha1.CapacitySpot { + t.Fatalf("expected capacity type read from annotation, got %q", fp.lastReq.CapacityType) + } + + got, err := h.GetPod(context.Background(), "default", "p1") + if err != nil { + t.Fatalf("GetPod: %v", err) + } + if got.Status.Phase != corev1.PodPending { + t.Fatalf("expected Pending after provision, got %q", got.Status.Phase) + } +} + +func TestCreatePod_ProvisionErrorSurfaces(t *testing.T) { + fp := &fakeProvider{provisionErr: errors.New("no capacity")} + h := NewHandler(fp) + pod := testPod("default", "p1") + + if err := h.CreatePod(context.Background(), pod); err == nil { + t.Fatal("expected CreatePod to return the provision error") + } + // The pod is tracked with a Failed status so state is observable / retriable. + got, err := h.GetPod(context.Background(), "default", "p1") + if err != nil { + t.Fatalf("GetPod: %v", err) + } + if got.Status.Phase != corev1.PodFailed { + t.Fatalf("expected Failed after provision error, got %q", got.Status.Phase) + } +} + +func TestDeletePod_TerminatesAndUntracks(t *testing.T) { + fp := &fakeProvider{provisionID: "inst-1"} + h := NewHandler(fp) + pod := testPod("default", "p1") + + if err := h.CreatePod(context.Background(), pod); err != nil { + t.Fatalf("CreatePod: %v", err) + } + if err := h.DeletePod(context.Background(), pod); err != nil { + t.Fatalf("DeletePod: %v", err) + } + if fp.terminateCnt != 1 { + t.Fatalf("expected 1 terminate, got %d", fp.terminateCnt) + } + if fp.terminateID != "inst-1" { + t.Fatalf("expected terminate of recorded instance id, got %q", fp.terminateID) + } + if _, err := h.GetPod(context.Background(), "default", "p1"); !errdefs.IsNotFound(err) { + t.Fatalf("expected NotFound after delete, got %v", err) + } +} + +func TestDeletePod_Idempotent(t *testing.T) { + fp := &fakeProvider{provisionID: "inst-1"} + h := NewHandler(fp) + pod := testPod("default", "p1") + _ = h.CreatePod(context.Background(), pod) + + _ = h.DeletePod(context.Background(), pod) + // A second DeletePod (VK may call more than once) must not error; Terminate is + // idempotent and there is no longer a tracked instance id. + if err := h.DeletePod(context.Background(), pod); err != nil { + t.Fatalf("second DeletePod: %v", err) + } +} + +func TestReconcileOnce_ReportsRunning(t *testing.T) { + fp := &fakeProvider{provisionID: "inst-1"} + h := NewHandler(fp) + pod := testPod("default", "p1") + _ = h.CreatePod(context.Background(), pod) + + // Capture notifications. + var mu sync.Mutex + var notified []*corev1.Pod + h.NotifyPods(context.Background(), func(p *corev1.Pod) { + mu.Lock() + notified = append(notified, p) + mu.Unlock() + }) + + // Provider now reports the instance running under the derived claim name. + fp.list = []provider.Instance{{ + ID: "inst-1", ClaimName: "default-p1", State: provider.InstanceRunning, Endpoint: "5.6.7.8", + }} + h.reconcileOnce(context.Background()) + + got, err := h.GetPod(context.Background(), "default", "p1") + if err != nil { + t.Fatalf("GetPod: %v", err) + } + if got.Status.Phase != corev1.PodRunning { + t.Fatalf("expected Running after poll, got %q", got.Status.Phase) + } + if got.Status.PodIP != "5.6.7.8" { + t.Fatalf("expected endpoint set as PodIP, got %q", got.Status.PodIP) + } + + mu.Lock() + defer mu.Unlock() + if len(notified) == 0 { + t.Fatal("expected a status notification on the running transition") + } +} + +func TestReconcileOnce_AbsentInstanceIsTerminated(t *testing.T) { + fp := &fakeProvider{provisionID: "inst-1"} + h := NewHandler(fp) + pod := testPod("default", "p1") + _ = h.CreatePod(context.Background(), pod) + + // Provider list is empty => the instance disappeared => reported Terminated. + fp.list = nil + h.reconcileOnce(context.Background()) + + got, _ := h.GetPod(context.Background(), "default", "p1") + if got.Status.Phase != corev1.PodFailed { + t.Fatalf("expected Failed when instance absent, got %q", got.Status.Phase) + } + if got.Status.Reason != "Terminated" { + t.Fatalf("expected Terminated reason, got %q", got.Status.Reason) + } +} + +func TestNewHandler_PollIntervalFromCapabilities(t *testing.T) { + // A provider that declares a cadence overrides the default. + custom := &fakeProvider{capabilities: provider.Capabilities{PollInterval: 5 * time.Second}} + if got := NewHandler(custom).pollEvery; got != 5*time.Second { + t.Fatalf("expected the provider's PollInterval, got %v", got) + } + // A provider that leaves it zero falls back to the vnode default. + if got := NewHandler(&fakeProvider{}).pollEvery; got != defaultPollInterval { + t.Fatalf("expected the default cadence, got %v", got) + } +} + +func TestGetPods_ReturnsTracked(t *testing.T) { + fp := &fakeProvider{provisionID: "inst-1"} + h := NewHandler(fp) + _ = h.CreatePod(context.Background(), testPod("default", "p1")) + _ = h.CreatePod(context.Background(), testPod("default", "p2")) + + pods, err := h.GetPods(context.Background()) + if err != nil { + t.Fatalf("GetPods: %v", err) + } + if len(pods) != 2 { + t.Fatalf("expected 2 tracked pods, got %d", len(pods)) + } +} diff --git a/pkg/vnode/node.go b/pkg/vnode/node.go new file mode 100644 index 0000000..b4086a9 --- /dev/null +++ b/pkg/vnode/node.go @@ -0,0 +1,306 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vnode + +import ( + "context" + "fmt" + "time" + + vknode "github.com/virtual-kubelet/virtual-kubelet/node" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/version" +) + +// informerResync is the full-resync period for the virtual node's informers. +const informerResync = time.Minute + +// NodeName returns the virtual node name for a provider: "nebula-". +// One static node per provider (see docs/architecture.md §3); the scheduler +// routes an ungated Pod to it via the ProviderLabel nodeSelector. +func NodeName(providerName string) string { + return "nebula-" + providerName +} + +// RBAC for the virtual kubelet. The VK pod controller reports Pod status back to +// the API server and reacts to the config/secret/service objects a Pod +// references; the node controller creates/maintains the virtual Node and its +// node lease and emits events. These grants back the node/pod controllers wired +// in Start. +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;update;patch;delete +// +kubebuilder:rbac:groups="",resources=pods/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=nodes/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=configmaps;secrets;services,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch +// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete + +// Runner is a manager.Runnable that owns one provider's virtual node. It starts +// the VK node/pod controllers and blocks until the manager's context is +// cancelled, so it participates in the manager's lifecycle (and, if enabled, +// leader election) like any other runnable. +// +// It deliberately wires the lower-level virtual-kubelet `node` package directly +// rather than the `nodeutil` convenience wrapper: nodeutil pulls in a kubelet +// HTTP/auth stack whose apiserver dependency does not compile against the k8s +// 0.33 line this project pins. Nebula does not serve the kubelet API (logs/exec +// are unsupported — see handler.go), so none of that surface is needed. +type Runner struct { + prov provider.Provider + client kubernetes.Interface + nodeName string +} + +// NewRunner builds the virtual-node runner for one provider. +func NewRunner(prov provider.Provider, client kubernetes.Interface) *Runner { + return &Runner{ + prov: prov, + client: client, + nodeName: NodeName(prov.Name()), + } +} + +var _ manager.Runnable = (*Runner)(nil) + +// Start builds and runs the virtual node until ctx is cancelled. It is invoked +// by the controller-runtime manager. +func (r *Runner) Start(ctx context.Context) error { + log := logf.FromContext(ctx).WithValues("virtualNode", r.nodeName, "provider", r.prov.Name()) + + handler := NewHandler(r.prov) + nodeSpec := nodeSpec(r.nodeName, r.prov.Name()) + + // Pod informer scoped to this node only (spec.nodeName == nodeName), so the + // pod controller reacts to exactly the Pods the scheduler bound here. + podFactory := informers.NewSharedInformerFactoryWithOptions( + r.client, informerResync, + informers.WithTweakListOptions(func(o *metav1.ListOptions) { + o.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", r.nodeName).String() + }), + ) + // Cluster-wide factory for the config/secret/service informers the pod + // controller needs to resolve pod references. + scmFactory := informers.NewSharedInformerFactoryWithOptions(r.client, informerResync) + + podInformer := podFactory.Core().V1().Pods() + secretInformer := scmFactory.Core().V1().Secrets() + configMapInformer := scmFactory.Core().V1().ConfigMaps() + serviceInformer := scmFactory.Core().V1().Services() + + eb := record.NewBroadcaster() + recorder := eb.NewRecorder(scheme.Scheme, corev1.EventSource{Component: r.nodeName + "/pod-controller"}) + + pc, err := vknode.NewPodController(vknode.PodControllerConfig{ + PodClient: r.client.CoreV1(), + EventRecorder: recorder, + Provider: handler, + PodInformer: podInformer, + SecretInformer: secretInformer, + ConfigMapInformer: configMapInformer, + ServiceInformer: serviceInformer, + }) + if err != nil { + return fmt.Errorf("build pod controller for %q: %w", r.nodeName, err) + } + + // A NaiveNodeProvider marks the node Ready and answers pings; our Handler + // owns only pod lifecycle, not node health. + np := vknode.NewNaiveNodeProvider() + nc, err := vknode.NewNodeController( + np, nodeSpec, r.client.CoreV1().Nodes(), + vknode.WithNodeEnableLeaseV1(r.client.CoordinationV1().Leases(corev1.NamespaceNodeLease), vknode.DefaultLeaseDuration), + ) + if err != nil { + return fmt.Errorf("build node controller for %q: %w", r.nodeName, err) + } + + eb.StartLogging(func(format string, args ...interface{}) { log.Info(fmt.Sprintf(format, args...)) }) + defer eb.Shutdown() + + go podFactory.Start(ctx.Done()) + go scmFactory.Start(ctx.Done()) + + log.Info("starting virtual node") + if err := r.run(ctx, pc, nc, nodeSpec, np); err != nil && ctx.Err() == nil { + return err + } + log.Info("virtual node stopped") + return nil +} + +// run starts the pod and node controllers in the required order (pods ready +// before the node advertises Ready) and blocks until ctx is cancelled or a +// controller exits. +func (r *Runner) run(ctx context.Context, pc *vknode.PodController, nc *vknode.NodeController, nodeSpec *corev1.Node, np *vknode.NaiveNodeProviderV2) error { + go pc.Run(ctx, 1) //nolint:errcheck + + select { + case <-ctx.Done(): + return nil + case <-pc.Ready(): + case <-pc.Done(): + return pc.Err() + } + + go nc.Run(ctx) //nolint:errcheck + + select { + case <-ctx.Done(): + return nil + case <-nc.Ready(): + case <-nc.Done(): + return nc.Err() + } + + // Mark the node Ready now that both controllers are up. + markNodeReady(nodeSpec) + if err := np.UpdateStatus(ctx, nodeSpec); err != nil { + return fmt.Errorf("mark virtual node ready: %w", err) + } + + select { + case <-ctx.Done(): + return nil + case <-nc.Done(): + return nc.Err() + case <-pc.Done(): + return pc.Err() + } +} + +// nodeSpec produces the Node object for a provider's virtual node: the +// ProviderLabel the placement controller selects on, and a NoSchedule taint so +// only Nebula-placed Pods (which the placement controller tolerates) ever land +// here. +func nodeSpec(nodeName, providerName string) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{ + // hostname is a well-known label the scheduler's default topology + // spread keys off. ProviderLabel routes a Pod to this provider's node + // (the placement controller's nodeSelector, paired with the NoSchedule + // taint of the same key below). ManagedByLabel is the management-scoped + // "this object is Nebula's" marker for tooling, policies, and queries. + corev1.LabelHostname: nodeName, + nebulav1alpha1.ProviderLabel: providerName, + nebulav1alpha1.ManagedByLabel: nebulav1alpha1.ManagedByValue, + // Populate the ROLES column of `kubectl get nodes`, which is derived + // solely from node-role.kubernetes.io/ label KEYS (the value is + // ignored, so it is conventionally ""). Two roles are advertised: + // "worker" so the node reads as a schedulable worker like any other (it + // runs Pods), and "nebula" so Nebula's virtual nodes are identifiable at + // a glance rather than blending in with real workers. + "node-role.kubernetes.io/worker": "", + "node-role.kubernetes.io/nebula": "", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{{ + Key: nebulav1alpha1.ProviderLabel, + Value: providerName, + Effect: corev1.TaintEffectNoSchedule, + }}, + }, + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ + // Surfaces in the VERSION column of `kubectl get nodes`. Stamped at + // build time via -ldflags (see pkg/version); an unstamped build reports + // "nebula-dev". + KubeletVersion: version.Get(), + OperatingSystem: "linux", + // Architecture feeds the well-known kubernetes.io/arch label, which + // workloads pin via nodeSelector/affinity. amd64 is correct for every + // currently-wired provider (Modal/RunPod are x86). The true arch isn't + // known at node-creation time (no instance exists yet), so this is a + // per-provider default, not a universal truth. + // TODO: source arch from the provider (e.g. via Capabilities) when an + // arm64 provider (e.g. Grace-Hopper/GH200) lands, or a Pod pinning + // kubernetes.io/arch=arm64 will fail to schedule onto this node. + Architecture: "amd64", + }, + // Advertise generous virtual capacity; the external provider, not the + // kubelet, enforces the real shape. Placement is driven by the + // nodeSelector, not by these numbers. + Capacity: virtualCapacity(), + Allocatable: virtualCapacity(), + Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady}}, + }, + } +} + +// markNodeReady flips the node's Ready condition to True. Called once both +// controllers are up so the scheduler starts binding Pods to it. +func markNodeReady(n *corev1.Node) { + n.Status.Phase = corev1.NodeRunning + for i := range n.Status.Conditions { + if n.Status.Conditions[i].Type != corev1.NodeReady { + continue + } + n.Status.Conditions[i].Status = corev1.ConditionTrue + n.Status.Conditions[i].Reason = "KubeletReady" + n.Status.Conditions[i].Message = "virtual node ready" + return + } + n.Status.Conditions = append(n.Status.Conditions, corev1.NodeCondition{ + Type: corev1.NodeReady, Status: corev1.ConditionTrue, Reason: "KubeletReady", + }) +} + +// nvidiaGPUResource is the extended-resource key GPU Pods may request under. A +// Nebula Pod expresses its accelerator type+count on the accelerators annotation +// (the provider reads it from there, not from limits), but a Pod author may +// still set nvidia.com/gpu limits for portability; if they do, the virtual node +// must advertise the resource or the scheduler rejects the Pod with "Insufficient +// nvidia.com/gpu" before provisioning ever starts. +const nvidiaGPUResource = "nvidia.com/gpu" + +// virtualCapacity is the nominal capacity advertised by a virtual node. The +// numbers are deliberately synthetic and effectively infinite: a virtual node +// has no real machine behind it, so capacity's only job is to clear the +// scheduler's fit check (requests ≤ allocatable) and let the Pod bind. The +// provider enforces the real shape and rejects (→ failover) if the workload +// can't actually be satisfied. GPUs are advertised for the same reason as +// cpu/memory — a GPU Pod carries nvidia.com/gpu in its limits, and without an +// allocatable count here the scheduler would never bind it. +// +// TODO: this advertises only nvidia.com/gpu, the sole key the current (Modal) +// provider reads. A provider serving a different accelerator resource key (e.g. +// amd.com/gpu, or a typed MIG key) would have its GPU Pods rejected by the +// scheduler before provisioning. When such a provider lands, drive capacity from +// the provider instead of this shared list — e.g. have Capabilities declare the +// accelerator resource keys it serves and build the node capacity from that. +func virtualCapacity() corev1.ResourceList { + return corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1k"), + corev1.ResourceMemory: resource.MustParse("10Ti"), + corev1.ResourcePods: resource.MustParse("1k"), + nvidiaGPUResource: resource.MustParse("1k"), + } +} diff --git a/pkg/vnode/status.go b/pkg/vnode/status.go new file mode 100644 index 0000000..d044795 --- /dev/null +++ b/pkg/vnode/status.go @@ -0,0 +1,140 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vnode + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/InftyAI/Nebula/pkg/provider" +) + +// Pod status reasons the virtual node stamps on the Pod it reports. The Pod is +// the source of truth for the external instance's runtime state (the NodeClaim +// is a passive ledger and does not mirror this), so these live here in the +// vnode, not on the API type. +const ( + // reasonProvisioning: a provider Provision call has been issued and the + // instance is not yet running. + reasonProvisioning = "Provisioning" + // reasonRunning: the provider reports the instance running. + reasonRunning = "Running" + // reasonProvisionFailed: the provider rejected or failed the Provision call. + reasonProvisionFailed = "ProvisionFailed" + // reasonFailed: the provider reports the instance in a failed state. + reasonFailed = "Failed" + // reasonTerminated: the instance is gone from the provider (torn down, + // reclaimed, or exited). Disappearance alone does not say WHY, so this is the + // neutral term rather than "Preempted". + reasonTerminated = "Terminated" +) + +// applyState maps a provider Instance state onto the Pod status the virtual node +// reports. The Pod is the object the scheduler and user see, so the external +// instance's lifecycle is projected onto standard Pod phases/conditions: +// +// Running -> PodRunning, Ready=True +// Pending -> PodPending, Ready=False (starting) +// Failed -> PodFailed +// Terminated -> PodFailed (instance gone: torn down or reclaimed out-of-band) +func applyState(pod *corev1.Pod, state provider.InstanceState, endpoint string, now metav1.Time) { + switch state { + case provider.InstanceRunning: + setPhase(pod, corev1.PodRunning, reasonRunning, "external instance is running", now) + setReady(pod, corev1.ConditionTrue, now) + setContainerStatuses(pod, corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: now}}, true, now) + if endpoint != "" { + pod.Status.PodIP = endpoint + } + case provider.InstancePending: + setPhase(pod, corev1.PodPending, reasonProvisioning, "external instance is starting", now) + setReady(pod, corev1.ConditionFalse, now) + setContainerStatuses(pod, corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{Reason: reasonProvisioning, Message: "external instance is starting"}, + }, false, now) + case provider.InstanceFailed: + setPhase(pod, corev1.PodFailed, reasonFailed, "external instance entered a failed state", now) + setReady(pod, corev1.ConditionFalse, now) + setContainerStatuses(pod, corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{Reason: reasonFailed, Message: "external instance entered a failed state", FinishedAt: now}, + }, false, now) + case provider.InstanceTerminated: + // The instance is simply gone — absent from the provider's List, whether + // torn down by us, deleted out-of-band, or reclaimed. We cannot tell WHY + // from disappearance alone, so report the neutral, accurate "Terminated" + // rather than "Preempted", which would falsely assert a provider reclaim. + setPhase(pod, corev1.PodFailed, reasonTerminated, "external instance is gone", now) + setReady(pod, corev1.ConditionFalse, now) + setContainerStatuses(pod, corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{Reason: reasonTerminated, Message: "external instance is gone", FinishedAt: now}, + }, false, now) + } +} + +// setContainerStatuses projects the single external-instance lifecycle onto a +// per-container status for every container in the Pod spec. Kubernetes has no +// notion of the external instance — the READY column, `kubectl wait`, and any +// controller keying off container readiness all read Status.ContainerStatuses, +// so a Pod with an empty array reads as 0/N even when its Ready condition is +// True. There is one real workload behind the whole Pod, so every container +// mirrors the same state. The container's Image is echoed back (required field); +// RestartCount stays 0 (the provider owns restarts, not the kubelet). +func setContainerStatuses(pod *corev1.Pod, state corev1.ContainerState, ready bool, now metav1.Time) { + statuses := make([]corev1.ContainerStatus, 0, len(pod.Spec.Containers)) + for i := range pod.Spec.Containers { + c := &pod.Spec.Containers[i] + statuses = append(statuses, corev1.ContainerStatus{ + Name: c.Name, + Image: c.Image, + State: state, + Ready: ready, + Started: &ready, + }) + } + pod.Status.ContainerStatuses = statuses +} + +// setPhase sets the coarse Pod phase plus a human-readable reason/message. The +// start time is stamped once so repeated updates keep a stable value. +func setPhase(pod *corev1.Pod, phase corev1.PodPhase, reason, msg string, now metav1.Time) { + pod.Status.Phase = phase + pod.Status.Reason = reason + pod.Status.Message = msg + if pod.Status.StartTime == nil { + pod.Status.StartTime = &now + } +} + +// setReady sets the PodReady condition, preserving the transition time when the +// status is unchanged so watchers don't see spurious flips. +func setReady(pod *corev1.Pod, status corev1.ConditionStatus, now metav1.Time) { + for i := range pod.Status.Conditions { + if pod.Status.Conditions[i].Type != corev1.PodReady { + continue + } + if pod.Status.Conditions[i].Status != status { + pod.Status.Conditions[i].Status = status + pod.Status.Conditions[i].LastTransitionTime = now + } + return + } + pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{ + Type: corev1.PodReady, + Status: status, + LastTransitionTime: now, + }) +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go new file mode 100644 index 0000000..5ef295b --- /dev/null +++ b/test/e2e/e2e_suite_test.go @@ -0,0 +1,89 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os" + "os/exec" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/InftyAI/Nebula/test/utils" +) + +var ( + // Optional Environment Variables: + // - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup. + // These variables are useful if CertManager is already installed, avoiding + // re-installation and conflicts. + skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" + // isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster + isCertManagerAlreadyInstalled = false + + // projectImage is the name of the image which will be build and loaded + // with the code source changes to be tested. + projectImage = "example.com/nebula:v0.0.1" +) + +// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, +// temporary environment to validate project changes with the purpose of being used in CI jobs. +// The default setup requires Kind, builds/loads the Manager Docker image locally, and installs +// CertManager. +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting nebula integration test suite\n") + RunSpecs(t, "e2e suite") +} + +var _ = BeforeSuite(func() { + By("building the manager(Operator) image") + cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage)) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image") + + // TODO(user): If you want to change the e2e test vendor from Kind, ensure the image is + // built and available before running the tests. Also, remove the following block. + By("loading the manager(Operator) image on Kind") + err = utils.LoadImageToKindClusterWithName(projectImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind") + + // The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing. + // To prevent errors when tests run in environments with CertManager already installed, + // we check for its presence before execution. + // Setup CertManager before the suite if not skipped and if not already installed + if !skipCertManagerInstall { + By("checking if cert manager is installed already") + isCertManagerAlreadyInstalled = utils.IsCertManagerCRDsInstalled() + if !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n") + Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n") + } + } +}) + +var _ = AfterSuite(func() { + // Teardown CertManager after the suite if not skipped and if it was not already installed + if !skipCertManagerInstall && !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n") + utils.UninstallCertManager() + } +}) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 0000000..41c1006 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,354 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/InftyAI/Nebula/test/utils" +) + +// namespace where the project is deployed in +const namespace = "nebula-system" + +// serviceAccountName created for the project +const serviceAccountName = "nebula-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "nebula-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "nebula-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { + var controllerPodName string + + // Before running the tests, set up the environment by creating the namespace, + // enforce the restricted security policy to the namespace, installing CRDs, + // and deploying the controller. + BeforeAll(func() { + By("creating manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + }) + + // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, + // and deleting the namespace. + AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + + By("undeploying the controller-manager") + cmd = exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) + + By("uninstalling CRDs") + cmd = exec.Command("make", "uninstall") + _, _ = utils.Run(cmd) + + By("removing manager namespace") + cmd = exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + }) + + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } + + By("Fetching Kubernetes events") + cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } + + By("Fetching curl-metrics logs") + cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } + + By("Fetching controller manager pod description") + cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := utils.Run(cmd) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) + + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + + Context("Manager", func() { + It("should run successfully", func() { + By("validating that the controller-manager pod is running as expected") + verifyControllerUp := func(g Gomega) { + // Get the name of the controller-manager pod + cmd := exec.Command("kubectl", "get", + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}"+ + "{{ if not .metadata.deletionTimestamp }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + "-n", namespace, + ) + + podOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := utils.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") + controllerPodName = podNames[0] + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) + + // Validate the pod's status + cmd = exec.Command("kubectl", "get", + "pods", controllerPodName, "-o", "jsonpath={.status.phase}", + "-n", namespace, + ) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") + } + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=nebula-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:latest", + "--overrides", + fmt.Sprintf(`{ + "spec": { + "containers": [{ + "name": "curl", + "image": "curlimages/curl:latest", + "command": ["/bin/sh", "-c"], + "args": ["curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics"], + "securityContext": { + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } + }], + "serviceAccountName": "%s" + } + }`, token, metricsServiceName, namespace, serviceAccountName)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + It("should provisioned cert-manager", func() { + By("validating that cert-manager has the certificate Secret") + verifyCertManager := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace) + _, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + } + Eventually(verifyCertManager).Should(Succeed()) + }) + + It("should have CA injection for mutating webhooks", func() { + By("checking CA injection for mutating webhooks") + verifyCAInjection := func(g Gomega) { + cmd := exec.Command("kubectl", "get", + "mutatingwebhookconfigurations.admissionregistration.k8s.io", + "nebula-mutating-webhook-configuration", + "-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}") + mwhOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(mwhOutput)).To(BeNumerically(">", 10)) + } + Eventually(verifyCAInjection).Should(Succeed()) + }) + + // +kubebuilder:scaffold:e2e-webhooks-checks + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) + }) +}) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutput +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/test/utils/utils.go b/test/utils/utils.go new file mode 100644 index 0000000..448055b --- /dev/null +++ b/test/utils/utils.go @@ -0,0 +1,254 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "bufio" + "bytes" + "fmt" + "os" + "os/exec" + "strings" + + . "github.com/onsi/ginkgo/v2" // nolint:revive,staticcheck +) + +const ( + prometheusOperatorVersion = "v0.77.1" + prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + + "releases/download/%s/bundle.yaml" + + certmanagerVersion = "v1.16.3" + certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" +) + +func warnError(err error) { + _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) +} + +// Run executes the provided command within this context +func Run(cmd *exec.Cmd) (string, error) { + dir, _ := GetProjectDir() + cmd.Dir = dir + + if err := os.Chdir(cmd.Dir); err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %q\n", err) + } + + cmd.Env = append(os.Environ(), "GO111MODULE=on") + command := strings.Join(cmd.Args, " ") + _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command) + output, err := cmd.CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("%q failed with error %q: %w", command, string(output), err) + } + + return string(output), nil +} + +// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. +func InstallPrometheusOperator() error { + url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) + cmd := exec.Command("kubectl", "create", "-f", url) + _, err := Run(cmd) + return err +} + +// UninstallPrometheusOperator uninstalls the prometheus +func UninstallPrometheusOperator() { + url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) + cmd := exec.Command("kubectl", "delete", "-f", url) + if _, err := Run(cmd); err != nil { + warnError(err) + } +} + +// IsPrometheusCRDsInstalled checks if any Prometheus CRDs are installed +// by verifying the existence of key CRDs related to Prometheus. +func IsPrometheusCRDsInstalled() bool { + // List of common Prometheus CRDs + prometheusCRDs := []string{ + "prometheuses.monitoring.coreos.com", + "prometheusrules.monitoring.coreos.com", + "prometheusagents.monitoring.coreos.com", + } + + cmd := exec.Command("kubectl", "get", "crds", "-o", "custom-columns=NAME:.metadata.name") + output, err := Run(cmd) + if err != nil { + return false + } + crdList := GetNonEmptyLines(output) + for _, crd := range prometheusCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// UninstallCertManager uninstalls the cert manager +func UninstallCertManager() { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "delete", "-f", url) + if _, err := Run(cmd); err != nil { + warnError(err) + } +} + +// InstallCertManager installs the cert manager bundle. +func InstallCertManager() error { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "apply", "-f", url) + if _, err := Run(cmd); err != nil { + return err + } + // Wait for cert-manager-webhook to be ready, which can take time if cert-manager + // was re-installed after uninstalling on a cluster. + cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "5m", + ) + + _, err := Run(cmd) + return err +} + +// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed +// by verifying the existence of key CRDs related to Cert Manager. +func IsCertManagerCRDsInstalled() bool { + // List of common Cert Manager CRDs + certManagerCRDs := []string{ + "certificates.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "certificaterequests.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + } + + // Execute the kubectl command to get all CRDs + cmd := exec.Command("kubectl", "get", "crds") + output, err := Run(cmd) + if err != nil { + return false + } + + // Check if any of the Cert Manager CRDs are present + crdList := GetNonEmptyLines(output) + for _, crd := range certManagerCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// LoadImageToKindClusterWithName loads a local docker image to the kind cluster +func LoadImageToKindClusterWithName(name string) error { + cluster := "kind" + if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { + cluster = v + } + kindOptions := []string{"load", "docker-image", name, "--name", cluster} + cmd := exec.Command("kind", kindOptions...) + _, err := Run(cmd) + return err +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + var res []string + elements := strings.Split(output, "\n") + for _, element := range elements { + if element != "" { + res = append(res, element) + } + } + + return res +} + +// GetProjectDir will return the directory where the project is +func GetProjectDir() (string, error) { + wd, err := os.Getwd() + if err != nil { + return wd, fmt.Errorf("failed to get current working directory: %w", err) + } + wd = strings.ReplaceAll(wd, "/test/e2e", "") + return wd, nil +} + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read file %q: %w", filename, err) + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %q to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + if _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err = out.WriteString("\n"); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + } + + if _, err = out.Write(content[idx+len(target):]); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + // false positive + // nolint:gosec + if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { + return fmt.Errorf("failed to write file %q: %w", filename, err) + } + + return nil +} From 49bb775b09b1ce2e42a82e1dfaea862b379aa8e6 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 27 Jul 2026 00:48:33 +0100 Subject: [PATCH 02/12] fix bug Signed-off-by: kerthcet --- Makefile | 4 + config/samples/nebula_v1alpha1_nodepool.yaml | 2 +- pkg/provider/modal/client.go | 69 +++++++++++++---- pkg/provider/modal/modal.go | 40 ++++++---- pkg/provider/modal/modal_test.go | 81 +++++++++++++++++--- 5 files changed, 153 insertions(+), 43 deletions(-) diff --git a/Makefile b/Makefile index e51afc0..4c8d8d1 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,10 @@ verify-catalog: kustomize ## Verify the price catalog CSVs parse and the catalog fmt: ## Run go fmt against code. go fmt ./... +.PHONY: format +format: golangci-lint ## Format code with the configured formatters (gofmt, goimports). + $(GOLANGCI_LINT) fmt + .PHONY: vet vet: ## Run go vet against code. go vet ./... diff --git a/config/samples/nebula_v1alpha1_nodepool.yaml b/config/samples/nebula_v1alpha1_nodepool.yaml index 4eed653..55ba7a1 100644 --- a/config/samples/nebula_v1alpha1_nodepool.yaml +++ b/config/samples/nebula_v1alpha1_nodepool.yaml @@ -10,7 +10,7 @@ spec: providers: - name: modal # - name: runpod - # Outer axis: try spot on every provider first, fall back to on-demand. + # Outer axis: try Reserved on every provider first, fall back to on-demand, then spot. capacityTypes: - Reserved - OnDemand diff --git a/pkg/provider/modal/client.go b/pkg/provider/modal/client.go index 2e36f62..18c0cf5 100644 --- a/pkg/provider/modal/client.go +++ b/pkg/provider/modal/client.go @@ -24,6 +24,7 @@ import ( "time" modal "github.com/modal-labs/modal-client/go" + corev1 "k8s.io/api/core/v1" "github.com/InftyAI/Nebula/pkg/provider/catalog" ) @@ -93,6 +94,11 @@ func (c *sdkClient) CreateSandbox(ctx context.Context, spec SandboxSpec) (string } image := c.mc.Images.FromRegistry(spec.Image, nil) + probe, err := modalProbe(spec.ReadinessProbe) + if err != nil { + return "", fmt.Errorf("modal: readiness probe: %w", err) + } + sb, err := c.mc.Sandboxes.Create(ctx, app, image, &modal.SandboxCreateParams{ Command: spec.Command, Env: spec.Env, @@ -102,6 +108,7 @@ func (c *sdkClient) CreateSandbox(ctx context.Context, spec SandboxSpec) (string EncryptedPorts: spec.Ports, Timeout: spec.Timeout, Tags: spec.Tags, + ReadinessProbe: probe, }) if err != nil { return "", err @@ -109,6 +116,34 @@ func (c *sdkClient) CreateSandbox(ctx context.Context, spec SandboxSpec) (string return sb.SandboxID, nil } +// modalProbe maps a Pod readinessProbe onto Modal's Probe. Modal supports only +// TCP and Exec probes, so an HTTPGet probe degrades to a TCP probe on its port +// (readiness ≈ the port accepting connections). Returns (nil, nil) when p is nil +// or names no supported handler, so a probe-less (or unsupported) workload simply +// gets no Modal probe. PeriodSeconds maps to the probe interval; zero leaves the +// SDK default (a zero interval is rejected by the SDK constructors). +func modalProbe(p *corev1.Probe) (*modal.Probe, error) { + if p == nil { + return nil, nil + } + // PeriodSeconds maps to the probe interval; zero leaves the SDK default (the + // SDK constructors reject a zero interval). + var interval time.Duration + if p.PeriodSeconds > 0 { + interval = time.Duration(p.PeriodSeconds) * time.Second + } + switch { + case p.Exec != nil && len(p.Exec.Command) > 0: + return modal.NewExecProbe(p.Exec.Command, &modal.ExecProbeParams{Interval: interval}) + case p.TCPSocket != nil: + return modal.NewTCPProbe(p.TCPSocket.Port.IntValue(), &modal.TCPProbeParams{Interval: interval}) + case p.HTTPGet != nil: + return modal.NewTCPProbe(p.HTTPGet.Port.IntValue(), &modal.TCPProbeParams{Interval: interval}) + default: + return nil, nil + } +} + // TerminateSandbox implements Client. Idempotent: a sandbox that no longer // exists resolves to a not-found from FromID, which we treat as already gone. func (c *sdkClient) TerminateSandbox(ctx context.Context, id string) error { @@ -166,20 +201,28 @@ func (c *sdkClient) ListSandboxes(ctx context.Context) ([]Sandbox, error) { // status (from Poll), tags (from GetTags), and a best-effort endpoint (from // Tunnels). Tag/tunnel/poll errors are tolerated so a single flaky sandbox // doesn't fail the whole List — the poll loop will re-observe next tick. +// +// observe is a POINT-IN-TIME read and never blocks: it reports the status as +// currently known and returns immediately. It deliberately does not call +// WaitUntilReady (the only Modal readiness signal), because that blocks until the +// probe first passes and would stall the List for as long as a sandbox takes to +// come up. func (c *sdkClient) observe(ctx context.Context, sb *modal.Sandbox) Sandbox { out := Sandbox{ID: sb.SandboxID} - // Status. Poll (== sandboxWait(0)) reports whether the sandbox PROCESS HAS - // EXITED: a non-nil exit code means it is gone (terminated), nil means it is - // still live. We treat "still live" as running. This does fold the brief - // startup window (queued, image pull, GPU attach, container boot) into - // "running" rather than "pending", because the SDK at this version exposes no - // lightweight readiness readback: WaitUntilReady only resolves against a - // readiness probe configured at create time (which we do not set) and requires - // a live task-command-router connection, so it can never report ready here. - // Poll is the only reliable signal, and reporting a starting sandbox as running - // a few seconds early is far better than the alternative — never reporting it - // ready at all, which wedges the Pod at Pending and the NodeClaim at Provisioning. + // Tags carry Nebula identity (ClaimTagKey), recovered by toInstance. + if tags, err := sb.GetTags(ctx, &modal.SandboxGetTagsParams{}); err == nil { + out.Tags = tags + } + + // Status. Poll (== sandboxWait(0)) is the only cheap point-in-time signal: it + // reports whether the sandbox PROCESS HAS EXITED — a non-nil exit code means it + // is gone (terminated), nil means it is still live. Poll cannot tell "still + // scheduling" (queued, image pull, GPU attach, container boot) apart from + // "running and serving" — both read as nil — and Modal exposes no cheap + // readiness readback (WaitUntilReady blocks, so we do not call it here). So a + // live sandbox is reported "running" as soon as its process exists, folding the + // brief startup window into running rather than blocking to confirm readiness. if code, err := sb.Poll(ctx, &modal.SandboxPollParams{}); err == nil { if code != nil { out.Status = "terminated" @@ -188,10 +231,6 @@ func (c *sdkClient) observe(ctx context.Context, sb *modal.Sandbox) Sandbox { } } - if tags, err := sb.GetTags(ctx, &modal.SandboxGetTagsParams{}); err == nil { - out.Tags = tags - } - // Endpoint is only meaningful once running; look it up best-effort. if out.Status == "running" { tctx, cancel := context.WithTimeout(ctx, c.endpointTimeout) diff --git a/pkg/provider/modal/modal.go b/pkg/provider/modal/modal.go index d4ed6d2..85f9fba 100644 --- a/pkg/provider/modal/modal.go +++ b/pkg/provider/modal/modal.go @@ -108,6 +108,13 @@ type SandboxSpec struct { Timeout time.Duration // Tags carry Nebula identity; ClaimTagKey holds the NodeClaim name. Tags map[string]string + // ReadinessProbe, when non-nil, is the Pod's first-container readinessProbe + // carried through so the Client can configure Modal's own readiness probe at + // create time. Modal enforces the probe internally (it gates its own traffic + // routing on it); Nebula does not read the result back — observe reports status + // from the cheap point-in-time Poll signal and never blocks on WaitUntilReady. + // We only ever pass a user-supplied probe; the adapter never fabricates one. + ReadinessProbe *corev1.Probe } // Sandbox is the adapter-level view of a Modal sandbox as observed. @@ -261,14 +268,15 @@ func (p *Provider) sandboxSpecFromPod(pod *corev1.Pod, req provider.ProvisionReq } spec := SandboxSpec{ - Image: c.Image, - Command: append(append([]string{}, c.Command...), c.Args...), - Env: env, - CPU: cpuCores(&c), - MemoryMiB: memoryMiB(&c), - Ports: containerPorts(&c), - Timeout: sandboxTimeout(pod), - Tags: map[string]string{ClaimTagKey: req.ClaimName}, + Image: c.Image, + Command: append(append([]string{}, c.Command...), c.Args...), + Env: env, + CPU: cpuCores(&c), + MemoryMiB: memoryMiB(&c), + Ports: containerPorts(&c), + Timeout: sandboxTimeout(pod), + Tags: map[string]string{ClaimTagKey: req.ClaimName}, + ReadinessProbe: c.ReadinessProbe, } // Accelerator type comes from the AcceleratorTypeLabel; count from the @@ -361,19 +369,19 @@ func (p *Provider) toInstance(sb Sandbox) provider.Instance { } } -// toState maps Modal's status strings to the provider-agnostic lifecycle state. -// Unknown statuses map to Pending so the poll loop keeps watching rather than -// declaring a premature terminal state. +// toState maps the status strings observe produces to the provider-agnostic +// lifecycle state. observe derives status from Poll (+ the readiness probe), so +// it only ever emits four values: "running"/"ready" (live), "pending" (live but +// the readiness probe has not passed yet), and "terminated" (process exited). +// Anything else — including the empty string observe leaves when Poll itself +// errors — maps to Pending, so the poll loop keeps watching rather than declaring +// a premature terminal state. func toState(modalStatus string) provider.InstanceState { switch strings.ToLower(modalStatus) { case "running", "ready": return provider.InstanceRunning - case "terminated", "stopped", "completed": + case "terminated": return provider.InstanceTerminated - case "error", "failed": - return provider.InstanceFailed - case "pending", "starting", "queued", "": - return provider.InstancePending default: return provider.InstancePending } diff --git a/pkg/provider/modal/modal_test.go b/pkg/provider/modal/modal_test.go index 8c4b708..084ebdb 100644 --- a/pkg/provider/modal/modal_test.go +++ b/pkg/provider/modal/modal_test.go @@ -25,6 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" "github.com/InftyAI/Nebula/pkg/provider" @@ -304,22 +305,16 @@ func TestOfferings(t *testing.T) { } func TestToState(t *testing.T) { - // The status→state mapping is load-bearing: "pending" must map to - // InstancePending, not InstanceRunning. A still-starting sandbox is reported - // pending by observe (via the readiness probe), and mapping it to Running here - // would resurface the "Pod Ready while the instance is still coming up" bug. + // The status→state mapping is load-bearing. observe emits only "running"/ + // "ready" (live), "terminated" (exited), or "" (Poll errored); everything else, + // including the empty string, must map to Pending so the poll loop keeps + // watching rather than declaring a premature terminal state. cases := map[string]provider.InstanceState{ "running": provider.InstanceRunning, "ready": provider.InstanceRunning, "pending": provider.InstancePending, - "starting": provider.InstancePending, - "queued": provider.InstancePending, - "": provider.InstancePending, + "": provider.InstancePending, // Poll errored, status left unset "terminated": provider.InstanceTerminated, - "stopped": provider.InstanceTerminated, - "completed": provider.InstanceTerminated, - "error": provider.InstanceFailed, - "failed": provider.InstanceFailed, "weird-new": provider.InstancePending, // unknown => keep watching, not terminal } for in, want := range cases { @@ -328,3 +323,67 @@ func TestToState(t *testing.T) { } } } + +func TestProvision_ReadinessProbeCarriedThrough(t *testing.T) { + f := &fakeClient{createID: "sb-1"} + p := newTestProvider(f) + + pod := gpuPod("claim-a", "H100", 1) + probe := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt(8000)}, + }, + } + pod.Spec.Containers[0].ReadinessProbe = probe + + if _, err := p.Provision(context.Background(), pod, provider.ProvisionRequest{ClaimName: "claim-a"}); err != nil { + t.Fatalf("Provision: %v", err) + } + // The probe is carried onto the spec so the Client can configure Modal's own + // readiness probe; Modal enforces it internally (Nebula does not read it back). + if f.lastSpec.ReadinessProbe != probe { + t.Fatalf("ReadinessProbe not carried onto the spec: got %v", f.lastSpec.ReadinessProbe) + } +} + +func TestProvision_NoProbeLeavesSpecUnset(t *testing.T) { + f := &fakeClient{createID: "sb-1"} + p := newTestProvider(f) + + if _, err := p.Provision(context.Background(), gpuPod("claim-a", "H100", 1), provider.ProvisionRequest{ClaimName: "claim-a"}); err != nil { + t.Fatalf("Provision: %v", err) + } + if f.lastSpec.ReadinessProbe != nil { + t.Fatalf("ReadinessProbe should be nil when the Pod declares none, got %v", f.lastSpec.ReadinessProbe) + } +} + +func TestModalProbe(t *testing.T) { + // nil Pod probe => no Modal probe (probe-less workload). + if got, err := modalProbe(nil); err != nil || got != nil { + t.Fatalf("modalProbe(nil) = (%v, %v), want (nil, nil)", got, err) + } + + // A probe with no supported handler also degrades to no Modal probe rather + // than an error, so it never wedges provisioning. + empty := &corev1.Probe{} + if got, err := modalProbe(empty); err != nil || got != nil { + t.Fatalf("modalProbe(empty) = (%v, %v), want (nil, nil)", got, err) + } + + // Supported handlers each produce a Modal probe. HTTPGet degrades to TCP. + supported := []*corev1.Probe{ + {ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt(8000)}}}, + {ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"true"}}}}, + {ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Port: intstr.FromInt(80)}}, PeriodSeconds: 5}, + } + for i, pr := range supported { + got, err := modalProbe(pr) + if err != nil { + t.Fatalf("case %d: modalProbe error: %v", i, err) + } + if got == nil { + t.Fatalf("case %d: expected a Modal probe, got nil", i) + } + } +} From f846def5a7713aa0d0e227e6c169142259a4f9cf Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 27 Jul 2026 13:24:53 +0100 Subject: [PATCH 03/12] fix lint Signed-off-by: kerthcet --- config/samples/deployment.yaml | 2 +- pkg/util/accelerator_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml index 2d9b784..826652a 100644 --- a/config/samples/deployment.yaml +++ b/config/samples/deployment.yaml @@ -44,7 +44,7 @@ spec: # provider catalog (e.g. Modal offers T4/L4/A10G/A100-40GB/A100-80GB/ # H100/H200). The COUNT is the nvidia.com/gpu resource below — omit it # for a single GPU. - nebula.inftyai.com/accelerator-type: l4 + nebula.inftyai.com/accelerator-type: a100-40gb spec: # Do NOT set nodeName or a provider nodeSelector yourself — the placement # controller fills the nodeSelector in when it ungates the Pod. Setting diff --git a/pkg/util/accelerator_test.go b/pkg/util/accelerator_test.go index c7062d4..f7096c2 100644 --- a/pkg/util/accelerator_test.go +++ b/pkg/util/accelerator_test.go @@ -20,8 +20,8 @@ import ( "testing" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" ) From 1d0e0d860364ab7f31907a01201dc7e9bfcdb0f4 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 27 Jul 2026 13:46:08 +0100 Subject: [PATCH 04/12] fix lint issue Signed-off-by: kerthcet --- .golangci.yml | 9 +++ .../controller/pod_placement_controller.go | 8 +-- .../pod_placement_controller_test.go | 69 ++++++++++--------- pkg/provider/catalog/catalog.go | 2 +- pkg/provider/modal/client.go | 9 +-- pkg/provider/modal/modal.go | 23 ++++--- pkg/provider/modal/modal_test.go | 34 +++++---- pkg/vnode/handler.go | 4 +- pkg/vnode/node.go | 8 ++- pkg/vnode/status.go | 22 ++++-- 10 files changed, 115 insertions(+), 73 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index e5b21b0..d25815d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -36,6 +36,15 @@ linters: - dupl - lll path: internal/* + # Test helpers intentionally take name/namespace params for readable call + # sites even when the current tests all pass the same value; unparam's + # "always receives X" is noise there (and removing the param hurts clarity). + # goconst likewise flags repeated fixture strings ("inst-1", …) that read + # fine inline — extracting a const per fixture adds indirection, not clarity. + - linters: + - goconst + - unparam + path: _test\.go paths: - third_party$ - builtin$ diff --git a/internal/controller/pod_placement_controller.go b/internal/controller/pod_placement_controller.go index e799c69..dc5ad51 100644 --- a/internal/controller/pod_placement_controller.go +++ b/internal/controller/pod_placement_controller.go @@ -155,13 +155,13 @@ func needsPlacement(pod *corev1.Pod) bool { if pod.Spec.NodeName != "" { return false } - return hasGateNamed(pod, nebulav1alpha1.ProviderSelectionGate) + return hasGateNamed(pod) } -// hasGateNamed reports whether the Pod carries the named scheduling gate. -func hasGateNamed(pod *corev1.Pod, name string) bool { +// hasGateNamed reports whether the Pod carries the provider-selection scheduling gate. +func hasGateNamed(pod *corev1.Pod) bool { for _, g := range pod.Spec.SchedulingGates { - if g.Name == name { + if g.Name == nebulav1alpha1.ProviderSelectionGate { return true } } diff --git a/internal/controller/pod_placement_controller_test.go b/internal/controller/pod_placement_controller_test.go index 726073e..d6d65f5 100644 --- a/internal/controller/pod_placement_controller_test.go +++ b/internal/controller/pod_placement_controller_test.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" ) // newPlacementReconciler wires a PodPlacementReconciler over a fake client. @@ -73,7 +74,7 @@ func gatedPod(name, ns, uid, pool, gpu string) *corev1.Pod { } func poolWith(name string, capTypes []nebulav1alpha1.CapacityType, providers ...string) *nebulav1alpha1.NodePool { - var refs []nebulav1alpha1.ProviderRef + refs := make([]nebulav1alpha1.ProviderRef, 0, len(providers)) for _, p := range providers { refs = append(refs, nebulav1alpha1.ProviderRef{Name: p}) } @@ -106,19 +107,19 @@ func getPod(t *testing.T, c client.Client, ns, name string) *corev1.Pod { func TestPlacement_UngatesAndRoutesAndCreatesClaim(t *testing.T) { pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") - pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") - prov := &fakeProvider{name: "modal", gpus: []string{"H100"}} + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, provider.ProviderModal) + prov := &fakeProvider{name: provider.ProviderModal, gpus: []string{"H100"}} r, c := newPlacementReconciler(t, []client.Object{pod, pool}, prov) reconcilePod(t, r, "default", "p1") got := getPod(t, c, "default", "p1") // Gate removed. - if hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + if hasGateNamed(got) { t.Fatal("expected the provider-selection gate to be removed") } // Routed to the chosen provider. - if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != "modal" { + if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != provider.ProviderModal { t.Fatalf("expected nodeSelector provider=modal, got %v", got.Spec.NodeSelector) } // Capacity tier stamped for the VK handler. @@ -130,7 +131,7 @@ func TestPlacement_UngatesAndRoutesAndCreatesClaim(t *testing.T) { if err := c.Get(context.Background(), types.NamespacedName{Name: "default-p1"}, &nc); err != nil { t.Fatalf("expected NodeClaim default-p1: %v", err) } - if nc.Spec.Provider != "modal" || nc.Spec.PodRef.UID != "uid-1" || nc.Spec.PoolRef != "pool-a" { + if nc.Spec.Provider != provider.ProviderModal || nc.Spec.PodRef.UID != "uid-1" || nc.Spec.PoolRef != "pool-a" { t.Fatalf("unexpected claim spec: %+v", nc.Spec) } } @@ -139,15 +140,15 @@ func TestPlacement_FirstMatchingProviderWins(t *testing.T) { // runpod is listed first but does not offer H100; modal does. First MATCHING // provider wins, so modal is chosen. pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") - pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "runpod", "modal") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "runpod", provider.ProviderModal) runpod := &fakeProvider{name: "runpod", gpus: []string{"A100"}} - modal := &fakeProvider{name: "modal", gpus: []string{"H100"}} + modal := &fakeProvider{name: provider.ProviderModal, gpus: []string{"H100"}} r, c := newPlacementReconciler(t, []client.Object{pod, pool}, runpod, modal) reconcilePod(t, r, "default", "p1") got := getPod(t, c, "default", "p1") - if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != "modal" { + if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != provider.ProviderModal { t.Fatalf("expected modal (first matching), got %v", got.Spec.NodeSelector) } } @@ -155,9 +156,9 @@ func TestPlacement_FirstMatchingProviderWins(t *testing.T) { func TestPlacement_OrderedPrefersEarlierProvider(t *testing.T) { // Both offer H100; the first in the list wins. pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") - pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "runpod", "modal") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "runpod", provider.ProviderModal) runpod := &fakeProvider{name: "runpod", gpus: []string{"H100"}} - modal := &fakeProvider{name: "modal", gpus: []string{"H100"}} + modal := &fakeProvider{name: provider.ProviderModal, gpus: []string{"H100"}} r, c := newPlacementReconciler(t, []client.Object{pod, pool}, runpod, modal) reconcilePod(t, r, "default", "p1") @@ -177,7 +178,7 @@ func TestPlacement_NoMatchingProviderLeavesPodGated(t *testing.T) { reconcilePod(t, r, "default", "p1") got := getPod(t, c, "default", "p1") - if !hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + if !hasGateNamed(got) { t.Fatal("expected the Pod to stay gated when no provider matches") } if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != "" { @@ -188,14 +189,14 @@ func TestPlacement_NoMatchingProviderLeavesPodGated(t *testing.T) { func TestPlacement_CPUOnlyPodMatchesAnyProvider(t *testing.T) { // No GPU annotation => any provider matches; even one offering nothing. pod := gatedPod("p1", "default", "uid-1", "pool-a", "") - pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") - modal := &fakeProvider{name: "modal", gpus: []string{}} // offers no GPUs + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, provider.ProviderModal) + modal := &fakeProvider{name: provider.ProviderModal, gpus: []string{}} // offers no GPUs r, c := newPlacementReconciler(t, []client.Object{pod, pool}, modal) reconcilePod(t, r, "default", "p1") got := getPod(t, c, "default", "p1") - if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != "modal" { + if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != provider.ProviderModal { t.Fatalf("expected a CPU-only Pod to place on modal, got %v", got.Spec.NodeSelector) } } @@ -203,14 +204,14 @@ func TestPlacement_CPUOnlyPodMatchesAnyProvider(t *testing.T) { func TestPlacement_SkipsPodWithoutOptInLabel(t *testing.T) { pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") pod.Labels[nebulav1alpha1.EnabledLabel] = "false" - pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") - modal := &fakeProvider{name: "modal"} + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, provider.ProviderModal) + modal := &fakeProvider{name: provider.ProviderModal} r, c := newPlacementReconciler(t, []client.Object{pod, pool}, modal) reconcilePod(t, r, "default", "p1") got := getPod(t, c, "default", "p1") - if !hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + if !hasGateNamed(got) { t.Fatal("expected a non-opted-in Pod to be left untouched") } } @@ -218,8 +219,8 @@ func TestPlacement_SkipsPodWithoutOptInLabel(t *testing.T) { func TestPlacement_SkipsAlreadyScheduledPod(t *testing.T) { pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") pod.Spec.NodeName = "some-node" - pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") - modal := &fakeProvider{name: "modal"} + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, provider.ProviderModal) + modal := &fakeProvider{name: provider.ProviderModal} r, c := newPlacementReconciler(t, []client.Object{pod, pool}, modal) reconcilePod(t, r, "default", "p1") @@ -234,13 +235,13 @@ func TestPlacement_SkipsAlreadyScheduledPod(t *testing.T) { func TestPlacement_MissingPoolLeavesPodGated(t *testing.T) { pod := gatedPod("p1", "default", "uid-1", "ghost-pool", "H100") - modal := &fakeProvider{name: "modal"} + modal := &fakeProvider{name: provider.ProviderModal} r, c := newPlacementReconciler(t, []client.Object{pod}, modal) // no pool seeded reconcilePod(t, r, "default", "p1") got := getPod(t, c, "default", "p1") - if !hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + if !hasGateNamed(got) { t.Fatal("expected the Pod to stay gated when its pool is missing") } } @@ -250,16 +251,16 @@ func TestPlacement_StaleClaimForPriorPodBlocksUngate(t *testing.T) { // (different UID). Placement must NOT ungate against the wrong ledger: it // leaves the Pod gated and requeues until the backstop reaps the stale claim. pod := gatedPod("p1", "default", "uid-new", "pool-a", "H100") - pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, provider.ProviderModal) stale := &nebulav1alpha1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{Name: "default-p1"}, Spec: nebulav1alpha1.NodeClaimSpec{ PodRef: nebulav1alpha1.PodReference{Namespace: "default", Name: "p1", UID: "uid-old"}, - Provider: "modal", + Provider: provider.ProviderModal, PoolRef: "pool-a", }, } - prov := &fakeProvider{name: "modal", gpus: []string{"H100"}} + prov := &fakeProvider{name: provider.ProviderModal, gpus: []string{"H100"}} r, c := newPlacementReconciler(t, []client.Object{pod, pool, stale}, prov) res, err := r.Reconcile(context.Background(), reconcile.Request{ @@ -273,7 +274,7 @@ func TestPlacement_StaleClaimForPriorPodBlocksUngate(t *testing.T) { } got := getPod(t, c, "default", "p1") - if !hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + if !hasGateNamed(got) { t.Fatal("expected the Pod to stay gated against a stale claim") } // The stale claim must be left untouched (the backstop, not placement, owns it). @@ -290,25 +291,25 @@ func TestPlacement_AdoptsOwnClaimOnRetry(t *testing.T) { // A claim of this name already exists AND pins this Pod's UID: a genuine retry // after a crash between create and ungate. Placement adopts it and ungates. pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") - pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, provider.ProviderModal) mine := &nebulav1alpha1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{Name: "default-p1"}, Spec: nebulav1alpha1.NodeClaimSpec{ PodRef: nebulav1alpha1.PodReference{Namespace: "default", Name: "p1", UID: "uid-1"}, - Provider: "modal", + Provider: provider.ProviderModal, PoolRef: "pool-a", }, } - prov := &fakeProvider{name: "modal", gpus: []string{"H100"}} + prov := &fakeProvider{name: provider.ProviderModal, gpus: []string{"H100"}} r, c := newPlacementReconciler(t, []client.Object{pod, pool, mine}, prov) reconcilePod(t, r, "default", "p1") got := getPod(t, c, "default", "p1") - if hasGateNamed(got, nebulav1alpha1.ProviderSelectionGate) { + if hasGateNamed(got) { t.Fatal("expected the Pod placed when the existing claim is its own") } - if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != "modal" { + if got.Spec.NodeSelector[nebulav1alpha1.ProviderLabel] != provider.ProviderModal { t.Fatalf("expected routing to modal, got %v", got.Spec.NodeSelector) } } @@ -317,8 +318,8 @@ func TestPlacement_IdempotentOnRetry(t *testing.T) { // A second reconcile after a successful placement must not error (claim // AlreadyExists is success) and must leave the Pod placed. pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") - pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, "modal") - prov := &fakeProvider{name: "modal", gpus: []string{"H100"}} + pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, provider.ProviderModal) + prov := &fakeProvider{name: provider.ProviderModal, gpus: []string{"H100"}} r, c := newPlacementReconciler(t, []client.Object{pod, pool}, prov) reconcilePod(t, r, "default", "p1") @@ -331,7 +332,7 @@ func TestPlacement_IdempotentOnRetry(t *testing.T) { reconcilePod(t, r, "default", "p1") // must not error on AlreadyExists final := getPod(t, c, "default", "p1") - if hasGateNamed(final, nebulav1alpha1.ProviderSelectionGate) { + if hasGateNamed(final) { t.Fatal("expected the Pod placed again on retry") } } diff --git a/pkg/provider/catalog/catalog.go b/pkg/provider/catalog/catalog.go index d87dd4b..40529e8 100644 --- a/pkg/provider/catalog/catalog.go +++ b/pkg/provider/catalog/catalog.go @@ -140,7 +140,7 @@ func parseCSV(r io.Reader) ([]provider.Offering, error) { return nil, err } - var offerings []provider.Offering + offerings := make([]provider.Offering, 0, len(records)) for i, rec := range records { if i == 0 { continue // header diff --git a/pkg/provider/modal/client.go b/pkg/provider/modal/client.go index 18c0cf5..53b03ba 100644 --- a/pkg/provider/modal/client.go +++ b/pkg/provider/modal/client.go @@ -187,7 +187,8 @@ func (c *sdkClient) ListSandboxes(ctx context.Context) ([]Sandbox, error) { if err != nil { return nil, err } - var out []Sandbox + // seq is an iterator with no length, so out is grown as sandboxes arrive. + out := make([]Sandbox, 0) //nolint:prealloc // unknown length: iterator, not a slice for sb, err := range seq { if err != nil { return nil, err @@ -225,14 +226,14 @@ func (c *sdkClient) observe(ctx context.Context, sb *modal.Sandbox) Sandbox { // brief startup window into running rather than blocking to confirm readiness. if code, err := sb.Poll(ctx, &modal.SandboxPollParams{}); err == nil { if code != nil { - out.Status = "terminated" + out.Status = statusTerminated } else { - out.Status = "running" + out.Status = statusRunning } } // Endpoint is only meaningful once running; look it up best-effort. - if out.Status == "running" { + if out.Status == statusRunning { tctx, cancel := context.WithTimeout(ctx, c.endpointTimeout) if tunnels, err := sb.Tunnels(tctx, c.endpointTimeout, &modal.SandboxTunnelsParams{}); err == nil { for _, t := range tunnels { diff --git a/pkg/provider/modal/modal.go b/pkg/provider/modal/modal.go index 85f9fba..25e289a 100644 --- a/pkg/provider/modal/modal.go +++ b/pkg/provider/modal/modal.go @@ -369,18 +369,25 @@ func (p *Provider) toInstance(sb Sandbox) provider.Instance { } } +// Sandbox status strings observe produces (and toState consumes). observe +// derives status from Poll, so it only ever emits statusRunning (live) or +// statusTerminated (process exited); an unset status ("") means Poll errored. +const ( + statusRunning = "running" + statusTerminated = "terminated" +) + // toState maps the status strings observe produces to the provider-agnostic -// lifecycle state. observe derives status from Poll (+ the readiness probe), so -// it only ever emits four values: "running"/"ready" (live), "pending" (live but -// the readiness probe has not passed yet), and "terminated" (process exited). -// Anything else — including the empty string observe leaves when Poll itself -// errors — maps to Pending, so the poll loop keeps watching rather than declaring -// a premature terminal state. +// lifecycle state. observe emits only statusRunning (live) or statusTerminated +// (process exited); "ready" is also accepted as a live synonym. Anything else — +// including the empty string observe leaves when Poll itself errors — maps to +// Pending, so the poll loop keeps watching rather than declaring a premature +// terminal state. func toState(modalStatus string) provider.InstanceState { switch strings.ToLower(modalStatus) { - case "running", "ready": + case statusRunning, "ready": return provider.InstanceRunning - case "terminated": + case statusTerminated: return provider.InstanceTerminated default: return provider.InstancePending diff --git a/pkg/provider/modal/modal_test.go b/pkg/provider/modal/modal_test.go index 084ebdb..1d1de1d 100644 --- a/pkg/provider/modal/modal_test.go +++ b/pkg/provider/modal/modal_test.go @@ -200,7 +200,8 @@ func TestProvision_DefaultsTimeoutWhenNoDeadline(t *testing.T) { // No activeDeadlineSeconds: the adapter must still set a non-zero timeout, else // Modal applies its 5-minute default and the workload dies almost immediately. - if _, err := p.Provision(context.Background(), gpuPod("claim-dt", "H100", 1), provider.ProvisionRequest{ClaimName: "claim-dt"}); err != nil { + req := provider.ProvisionRequest{ClaimName: "claim-dt"} + if _, err := p.Provision(context.Background(), gpuPod("claim-dt", "H100", 1), req); err != nil { t.Fatalf("Provision: %v", err) } if f.lastSpec.Timeout != defaultSandboxTimeout { @@ -226,11 +227,16 @@ func TestProvision_CPUOnly(t *testing.T) { func TestProvision_Idempotent(t *testing.T) { f := &fakeClient{ - sandboxes: []Sandbox{{ID: "sb-existing", Tags: map[string]string{ClaimTagKey: "claim-a"}, Status: "running"}}, + sandboxes: []Sandbox{{ + ID: "sb-existing", + Tags: map[string]string{ClaimTagKey: "claim-a"}, + Status: "running", + }}, } p := newTestProvider(f) - id, err := p.Provision(context.Background(), gpuPod("claim-a", "H100", 1), provider.ProvisionRequest{ClaimName: "claim-a"}) + req := provider.ProvisionRequest{ClaimName: "claim-a"} + id, err := p.Provision(context.Background(), gpuPod("claim-a", "H100", 1), req) if err != nil { t.Fatalf("Provision: %v", err) } @@ -245,7 +251,8 @@ func TestProvision_Idempotent(t *testing.T) { func TestProvision_UnsupportedAccelerator(t *testing.T) { f := &fakeClient{} p := newTestProvider(f) - _, err := p.Provision(context.Background(), gpuPod("claim-x", "TPU-v4", 1), provider.ProvisionRequest{ClaimName: "claim-x"}) + req := provider.ProvisionRequest{ClaimName: "claim-x"} + _, err := p.Provision(context.Background(), gpuPod("claim-x", "TPU-v4", 1), req) if err == nil { t.Fatal("expected error for unsupported accelerator") } @@ -253,18 +260,20 @@ func TestProvision_UnsupportedAccelerator(t *testing.T) { func TestClassifyProvisionError(t *testing.T) { p := newTestProvider(&fakeClient{}) + denyAll := provider.BlockScope{DenyAll: true} + onDemand := provider.BlockScope{CapacityType: nebulav1alpha1.CapacityOnDemand} tests := []struct { name string err error want provider.BlockScope }{ - {"auth sentinel", provider.ErrAuth, provider.BlockScope{DenyAll: true}}, - {"quota sentinel", provider.ErrQuota, provider.BlockScope{DenyAll: true}}, - {"capacity sentinel", provider.ErrNoCapacity, provider.BlockScope{CapacityType: nebulav1alpha1.CapacityOnDemand}}, - {"wrapped capacity", fmt.Errorf("provision: %w", provider.ErrNoCapacity), provider.BlockScope{CapacityType: nebulav1alpha1.CapacityOnDemand}}, - {"string unauthorized", fmt.Errorf("401 unauthorized"), provider.BlockScope{DenyAll: true}}, - {"string no capacity", fmt.Errorf("no capacity available in region"), provider.BlockScope{CapacityType: nebulav1alpha1.CapacityOnDemand}}, - {"unknown", fmt.Errorf("weird transient blip"), provider.BlockScope{DenyAll: true}}, + {"auth sentinel", provider.ErrAuth, denyAll}, + {"quota sentinel", provider.ErrQuota, denyAll}, + {"capacity sentinel", provider.ErrNoCapacity, onDemand}, + {"wrapped capacity", fmt.Errorf("provision: %w", provider.ErrNoCapacity), onDemand}, + {"string unauthorized", fmt.Errorf("401 unauthorized"), denyAll}, + {"string no capacity", fmt.Errorf("no capacity available in region"), onDemand}, + {"unknown", fmt.Errorf("weird transient blip"), denyAll}, {"nil", nil, provider.BlockScope{}}, } for _, tt := range tests { @@ -350,7 +359,8 @@ func TestProvision_NoProbeLeavesSpecUnset(t *testing.T) { f := &fakeClient{createID: "sb-1"} p := newTestProvider(f) - if _, err := p.Provision(context.Background(), gpuPod("claim-a", "H100", 1), provider.ProvisionRequest{ClaimName: "claim-a"}); err != nil { + req := provider.ProvisionRequest{ClaimName: "claim-a"} + if _, err := p.Provision(context.Background(), gpuPod("claim-a", "H100", 1), req); err != nil { t.Fatalf("Provision: %v", err) } if f.lastSpec.ReadinessProbe != nil { diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index 5347f4e..ee820fc 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -316,7 +316,9 @@ func ptrNow(t metav1.Time) *metav1.Time { return &t } // These satisfy the nodeutil.Provider interface required by nodeutil.NewNode and // return a NotFound so the VK core reports them cleanly rather than panicking. -func (h *Handler) GetContainerLogs(context.Context, string, string, string, vkapi.ContainerLogOpts) (io.ReadCloser, error) { +func (h *Handler) GetContainerLogs( + context.Context, string, string, string, vkapi.ContainerLogOpts, +) (io.ReadCloser, error) { return nil, errdefs.NotFound("container logs are not supported by the Nebula virtual node") } diff --git a/pkg/vnode/node.go b/pkg/vnode/node.go index b4086a9..d8133e7 100644 --- a/pkg/vnode/node.go +++ b/pkg/vnode/node.go @@ -132,9 +132,10 @@ func (r *Runner) Start(ctx context.Context) error { // A NaiveNodeProvider marks the node Ready and answers pings; our Handler // owns only pod lifecycle, not node health. np := vknode.NewNaiveNodeProvider() + leases := r.client.CoordinationV1().Leases(corev1.NamespaceNodeLease) nc, err := vknode.NewNodeController( np, nodeSpec, r.client.CoreV1().Nodes(), - vknode.WithNodeEnableLeaseV1(r.client.CoordinationV1().Leases(corev1.NamespaceNodeLease), vknode.DefaultLeaseDuration), + vknode.WithNodeEnableLeaseV1(leases, vknode.DefaultLeaseDuration), ) if err != nil { return fmt.Errorf("build node controller for %q: %w", r.nodeName, err) @@ -157,7 +158,10 @@ func (r *Runner) Start(ctx context.Context) error { // run starts the pod and node controllers in the required order (pods ready // before the node advertises Ready) and blocks until ctx is cancelled or a // controller exits. -func (r *Runner) run(ctx context.Context, pc *vknode.PodController, nc *vknode.NodeController, nodeSpec *corev1.Node, np *vknode.NaiveNodeProviderV2) error { +func (r *Runner) run( + ctx context.Context, pc *vknode.PodController, nc *vknode.NodeController, + nodeSpec *corev1.Node, np *vknode.NaiveNodeProviderV2, +) error { go pc.Run(ctx, 1) //nolint:errcheck select { diff --git a/pkg/vnode/status.go b/pkg/vnode/status.go index d044795..ff5d5cd 100644 --- a/pkg/vnode/status.go +++ b/pkg/vnode/status.go @@ -56,7 +56,7 @@ func applyState(pod *corev1.Pod, state provider.InstanceState, endpoint string, case provider.InstanceRunning: setPhase(pod, corev1.PodRunning, reasonRunning, "external instance is running", now) setReady(pod, corev1.ConditionTrue, now) - setContainerStatuses(pod, corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: now}}, true, now) + setContainerStatuses(pod, corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: now}}, true) if endpoint != "" { pod.Status.PodIP = endpoint } @@ -65,13 +65,17 @@ func applyState(pod *corev1.Pod, state provider.InstanceState, endpoint string, setReady(pod, corev1.ConditionFalse, now) setContainerStatuses(pod, corev1.ContainerState{ Waiting: &corev1.ContainerStateWaiting{Reason: reasonProvisioning, Message: "external instance is starting"}, - }, false, now) + }, false) case provider.InstanceFailed: setPhase(pod, corev1.PodFailed, reasonFailed, "external instance entered a failed state", now) setReady(pod, corev1.ConditionFalse, now) setContainerStatuses(pod, corev1.ContainerState{ - Terminated: &corev1.ContainerStateTerminated{Reason: reasonFailed, Message: "external instance entered a failed state", FinishedAt: now}, - }, false, now) + Terminated: &corev1.ContainerStateTerminated{ + Reason: reasonFailed, + Message: "external instance entered a failed state", + FinishedAt: now, + }, + }, false) case provider.InstanceTerminated: // The instance is simply gone — absent from the provider's List, whether // torn down by us, deleted out-of-band, or reclaimed. We cannot tell WHY @@ -80,8 +84,12 @@ func applyState(pod *corev1.Pod, state provider.InstanceState, endpoint string, setPhase(pod, corev1.PodFailed, reasonTerminated, "external instance is gone", now) setReady(pod, corev1.ConditionFalse, now) setContainerStatuses(pod, corev1.ContainerState{ - Terminated: &corev1.ContainerStateTerminated{Reason: reasonTerminated, Message: "external instance is gone", FinishedAt: now}, - }, false, now) + Terminated: &corev1.ContainerStateTerminated{ + Reason: reasonTerminated, + Message: "external instance is gone", + FinishedAt: now, + }, + }, false) } } @@ -93,7 +101,7 @@ func applyState(pod *corev1.Pod, state provider.InstanceState, endpoint string, // True. There is one real workload behind the whole Pod, so every container // mirrors the same state. The container's Image is echoed back (required field); // RestartCount stays 0 (the provider owns restarts, not the kubelet). -func setContainerStatuses(pod *corev1.Pod, state corev1.ContainerState, ready bool, now metav1.Time) { +func setContainerStatuses(pod *corev1.Pod, state corev1.ContainerState, ready bool) { statuses := make([]corev1.ContainerStatus, 0, len(pod.Spec.Containers)) for i := range pod.Spec.Containers { c := &pod.Spec.Containers[i] From 5c86b0452a0420571ba2e304d1dc1f61c1b6a9e7 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 27 Jul 2026 14:02:28 +0100 Subject: [PATCH 05/12] fix tests Signed-off-by: kerthcet --- .env.example | 2 +- .../controller/pod_placement_controller.go | 10 +---- internal/controller/pod_placement_helpers.go | 2 +- internal/webhook/v1/pod_webhook.go | 12 ++++-- pkg/provider/modal/client.go | 40 +++++++++++++++++-- pkg/provider/modal/modal_test.go | 12 ++++++ test/e2e/e2e_test.go | 18 +++++++++ 7 files changed, 79 insertions(+), 17 deletions(-) diff --git a/.env.example b/.env.example index f817380..75e17ec 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,7 @@ # Consumed by hack/deploy.sh (make deploy-all), which turns these into one # Kubernetes Secret PER PROVIDER. Non-secret config (image, namespace, Kind # cluster) is NOT here — pass it as make variables, e.g. -# make deploy-all IMG=myrepo/nebula:v1 KIND_CLUSTER=nebula-test-e2e +# make deploy-all IMG=myrepo/nebula:v1 DEPLOY_KIND_CLUSTER=nebula-test-e2e # # cp .env.example .env # # edit .env diff --git a/internal/controller/pod_placement_controller.go b/internal/controller/pod_placement_controller.go index dc5ad51..38c923b 100644 --- a/internal/controller/pod_placement_controller.go +++ b/internal/controller/pod_placement_controller.go @@ -18,7 +18,6 @@ package controller import ( "context" - "fmt" "time" corev1 "k8s.io/api/core/v1" @@ -29,6 +28,7 @@ import ( nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/util" ) // staleClaimRequeue is how long placement waits before re-checking when a @@ -125,7 +125,7 @@ func (r *PodPlacementReconciler) Reconcile(ctx context.Context, req ctrl.Request // against the wrong ledger; wait for the NodeClaim backstop to reap it, // then a requeue re-creates the claim with this Pod's UID. log.Info("waiting for a stale NodeClaim to be reclaimed before placing", - "pod", pod.Name, "claim", claimNameForPod(&pod)) + "pod", pod.Name, "claim", util.ClaimName(pod.Namespace, pod.Name)) return ctrl.Result{RequeueAfter: staleClaimRequeue}, nil } @@ -176,9 +176,3 @@ func (r *PodPlacementReconciler) provider(name string) (provider.Provider, bool) } return provider.Get(name) } - -// claimName is the fixed, Pod-derived NodeClaim name, so placement is idempotent -// across retries: one Pod always maps to one claim of this name. -func claimNameForPod(pod *corev1.Pod) string { - return fmt.Sprintf("%s-%s", pod.Namespace, pod.Name) -} diff --git a/internal/controller/pod_placement_helpers.go b/internal/controller/pod_placement_helpers.go index 00e0dd3..444dab1 100644 --- a/internal/controller/pod_placement_helpers.go +++ b/internal/controller/pod_placement_helpers.go @@ -112,7 +112,7 @@ func preferredCapacityType(pool *nebulav1alpha1.NodePool) nebulav1alpha1.Capacit func (r *PodPlacementReconciler) ensureClaim(ctx context.Context, pod *corev1.Pod, pool *nebulav1alpha1.NodePool, p placement) (ready bool, err error) { claim := &nebulav1alpha1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: claimNameForPod(pod), + Name: util.ClaimName(pod.Namespace, pod.Name), Labels: map[string]string{ nebulav1alpha1.ManagedByLabel: nebulav1alpha1.ManagedByValue, nebulav1alpha1.PoolLabel: pool.Name, diff --git a/internal/webhook/v1/pod_webhook.go b/internal/webhook/v1/pod_webhook.go index 66c8341..74a75dc 100644 --- a/internal/webhook/v1/pod_webhook.go +++ b/internal/webhook/v1/pod_webhook.go @@ -118,14 +118,20 @@ func hasGate(pod *corev1.Pod, name string) bool { } // hasProviderToleration reports whether the Pod already tolerates the -// provider-node taint. An existing toleration counts if it would match the -// NoSchedule taint on key nebula.inftyai.com/provider — either a key-only -// Exists toleration or the user's own equivalent — so we never add a duplicate. +// provider-node taint for ANY provider value. Only a key-only Exists toleration +// qualifies: the provider — and thus the taint value — is not chosen at +// admission, so an Operator=Equal toleration matches just one specific value and +// would leave the Pod unable to bind if placement later routes it to a different +// provider's node. Treating such an Equal toleration as sufficient would suppress +// the required Exists toleration, so we require Exists here. func hasProviderToleration(pod *corev1.Pod) bool { for _, t := range pod.Spec.Tolerations { if t.Key != nebulav1alpha1.ProviderLabel { continue } + if t.Operator != corev1.TolerationOpExists { + continue + } if t.Effect != "" && t.Effect != corev1.TaintEffectNoSchedule { continue } diff --git a/pkg/provider/modal/client.go b/pkg/provider/modal/client.go index 53b03ba..f5701de 100644 --- a/pkg/provider/modal/client.go +++ b/pkg/provider/modal/client.go @@ -25,6 +25,7 @@ import ( modal "github.com/modal-labs/modal-client/go" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" "github.com/InftyAI/Nebula/pkg/provider/catalog" ) @@ -122,6 +123,12 @@ func (c *sdkClient) CreateSandbox(ctx context.Context, spec SandboxSpec) (string // or names no supported handler, so a probe-less (or unsupported) workload simply // gets no Modal probe. PeriodSeconds maps to the probe interval; zero leaves the // SDK default (a zero interval is rejected by the SDK constructors). +// +// A TCP/HTTPGet probe with a NAMED port (e.g. port: http) is treated as +// unsupported: resolving the name to a number needs the container's ports list, +// which this helper does not have, and passing the intstr's 0 fallback would +// create an invalid Modal TCP probe on port 0. So a named port omits the probe +// rather than emitting a bogus one. func modalProbe(p *corev1.Probe) (*modal.Probe, error) { if p == nil { return nil, nil @@ -136,14 +143,34 @@ func modalProbe(p *corev1.Probe) (*modal.Probe, error) { case p.Exec != nil && len(p.Exec.Command) > 0: return modal.NewExecProbe(p.Exec.Command, &modal.ExecProbeParams{Interval: interval}) case p.TCPSocket != nil: - return modal.NewTCPProbe(p.TCPSocket.Port.IntValue(), &modal.TCPProbeParams{Interval: interval}) + if port, ok := numericPort(p.TCPSocket.Port); ok { + return modal.NewTCPProbe(port, &modal.TCPProbeParams{Interval: interval}) + } + return nil, nil // named port: unsupported here (see doc) case p.HTTPGet != nil: - return modal.NewTCPProbe(p.HTTPGet.Port.IntValue(), &modal.TCPProbeParams{Interval: interval}) + if port, ok := numericPort(p.HTTPGet.Port); ok { + return modal.NewTCPProbe(port, &modal.TCPProbeParams{Interval: interval}) + } + return nil, nil // named port: unsupported here (see doc) default: return nil, nil } } +// numericPort returns an intstr port as an int when it is numeric (>0), reporting +// ok=false for a named port or a non-positive value. IntValue() yields 0 for a +// named port, which is not a usable Modal probe target, so callers must skip the +// probe rather than emit port 0. +func numericPort(p intstr.IntOrString) (int, bool) { + if p.Type != intstr.Int { + return 0, false + } + if v := p.IntValue(); v > 0 { + return v, true + } + return 0, false +} + // TerminateSandbox implements Client. Idempotent: a sandbox that no longer // exists resolves to a not-found from FromID, which we treat as already gone. func (c *sdkClient) TerminateSandbox(ctx context.Context, id string) error { @@ -176,8 +203,13 @@ func (c *sdkClient) GetSandbox(ctx context.Context, id string) (*Sandbox, error) return &out, nil } -// ListSandboxes implements Client. Filters server-side by the Nebula claim tag -// key so only Nebula-owned sandboxes are returned. +// ListSandboxes implements Client. It scopes the list server-side to Nebula's +// own Modal App (AppID), which is what isolates Nebula-owned sandboxes: every +// sandbox Nebula creates lives under this one App, so sandboxes in other Apps +// are never returned. It does NOT filter by the claim tag — the SDK's Tags +// filter matches exact key=value pairs, but ClaimTagKey carries a distinct +// value (the claim name) per sandbox, so there is no "has this key with any +// value" filter to select all Nebula sandboxes by. App scoping already does it. func (c *sdkClient) ListSandboxes(ctx context.Context) ([]Sandbox, error) { app, err := c.app(ctx) if err != nil { diff --git a/pkg/provider/modal/modal_test.go b/pkg/provider/modal/modal_test.go index 1d1de1d..18c2610 100644 --- a/pkg/provider/modal/modal_test.go +++ b/pkg/provider/modal/modal_test.go @@ -396,4 +396,16 @@ func TestModalProbe(t *testing.T) { t.Fatalf("case %d: expected a Modal probe, got nil", i) } } + + // A NAMED port cannot be resolved here (needs the container's ports list), so + // TCP/HTTPGet with a named port omits the probe rather than emitting port 0. + named := []*corev1.Probe{ + {ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString("http")}}}, + {ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Port: intstr.FromString("http")}}}, + } + for i, pr := range named { + if got, err := modalProbe(pr); err != nil || got != nil { + t.Fatalf("named-port case %d: modalProbe = (%v, %v), want (nil, nil)", i, got, err) + } + } } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 41c1006..d5aff60 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -65,10 +65,28 @@ var _ = Describe("Manager", Ordered, func() { _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + // The manager mounts webhook-server-cert as a REQUIRED volume, so the + // self-signed cert Secret must exist before the pod starts, or it wedges + // in ContainerCreating (Pending) forever. Nebula does not use cert-manager + // (see config/default/kustomization.yaml); hack/gen-webhook-cert.sh is the + // source of truth, matching the ordering hack/deploy.sh uses in prod. + By("provisioning the webhook serving certificate Secret") + cmd = exec.Command("hack/gen-webhook-cert.sh", "secret") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to provision the webhook serving cert Secret") + By("deploying the controller-manager") cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + + // caBundle injection is server-side (only the API server reads it) and + // requires the MutatingWebhookConfiguration created by `make deploy`, so it + // runs after the deploy. The manager pod is untouched — no restart needed. + By("injecting the webhook CA bundle") + cmd = exec.Command("hack/gen-webhook-cert.sh", "cabundle") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to inject the webhook CA bundle") }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, From 40219df451055b97b32388e4d967c18e5c70893a Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 27 Jul 2026 17:08:32 +0100 Subject: [PATCH 06/12] address comments Signed-off-by: kerthcet --- test/utils/utils.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/utils/utils.go b/test/utils/utils.go index 448055b..8ae13e7 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -42,12 +42,14 @@ func warnError(err error) { // Run executes the provided command within this context func Run(cmd *exec.Cmd) (string, error) { - dir, _ := GetProjectDir() - cmd.Dir = dir - - if err := os.Chdir(cmd.Dir); err != nil { - _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %q\n", err) + // Isolate the working directory to the command via cmd.Dir rather than + // os.Chdir: os.Chdir mutates the process-wide cwd, which is not goroutine-safe + // and would corrupt parallel commands (and anything else relying on cwd). + dir, err := GetProjectDir() + if err != nil { + return "", fmt.Errorf("get project dir: %w", err) } + cmd.Dir = dir cmd.Env = append(os.Environ(), "GO111MODULE=on") command := strings.Join(cmd.Args, " ") @@ -214,7 +216,7 @@ func UncommentCode(filename, target, prefix string) error { idx := strings.Index(strContent, target) if idx < 0 { - return fmt.Errorf("unable to find the code %q to be uncomment", target) + return fmt.Errorf("unable to find the code %q to uncomment", target) } out := new(bytes.Buffer) From 89ad6b0162d80c99992f5f8330d16d906295ba62 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 27 Jul 2026 17:29:28 +0100 Subject: [PATCH 07/12] add fake provider Signed-off-by: kerthcet --- .env.example | 3 - cmd/main.go | 11 ++ config/manager/manager.yaml | 3 +- config/manager/modal-credentials.example.yaml | 6 - docs/deploy.md | 4 +- hack/deploy.sh | 3 +- pkg/provider/fake/fake.go | 165 ++++++++++++++++++ pkg/provider/fake/fake_test.go | 129 ++++++++++++++ test/e2e/e2e_test.go | 103 +++++++++++ 9 files changed, 412 insertions(+), 15 deletions(-) create mode 100644 pkg/provider/fake/fake.go create mode 100644 pkg/provider/fake/fake_test.go diff --git a/.env.example b/.env.example index 75e17ec..388c2fb 100644 --- a/.env.example +++ b/.env.example @@ -15,9 +15,6 @@ # provider is then skipped at registration — not fatal). MODAL_TOKEN_ID= MODAL_TOKEN_SECRET= -# Optional: Modal workspace environment and the App all sandboxes run under. -MODAL_ENVIRONMENT= -MODAL_APP_NAME=nebula # --- Additional providers (add as adapters land) --------------------------- # Each provider gets its OWN secret (see hack/deploy.sh PROVIDER_SECRETS), e.g.: diff --git a/cmd/main.go b/cmd/main.go index b505780..2e530a2 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -46,6 +46,7 @@ import ( "github.com/InftyAI/Nebula/internal/controller" webhookv1 "github.com/InftyAI/Nebula/internal/webhook/v1" "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/provider/fake" "github.com/InftyAI/Nebula/pkg/provider/modal" "github.com/InftyAI/Nebula/pkg/vnode" // +kubebuilder:scaffold:imports @@ -325,4 +326,14 @@ func registerProviders(ctx context.Context) { provider.Register(p) setupLog.Info("registered provider", "provider", p.Name()) } + + // The fake provider is an in-memory backend used only by the e2e suite to + // exercise the full control-plane loop without cloud credentials. It ships in + // the binary but registers ONLY when explicitly enabled, so it can never place + // real workloads in production. + if os.Getenv("NEBULA_ENABLE_FAKE_PROVIDER") == "true" { + p := fake.New() + provider.Register(p) + setupLog.Info("registered provider", "provider", p.Name()) + } } diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 64fe631..895d52b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -81,8 +81,7 @@ spec: # provider. Providers namespace their env vars (MODAL_TOKEN_ID, # RUNPOD_API_KEY, ...), so the merged environment never collides. # - # The Modal SDK reads MODAL_TOKEN_ID / MODAL_TOKEN_SECRET (and optionally - # MODAL_ENVIRONMENT / MODAL_APP_NAME) from the environment. + # The Modal SDK reads MODAL_TOKEN_ID / MODAL_TOKEN_SECRET from the environment. - secretRef: name: nebula-modal-credentials optional: true diff --git a/config/manager/modal-credentials.example.yaml b/config/manager/modal-credentials.example.yaml index c733ab7..6f7c0c2 100644 --- a/config/manager/modal-credentials.example.yaml +++ b/config/manager/modal-credentials.example.yaml @@ -16,10 +16,6 @@ # --from-literal=MODAL_TOKEN_ID=ak-xxxxxxxxxxxxxxxx \ # --from-literal=MODAL_TOKEN_SECRET=as-xxxxxxxxxxxxxxxx # -# Optional keys: MODAL_ENVIRONMENT (Modal workspace environment) and -# MODAL_APP_NAME (the Modal App all Nebula sandboxes are created under; -# defaults to "nebula"). -# # The key NAMES matter: the Modal SDK reads MODAL_TOKEN_ID / MODAL_TOKEN_SECRET # straight from the environment, and envFrom projects every key in this Secret # as an env var of the same name. @@ -36,5 +32,3 @@ type: Opaque stringData: MODAL_TOKEN_ID: "ak-REPLACE_ME" MODAL_TOKEN_SECRET: "as-REPLACE_ME" - # MODAL_ENVIRONMENT: "main" - # MODAL_APP_NAME: "nebula" diff --git a/docs/deploy.md b/docs/deploy.md index deec82a..f8e0933 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -134,8 +134,6 @@ cert-manager, and drop the `gen-webhook-cert.sh` calls from `hack/deploy.sh`. |---|---|---|---| | `MODAL_TOKEN_ID` | Modal | yes | From `modal token new` | | `MODAL_TOKEN_SECRET` | Modal | yes | From `modal token new` | -| `MODAL_ENVIRONMENT` | Modal | no | Modal workspace environment | -| `MODAL_APP_NAME` | Modal | no | Modal App for all sandboxes (default `nebula`) | Non-secret config, passed as `make` variables: @@ -156,7 +154,7 @@ When a new provider adapter lands, wire its credentials in two places: ```bash PROVIDER_SECRETS=( - "nebula-modal-credentials|MODAL_TOKEN_ID MODAL_TOKEN_SECRET|MODAL_ENVIRONMENT MODAL_APP_NAME" + "nebula-modal-credentials|MODAL_TOKEN_ID MODAL_TOKEN_SECRET" "nebula-runpod-credentials|RUNPOD_API_KEY|" ) ``` diff --git a/hack/deploy.sh b/hack/deploy.sh index 9b358d1..05a92e2 100755 --- a/hack/deploy.sh +++ b/hack/deploy.sh @@ -60,7 +60,8 @@ fi # SDKs read them by these exact names). To add a provider, append a row — # nothing else in this script needs to change. PROVIDER_SECRETS=( - "nebula-modal-credentials|MODAL_TOKEN_ID MODAL_TOKEN_SECRET|MODAL_ENVIRONMENT MODAL_APP_NAME" + # Only secrets belong here. + "nebula-modal-credentials|MODAL_TOKEN_ID MODAL_TOKEN_SECRET|" # "nebula-runpod-credentials|RUNPOD_API_KEY|" ) diff --git a/pkg/provider/fake/fake.go b/pkg/provider/fake/fake.go new file mode 100644 index 0000000..baa03ec --- /dev/null +++ b/pkg/provider/fake/fake.go @@ -0,0 +1,165 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package fake implements a fully in-memory provider.Provider. It provisions +// nothing real: Provision records the claim in a map and reports the instance +// Running immediately, List/Get read that map, and Terminate forgets it. This +// lets the whole control-plane loop — webhook gate → placement → NodeClaim → +// virtual-kubelet Provision/List → Pod Running — run against a plain Kind +// cluster with no cloud credentials, which is exactly what the e2e suite needs. +// +// It is NOT registered by default. cmd/main.go wires it only when +// NEBULA_ENABLE_FAKE_PROVIDER=true, so it never registers in production even +// though it ships in the binary (see registerProviders). +package fake + +import ( + "context" + "fmt" + "sync" + + corev1 "k8s.io/api/core/v1" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/provider/catalog" +) + +// ProviderName is the fake provider's stable identifier. It is intentionally not +// one of the canonical NeoCloud names in pkg/provider (runpod, modal, ...): a +// NodePool referencing "fake" can only resolve when the fake is explicitly +// enabled, so a real cluster never places onto it by accident. +const ProviderName = "fake" + +// compile-time assertion that Provider satisfies the interface. +var _ provider.Provider = (*Provider)(nil) + +// Provider is the in-memory backend. Offerings/Name/MapAccelerator come from the +// embedded catalog.Base (fed a static in-memory catalog); the lifecycle methods +// operate on the guarded instances map. +type Provider struct { + catalog.Base + + mu sync.Mutex + instances map[string]*provider.Instance // key: instance id + nextID int +} + +// New returns a fake Provider whose catalog offers a small, fixed set of +// accelerators (so GPU Pods match in selectPlacement) plus CPU-only workloads, +// all as OnDemand. +func New() *Provider { + return &Provider{ + Base: catalog.Base{ProviderName: ProviderName, Catalog: fixedCatalog{}}, + instances: make(map[string]*provider.Instance), + } +} + +// Capabilities reports a plain OnDemand-only, tag-bearing backend with no +// preemption — the simplest shape, so the control plane exercises its default +// paths. +func (p *Provider) Capabilities() provider.Capabilities { + return provider.Capabilities{ + SupportsStop: false, + SupportsSpot: false, + NativeTags: true, + PreemptionNotice: 0, + PollInterval: 0, // use the vnode default cadence + } +} + +// Provision records one instance for the claim and reports it Running at once. +// Idempotent on ClaimName: a repeat returns the existing instance's id rather +// than creating a second (matching the real adapters' contract). +func (p *Provider) Provision(_ context.Context, pod *corev1.Pod, req provider.ProvisionRequest) (string, error) { + if pod == nil { + return "", fmt.Errorf("fake: nil pod") + } + if req.ClaimName == "" { + return "", fmt.Errorf("fake: empty ClaimName in ProvisionRequest") + } + + p.mu.Lock() + defer p.mu.Unlock() + + for _, inst := range p.instances { + if inst.ClaimName == req.ClaimName { + return inst.ID, nil // idempotent reuse + } + } + + p.nextID++ + id := fmt.Sprintf("fake-%d", p.nextID) + p.instances[id] = &provider.Instance{ + ID: id, + ClaimName: req.ClaimName, + State: provider.InstanceRunning, + Endpoint: fmt.Sprintf("fake://%s", id), + CapacityType: req.CapacityType, + } + return id, nil +} + +// Terminate forgets the instance. Idempotent: terminating an already-gone (or +// never-created) instance returns nil. +func (p *Provider) Terminate(_ context.Context, instanceID string) error { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.instances, instanceID) + return nil +} + +// Get returns one instance, or (nil, nil) if it no longer exists. +func (p *Provider) Get(_ context.Context, instanceID string) (*provider.Instance, error) { + p.mu.Lock() + defer p.mu.Unlock() + inst, ok := p.instances[instanceID] + if !ok { + return nil, nil + } + out := *inst // copy so callers can't mutate our map entry + return &out, nil +} + +// List returns every instance this fake currently holds. +func (p *Provider) List(_ context.Context) ([]provider.Instance, error) { + p.mu.Lock() + defer p.mu.Unlock() + out := make([]provider.Instance, 0, len(p.instances)) + for _, inst := range p.instances { + out = append(out, *inst) + } + return out, nil +} + +// ClassifyProvisionError never really fails to provision, so any error it is +// asked to classify is treated as a whole-provider block on OnDemand — the same +// shared derivation the real adapters use. +func (p *Provider) ClassifyProvisionError(err error) provider.BlockScope { + return provider.ClassifyError(err, nebulav1alpha1.CapacityOnDemand) +} + +// fixedCatalog is a trivial catalog.Lookup: a static set of OnDemand offerings +// so a GPU Pod matches the fake in selectPlacement (via the embedded Base's +// MapAccelerator) without loading any CSV. +type fixedCatalog struct{} + +func (fixedCatalog) Offerings(string) []provider.Offering { + return []provider.Offering{ + {AcceleratorType: "H100", CapacityType: nebulav1alpha1.CapacityOnDemand, PricePerHour: 1, Available: true}, + {AcceleratorType: "A100-80GB", CapacityType: nebulav1alpha1.CapacityOnDemand, PricePerHour: 1, Available: true}, + } +} diff --git a/pkg/provider/fake/fake_test.go b/pkg/provider/fake/fake_test.go new file mode 100644 index 0000000..d727ea2 --- /dev/null +++ b/pkg/provider/fake/fake_test.go @@ -0,0 +1,129 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" +) + +func testPod() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "default"}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "main", Image: "img"}}}, + } +} + +func TestProvisionReportsRunningAndLists(t *testing.T) { + p := New() + ctx := context.Background() + + id, err := p.Provision(ctx, testPod(), provider.ProvisionRequest{ + ClaimName: "claim-a", + CapacityType: nebulav1alpha1.CapacityOnDemand, + }) + if err != nil { + t.Fatalf("Provision: %v", err) + } + if id == "" { + t.Fatal("expected a non-empty instance id") + } + + // Get reports it Running with the claim recovered. + inst, err := p.Get(ctx, id) + if err != nil || inst == nil { + t.Fatalf("Get = (%v, %v), want a live instance", inst, err) + } + if inst.State != provider.InstanceRunning { + t.Fatalf("state = %q, want Running", inst.State) + } + if inst.ClaimName != "claim-a" { + t.Fatalf("claim = %q, want claim-a", inst.ClaimName) + } + + // List surfaces exactly the one instance. + list, err := p.List(ctx) + if err != nil { + t.Fatalf("List: %v", err) + } + if len(list) != 1 || list[0].ID != id { + t.Fatalf("List = %v, want one instance %q", list, id) + } +} + +func TestProvisionIdempotentOnClaim(t *testing.T) { + p := New() + ctx := context.Background() + req := provider.ProvisionRequest{ClaimName: "claim-a"} + + id1, err := p.Provision(ctx, testPod(), req) + if err != nil { + t.Fatalf("Provision #1: %v", err) + } + id2, err := p.Provision(ctx, testPod(), req) + if err != nil { + t.Fatalf("Provision #2: %v", err) + } + if id1 != id2 { + t.Fatalf("ids differ (%q vs %q); Provision must be idempotent on ClaimName", id1, id2) + } + if list, _ := p.List(ctx); len(list) != 1 { + t.Fatalf("expected 1 instance after idempotent re-provision, got %d", len(list)) + } +} + +func TestTerminateIsIdempotent(t *testing.T) { + p := New() + ctx := context.Background() + + id, err := p.Provision(ctx, testPod(), provider.ProvisionRequest{ClaimName: "claim-a"}) + if err != nil { + t.Fatalf("Provision: %v", err) + } + if err := p.Terminate(ctx, id); err != nil { + t.Fatalf("Terminate: %v", err) + } + // Gone from Get/List. + if inst, _ := p.Get(ctx, id); inst != nil { + t.Fatalf("Get after Terminate = %v, want nil (terminated)", inst) + } + // A repeat Terminate (and terminating an unknown id) is a no-op, not an error. + if err := p.Terminate(ctx, id); err != nil { + t.Fatalf("repeat Terminate: %v", err) + } + if err := p.Terminate(ctx, "never-existed"); err != nil { + t.Fatalf("Terminate unknown id: %v", err) + } +} + +func TestMapAcceleratorFromCatalog(t *testing.T) { + p := New() + // A GPU in the fixed catalog resolves (case-insensitively); one that is not + // offered reports ok=false, so selectPlacement skips the fake for it. + if _, ok := p.MapAccelerator("h100"); !ok { + t.Fatal("expected H100 to be offered by the fake catalog") + } + if _, ok := p.MapAccelerator("TPU-v4"); ok { + t.Fatal("did not expect TPU-v4 to be offered by the fake catalog") + } +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index d5aff60..3ec294d 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -42,6 +42,17 @@ const metricsServiceName = "nebula-controller-manager-metrics-service" // metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data const metricsRoleBindingName = "nebula-metrics-binding" +// Fake-provider placement-flow fixtures. fakeProviderName must match +// fake.ProviderName; fakeVirtualNode is vnode.NodeName(fakeProviderName) +// ("nebula-"). Kept as literals here so the e2e package stays free of +// production imports. +const ( + fakeProviderName = "fake" + fakeVirtualNode = "nebula-fake" + fakePoolName = "e2e-fake-pool" + fakeWorkloadPod = "e2e-fake-workload" +) + var _ = Describe("Manager", Ordered, func() { var controllerPodName string @@ -80,6 +91,17 @@ var _ = Describe("Manager", Ordered, func() { _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + // Enable the in-memory fake provider so the placement→NodeClaim→virtual-node + // flow can be exercised end-to-end without cloud credentials (no real + // provider registers in a Kind cluster). It is gated on this env var and + // never registers in production. Set it AFTER deploy and wait for the + // rollout so the manager restarts with the fake registered. + By("enabling the fake provider on the controller-manager") + cmd = exec.Command("kubectl", "set", "env", "deployment/nebula-controller-manager", + "-n", namespace, "NEBULA_ENABLE_FAKE_PROVIDER=true") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to enable the fake provider") + // caBundle injection is server-side (only the API server reads it) and // requires the MutatingWebhookConfiguration created by `make deploy`, so it // runs after the deploy. The manager pod is untouched — no restart needed. @@ -87,6 +109,12 @@ var _ = Describe("Manager", Ordered, func() { cmd = exec.Command("hack/gen-webhook-cert.sh", "cabundle") _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to inject the webhook CA bundle") + + By("waiting for the controller-manager rollout to finish") + cmd = exec.Command("kubectl", "rollout", "status", + "deployment/nebula-controller-manager", "-n", namespace, "--timeout=120s") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "controller-manager rollout did not complete") }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, @@ -96,6 +124,11 @@ var _ = Describe("Manager", Ordered, func() { cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) _, _ = utils.Run(cmd) + By("cleaning up the fake-provider workload and pool") + _, _ = utils.Run(exec.Command("kubectl", "delete", "pod", fakeWorkloadPod, + "-n", namespace, "--ignore-not-found=true")) + _, _ = utils.Run(exec.Command("kubectl", "delete", "nodepool", fakePoolName, "--ignore-not-found=true")) + By("undeploying the controller-manager") cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) @@ -299,6 +332,76 @@ var _ = Describe("Manager", Ordered, func() { Eventually(verifyCAInjection).Should(Succeed()) }) + It("should place an opted-in Pod onto the fake provider's virtual node", func() { + // This exercises the whole control-plane loop against the in-memory fake + // provider: webhook gate → placement picks the pool's provider → NodeClaim + // ledger → the fake's virtual node provisions → the Pod binds and runs. + + By("waiting for the fake provider's virtual node to register") + verifyNode := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "node", fakeVirtualNode) + _, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "fake virtual node not registered") + } + Eventually(verifyNode).Should(Succeed()) + + By("creating a NodePool that allows the fake provider") + manifest := fmt.Sprintf(`apiVersion: nebula.inftyai.com/v1alpha1 +kind: NodePool +metadata: + name: %s +spec: + providers: + - name: %s + capacityTypes: + - OnDemand + strategy: Ordered +--- +apiVersion: v1 +kind: Pod +metadata: + name: %s + namespace: %s + labels: + nebula.inftyai.com/enabled: "true" + nebula.inftyai.com/nodepool: %s +spec: + containers: + - name: main + image: registry.k8s.io/pause:3.10 +`, fakePoolName, fakeProviderName, fakeWorkloadPod, namespace, fakePoolName) + manifestFile := filepath.Join("/tmp", "nebula-fake-workload.yaml") + Expect(os.WriteFile(manifestFile, []byte(manifest), 0o644)).To(Succeed()) + cmd := exec.Command("kubectl", "apply", "-f", manifestFile) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to apply the NodePool + Pod") + + By("verifying the placement controller creates a NodeClaim for the Pod") + verifyClaim := func(g Gomega) { + // util.ClaimName(namespace, name) == "-". + claim := fmt.Sprintf("%s-%s", namespace, fakeWorkloadPod) + cmd := exec.Command("kubectl", "get", "nodeclaim", claim, + "-o", "jsonpath={.spec.provider}") + out, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "NodeClaim not created") + g.Expect(out).To(Equal(fakeProviderName), "NodeClaim names the wrong provider") + } + Eventually(verifyClaim).Should(Succeed()) + + By("verifying the Pod is ungated and bound to the fake virtual node") + verifyBound := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pod", fakeWorkloadPod, "-n", namespace, + "-o", "jsonpath={.spec.nodeName}") + out, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).To(Equal(fakeVirtualNode), "Pod not bound to the fake virtual node") + } + Eventually(verifyBound).Should(Succeed()) + + By("cleaning up the fake workload") + _, _ = utils.Run(exec.Command("kubectl", "delete", "-f", manifestFile, "--ignore-not-found=true")) + }) + // +kubebuilder:scaffold:e2e-webhooks-checks // TODO: Customize the e2e test suite with scenarios specific to your project. From 9f23b808ec24c39cbfc80844fef6c37b94d8928e Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 27 Jul 2026 17:45:53 +0100 Subject: [PATCH 08/12] fix tests Signed-off-by: kerthcet --- Makefile | 5 ++++ config/e2e/enable_fake_provider_patch.yaml | 12 +++++++++ config/e2e/kustomization.yaml | 14 ++++++++++ test/e2e/e2e_test.go | 31 ++++++++-------------- 4 files changed, 42 insertions(+), 20 deletions(-) create mode 100644 config/e2e/enable_fake_provider_patch.yaml create mode 100644 config/e2e/kustomization.yaml diff --git a/Makefile b/Makefile index 4c8d8d1..e9b163f 100644 --- a/Makefile +++ b/Makefile @@ -197,6 +197,11 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_FLAGS) config/default | $(KUBECTL) apply -f - +.PHONY: deploy-e2e +deploy-e2e: manifests kustomize ## Deploy for e2e: config/default plus the fake-provider env var (baked in at deploy time, not via a post-deploy rollout). + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_FLAGS) config/e2e | $(KUBECTL) apply -f - + .PHONY: undeploy undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(KUSTOMIZE) build $(KUSTOMIZE_BUILD_FLAGS) config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - diff --git a/config/e2e/enable_fake_provider_patch.yaml b/config/e2e/enable_fake_provider_patch.yaml new file mode 100644 index 0000000..3e929dd --- /dev/null +++ b/config/e2e/enable_fake_provider_patch.yaml @@ -0,0 +1,12 @@ +# e2e-only: enable the in-memory fake provider so the placement→NodeClaim→ +# virtual-node flow can be exercised without cloud credentials. This overlay is +# built ONLY by `make deploy-e2e` (see the e2e suite); production `make deploy` +# uses config/default, which never sets this env, so the fake never registers in +# a real cluster. Baked in at deploy time (not `kubectl set env`) so the manager +# boots once with the fake already registered — no extra rollout, and thus no +# leader-election re-acquire window where metrics serve before the first reconcile. +- op: add + path: /spec/template/spec/containers/0/env/- + value: + name: NEBULA_ENABLE_FAKE_PROVIDER + value: "true" diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml new file mode 100644 index 0000000..f76bd7f --- /dev/null +++ b/config/e2e/kustomization.yaml @@ -0,0 +1,14 @@ +# e2e overlay: the production config/default plus the fake-provider env var, so +# the e2e suite can drive the full placement flow with no cloud credentials. +# Built by `make deploy-e2e`; never used by production `make deploy`. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../default + +patches: +- path: enable_fake_provider_patch.yaml + target: + kind: Deployment + name: controller-manager diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 3ec294d..34b8dac 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -86,35 +86,26 @@ var _ = Describe("Manager", Ordered, func() { _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to provision the webhook serving cert Secret") - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) + // Deploy via the e2e overlay, which bakes NEBULA_ENABLE_FAKE_PROVIDER=true + // into the manager env so the in-memory fake provider registers at first + // boot. This drives the placement→NodeClaim→virtual-node flow without cloud + // credentials (no real provider registers in a Kind cluster). Baking it in + // (vs. a post-deploy `kubectl set env`) means the manager boots ONCE — no + // second rollout, so no leader-election re-acquire window where the metrics + // endpoint serves before the first reconcile. The fake ships in the binary + // but only registers on this env var, so production `make deploy` never has it. + By("deploying the controller-manager (e2e overlay: fake provider enabled)") + cmd = exec.Command("make", "deploy-e2e", fmt.Sprintf("IMG=%s", projectImage)) _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") - // Enable the in-memory fake provider so the placement→NodeClaim→virtual-node - // flow can be exercised end-to-end without cloud credentials (no real - // provider registers in a Kind cluster). It is gated on this env var and - // never registers in production. Set it AFTER deploy and wait for the - // rollout so the manager restarts with the fake registered. - By("enabling the fake provider on the controller-manager") - cmd = exec.Command("kubectl", "set", "env", "deployment/nebula-controller-manager", - "-n", namespace, "NEBULA_ENABLE_FAKE_PROVIDER=true") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to enable the fake provider") - // caBundle injection is server-side (only the API server reads it) and - // requires the MutatingWebhookConfiguration created by `make deploy`, so it + // requires the MutatingWebhookConfiguration created by the deploy, so it // runs after the deploy. The manager pod is untouched — no restart needed. By("injecting the webhook CA bundle") cmd = exec.Command("hack/gen-webhook-cert.sh", "cabundle") _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to inject the webhook CA bundle") - - By("waiting for the controller-manager rollout to finish") - cmd = exec.Command("kubectl", "rollout", "status", - "deployment/nebula-controller-manager", "-n", namespace, "--timeout=120s") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "controller-manager rollout did not complete") }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, From 8420564c6a6e4d366f99282b4a4d8ea13f2259db Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 27 Jul 2026 17:49:32 +0100 Subject: [PATCH 09/12] trim the CM Signed-off-by: kerthcet --- .github/workflows/test-e2e.yml | 5 +++++ test/e2e/e2e_test.go | 10 ++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 68fd1ed..da887e1 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -27,6 +27,11 @@ jobs: run: kind version - name: Running Test e2e + # Nebula does not use cert-manager: the webhook serving cert is a + # self-signed Secret from hack/gen-webhook-cert.sh (see the e2e BeforeAll), + # so skip the suite's cert-manager install/uninstall entirely. + env: + CERT_MANAGER_INSTALL_SKIP: "true" run: | go mod tidy make test-e2e diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 34b8dac..be2c821 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -299,14 +299,16 @@ var _ = Describe("Manager", Ordered, func() { )) }) - It("should provisioned cert-manager", func() { - By("validating that cert-manager has the certificate Secret") - verifyCertManager := func(g Gomega) { + It("should have the self-signed webhook serving cert Secret", func() { + // Nebula does not use cert-manager; hack/gen-webhook-cert.sh creates this + // self-signed Secret in the e2e BeforeAll (and hack/deploy.sh in prod). + By("validating that the webhook serving cert Secret exists") + verifyCertSecret := func(g Gomega) { cmd := exec.Command("kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace) _, err := utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred()) } - Eventually(verifyCertManager).Should(Succeed()) + Eventually(verifyCertSecret).Should(Succeed()) }) It("should have CA injection for mutating webhooks", func() { From a831f2e656a8560c2bf12f933857db53ee2346e9 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 27 Jul 2026 17:56:21 +0100 Subject: [PATCH 10/12] fix test Signed-off-by: kerthcet --- test/e2e/e2e_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index be2c821..d368b79 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -359,9 +359,18 @@ metadata: nebula.inftyai.com/enabled: "true" nebula.inftyai.com/nodepool: %s spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault containers: - name: main image: registry.k8s.io/pause:3.10 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL `, fakePoolName, fakeProviderName, fakeWorkloadPod, namespace, fakePoolName) manifestFile := filepath.Join("/tmp", "nebula-fake-workload.yaml") Expect(os.WriteFile(manifestFile, []byte(manifest), 0o644)).To(Succeed()) From 3d0c63f5b23033f5ccd43d0a121cab68d5db67ce Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 27 Jul 2026 18:05:45 +0100 Subject: [PATCH 11/12] fix test Signed-off-by: kerthcet --- test/e2e/e2e_test.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index d368b79..f5b2d29 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -51,6 +51,11 @@ const ( fakeVirtualNode = "nebula-fake" fakePoolName = "e2e-fake-pool" fakeWorkloadPod = "e2e-fake-workload" + // fakeWorkloadNS is a dedicated namespace for the placement-flow workload. It + // must NOT be the manager namespace: the mutating webhook's namespaceSelector + // excludes nebula-system (see config/webhook/selector_patch.yaml), so a Pod + // there would never get the scheduling gate and placement would never run. + fakeWorkloadNS = "nebula-e2e-workload" ) var _ = Describe("Manager", Ordered, func() { @@ -115,10 +120,11 @@ var _ = Describe("Manager", Ordered, func() { cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) _, _ = utils.Run(cmd) - By("cleaning up the fake-provider workload and pool") + By("cleaning up the fake-provider workload, pool, and namespace") _, _ = utils.Run(exec.Command("kubectl", "delete", "pod", fakeWorkloadPod, - "-n", namespace, "--ignore-not-found=true")) + "-n", fakeWorkloadNS, "--ignore-not-found=true")) _, _ = utils.Run(exec.Command("kubectl", "delete", "nodepool", fakePoolName, "--ignore-not-found=true")) + _, _ = utils.Run(exec.Command("kubectl", "delete", "ns", fakeWorkloadNS, "--ignore-not-found=true")) By("undeploying the controller-manager") cmd = exec.Command("make", "undeploy") @@ -338,6 +344,13 @@ var _ = Describe("Manager", Ordered, func() { } Eventually(verifyNode).Should(Succeed()) + By("creating the workload namespace (webhook-eligible, restricted policy)") + _, _ = utils.Run(exec.Command("kubectl", "create", "ns", fakeWorkloadNS)) + cmd := exec.Command("kubectl", "label", "--overwrite", "ns", fakeWorkloadNS, + "pod-security.kubernetes.io/enforce=restricted") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label the workload namespace") + By("creating a NodePool that allows the fake provider") manifest := fmt.Sprintf(`apiVersion: nebula.inftyai.com/v1alpha1 kind: NodePool @@ -371,17 +384,17 @@ spec: capabilities: drop: - ALL -`, fakePoolName, fakeProviderName, fakeWorkloadPod, namespace, fakePoolName) +`, fakePoolName, fakeProviderName, fakeWorkloadPod, fakeWorkloadNS, fakePoolName) manifestFile := filepath.Join("/tmp", "nebula-fake-workload.yaml") Expect(os.WriteFile(manifestFile, []byte(manifest), 0o644)).To(Succeed()) - cmd := exec.Command("kubectl", "apply", "-f", manifestFile) - _, err := utils.Run(cmd) + cmd = exec.Command("kubectl", "apply", "-f", manifestFile) + _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to apply the NodePool + Pod") By("verifying the placement controller creates a NodeClaim for the Pod") verifyClaim := func(g Gomega) { // util.ClaimName(namespace, name) == "-". - claim := fmt.Sprintf("%s-%s", namespace, fakeWorkloadPod) + claim := fmt.Sprintf("%s-%s", fakeWorkloadNS, fakeWorkloadPod) cmd := exec.Command("kubectl", "get", "nodeclaim", claim, "-o", "jsonpath={.spec.provider}") out, err := utils.Run(cmd) @@ -392,7 +405,7 @@ spec: By("verifying the Pod is ungated and bound to the fake virtual node") verifyBound := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "pod", fakeWorkloadPod, "-n", namespace, + cmd := exec.Command("kubectl", "get", "pod", fakeWorkloadPod, "-n", fakeWorkloadNS, "-o", "jsonpath={.spec.nodeName}") out, err := utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred()) From c6a0289cf067c7f6e1ce9b30cd446147b2f63cf8 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 27 Jul 2026 18:16:14 +0100 Subject: [PATCH 12/12] decrease the time interval to 15s Signed-off-by: kerthcet --- pkg/provider/provider.go | 2 +- pkg/vnode/handler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 3a0261a..b8f75d5 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -136,7 +136,7 @@ type Capabilities struct { // provider: a spot-heavy backend where preemption is common and costly wants a // short interval to notice reclaims quickly, while an OnDemand-only backend // that never preempts can poll lazily. Zero means "use the vnode default" - // (30s). The happy-path transitions (provision start, teardown) do not wait on + // (15s). The happy-path transitions (provision start, teardown) do not wait on // this — they are pushed synchronously by CreatePod/DeletePod — so this only // bounds detection latency for events no provider pushes. PollInterval time.Duration diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index ee820fc..4b0f330 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -42,7 +42,7 @@ import ( // only the fallback: a provider may override the cadence via // Capabilities.PollInterval (e.g. a spot-heavy backend polls faster to notice // reclaims sooner; an OnDemand-only one can poll lazily). -const defaultPollInterval = 30 * time.Second +const defaultPollInterval = 15 * time.Second // Handler bridges one provider into the virtual kubelet: it implements the VK // PodLifecycleHandler (+ PodNotifier) so the pod controller's CreatePod