From 1d0889f761f024ab1818811b1f3817c02f3b092b Mon Sep 17 00:00:00 2001 From: Victor Bona Date: Mon, 27 Jul 2026 14:33:38 -0300 Subject: [PATCH] feat: add namespace-isolated multi-tenancy --- .github/workflows/ci.yml | 2 + .gitignore | 3 + CHANGELOG.md | 22 + CONTRIBUTING.md | 3 +- Makefile | 1 + PRODUCT.md | 54 ++ README.md | 60 +- SECURITY.md | 26 +- charts/devboxes/templates/NOTES.txt | 25 + charts/devboxes/templates/_helpers.tpl | 4 + charts/devboxes/templates/deployment.yaml | 23 +- charts/devboxes/templates/rbac.yaml | 50 ++ .../devboxes/templates/serviceaccounts.yaml | 21 +- charts/devboxes/templates/tenancy.yaml | 138 +++++ charts/devboxes/values.schema.json | 209 +++++++ charts/devboxes/values.yaml | 50 ++ cli/README.md | 9 +- cli/src/client.rs | 27 +- cli/src/config.rs | 89 ++- cli/src/main.rs | 173 +++++- cli/src/models.rs | 58 +- controller/README.md | 11 +- controller/src/devboxes_controller/app.py | 238 ++++++-- controller/src/devboxes_controller/auth.py | 339 +++++++++-- controller/src/devboxes_controller/config.py | 497 +++++++++++++++- .../devboxes_controller/insights_service.py | 20 +- .../src/devboxes_controller/insights_store.py | 150 +++-- controller/src/devboxes_controller/manager.py | 539 +++++++++++++----- controller/src/devboxes_controller/models.py | 26 + .../src/devboxes_controller/resources.py | 62 +- .../src/devboxes_controller/static/app.js | 28 +- .../devboxes_controller/static/insights.js | 2 + .../src/devboxes_controller/static/styles.css | 47 ++ .../src/devboxes_controller/static/tenant.js | 7 + .../templates/_tenant_switcher.html | 12 + .../templates/cli_authorize.html | 4 +- .../devboxes_controller/templates/docs.html | 113 +++- .../devboxes_controller/templates/index.html | 63 +- .../templates/insights.html | 24 +- .../devboxes_controller/templates/login.html | 2 +- controller/tests/fakes.py | 73 ++- controller/tests/preview_app.py | 45 +- controller/tests/test_app.py | 175 +++++- controller/tests/test_auth.py | 90 ++- controller/tests/test_config.py | 263 +++++++++ controller/tests/test_insights_app.py | 83 ++- controller/tests/test_insights_store.py | 120 +++- controller/tests/test_manager.py | 231 +++++++- controller/tests/test_resources.py | 3 + docs/api.md | 94 ++- docs/architecture.md | 91 ++- docs/cli.md | 53 +- docs/configuration.md | 87 ++- docs/credentials.md | 51 +- docs/golden-path.md | 24 +- docs/gpu.md | 6 +- docs/images.md | 7 +- docs/index.md | 9 +- docs/insights.md | 22 +- docs/multi-tenancy.md | 254 +++++++++ docs/operations.md | 41 +- docs/troubleshooting.md | 38 +- scripts/fixtures/tenancy-values.yaml | 44 ++ scripts/kind-e2e.sh | 294 +++++++++- scripts/test-helm-tenancy.sh | 109 ++++ 65 files changed, 5055 insertions(+), 483 deletions(-) create mode 100644 PRODUCT.md create mode 100644 charts/devboxes/templates/tenancy.yaml create mode 100644 controller/src/devboxes_controller/static/tenant.js create mode 100644 controller/src/devboxes_controller/templates/_tenant_switcher.html create mode 100644 docs/multi-tenancy.md create mode 100644 scripts/fixtures/tenancy-values.yaml create mode 100755 scripts/test-helm-tenancy.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1760d17..8cfcd1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,6 +109,8 @@ jobs: run: scripts/test-helm-insights.sh - name: Helm custom image contracts run: scripts/test-helm-images.sh + - name: Helm multi-tenancy contracts + run: scripts/test-helm-tenancy.sh - name: Render LoadBalancer installation run: helm template devboxes charts/devboxes --namespace devboxes > /tmp/devboxes.yaml - name: Validate Kubernetes resources diff --git a/.gitignore b/.gitignore index 0127ad5..0aeb286 100644 --- a/.gitignore +++ b/.gitignore @@ -19,5 +19,8 @@ controller/**/*.pyc cli/target/ +workspace/**/__pycache__/ +workspace/**/*.pyc + dist/ *.tgz diff --git a/CHANGELOG.md b/CHANGELOG.md index c09e73c..f64dad2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,28 @@ All notable changes to Devboxes are documented here. The project follows [Keep a ## [Unreleased] +### Added + +- Added opt-in namespace-per-tenant isolation across Helm, the controller API, CLI, + dashboard, authenticated documentation, and public documentation, with principal + memberships, `viewer`/`member`/`admin` roles, tenant switching, creator attribution, and + backward-compatible single-operator defaults. +- Added per-tenant workspace Secrets, tokenless ServiceAccounts, namespaced controller RBAC, + retained namespace creation, ResourceQuota, SSH-only workspace ingress NetworkPolicy, + StorageClass and TTL policy, and preset, GPU, and custom-image allowlists. +- Added tenant-scoped Insights ingest credentials, storage, queries, exports, migrations, and + purge, including sanitized online SQLite backups and legacy-history assignment to the + configured default tenant. +- Added Helm tenancy contract tests plus controller, API, authorization, manager, Insights, + and CLI coverage for isolation, role enforcement, quota failures, migration, and + configuration precedence. + +### Security + +- Keep principal token values out of Helm values by loading unique strong credentials from a + read-only existing Secret, bind sessions and CLI tokens to credential epochs, and resolve + live tenant membership before every request. + ## [0.5.1] - 2026-07-27 ### Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 536658d..8fb972d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,8 @@ By participating, you agree to follow [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) a ## Engineering expectations - Preserve explicit persistence semantics: TTL and stop never delete a PVC. -- Keep Kubernetes permissions namespace-scoped and workspace service accounts tokenless. +- Keep Kubernetes permissions namespace-scoped and workspace service accounts tokenless; + tenancy changes must prove cross-namespace and cross-principal denial. - Do not add plaintext credentials, example secrets, telemetry, or external calls without explicit user control. - Maintain both `LoadBalancer` and `NodePort` SSH paths. - Keep the CLI usable in scripts: errors go to stderr, JSON remains stable within a release line, and tokens should not be required on command lines. diff --git a/Makefile b/Makefile index 4dc81a0..f16a94d 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ helm: scripts/test-helm-insights.sh scripts/test-helm-gpu.sh scripts/test-helm-images.sh + scripts/test-helm-tenancy.sh helm template devboxes charts/devboxes --namespace devboxes --set workspace.sshService.type=NodePort --set workspace.sshService.host=192.0.2.10 >/dev/null images: diff --git a/PRODUCT.md b/PRODUCT.md new file mode 100644 index 0000000..09e9844 --- /dev/null +++ b/PRODUCT.md @@ -0,0 +1,54 @@ +# Product + +## Register + +product + +## Users + +Devboxes serves developers who need durable, prepared development environments and Kubernetes +operators who make those environments safe and supportable. Developers work from the CLI or +browser workbench and expect fast creation, reliable SSH reconnection, and explicit data +lifecycle behavior. Operators configure cluster policy, credentials, storage, networking, +images, accelerators, tenancy, and capacity without exposing raw Kubernetes controls to users. + +## Product Purpose + +Devboxes turns approved Kubernetes capacity into ready-to-use development machines. Compute is +disposable, `/home/dev` is durable until an explicit purge, and the same lifecycle contract is +available through the API, CLI, and dashboard. Success means a developer can understand and +control a workspace without learning Kubernetes internals while an operator can prove which +policy, identity, namespace, credentials, and resources govern it. + +## Brand Personality + +Clear, operational, trustworthy. The product speaks directly, makes destructive behavior +unmistakable, and presents cluster state without hype or false certainty. + +## Anti-references + +Devboxes should not resemble a decorative SaaS control panel, a consumer cloud drive, or a +generic generated admin template. Avoid hidden automation, ornamental metrics, vague resource +labels, invented interaction patterns, and UI-only security controls. Do not expose arbitrary +Kubernetes fields merely to make a workflow look flexible. + +## Design Principles + +1. Keep operator policy separate from user intent. Users select stable, named capabilities; + operators own namespaces, credentials, scheduling, quotas, and trusted catalogs. +2. Make persistence and destructive actions explicit. Stop, delete, and purge must remain + visibly distinct in every interface. +3. Preserve terminal, API, and dashboard parity. The same identity, tenancy, capability, and + lifecycle contract should be discoverable everywhere. +4. Show real state and useful recovery paths. Pending capacity, policy rejection, degraded + workloads, and retained storage should be described honestly and specifically. +5. Prefer familiar, accessible controls over novelty. Keyboard operation, visible focus, + semantic markup, reduced motion, readable contrast, and responsive task flows are baseline + product behavior. + +## Accessibility & Inclusion + +Preserve the existing keyboard-first and screen-reader-aware interface conventions: skip links, +semantic headings and tables, explicit labels, live regions, focus restoration, and native +dialogs and controls. State must never depend on color alone. Maintain readable contrast, +usable narrow-screen layouts, and reduced-motion behavior across all authenticated surfaces. diff --git a/README.md b/README.md index a81cdc2..607af3e 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,19 @@ Each workspace includes Rust, Node.js, Python, `uv`, GitHub CLI, Codex CLI, Clau - A Rust `devbox` CLI for create, list, inspect, SSH, start, stop, delete, custom-image profiles, and opt-in Insights workflows. - A FastAPI controller with an authenticated API, accessible browser workbench, Insights dashboard, documentation, metrics, health checks, and TTL cleanup. - A versioned Helm chart with values schema validation and namespace-scoped RBAC. +- Optional namespace-per-tenant isolation with principal roles, quotas, policy catalogs, and + tenant-scoped Insights. - Optional operator-approved GPU profiles for NVIDIA, AMD, Intel, partitioned, or shared accelerators. - Optional operator-approved custom image profiles for pod-local services or vetted workspace derivatives. - Multi-architecture controller and workspace images for `linux/amd64` and `linux/arm64`. - Persistent SSH host identity, shell state, tool installs, account state, and source under `/home/dev`. - GitHub Releases with macOS and Linux CLI binaries and SHA-256 checksums. -Devboxes is currently a single-operator system: one shared token controls every box in one installation. It is suitable for a trusted personal cluster or trusted operator group, not mutually untrusted tenants. +Devboxes defaults to a backward-compatible single-operator installation. Operators can +enable [multi-tenancy](docs/multi-tenancy.md) to give users or teams independent Kubernetes +namespaces, credentials, storage, SSH identity, policy catalogs, resource quotas, and +tenant-scoped lifecycle and Insights access. The master token remains an +installation-wide operator credential. ## Requirements @@ -94,6 +100,53 @@ helm install devboxes oci://ghcr.io/vicotrbb/charts/devboxes \ The chart never embeds credential values. It references existing Kubernetes Secrets so it also works with External Secrets Operator, Sealed Secrets, SOPS, Infisical, Vault, and other secret-management workflows. +### Enable multi-tenancy + +Multi-tenancy is opt-in and uses one Kubernetes namespace per tenant. Operators define +principal memberships and roles in Helm while storing every principal token in an existing +Secret: + +```yaml +tenancy: + enabled: true + defaultTenant: platform + existingPrincipalSecret: devboxes-principals + tenants: + - id: platform + displayName: Platform Engineering + namespace: devboxes-platform + createNamespace: true + workspaceSecretName: devboxes-platform-workspace + allowedPresets: [small, medium] + maxTtlHours: 72 + resourceQuota: + enabled: true + hard: + requests.cpu: "8" + requests.memory: 32Gi + requests.storage: 500Gi + principals: + - subject: alice@example.com + displayName: Alice + tokenKey: alice-token + defaultTenant: platform + memberships: + - tenant: platform + role: admin +``` + +Users can switch their authorized context consistently in the CLI or dashboard: + +```bash +devbox tenant list +devbox tenant use platform +devbox --tenant platform create atlas --preset medium --ssh +``` + +Read [multi-tenancy](docs/multi-tenancy.md) before rollout for the complete role matrix, +Secret provisioning, isolation guarantees, quota behavior, Insights migration, and +rollback contract. + ### Configure dashboard ingress This example uses a generic nginx ingress class and an existing TLS Secret: @@ -322,7 +375,10 @@ Devboxes controller ─── Kubernetes API └─ Insights SQLite PVC (optional central history) ``` -The controller watches only its release namespace and receives namespace-scoped RBAC. Workspace pods do not receive Kubernetes service-account tokens. See [architecture](docs/architecture.md) for resource ownership, persistence, readiness, and threat boundaries. +The controller receives only namespaced RBAC: the release namespace in default mode or each +explicitly configured tenant namespace in multi-tenant mode. Workspace pods do not receive +Kubernetes service-account tokens. See [architecture](docs/architecture.md) for resource +ownership, persistence, readiness, and threat boundaries. ## Development diff --git a/SECURITY.md b/SECURITY.md index 1162a4f..54196af 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -18,16 +18,31 @@ You should receive an acknowledgement within seven days. Please allow maintainer ## Security model -Devboxes assumes a trusted single operator or trusted operator group. The shared access token can control and permanently purge every devbox in an installation. Workspaces are development machines with passwordless `sudo`; they are not sandboxes for untrusted code or mutually untrusted users. +Devboxes defaults to a trusted single operator or operator group. Opt-in multi-tenancy adds +principal roles and a dedicated Kubernetes namespace, credentials, storage, quota, and +telemetry scope per tenant. The master access token remains an installation-wide operator +credential that can control and permanently purge every tenant. + +Workspaces are development machines with passwordless `sudo`; they are not sandboxes for +hostile code. Namespace isolation, application authorization, RBAC, ResourceQuota, and +NetworkPolicy provide administrative tenant separation and defense in depth, not a hardened +kernel or node boundary. Use separate clusters or appropriately sandboxed runtimes when +tenants must run mutually hostile workloads. Recommended deployment controls: - expose the controller only through HTTPS or local port-forwarding; - restrict controller ingress and SSH Services to trusted client networks; - use a 256-bit or stronger controller token and rotate it after suspected exposure; +- give tenant users unique principal tokens and least-privilege `viewer`, `member`, or + `admin` memberships instead of the master token; +- use one namespace per person when independent ownership or quota is required, and keep + tenant workspace Secrets separate; - use fine-grained, least-privilege provider tokens; - enable Kubernetes Secret encryption at rest and an auditable secret manager; - enforce the chart's namespace-scoped RBAC and tokenless workspace service account; +- use a NetworkPolicy-capable CNI, review the chart's SSH-only ingress policy, and add + environment-specific source and egress restrictions where required; - expose GPU devices only through reviewed operator-owned profiles and trusted derived images; - back up important PVCs and test restore procedures; - keep Kubernetes, ingress, CSI, images, chart, and CLI releases current; @@ -42,6 +57,15 @@ tmux starts. GPU profiles preserve this trust model. Clients can select only a configured name; they cannot inject an image, extended resource, RuntimeClass, supplemental group, selector, toleration, privileged mode, host path, or device path. The assigned device is fully available to the trusted workspace user, including through passwordless `sudo`, so GPU profiles do not make Devboxes suitable for untrusted workloads. Vendor drivers, device plugins, runtimes, and derived GPU images are part of the operator's trusted supply chain. +Multi-tenant clients select only a configured tenant ID and an operator-approved policy +catalog. Unknown and unauthorized IDs use the same forbidden response. The controller has +one namespaced RoleBinding per configured tenant and no ClusterRole; workspaces use +ServiceAccounts without API tokens or RBAC bindings. Tenant Insights exports, including +SQLite backups, are sanitized to the active tenant before leaving the controller. +Application roles do not replace SSH authorization: shell access is controlled by the public +keys in each tenant workspace Secret. A viewer with an authorized key can modify workspace +files but cannot change the workspace lifecycle through the Devboxes API. + ## Dependency and supply-chain policy Dependabot monitors npm, Cargo, Python, Docker, and GitHub Actions dependencies. Pull requests receive dependency review, and CI audits npm, Python, and Cargo dependencies, builds both images, performs strict JavaScript, documentation, controller, and CLI checks, validates Helm output, and installs into a clean Kind cluster. Release images are published for amd64 and arm64 with GitHub artifact attestations; CLI archives include SHA-256 checksums. diff --git a/charts/devboxes/templates/NOTES.txt b/charts/devboxes/templates/NOTES.txt index d175d82..feef448 100644 --- a/charts/devboxes/templates/NOTES.txt +++ b/charts/devboxes/templates/NOTES.txt @@ -5,8 +5,22 @@ Devboxes is installed in namespace {{ .Release.Namespace }}. kubectl -n {{ .Release.Namespace }} create secret generic {{ .Values.controller.existingSecret }} \ --from-literal={{ .Values.controller.accessTokenKey }}='' +{{- if .Values.tenancy.enabled }} +{{- if gt (len .Values.tenancy.principals) 0 }} + + kubectl -n {{ .Release.Namespace }} create secret generic {{ .Values.tenancy.existingPrincipalSecret }} \ + --from-literal={{ (index .Values.tenancy.principals 0).tokenKey }}='' +{{- end }} +{{- range $tenant := .Values.tenancy.tenants }} + + kubectl -n {{ $tenant.namespace }} create secret generic {{ default $.Values.workspace.existingSecret $tenant.workspaceSecretName }} \ + --from-file=SSH_AUTHORIZED_KEYS="$HOME/.ssh/id_ed25519.pub" +{{- end }} +{{- else }} + kubectl -n {{ .Release.Namespace }} create secret generic {{ .Values.workspace.existingSecret }} \ --from-file=SSH_AUTHORIZED_KEYS="$HOME/.ssh/id_ed25519.pub" +{{- end }} 2. Wait for the controller: @@ -26,6 +40,17 @@ Devboxes is installed in namespace {{ .Release.Namespace }}. Important: every workspace needs an externally reachable SSH service. Configure workspace.sshService for your cluster's LoadBalancer implementation or use NodePort with workspace.sshService.host. +{{- if .Values.tenancy.enabled }} + +Multi-tenancy is enabled with {{ len .Values.tenancy.tenants }} namespace-isolated tenant(s). +Verify principal membership and select a tenant with: + + devbox tenant list + devbox tenant use {{ .Values.tenancy.defaultTenant }} + +Review ResourceQuota and NetworkPolicy status in every tenant namespace before onboarding +users. The master controller token remains an installation-wide operator credential. +{{- end }} {{- if .Values.gpu.enabled }} GPU acceleration is enabled with {{ len .Values.gpu.profiles }} operator-approved profile(s). diff --git a/charts/devboxes/templates/_helpers.tpl b/charts/devboxes/templates/_helpers.tpl index ac90f55..04eeab0 100644 --- a/charts/devboxes/templates/_helpers.tpl +++ b/charts/devboxes/templates/_helpers.tpl @@ -46,6 +46,10 @@ app.kubernetes.io/component: controller {{- end }} {{- end }} +{{- define "devboxes.tenantWorkspaceServiceAccountName" -}} +{{- default (include "devboxes.workspaceServiceAccountName" .root) .tenant.workspaceServiceAccountName }} +{{- end }} + {{- define "devboxes.controllerImage" -}} {{- printf "%s:%s" .Values.controller.image.repository (default .Chart.AppVersion .Values.controller.image.tag) }} {{- end }} diff --git a/charts/devboxes/templates/deployment.yaml b/charts/devboxes/templates/deployment.yaml index f6c7383..3f165f3 100644 --- a/charts/devboxes/templates/deployment.yaml +++ b/charts/devboxes/templates/deployment.yaml @@ -118,6 +118,16 @@ spec: value: {{ .Values.controller.displayName | quote }} - name: DEVBOXES_CLUSTER_NAME value: {{ .Values.controller.clusterName | quote }} + - name: DEVBOXES_MULTI_TENANCY_ENABLED + value: {{ .Values.tenancy.enabled | quote }} + - name: DEVBOXES_DEFAULT_TENANT + value: {{ .Values.tenancy.defaultTenant | quote }} + - name: DEVBOXES_TENANTS + value: {{ .Values.tenancy.tenants | toJson | quote }} + - name: DEVBOXES_PRINCIPALS + value: {{ .Values.tenancy.principals | toJson | quote }} + - name: DEVBOXES_PRINCIPAL_TOKEN_DIRECTORY + value: {{ .Values.tenancy.principalTokenMountPath | quote }} - name: DEVBOXES_COOKIE_SECURE value: {{ .Values.controller.cookieSecure | quote }} - name: DEVBOXES_DEFAULT_TTL_HOURS @@ -157,7 +167,7 @@ spec: - name: DEVBOXES_INSIGHTS_DATABASE_WARNING_BYTES value: {{ int .Values.insights.storage.warningBytes | quote }} - name: DEVBOXES_INSIGHTS_CONTROLLER_URL - value: {{ printf "http://%s:%v" (include "devboxes.fullname" .) .Values.service.port | quote }} + value: {{ printf "http://%s.%s.svc:%v" (include "devboxes.fullname" .) .Release.Namespace .Values.service.port | quote }} - name: DEVBOXES_INSIGHTS_RETENTION_RAW_DAYS value: {{ int .Values.insights.retention.rawDays | quote }} - name: DEVBOXES_INSIGHTS_RETENTION_HOURLY_DAYS @@ -229,6 +239,11 @@ spec: volumeMounts: - name: tmp mountPath: /tmp + {{- if gt (len .Values.tenancy.principals) 0 }} + - name: principal-tokens + mountPath: {{ .Values.tenancy.principalTokenMountPath }} + readOnly: true + {{- end }} {{- if .Values.insights.enabled }} - name: insights mountPath: /var/lib/devboxes @@ -236,6 +251,12 @@ spec: volumes: - name: tmp emptyDir: {} + {{- if gt (len .Values.tenancy.principals) 0 }} + - name: principal-tokens + secret: + secretName: {{ .Values.tenancy.existingPrincipalSecret }} + defaultMode: 0440 + {{- end }} {{- if .Values.insights.enabled }} - name: insights persistentVolumeClaim: diff --git a/charts/devboxes/templates/rbac.yaml b/charts/devboxes/templates/rbac.yaml index d4d1f25..b761014 100644 --- a/charts/devboxes/templates/rbac.yaml +++ b/charts/devboxes/templates/rbac.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.tenancy.enabled }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -40,3 +41,52 @@ subjects: - kind: ServiceAccount name: {{ include "devboxes.controllerServiceAccountName" . }} namespace: {{ .Release.Namespace }} +{{- else }} +{{- range $tenant := .Values.tenancy.tenants }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "devboxes.fullname" $ }} + namespace: {{ $tenant.namespace }} + labels: + {{- include "devboxes.labels" $ | nindent 4 }} + devboxes.bonalab.org/tenant: {{ $tenant.id | quote }} +rules: + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["create", "delete", "get", "list", "patch"] + - apiGroups: ["apps"] + resources: ["deployments/scale"] + verbs: ["get", "patch"] + - apiGroups: [""] + resources: ["pods", "services"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["services"] + verbs: ["create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "delete", "get", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete", "get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "devboxes.fullname" $ }} + namespace: {{ $tenant.namespace }} + labels: + {{- include "devboxes.labels" $ | nindent 4 }} + devboxes.bonalab.org/tenant: {{ $tenant.id | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "devboxes.fullname" $ }} +subjects: + - kind: ServiceAccount + name: {{ include "devboxes.controllerServiceAccountName" $ }} + namespace: {{ $.Release.Namespace }} +{{- end }} +{{- end }} diff --git a/charts/devboxes/templates/serviceaccounts.yaml b/charts/devboxes/templates/serviceaccounts.yaml index b88cdb1..299d126 100644 --- a/charts/devboxes/templates/serviceaccounts.yaml +++ b/charts/devboxes/templates/serviceaccounts.yaml @@ -7,7 +7,7 @@ metadata: {{- include "devboxes.labels" . | nindent 4 }} app.kubernetes.io/component: controller automountServiceAccountToken: true -{{- if .Values.workspace.serviceAccount.create }} +{{- if and .Values.workspace.serviceAccount.create (not .Values.tenancy.enabled) }} --- apiVersion: v1 kind: ServiceAccount @@ -23,3 +23,22 @@ metadata: {{- end }} automountServiceAccountToken: false {{- end }} +{{- if and .Values.workspace.serviceAccount.create .Values.tenancy.enabled }} +{{- range $tenant := .Values.tenancy.tenants }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "devboxes.tenantWorkspaceServiceAccountName" (dict "root" $ "tenant" $tenant) }} + namespace: {{ $tenant.namespace }} + labels: + {{- include "devboxes.labels" $ | nindent 4 }} + app.kubernetes.io/component: workspace + devboxes.bonalab.org/tenant: {{ $tenant.id | quote }} + {{- with $.Values.workspace.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: false +{{- end }} +{{- end }} diff --git a/charts/devboxes/templates/tenancy.yaml b/charts/devboxes/templates/tenancy.yaml new file mode 100644 index 0000000..a3c51e6 --- /dev/null +++ b/charts/devboxes/templates/tenancy.yaml @@ -0,0 +1,138 @@ +{{- if .Values.tenancy.enabled }} +{{- $tenantIds := dict }} +{{- $tenantNamespaces := dict }} +{{- $gpuProfiles := dict }} +{{- $imageProfiles := dict }} +{{- range .Values.gpu.profiles }} +{{- $_ := set $gpuProfiles .name true }} +{{- end }} +{{- range .Values.workspace.customImages.profiles }} +{{- $_ := set $imageProfiles .name true }} +{{- end }} +{{- range $tenant := .Values.tenancy.tenants }} +{{- if hasKey $tenantIds $tenant.id }} +{{- fail (printf "tenancy.tenants contains duplicate id %q" $tenant.id) }} +{{- end }} +{{- if hasKey $tenantNamespaces $tenant.namespace }} +{{- fail (printf "tenancy.tenants contains duplicate namespace %q" $tenant.namespace) }} +{{- end }} +{{- $_ := set $tenantIds $tenant.id true }} +{{- $_ := set $tenantNamespaces $tenant.namespace true }} +{{- $namespaceLabels := default (dict) $tenant.namespaceLabels }} +{{- if or (eq $tenant.namespace "default") (hasPrefix "kube-" $tenant.namespace) }} +{{- fail (printf "tenant %q must use a non-system Kubernetes namespace" $tenant.id) }} +{{- end }} +{{- if or (hasKey $namespaceLabels "app.kubernetes.io/part-of") (hasKey $namespaceLabels "devboxes.bonalab.org/tenant") (hasKey $namespaceLabels "kubernetes.io/metadata.name") }} +{{- fail (printf "tenant %q namespaceLabels cannot replace Devboxes ownership labels" $tenant.id) }} +{{- end }} +{{- if and $tenant.maxTtlHours (gt (int $tenant.maxTtlHours) (int $.Values.controller.maxTtlHours)) }} +{{- fail (printf "tenant %q maxTtlHours cannot exceed controller.maxTtlHours" $tenant.id) }} +{{- end }} +{{- range $profile := $tenant.allowedGpuProfiles }} +{{- if not (hasKey $gpuProfiles $profile) }} +{{- fail (printf "tenant %q references unknown GPU profile %q" $tenant.id $profile) }} +{{- end }} +{{- end }} +{{- if $tenant.defaultGpuProfile }} +{{- if not $.Values.gpu.enabled }} +{{- fail (printf "tenant %q defaultGpuProfile requires gpu.enabled=true" $tenant.id) }} +{{- end }} +{{- if not (hasKey $gpuProfiles $tenant.defaultGpuProfile) }} +{{- fail (printf "tenant %q references unknown default GPU profile %q" $tenant.id $tenant.defaultGpuProfile) }} +{{- end }} +{{- if and (hasKey $tenant "allowedGpuProfiles") (not (has $tenant.defaultGpuProfile $tenant.allowedGpuProfiles)) }} +{{- fail (printf "tenant %q defaultGpuProfile must be included in allowedGpuProfiles" $tenant.id) }} +{{- end }} +{{- end }} +{{- range $profile := $tenant.allowedImageProfiles }} +{{- if not (hasKey $imageProfiles $profile) }} +{{- fail (printf "tenant %q references unknown image profile %q" $tenant.id $profile) }} +{{- end }} +{{- end }} +{{- end }} +{{- if not (hasKey $tenantIds .Values.tenancy.defaultTenant) }} +{{- fail "tenancy.defaultTenant must name an entry in tenancy.tenants" }} +{{- end }} +{{- if and (gt (len .Values.tenancy.principals) 0) (not .Values.tenancy.existingPrincipalSecret) }} +{{- fail "tenancy.existingPrincipalSecret is required when principals are configured" }} +{{- end }} +{{- $subjects := dict }} +{{- $tokenKeys := dict }} +{{- range $principal := .Values.tenancy.principals }} +{{- if hasKey $subjects $principal.subject }} +{{- fail (printf "tenancy.principals contains duplicate subject %q" $principal.subject) }} +{{- end }} +{{- if hasKey $tokenKeys $principal.tokenKey }} +{{- fail (printf "tenancy.principals contains duplicate tokenKey %q" $principal.tokenKey) }} +{{- end }} +{{- $_ := set $subjects $principal.subject true }} +{{- $_ := set $tokenKeys $principal.tokenKey true }} +{{- $memberships := dict }} +{{- range $membership := $principal.memberships }} +{{- if not (hasKey $tenantIds $membership.tenant) }} +{{- fail (printf "principal %q references unknown tenant %q" $principal.subject $membership.tenant) }} +{{- end }} +{{- if hasKey $memberships $membership.tenant }} +{{- fail (printf "principal %q has duplicate membership for tenant %q" $principal.subject $membership.tenant) }} +{{- end }} +{{- $_ := set $memberships $membership.tenant true }} +{{- end }} +{{- if and $principal.defaultTenant (not (hasKey $memberships $principal.defaultTenant)) }} +{{- fail (printf "principal %q defaultTenant must name one of its memberships" $principal.subject) }} +{{- end }} +{{- end }} +{{- range $tenant := .Values.tenancy.tenants }} +{{- if $tenant.createNamespace }} +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $tenant.namespace }} + labels: + app.kubernetes.io/part-of: devboxes + devboxes.bonalab.org/tenant: {{ $tenant.id | quote }} + {{- with $tenant.namespaceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + helm.sh/resource-policy: keep +{{- end }} +{{- with $tenant.resourceQuota }} +{{- if .enabled }} +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: {{ printf "%s-workspaces" (include "devboxes.fullname" $) | trunc 63 | trimSuffix "-" }} + namespace: {{ $tenant.namespace }} + labels: + {{- include "devboxes.labels" $ | nindent 4 }} + devboxes.bonalab.org/tenant: {{ $tenant.id | quote }} +spec: + hard: + {{- toYaml .hard | nindent 4 }} +{{- end }} +{{- end }} +{{- if $.Values.tenancy.networkPolicy.enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ printf "%s-workspace-ingress" (include "devboxes.fullname" $) | trunc 63 | trimSuffix "-" }} + namespace: {{ $tenant.namespace }} + labels: + {{- include "devboxes.labels" $ | nindent 4 }} + devboxes.bonalab.org/tenant: {{ $tenant.id | quote }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/managed-by: devboxes-controller + policyTypes: + - Ingress + ingress: + - ports: + - protocol: TCP + port: 2222 +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/devboxes/values.schema.json b/charts/devboxes/values.schema.json index 1d90a48..eddc82b 100644 --- a/charts/devboxes/values.schema.json +++ b/charts/devboxes/values.schema.json @@ -36,6 +36,73 @@ "affinity": {"type": "object"} } }, + "tenancy": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "defaultTenant", + "existingPrincipalSecret", + "principalTokenMountPath", + "networkPolicy", + "tenants", + "principals" + ], + "properties": { + "enabled": {"type": "boolean"}, + "defaultTenant": {"$ref": "#/definitions/tenantId"}, + "existingPrincipalSecret": {"type": "string", "maxLength": 253}, + "principalTokenMountPath": { + "type": "string", + "minLength": 2, + "maxLength": 256, + "pattern": "^/.*[^/]$", + "not": {"pattern": "(^|/)\\.\\.(/|$)"} + }, + "networkPolicy": { + "type": "object", + "additionalProperties": false, + "required": ["enabled"], + "properties": { + "enabled": {"type": "boolean"} + } + }, + "tenants": { + "type": "array", + "maxItems": 64, + "items": {"$ref": "#/definitions/tenant"} + }, + "principals": { + "type": "array", + "maxItems": 256, + "items": {"$ref": "#/definitions/principal"} + } + }, + "allOf": [ + { + "if": {"properties": {"enabled": {"const": true}}}, + "then": { + "properties": { + "tenants": {"minItems": 1} + } + }, + "else": { + "properties": { + "tenants": {"maxItems": 0}, + "principals": {"maxItems": 0} + } + } + }, + { + "if": {"properties": {"principals": {"minItems": 1}}}, + "then": { + "properties": { + "existingPrincipalSecret": {"minLength": 1} + } + } + } + ] + }, "workspace": { "type": "object", "additionalProperties": false, @@ -274,6 +341,148 @@ } }, "definitions": { + "tenantId": { + "type": "string", + "minLength": 1, + "maxLength": 40, + "pattern": "^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$" + }, + "tenant": { + "type": "object", + "additionalProperties": false, + "required": ["id", "displayName", "namespace"], + "properties": { + "id": {"$ref": "#/definitions/tenantId"}, + "displayName": {"type": "string", "minLength": 1, "maxLength": 80}, + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 63, + "pattern": "^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$", + "not": {"pattern": "^(?:default$|kube-)"} + }, + "createNamespace": {"type": "boolean"}, + "namespaceLabels": { + "type": "object", + "maxProperties": 16, + "propertyNames": { + "maxLength": 317, + "pattern": "^(?:[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?(?:\\.[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?)*/)?[A-Za-z0-9](?:[-_.A-Za-z0-9]{0,61}[A-Za-z0-9])?$", + "not": { + "enum": [ + "app.kubernetes.io/part-of", + "devboxes.bonalab.org/tenant", + "kubernetes.io/metadata.name" + ] + } + }, + "additionalProperties": { + "type": "string", + "maxLength": 63, + "pattern": "^(?:[A-Za-z0-9](?:[-_.A-Za-z0-9]{0,61}[A-Za-z0-9])?)?$" + } + }, + "workspaceSecretName": { + "type": "string", + "maxLength": 253, + "pattern": "^(?:[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?(?:\\.[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?)*)?$" + }, + "workspaceServiceAccountName": { + "type": "string", + "maxLength": 253, + "pattern": "^(?:[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?(?:\\.[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?)*)?$" + }, + "storageClass": { + "type": "string", + "maxLength": 253, + "pattern": "^(?:[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?(?:\\.[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?)*)?$" + }, + "maxTtlHours": {"type": "integer", "minimum": 1, "maximum": 168}, + "allowedPresets": { + "type": "array", + "minItems": 1, + "maxItems": 3, + "uniqueItems": true, + "items": {"type": "string", "enum": ["small", "medium", "large"]} + }, + "allowedGpuProfiles": { + "type": "array", + "maxItems": 32, + "uniqueItems": true, + "items": {"$ref": "#/definitions/tenantId"} + }, + "defaultGpuProfile": {"$ref": "#/definitions/tenantId"}, + "allowedImageProfiles": { + "type": "array", + "maxItems": 32, + "uniqueItems": true, + "items": {"$ref": "#/definitions/tenantId"} + }, + "resourceQuota": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "hard"], + "properties": { + "enabled": {"type": "boolean"}, + "hard": { + "type": "object", + "maxProperties": 32, + "propertyNames": { + "pattern": "^(?:requests\\.(?:cpu|memory|storage)|limits\\.(?:cpu|memory)|count/(?:pods|services|persistentvolumeclaims|secrets|deployments\\.apps|services\\.loadbalancers)|requests\\.[a-z0-9](?:[-a-z0-9.]{0,251}[a-z0-9])?/[A-Za-z0-9](?:[-_.A-Za-z0-9]{0,61}[A-Za-z0-9])?)$" + }, + "additionalProperties": { + "type": "string", + "minLength": 1 + } + } + }, + "allOf": [ + { + "if": {"properties": {"enabled": {"const": true}}}, + "then": {"properties": {"hard": {"minProperties": 1}}} + } + ] + } + } + }, + "principal": { + "type": "object", + "additionalProperties": false, + "required": ["subject", "displayName", "tokenKey", "memberships"], + "properties": { + "subject": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9](?:[-_.@A-Za-z0-9]{0,126}[A-Za-z0-9])?$" + }, + "displayName": {"type": "string", "minLength": 1, "maxLength": 80}, + "tokenKey": { + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^[A-Za-z0-9](?:[-_.A-Za-z0-9]{0,251}[A-Za-z0-9])?$" + }, + "defaultTenant": {"$ref": "#/definitions/tenantId"}, + "memberships": { + "type": "array", + "minItems": 1, + "maxItems": 32, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["tenant", "role"], + "properties": { + "tenant": {"$ref": "#/definitions/tenantId"}, + "role": { + "type": "string", + "enum": ["viewer", "member", "admin"] + } + } + } + } + } + }, "image": { "type": "object", "additionalProperties": false, diff --git a/charts/devboxes/values.yaml b/charts/devboxes/values.yaml index 34ca620..340f3c0 100644 --- a/charts/devboxes/values.yaml +++ b/charts/devboxes/values.yaml @@ -37,6 +37,56 @@ controller: tolerations: [] affinity: {} +# Multi-tenancy is opt-in so existing installations retain their single-operator +# behavior. When enabled, each tenant is an authorization and Kubernetes namespace +# boundary. Principal tokens are read from an existing Secret and are never stored +# in Helm values. +tenancy: + enabled: false + defaultTenant: default + existingPrincipalSecret: "" + principalTokenMountPath: /var/run/secrets/devboxes/principals + networkPolicy: + # Restrict tenant workspace ingress to SSH. Egress remains available for + # repositories, package registries, and the Insights controller. + enabled: true + # Example: + # tenants: + # - id: platform + # displayName: Platform Engineering + # namespace: devboxes-platform + # createNamespace: true + # namespaceLabels: + # owner: platform + # workspaceSecretName: devboxes-workspace + # workspaceServiceAccountName: devboxes-workspace + # storageClass: fast-rwo + # maxTtlHours: 72 + # allowedPresets: [small, medium] + # allowedGpuProfiles: [nvidia-l4] + # defaultGpuProfile: nvidia-l4 + # allowedImageProfiles: [] + # resourceQuota: + # enabled: true + # hard: + # requests.cpu: "8" + # requests.memory: 32Gi + # requests.storage: 500Gi + # count/deployments.apps: "10" + # count/pods: "10" + tenants: [] + # Example principal. Create existingPrincipalSecret in the controller namespace + # with a key matching tokenKey and at least 32 random characters. + # principals: + # - subject: alice + # displayName: Alice + # tokenKey: alice-token + # defaultTenant: platform + # memberships: + # - tenant: platform + # role: admin + principals: [] + workspace: image: repository: ghcr.io/vicotrbb/devboxes-workspace diff --git a/cli/README.md b/cli/README.md index 0a5f754..72d0146 100644 --- a/cli/README.md +++ b/cli/README.md @@ -10,6 +10,13 @@ devbox gpu profiles devbox create inference --gpu --ssh devbox image profiles devbox create docs-preview --image nginx --ssh +devbox tenant list +devbox --tenant platform list ``` -See the [CLI reference](../docs/cli.md) for every command, option, environment variable, output contract, and SSH workflow. [GPU acceleration](../docs/gpu.md) covers operator-approved accelerator profiles, and [custom image profiles](../docs/images.md) covers approved service and workspace images. The [golden path](../docs/golden-path.md) covers the recommended installation and performance setup. +See the [CLI reference](../docs/cli.md) for every command, option, environment variable, +tenant-selection rule, output contract, and SSH workflow. [Multi-tenancy](../docs/multi-tenancy.md) +covers roles and isolation, [GPU acceleration](../docs/gpu.md) covers operator-approved +accelerator profiles, and [custom image profiles](../docs/images.md) covers approved service +and workspace images. The [golden path](../docs/golden-path.md) covers the recommended +installation and performance setup. diff --git a/cli/src/client.rs b/cli/src/client.rs index 513e811..405633d 100644 --- a/cli/src/client.rs +++ b/cli/src/client.rs @@ -1,7 +1,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; -use reqwest::{Client, Method, StatusCode}; +use reqwest::{Client, Method, RequestBuilder, StatusCode}; use serde::Serialize; use serde::de::DeserializeOwned; use serde_json::Value; @@ -15,14 +15,16 @@ use crate::models::{ pub struct ApiClient { base_url: String, token: String, + tenant: Option, http: Client, } impl ApiClient { - pub fn new(base_url: String, token: String) -> Result { + pub fn new(base_url: String, token: String, tenant: Option) -> Result { Ok(Self { base_url, token, + tenant, http: http_client()?, }) } @@ -133,9 +135,7 @@ impl ApiClient { pub async fn insights_export(&self, query: &[(String, String)]) -> Result { let response = self - .http - .get(self.url("/api/v1/insights/export", query)?) - .bearer_auth(&self.token) + .authorize(self.http.get(self.url("/api/v1/insights/export", query)?)) .header("Accept", "application/json, text/csv") .send() .await @@ -169,9 +169,7 @@ impl ApiClient { R: DeserializeOwned, { let mut request = self - .http - .request(method, self.url(path, &[])?) - .bearer_auth(&self.token) + .authorize(self.http.request(method, self.url(path, &[])?)) .header("Accept", "application/json"); if let Some(body) = body { request = request.json(body); @@ -193,9 +191,7 @@ impl ApiClient { R: DeserializeOwned, { let response = self - .http - .request(method, self.url(path, query)?) - .bearer_auth(&self.token) + .authorize(self.http.request(method, self.url(path, query)?)) .header("Accept", "application/json") .send() .await @@ -212,6 +208,15 @@ impl ApiClient { } Ok(url) } + + fn authorize(&self, request: RequestBuilder) -> RequestBuilder { + let request = request.bearer_auth(&self.token); + if let Some(tenant) = &self.tenant { + request.header("X-Devboxes-Tenant", tenant) + } else { + request + } + } } fn http_client() -> Result { diff --git a/cli/src/config.rs b/cli/src/config.rs index c91865e..28bd23b 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -12,11 +12,14 @@ use serde::{Deserialize, Serialize}; pub struct StoredConfig { pub url: Option, pub token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tenant: Option, } pub struct ResolvedConfig { pub url: String, pub token: String, + pub tenant: Option, pub server_alias: String, } @@ -31,7 +34,12 @@ impl StoredConfig { toml::from_str(&contents).with_context(|| format!("invalid config at {}", path.display())) } - pub fn resolve(self, url: Option, token: Option) -> Result { + pub fn resolve_with_tenant( + self, + url: Option, + token: Option, + tenant: Option, + ) -> Result { let resolved_url = self.resolve_url(url)?; let token = token .or_else(|| std::env::var("DEVBOX_TOKEN").ok()) @@ -40,11 +48,18 @@ impl StoredConfig { .ok_or_else(|| { anyhow::anyhow!("not logged in; run `devbox login` or set DEVBOX_TOKEN") })?; + let tenant = tenant + .or_else(|| std::env::var("DEVBOX_TENANT").ok()) + .or(self.tenant) + .filter(|value| !value.trim().is_empty()) + .map(|value| normalize_tenant(&value)) + .transpose()?; let parsed = Url::parse(&resolved_url).context("invalid Devboxes API URL")?; let server_alias = server_alias(&parsed); Ok(ResolvedConfig { url: resolved_url, token, + tenant, server_alias, }) } @@ -74,7 +89,7 @@ impl StoredConfig { Ok(url.trim_end_matches('/').to_owned()) } - pub fn save(url: &str, token: &str) -> Result { + pub fn save_with_tenant(url: &str, token: &str, tenant: Option<&str>) -> Result { let path = config_path()?; let parent = path.parent().context("config path has no parent")?; fs::create_dir_all(parent) @@ -82,6 +97,7 @@ impl StoredConfig { let content = toml::to_string_pretty(&Self { url: Some(url.trim_end_matches('/').to_owned()), token: Some(token.to_owned()), + tenant: tenant.map(str::to_owned), })?; let mut options = fs::OpenOptions::new(); @@ -98,6 +114,29 @@ impl StoredConfig { } } +fn normalize_tenant(value: &str) -> Result { + let value = value.trim().to_ascii_lowercase(); + let valid_length = (1..=40).contains(&value.len()); + let valid_edges = value + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && value + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric); + let valid_characters = value.bytes().all(|character| { + character.is_ascii_lowercase() || character.is_ascii_digit() || character == b'-' + }); + if valid_length && valid_edges && valid_characters { + Ok(value) + } else { + bail!( + "invalid tenant ID; use 1-40 lowercase letters, digits, or hyphens and start and end alphanumeric" + ) + } +} + fn server_alias(url: &Url) -> String { let identity = format!( "{}-{}{}", @@ -135,26 +174,30 @@ mod tests { let config = StoredConfig { url: Some("http://devboxes.example.com".to_owned()), token: Some("secret".to_owned()), + tenant: None, }; - assert!(config.resolve(None, None).is_err()); + assert!(config.resolve_with_tenant(None, None, None).is_err()); let lookalike = StoredConfig { url: Some("http://localhost.example.com".to_owned()), token: Some("secret".to_owned()), + tenant: None, }; - assert!(lookalike.resolve(None, None).is_err()); + assert!(lookalike.resolve_with_tenant(None, None, None).is_err()); let local = StoredConfig { url: Some("http://localhost:8000".to_owned()), token: Some("secret".to_owned()), + tenant: None, }; - assert!(local.resolve(None, None).is_ok()); + assert!(local.resolve_with_tenant(None, None, None).is_ok()); let loopback = StoredConfig { url: Some("http://127.0.0.1:8000".to_owned()), token: Some("secret".to_owned()), + tenant: None, }; - assert!(loopback.resolve(None, None).is_ok()); + assert!(loopback.resolve_with_tenant(None, None, None).is_ok()); } #[test] @@ -162,9 +205,10 @@ mod tests { let config = StoredConfig { url: None, token: Some("secret".to_owned()), + tenant: None, }; - assert!(config.resolve(None, None).is_err()); + assert!(config.resolve_with_tenant(None, None, None).is_err()); } #[test] @@ -172,14 +216,16 @@ mod tests { let first = StoredConfig { url: Some("https://devboxes-one.example.com".to_owned()), token: Some("secret".to_owned()), + tenant: None, } - .resolve(None, None) + .resolve_with_tenant(None, None, None) .unwrap(); let second = StoredConfig { url: Some("https://devboxes-two.example.com".to_owned()), token: Some("secret".to_owned()), + tenant: None, } - .resolve(None, None) + .resolve_with_tenant(None, None, None) .unwrap(); assert_ne!(first.server_alias, second.server_alias); @@ -194,9 +240,32 @@ mod tests { let config = StoredConfig { url: Some(url.to_owned()), token: Some("secret".to_owned()), + tenant: None, }; - assert!(config.resolve(None, None).is_err()); + assert!(config.resolve_with_tenant(None, None, None).is_err()); + } + } + + #[test] + fn tenant_selection_is_normalized_and_validated() { + let selected = StoredConfig { + url: Some("https://devboxes.example.com".to_owned()), + token: Some("secret".to_owned()), + tenant: Some("platform".to_owned()), } + .resolve_with_tenant(None, None, Some(" Research ".to_owned())) + .unwrap(); + + assert_eq!(selected.tenant.as_deref(), Some("research")); + assert!( + StoredConfig { + url: Some("https://devboxes.example.com".to_owned()), + token: Some("secret".to_owned()), + tenant: None, + } + .resolve_with_tenant(None, None, Some("../other".to_owned())) + .is_err() + ); } #[test] diff --git a/cli/src/main.rs b/cli/src/main.rs index d5756a4..2b1584d 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -36,6 +36,10 @@ struct Cli { #[arg(long, global = true)] token: Option, + /// Select a tenant for this command (or set `DEVBOX_TENANT`). + #[arg(long, global = true, value_parser = validate_name)] + tenant: Option, + /// Print machine-readable JSON where supported. #[arg(long, global = true)] json: bool, @@ -68,6 +72,8 @@ enum Commands { Gpu(GpuArgs), /// Inspect custom image profiles configured by the operator. Image(ImageArgs), + /// List or select the tenants available to this identity. + Tenant(TenantArgs), } #[derive(Args)] @@ -130,6 +136,23 @@ struct ImageArgs { command: Option, } +#[derive(Args)] +struct TenantArgs { + #[command(subcommand)] + command: TenantCommand, +} + +#[derive(Subcommand)] +enum TenantCommand { + /// List tenant memberships and roles. + List, + /// Persist the tenant used by subsequent commands. + Use { + #[arg(value_parser = validate_name)] + tenant: String, + }, +} + #[derive(Subcommand)] enum GpuCommand { /// List the GPU profiles available for new devboxes. @@ -273,12 +296,25 @@ impl std::fmt::Display for MetricsExportFormat { async fn main() -> Result<()> { let cli = Cli::parse(); if let Commands::Login(args) = &cli.command { - return login(cli.url.as_deref(), cli.token.as_deref(), args, cli.json).await; + return login( + cli.url.as_deref(), + cli.token.as_deref(), + cli.tenant.as_deref(), + args, + cli.json, + ) + .await; } - let resolved = StoredConfig::load()?.resolve(cli.url.clone(), cli.token.clone())?; + let resolved = StoredConfig::load()?.resolve_with_tenant( + cli.url.clone(), + cli.token.clone(), + cli.tenant.clone(), + )?; let server_alias = resolved.server_alias.clone(); - let client = ApiClient::new(resolved.url, resolved.token)?; + let api_url = resolved.url.clone(); + let api_token = resolved.token.clone(); + let client = ApiClient::new(resolved.url, resolved.token, resolved.tenant)?; match cli.command { Commands::Login(_) => unreachable!(), @@ -292,12 +328,14 @@ async fn main() -> Result<()> { Commands::Metrics(args) => metrics(&client, args, cli.json).await, Commands::Gpu(args) => gpu(&client, args, cli.json).await, Commands::Image(args) => image(&client, args, cli.json).await, + Commands::Tenant(args) => tenant(&client, args, cli.json, &api_url, &api_token).await, } } async fn login( url: Option<&str>, provided_token: Option<&str>, + tenant: Option<&str>, args: &LoginArgs, json: bool, ) -> Result<()> { @@ -339,26 +377,41 @@ async fn login( } response.access_token }; - let resolved = stored.resolve(Some(configured_url), Some(token.clone()))?; - let identity = ApiClient::new(resolved.url.clone(), resolved.token)? + // A previous principal's saved tenant must not override the server-selected + // default during a new login. Explicit CLI/environment selection still wins. + let login_config = StoredConfig { + tenant: None, + ..stored + }; + let resolved = login_config.resolve_with_tenant( + Some(configured_url), + Some(token.clone()), + tenant.map(str::to_owned), + )?; + let identity = ApiClient::new(resolved.url.clone(), resolved.token, resolved.tenant)? .whoami() .await .context("token verification failed")?; - let path = StoredConfig::save(&resolved.url, &token)?; + let path = StoredConfig::save_with_tenant(&resolved.url, &token, Some(&identity.tenant))?; if json { println!( "{}", serde_json::json!({ "user": identity.user, + "subject": identity.subject, "mode": identity.mode, + "tenant": identity.tenant, + "role": identity.role, "config": path, }) ); } else { println!( - "✓ authenticated as {} via {}\n config: {}", + "✓ authenticated as {} via {}\n tenant: {} ({})\n config: {}", identity.user, identity.mode, + identity.tenant, + identity.role, path.display() ); } @@ -622,6 +675,71 @@ async fn image(client: &ApiClient, args: ImageArgs, json: bool) -> Result<()> { Ok(()) } +async fn tenant( + client: &ApiClient, + args: TenantArgs, + json: bool, + api_url: &str, + api_token: &str, +) -> Result<()> { + match args.command { + TenantCommand::List => { + let identity = client.whoami().await?; + if json { + println!( + "{}", + serde_json::json!({ + "current": identity.tenant, + "tenants": identity.tenants, + }) + ); + } else { + println!("{:<3} {:<20} {:<28} ROLE", "", "TENANT", "NAME"); + for item in identity.tenants { + let marker = if item.id == identity.tenant { + "→" + } else { + "" + }; + println!( + "{marker:<3} {:<20} {:<28} {}", + item.id, item.display_name, item.role + ); + } + } + } + TenantCommand::Use { tenant } => { + let selected = ApiClient::new( + api_url.to_owned(), + api_token.to_owned(), + Some(tenant.clone()), + )? + .whoami() + .await + .with_context(|| format!("tenant {tenant:?} is not available to this identity"))?; + let path = StoredConfig::save_with_tenant(api_url, api_token, Some(&selected.tenant))?; + if json { + println!( + "{}", + serde_json::json!({ + "tenant": selected.tenant, + "role": selected.role, + "config": path, + }) + ); + } else { + println!( + "✓ using tenant {} ({})\n config: {}", + selected.tenant, + selected.role, + path.display() + ); + } + } + } + Ok(()) +} + fn print_gpu_profiles(capabilities: &GpuCapabilities) { if !capabilities.enabled { println!("GPU acceleration is disabled by the operator."); @@ -872,6 +990,7 @@ async fn run_ssh(box_info: &Devbox, extra_args: &[String], server_alias: &str) - .context("devbox does not have an SSH address yet")?; let arguments = ssh_arguments( server_alias, + &box_info.tenant, &box_info.name, host, box_info.ssh_port, @@ -892,6 +1011,7 @@ async fn run_ssh(box_info: &Devbox, extra_args: &[String], server_alias: &str) - fn ssh_arguments( server_alias: &str, + tenant: &str, name: &str, host: &str, port: u16, @@ -902,7 +1022,7 @@ fn ssh_arguments( "-p".to_owned(), port.to_string(), "-o".to_owned(), - format!("HostKeyAlias={server_alias}-{name}"), + format!("HostKeyAlias={server_alias}-{tenant}-{name}"), "-o".to_owned(), "StrictHostKeyChecking=accept-new".to_owned(), "-o".to_owned(), @@ -919,6 +1039,7 @@ fn print_box(box_info: &Devbox, json: bool) -> Result<()> { return Ok(()); } println!("{} {}", box_info.name, box_info.state); + println!(" tenant: {}", box_info.tenant); println!(" preset: {}", box_info.preset); println!(" storage: {}", box_info.storage_size); if let Some(gpu) = &box_info.gpu { @@ -982,8 +1103,8 @@ mod tests { use clap::Parser; use super::{ - Cli, Commands, GpuCommand, ImageCommand, MetricsCommand, human_duration, metrics_query, - resolve_login_token, ssh_arguments, validate_image_selector, validate_name, + Cli, Commands, GpuCommand, ImageCommand, MetricsCommand, TenantCommand, human_duration, + metrics_query, resolve_login_token, ssh_arguments, validate_image_selector, validate_name, }; #[test] @@ -1027,7 +1148,14 @@ mod tests { #[test] fn extra_ssh_options_come_before_the_destination() { let extra = vec!["-L".to_owned(), "3000:127.0.0.1:3000".to_owned()]; - let arguments = ssh_arguments("devboxes-cluster-one", "atlas", "192.0.2.10", 22, &extra); + let arguments = ssh_arguments( + "devboxes-cluster-one", + "platform", + "atlas", + "192.0.2.10", + 22, + &extra, + ); let forwarding = arguments.iter().position(|item| item == "-L").unwrap(); let destination = arguments .iter() @@ -1035,7 +1163,7 @@ mod tests { .unwrap(); assert!(forwarding < destination); - assert!(arguments.contains(&"HostKeyAlias=devboxes-cluster-one-atlas".to_owned())); + assert!(arguments.contains(&"HostKeyAlias=devboxes-cluster-one-platform-atlas".to_owned())); } #[test] @@ -1084,6 +1212,27 @@ mod tests { assert!(Cli::try_parse_from(["devbox", "metrics", "--box", "Invalid"]).is_err()); } + #[test] + fn tenant_context_and_commands_are_discoverable() { + let list = + Cli::try_parse_from(["devbox", "--tenant", "platform", "tenant", "list"]).unwrap(); + assert_eq!(list.tenant.as_deref(), Some("platform")); + let Commands::Tenant(args) = list.command else { + panic!("expected tenant command"); + }; + assert!(matches!(args.command, TenantCommand::List)); + + let select = Cli::try_parse_from(["devbox", "tenant", "use", "research"]).unwrap(); + let Commands::Tenant(args) = select.command else { + panic!("expected tenant command"); + }; + assert!(matches!( + args.command, + TenantCommand::Use { tenant } if tenant == "research" + )); + assert!(Cli::try_parse_from(["devbox", "--tenant", "../other", "list"]).is_err()); + } + #[test] fn create_accepts_default_and_named_gpu_profiles() { let default_gpu = Cli::try_parse_from(["devbox", "create", "atlas", "--gpu"]).unwrap(); diff --git a/cli/src/models.rs b/cli/src/models.rs index e392844..2efc585 100644 --- a/cli/src/models.rs +++ b/cli/src/models.rs @@ -65,6 +65,8 @@ pub struct CustomImageAllocation { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Devbox { + #[serde(default = "default_tenant")] + pub tenant: String, pub name: String, pub state: String, pub preset: Preset, @@ -80,6 +82,8 @@ pub struct Devbox { pub storage_size: String, pub message: Option, #[serde(default)] + pub created_by: Option, + #[serde(default)] pub gpu: Option, #[serde(default)] pub image: Option, @@ -98,7 +102,22 @@ pub struct DeleteResult { #[derive(Debug, Deserialize)] pub struct WhoAmI { pub user: String, + #[serde(default)] + pub subject: String, pub mode: String, + #[serde(default = "default_tenant")] + pub tenant: String, + #[serde(default = "default_operator_role")] + pub role: String, + #[serde(default)] + pub tenants: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct TenantSummary { + pub id: String, + pub display_name: String, + pub role: String, } #[derive(Debug, Deserialize, Serialize)] @@ -106,6 +125,18 @@ pub struct Capabilities { pub gpu: GpuCapabilities, #[serde(default)] pub images: CustomImageCapabilities, + #[serde(default)] + pub tenancy: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct TenancyCapabilities { + pub enabled: bool, + pub current: TenantSummary, + pub tenants: Vec, + pub max_ttl_hours: u16, + pub allowed_presets: Vec, + pub resource_quota: std::collections::BTreeMap, } #[derive(Debug, Deserialize, Serialize)] @@ -164,6 +195,8 @@ pub struct InsightsEnvelope { pub schema_version: u32, pub generated_at: String, pub enabled: bool, + #[serde(default)] + pub tenant: Option, pub effective_range: Option, pub coverage: InsightsCoverage, pub capabilities: Value, @@ -287,14 +320,24 @@ pub struct InsightsPurgeResult { pub generated_at: String, #[serde(rename = "box")] pub box_name: String, + #[serde(default = "default_tenant")] + pub tenant: String, pub purged_instances: u64, } +fn default_tenant() -> String { + "default".to_owned() +} + +fn default_operator_role() -> String { + "operator".to_owned() +} + #[cfg(test)] mod tests { use serde_json::json; - use super::{CreateDevbox, Devbox, GpuRequest, Preset}; + use super::{CreateDevbox, Devbox, GpuRequest, Preset, WhoAmI}; #[test] fn create_payload_omits_cpu_only_gpu_state() { @@ -385,4 +428,17 @@ mod tests { assert!(box_info.gpu.is_none()); } + + #[test] + fn identity_deserialization_accepts_pre_tenancy_controller_responses() { + let identity: WhoAmI = serde_json::from_value(json!({ + "user": "operator", + "mode": "bearer" + })) + .unwrap(); + + assert_eq!(identity.tenant, "default"); + assert_eq!(identity.role, "operator"); + assert!(identity.tenants.is_empty()); + } } diff --git a/controller/README.md b/controller/README.md index 76eff13..9a4c788 100644 --- a/controller/README.md +++ b/controller/README.md @@ -2,8 +2,10 @@ FastAPI controller, authenticated REST API, and server-rendered dashboard for Devboxes. -Authentication supports the master bearer token, signed browser sessions with CSRF, and a -native CLI Authorization Code plus PKCE flow that issues scoped expiring bearer tokens. +Authentication supports the master bearer token, opt-in Secret-backed tenant principals, +signed browser sessions with CSRF, and a native CLI Authorization Code plus PKCE flow that +issues scoped expiring bearer tokens. Requests resolve one live tenant membership and role +before Kubernetes or Insights access. ```bash uv sync --extra dev @@ -13,4 +15,7 @@ uv run pytest For local development, set `DEVBOXES_KUBECONFIG_CONTEXT` to a disposable Kubernetes context and provide a non-production `DEVBOXES_ACCESS_TOKEN`. -See the [API reference](../docs/api.md), [architecture](../docs/architecture.md), [GPU acceleration](../docs/gpu.md), [custom image profiles](../docs/images.md), and [operations runbook](../docs/operations.md) for supported behavior and deployment guidance. +See the [API reference](../docs/api.md), [architecture](../docs/architecture.md), +[multi-tenancy](../docs/multi-tenancy.md), [GPU acceleration](../docs/gpu.md), [custom image +profiles](../docs/images.md), and [operations runbook](../docs/operations.md) for supported +behavior and deployment guidance. diff --git a/controller/src/devboxes_controller/app.py b/controller/src/devboxes_controller/app.py index a4f0f33..fe09146 100644 --- a/controller/src/devboxes_controller/app.py +++ b/controller/src/devboxes_controller/app.py @@ -47,7 +47,12 @@ record_insights_rejection, ) from .insights_store import QueryFilters -from .manager import DevboxConflictError, DevboxManager, DevboxNotFoundError +from .manager import ( + DevboxConflictError, + DevboxManager, + DevboxNotFoundError, + TenantQuotaExceededError, +) from .models import ( Capabilities, CliTokenRequest, @@ -61,6 +66,9 @@ DevboxList, GpuCapabilities, GpuProfileSummary, + Preset, + TenancyCapabilities, + TenantSummary, WhoAmI, ) from .resources import PRESETS @@ -78,12 +86,29 @@ ] -def _capabilities(settings: Settings) -> Capabilities: +def _tenant_summaries(auth: AuthContext) -> list[TenantSummary]: + """Build the tenant switcher contract available to the current identity.""" + return [ + TenantSummary( + id=tenant.id, + display_name=tenant.display_name, + role=tenant.role.value, + ) + for tenant in auth.tenants + ] + + +def _capabilities(settings: Settings, auth: AuthContext) -> Capabilities: """Build the safe installation feature contract exposed to clients.""" + tenant = auth.tenant + gpu_profiles = settings.tenant_gpu_profiles(tenant) + default_gpu_profile = settings.tenant_default_gpu_profile(tenant) + image_profiles = settings.tenant_custom_images(tenant) + tenant_summaries = _tenant_summaries(auth) return Capabilities( gpu=GpuCapabilities( - enabled=settings.gpu_enabled, - default_profile=settings.gpu_default_profile if settings.gpu_enabled else None, + enabled=bool(gpu_profiles), + default_profile=default_gpu_profile, profiles=[ GpuProfileSummary( name=profile.name, @@ -91,14 +116,13 @@ def _capabilities(settings: Settings) -> Capabilities: description=profile.description, resource_name=profile.resource_name, count=profile.count, - default=profile.name == settings.gpu_default_profile, + default=profile.name == default_gpu_profile, ) - for profile in settings.gpu_profiles - if settings.gpu_enabled + for profile in gpu_profiles ], ), images=CustomImageCapabilities( - enabled=settings.custom_images_enabled, + enabled=bool(image_profiles), profiles=[ CustomImageProfileSummary( name=profile.name, @@ -114,10 +138,25 @@ def _capabilities(settings: Settings) -> Capabilities: for port in profile.ports ], ) - for profile in settings.custom_images - if settings.custom_images_enabled + for profile in image_profiles ], ), + tenancy=TenancyCapabilities( + enabled=settings.multi_tenancy_enabled, + current=TenantSummary( + id=tenant.id, + display_name=tenant.display_name, + role=auth.role.value, + ), + tenants=tenant_summaries, + max_ttl_hours=settings.tenant_max_ttl_hours(tenant), + allowed_presets=list(tenant.allowed_presets), + resource_quota=( + tenant.resource_quota.hard + if tenant.resource_quota is not None and tenant.resource_quota.enabled + else {} + ), + ), ) @@ -136,7 +175,6 @@ def create_app( maximum_codes=settings.authorization_code_store_size, ) templates = Jinja2Templates(directory=PACKAGE_DIR / "templates") - capabilities = _capabilities(settings) @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: @@ -216,7 +254,10 @@ async def ready() -> Response: @app.get("/metrics", include_in_schema=False) async def metrics() -> Response: - boxes = await manager.list() + tenant_boxes = await asyncio.gather( + *[manager.list(tenant) for tenant in settings.tenant_configs] + ) + boxes = [box for items in tenant_boxes for box in items] counts = dict.fromkeys(("starting", "ready", "stopped", "degraded"), 0) for box in boxes: counts[box.state.value] += 1 @@ -251,7 +292,8 @@ async def login( raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid return path" ) - if not authenticator.validate_access_token(token): + identity = authenticator.authenticate_access_token(token) + if identity is None: return templates.TemplateResponse( request, "login.html", @@ -262,7 +304,7 @@ async def login( }, status_code=status.HTTP_401_UNAUTHORIZED, ) - session, csrf = authenticator.issue_session() + session, csrf = authenticator.issue_session(identity.subject) response = RedirectResponse(next_target, status_code=status.HTTP_303_SEE_OTHER) response.set_cookie( SESSION_COOKIE, @@ -410,27 +452,40 @@ async def logout(_: Auth) -> Response: return response @app.get("/", response_class=HTMLResponse, include_in_schema=False) - async def dashboard(request: Request) -> Response: - if not authenticator.browser_session_valid(request): + async def dashboard( + request: Request, + tenant_id: Annotated[str | None, Query(alias="tenant", max_length=40)] = None, + ) -> Response: + auth = authenticator.browser_context(request, tenant_id) + if auth is None: return RedirectResponse("/login", status_code=status.HTTP_303_SEE_OTHER) + capabilities = _capabilities(settings, auth) + maximum_ttl = settings.tenant_max_ttl_hours(auth.tenant) return templates.TemplateResponse( request, "index.html", { - "default_ttl_hours": settings.default_ttl_hours, - "max_ttl_hours": settings.max_ttl_hours, + "default_ttl_hours": min(settings.default_ttl_hours, maximum_ttl), + "max_ttl_hours": maximum_ttl, "cluster_name": settings.cluster_name, - "storage_class": settings.storage_class or "cluster default", + "storage_class": (settings.tenant_storage_class(auth.tenant) or "cluster default"), "workspace_service_type": settings.workspace_service_type, "gpu": capabilities.gpu, "images": capabilities.images, + "tenancy": capabilities.tenancy, + "can_manage": auth.can_manage, + "can_purge": auth.can_purge, "version": __version__, }, ) @app.get("/insights", response_class=HTMLResponse, include_in_schema=False) - async def insights_dashboard(request: Request) -> Response: - if not authenticator.browser_session_valid(request): + async def insights_dashboard( + request: Request, + tenant_id: Annotated[str | None, Query(alias="tenant", max_length=40)] = None, + ) -> Response: + auth = authenticator.browser_context(request, tenant_id) + if auth is None: return RedirectResponse("/login", status_code=status.HTTP_303_SEE_OTHER) return templates.TemplateResponse( request, @@ -438,23 +493,43 @@ async def insights_dashboard(request: Request) -> Response: { "cluster_name": settings.cluster_name, "insights_enabled": insights.enabled, + "tenancy": _capabilities(settings, auth).tenancy, + "can_purge": auth.can_purge, "version": __version__, }, ) @app.get("/docs", response_class=HTMLResponse, include_in_schema=False) - async def documentation(request: Request) -> Response: - if not authenticator.browser_session_valid(request): + async def documentation( + request: Request, + tenant_id: Annotated[str | None, Query(alias="tenant", max_length=40)] = None, + ) -> Response: + auth = authenticator.browser_context(request, tenant_id) + if auth is None: return RedirectResponse("/login", status_code=status.HTTP_303_SEE_OTHER) + capabilities = _capabilities(settings, auth) + configured_presets = set(auth.tenant.allowed_presets) + allowed_presets = [preset for preset in PRESETS if preset in configured_presets] + maximum_ttl = settings.tenant_max_ttl_hours(auth.tenant) + default_preset = Preset.MEDIUM if Preset.MEDIUM in allowed_presets else allowed_presets[0] return templates.TemplateResponse( request, "docs.html", { - "default_ttl_hours": settings.default_ttl_hours, - "max_ttl_hours": settings.max_ttl_hours, + "default_ttl_hours": min( + settings.default_ttl_hours, + maximum_ttl, + ), + "max_ttl_hours": maximum_ttl, + "example_ttl_hours": min(72, maximum_ttl), + "default_preset": default_preset.value, + "smallest_preset": allowed_presets[0].value, + "largest_preset": allowed_presets[-1].value, + "allowed_presets": [preset.value for preset in allowed_presets], "external_url": settings.external_url, - "namespace": settings.namespace, - "storage_class": settings.storage_class or "cluster default", + "namespace": auth.tenant.namespace, + "controller_namespace": settings.namespace, + "storage_class": (settings.tenant_storage_class(auth.tenant) or "cluster default"), "workspace_service_type": settings.workspace_service_type, "cluster_name": settings.cluster_name, "presets": [ @@ -466,60 +541,80 @@ async def documentation(request: Request) -> Response: "storage": resources["storage"], } for preset, resources in PRESETS.items() + if preset in allowed_presets ], "insights_enabled": insights.enabled, "gpu": capabilities.gpu, "images": capabilities.images, + "tenancy": capabilities.tenancy, "version": __version__, }, ) @app.get("/api/v1/whoami", tags=["auth"]) async def whoami(auth: Auth) -> WhoAmI: - return WhoAmI(user=auth.subject, mode=auth.mode) + return WhoAmI( + user=auth.display_name, + subject=auth.subject, + mode=auth.mode, + tenant=auth.tenant.id, + role=auth.role.value, + tenants=_tenant_summaries(auth), + ) @app.get("/api/v1/capabilities", tags=["system"]) - async def installation_capabilities(_: Auth) -> Capabilities: - return capabilities + async def installation_capabilities(auth: Auth) -> Capabilities: + return _capabilities(settings, auth) @app.get("/api/v1/devboxes", tags=["devboxes"]) - async def list_devboxes(_: Auth) -> DevboxList: - return DevboxList(items=await manager.list()) + async def list_devboxes(auth: Auth) -> DevboxList: + return DevboxList(items=await manager.list(auth.tenant)) @app.post( "/api/v1/devboxes", status_code=status.HTTP_201_CREATED, tags=["devboxes"], ) - async def create_devbox(payload: CreateDevboxRequest, _: Auth) -> Devbox: + async def create_devbox(payload: CreateDevboxRequest, auth: Auth) -> Devbox: + _require_manage(auth) try: - return await manager.create(payload) + return await manager.create(payload, auth.tenant, auth.subject) except DevboxConflictError as error: raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(error)) from error + except TenantQuotaExceededError as error: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(error)) from error except ValueError as error: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(error) ) from error @app.get("/api/v1/devboxes/{name}", tags=["devboxes"]) - async def get_devbox(name: DevboxName, _: Auth) -> Devbox: - return await _not_found(manager.get(name), name) + async def get_devbox(name: DevboxName, auth: Auth) -> Devbox: + return await _not_found(manager.get(name, auth.tenant), name) @app.post("/api/v1/devboxes/{name}/start", tags=["devboxes"]) - async def start_devbox(name: DevboxName, _: Auth) -> Devbox: - return await _not_found(manager.scale(name, 1), name) + async def start_devbox(name: DevboxName, auth: Auth) -> Devbox: + _require_manage(auth) + try: + return await _not_found(manager.scale(name, 1, auth.tenant), name) + except TenantQuotaExceededError as error: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(error)) from error @app.post("/api/v1/devboxes/{name}/stop", tags=["devboxes"]) - async def stop_devbox(name: DevboxName, _: Auth) -> Devbox: - return await _not_found(manager.scale(name, 0), name) + async def stop_devbox(name: DevboxName, auth: Auth) -> Devbox: + _require_manage(auth) + return await _not_found(manager.scale(name, 0, auth.tenant), name) @app.delete("/api/v1/devboxes/{name}", tags=["devboxes"]) async def delete_devbox( name: DevboxName, - _: Auth, + auth: Auth, purge: Annotated[bool, Query()] = False, ) -> DeleteResult: - return await _not_found(manager.delete(name, purge), name) + _require_manage(auth) + if purge: + _require_purge(auth) + return await _not_found(manager.delete(name, purge, auth.tenant), name) @app.post("/internal/v1/insights/batches", include_in_schema=False) async def ingest_insights_batch(request: Request) -> dict[str, Any]: @@ -545,10 +640,11 @@ async def ingest_insights_batch(request: Request) -> dict[str, Any]: status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail="Insights batches must use application/json", ) - instance_id, box_name = scope + tenant_id, instance_id, box_name = scope try: body = await _read_limited_body(request, settings.insights_max_compressed_bytes) return await insights.ingest( + tenant_id=tenant_id, instance_id=instance_id, box_name=box_name, compressed_body=body, @@ -574,7 +670,7 @@ async def ingest_insights_batch(request: Request) -> dict[str, Any]: @app.get("/api/v1/insights/summary", tags=["insights"]) async def insights_summary( - _: Auth, + auth: Auth, since: Annotated[str, Query(min_length=2, max_length=64)] = "7d", until: Annotated[str | None, Query(max_length=64)] = None, box: Annotated[str | None, Query(max_length=40)] = None, @@ -593,6 +689,7 @@ async def insights_summary( return insights.disabled_envelope() filters = _insights_filters( insights, + auth.tenant.id, since, until, _query_alias(box, devbox, "box", "devbox"), @@ -607,7 +704,7 @@ async def insights_summary( @app.get("/api/v1/insights/timeseries", tags=["insights"]) async def insights_timeseries( - _: Auth, + auth: Auth, metric: Annotated[str, Query(max_length=32)], since: Annotated[str, Query(min_length=2, max_length=64)] = "7d", until: Annotated[str | None, Query(max_length=64)] = None, @@ -624,6 +721,7 @@ async def insights_timeseries( return insights.disabled_envelope() filters = _insights_filters( insights, + auth.tenant.id, since, until, _query_alias(box, devbox, "box", "devbox"), @@ -648,7 +746,7 @@ async def insights_timeseries( @app.get("/api/v1/insights/activity", tags=["insights"]) async def insights_activity( - _: Auth, + auth: Auth, since: Annotated[str, Query(min_length=2, max_length=64)] = "7d", until: Annotated[str | None, Query(max_length=64)] = None, box: Annotated[str | None, Query(max_length=40)] = None, @@ -665,6 +763,7 @@ async def insights_activity( return insights.disabled_envelope() filters = _insights_filters( insights, + auth.tenant.id, since, until, _query_alias(box, devbox, "box", "devbox"), @@ -688,7 +787,7 @@ async def insights_activity( @app.get("/api/v1/insights/capabilities", tags=["insights"]) async def insights_capabilities( - _: Auth, + auth: Auth, since: Annotated[str, Query(min_length=2, max_length=64)] = "7d", until: Annotated[str | None, Query(max_length=64)] = None, box: Annotated[str | None, Query(max_length=40)] = None, @@ -707,6 +806,7 @@ async def insights_capabilities( return insights.disabled_envelope() filters = _insights_filters( insights, + auth.tenant.id, since, until, _query_alias(box, devbox, "box", "devbox"), @@ -721,7 +821,7 @@ async def insights_capabilities( @app.get("/api/v1/insights/export", tags=["insights"]) async def insights_export( - _: Auth, + auth: Auth, format: Annotated[str, Query(pattern="^(json|csv|sqlite)$")] = "json", since: Annotated[str, Query(min_length=2, max_length=64)] = "30d", until: Annotated[str | None, Query(max_length=64)] = None, @@ -744,6 +844,7 @@ async def insights_export( ) filters = _insights_filters( insights, + auth.tenant.id, since, until, _query_alias(box, devbox, "box", "devbox"), @@ -762,7 +863,7 @@ async def insights_export( ) os.close(descriptor) try: - await insights.backup(backup_path) + await insights.backup(backup_path, auth.tenant.id) except (OSError, sqlite3.Error): await asyncio.to_thread(Path(backup_path).unlink, missing_ok=True) raise @@ -791,11 +892,12 @@ async def insights_export( @app.delete("/api/v1/insights", tags=["insights"]) async def purge_insights_query( - _: Auth, + auth: Auth, instance_id: Annotated[str | None, Query(max_length=36)] = None, box: Annotated[str | None, Query(max_length=40)] = None, devbox: Annotated[str | None, Query(max_length=40)] = None, ) -> dict[str, Any]: + _require_purge(auth) if not insights.enabled: raise HTTPException( status_code=status.HTTP_409_CONFLICT, @@ -803,7 +905,11 @@ async def purge_insights_query( ) selected_box = _query_alias(box, devbox, "box", "devbox") try: - return await insights.purge(selected_box, instance_id=instance_id) + return await insights.purge( + selected_box, + instance_id=instance_id, + tenant_id=auth.tenant.id, + ) except ValueError as error: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, @@ -811,13 +917,14 @@ async def purge_insights_query( ) from error @app.delete("/api/v1/insights/devboxes/{name}", tags=["insights"]) - async def purge_insights_compatibility(name: DevboxName, _: Auth) -> dict[str, Any]: + async def purge_insights_compatibility(name: DevboxName, auth: Auth) -> dict[str, Any]: + _require_purge(auth) if not insights.enabled: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail="Insights is disabled", ) - return await insights.purge(box_name=name) + return await insights.purge(box_name=name, tenant_id=auth.tenant.id) return app @@ -829,8 +936,9 @@ async def _with_workspace_insights( ) -> dict[str, Any]: """Merge Kubernetes rollout state into collector coverage without blocking queries.""" try: - boxes = await manager.list() - except (ApiException, OSError): + tenant = manager.settings.resolve_tenant(filters.tenant_id) + boxes = await manager.list(tenant) + except (ApiException, OSError, ValueError): return envelope collectors = list(envelope.get("coverage", {}).get("collectors", [])) represented = {str(item.get("box")) for item in collectors} @@ -890,6 +998,24 @@ async def _not_found[T](awaitable: Awaitable[T], name: str) -> T: ) from error +def _require_manage(auth: AuthContext) -> None: + """Require a role allowed to mutate ordinary tenant resources.""" + if not auth.can_manage: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="The current tenant role is read-only", + ) + + +def _require_purge(auth: AuthContext) -> None: + """Require an administrative role for irreversible tenant data deletion.""" + if not auth.can_purge: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="The current tenant role cannot purge persistent data", + ) + + def _append_redirect_query(redirect_uri: str, values: dict[str, str]) -> str: parsed = urlsplit(redirect_uri) return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(values), "")) @@ -948,6 +1074,7 @@ async def _read_limited_body(request: Request, maximum: int) -> bytes: def _insights_filters( insights: InsightsService, + tenant_id: str, since: str, until: str | None, box: str | None, @@ -962,6 +1089,7 @@ def _insights_filters( ) -> QueryFilters: try: return insights.filters( + tenant_id=tenant_id, since=since, until=until, box=box, diff --git a/controller/src/devboxes_controller/auth.py b/controller/src/devboxes_controller/auth.py index 4709162..e3088ca 100644 --- a/controller/src/devboxes_controller/auth.py +++ b/controller/src/devboxes_controller/auth.py @@ -5,11 +5,13 @@ import hashlib import hmac import ipaddress +import json import re import secrets import time from collections import OrderedDict from dataclasses import dataclass +from pathlib import Path from typing import Final from urllib.parse import parse_qs, urlsplit @@ -17,14 +19,16 @@ from fastapi import HTTPException, Request, status from jwt import InvalidTokenError -from .config import Settings +from .config import Settings, TenantConfig, TenantRole SESSION_COOKIE = "devboxes_session" CSRF_COOKIE = "devboxes_csrf" CSRF_HEADER = "X-Devboxes-CSRF" +TENANT_HEADER = "X-Devboxes-Tenant" CLI_CLIENT_ID: Final = "devbox-cli" CLI_AUDIENCE: Final = "devbox-cli" -CLI_TOKEN_TYPE: Final = "devboxes-cli-v1" # noqa: S105 - public token type claim +CLI_TOKEN_TYPE: Final = "devboxes-cli-v2" # noqa: S105 - public token type claim +LEGACY_CLI_TOKEN_TYPE: Final = "devboxes-cli-v1" # noqa: S105 - public token type claim CLI_SCOPE: Final = "devboxes:manage" CLI_CALLBACK_PATH: Final = "/callback" CLI_RESPONSE_TYPE: Final = "code" @@ -35,18 +39,61 @@ _CODE_RE = re.compile(r"^[A-Za-z0-9_-]{32,256}$") _SIGNING_DOMAIN = b"devboxes:cli-token-signing-key:v1" _INSIGHTS_SIGNING_DOMAIN = b"devboxes:insights-ingest-signing-key:v1" -_INSIGHTS_TOKEN_RE = re.compile( +_LEGACY_INSIGHTS_TOKEN_RE = re.compile( r"^v1\.([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\." r"([a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?)\.([A-Za-z0-9_-]{43})$" ) +_INSIGHTS_TOKEN_RE = re.compile( + r"^v2\.([a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?)\." + r"([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\." + r"([a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?)\.([A-Za-z0-9_-]{43})$" +) + + +@dataclass(frozen=True) +class TenantAccess: + """Describe one tenant visible to an authenticated principal.""" + + id: str + display_name: str + role: TenantRole + + +@dataclass(frozen=True) +class _Identity: + """Hold one loaded credential and its current tenant memberships.""" + + subject: str + display_name: str + credential_epoch: str + memberships: tuple[TenantAccess, ...] + default_tenant: str + + def access(self, tenant_id: str) -> TenantAccess | None: + """Return this identity's role for a configured tenant.""" + return next((item for item in self.memberships if item.id == tenant_id), None) @dataclass(frozen=True) class AuthContext: - """Describe the authentication mechanism accepted for a request.""" + """Describe one authenticated and tenant-authorized request.""" mode: str subject: str + display_name: str + tenant: TenantConfig + role: TenantRole + tenants: tuple[TenantAccess, ...] + + @property + def can_manage(self) -> bool: + """Return whether ordinary lifecycle mutations are authorized.""" + return self.role.can_manage + + @property + def can_purge(self) -> bool: + """Return whether irreversible tenant data deletion is authorized.""" + return self.role.can_purge @dataclass(frozen=True) @@ -158,11 +205,30 @@ class Authenticator: """Issue and validate controller credentials without storing browser sessions.""" def __init__(self, settings: Settings) -> None: + self._settings = settings self._token = settings.access_token.get_secret_value().encode() self._session_ttl = settings.session_ttl_seconds self._issuer = settings.external_url self._subject = settings.display_name self._cli_token_ttl = settings.cli_token_ttl_seconds + operator_memberships = tuple( + TenantAccess( + id=tenant.id, + display_name=tenant.display_name, + role=TenantRole.OPERATOR, + ) + for tenant in settings.tenant_configs + ) + self._operator = _Identity( + subject=self._subject, + display_name=settings.display_name, + credential_epoch=_credential_epoch(self._token), + memberships=operator_memberships, + default_tenant=settings.default_tenant, + ) + self._identities: dict[str, _Identity] = {self._operator.subject: self._operator} + self._principal_credentials: list[tuple[bytes, _Identity]] = [] + self._load_principals(settings) configured_signing_key = ( settings.cli_signing_key.get_secret_value().encode() if settings.cli_signing_key is not None @@ -182,11 +248,66 @@ def __init__(self, settings: Settings) -> None: or hmac.new(self._token, _INSIGHTS_SIGNING_DOMAIN, hashlib.sha256).digest() ) - def issue_session(self) -> tuple[str, str]: + def _load_principals(self, settings: Settings) -> None: + """Load tenant principal tokens from one mounted Secret directory.""" + if not settings.multi_tenancy_enabled: + return + base = Path(settings.principal_token_directory) + credential_digests: set[bytes] = set() + for principal in settings.principals: + if principal.subject in self._identities: + raise ValueError(f"principal subject {principal.subject!r} conflicts with operator") + path = base / principal.token_key + try: + token = path.read_text(encoding="utf-8").strip().encode() + except OSError as error: + raise ValueError( + f"failed to read token key {principal.token_key!r} for " + f"principal {principal.subject!r}" + ) from error + if len(token) < 32: + raise ValueError( + f"token for principal {principal.subject!r} must contain at least 32 characters" + ) + digest = hashlib.sha256(token).digest() + if digest in credential_digests or hmac.compare_digest(token, self._token): + raise ValueError("principal tokens and the operator token must be unique") + credential_digests.add(digest) + memberships = tuple( + TenantAccess( + id=membership.tenant, + display_name=settings.resolve_tenant(membership.tenant).display_name, + role=TenantRole(membership.role), + ) + for membership in principal.memberships + ) + identity = _Identity( + subject=principal.subject, + display_name=principal.display_name, + credential_epoch=_credential_epoch(token), + memberships=memberships, + default_tenant=principal.default_tenant or memberships[0].id, + ) + self._identities[identity.subject] = identity + self._principal_credentials.append((digest, identity)) + + def issue_session(self, subject: str | None = None) -> tuple[str, str]: """Issue a signed browser session and its matching CSRF token.""" + identity = self._identity(subject or self._subject) + if identity is None: + raise ValueError("cannot issue a session for an unknown principal") csrf = secrets.token_urlsafe(24) - issued_at = str(int(time.time())) - payload = f"{issued_at}:{csrf}".encode() + payload = json.dumps( + { + "v": 2, + "iat": int(time.time()), + "sub": identity.subject, + "epoch": identity.credential_epoch, + "csrf": csrf, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() signature = hmac.new(self._token, payload, hashlib.sha256).digest() return f"{_b64(payload)}.{_b64(signature)}", csrf @@ -194,19 +315,35 @@ def validate_access_token(self, candidate: str) -> bool: """Compare a candidate controller token in constant time.""" return hmac.compare_digest(candidate.encode(), self._token) + def authenticate_access_token(self, candidate: str) -> _Identity | None: + """Resolve an operator or principal access token in constant time.""" + encoded = candidate.encode() + if hmac.compare_digest(encoded, self._token): + return self._operator + digest = hashlib.sha256(encoded).digest() + matched: _Identity | None = None + for expected, identity in self._principal_credentials: + if hmac.compare_digest(digest, expected): + matched = identity + return matched + def issue_cli_token(self, subject: str) -> tuple[str, int]: """Issue a signed, scoped, expiring CLI bearer token.""" + identity = self._identity(subject) + if identity is None: + raise ValueError("cannot issue a CLI token for an unknown principal") now = int(time.time()) claims = { "iss": self._issuer, "aud": CLI_AUDIENCE, - "sub": subject, + "sub": identity.subject, "iat": now, "nbf": now, "exp": now + self._cli_token_ttl, "jti": secrets.token_urlsafe(18), "token_type": CLI_TOKEN_TYPE, "scope": CLI_SCOPE, + "auth_epoch": identity.credential_epoch, } token = jwt.encode( claims, @@ -247,23 +384,49 @@ def validate_cli_token(self, candidate: str) -> str | None: ) issued_at = int(claims["iat"]) expires_at = int(claims["exp"]) + subject = claims.get("sub") + identity = self._identity(subject if isinstance(subject, str) else "") + token_type = claims.get("token_type") + epoch_matches = ( + isinstance(claims.get("auth_epoch"), str) + and identity is not None + and hmac.compare_digest( + str(claims["auth_epoch"]), + identity.credential_epoch, + ) + ) + legacy_operator = ( + token_type == LEGACY_CLI_TOKEN_TYPE + and identity is self._operator + and "auth_epoch" not in claims + ) if ( - claims.get("token_type") != CLI_TOKEN_TYPE + token_type not in {CLI_TOKEN_TYPE, LEGACY_CLI_TOKEN_TYPE} or claims.get("scope") != CLI_SCOPE - or not isinstance(claims.get("sub"), str) - or not claims["sub"] + or identity is None + or not (epoch_matches or legacy_operator) or expires_at <= issued_at or expires_at - issued_at > self._cli_token_ttl or issued_at > int(time.time()) + 30 ): return None - return str(claims["sub"]) + return identity.subject except (InvalidTokenError, TypeError, ValueError): return None - def issue_insights_token(self, instance_id: str, box_name: str) -> str: + def issue_insights_token( + self, + instance_id: str, + box_name: str, + tenant_id: str | None = None, + ) -> str: """Issue a deterministic, write-only token scoped to one retained instance.""" - payload = f"v1.{instance_id}.{box_name}" + tenant = self._settings.resolve_tenant(tenant_id).id + payload = ( + f"v2.{tenant}.{instance_id}.{box_name}" + if self._settings.multi_tenancy_enabled + else f"v1.{instance_id}.{box_name}" + ) signature = hmac.new( self._insights_signing_key, payload.encode(), @@ -271,13 +434,23 @@ def issue_insights_token(self, instance_id: str, box_name: str) -> str: ).digest() return f"{payload}.{_b64(signature)}" - def validate_insights_token(self, candidate: str) -> tuple[str, str] | None: + def validate_insights_token(self, candidate: str) -> tuple[str, str, str] | None: """Validate an ingest-only credential and return its authoritative scope.""" match = _INSIGHTS_TOKEN_RE.fullmatch(candidate) - if match is None: - return None - instance_id, box_name, encoded_signature = match.groups() - payload = f"v1.{instance_id}.{box_name}" + if match is not None: + tenant_id, instance_id, box_name, encoded_signature = match.groups() + try: + self._settings.resolve_tenant(tenant_id) + except ValueError: + return None + payload = f"v2.{tenant_id}.{instance_id}.{box_name}" + else: + legacy = _LEGACY_INSIGHTS_TOKEN_RE.fullmatch(candidate) + if legacy is None: + return None + instance_id, box_name, encoded_signature = legacy.groups() + tenant_id = self._settings.default_tenant + payload = f"v1.{instance_id}.{box_name}" try: signature = _unb64(encoded_signature) except ValueError: @@ -289,10 +462,15 @@ def validate_insights_token(self, candidate: str) -> tuple[str, str] | None: ).digest() if not hmac.compare_digest(signature, expected): return None - return instance_id, box_name + return tenant_id, instance_id, box_name def validate_session(self, candidate: str) -> str | None: """Return the CSRF value for a valid unexpired session.""" + claims = self._session_claims(candidate) + return claims[1] if claims is not None else None + + def _session_claims(self, candidate: str) -> tuple[_Identity, str] | None: + """Validate a signed browser session and return its live identity.""" try: encoded_payload, encoded_signature = candidate.split(".", maxsplit=1) payload = _unb64(encoded_payload) @@ -300,59 +478,136 @@ def validate_session(self, candidate: str) -> str | None: expected = hmac.new(self._token, payload, hashlib.sha256).digest() if not hmac.compare_digest(signature, expected): return None - issued_at_raw, csrf = payload.decode().split(":", maxsplit=1) - issued_at = int(issued_at_raw) - except (ValueError, UnicodeDecodeError): + decoded = payload.decode() + if decoded.startswith("{"): + values = json.loads(decoded) + if values.get("v") != 2: + return None + issued_at = int(values["iat"]) + csrf = str(values["csrf"]) + identity = self._identity(str(values["sub"])) + if ( + identity is None + or not isinstance(values.get("epoch"), str) + or not hmac.compare_digest(values["epoch"], identity.credential_epoch) + ): + return None + else: + # Sessions from releases before tenancy were always operator sessions. + issued_at_raw, csrf = decoded.split(":", maxsplit=1) + issued_at = int(issued_at_raw) + identity = self._operator + except (KeyError, TypeError, ValueError, UnicodeDecodeError, json.JSONDecodeError): return None if issued_at > time.time() + 30 or time.time() - issued_at > self._session_ttl: return None - return csrf + if not csrf or len(csrf) > 256: + return None + return identity, csrf async def require(self, request: Request) -> AuthContext: """Require a master/CLI bearer token or a CSRF-protected browser session.""" authorization = request.headers.get("Authorization", "") if authorization.startswith("Bearer "): candidate = authorization.removeprefix("Bearer ").strip() - if self.validate_access_token(candidate): - return AuthContext(mode="master-bearer", subject=self._subject) + identity = self.authenticate_access_token(candidate) + if identity is not None: + mode = "master-bearer" if identity is self._operator else "principal-bearer" + return self._context(identity, request.headers.get(TENANT_HEADER), mode) subject = self.validate_cli_token(candidate) if subject is not None: - return AuthContext(mode="cli-bearer", subject=subject) + identity = self._identity(subject) + if identity is not None: + return self._context( + identity, + request.headers.get(TENANT_HEADER), + "cli-bearer", + ) raise _unauthorized() - csrf = self.browser_csrf(request) - if csrf is None: + browser = self._browser_claims(request) + if browser is None: raise _unauthorized() + identity, csrf = browser if request.method not in {"GET", "HEAD", "OPTIONS"}: supplied_csrf = request.headers.get(CSRF_HEADER, "") if not supplied_csrf or not hmac.compare_digest(supplied_csrf, csrf): raise _csrf_error() - return AuthContext(mode="browser-session", subject=self._subject) + return self._context( + identity, + request.headers.get(TENANT_HEADER), + "browser-session", + ) def browser_csrf(self, request: Request) -> str | None: """Return the CSRF value when both browser-session cookies agree.""" + claims = self._browser_claims(request) + return claims[1] if claims is not None else None + + def _browser_claims(self, request: Request) -> tuple[_Identity, str] | None: + """Return the current identity and CSRF value when browser cookies agree.""" session = request.cookies.get(SESSION_COOKIE) cookie_csrf = request.cookies.get(CSRF_COOKIE, "") if not session or not cookie_csrf: return None - csrf = self.validate_session(session) - if csrf is None or not hmac.compare_digest(cookie_csrf, csrf): + claims = self._session_claims(session) + if claims is None or not hmac.compare_digest(cookie_csrf, claims[1]): + return None + return claims + + def browser_context( + self, + request: Request, + tenant_id: str | None = None, + ) -> AuthContext | None: + """Resolve a browser session to one authorized tenant context.""" + claims = self._browser_claims(request) + if claims is None: return None - return csrf + return self._context(claims[0], tenant_id, "browser-session") def require_form_csrf(self, request: Request, supplied_csrf: str) -> AuthContext: """Require a browser session and matching HTML form CSRF token.""" - csrf = self.browser_csrf(request) - if csrf is None: + claims = self._browser_claims(request) + if claims is None: raise _unauthorized() + identity, csrf = claims if not supplied_csrf or not hmac.compare_digest(supplied_csrf, csrf): raise _csrf_error() - return AuthContext(mode="browser-session", subject=self._subject) + return self._context(identity, None, "browser-session") def browser_session_valid(self, request: Request) -> bool: """Return whether a request carries a valid browser session.""" return self.browser_csrf(request) is not None + def _identity(self, subject: str) -> _Identity | None: + """Return a configured identity by its signed stable subject.""" + return self._identities.get(subject) + + def _context( + self, + identity: _Identity, + tenant_id: str | None, + mode: str, + ) -> AuthContext: + """Resolve and authorize exactly one tenant before application work begins.""" + requested = (tenant_id or identity.default_tenant).strip().lower() + access = identity.access(requested) + if access is None: + raise _tenant_forbidden() + try: + tenant = self._settings.resolve_tenant(requested) + except ValueError: + raise _tenant_forbidden() from None + return AuthContext( + mode=mode, + subject=identity.subject, + display_name=identity.display_name, + tenant=tenant, + role=access.role, + tenants=identity.memberships, + ) + def validate_authorization_request( *, @@ -473,6 +728,13 @@ def _csrf_error() -> HTTPException: ) +def _tenant_forbidden() -> HTTPException: + return HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="The requested tenant is not available to this identity", + ) + + def _oauth_error(message: str) -> HTTPException: return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) @@ -483,3 +745,8 @@ def _b64(value: bytes) -> str: def _unb64(value: str) -> bytes: return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + + +def _credential_epoch(token: bytes) -> str: + """Derive a non-secret revocation marker from one high-entropy credential.""" + return hashlib.sha256(token).hexdigest()[:24] diff --git a/controller/src/devboxes_controller/config.py b/controller/src/devboxes_controller/config.py index 98d19a4..d44c5a4 100644 --- a/controller/src/devboxes_controller/config.py +++ b/controller/src/devboxes_controller/config.py @@ -1,7 +1,9 @@ """Validate controller configuration loaded from environment variables.""" import re +from enum import StrEnum from functools import lru_cache +from pathlib import Path from typing import Annotated, Literal, Self from urllib.parse import urlsplit @@ -9,10 +11,15 @@ from pydantic import BaseModel, ConfigDict, Field, SecretStr, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict +from .models import Preset + GPU_PROFILE_NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$") DNS_LABEL_RE = re.compile(r"^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$") QUALIFIED_NAME_PART_RE = re.compile(r"^[A-Za-z0-9](?:[-_.A-Za-z0-9]{0,61}[A-Za-z0-9])?$") LABEL_VALUE_RE = re.compile(r"^(?:[A-Za-z0-9](?:[-_.A-Za-z0-9]{0,61}[A-Za-z0-9])?)?$") +PRINCIPAL_SUBJECT_RE = re.compile(r"^[A-Za-z0-9](?:[-_.@A-Za-z0-9]{0,126}[A-Za-z0-9])?$") +SECRET_KEY_RE = re.compile(r"^[A-Za-z0-9](?:[-_.A-Za-z0-9]{0,251}[A-Za-z0-9])?$") +RESOURCE_QUOTA_KEY_RE = re.compile(r"^[A-Za-z0-9](?:[-./A-Za-z0-9]{0,251}[A-Za-z0-9])?$") SupplementalGroup = Annotated[int, Field(strict=True, ge=1, le=2_147_483_647)] @@ -39,6 +46,297 @@ def _valid_qualified_name(value: str, *, require_prefix: bool) -> bool: return not require_prefix and bool(QUALIFIED_NAME_PART_RE.fullmatch(value)) +class TenantRole(StrEnum): + """Define the application permissions granted inside one tenant.""" + + VIEWER = "viewer" + MEMBER = "member" + ADMIN = "admin" + OPERATOR = "operator" + + @property + def can_manage(self) -> bool: + """Return whether the role may change ordinary workspace lifecycle state.""" + return self in {self.MEMBER, self.ADMIN, self.OPERATOR} + + @property + def can_purge(self) -> bool: + """Return whether the role may permanently remove retained data.""" + return self in {self.ADMIN, self.OPERATOR} + + +class TenantResourceQuota(BaseModel): + """Describe the namespace ResourceQuota policy shown to authenticated clients.""" + + model_config = ConfigDict(extra="forbid", populate_by_name=True, frozen=True) + + enabled: bool = True + hard: dict[str, str] = Field(default_factory=dict, max_length=32) + + @field_validator("hard") + @classmethod + def hard_limits_are_valid(cls, value: dict[str, str]) -> dict[str, str]: + """Validate bounded positive Kubernetes resource quantities.""" + normalized: dict[str, str] = {} + for raw_key, raw_quantity in value.items(): + key = raw_key.strip() + quantity = str(raw_quantity).strip() + if not RESOURCE_QUOTA_KEY_RE.fullmatch(key): + raise ValueError(f"invalid ResourceQuota key {raw_key!r}") + try: + if parse_quantity(quantity) <= 0: + raise ValueError + except ValueError as error: + raise ValueError(f"ResourceQuota {key!r} must be a positive quantity") from error + normalized[key] = quantity + return normalized + + @model_validator(mode="after") + def enabled_quota_has_limits(self) -> Self: + """Reject a quota object that claims enforcement without any limits.""" + if self.enabled and not self.hard: + raise ValueError("enabled ResourceQuota requires at least one hard limit") + return self + + +class TenantConfig(BaseModel): + """Define one operator-owned namespace and policy domain.""" + + model_config = ConfigDict(extra="forbid", populate_by_name=True, frozen=True) + + id: str = Field(min_length=1, max_length=40) + display_name: str = Field(alias="displayName", min_length=1, max_length=80) + namespace: str = Field(min_length=1, max_length=63) + create_namespace: bool = Field(default=False, alias="createNamespace") + namespace_labels: dict[str, str] = Field( + default_factory=dict, + alias="namespaceLabels", + max_length=16, + ) + workspace_secret_name: str | None = Field( + default=None, + alias="workspaceSecretName", + max_length=253, + ) + workspace_service_account_name: str | None = Field( + default=None, + alias="workspaceServiceAccountName", + max_length=253, + ) + storage_class: str | None = Field(default=None, alias="storageClass", max_length=253) + max_ttl_hours: int | None = Field(default=None, alias="maxTtlHours", ge=1, le=168) + allowed_presets: list[Preset] = Field( + default_factory=lambda: list(Preset), + alias="allowedPresets", + min_length=1, + max_length=3, + ) + allowed_gpu_profiles: list[str] | None = Field( + default=None, + alias="allowedGpuProfiles", + max_length=32, + ) + default_gpu_profile: str | None = Field( + default=None, + alias="defaultGpuProfile", + max_length=40, + ) + allowed_image_profiles: list[str] | None = Field( + default=None, + alias="allowedImageProfiles", + max_length=32, + ) + resource_quota: TenantResourceQuota | None = Field(default=None, alias="resourceQuota") + + @field_validator("id") + @classmethod + def id_is_safe(cls, value: str) -> str: + """Keep tenant IDs stable across HTTP, CLI, labels, and token claims.""" + value = value.strip().lower() + if not GPU_PROFILE_NAME_RE.fullmatch(value): + raise ValueError( + "use 1-40 lowercase letters, digits, or hyphens; start and end alphanumeric" + ) + return value + + @field_validator("display_name") + @classmethod + def display_name_is_not_blank(cls, value: str) -> str: + """Normalize the user-visible tenant name.""" + value = value.strip() + if not value: + raise ValueError("must not be blank") + return value + + @field_validator("namespace") + @classmethod + def namespace_is_safe(cls, value: str) -> str: + """Require the namespace-per-tenant isolation unit to be a DNS label.""" + value = value.strip().lower() + if not DNS_LABEL_RE.fullmatch(value) or value == "default" or value.startswith("kube-"): + raise ValueError("must be a valid non-system Kubernetes namespace name") + return value + + @field_validator("namespace_labels") + @classmethod + def namespace_labels_are_safe(cls, value: dict[str, str]) -> dict[str, str]: + """Validate labels without allowing tenant ownership metadata to be replaced.""" + reserved = { + "app.kubernetes.io/part-of", + "devboxes.bonalab.org/tenant", + "kubernetes.io/metadata.name", + } + normalized: dict[str, str] = {} + for raw_key, raw_value in value.items(): + key = raw_key.strip() + label_value = str(raw_value).strip() + if key in reserved: + raise ValueError(f"namespaceLabels cannot override reserved label {key!r}") + if not _valid_qualified_name(key, require_prefix=False): + raise ValueError(f"invalid namespace label key {raw_key!r}") + if not LABEL_VALUE_RE.fullmatch(label_value): + raise ValueError(f"invalid namespace label value for {key!r}") + normalized[key] = label_value + return normalized + + @field_validator( + "workspace_secret_name", + "workspace_service_account_name", + ) + @classmethod + def optional_resource_name_is_safe(cls, value: str | None) -> str | None: + """Validate optional namespaced Kubernetes resource names.""" + if value is None or not value.strip(): + return None + value = value.strip().lower() + if not _valid_dns_subdomain(value): + raise ValueError("must be a valid Kubernetes DNS subdomain") + return value + + @field_validator("storage_class") + @classmethod + def optional_storage_class_is_safe(cls, value: str | None) -> str | None: + """Normalize the optional per-tenant StorageClass override.""" + if value is None or not value.strip(): + return None + value = value.strip() + if not _valid_dns_subdomain(value): + raise ValueError("must be a valid Kubernetes DNS subdomain") + return value + + @field_validator("allowed_presets") + @classmethod + def presets_are_unique(cls, value: list[Preset]) -> list[Preset]: + """Reject ambiguous duplicate policy entries.""" + if len(value) != len(set(value)): + raise ValueError("allowedPresets must contain unique values") + return value + + @field_validator("allowed_gpu_profiles", "allowed_image_profiles") + @classmethod + def profile_names_are_safe(cls, value: list[str] | None) -> list[str] | None: + """Normalize optional allowlists; an empty list deliberately allows none.""" + if value is None: + return None + normalized = [item.strip().lower() for item in value] + if any(not GPU_PROFILE_NAME_RE.fullmatch(item) for item in normalized): + raise ValueError("profile allowlists must contain valid profile names") + if len(normalized) != len(set(normalized)): + raise ValueError("profile allowlists must contain unique values") + return normalized + + @field_validator("default_gpu_profile") + @classmethod + def default_gpu_profile_is_safe(cls, value: str | None) -> str | None: + """Normalize an optional tenant-specific accelerator default.""" + if value is None or not value.strip(): + return None + value = value.strip().lower() + if not GPU_PROFILE_NAME_RE.fullmatch(value): + raise ValueError("defaultGpuProfile must contain a valid profile name") + return value + + +class TenantMembership(BaseModel): + """Grant one principal a role in one configured tenant.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + tenant: str = Field(min_length=1, max_length=40) + role: Literal["viewer", "member", "admin"] = "member" + + @field_validator("tenant") + @classmethod + def tenant_is_safe(cls, value: str) -> str: + """Normalize the referenced tenant ID.""" + value = value.strip().lower() + if not GPU_PROFILE_NAME_RE.fullmatch(value): + raise ValueError("must reference a valid tenant ID") + return value + + +class PrincipalConfig(BaseModel): + """Define one human or automation identity without embedding its token.""" + + model_config = ConfigDict(extra="forbid", populate_by_name=True, frozen=True) + + subject: str = Field(min_length=1, max_length=128) + display_name: str = Field(alias="displayName", min_length=1, max_length=80) + token_key: str = Field(alias="tokenKey", min_length=1, max_length=253) + memberships: list[TenantMembership] = Field(min_length=1, max_length=32) + default_tenant: str | None = Field(default=None, alias="defaultTenant", max_length=40) + + @field_validator("subject") + @classmethod + def subject_is_safe(cls, value: str) -> str: + """Keep subjects unambiguous in signed sessions and audit annotations.""" + value = value.strip() + if not PRINCIPAL_SUBJECT_RE.fullmatch(value): + raise ValueError( + "use 1-128 letters, digits, dots, underscores, hyphens, or @ characters" + ) + return value + + @field_validator("display_name") + @classmethod + def display_name_is_not_blank(cls, value: str) -> str: + """Normalize the user-facing principal name.""" + value = value.strip() + if not value: + raise ValueError("must not be blank") + return value + + @field_validator("token_key") + @classmethod + def token_key_is_safe(cls, value: str) -> str: + """Prevent traversal outside the mounted Kubernetes Secret directory.""" + value = value.strip() + if not SECRET_KEY_RE.fullmatch(value): + raise ValueError("must be a valid Kubernetes Secret data key") + return value + + @field_validator("default_tenant") + @classmethod + def default_tenant_is_safe(cls, value: str | None) -> str | None: + """Normalize an optional per-principal default tenant.""" + if value is None or not value.strip(): + return None + value = value.strip().lower() + if not GPU_PROFILE_NAME_RE.fullmatch(value): + raise ValueError("must reference a valid tenant ID") + return value + + @model_validator(mode="after") + def memberships_are_unique(self) -> Self: + """Require exactly one role per tenant and a reachable default.""" + tenant_ids = [membership.tenant for membership in self.memberships] + if len(tenant_ids) != len(set(tenant_ids)): + raise ValueError("principal memberships must reference unique tenants") + if self.default_tenant is not None and self.default_tenant not in tenant_ids: + raise ValueError("defaultTenant must name one of the principal memberships") + return self + + class GpuToleration(BaseModel): """Describe one operator-approved toleration applied to GPU workspaces.""" @@ -347,6 +645,11 @@ class Settings(BaseSettings): workspace_load_balancer_class: str | None = None workspace_external_traffic_policy: Literal["Cluster", "Local"] = "Cluster" workspace_load_balancer_source_ranges: list[str] = Field(default_factory=list) + multi_tenancy_enabled: bool = False + default_tenant: str = "default" + tenants: list[TenantConfig] = Field(default_factory=list, max_length=64) + principals: list[PrincipalConfig] = Field(default_factory=list, max_length=256) + principal_token_directory: str = "/var/run/secrets/devboxes/principals" # noqa: S105 - mounted Secret path access_token: SecretStr default_ttl_hours: int = Field(default=24, le=168) max_ttl_hours: int = Field(default=168, le=168) @@ -475,6 +778,25 @@ def access_token_is_not_blank(cls, value: SecretStr) -> SecretStr: raise ValueError("access token must contain at least 32 characters") return value + @field_validator("default_tenant") + @classmethod + def default_tenant_is_safe(cls, value: str) -> str: + """Keep the installation default usable as a tenant identifier.""" + value = value.strip().lower() + if not GPU_PROFILE_NAME_RE.fullmatch(value): + raise ValueError("default_tenant must be a valid tenant ID") + return value + + @field_validator("principal_token_directory") + @classmethod + def principal_token_directory_is_absolute(cls, value: str) -> str: + """Keep principal Secret reads inside one explicit mounted directory.""" + value = value.strip().rstrip("/") + path = Path(value) + if not value or not path.is_absolute() or ".." in path.parts: + raise ValueError("principal_token_directory must be an absolute non-root path") + return value + @model_validator(mode="after") def settings_are_consistent(self) -> Self: """Validate relationships between independently parsed settings.""" @@ -510,27 +832,188 @@ def settings_are_consistent(self) -> Self: raise ValueError("custom image profile references must be unique") if self.custom_images_enabled and not custom_image_names: raise ValueError("custom_images_enabled requires at least one custom image profile") + tenant_ids = [tenant.id for tenant in self.tenants] + tenant_namespaces = [tenant.namespace for tenant in self.tenants] + if len(tenant_ids) != len(set(tenant_ids)): + raise ValueError("tenant IDs must be unique") + if len(tenant_namespaces) != len(set(tenant_namespaces)): + raise ValueError("tenant namespaces must be unique") + principal_subjects = [principal.subject for principal in self.principals] + principal_token_keys = [principal.token_key for principal in self.principals] + if len(principal_subjects) != len(set(principal_subjects)): + raise ValueError("principal subjects must be unique") + if len(principal_token_keys) != len(set(principal_token_keys)): + raise ValueError("principal token keys must be unique") + if self.multi_tenancy_enabled: + if not self.tenants: + raise ValueError("multi_tenancy_enabled requires at least one tenant") + if self.default_tenant not in tenant_ids: + raise ValueError("default_tenant must name a configured tenant") + configured_gpu_profiles = set(profile_names) + configured_image_profiles = set(custom_image_names) + for tenant in self.tenants: + tenant_max_ttl = tenant.max_ttl_hours or self.max_ttl_hours + if tenant_max_ttl > self.max_ttl_hours: + raise ValueError( + f"tenant {tenant.id!r} maxTtlHours cannot exceed controller maximum" + ) + unavailable_gpu = set(tenant.allowed_gpu_profiles or []) - configured_gpu_profiles + if unavailable_gpu: + raise ValueError( + f"tenant {tenant.id!r} references unknown GPU profiles: " + f"{', '.join(sorted(unavailable_gpu))}" + ) + if tenant.default_gpu_profile is not None: + if not self.gpu_enabled: + raise ValueError( + f"tenant {tenant.id!r} defaultGpuProfile requires GPU acceleration" + ) + if tenant.default_gpu_profile not in configured_gpu_profiles: + raise ValueError( + f"tenant {tenant.id!r} references unknown default GPU profile " + f"{tenant.default_gpu_profile!r}" + ) + if ( + tenant.allowed_gpu_profiles is not None + and tenant.default_gpu_profile not in tenant.allowed_gpu_profiles + ): + raise ValueError( + f"tenant {tenant.id!r} defaultGpuProfile must be included in " + "allowedGpuProfiles" + ) + unavailable_images = ( + set(tenant.allowed_image_profiles or []) - configured_image_profiles + ) + if unavailable_images: + raise ValueError( + f"tenant {tenant.id!r} references unknown image profiles: " + f"{', '.join(sorted(unavailable_images))}" + ) + configured_tenants = set(tenant_ids) + for principal in self.principals: + unavailable_tenants = { + membership.tenant for membership in principal.memberships + } - configured_tenants + if unavailable_tenants: + raise ValueError( + f"principal {principal.subject!r} references unknown tenants: " + f"{', '.join(sorted(unavailable_tenants))}" + ) + elif self.tenants or self.principals: + raise ValueError( + "tenants and principals require multi_tenancy_enabled=true to avoid " + "silently unenforced policy" + ) return self - def resolve_gpu_profile(self, requested_name: str | None) -> GpuProfile: + @property + def tenant_configs(self) -> tuple[TenantConfig, ...]: + """Return configured tenants or the backward-compatible installation tenant.""" + if self.multi_tenancy_enabled: + return tuple(self.tenants) + return ( + TenantConfig( + id=self.default_tenant, + displayName=self.display_name, + namespace=self.namespace, + workspaceSecretName=self.workspace_secret_name, + workspaceServiceAccountName=self.workspace_service_account_name, + storageClass=self.storage_class, + maxTtlHours=self.max_ttl_hours, + ), + ) + + def resolve_tenant(self, tenant_id: str | None) -> TenantConfig: + """Resolve one configured tenant without falling back on an unknown value.""" + requested = (tenant_id or self.default_tenant).strip().lower() + for tenant in self.tenant_configs: + if tenant.id == requested: + return tenant + raise ValueError(f"unknown tenant {requested!r}") + + def tenant_max_ttl_hours(self, tenant: TenantConfig) -> int: + """Return the effective TTL ceiling for one tenant.""" + return min(tenant.max_ttl_hours or self.max_ttl_hours, self.max_ttl_hours) + + def tenant_workspace_secret_name(self, tenant: TenantConfig) -> str: + """Return the tenant-local Secret mounted into its workspaces.""" + return tenant.workspace_secret_name or self.workspace_secret_name + + def tenant_workspace_service_account_name(self, tenant: TenantConfig) -> str: + """Return the tenant-local no-RBAC workspace ServiceAccount.""" + return tenant.workspace_service_account_name or self.workspace_service_account_name + + def tenant_storage_class(self, tenant: TenantConfig) -> str | None: + """Return the per-tenant or installation StorageClass selection.""" + return tenant.storage_class or self.storage_class + + def tenant_gpu_profiles(self, tenant: TenantConfig) -> list[GpuProfile]: + """Return GPU profiles admitted by installation and tenant policy.""" + allowed = ( + set(tenant.allowed_gpu_profiles) if tenant.allowed_gpu_profiles is not None else None + ) + return [ + profile + for profile in self.gpu_profiles + if self.gpu_enabled and (allowed is None or profile.name in allowed) + ] + + def tenant_default_gpu_profile(self, tenant: TenantConfig) -> str | None: + """Return the explicit or inherited GPU default available to one tenant.""" + available = {profile.name for profile in self.tenant_gpu_profiles(tenant)} + if tenant.default_gpu_profile in available: + return tenant.default_gpu_profile + return self.gpu_default_profile if self.gpu_default_profile in available else None + + def tenant_custom_images(self, tenant: TenantConfig) -> list[CustomImageProfile]: + """Return custom images admitted by installation and tenant policy.""" + allowed = ( + set(tenant.allowed_image_profiles) + if tenant.allowed_image_profiles is not None + else None + ) + return [ + profile + for profile in self.custom_images + if self.custom_images_enabled and (allowed is None or profile.name in allowed) + ] + + def resolve_gpu_profile( + self, + requested_name: str | None, + tenant: TenantConfig | None = None, + ) -> GpuProfile: """Resolve a user request to one trusted installation profile.""" if not self.gpu_enabled: raise ValueError("GPU acceleration is disabled by the operator") - profile_name = requested_name or self.gpu_default_profile - for profile in self.gpu_profiles: + selected_tenant = tenant or self.resolve_tenant(None) + profiles = self.tenant_gpu_profiles(selected_tenant) + profile_name = requested_name or self.tenant_default_gpu_profile(selected_tenant) + if profile_name is None: + available = ", ".join(profile.name for profile in profiles) or "none" + raise ValueError( + "no default GPU profile is available to this tenant; " + f"select an explicit profile from: {available}" + ) + for profile in profiles: if profile.name == profile_name: return profile - available = ", ".join(profile.name for profile in self.gpu_profiles) + available = ", ".join(profile.name for profile in profiles) or "none" raise ValueError(f"unknown GPU profile {profile_name!r}; available profiles: {available}") - def resolve_custom_image(self, selector: str) -> CustomImageProfile: + def resolve_custom_image( + self, + selector: str, + tenant: TenantConfig | None = None, + ) -> CustomImageProfile: """Resolve a profile name or exact approved image reference before pod creation.""" if not self.custom_images_enabled: raise ValueError("custom images are disabled by the operator") - for profile in self.custom_images: + profiles = self.tenant_custom_images(tenant or self.resolve_tenant(None)) + for profile in profiles: if selector in {profile.name, profile.image}: return profile - available = ", ".join(profile.name for profile in self.custom_images) + available = ", ".join(profile.name for profile in profiles) or "none" raise ValueError(f"unknown custom image {selector!r}; available profiles: {available}") diff --git a/controller/src/devboxes_controller/insights_service.py b/controller/src/devboxes_controller/insights_service.py index c1f9e59..c624ae7 100644 --- a/controller/src/devboxes_controller/insights_service.py +++ b/controller/src/devboxes_controller/insights_service.py @@ -108,6 +108,7 @@ def __init__(self, settings: Settings, store: InsightsStore | None = None) -> No raw_days=settings.insights_retention_raw_days, hourly_days=settings.insights_retention_hourly_days, daily_days=settings.insights_retention_daily_days, + default_tenant_id=settings.default_tenant, ) self._rates: dict[str, deque[float]] = defaultdict(deque) self._rate_lock = asyncio.Lock() @@ -154,6 +155,7 @@ async def check_rate(self, instance_id: str) -> None: async def ingest( self, *, + tenant_id: str = "default", instance_id: str, box_name: str, compressed_body: bytes, @@ -170,6 +172,7 @@ async def ingest( payload.update(_collector_metadata(batch)) started = time.perf_counter() result = await self.store.ingest( + tenant_id=tenant_id, instance_id=instance_id, box_name=box_name, batch_id=str(batch["batch_id"]), @@ -282,6 +285,7 @@ async def purge( box_name: str | None = None, *, instance_id: str | None = None, + tenant_id: str = "default", ) -> dict[str, Any]: """Delete central history without touching any Kubernetes resources.""" self._require_enabled() @@ -290,9 +294,9 @@ async def purge( if instance_id is not None: _validate_instance_id(instance_id) count = ( - await self.store.purge_instance(instance_id) + await self.store.purge_instance(instance_id, tenant_id) if instance_id is not None - else await self.store.purge_box(str(box_name)) + else await self.store.purge_box(str(box_name), tenant_id) ) database_bytes = await self.store.database_size() INSIGHTS_DATABASE_BYTES.set(database_bytes) @@ -314,6 +318,7 @@ async def purge( }, "box": box_name or "", "instance_id": instance_id, + "tenant": tenant_id, "purged_instances": count, } @@ -329,10 +334,14 @@ async def maintain(self) -> dict[str, int]: result["database_bytes"] = database_bytes return result - async def backup(self, destination: str | Path) -> None: + async def backup( + self, + destination: str | Path, + tenant_id: str | None = None, + ) -> None: """Create an authenticated, transactionally consistent SQLite snapshot.""" self._require_enabled() - await self.store.backup(destination) + await self.store.backup(destination, tenant_id) def filters( self, @@ -344,6 +353,7 @@ def filters( model: str | None, repo: str | None, maximum_days: int, + tenant_id: str = "default", instance_id: str | None = None, group_by: str | None = None, bucket: str | None = None, @@ -366,6 +376,7 @@ def filters( return QueryFilters( since=start, until=end, + tenant_id=tenant_id, box=box, instance_id=instance_id, provider=provider, @@ -488,6 +499,7 @@ def _envelope( "schema_version": SCHEMA_VERSION, "generated_at": datetime.now(UTC).isoformat(), "enabled": True, + "tenant": filters.tenant_id, "effective_range": { "since": filters.since.isoformat(), "until": filters.until.isoformat(), diff --git a/controller/src/devboxes_controller/insights_store.py b/controller/src/devboxes_controller/insights_store.py index f83a86a..c5bc700 100644 --- a/controller/src/devboxes_controller/insights_store.py +++ b/controller/src/devboxes_controller/insights_store.py @@ -13,7 +13,10 @@ from pathlib import Path from typing import Any, Final +# The public ingest/query envelope remains compatible with v1 collectors. The +# private SQLite migration version advances independently. SCHEMA_VERSION: Final = 1 +DATABASE_SCHEMA_VERSION: Final = 2 _MINIMUM_TIMESTAMP = datetime(2020, 1, 1, tzinfo=UTC) @@ -33,6 +36,7 @@ class QueryFilters: since: datetime until: datetime + tenant_id: str = "default" box: str | None = None instance_id: str | None = None provider: str | None = None @@ -52,11 +56,13 @@ def __init__( raw_days: int = 30, hourly_days: int = 90, daily_days: int = 365, + default_tenant_id: str = "default", ) -> None: self.path = Path(path) self.raw_days = raw_days self.hourly_days = hourly_days self.daily_days = daily_days + self.default_tenant_id = default_tenant_id self._writer_lock = asyncio.Lock() async def initialize(self) -> None: @@ -74,6 +80,7 @@ async def ready(self) -> bool: async def ingest( self, *, + tenant_id: str = "default", instance_id: str, box_name: str, batch_id: str, @@ -87,6 +94,7 @@ async def ingest( async with self._writer_lock: return await asyncio.to_thread( self._ingest_sync, + tenant_id, instance_id, box_name, batch_id, @@ -119,25 +127,33 @@ async def collector_status(self, filters: QueryFilters) -> list[dict[str, Any]]: """Return collector freshness and loss counters for coverage reporting.""" return await asyncio.to_thread(self._collector_status_sync, filters) - async def purge_box(self, box_name: str) -> int: + async def purge_box(self, box_name: str, tenant_id: str = "default") -> int: """Delete central Insights records for every instance that used a box name.""" async with self._writer_lock: - return await asyncio.to_thread(self._purge_box_sync, box_name) + return await asyncio.to_thread(self._purge_box_sync, tenant_id, box_name) - async def purge_instance(self, instance_id: str) -> int: + async def purge_instance( + self, + instance_id: str, + tenant_id: str = "default", + ) -> int: """Delete central Insights records for one stable workspace instance.""" async with self._writer_lock: - return await asyncio.to_thread(self._purge_instance_sync, instance_id) + return await asyncio.to_thread( + self._purge_instance_sync, + tenant_id, + instance_id, + ) async def maintain(self) -> dict[str, int]: """Materialize rollups, enforce retention, and checkpoint the WAL.""" async with self._writer_lock: return await asyncio.to_thread(self._maintain_sync) - async def backup(self, destination: str | Path) -> None: - """Create a consistent live snapshot through SQLite's online backup API.""" + async def backup(self, destination: str | Path, tenant_id: str | None = None) -> None: + """Create a consistent, optionally tenant-scoped SQLite snapshot.""" async with self._writer_lock: - await asyncio.to_thread(self._backup_sync, Path(destination)) + await asyncio.to_thread(self._backup_sync, Path(destination), tenant_id) async def database_size(self) -> int: """Return the current SQLite main, WAL, and shared-memory footprint.""" @@ -158,6 +174,30 @@ def _initialize_sync(self) -> None: connection.execute("PRAGMA journal_mode=WAL") connection.execute("PRAGMA synchronous=FULL") connection.executescript(f"BEGIN IMMEDIATE;\n{_SCHEMA}\nCOMMIT;") + connection.execute("BEGIN IMMEDIATE") + columns = { + str(row["name"]) + for row in connection.execute("PRAGMA table_info(devbox_instances)") + } + if "tenant_id" not in columns: + connection.execute( + "ALTER TABLE devbox_instances " + "ADD COLUMN tenant_id TEXT NOT NULL DEFAULT 'default'" + ) + connection.execute( + "UPDATE devbox_instances SET tenant_id = ?", + (self.default_tenant_id,), + ) + connection.execute( + "CREATE INDEX IF NOT EXISTS idx_instances_tenant_name " + "ON devbox_instances(tenant_id, name)" + ) + connection.execute( + "INSERT OR IGNORE INTO schema_migrations(version, applied_at) " + "VALUES (?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))", + (DATABASE_SCHEMA_VERSION,), + ) + connection.commit() except Exception: connection.rollback() raise @@ -171,12 +211,13 @@ def _ready_sync(self) -> bool: "SELECT MAX(version) AS version FROM schema_migrations" ).fetchone() connection.execute("SELECT 1 FROM metric_points LIMIT 1").fetchone() - return row is not None and row["version"] == SCHEMA_VERSION + return row is not None and row["version"] == DATABASE_SCHEMA_VERSION finally: connection.close() def _ingest_sync( self, + tenant_id: str, instance_id: str, box_name: str, batch_id: str, @@ -211,6 +252,12 @@ def _ingest_sync( ).hexdigest() try: connection.execute("BEGIN IMMEDIATE") + existing_instance = connection.execute( + "SELECT tenant_id FROM devbox_instances WHERE id = ?", + (instance_id,), + ).fetchone() + if existing_instance is not None and existing_instance["tenant_id"] != tenant_id: + raise ValueError("instance identifier conflicts with another tenant") duplicate = connection.execute( "SELECT instance_id, fingerprint FROM ingest_batches WHERE batch_id = ?", (batch_id,), @@ -225,13 +272,15 @@ def _ingest_sync( return IngestResult(False, True, 0, ()) connection.execute( """ - INSERT INTO devbox_instances(id, name, first_seen_at, last_seen_at) - VALUES (?, ?, ?, ?) + INSERT INTO devbox_instances( + id, tenant_id, name, first_seen_at, last_seen_at + ) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET name=excluded.name, last_seen_at=excluded.last_seen_at, deleted_at=NULL """, - (instance_id, box_name, received_at, received_at), + (instance_id, tenant_id, box_name, received_at, received_at), ) if kind == "otlp": inserted_points, providers = self._insert_otlp( @@ -627,8 +676,8 @@ def _group_keys(self, filters: QueryFilters, field: str) -> list[str]: connection = self._connect() try: if field == "box": - conditions = ["1=1"] - parameters: list[Any] = [] + conditions = ["tenant_id = ?"] + parameters: list[Any] = [filters.tenant_id] if filters.instance_id: conditions.append("id = ?") parameters.append(filters.instance_id) @@ -641,33 +690,35 @@ def _group_keys(self, filters: QueryFilters, field: str) -> list[str]: parameters, ).fetchall() elif field == "model": - conditions = ["model IS NOT NULL"] - parameters = [] + conditions = ["s.model IS NOT NULL", "i.tenant_id = ?"] + parameters = [filters.tenant_id] if filters.provider: - conditions.append("provider = ?") + conditions.append("s.provider = ?") parameters.append(filters.provider) if filters.model: - conditions.append("model = ?") + conditions.append("s.model = ?") parameters.append(filters.model) if filters.instance_id: - conditions.append("instance_id = ?") + conditions.append("s.instance_id = ?") parameters.append(filters.instance_id) rows = connection.execute( - f"SELECT DISTINCT model AS value FROM metric_series " + f"SELECT DISTINCT s.model AS value FROM metric_series s " + f"JOIN devbox_instances i ON i.id = s.instance_id " f"WHERE {' AND '.join(conditions)} ORDER BY value LIMIT 200", parameters, ).fetchall() else: - conditions = ["1=1"] - parameters = [] + conditions = ["i.tenant_id = ?"] + parameters = [filters.tenant_id] if filters.repo: - conditions.append("repo_key = ?") + conditions.append("r.repo_key = ?") parameters.append(filters.repo) if filters.instance_id: - conditions.append("instance_id = ?") + conditions.append("r.instance_id = ?") parameters.append(filters.instance_id) rows = connection.execute( - f"SELECT DISTINCT repo_key AS value FROM repositories " + f"SELECT DISTINCT r.repo_key AS value FROM repositories r " + f"JOIN devbox_instances i ON i.id = r.instance_id " f"WHERE {' AND '.join(conditions)} ORDER BY value LIMIT 200", parameters, ).fetchall() @@ -678,8 +729,8 @@ def _group_keys(self, filters: QueryFilters, field: str) -> list[str]: def _latest_worktree( self, connection: sqlite3.Connection, filters: QueryFilters ) -> dict[str, int]: - conditions = ["1=1"] - parameters: list[Any] = [] + conditions = ["i.tenant_id = ?"] + parameters: list[Any] = [filters.tenant_id] if filters.box: conditions.append("i.name = ?") parameters.append(filters.box) @@ -860,8 +911,8 @@ def _activity_sync( def _collector_status_sync(self, filters: QueryFilters) -> list[dict[str, Any]]: connection = self._connect() try: - conditions = ["1=1"] - parameters: list[Any] = [] + conditions = ["i.tenant_id = ?"] + parameters: list[Any] = [filters.tenant_id] if filters.box: conditions.append("i.name = ?") parameters.append(filters.box) @@ -913,14 +964,18 @@ def _collector_status_sync(self, filters: QueryFilters) -> list[dict[str, Any]]: finally: connection.close() - def _purge_box_sync(self, box_name: str) -> int: + def _purge_box_sync(self, tenant_id: str, box_name: str) -> int: connection = self._connect() try: connection.execute("BEGIN IMMEDIATE") identifiers = connection.execute( - "SELECT id FROM devbox_instances WHERE name = ?", (box_name,) + "SELECT id FROM devbox_instances WHERE tenant_id = ? AND name = ?", + (tenant_id, box_name), ).fetchall() - connection.execute("DELETE FROM devbox_instances WHERE name = ?", (box_name,)) + connection.execute( + "DELETE FROM devbox_instances WHERE tenant_id = ? AND name = ?", + (tenant_id, box_name), + ) connection.commit() return len(identifiers) except Exception: @@ -929,12 +984,13 @@ def _purge_box_sync(self, box_name: str) -> int: finally: connection.close() - def _purge_instance_sync(self, instance_id: str) -> int: + def _purge_instance_sync(self, tenant_id: str, instance_id: str) -> int: connection = self._connect() try: connection.execute("BEGIN IMMEDIATE") deleted = connection.execute( - "DELETE FROM devbox_instances WHERE id = ?", (instance_id,) + "DELETE FROM devbox_instances WHERE tenant_id = ? AND id = ?", + (tenant_id, instance_id), ).rowcount connection.commit() return max(deleted, 0) @@ -1037,12 +1093,22 @@ def _maintain_sync(self) -> dict[str, int]: finally: connection.close() - def _backup_sync(self, destination: Path) -> None: + def _backup_sync(self, destination: Path, tenant_id: str | None) -> None: destination.parent.mkdir(parents=True, exist_ok=True) source = self._connect() target = sqlite3.connect(destination) try: source.backup(target) + if tenant_id is not None: + target.execute("PRAGMA foreign_keys=ON") + target.execute("PRAGMA secure_delete=ON") + target.execute("BEGIN IMMEDIATE") + target.execute( + "DELETE FROM devbox_instances WHERE tenant_id != ?", + (tenant_id,), + ) + target.commit() + target.execute("VACUUM") finally: target.close() source.close() @@ -1165,8 +1231,8 @@ def _normalize_series(rows: Iterable[sqlite3.Row], metric: str) -> list[dict[str def _metric_dimension_filter(filters: QueryFilters) -> tuple[str, list[Any]]: - conditions = ["1=1"] - parameters: list[Any] = [] + conditions = ["i.tenant_id = ?"] + parameters: list[Any] = [filters.tenant_id] if filters.box: conditions.append("i.name = ?") parameters.append(filters.box) @@ -1183,8 +1249,16 @@ def _metric_dimension_filter(filters: QueryFilters) -> tuple[str, list[Any]]: def _git_filter(filters: QueryFilters) -> tuple[str, list[Any]]: - conditions = ["c.committed_at >= ?", "c.committed_at <= ?"] - parameters: list[Any] = [filters.since.isoformat(), filters.until.isoformat()] + conditions = [ + "i.tenant_id = ?", + "c.committed_at >= ?", + "c.committed_at <= ?", + ] + parameters: list[Any] = [ + filters.tenant_id, + filters.since.isoformat(), + filters.until.isoformat(), + ] if filters.box: conditions.append("i.name = ?") parameters.append(filters.box) diff --git a/controller/src/devboxes_controller/manager.py b/controller/src/devboxes_controller/manager.py index 7ca56fd..daca5cb 100644 --- a/controller/src/devboxes_controller/manager.py +++ b/controller/src/devboxes_controller/manager.py @@ -17,7 +17,7 @@ from kubernetes.utils.quantity import parse_quantity from .auth import Authenticator -from .config import CustomImageProfile, GpuProfile, Settings +from .config import CustomImageProfile, GpuProfile, Settings, TenantConfig from .models import ( CreateDevboxRequest, CustomImageAllocation, @@ -32,6 +32,7 @@ from .resources import ( ANNOTATION_AUTO_STOPPED_AT, ANNOTATION_CREATED_AT, + ANNOTATION_CREATED_BY, ANNOTATION_CUSTOM_IMAGE_CONFIG, ANNOTATION_CUSTOM_IMAGE_PROFILE, ANNOTATION_EXPIRES_AT, @@ -48,6 +49,7 @@ ANNOTATION_TTL_HOURS, LABEL_MANAGED_BY, LABEL_NAME, + LABEL_TENANT, MANAGED_BY, PRESETS, build_deployment, @@ -67,6 +69,10 @@ class DevboxConflictError(Exception): """Signal that a requested devbox name is already active.""" +class TenantQuotaExceededError(Exception): + """Signal that Kubernetes rejected work against a tenant ResourceQuota.""" + + @dataclass class _LifecycleLock: """Retain one keyed lifecycle lock while callers are using or awaiting it.""" @@ -128,28 +134,48 @@ def _load_config(settings: Settings) -> None: logger.info("loaded kubeconfig context %s", settings.kubeconfig_context or "current") async def ready(self) -> bool: - """Return whether the controller can list namespaced Deployments.""" + """Return whether the controller can list Deployments in every tenant.""" try: - await asyncio.to_thread( - self.apps.list_namespaced_deployment, - self.settings.namespace, - limit=1, + await asyncio.gather( + *[ + asyncio.to_thread( + self.apps.list_namespaced_deployment, + tenant.namespace, + limit=1, + ) + for tenant in self.settings.tenant_configs + ] ) except ApiException: return False return True - async def create(self, request: CreateDevboxRequest) -> Devbox: + async def create( + self, + request: CreateDevboxRequest, + tenant: TenantConfig | None = None, + created_by: str | None = None, + ) -> Devbox: """Create compute, SSH, and persistent storage for a devbox.""" - if request.ttl_hours > self.settings.max_ttl_hours: - raise ValueError(f"ttl_hours cannot exceed {self.settings.max_ttl_hours}") + tenant = tenant or self.settings.resolve_tenant(None) + maximum_ttl = self.settings.tenant_max_ttl_hours(tenant) + if request.ttl_hours > maximum_ttl: + raise ValueError(f"ttl_hours cannot exceed {maximum_ttl} for tenant {tenant.id!r}") + if request.preset not in tenant.allowed_presets: + available = ", ".join(preset.value for preset in tenant.allowed_presets) + raise ValueError( + f"preset {request.preset.value!r} is unavailable to tenant {tenant.id!r}; " + f"available presets: {available}" + ) gpu_profile = ( - self.settings.resolve_gpu_profile(request.gpu.profile) + self.settings.resolve_gpu_profile(request.gpu.profile, tenant) if request.gpu is not None else None ) custom_image = ( - self.settings.resolve_custom_image(request.image) if request.image is not None else None + self.settings.resolve_custom_image(request.image, tenant) + if request.image is not None + else None ) if ( custom_image is not None @@ -163,8 +189,15 @@ async def create(self, request: CreateDevboxRequest) -> Devbox: ) name = resource_name(request.name) - async with self._lock_lifecycle(name): - return await self._create_locked(request, gpu_profile, custom_image, name) + async with self._lock_lifecycle(f"{tenant.id}/{name}"): + return await self._create_locked( + request, + gpu_profile, + custom_image, + name, + tenant, + created_by, + ) async def _create_locked( self, @@ -172,16 +205,19 @@ async def _create_locked( gpu_profile: GpuProfile | None, custom_image: CustomImageProfile | None, name: str, + tenant: TenantConfig, + created_by: str | None, ) -> Devbox: """Create a devbox while holding its lifecycle lock.""" - if await self._deployment_exists(name): + if await self._deployment_exists(name, tenant, include_unmanaged=True): raise DevboxConflictError(f"devbox {request.name!r} already exists") pvc_name = f"{name}-home" + include_tenant_label = self.settings.multi_tenancy_enabled desired_storage = PRESETS[request.preset]["storage"] - existing_pvc = await self._read_pvc(pvc_name) + existing_pvc = await self._read_pvc(pvc_name, tenant) instance_id = ( - await self._instance_id(pvc_name, existing_pvc) + await self._instance_id(pvc_name, existing_pvc, tenant) if self.settings.insights_enabled else None ) @@ -189,15 +225,26 @@ async def _create_locked( if existing_pvc is None: pvc = build_pvc( request, - self.settings.namespace, - self.settings.storage_class, + tenant.namespace, + self.settings.tenant_storage_class(tenant), instance_id, + tenant.id, + created_by, + include_tenant_label, ) - await asyncio.to_thread( - self.core.create_namespaced_persistent_volume_claim, - self.settings.namespace, - pvc, - ) + try: + await asyncio.to_thread( + self.core.create_namespaced_persistent_volume_claim, + tenant.namespace, + pvc, + ) + except ApiException as error: + self._raise_quota(error, tenant) + if error.status == 409: + raise DevboxConflictError( + f"persistent volume claim {pvc_name!r} already exists" + ) from error + raise else: current_storage = str( (existing_pvc.spec.resources.requests or {}).get("storage", desired_storage) @@ -206,14 +253,14 @@ async def _create_locked( await asyncio.to_thread( self.core.patch_namespaced_persistent_volume_claim, pvc_name, - self.settings.namespace, + tenant.namespace, {"spec": {"resources": {"requests": {"storage": desired_storage}}}}, ) else: storage_size = current_storage insights_credential = ( - self.authenticator.issue_insights_token(instance_id, request.name) + self.authenticator.issue_insights_token(instance_id, request.name, tenant.id) if instance_id is not None else None ) @@ -223,13 +270,14 @@ async def _create_locked( request.name, instance_id, insights_credential, + tenant, ) deployment = build_deployment( request, - self.settings.namespace, + tenant.namespace, self.settings.workspace_image, - self.settings.workspace_secret_name, - self.settings.workspace_service_account_name, + self.settings.tenant_workspace_secret_name(tenant), + self.settings.tenant_workspace_service_account_name(tenant), self.settings.workspace_priority_class, self.settings.image_pull_secret, gpu_profile=gpu_profile, @@ -243,61 +291,86 @@ async def _create_locked( insights_repository_depth=self.settings.insights_agent_repository_depth, insights_max_queue_bytes=self.settings.insights_agent_max_queue_bytes, insights_max_queue_age_seconds=self.settings.insights_agent_max_queue_age_seconds, + tenant_id=tenant.id, + created_by=created_by, + include_tenant_label=include_tenant_label, ) deployment["metadata"]["annotations"][ANNOTATION_STORAGE] = storage_size service = build_service( request, - self.settings.namespace, + tenant.namespace, self.settings.workspace_service_type, self.settings.workspace_service_annotations, None, self.settings.workspace_load_balancer_class, self.settings.workspace_external_traffic_policy, self.settings.workspace_load_balancer_source_ranges, + tenant.id, + created_by, + include_tenant_label, ) + deployment_created = False try: await asyncio.to_thread( self.apps.create_namespaced_deployment, - self.settings.namespace, + tenant.namespace, deployment, ) + deployment_created = True await asyncio.to_thread( self.core.create_namespaced_service, - self.settings.namespace, + tenant.namespace, service, ) + except ApiException as error: + if deployment_created: + await self._delete_deployment(name, tenant) + if insights_secret_name: + await self._delete_secret(insights_secret_name, tenant) + self._raise_quota(error, tenant) + if error.status == 409: + raise DevboxConflictError( + f"a Kubernetes resource for devbox {request.name!r} already exists" + ) from error + raise except Exception: - await self._delete_deployment(name) + if deployment_created: + await self._delete_deployment(name, tenant) if insights_secret_name: - await self._delete_secret(insights_secret_name) + await self._delete_secret(insights_secret_name, tenant) raise - return await self.get(request.name) + return await self.get(request.name, tenant) - async def list(self) -> list[Devbox]: + async def list(self, tenant: TenantConfig | None = None) -> list[Devbox]: """List managed devboxes in reverse creation order.""" + tenant = tenant or self.settings.resolve_tenant(None) selector = f"{LABEL_MANAGED_BY}={MANAGED_BY}" deployments, services, pods = await asyncio.gather( asyncio.to_thread( self.apps.list_namespaced_deployment, - self.settings.namespace, + tenant.namespace, label_selector=selector, ), asyncio.to_thread( self.core.list_namespaced_service, - self.settings.namespace, + tenant.namespace, label_selector=selector, ), asyncio.to_thread( self.core.list_namespaced_pod, - self.settings.namespace, + tenant.namespace, label_selector=selector, ), ) - service_by_name = {item.metadata.labels.get(LABEL_NAME): item for item in services.items} + service_by_name = { + item.metadata.labels.get(LABEL_NAME): item + for item in services.items + if self._belongs_to_tenant(item, tenant) + } pod_by_name: dict[str, Any] = {} for item in pods.items: box_name = item.metadata.labels.get(LABEL_NAME) - if box_name: + if box_name and self._belongs_to_tenant(item, tenant): pod_by_name[box_name] = item result = [ @@ -305,29 +378,32 @@ async def list(self) -> list[Devbox]: deployment, service_by_name.get(deployment.metadata.labels.get(LABEL_NAME)), pod_by_name.get(deployment.metadata.labels.get(LABEL_NAME)), + tenant, ) for deployment in deployments.items + if self._belongs_to_tenant(deployment, tenant) ] return sorted(result, key=lambda item: item.created_at, reverse=True) - async def get(self, name: str) -> Devbox: + async def get(self, name: str, tenant: TenantConfig | None = None) -> Devbox: """Return the current model for one managed devbox.""" + tenant = tenant or self.settings.resolve_tenant(None) resource = resource_name(name) try: deployment, service, pods = await asyncio.gather( asyncio.to_thread( self.apps.read_namespaced_deployment, resource, - self.settings.namespace, + tenant.namespace, ), asyncio.to_thread( self.core.read_namespaced_service, f"{resource}-ssh", - self.settings.namespace, + tenant.namespace, ), asyncio.to_thread( self.core.list_namespaced_pod, - self.settings.namespace, + tenant.namespace, label_selector=f"{LABEL_NAME}={name}", ), ) @@ -335,96 +411,142 @@ async def get(self, name: str) -> Devbox: if error.status == 404: raise DevboxNotFoundError(name) from error raise - pod = pods.items[0] if pods.items else None - return self._to_model(deployment, service, pod) + if not self._belongs_to_tenant(deployment, tenant): + raise DevboxNotFoundError(name) + service = service if self._belongs_to_tenant(service, tenant) else None + pod = next( + (item for item in pods.items if self._belongs_to_tenant(item, tenant)), + None, + ) + return self._to_model(deployment, service, pod, tenant) - async def scale(self, name: str, replicas: int) -> Devbox: + async def scale( + self, + name: str, + replicas: int, + tenant: TenantConfig | None = None, + ) -> Devbox: """Start or stop a devbox while preserving its home volume.""" + tenant = tenant or self.settings.resolve_tenant(None) resource = resource_name(name) - async with self._lock_lifecycle(resource): - return await self._scale_locked(name, resource, replicas) + async with self._lock_lifecycle(f"{tenant.id}/{resource}"): + return await self._scale_locked(name, resource, replicas, tenant) - async def _scale_locked(self, name: str, resource: str, replicas: int) -> Devbox: + async def _scale_locked( + self, + name: str, + resource: str, + replicas: int, + tenant: TenantConfig, + ) -> Devbox: """Scale a devbox while holding its lifecycle lock.""" try: + deployment = await asyncio.to_thread( + self.apps.read_namespaced_deployment, + resource, + tenant.namespace, + ) + if not self._belongs_to_tenant(deployment, tenant): + raise DevboxNotFoundError(name) if replicas == 1: - deployment = await asyncio.to_thread( - self.apps.read_namespaced_deployment, - resource, - self.settings.namespace, - ) - deployment = await self._prepare_insights_template(deployment) + deployment = await self._prepare_insights_template(deployment, tenant) annotations = deployment.metadata.annotations or {} ttl_hours = _ttl_hours( annotations.get(ANNOTATION_TTL_HOURS), self.settings.default_ttl_hours, - self.settings.max_ttl_hours, + self.settings.tenant_max_ttl_hours(tenant), ) + metadata_patch: dict[str, Any] = { + "annotations": { + ANNOTATION_AUTO_STOPPED_AT: None, + ANNOTATION_EXPIRES_AT: ( + datetime.now(UTC) + timedelta(hours=ttl_hours) + ).isoformat(), + } + } + resource_version = getattr(deployment.metadata, "resource_version", None) + if resource_version: + metadata_patch["resourceVersion"] = resource_version await asyncio.to_thread( self.apps.patch_namespaced_deployment, resource, - self.settings.namespace, + tenant.namespace, { - "metadata": { - "annotations": { - ANNOTATION_AUTO_STOPPED_AT: None, - ANNOTATION_EXPIRES_AT: ( - datetime.now(UTC) + timedelta(hours=ttl_hours) - ).isoformat(), - } - }, + "metadata": metadata_patch, "spec": {"replicas": 1}, }, ) else: + scale_patch: dict[str, Any] = {"spec": {"replicas": replicas}} + resource_version = getattr(deployment.metadata, "resource_version", None) + if resource_version: + scale_patch["metadata"] = {"resourceVersion": resource_version} await asyncio.to_thread( self.apps.patch_namespaced_deployment_scale, resource, - self.settings.namespace, - {"spec": {"replicas": replicas}}, + tenant.namespace, + scale_patch, ) except ApiException as error: if error.status == 404: raise DevboxNotFoundError(name) from error + self._raise_quota(error, tenant) raise - return await self.get(name) + return await self.get(name, tenant) async def reconcile_insights(self) -> builtins.list[str]: """Reconcile identity and safe collector state without restarting active workspaces.""" if not self.settings.insights_enabled: return [] + changed: builtins.list[str] = [] + for tenant in self.settings.tenant_configs: + changed.extend(await self._reconcile_tenant_insights(tenant)) + return changed + + async def _reconcile_tenant_insights( + self, + tenant: TenantConfig, + ) -> builtins.list[str]: + """Reconcile Insights inside one tenant namespace.""" selector = f"{LABEL_MANAGED_BY}={MANAGED_BY}" deployments = await asyncio.to_thread( self.apps.list_namespaced_deployment, - self.settings.namespace, + tenant.namespace, label_selector=selector, ) changed: builtins.list[str] = [] for listed_deployment in deployments.items: + if not self._belongs_to_tenant(listed_deployment, tenant): + continue name = listed_deployment.metadata.labels.get(LABEL_NAME) if not name: continue resource = resource_name(name) - async with self._lock_lifecycle(resource): + async with self._lock_lifecycle(f"{tenant.id}/{resource}"): try: deployment = await asyncio.to_thread( self.apps.read_namespaced_deployment, resource, - self.settings.namespace, + tenant.namespace, ) except ApiException as error: if error.status == 404: continue raise + if not self._belongs_to_tenant(deployment, tenant): + continue if _deletion_in_progress(deployment): continue if (deployment.spec.replicas or 0) == 0: - prepared = await self._prepare_insights_template(deployment) + prepared = await self._prepare_insights_template(deployment, tenant) if prepared is not deployment: - changed.append(name) + changed.append(self._reported_name(tenant, name)) else: annotations = deployment.metadata.annotations or {} - desired, instance_id = await self._desired_insights_deployment(deployment) + desired, instance_id = await self._desired_insights_deployment( + deployment, + tenant, + ) if not _insights_template_matches(deployment, desired) and ( annotations.get(ANNOTATION_INSIGHTS_STATE) != InsightsState.RESTART_REQUIRED.value @@ -433,7 +555,7 @@ async def reconcile_insights(self) -> builtins.list[str]: await asyncio.to_thread( self.apps.patch_namespaced_deployment, deployment.metadata.name, - self.settings.namespace, + tenant.namespace, { "metadata": { "annotations": { @@ -445,22 +567,28 @@ async def reconcile_insights(self) -> builtins.list[str]: } }, ) - changed.append(name) + changed.append(self._reported_name(tenant, name)) return changed - async def delete(self, name: str, purge: bool) -> DeleteResult: + async def delete( + self, + name: str, + purge: bool, + tenant: TenantConfig | None = None, + ) -> DeleteResult: """Delete compute and SSH resources, optionally deleting storage.""" + tenant = tenant or self.settings.resolve_tenant(None) resource = resource_name(name) - async with self._lock_lifecycle(resource): - if not await self._deployment_exists(resource): + async with self._lock_lifecycle(f"{tenant.id}/{resource}"): + if not await self._deployment_exists(resource, tenant): raise DevboxNotFoundError(name) await asyncio.gather( - self._delete_deployment(resource), - self._delete_service(f"{resource}-ssh"), - self._delete_secret(f"{resource}-insights"), + self._delete_deployment(resource, tenant), + self._delete_service(f"{resource}-ssh", tenant), + self._delete_secret(f"{resource}-insights", tenant), ) if purge: - await self._delete_pvc(f"{resource}-home") + await self._delete_pvc(f"{resource}-home", tenant) return DeleteResult( name=name, purged=purge, @@ -475,47 +603,94 @@ async def stop_expired(self) -> builtins.list[str]: """Stop every active devbox whose TTL has expired.""" now = datetime.now(UTC) stopped: builtins.list[str] = [] - for box in await self.list(): - if box.state is not DevboxState.STOPPED and box.expires_at <= now: - resource = resource_name(box.name) - await asyncio.to_thread( - self.apps.patch_namespaced_deployment, - resource, - self.settings.namespace, - { - "metadata": {"annotations": {ANNOTATION_AUTO_STOPPED_AT: now.isoformat()}}, - "spec": {"replicas": 0}, - }, - ) - stopped.append(box.name) + for tenant in self.settings.tenant_configs: + for box in await self.list(tenant): + if box.state is not DevboxState.STOPPED and box.expires_at <= now: + resource = resource_name(box.name) + async with self._lock_lifecycle(f"{tenant.id}/{resource}"): + try: + deployment = await asyncio.to_thread( + self.apps.read_namespaced_deployment, + resource, + tenant.namespace, + ) + except ApiException as error: + if error.status == 404: + continue + raise + if not self._belongs_to_tenant(deployment, tenant): + continue + annotations = deployment.metadata.annotations or {} + current_expiry = _parse_datetime( + annotations.get(ANNOTATION_EXPIRES_AT), + deployment.metadata.creation_timestamp, + ) + if (deployment.spec.replicas or 0) == 0 or current_expiry > now: + continue + metadata_patch: dict[str, Any] = { + "annotations": {ANNOTATION_AUTO_STOPPED_AT: now.isoformat()} + } + resource_version = getattr( + deployment.metadata, + "resource_version", + None, + ) + if resource_version: + metadata_patch["resourceVersion"] = resource_version + await asyncio.to_thread( + self.apps.patch_namespaced_deployment, + resource, + tenant.namespace, + { + "metadata": metadata_patch, + "spec": {"replicas": 0}, + }, + ) + stopped.append(self._reported_name(tenant, box.name)) return stopped - async def _deployment_exists(self, name: str) -> bool: + async def _deployment_exists( + self, + name: str, + tenant: TenantConfig, + *, + include_unmanaged: bool = False, + ) -> bool: try: - await asyncio.to_thread( + deployment = await asyncio.to_thread( self.apps.read_namespaced_deployment, name, - self.settings.namespace, + tenant.namespace, ) except ApiException as error: if error.status == 404: return False raise - return True + return include_unmanaged or self._belongs_to_tenant(deployment, tenant) - async def _read_pvc(self, name: str) -> Any | None: + async def _read_pvc(self, name: str, tenant: TenantConfig) -> Any | None: try: - return await asyncio.to_thread( + pvc = await asyncio.to_thread( self.core.read_namespaced_persistent_volume_claim, name, - self.settings.namespace, + tenant.namespace, ) except ApiException as error: if error.status == 404: return None raise + if not self._belongs_to_tenant(pvc, tenant): + raise DevboxConflictError( + f"persistent volume claim {name!r} is not owned by tenant {tenant.id!r}" + ) + return pvc - async def _instance_id(self, pvc_name: str, pvc: Any | None) -> str: + async def _instance_id( + self, + pvc_name: str, + pvc: Any | None, + tenant: TenantConfig, + ) -> str: if pvc is not None: annotations = getattr(getattr(pvc, "metadata", None), "annotations", None) or {} candidate = annotations.get(ANNOTATION_INSTANCE_ID) @@ -526,7 +701,7 @@ async def _instance_id(self, pvc_name: str, pvc: Any | None) -> str: await asyncio.to_thread( self.core.patch_namespaced_persistent_volume_claim, pvc_name, - self.settings.namespace, + tenant.namespace, {"metadata": {"annotations": {ANNOTATION_INSTANCE_ID: instance_id}}}, ) return instance_id @@ -536,6 +711,7 @@ async def _ensure_insights_secret( box_name: str, instance_id: str, credential: str, + tenant: TenantConfig, ) -> str: secret_name = f"{resource_name(box_name)}-insights" body = { @@ -543,10 +719,11 @@ async def _ensure_insights_secret( "kind": "Secret", "metadata": { "name": secret_name, - "namespace": self.settings.namespace, + "namespace": tenant.namespace, "labels": { LABEL_MANAGED_BY: MANAGED_BY, LABEL_NAME: box_name, + **({LABEL_TENANT: tenant.id} if self.settings.multi_tenancy_enabled else {}), }, "annotations": {ANNOTATION_INSTANCE_ID: instance_id}, }, @@ -557,17 +734,21 @@ async def _ensure_insights_secret( existing = await asyncio.to_thread( self.core.read_namespaced_secret, secret_name, - self.settings.namespace, + tenant.namespace, ) except ApiException as error: if error.status != 404: raise await asyncio.to_thread( self.core.create_namespaced_secret, - self.settings.namespace, + tenant.namespace, body, ) else: + if not self._belongs_to_tenant(existing, tenant): + raise DevboxConflictError( + f"Insights Secret {secret_name!r} is not owned by tenant {tenant.id!r}" + ) annotations = getattr(getattr(existing, "metadata", None), "annotations", None) or {} data = getattr(existing, "data", None) encoded = data.get("credential") if isinstance(data, dict) else None @@ -581,7 +762,7 @@ async def _ensure_insights_secret( await asyncio.to_thread( self.core.patch_namespaced_secret, secret_name, - self.settings.namespace, + tenant.namespace, { "metadata": {"annotations": {ANNOTATION_INSTANCE_ID: instance_id}}, "stringData": {"credential": credential}, @@ -589,10 +770,14 @@ async def _ensure_insights_secret( ) return secret_name - async def _prepare_insights_template(self, deployment: Any) -> Any: + async def _prepare_insights_template( + self, + deployment: Any, + tenant: TenantConfig, + ) -> Any: if not self.settings.insights_enabled: return deployment - desired, instance_id = await self._desired_insights_deployment(deployment) + desired, instance_id = await self._desired_insights_deployment(deployment, tenant) annotations = deployment.metadata.annotations or {} if _insights_template_matches(deployment, desired): if annotations.get(ANNOTATION_INSIGHTS_STATE) == InsightsState.COLLECTING.value: @@ -600,7 +785,7 @@ async def _prepare_insights_template(self, deployment: Any) -> Any: await asyncio.to_thread( self.apps.patch_namespaced_deployment, deployment.metadata.name, - self.settings.namespace, + tenant.namespace, { "metadata": { "annotations": { @@ -613,12 +798,12 @@ async def _prepare_insights_template(self, deployment: Any) -> Any: return await asyncio.to_thread( self.apps.read_namespaced_deployment, deployment.metadata.name, - self.settings.namespace, + tenant.namespace, ) await asyncio.to_thread( self.apps.patch_namespaced_deployment, deployment.metadata.name, - self.settings.namespace, + tenant.namespace, { "metadata": { "annotations": { @@ -635,36 +820,45 @@ async def _prepare_insights_template(self, deployment: Any) -> Any: return await asyncio.to_thread( self.apps.read_namespaced_deployment, deployment.metadata.name, - self.settings.namespace, + tenant.namespace, ) - async def _desired_insights_deployment(self, deployment: Any) -> tuple[dict[str, Any], str]: + async def _desired_insights_deployment( + self, + deployment: Any, + tenant: TenantConfig, + ) -> tuple[dict[str, Any], str]: annotations = deployment.metadata.annotations or {} name = deployment.metadata.labels[LABEL_NAME] pvc_name = f"{resource_name(name)}-home" - pvc = await self._read_pvc(pvc_name) - instance_id = await self._instance_id(pvc_name, pvc) + pvc = await self._read_pvc(pvc_name, tenant) + instance_id = await self._instance_id(pvc_name, pvc, tenant) request = CreateDevboxRequest( name=name, preset=Preset(annotations.get(ANNOTATION_PRESET, Preset.SMALL.value)), ttl_hours=_ttl_hours( annotations.get(ANNOTATION_TTL_HOURS), self.settings.default_ttl_hours, - self.settings.max_ttl_hours, + self.settings.tenant_max_ttl_hours(tenant), ), repository=annotations.get(ANNOTATION_REPOSITORY), image=annotations.get(ANNOTATION_CUSTOM_IMAGE_PROFILE), ) gpu_profile = _resolved_gpu_profile(annotations, self.settings) custom_image = _resolved_custom_image(annotations, self.settings) - credential = self.authenticator.issue_insights_token(instance_id, name) - secret_name = await self._ensure_insights_secret(name, instance_id, credential) + credential = self.authenticator.issue_insights_token(instance_id, name, tenant.id) + secret_name = await self._ensure_insights_secret( + name, + instance_id, + credential, + tenant, + ) desired = build_deployment( request, - self.settings.namespace, + tenant.namespace, self.settings.workspace_image, - self.settings.workspace_secret_name, - self.settings.workspace_service_account_name, + self.settings.tenant_workspace_secret_name(tenant), + self.settings.tenant_workspace_service_account_name(tenant), self.settings.workspace_priority_class, self.settings.image_pull_secret, gpu_profile=gpu_profile, @@ -678,38 +872,41 @@ async def _desired_insights_deployment(self, deployment: Any) -> tuple[dict[str, insights_repository_depth=self.settings.insights_agent_repository_depth, insights_max_queue_bytes=self.settings.insights_agent_max_queue_bytes, insights_max_queue_age_seconds=self.settings.insights_agent_max_queue_age_seconds, + tenant_id=tenant.id, + created_by=annotations.get(ANNOTATION_CREATED_BY), + include_tenant_label=self.settings.multi_tenancy_enabled, ) return desired, instance_id - async def _delete_deployment(self, name: str) -> None: + async def _delete_deployment(self, name: str, tenant: TenantConfig) -> None: await self._ignore_not_found( self.apps.delete_namespaced_deployment, name, - self.settings.namespace, + tenant.namespace, body=client.V1DeleteOptions(propagation_policy="Foreground"), ) - async def _delete_service(self, name: str) -> None: + async def _delete_service(self, name: str, tenant: TenantConfig) -> None: await self._ignore_not_found( self.core.delete_namespaced_service, name, - self.settings.namespace, + tenant.namespace, body=client.V1DeleteOptions(), ) - async def _delete_pvc(self, name: str) -> None: + async def _delete_pvc(self, name: str, tenant: TenantConfig) -> None: await self._ignore_not_found( self.core.delete_namespaced_persistent_volume_claim, name, - self.settings.namespace, + tenant.namespace, body=client.V1DeleteOptions(), ) - async def _delete_secret(self, name: str) -> None: + async def _delete_secret(self, name: str, tenant: TenantConfig) -> None: await self._ignore_not_found( self.core.delete_namespaced_secret, name, - self.settings.namespace, + tenant.namespace, body=client.V1DeleteOptions(), ) @@ -721,7 +918,43 @@ async def _ignore_not_found(function: Any, *args: Any, **kwargs: Any) -> None: if error.status != 404: raise - def _to_model(self, deployment: Any, service: Any | None, pod: Any | None) -> Devbox: + def _belongs_to_tenant(self, resource: Any, tenant: TenantConfig) -> bool: + """Enforce tenant ownership labels while admitting the legacy default namespace.""" + raw_labels = getattr(getattr(resource, "metadata", None), "labels", None) + labels = raw_labels if isinstance(raw_labels, dict) else {} + if labels.get(LABEL_MANAGED_BY) != MANAGED_BY: + return False + labeled_tenant = labels.get(LABEL_TENANT) + if labeled_tenant is not None: + return bool(labeled_tenant == tenant.id) + return ( + tenant.id == self.settings.default_tenant + and tenant.namespace == self.settings.namespace + ) + + def _reported_name(self, tenant: TenantConfig, name: str) -> str: + """Keep single-tenant maintenance output backward compatible.""" + return f"{tenant.id}/{name}" if self.settings.multi_tenancy_enabled else name + + @staticmethod + def _raise_quota(error: ApiException, tenant: TenantConfig) -> None: + """Translate Kubernetes quota admission into a stable tenant-facing error.""" + detail = " ".join( + value for value in [str(error.reason or ""), str(error.body or "")] if value + ) + if error.status == 403 and "quota" in detail.lower(): + raise TenantQuotaExceededError( + f"tenant {tenant.id!r} exceeded its Kubernetes ResourceQuota" + ) from error + + def _to_model( + self, + deployment: Any, + service: Any | None, + pod: Any | None, + tenant: TenantConfig | None = None, + ) -> Devbox: + tenant = tenant or self.settings.resolve_tenant(None) annotations = deployment.metadata.annotations or {} box_name = deployment.metadata.labels[LABEL_NAME] created_at = _parse_datetime( @@ -732,10 +965,12 @@ def _to_model(self, deployment: Any, service: Any | None, pod: Any | None) -> De desired = deployment.spec.replicas or 0 pod_ready = _pod_ready(pod) host, port = _service_endpoint(service, self.settings.workspace_service_host) - state, message = _state(desired, pod, pod_ready, host) + state, message = _state(desired, pod, pod_ready, host, deployment) restarts = sum(status.restart_count or 0 for status in _container_statuses(pod)) return Devbox( name=box_name, + tenant=tenant.id, + created_by=annotations.get(ANNOTATION_CREATED_BY), state=state, preset=preset, created_at=created_at, @@ -822,11 +1057,36 @@ def _state( pod: Any | None, pod_ready: bool, host: str | None, + deployment: Any | None = None, ) -> tuple[DevboxState, str | None]: if desired == 0: return DevboxState.STOPPED, "Compute stopped; home volume retained" if pod_ready and host: return DevboxState.READY, None + replica_failure = next( + ( + condition + for condition in ( + getattr(getattr(deployment, "status", None), "conditions", None) or [] + ) + if getattr(condition, "type", None) == "ReplicaFailure" + and getattr(condition, "status", None) == "True" + ), + None, + ) + if replica_failure is not None: + detail = str(getattr(replica_failure, "message", "") or "") + if "quota" in detail.lower(): + return ( + DevboxState.DEGRADED, + f"Tenant quota blocked this workspace: {detail[:280]}", + ) + return ( + DevboxState.DEGRADED, + f"Workspace replica creation failed: {detail[:280]}" + if detail + else "Workspace replica creation failed", + ) if pod is not None: phase = getattr(pod.status, "phase", None) waiting_reasons = { @@ -853,9 +1113,12 @@ def _state( None, ) if scheduling_condition is not None: - detail = getattr(scheduling_condition, "message", None) - if detail: - return DevboxState.STARTING, f"Scheduling blocked: {str(detail)[:320]}" + scheduling_detail = getattr(scheduling_condition, "message", None) + if scheduling_detail: + return ( + DevboxState.STARTING, + f"Scheduling blocked: {str(scheduling_detail)[:320]}", + ) return DevboxState.STARTING, "Waiting for a node with the requested resources" if pod_ready: return DevboxState.STARTING, "Waiting for an SSH service address" diff --git a/controller/src/devboxes_controller/models.py b/controller/src/devboxes_controller/models.py index 1d78f12..a237164 100644 --- a/controller/src/devboxes_controller/models.py +++ b/controller/src/devboxes_controller/models.py @@ -137,6 +137,8 @@ class Devbox(BaseModel): """Represent the observable state of one managed devbox.""" name: str + tenant: str = "default" + created_by: str | None = None state: DevboxState preset: Preset created_at: datetime @@ -162,11 +164,34 @@ class DevboxList(BaseModel): items: list[Devbox] +class TenantSummary(BaseModel): + """Expose one tenant context available to the current identity.""" + + id: str + display_name: str + role: str + + class WhoAmI(BaseModel): """Describe the authenticated controller identity and mechanism.""" user: str + subject: str | None = None mode: str + tenant: str = "default" + role: str = "operator" + tenants: list[TenantSummary] = Field(default_factory=list) + + +class TenancyCapabilities(BaseModel): + """Describe the active tenant and its operator-owned policy.""" + + enabled: bool + current: TenantSummary + tenants: list[TenantSummary] + max_ttl_hours: int + allowed_presets: list[Preset] + resource_quota: dict[str, str] = Field(default_factory=dict) class GpuProfileSummary(BaseModel): @@ -210,6 +235,7 @@ class Capabilities(BaseModel): gpu: GpuCapabilities images: CustomImageCapabilities + tenancy: TenancyCapabilities | None = None class CliTokenRequest(BaseModel): diff --git a/controller/src/devboxes_controller/resources.py b/controller/src/devboxes_controller/resources.py index 1462af0..ebd40b0 100644 --- a/controller/src/devboxes_controller/resources.py +++ b/controller/src/devboxes_controller/resources.py @@ -11,7 +11,9 @@ MANAGED_BY = "devboxes-controller" LABEL_MANAGED_BY = "app.kubernetes.io/managed-by" LABEL_NAME = "devboxes.bonalab.org/name" +LABEL_TENANT = "devboxes.bonalab.org/tenant" ANNOTATION_CREATED_AT = "devboxes.bonalab.org/created-at" +ANNOTATION_CREATED_BY = "devboxes.bonalab.org/created-by" ANNOTATION_EXPIRES_AT = "devboxes.bonalab.org/expires-at" ANNOTATION_TTL_HOURS = "devboxes.bonalab.org/ttl-hours" ANNOTATION_REPOSITORY = "devboxes.bonalab.org/repository" @@ -56,15 +58,18 @@ def resource_name(name: str) -> str: return f"devbox-{name}" -def labels(name: str) -> dict[str, str]: - """Return ownership and lookup labels for a managed devbox.""" - return { +def labels(name: str, tenant_id: str | None = None) -> dict[str, str]: + """Return ownership labels, adding tenant identity only in multi-tenant mode.""" + result = { "app.kubernetes.io/name": "devbox", "app.kubernetes.io/instance": resource_name(name), "app.kubernetes.io/part-of": "devboxes", LABEL_MANAGED_BY: MANAGED_BY, LABEL_NAME: name, } + if tenant_id is not None: + result[LABEL_TENANT] = tenant_id + return result def annotations( @@ -73,6 +78,7 @@ def annotations( instance_id: str | None = None, gpu_profile: GpuProfile | None = None, custom_image: CustomImageProfile | None = None, + created_by: str | None = None, ) -> dict[str, str]: """Return lifecycle and user-input annotations for a new devbox.""" now = now or datetime.now(UTC) @@ -85,6 +91,8 @@ def annotations( } if request.repository: result[ANNOTATION_REPOSITORY] = request.repository + if created_by: + result[ANNOTATION_CREATED_BY] = created_by if instance_id: result[ANNOTATION_INSTANCE_ID] = instance_id if gpu_profile is not None: @@ -119,6 +127,9 @@ def build_pvc( namespace: str, storage_class: str | None, instance_id: str | None = None, + tenant_id: str = "default", + created_by: str | None = None, + include_tenant_label: bool = False, ) -> dict[str, Any]: """Build the persistent home volume claim for a devbox.""" manifest: dict[str, Any] = { @@ -127,9 +138,15 @@ def build_pvc( "metadata": { "name": f"{resource_name(request.name)}-home", "namespace": namespace, - "labels": labels(request.name), + "labels": labels( + request.name, + tenant_id if include_tenant_label else None, + ), "annotations": ( - {ANNOTATION_INSTANCE_ID: instance_id} if instance_id is not None else {} + { + **({ANNOTATION_INSTANCE_ID: instance_id} if instance_id is not None else {}), + **({ANNOTATION_CREATED_BY: created_by} if created_by else {}), + } ), }, "spec": { @@ -162,10 +179,16 @@ def build_deployment( insights_repository_depth: int = 4, insights_max_queue_bytes: int = 134_217_728, insights_max_queue_age_seconds: int = 604_800, + tenant_id: str = "default", + created_by: str | None = None, + include_tenant_label: bool = False, ) -> dict[str, Any]: """Build the disposable workspace Deployment for a devbox.""" name = resource_name(request.name) - box_labels = labels(request.name) + box_labels = labels( + request.name, + tenant_id if include_tenant_label else None, + ) effective_workspace_image = workspace_image effective_workspace_pull_policy = "IfNotPresent" if gpu_profile is not None and gpu_profile.workspace_image is not None: @@ -176,6 +199,7 @@ def build_deployment( env = [ {"name": "DEVBOX_NAME", "value": request.name}, {"name": "DEVBOX_PRESET", "value": request.preset.value}, + {"name": "DEVBOX_TENANT", "value": tenant_id}, ] if request.repository: env.append({"name": "DEVBOX_REPOSITORY", "value": request.repository}) @@ -427,6 +451,7 @@ def build_deployment( instance_id, gpu_profile, custom_image, + created_by, ) deployment_annotations[ANNOTATION_INSIGHTS_STATE] = ( "collecting" if insights_enabled else "disabled" @@ -458,7 +483,12 @@ def build_deployment( # cold node or slower registry path. "progressDeadlineSeconds": 1800, "strategy": {"type": "Recreate"}, - "selector": {"matchLabels": {LABEL_NAME: request.name}}, + "selector": { + "matchLabels": { + LABEL_NAME: request.name, + **({LABEL_TENANT: tenant_id} if include_tenant_label else {}), + } + }, "template": { "metadata": { "labels": box_labels, @@ -497,6 +527,9 @@ def build_service( load_balancer_class: str | None = None, external_traffic_policy: str = "Cluster", load_balancer_source_ranges: list[str] | None = None, + tenant_id: str = "default", + created_by: str | None = None, + include_tenant_label: bool = False, ) -> dict[str, Any]: """Build the externally reachable SSH Service for a devbox.""" port: dict[str, Any] = { @@ -508,10 +541,13 @@ def build_service( if node_port is not None: port["nodePort"] = node_port + selector = {LABEL_NAME: request.name} + if include_tenant_label: + selector[LABEL_TENANT] = tenant_id spec: dict[str, Any] = { "type": service_type, "externalTrafficPolicy": external_traffic_policy, - "selector": {LABEL_NAME: request.name}, + "selector": selector, "ports": [port], } if load_balancer_class: @@ -525,8 +561,14 @@ def build_service( "metadata": { "name": f"{resource_name(request.name)}-ssh", "namespace": namespace, - "labels": labels(request.name), - "annotations": annotations or {}, + "labels": labels( + request.name, + tenant_id if include_tenant_label else None, + ), + "annotations": { + **(annotations or {}), + **({ANNOTATION_CREATED_BY: created_by} if created_by else {}), + }, }, "spec": spec, } diff --git a/controller/src/devboxes_controller/static/app.js b/controller/src/devboxes_controller/static/app.js index 753addc..a226c5b 100644 --- a/controller/src/devboxes_controller/static/app.js +++ b/controller/src/devboxes_controller/static/app.js @@ -2,6 +2,9 @@ const state = { boxes: [], deleteName: null, deleteTrigger: null, + tenant: document.body.dataset.tenant || "default", + canManage: document.body.dataset.canManage === "true", + canPurge: document.body.dataset.canPurge === "true", }; const elements = { @@ -37,6 +40,7 @@ async function api(path, options = {}) { const method = options.method || "GET"; const headers = new Headers(options.headers || {}); headers.set("Accept", "application/json"); + headers.set("X-Devboxes-Tenant", state.tenant); if (options.body) { headers.set("Content-Type", "application/json"); } @@ -173,12 +177,14 @@ function renderBox(box) { if (box.ssh_command && box.state !== "stopped") { actions.append(actionButton("Copy SSH", "copy", box)); } - if (box.state === "stopped") { - actions.append(actionButton("Start", "start", box)); - } else { - actions.append(actionButton("Stop", "stop", box)); + if (state.canManage) { + if (box.state === "stopped") { + actions.append(actionButton("Start", "start", box)); + } else { + actions.append(actionButton("Stop", "stop", box)); + } + actions.append(actionButton("Delete", "delete", box, true)); } - actions.append(actionButton("Delete", "delete", box, true)); actionsCell.append(actions); row.append(nameCell, stateCell, workspaceCell, expiryCell, actionsCell); @@ -275,6 +281,9 @@ function toast(message, error = false) { elements.createForm.addEventListener("submit", async (event) => { event.preventDefault(); + if (!state.canManage) { + return; + } elements.createError.hidden = true; const submit = elements.createForm.querySelector("button[type='submit']"); const form = new FormData(elements.createForm); @@ -364,6 +373,10 @@ elements.confirmDelete.addEventListener("click", async (event) => { } const name = state.deleteName; const purge = elements.purgeVolume.checked; + if (purge && !state.canPurge) { + toast("Admin access is required to purge persistent data.", true); + return; + } elements.confirmDelete.disabled = true; try { const result = await api( @@ -411,5 +424,10 @@ document.addEventListener("click", (event) => { }); updateImageHelp(); +if (!state.canManage) { + for (const control of elements.createForm.elements) { + control.disabled = true; + } +} loadBoxes(); window.setInterval(() => loadBoxes({ quiet: true }), 8_000); diff --git a/controller/src/devboxes_controller/static/insights.js b/controller/src/devboxes_controller/static/insights.js index a0b292f..fb40185 100644 --- a/controller/src/devboxes_controller/static/insights.js +++ b/controller/src/devboxes_controller/static/insights.js @@ -1,4 +1,5 @@ const enabled = document.body.dataset.insightsEnabled === "true"; +const tenant = document.body.dataset.tenant || "default"; const logoutButton = document.querySelector("#logout-button"); const refreshButton = document.querySelector("#insights-refresh"); @@ -14,6 +15,7 @@ async function api(path, options = {}) { const method = options.method || "GET"; const headers = new Headers(options.headers || {}); headers.set("Accept", "application/json"); + headers.set("X-Devboxes-Tenant", tenant); if (!["GET", "HEAD", "OPTIONS"].includes(method)) { headers.set( "X-Devboxes-CSRF", diff --git a/controller/src/devboxes_controller/static/styles.css b/controller/src/devboxes_controller/static/styles.css index e220bcf..d3ed20c 100644 --- a/controller/src/devboxes_controller/static/styles.css +++ b/controller/src/devboxes_controller/static/styles.css @@ -240,6 +240,25 @@ h3 { gap: var(--space-2); } +.tenant-switcher { + display: inline-flex; + align-items: center; + gap: var(--space-2); + color: var(--muted); + font-size: 0.74rem; + font-weight: 700; +} + +.tenant-switcher select { + width: min(14rem, 24vw); + min-height: 2.2rem; + padding: 0.42rem 1.9rem 0.42rem 0.62rem; + border-color: var(--line); + background: var(--surface); + font-size: 0.78rem; + font-weight: 650; +} + .cluster-state { display: inline-flex; align-items: center; @@ -355,6 +374,20 @@ h3 { font-weight: 700; } +.role-notice { + margin: calc(-1 * var(--space-2)) 0 var(--space-4); + padding: var(--space-3) var(--space-4); + border: 1px solid oklch(0.79 0.075 76); + border-radius: var(--radius-md); + background: var(--warning-soft); + color: var(--muted); + font-size: 0.86rem; +} + +.role-notice strong { + color: var(--ink); +} + .create-form { display: grid; grid-template-columns: minmax(9rem, 1fr) minmax(9rem, 0.8fr) minmax(9rem, 0.8fr) minmax(11rem, 1fr) minmax(14rem, 1.4fr) auto; @@ -1577,6 +1610,14 @@ kbd { display: none; } + .tenant-switcher > span { + display: none; + } + + .tenant-switcher select { + width: 10rem; + } + .app-main { padding: var(--space-6) var(--space-4) var(--space-7); } @@ -1682,7 +1723,9 @@ kbd { } .app-header { + flex-wrap: wrap; gap: var(--space-2); + padding-block: var(--space-2); } .header-leading { @@ -1697,6 +1740,10 @@ kbd { padding-inline: 0.5rem; } + .tenant-switcher select { + width: 8.5rem; + } + .create-workbench { padding: var(--space-5) var(--space-4); } diff --git a/controller/src/devboxes_controller/static/tenant.js b/controller/src/devboxes_controller/static/tenant.js new file mode 100644 index 0000000..a410e83 --- /dev/null +++ b/controller/src/devboxes_controller/static/tenant.js @@ -0,0 +1,7 @@ +const tenantSwitcher = document.querySelector("#tenant-switcher"); + +tenantSwitcher?.addEventListener("change", () => { + const destination = new URL(window.location.href); + destination.searchParams.set("tenant", tenantSwitcher.value); + window.location.assign(destination); +}); diff --git a/controller/src/devboxes_controller/templates/_tenant_switcher.html b/controller/src/devboxes_controller/templates/_tenant_switcher.html new file mode 100644 index 0000000..f981437 --- /dev/null +++ b/controller/src/devboxes_controller/templates/_tenant_switcher.html @@ -0,0 +1,12 @@ +{% if tenancy.enabled %} + +{% endif %} diff --git a/controller/src/devboxes_controller/templates/cli_authorize.html b/controller/src/devboxes_controller/templates/cli_authorize.html index 1487f9c..9380db9 100644 --- a/controller/src/devboxes_controller/templates/cli_authorize.html +++ b/controller/src/devboxes_controller/templates/cli_authorize.html @@ -25,7 +25,7 @@

Allow the Devbox CLI to access this installation? A local Devbox CLI is waiting for your decision. Approval creates a scoped, expiring token for {{ display_name }}; it does not reveal or copy - the operator access token. + the submitted access token.

@@ -36,7 +36,7 @@

Allow the Devbox CLI to access this installation?
Access
-
Create and manage devboxes as the current operator
+
Operate within the current identity’s authorized tenant roles
Return address
diff --git a/controller/src/devboxes_controller/templates/docs.html b/controller/src/devboxes_controller/templates/docs.html index b8edaa0..dfaf1d1 100644 --- a/controller/src/devboxes_controller/templates/docs.html +++ b/controller/src/devboxes_controller/templates/docs.html @@ -7,23 +7,26 @@ Documentation · Devboxes + - + + {% set tenant_query = '?tenant=' ~ tenancy.current.id if tenancy.enabled else '' %}
- + devboxes
+ {% include "_tenant_switcher.html" %} {{ cluster_name }} connected
@@ -35,6 +38,7 @@

Using Devboxes

@@ -85,8 +92,8 @@

Quick start

curl -fsSLO https://raw.githubusercontent.com/vicotrbb/devboxes/main/scripts/install-devbox-cli.sh
 sh install-devbox-cli.sh
-devbox login --url {{ external_url }}
-devbox create atlas --preset medium --ttl {{ default_ttl_hours }} --repo owner/your-project --ssh
+devbox login --url {{ external_url }}{% if tenancy.enabled %} --tenant {{ tenancy.current.id }}{% endif %} +devbox create atlas --preset {{ default_preset }} --ttl {{ default_ttl_hours }} --repo owner/your-project --ssh
    @@ -114,6 +121,51 @@

    Quick start

    +
    +

    Know your tenant boundary

    + {% if tenancy.enabled %} +

    + You are viewing {{ tenancy.current.display_name }} + ({{ tenancy.current.id }}) with the + {{ tenancy.current.role }} role. Its workspaces, home volumes, + SSH credentials, quotas, and Insights history are isolated in namespace + {{ namespace }}. +

    +
    +
    + Inspect or change CLI tenant context + +
    + 
    devbox tenant list
+devbox tenant use {{ tenancy.current.id }}
+devbox --tenant {{ tenancy.current.id }} list
    +
    +
      +
    • viewer can inspect workspaces and metrics.
      • +
    • member can create, start, stop, and delete compute while retaining volumes.
      • +
    • admin can also purge home volumes and central Insights history.
      • +
    • The installation operator can access every configured tenant for recovery.
      • +
    • These roles govern the Devboxes API. Shell access is separately controlled by the tenant workspace Secret’s SSH keys.
      • +
    + {% if tenancy.resource_quota %} +
    + Tenant quota +

    + {% for key, value in tenancy.resource_quota.items() %} + {{ key }}={{ value }}{% if not loop.last %} · {% endif %} + {% endfor %} +

    +
    + {% endif %} + {% else %} +

    + Multi-tenancy is disabled, so this installation retains its backward-compatible + single-operator namespace. Enable the Helm tenancy contract before granting access to + mutually untrusted users or teams. +

    + {% endif %} +
    +

    Know exactly what persists

    @@ -185,11 +237,11 @@

    Create the right box

    # Empty box, connect when ready
 devbox create scratch --ssh
 
-# Clone a GitHub repository and use a larger workspace
-devbox create compiler --preset large --ttl 72 --repo owner/compiler --ssh
+# Clone a GitHub repository and use the largest available workspace
+devbox create compiler --preset {{ largest_preset }} --ttl {{ example_ttl_hours }} --repo owner/compiler --ssh
 
 # Provision asynchronously for later
-devbox create nightly --preset small --no-wait
    +devbox create nightly --preset {{ smallest_preset }} --no-wait
    @@ -218,9 +270,15 @@

    Create the right box

      + {% if "small" in allowed_presets %}
    • small is comfortable for scripts, reviews, and lightweight services.
      • + {% endif %} + {% if "medium" in allowed_presets %}
    • medium is the everyday default for compilers, agents, and local app stacks.
      • + {% endif %} + {% if "large" in allowed_presets %}
    • large is for memory-heavy builds, databases, or parallel tooling.
      • + {% endif %}
    • Reusing a retained volume can expand it for a larger preset; volumes are never shrunk.
    @@ -230,7 +288,8 @@

    Use an operator-approved GPU

    {% if gpu.enabled %}

    This installation exposes {{ gpu.profiles | length }} GPU profile{% if gpu.profiles | length != 1 %}s{% endif %}. - CPU-only remains the default. In the CLI, --gpu selects the operator's default and + CPU-only remains the default. In the CLI, + {% if gpu.default_profile %}--gpu selects this tenant's default and {% endif %} --gpu-profile selects an exact profile. The workbench offers the same catalog.

    @@ -263,8 +322,8 @@

    Use an operator-approved GPU

    devbox gpu profiles
-devbox create inference --gpu --ssh
-devbox create training --gpu-profile {{ gpu.profiles[0].name }} --preset large --ssh
    +{% if gpu.default_profile %}devbox create inference --gpu --ssh +{% endif %}devbox create training --gpu-profile {{ gpu.profiles[0].name }} --preset {{ largest_preset }} --ssh

    @@ -442,6 +501,7 @@

    Use Devboxes from scripts

    create, list, status, start, and stop support global --json. Use DEVBOX_TOKEN for non-interactive authentication so a token never appears in command history. + {% if tenancy.enabled %}Set DEVBOX_TENANT to make the authorization boundary explicit.{% endif %}

    @@ -451,12 +511,15 @@

    Use Devboxes from scripts

    DEVBOX_URL="{{ external_url }}" \
 DEVBOX_TOKEN="$TOKEN_FROM_YOUR_SECRET_STORE" \
+{% if tenancy.enabled %}DEVBOX_TENANT="{{ tenancy.current.id }}" \
+{% endif %}
   devbox --json list | jq -r '.[] | select(.state == "ready") | .name'
    DEVBOX_TOKEN
    Overrides the saved access token.
    DEVBOX_URL
    Overrides the saved API URL.
    + {% if tenancy.enabled %}
    DEVBOX_TENANT
    Selects one authorized tenant without changing saved context.
    {% endif %}
    DEVBOX_CONFIG
    Uses a separate config file for an isolated profile or test.
    --url / --token
    Highest-precedence one-command overrides; prefer the token environment variable for secrecy.
    @@ -485,7 +548,7 @@

    Understand work with Insights

    computes a productivity score or an AI-written percentage, and it does not collect prompts, responses, commands, paths, file contents, Git authors, or commit messages.

    - Open Insights + Open Insights {% else %}

    Insights is disabled for this installation. An operator can opt in through the Helm @@ -507,9 +570,9 @@

    Use the browser workbench

  1. When enabled, the create form presents reviewed image profiles rather than accepting a raw container reference.
  2. State is written as text, ready, starting, stopped, or degraded, rather than conveyed by color alone.
  3. Deleting opens a confirmation that distinguishes retained storage from permanent purge.
    4. -
  4. The same access token creates a signed browser session; the raw token is not kept in the session cookie.
    5. +
  5. The submitted access token creates a signed browser session; the raw token is not kept in the session cookie.
    6. - Open the workbench + Open the workbench
    @@ -524,7 +587,9 @@

    Command reference

    - devbox loginVerifies and saves API access.--url, --token + devbox loginVerifies and saves API access.--url, --token, --tenant + {% if tenancy.enabled %}devbox tenantLists memberships or saves a verified tenant context.list, use{% endif %} + {% if tenancy.enabled %}devbox --tenant TENANT COMMANDRuns one command inside an authorized tenant boundary.--json{% endif %} devbox create NAMECreates a prepared box and waits for readiness.--preset, --ttl, --repo, --image, --ssh, --no-wait devbox image profilesLists the operator-approved image catalog.--json devbox listLists every active or stopped box.--json @@ -544,7 +609,7 @@

    Troubleshooting

    The CLI says authentication is required -

    Run devbox login --url {{ external_url }} and enter the access token created during installation. For a script, confirm DEVBOX_TOKEN is present and non-empty.

    +

    Run devbox login --url {{ external_url }} and enter the access token issued by your cluster operator. For a script, confirm DEVBOX_TOKEN is present and non-empty.

    The TLS certificate is not trusted @@ -560,11 +625,11 @@

    Troubleshooting

    The repository did not clone -

    Connect to the empty box, run gh auth status, then retry with gh repo clone OWNER/REPOSITORY ~/workspace/project. Confirm the shared GitHub token can read the repository.

    +

    Connect to the empty box, run gh auth status, then retry with gh repo clone OWNER/REPOSITORY ~/workspace/project. Confirm the active tenant workspace credential can read the repository.

    SSH reports a changed host key after an intentional purge -

    A purged home creates a new host identity. Copy the install-scoped alias from the SSH warning, remove only that entry with ssh-keygen -R ALIAS, then reconnect and verify the new key.

    +

    A purged home creates a new host identity. Copy the installation-, tenant-, and box-scoped alias from the SSH warning, remove only that entry with ssh-keygen -R ALIAS, then reconnect and verify the new key.

    Codex or Claude asks you to sign in again @@ -578,9 +643,9 @@

    Operator recovery

    Cluster diagnostics - 
    helm status devboxes -n {{ namespace }}
+            
    helm status devboxes -n {{ controller_namespace }}
 kubectl get deployment,service,pvc -n {{ namespace }}
-kubectl logs -n {{ namespace }} deployment/devboxes --tail=100
+kubectl logs -n {{ controller_namespace }} deployment/devboxes --tail=100
 kubectl get events -n {{ namespace }} --sort-by=.lastTimestamp
    @@ -590,7 +655,7 @@

    Operator recovery

    The shortest useful loop

    Create, connect, detach, reconnect. Stop only when you want the compute back.

    - Create a devbox + Create a devbox diff --git a/controller/src/devboxes_controller/templates/index.html b/controller/src/devboxes_controller/templates/index.html index 48fa1e8..9d4701b 100644 --- a/controller/src/devboxes_controller/templates/index.html +++ b/controller/src/devboxes_controller/templates/index.html @@ -7,23 +7,30 @@ Devboxes + - + + {% set tenant_query = '?tenant=' ~ tenancy.current.id if tenancy.enabled else '' %}
    - + devboxes
    + {% include "_tenant_switcher.html" %} {{ cluster_name }} connected @@ -33,7 +40,9 @@
    -

    Ephemeral compute · persistent workspace

    +

    + {{ tenancy.current.display_name }} · {{ tenancy.current.role }} access +

    Development machines, on demand.

    Create a prepared environment here or from the terminal. Disconnect freely, while @@ -41,8 +50,8 @@

    Development machines, on demand.

    - devbox create <name> --ssh -
    @@ -55,10 +64,17 @@

    Create a devbox

    Compute expires into a safe stopped state. Your volume is never deleted automatically.

    - {{ storage_class }} storage · {{ workspace_service_type }} SSH{% if gpu.enabled %} · GPU profiles{% endif %}{% if images.enabled %} · approved images{% endif %} + {{ storage_class }} storage · {{ workspace_service_type }} SSH · {{ tenancy.current.role }}{% if gpu.enabled %} · GPU profiles{% endif %}{% if images.enabled %} · approved images{% endif %} + {% if not can_manage %} +
    + Read-only tenant access. + You can inspect workspaces and copy SSH commands, but lifecycle changes require member or admin access. + Shell authorization is controlled separately by this tenant’s workspace SSH keys. +
    + {% endif %}
    @@ -67,27 +83,20 @@

    Create a devbox

    - +
    @@ -153,8 +162,8 @@

    Your devboxes

    @@ -186,10 +195,10 @@

    Delete ?

    Compute and its SSH address will be removed immediately.

    diff --git a/controller/src/devboxes_controller/templates/insights.html b/controller/src/devboxes_controller/templates/insights.html index 459bc0e..8d01cc8 100644 --- a/controller/src/devboxes_controller/templates/insights.html +++ b/controller/src/devboxes_controller/templates/insights.html @@ -7,23 +7,31 @@ Insights · Devboxes + - + + {% set tenant_query = '?tenant=' ~ tenancy.current.id if tenancy.enabled else '' %}
    - + devboxes
    + {% include "_tenant_switcher.html" %} {{ cluster_name }} connected @@ -33,7 +41,9 @@
    -

    Private by design · opt-in collection

    +

    + {{ tenancy.current.display_name }} · tenant-scoped metrics +

    Understand the work, not the worker.

    AI usage and source activity stay separate. Insights stores numeric metrics and Git @@ -55,7 +65,7 @@

    Insights is off.

    insights:
   enabled: true
    - Read the privacy and operations guide + Read the privacy and operations guide
    {% else %} diff --git a/controller/src/devboxes_controller/templates/login.html b/controller/src/devboxes_controller/templates/login.html index bdc117a..381179f 100644 --- a/controller/src/devboxes_controller/templates/login.html +++ b/controller/src/devboxes_controller/templates/login.html @@ -32,7 +32,7 @@

    Ready when the idea is.

    Authenticated access

    Open the workbench

    -

    Use the access token your cluster operator created during installation.

    +

    Use the access token issued by your cluster operator.

    diff --git a/controller/tests/fakes.py b/controller/tests/fakes.py index 220b305..761b1cb 100644 --- a/controller/tests/fakes.py +++ b/controller/tests/fakes.py @@ -2,6 +2,7 @@ from datetime import UTC, datetime, timedelta +from devboxes_controller.config import Settings, TenantConfig from devboxes_controller.manager import DevboxConflictError, DevboxNotFoundError from devboxes_controller.models import ( CreateDevboxRequest, @@ -20,10 +21,12 @@ def sample_devbox( state: DevboxState = DevboxState.READY, preset: Preset = Preset.MEDIUM, gpu: GpuAllocation | None = None, + tenant: str = "default", ) -> Devbox: now = datetime.now(UTC) ready = state is DevboxState.READY return Devbox( + tenant=tenant, name=name, state=state, preset=preset, @@ -50,7 +53,11 @@ def sample_devbox( class FakeManager: - def __init__(self) -> None: + def __init__(self, settings: Settings | None = None) -> None: + self.settings = settings or Settings( + access_token="test-access-token-at-least-32-characters" + ) + default_tenant = self.settings.default_tenant self.boxes = [ sample_devbox( gpu=GpuAllocation( @@ -58,28 +65,53 @@ def __init__(self) -> None: display_name="NVIDIA L4", resource_name="nvidia.com/gpu", count=1, - ) + ), + tenant=default_tenant, + ), + sample_devbox( + "paperclip", + DevboxState.STOPPED, + Preset.SMALL, + tenant=default_tenant, + ), + sample_devbox( + "nightly", + DevboxState.STARTING, + Preset.LARGE, + tenant=default_tenant, ), - sample_devbox("paperclip", DevboxState.STOPPED, Preset.SMALL), - sample_devbox("nightly", DevboxState.STARTING, Preset.LARGE), ] async def ready(self) -> bool: return True - async def list(self) -> list[Devbox]: - return self.boxes + async def list(self, tenant: TenantConfig | None = None) -> list[Devbox]: + tenant_id = (tenant or self.settings.resolve_tenant(None)).id + return [box for box in self.boxes if box.tenant == tenant_id] - async def get(self, name: str) -> Devbox: + async def get(self, name: str, tenant: TenantConfig | None = None) -> Devbox: + tenant_id = (tenant or self.settings.resolve_tenant(None)).id try: - return next(box for box in self.boxes if box.name == name) + return next(box for box in self.boxes if box.name == name and box.tenant == tenant_id) except StopIteration as error: raise DevboxNotFoundError(name) from error - async def create(self, request: CreateDevboxRequest) -> Devbox: - if any(box.name == request.name for box in self.boxes): + async def create( + self, + request: CreateDevboxRequest, + tenant: TenantConfig | None = None, + created_by: str | None = None, + ) -> Devbox: + tenant = tenant or self.settings.resolve_tenant(None) + if any(box.name == request.name and box.tenant == tenant.id for box in self.boxes): raise DevboxConflictError(f"devbox {request.name!r} already exists") - box = sample_devbox(request.name, DevboxState.STARTING, request.preset) + box = sample_devbox( + request.name, + DevboxState.STARTING, + request.preset, + tenant=tenant.id, + ) + box.created_by = created_by box.repository = request.repository if request.gpu is not None: profile = request.gpu.profile or "nvidia-l4" @@ -116,8 +148,13 @@ async def create(self, request: CreateDevboxRequest) -> Devbox: self.boxes.insert(0, box) return box - async def scale(self, name: str, replicas: int) -> Devbox: - box = await self.get(name) + async def scale( + self, + name: str, + replicas: int, + tenant: TenantConfig | None = None, + ) -> Devbox: + box = await self.get(name, tenant) box.state = DevboxState.STARTING if replicas else DevboxState.STOPPED box.pod_ready = False box.message = "Preparing workspace and SSH" if replicas else "Compute stopped" @@ -125,8 +162,14 @@ async def scale(self, name: str, replicas: int) -> Devbox: box.expires_at = datetime.now(UTC) + timedelta(hours=24) return box - async def delete(self, name: str, purge: bool) -> DeleteResult: - self.boxes = [box for box in self.boxes if box.name != name] + async def delete( + self, + name: str, + purge: bool, + tenant: TenantConfig | None = None, + ) -> DeleteResult: + tenant_id = (tenant or self.settings.resolve_tenant(None)).id + self.boxes = [box for box in self.boxes if box.name != name or box.tenant != tenant_id] return DeleteResult(name=name, purged=purge, message=f"{name} deleted") async def stop_expired(self) -> list[str]: diff --git a/controller/tests/preview_app.py b/controller/tests/preview_app.py index 199a726..f5907fa 100644 --- a/controller/tests/preview_app.py +++ b/controller/tests/preview_app.py @@ -1,7 +1,8 @@ from devboxes_controller.app import create_app from devboxes_controller.config import CustomImagePort, CustomImageProfile, GpuProfile, Settings +from devboxes_controller.models import DevboxState, Preset -from .fakes import FakeManager +from .fakes import FakeManager, sample_devbox settings = Settings( access_token="preview-access-token-at-least-32-characters", @@ -36,5 +37,45 @@ ports=[CustomImagePort(name="http", containerPort=8080)], ) ], + multi_tenancy_enabled=True, + default_tenant="platform", + tenants=[ + { + "id": "platform", + "displayName": "Platform Engineering", + "namespace": "devboxes", + "allowedPresets": ["small", "medium"], + "allowedGpuProfiles": ["nvidia-l4"], + "allowedImageProfiles": ["nginx"], + "resourceQuota": { + "enabled": True, + "hard": { + "requests.cpu": "8", + "requests.memory": "32Gi", + }, + }, + }, + { + "id": "research", + "displayName": "Research", + "namespace": "devboxes-research", + "allowedPresets": ["medium", "large"], + "allowedGpuProfiles": ["nvidia-shared"], + "allowedImageProfiles": [], + "maxTtlHours": 72, + }, + ], +) +manager = FakeManager(settings) +manager.boxes.extend( + [ + sample_devbox("simulation", preset=Preset.LARGE, tenant="research"), + sample_devbox( + "notebook", + state=DevboxState.STOPPED, + preset=Preset.MEDIUM, + tenant="research", + ), + ] ) -app = create_app(settings, FakeManager()) # type: ignore[arg-type] +app = create_app(settings, manager) # type: ignore[arg-type] diff --git a/controller/tests/test_app.py b/controller/tests/test_app.py index 8757a0b..f78ee27 100644 --- a/controller/tests/test_app.py +++ b/controller/tests/test_app.py @@ -1,3 +1,4 @@ +from pathlib import Path from urllib.parse import parse_qs, urlencode, urlsplit from fastapi.testclient import TestClient @@ -15,7 +16,7 @@ def app_client(settings: Settings | None = None) -> TestClient: cookie_secure=False, cleanup_interval_seconds=3600, ) - return TestClient(create_app(settings, FakeManager())) # type: ignore[arg-type] + return TestClient(create_app(settings, FakeManager(settings))) # type: ignore[arg-type] def authorization_parameters(port: int = 49152) -> tuple[dict[str, str], str]: @@ -155,7 +156,17 @@ def test_cli_bearer_can_inspect_and_control_a_devbox() -> None: with app_client() as client: assert client.get("/api/v1/whoami", headers=headers).json() == { "user": "operator", + "subject": "operator", "mode": "master-bearer", + "tenant": "default", + "role": "operator", + "tenants": [ + { + "id": "default", + "display_name": "operator", + "role": "operator", + } + ], } assert client.get("/api/v1/devboxes/atlas", headers=headers).status_code == 200 @@ -245,6 +256,24 @@ def test_gpu_capabilities_expose_only_safe_profile_metadata() -> None: ], }, "images": {"enabled": False, "profiles": []}, + "tenancy": { + "enabled": False, + "current": { + "id": "default", + "display_name": "operator", + "role": "operator", + }, + "tenants": [ + { + "id": "default", + "display_name": "operator", + "role": "operator", + } + ], + "max_ttl_hours": 168, + "allowed_presets": ["small", "medium", "large"], + "resource_quota": {}, + }, } assert "workspaceImage" not in response.text assert "runtimeClassName" not in response.text @@ -394,7 +423,17 @@ def test_cli_approval_exchange_whoami_replay_and_no_store() -> None: cli_headers = {"Authorization": f"Bearer {payload['access_token']}"} assert client.get("/api/v1/whoami", headers=cli_headers).json() == { "user": "operator", + "subject": "operator", "mode": "cli-bearer", + "tenant": "default", + "role": "operator", + "tenants": [ + { + "id": "default", + "display_name": "operator", + "role": "operator", + } + ], } assert client.get("/api/v1/devboxes", headers=cli_headers).status_code == 200 @@ -492,3 +531,137 @@ def test_cli_exchange_rejects_malformed_payload_without_reflecting_secrets() -> assert invalid_json.status_code == 400 assert invalid_json.json() == {"detail": "Invalid authorization code exchange"} assert secret not in invalid_json.text + + +def test_tenant_headers_roles_and_dashboard_context_are_enforced( + tmp_path: Path, +) -> None: + credentials = { + "viewer-token": "viewer-principal-token-at-least-32-characters", + "member-token": "member-principal-token-at-least-32-characters", + "admin-token": "admin-principal-token-at-least-32-characters", + } + for key, value in credentials.items(): + (tmp_path / key).write_text(value) + settings = Settings( + access_token="test-access-token-at-least-32-characters", + cookie_secure=False, + cleanup_interval_seconds=3600, + multi_tenancy_enabled=True, + default_tenant="platform", + principal_token_directory=str(tmp_path), + tenants=[ + { + "id": "platform", + "displayName": "Platform", + "namespace": "devboxes-platform", + "allowedPresets": ["small", "medium"], + "maxTtlHours": 72, + }, + { + "id": "research", + "displayName": "Research", + "namespace": "devboxes-research", + }, + ], + principals=[ + { + "subject": "viewer", + "displayName": "View Only", + "tokenKey": "viewer-token", + "memberships": [{"tenant": "platform", "role": "viewer"}], + }, + { + "subject": "member", + "displayName": "Member", + "tokenKey": "member-token", + "memberships": [{"tenant": "platform", "role": "member"}], + }, + { + "subject": "admin", + "displayName": "Admin", + "tokenKey": "admin-token", + "memberships": [ + {"tenant": "platform", "role": "admin"}, + {"tenant": "research", "role": "admin"}, + ], + }, + ], + ) + manager = FakeManager(settings) + app = TestClient(create_app(settings, manager)) # type: ignore[arg-type] + viewer_headers = {"Authorization": f"Bearer {credentials['viewer-token']}"} + member_headers = {"Authorization": f"Bearer {credentials['member-token']}"} + admin_headers = {"Authorization": f"Bearer {credentials['admin-token']}"} + operator_headers = { + "Authorization": "Bearer test-access-token-at-least-32-characters", + "X-Devboxes-Tenant": "research", + } + + with app: + whoami = app.get("/api/v1/whoami", headers=viewer_headers).json() + assert whoami["tenant"] == "platform" + assert whoami["role"] == "viewer" + assert len(app.get("/api/v1/devboxes", headers=viewer_headers).json()["items"]) == 3 + assert ( + app.post( + "/api/v1/devboxes", + headers=viewer_headers, + json={"name": "blocked"}, + ).status_code + == 403 + ) + assert ( + app.get( + "/api/v1/devboxes", + headers={**viewer_headers, "X-Devboxes-Tenant": "research"}, + ).status_code + == 403 + ) + + created = app.post( + "/api/v1/devboxes", + headers=member_headers, + json={"name": "member-box", "preset": "small", "ttl_hours": 72}, + ) + assert created.status_code == 201 + assert created.json()["tenant"] == "platform" + assert created.json()["created_by"] == "member" + assert ( + app.delete( + "/api/v1/devboxes/member-box?purge=true", + headers=member_headers, + ).status_code + == 403 + ) + assert app.delete("/api/v1/devboxes/member-box", headers=member_headers).status_code == 200 + + assert ( + app.delete( + "/api/v1/devboxes/paperclip?purge=true", + headers=admin_headers, + ).status_code + == 200 + ) + same_name = app.post( + "/api/v1/devboxes", + headers=operator_headers, + json={"name": "atlas"}, + ) + assert same_name.status_code == 201 + research_items = app.get( + "/api/v1/devboxes", + headers=operator_headers, + ).json()["items"] + assert [item["name"] for item in research_items] == ["atlas"] + + login = app.post( + "/auth/login", + data={"token": credentials["viewer-token"]}, + follow_redirects=False, + ) + assert login.status_code == 303 + dashboard = app.get("/?tenant=platform") + assert "Platform · viewer access" in dashboard.text + assert "Read-only tenant access" in dashboard.text + assert 'id="tenant-switcher"' in dashboard.text diff --git a/controller/tests/test_auth.py b/controller/tests/test_auth.py index 9664e13..85b9abf 100644 --- a/controller/tests/test_auth.py +++ b/controller/tests/test_auth.py @@ -1,5 +1,7 @@ import asyncio +import hashlib import time +from pathlib import Path from urllib.parse import urlencode import jwt @@ -218,6 +220,7 @@ def _custom_cli_token(**overrides: object) -> str: "jti": "test-token-id", "token_type": CLI_TOKEN_TYPE, "scope": CLI_SCOPE, + "auth_epoch": hashlib.sha256(MASTER_TOKEN.encode()).hexdigest()[:24], } claims.update(overrides) return jwt.encode(claims, SIGNING_KEY, algorithm="HS256", headers={"typ": "JWT"}) @@ -259,8 +262,93 @@ def test_insights_tokens_are_instance_scoped_domain_separated_and_rotatable() -> token = first.issue_insights_token(instance_id, "atlas") cli_token, _ = first.issue_cli_token("operator") - assert same.validate_insights_token(token) == (instance_id, "atlas") + assert same.validate_insights_token(token) == ("default", instance_id, "atlas") + assert token.startswith(f"v1.{instance_id}.atlas.") assert rotated.validate_insights_token(token) is None assert first.validate_insights_token(token.replace("atlas", "other")) is None assert first.validate_insights_token(cli_token) is None assert MASTER_TOKEN not in token + + +def test_principal_tokens_resolve_tenant_roles_and_revoke_derived_credentials( + tmp_path: Path, +) -> None: + principal_token = "alice-principal-token-at-least-32-characters" + token_path = tmp_path / "alice-token" + token_path.write_text(principal_token) + tenant_settings = settings( + multi_tenancy_enabled=True, + default_tenant="platform", + principal_token_directory=str(tmp_path), + tenants=[ + { + "id": "platform", + "displayName": "Platform", + "namespace": "devboxes-platform", + }, + { + "id": "research", + "displayName": "Research", + "namespace": "devboxes-research", + }, + ], + principals=[ + { + "subject": "alice@example.com", + "displayName": "Alice", + "tokenKey": "alice-token", + "defaultTenant": "platform", + "memberships": [ + {"tenant": "platform", "role": "admin"}, + {"tenant": "research", "role": "viewer"}, + ], + } + ], + ) + authenticator = Authenticator(tenant_settings) + tenant_insights_token = authenticator.issue_insights_token( + "99999999-9999-4999-8999-999999999999", + "atlas", + "platform", + ) + assert tenant_insights_token.startswith( + "v2.platform.99999999-9999-4999-8999-999999999999.atlas." + ) + app = FastAPI() + + @app.get("/checked") + async def checked(request: Request): + context = await authenticator.require(request) + return { + "subject": context.subject, + "tenant": context.tenant.id, + "role": context.role, + } + + client = TestClient(app) + headers = {"Authorization": f"Bearer {principal_token}"} + assert client.get("/checked", headers=headers).json() == { + "subject": "alice@example.com", + "tenant": "platform", + "role": "admin", + } + assert ( + client.get( + "/checked", + headers={**headers, "X-Devboxes-Tenant": "research"}, + ).json()["role"] + == "viewer" + ) + assert ( + client.get( + "/checked", + headers={**headers, "X-Devboxes-Tenant": "missing"}, + ).status_code + == 403 + ) + + cli_token, _ = authenticator.issue_cli_token("alice@example.com") + assert authenticator.validate_cli_token(cli_token) == "alice@example.com" + token_path.write_text("rotated-alice-principal-token-at-least-32-characters") + rotated = Authenticator(tenant_settings) + assert rotated.validate_cli_token(cli_token) is None diff --git a/controller/tests/test_config.py b/controller/tests/test_config.py index 99b6de6..1d1ab52 100644 --- a/controller/tests/test_config.py +++ b/controller/tests/test_config.py @@ -314,3 +314,266 @@ def test_custom_image_profiles_load_from_the_helm_json_environment( assert profile.description == "Serve static content" assert profile.resources is not None assert profile.ports[0].container_port == 8080 + + +def test_multi_tenancy_policy_loads_from_helm_json_and_resolves_effective_limits( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DEVBOXES_ACCESS_TOKEN", "test-access-token-at-least-32-characters") + monkeypatch.setenv("DEVBOXES_MULTI_TENANCY_ENABLED", "true") + monkeypatch.setenv("DEVBOXES_DEFAULT_TENANT", "platform") + monkeypatch.setenv( + "DEVBOXES_TENANTS", + json.dumps( + [ + { + "id": "platform", + "displayName": " Platform Engineering ", + "namespace": "devboxes-platform", + "workspaceSecretName": "platform-workspace", + "maxTtlHours": 72, + "allowedPresets": ["small", "medium"], + "allowedGpuProfiles": [], + "resourceQuota": { + "enabled": True, + "hard": { + "requests.cpu": "8", + "requests.memory": "32Gi", + "requests.nvidia.com/gpu": "2", + }, + }, + }, + { + "id": "research", + "displayName": "Research", + "namespace": "devboxes-research", + }, + ] + ), + ) + monkeypatch.setenv( + "DEVBOXES_PRINCIPALS", + json.dumps( + [ + { + "subject": "alice@example.com", + "displayName": "Alice", + "tokenKey": "alice-token", + "defaultTenant": "platform", + "memberships": [ + {"tenant": "platform", "role": "admin"}, + {"tenant": "research", "role": "viewer"}, + ], + } + ] + ), + ) + + settings = Settings(_env_file=None) + platform = settings.resolve_tenant("platform") + + assert platform.display_name == "Platform Engineering" + assert settings.tenant_max_ttl_hours(platform) == 72 + assert [preset.value for preset in platform.allowed_presets] == ["small", "medium"] + assert settings.tenant_workspace_secret_name(platform) == "platform-workspace" + assert platform.resource_quota is not None + assert platform.resource_quota.hard["requests.nvidia.com/gpu"] == "2" + + +@pytest.mark.parametrize( + ("override", "message"), + [ + ({"default_tenant": "missing"}, "default_tenant"), + ( + { + "tenants": [ + { + "id": "platform", + "displayName": "Platform", + "namespace": "shared", + }, + { + "id": "research", + "displayName": "Research", + "namespace": "shared", + }, + ] + }, + "namespaces must be unique", + ), + ( + { + "principals": [ + { + "subject": "alice", + "displayName": "Alice", + "tokenKey": "alice-token", + "memberships": [{"tenant": "missing", "role": "member"}], + } + ] + }, + "unknown tenants", + ), + ], +) +def test_multi_tenancy_rejects_ambiguous_or_unreachable_policy( + override: dict[str, object], + message: str, +) -> None: + values: dict[str, object] = { + "access_token": "test-access-token-at-least-32-characters", + "multi_tenancy_enabled": True, + "default_tenant": "platform", + "tenants": [ + { + "id": "platform", + "displayName": "Platform", + "namespace": "devboxes-platform", + } + ], + "_env_file": None, + } + values.update(override) + + with pytest.raises(ValidationError, match=message): + Settings(**values) # type: ignore[arg-type] + + +def test_tenant_gpu_default_is_explicit_and_must_remain_inside_the_allowlist() -> None: + profiles = [ + GpuProfile( + name=name, + displayName=name.upper(), + resourceName=f"example.com/{name}", + count=1, + ) + for name in ("shared", "dedicated") + ] + settings = Settings( + access_token="test-access-token-at-least-32-characters", + gpu_enabled=True, + gpu_default_profile="shared", + gpu_profiles=profiles, + multi_tenancy_enabled=True, + default_tenant="platform", + tenants=[ + { + "id": "platform", + "displayName": "Platform", + "namespace": "devboxes-platform", + "allowedGpuProfiles": ["dedicated"], + "defaultGpuProfile": "dedicated", + } + ], + _env_file=None, + ) + tenant = settings.resolve_tenant("platform") + + assert settings.tenant_default_gpu_profile(tenant) == "dedicated" + assert settings.resolve_gpu_profile(None, tenant).name == "dedicated" + + with pytest.raises(ValidationError, match="included in allowedGpuProfiles"): + Settings( + access_token="test-access-token-at-least-32-characters", + gpu_enabled=True, + gpu_default_profile="shared", + gpu_profiles=profiles, + multi_tenancy_enabled=True, + default_tenant="platform", + tenants=[ + { + "id": "platform", + "displayName": "Platform", + "namespace": "devboxes-platform", + "allowedGpuProfiles": ["dedicated"], + "defaultGpuProfile": "shared", + } + ], + _env_file=None, + ) + + with pytest.raises(ValidationError, match="requires GPU acceleration"): + Settings( + access_token="test-access-token-at-least-32-characters", + gpu_enabled=False, + gpu_profiles=profiles, + multi_tenancy_enabled=True, + default_tenant="platform", + tenants=[ + { + "id": "platform", + "displayName": "Platform", + "namespace": "devboxes-platform", + "defaultGpuProfile": "shared", + } + ], + _env_file=None, + ) + + +@pytest.mark.parametrize( + ("values", "message"), + [ + ({"principal_token_directory": "/"}, "absolute non-root"), + ({"principal_token_directory": "//"}, "absolute non-root"), + ({"principal_token_directory": "/var/run/../secrets"}, "absolute non-root"), + ( + { + "multi_tenancy_enabled": True, + "default_tenant": "platform", + "tenants": [ + { + "id": "platform", + "displayName": "Platform", + "namespace": "default", + } + ], + }, + "non-system", + ), + ( + { + "multi_tenancy_enabled": True, + "default_tenant": "platform", + "tenants": [ + { + "id": "platform", + "displayName": "Platform", + "namespace": "devboxes-platform", + "namespaceLabels": { + "devboxes.bonalab.org/tenant": "other", + }, + } + ], + }, + "reserved label", + ), + ( + { + "multi_tenancy_enabled": True, + "default_tenant": "platform", + "tenants": [ + { + "id": "platform", + "displayName": "Platform", + "namespace": "devboxes-platform", + "namespaceLabels": { + "kubernetes.io/metadata.name": "other", + }, + } + ], + }, + "reserved label", + ), + ], +) +def test_tenancy_rejects_unsafe_mount_and_namespace_metadata( + values: dict[str, object], + message: str, +) -> None: + with pytest.raises(ValidationError, match=message): + Settings( + access_token="test-access-token-at-least-32-characters", + _env_file=None, + **values, + ) diff --git a/controller/tests/test_insights_app.py b/controller/tests/test_insights_app.py index fd98697..0356409 100644 --- a/controller/tests/test_insights_app.py +++ b/controller/tests/test_insights_app.py @@ -27,7 +27,7 @@ def enabled_app(tmp_path: Path) -> tuple[TestClient, Settings]: insights_db_path=str(tmp_path / "insights.db"), ) service = InsightsService(settings) - return TestClient(create_app(settings, FakeManager(), service)), settings # type: ignore[arg-type] + return TestClient(create_app(settings, FakeManager(settings), service)), settings # type: ignore[arg-type] def otlp_batch(batch_id: str = "a" * 64) -> bytes: @@ -57,7 +57,7 @@ def test_insights_dashboard_disabled_state_and_read_contract() -> None: cookie_secure=False, cleanup_interval_seconds=3600, ) - with TestClient(create_app(settings, FakeManager())) as client: # type: ignore[arg-type] + with TestClient(create_app(settings, FakeManager(settings))) as client: # type: ignore[arg-type] browser_login(client) dashboard = client.get("/insights") assert dashboard.status_code == 200 @@ -237,3 +237,82 @@ def test_insights_query_validation_and_authentication(tmp_path: Path) -> None: == 422 ) assert client.delete("/api/v1/insights", headers=headers).status_code == 422 + + +def test_insights_ingest_query_and_purge_are_tenant_scoped(tmp_path: Path) -> None: + settings = Settings( + access_token=MASTER_TOKEN, + cookie_secure=False, + cleanup_interval_seconds=3600, + insights_enabled=True, + insights_db_path=str(tmp_path / "insights.db"), + multi_tenancy_enabled=True, + default_tenant="platform", + tenants=[ + { + "id": "platform", + "displayName": "Platform", + "namespace": "devboxes-platform", + }, + { + "id": "research", + "displayName": "Research", + "namespace": "devboxes-research", + }, + ], + ) + client = TestClient( + create_app(settings, FakeManager(settings), InsightsService(settings)) # type: ignore[arg-type] + ) + authenticator = Authenticator(settings) + instances = { + "platform": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "research": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + } + + with client: + for index, (tenant, instance_id) in enumerate(instances.items()): + credential = authenticator.issue_insights_token( + instance_id, + "atlas", + tenant, + ) + accepted = client.post( + "/internal/v1/insights/batches", + content=gzip.compress(otlp_batch(chr(ord("c") + index) * 64)), + headers={ + "Authorization": f"Bearer {credential}", + "Content-Type": "application/json", + "Content-Encoding": "gzip", + }, + ) + assert accepted.status_code == 200 + + for tenant in instances: + summary = client.get( + "/api/v1/insights/summary?since=24h", + headers={ + "Authorization": f"Bearer {MASTER_TOKEN}", + "X-Devboxes-Tenant": tenant, + }, + ) + assert summary.status_code == 200 + assert summary.json()["tenant"] == tenant + assert summary.json()["data"]["ai"]["totals"]["tokens"] == 15 + + purged = client.delete( + "/api/v1/insights?box=atlas", + headers={ + "Authorization": f"Bearer {MASTER_TOKEN}", + "X-Devboxes-Tenant": "platform", + }, + ) + assert purged.json()["purged_instances"] == 1 + research = client.get( + "/api/v1/insights/summary?since=24h", + headers={ + "Authorization": f"Bearer {MASTER_TOKEN}", + "X-Devboxes-Tenant": "research", + }, + ) + assert research.json()["data"]["ai"]["totals"]["tokens"] == 15 diff --git a/controller/tests/test_insights_store.py b/controller/tests/test_insights_store.py index b40cea9..966f4d5 100644 --- a/controller/tests/test_insights_store.py +++ b/controller/tests/test_insights_store.py @@ -26,6 +26,7 @@ def provider_fixture(name: str) -> tuple[dict[str, object], int]: def ingest( store: InsightsStore, *, + tenant: str = "default", instance_id: str, box: str, batch_id: str, @@ -36,6 +37,7 @@ def ingest( ): return asyncio.run( store.ingest( + tenant_id=tenant, instance_id=instance_id, box_name=box, batch_id=batch_id, @@ -83,7 +85,7 @@ def test_store_migrates_readies_and_creates_an_online_backup(tmp_path: Path) -> asyncio.run(store.backup(backup)) assert backup.exists() with closing(sqlite3.connect(backup)) as copy: - assert copy.execute("SELECT MAX(version) FROM schema_migrations").fetchone()[0] == 1 + assert copy.execute("SELECT MAX(version) FROM schema_migrations").fetchone()[0] == 2 def test_database_size_tolerates_a_disappearing_sqlite_auxiliary_file( @@ -126,7 +128,7 @@ def test_store_migrates_a_previous_empty_schema_transactionally(tmp_path: Path) assert asyncio.run(store.ready()) is True with closing(sqlite3.connect(database)) as connection: - assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone()[0] == 1 + assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone()[0] == 2 assert ( connection.execute( "SELECT COUNT(*) FROM sqlite_master " @@ -492,3 +494,117 @@ def test_store_rolls_back_invalid_batch_kinds_and_timestamps(tmp_path: Path) -> def test_uninitialized_store_is_not_ready(tmp_path: Path) -> None: store = InsightsStore(tmp_path / "missing" / "insights.db") assert asyncio.run(store.ready()) is False + + +def test_store_isolates_queries_purges_and_sqlite_exports_by_tenant( + tmp_path: Path, +) -> None: + store = InsightsStore(tmp_path / "insights.db") + asyncio.run(store.initialize()) + claude, points = provider_fixture("claude-2.1.205-otlp.json") + platform_instance = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + research_instance = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" + ingest( + store, + tenant="platform", + instance_id=platform_instance, + box="atlas", + batch_id="c" * 64, + collector="otel", + kind="otlp", + payload=claude, + points=points, + ) + ingest( + store, + tenant="research", + instance_id=research_instance, + box="atlas", + batch_id="d" * 64, + collector="otel", + kind="otlp", + payload=claude, + points=points, + ) + + platform = asyncio.run( + store.summary( + QueryFilters( + since=RANGE.since, + until=RANGE.until, + tenant_id="platform", + ) + ) + ) + research = asyncio.run( + store.summary( + QueryFilters( + since=RANGE.since, + until=RANGE.until, + tenant_id="research", + ) + ) + ) + missing = asyncio.run( + store.summary( + QueryFilters( + since=RANGE.since, + until=RANGE.until, + tenant_id="missing", + ) + ) + ) + assert platform["ai"]["totals"]["tokens"] == 15 + assert research["ai"]["totals"]["tokens"] == 15 + assert missing["ai"]["totals"]["tokens"] == 0 + + export = tmp_path / "platform.db" + asyncio.run(store.backup(export, "platform")) + with closing(sqlite3.connect(export)) as connection: + assert connection.execute("SELECT tenant_id, id FROM devbox_instances").fetchall() == [ + ("platform", platform_instance) + ] + + assert asyncio.run(store.purge_box("atlas", "platform")) == 1 + with closing(sqlite3.connect(store.path)) as connection: + assert connection.execute( + "SELECT tenant_id, id FROM devbox_instances ORDER BY tenant_id" + ).fetchall() == [("research", research_instance)] + + +def test_store_migrates_legacy_instances_into_the_configured_default_tenant( + tmp_path: Path, +) -> None: + database = tmp_path / "legacy.db" + with closing(sqlite3.connect(database)) as connection: + connection.executescript( + """ + CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL + ); + CREATE TABLE devbox_instances ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + deleted_at TEXT + ); + INSERT INTO schema_migrations VALUES (1, '2026-01-01T00:00:00Z'); + INSERT INTO devbox_instances VALUES ( + 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + 'atlas', + '2026-01-01T00:00:00Z', + '2026-01-01T00:00:00Z', + NULL + ); + """ + ) + store = InsightsStore(database, default_tenant_id="platform") + + asyncio.run(store.initialize()) + + with closing(sqlite3.connect(database)) as connection: + assert connection.execute("SELECT tenant_id FROM devbox_instances").fetchone() == ( + "platform", + ) diff --git a/controller/tests/test_manager.py b/controller/tests/test_manager.py index 1c585d3..8a698f0 100644 --- a/controller/tests/test_manager.py +++ b/controller/tests/test_manager.py @@ -11,6 +11,7 @@ DevboxConflictError, DevboxManager, DevboxNotFoundError, + TenantQuotaExceededError, _custom_image_allocation, _gpu_allocation, _resolved_custom_image, @@ -22,6 +23,7 @@ from devboxes_controller.models import CreateDevboxRequest, DevboxState, GpuRequest, Preset from devboxes_controller.resources import ( ANNOTATION_CREATED_AT, + ANNOTATION_CREATED_BY, ANNOTATION_CUSTOM_IMAGE_CONFIG, ANNOTATION_CUSTOM_IMAGE_PROFILE, ANNOTATION_EXPIRES_AT, @@ -32,12 +34,25 @@ ANNOTATION_PRESET, ANNOTATION_STORAGE, ANNOTATION_TTL_HOURS, + LABEL_MANAGED_BY, LABEL_NAME, + LABEL_TENANT, + MANAGED_BY, ) from .fakes import sample_devbox +def _managed_labels(name: str = "atlas", tenant: str | None = None) -> dict[str, str]: + labels = { + LABEL_MANAGED_BY: MANAGED_BY, + LABEL_NAME: name, + } + if tenant is not None: + labels[LABEL_TENANT] = tenant + return labels + + def test_ttl_hours_rejects_missing_invalid_and_out_of_range_values() -> None: assert _ttl_hours(None, default=24, maximum=168) == 24 assert _ttl_hours("invalid", default=24, maximum=168) == 24 @@ -97,9 +112,50 @@ def test_create_removes_deployment_when_service_creation_fails() -> None: core.create_namespaced_persistent_volume_claim.assert_called_once() +def test_create_rejects_an_unmanaged_same_name_deployment_without_deleting_it() -> None: + apps = Mock() + apps.read_namespaced_deployment.return_value = SimpleNamespace( + metadata=SimpleNamespace(labels={"app": "someone-else"}) + ) + core = Mock() + manager = DevboxManager( + Settings(access_token="test-access-token-at-least-32-characters"), + apps_api=apps, + core_api=core, + ) + + with pytest.raises(DevboxConflictError, match="already exists"): + asyncio.run(manager.create(CreateDevboxRequest(name="atlas"))) + + apps.create_namespaced_deployment.assert_not_called() + apps.delete_namespaced_deployment.assert_not_called() + core.read_namespaced_persistent_volume_claim.assert_not_called() + + +def test_create_does_not_delete_a_deployment_that_wins_the_create_race() -> None: + apps = Mock() + apps.read_namespaced_deployment.side_effect = ApiException(status=404) + apps.create_namespaced_deployment.side_effect = ApiException(status=409) + core = Mock() + core.read_namespaced_persistent_volume_claim.side_effect = ApiException(status=404) + manager = DevboxManager( + Settings(access_token="test-access-token-at-least-32-characters"), + apps_api=apps, + core_api=core, + ) + + with pytest.raises(DevboxConflictError, match="already exists"): + asyncio.run(manager.create(CreateDevboxRequest(name="atlas"))) + + apps.delete_namespaced_deployment.assert_not_called() + + @pytest.mark.parametrize("purge", [False, True]) def test_delete_preserves_or_purges_storage_explicitly(purge: bool) -> None: apps = Mock() + apps.read_namespaced_deployment.return_value = SimpleNamespace( + metadata=SimpleNamespace(labels=_managed_labels()) + ) core = Mock() manager = DevboxManager( Settings(access_token="test-access-token-at-least-32-characters"), @@ -143,6 +199,14 @@ def test_stop_expired_stops_only_active_expired_boxes() -> None: apps_api=apps, core_api=Mock(), ) + apps.read_namespaced_deployment.return_value = SimpleNamespace( + metadata=SimpleNamespace( + annotations={ANNOTATION_EXPIRES_AT: expired.expires_at.isoformat()}, + creation_timestamp=expired.created_at, + labels=_managed_labels("expired"), + ), + spec=SimpleNamespace(replicas=1), + ) manager.list = AsyncMock(return_value=[expired, stopped, active]) # type: ignore[method-assign] assert asyncio.run(manager.stop_expired()) == ["expired"] @@ -207,7 +271,10 @@ def test_state_surfaces_scheduler_capacity_failures() -> None: def test_start_renews_the_original_ttl_atomically() -> None: apps = Mock() apps.read_namespaced_deployment.return_value = SimpleNamespace( - metadata=SimpleNamespace(annotations={ANNOTATION_TTL_HOURS: "72"}) + metadata=SimpleNamespace( + annotations={ANNOTATION_TTL_HOURS: "72"}, + labels=_managed_labels(), + ) ) settings = Settings( access_token="test-access-token-at-least-32-characters", @@ -232,6 +299,9 @@ def test_start_renews_the_original_ttl_atomically() -> None: def test_stop_uses_the_scale_subresource_without_renewing_ttl() -> None: apps = Mock() + apps.read_namespaced_deployment.return_value = SimpleNamespace( + metadata=SimpleNamespace(labels=_managed_labels()) + ) settings = Settings(access_token="test-access-token-at-least-32-characters") manager = DevboxManager(settings, apps_api=apps, core_api=Mock()) manager.get = AsyncMock(return_value=Mock()) # type: ignore[method-assign] @@ -246,12 +316,31 @@ def test_stop_uses_the_scale_subresource_without_renewing_ttl() -> None: apps.patch_namespaced_deployment.assert_not_called() +def test_stop_rejects_an_unmanaged_same_name_deployment() -> None: + apps = Mock() + apps.read_namespaced_deployment.return_value = SimpleNamespace( + metadata=SimpleNamespace(labels={"app": "someone-else"}) + ) + manager = DevboxManager( + Settings(access_token="test-access-token-at-least-32-characters"), + apps_api=apps, + core_api=Mock(), + ) + + with pytest.raises(DevboxNotFoundError): + asyncio.run(manager.scale("atlas", 0)) + + apps.patch_namespaced_deployment_scale.assert_not_called() + + def test_create_expands_a_retained_home_volume_for_a_larger_preset() -> None: apps = Mock() apps.read_namespaced_deployment.side_effect = ApiException(status=404) core = Mock() + core.read_namespaced_secret.side_effect = ApiException(status=404) core.read_namespaced_persistent_volume_claim.return_value = SimpleNamespace( - spec=SimpleNamespace(resources=SimpleNamespace(requests={"storage": "20Gi"})) + metadata=SimpleNamespace(labels=_managed_labels()), + spec=SimpleNamespace(resources=SimpleNamespace(requests={"storage": "20Gi"})), ) settings = Settings(access_token="test-access-token-at-least-32-characters") manager = DevboxManager(settings, apps_api=apps, core_api=core) @@ -273,6 +362,26 @@ def test_create_expands_a_retained_home_volume_for_a_larger_preset() -> None: assert result is expected +def test_create_rejects_an_unmanaged_retained_volume_before_mutating_it() -> None: + apps = Mock() + apps.read_namespaced_deployment.side_effect = ApiException(status=404) + core = Mock() + core.read_namespaced_persistent_volume_claim.return_value = SimpleNamespace( + metadata=SimpleNamespace(labels={"app": "someone-else"}) + ) + manager = DevboxManager( + Settings(access_token="test-access-token-at-least-32-characters"), + apps_api=apps, + core_api=core, + ) + + with pytest.raises(DevboxConflictError, match="not owned"): + asyncio.run(manager.create(CreateDevboxRequest(name="atlas"))) + + core.patch_namespaced_persistent_volume_claim.assert_not_called() + apps.create_namespaced_deployment.assert_not_called() + + def test_node_port_endpoint_uses_configured_node_address() -> None: service = SimpleNamespace( spec=SimpleNamespace( @@ -321,6 +430,9 @@ def test_create_uses_portable_node_port_and_default_storage_configuration() -> N deployment = apps.create_namespaced_deployment.call_args.args[1] service = core.create_namespaced_service.call_args.args[1] assert "storageClassName" not in pvc["spec"] + assert LABEL_TENANT not in pvc["metadata"]["labels"] + assert LABEL_TENANT not in deployment["metadata"]["labels"] + assert LABEL_TENANT not in service["metadata"]["labels"] assert deployment["spec"]["template"]["spec"]["serviceAccountName"] == "devboxes-workspace" assert "priorityClassName" not in deployment["spec"]["template"]["spec"] assert service["spec"]["type"] == "NodePort" @@ -471,7 +583,7 @@ def _legacy_deployment(replicas: int) -> SimpleNamespace: return SimpleNamespace( metadata=SimpleNamespace( name="devbox-atlas", - labels={LABEL_NAME: "atlas"}, + labels=_managed_labels(), annotations={ ANNOTATION_CREATED_AT: now.isoformat(), ANNOTATION_EXPIRES_AT: (now + timedelta(hours=24)).isoformat(), @@ -580,8 +692,9 @@ def test_active_legacy_workspace_is_marked_restart_required_without_template_pat apps.list_namespaced_deployment.return_value = SimpleNamespace(items=[deployment]) apps.read_namespaced_deployment.return_value = deployment core = Mock() + core.read_namespaced_secret.side_effect = ApiException(status=404) core.read_namespaced_persistent_volume_claim.return_value = SimpleNamespace( - metadata=SimpleNamespace(annotations={}) + metadata=SimpleNamespace(annotations={}, labels=_managed_labels()) ) manager = DevboxManager( Settings( @@ -663,9 +776,11 @@ def test_stopped_legacy_workspace_template_is_reconciled_without_starting_it() - apps.list_namespaced_deployment.return_value = SimpleNamespace(items=[deployment]) apps.read_namespaced_deployment.side_effect = [deployment, _legacy_deployment(0)] core = Mock() + core.read_namespaced_secret.side_effect = ApiException(status=404) core.read_namespaced_persistent_volume_claim.return_value = SimpleNamespace( metadata=SimpleNamespace( - annotations={ANNOTATION_INSTANCE_ID: "99999999-9999-4999-8999-999999999999"} + annotations={ANNOTATION_INSTANCE_ID: "99999999-9999-4999-8999-999999999999"}, + labels=_managed_labels(), ) ) manager = DevboxManager( @@ -710,9 +825,11 @@ def test_insights_reconciliation_uses_the_pinned_gpu_snapshot() -> None: apps.list_namespaced_deployment.return_value = SimpleNamespace(items=[deployment]) apps.read_namespaced_deployment.side_effect = [deployment, _legacy_deployment(0)] core = Mock() + core.read_namespaced_secret.side_effect = ApiException(status=404) core.read_namespaced_persistent_volume_claim.return_value = SimpleNamespace( metadata=SimpleNamespace( - annotations={ANNOTATION_INSTANCE_ID: "99999999-9999-4999-8999-999999999999"} + annotations={ANNOTATION_INSTANCE_ID: "99999999-9999-4999-8999-999999999999"}, + labels=_managed_labels(), ) ) manager = DevboxManager( @@ -779,3 +896,105 @@ def test_node_port_model_includes_allocated_port_in_ssh_command() -> None: assert box.ssh_host == "dev-node.example.test" assert box.ssh_port == 30222 assert box.ssh_command == "ssh -t -p 30222 dev@dev-node.example.test" + + +def test_multi_tenant_create_uses_only_the_resolved_namespace_and_policy() -> None: + settings = Settings( + access_token="test-access-token-at-least-32-characters", + multi_tenancy_enabled=True, + default_tenant="platform", + tenants=[ + { + "id": "platform", + "displayName": "Platform", + "namespace": "devboxes-platform", + }, + { + "id": "research", + "displayName": "Research", + "namespace": "devboxes-research", + "workspaceSecretName": "research-workspace", + "workspaceServiceAccountName": "research-workspace", + "storageClass": "fast-rwo", + "maxTtlHours": 24, + "allowedPresets": ["small"], + }, + ], + ) + research = settings.resolve_tenant("research") + apps = Mock() + apps.read_namespaced_deployment.side_effect = ApiException(status=404) + core = Mock() + core.read_namespaced_persistent_volume_claim.side_effect = ApiException(status=404) + manager = DevboxManager(settings, apps_api=apps, core_api=core) + manager.get = AsyncMock(return_value=sample_devbox(tenant="research")) # type: ignore[method-assign] + + result = asyncio.run( + manager.create( + CreateDevboxRequest(name="atlas", preset="small", ttl_hours=24), + research, + "alice@example.com", + ) + ) + + assert result.tenant == "research" + pvc = core.create_namespaced_persistent_volume_claim.call_args.args[1] + deployment = apps.create_namespaced_deployment.call_args.args[1] + service = core.create_namespaced_service.call_args.args[1] + for manifest in (pvc, deployment, service): + assert manifest["metadata"]["namespace"] == "devboxes-research" + assert manifest["metadata"]["labels"][LABEL_TENANT] == "research" + assert manifest["metadata"]["annotations"][ANNOTATION_CREATED_BY] == "alice@example.com" + assert pvc["spec"]["storageClassName"] == "fast-rwo" + pod_spec = deployment["spec"]["template"]["spec"] + assert pod_spec["serviceAccountName"] == "research-workspace" + secret = next(volume for volume in pod_spec["volumes"] if volume["name"] == "workspace-secrets") + assert secret["secret"]["secretName"] == "research-workspace" + assert all( + operation.call_args.args[0] == "devboxes-research" + for operation in ( + core.create_namespaced_persistent_volume_claim, + apps.create_namespaced_deployment, + core.create_namespaced_service, + ) + ) + + with pytest.raises(ValueError, match="unavailable"): + asyncio.run( + manager.create( + CreateDevboxRequest(name="large", preset="large"), + research, + ) + ) + + +def test_resource_quota_admission_is_translated_without_leaking_provider_details() -> None: + settings = Settings( + access_token="test-access-token-at-least-32-characters", + multi_tenancy_enabled=True, + default_tenant="platform", + tenants=[ + { + "id": "platform", + "displayName": "Platform", + "namespace": "devboxes-platform", + } + ], + ) + tenant = settings.resolve_tenant("platform") + apps = Mock() + apps.read_namespaced_deployment.side_effect = ApiException(status=404) + core = Mock() + core.read_namespaced_persistent_volume_claim.side_effect = ApiException(status=404) + core.create_namespaced_persistent_volume_claim.side_effect = ApiException( + status=403, + reason='exceeded quota: private-provider-detail="node-12"', + ) + manager = DevboxManager(settings, apps_api=apps, core_api=core) + + with pytest.raises( + TenantQuotaExceededError, + match="tenant 'platform' exceeded its Kubernetes ResourceQuota", + ) as error: + asyncio.run(manager.create(CreateDevboxRequest(name="atlas"), tenant)) + assert "node-12" not in str(error.value) diff --git a/controller/tests/test_resources.py b/controller/tests/test_resources.py index 136333d..21810f0 100644 --- a/controller/tests/test_resources.py +++ b/controller/tests/test_resources.py @@ -19,6 +19,7 @@ ANNOTATION_GPU_RESOURCE, ANNOTATION_INSTANCE_ID, ANNOTATION_TTL_HOURS, + LABEL_TENANT, build_deployment, build_pvc, build_service, @@ -52,6 +53,8 @@ def test_deployment_is_hardened_and_ready_for_tmux_workspace() -> None: assert deployment["spec"]["progressDeadlineSeconds"] == 1800 assert deployment["spec"]["strategy"]["type"] == "Recreate" assert deployment["metadata"]["annotations"][ANNOTATION_TTL_HOURS] == "24" + assert LABEL_TENANT not in deployment["metadata"]["labels"] + assert LABEL_TENANT not in deployment["spec"]["selector"]["matchLabels"] assert container["resources"]["limits"] == {"memory": "8Gi"} assert "cpu" not in container["resources"]["limits"] assert container["securityContext"]["allowPrivilegeEscalation"] is True diff --git a/docs/api.md b/docs/api.md index 4ef6cc8..a729012 100644 --- a/docs/api.md +++ b/docs/api.md @@ -4,19 +4,33 @@ The controller exposes a small JSON API under `/api/v1`. The Rust CLI is the ref ## Authentication -Automation requests may use the master controller token. Browser-authorized CLI requests -use a scoped, expiring token with the same bearer header: +Automation requests may use the master controller token or a configured principal token. +Browser-authorized CLI requests use a scoped, expiring token with the same bearer header: ```http Authorization: Bearer CONTROLLER_ACCESS_TOKEN Accept: application/json ``` -Browser login exchanges the master token for an HTTP-only, SameSite `devboxes_session` -cookie and a readable `devboxes_csrf` cookie. Every browser mutation sends the CSRF value -in `X-Devboxes-CSRF`. API clients using bearer authentication do not need a CSRF header. +Browser login exchanges a submitted master or principal token for an HTTP-only, SameSite +`devboxes_session` cookie and a readable `devboxes_csrf` cookie. Every browser mutation +sends the CSRF value in `X-Devboxes-CSRF`. API clients using bearer authentication do not +need a CSRF header. -The shared token controls every devbox in the installation, including permanent purge. Do not expose it in URLs, shell history, logs, or source control. +The master token controls every tenant, including permanent purge. Principal access is +limited by live tenant memberships. Do not expose either credential in URLs, shell history, +logs, or source control. + +When multi-tenancy is enabled, select a tenant with: + +```http +X-Devboxes-Tenant: platform +``` + +Omitting the header selects the identity's default tenant. The controller resolves and +authorizes the tenant before any lifecycle, capability, or Insights work. Unknown and +unauthorized tenant IDs both return `403`. Browser requests carry the same header from the +tenant switcher; HTML pages use the equivalent `?tenant=platform` navigation query. ## Endpoints @@ -29,7 +43,7 @@ The shared token controls every devbox in the installation, including permanent | `POST` | `/auth/cli/authorize` | 303 | Approve or deny a CSRF-protected CLI request | | `POST` | `/api/v1/auth/cli/token` | 200 | Exchange a one-time code and PKCE verifier | | `GET` | `/api/v1/whoami` | 200 | Verify authentication and identity | -| `GET` | `/api/v1/capabilities` | 200 | Discover installation GPU and custom image profiles | +| `GET` | `/api/v1/capabilities` | 200 | Discover active tenant policy, GPU, and image profiles | | `GET` | `/api/v1/devboxes` | 200 | List managed devboxes | | `POST` | `/api/v1/devboxes` | 201 | Create a devbox | | `GET` | `/api/v1/devboxes/{name}` | 200 | Read one devbox | @@ -43,7 +57,7 @@ The shared token controls every devbox in the installation, including permanent | `GET` | `/api/v1/insights/export` | 200 | Export a filtered JSON or CSV summary, or an online SQLite backup | | `DELETE` | `/api/v1/insights` | 200 | Explicitly purge history by box or instance ID | -The list endpoint is not paginated. One installation is intended for a small, trusted operator scope. +The list endpoint is not paginated and returns only the active tenant's boxes. The CLI authorization route accepts only `client_id=devbox-cli`, an exact numeric HTTP loopback redirect ending in `/callback`, a high-entropy state, a PKCE challenge, and @@ -54,10 +68,40 @@ responses are `no-store`, codes are single-use, and no refresh token is returned ## Installation capabilities -Authenticated clients use `GET /api/v1/capabilities` to discover optional installation features. Capability discovery returns only the safe user contract, not profile images, pull policies, resource limits, RuntimeClasses, supplemental groups, selectors, or tolerations: +Authenticated clients use `GET /api/v1/capabilities` to discover the selected tenant's +effective policy and optional installation features. Capability discovery returns only the +safe user contract, not namespaces outside the active tenant, workspace Secrets, profile +images, pull policies, resource limits, RuntimeClasses, supplemental groups, selectors, or +tolerations: ```json { + "tenancy": { + "enabled": true, + "current": { + "id": "platform", + "display_name": "Platform Engineering", + "role": "admin" + }, + "tenants": [ + { + "id": "platform", + "display_name": "Platform Engineering", + "role": "admin" + }, + { + "id": "research", + "display_name": "Research", + "role": "viewer" + } + ], + "max_ttl_hours": 72, + "allowed_presets": ["small", "medium"], + "resource_quota": { + "requests.cpu": "8", + "requests.memory": "32Gi" + } + }, "gpu": { "enabled": true, "default_profile": "nvidia-l4", @@ -89,7 +133,15 @@ Authenticated clients use `GET /api/v1/capabilities` to discover optional instal } ``` -When GPU support is disabled, `gpu.enabled` is false, `default_profile` is null, and its `profiles` is empty. When custom images are disabled, `images.enabled` is false and its `profiles` is empty. Clients should treat new top-level capabilities as additive. +Only memberships belonging to the current identity appear in `tenancy.tenants`. GPU and +image catalogs are intersected with tenant allowlists. When GPU support is disabled, +`gpu.enabled` is false, `default_profile` is null, and its `profiles` is empty. When custom +images are disabled, `images.enabled` is false and its `profiles` is empty. Clients should +treat new top-level capabilities as additive. + +`GET /api/v1/whoami` returns `user`, stable `subject`, authentication `mode`, active +`tenant`, active `role`, and the same safe membership summaries. In disabled mode, the +backward-compatible identity has tenant `default` and role `operator`. ## Create a devbox @@ -97,6 +149,7 @@ When GPU support is disabled, `gpu.enabled` is false, `default_profile` is null, curl --fail-with-body \ --request POST \ --header "Authorization: Bearer $DEVBOX_TOKEN" \ + --header 'X-Devboxes-Tenant: platform' \ --header 'Content-Type: application/json' \ --data '{ "name": "atlas", @@ -116,12 +169,12 @@ Request fields: | `ttl_hours` | integer | 1 to the configured maximum, default 24 | | `repository` | string or null | `owner/repository` or an HTTPS GitHub repository URL | | `gpu` | object or null | Optional GPU request; omit or use null for CPU-only | -| `gpu.profile` | string or null | Configured profile name; null selects the operator default | +| `gpu.profile` | string or null | Configured profile name; null selects the active tenant's effective default | | `image` | string or null | Operator-approved image profile name or its exact configured image reference | Unknown fields are rejected. Creating an existing name returns `409 Conflict`. Recreating a deleted name reuses its retained PVC, and expands it when the new preset requests more storage. -Request the operator's default GPU profile with an empty nested object: +Request the active tenant's effective default GPU profile with an empty nested object: ```json { @@ -141,6 +194,8 @@ Select a custom image with `"image": "nginx"`. The controller resolves the selec ```json { "name": "atlas", + "tenant": "platform", + "created_by": "alice@example.com", "state": "ready", "preset": "medium", "created_at": "2026-07-13T12:00:00Z", @@ -194,7 +249,11 @@ States are: - `stopped`, the Deployment has zero replicas and the home volume remains. - `degraded`, the pod failed or a known image or restart failure is visible. -Timestamps are RFC 3339 values. `ssh_host`, `ssh_command`, `pod_name`, and `message` can be null while resources converge. `gpu` is null for CPU-only boxes and `image` is null when no custom profile was selected. Both resolved allocations remain stable across stop and start. +Timestamps are RFC 3339 values. `tenant` is the stable application tenant ID and +`created_by` is the authenticated subject recorded at creation. `ssh_host`, `ssh_command`, +`pod_name`, and `message` can be null while resources converge. `gpu` is null for CPU-only +boxes and `image` is null when no custom profile was selected. Both resolved allocations +remain stable across stop and start. ## Lifecycle semantics @@ -217,7 +276,10 @@ Insights is disabled by default. Authenticated reads return an envelope with `en Read endpoints accept `since`, `until`, `box` or `devbox`, `instance_id`, `provider`, `model`, and `repo` or `repository` where applicable. Summary and capabilities accept `group_by=provider|model|box|repository`. Timeseries requires a supported `metric` and accepts `bucket=hour|day`. Activity accepts an opaque `cursor` and a `limit` from 1 to 200. -Export accepts `format=json|csv|sqlite`. JSON and CSV honor the filters. SQLite returns a consistent online backup of the complete database and intentionally ignores summary filters. Never copy only the live database file while write-ahead logging is active. +Export accepts `format=json|csv|sqlite`. JSON and CSV honor the filters. SQLite ignores +summary filters but still returns only the active tenant: the controller creates a +consistent online backup and removes all other-tenant rows before responding. Never copy +only the live database file while write-ahead logging is active. Purge requires exactly one selector: @@ -245,9 +307,9 @@ Validation failures contain a list of structured errors. Common status codes are | Status | Meaning | | ---: | --- | | 401 | Missing or invalid bearer token or browser session | -| 403 | Missing or invalid browser CSRF token | +| 403 | Invalid browser CSRF token, unauthorized tenant, read-only role, or insufficient purge role | | 404 | Devbox does not exist | -| 409 | Devbox name already exists | +| 409 | Devbox name already exists or a direct tenant ResourceQuota admission rejected create/start | | 422 | Invalid path, request field, repository, preset, TTL, disabled feature, or unknown GPU or image profile | | 503 | Controller cannot reach the Kubernetes API through `/ready` | diff --git a/docs/architecture.md b/docs/architecture.md index 6b68bfd..f23bca3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,14 +2,15 @@ Devboxes is a namespaced Kubernetes application with three independently consumable parts: the controller and dashboard image, the workspace image, and the Rust CLI. -## Request path +## Request and tenant resolution path -Automation may authenticate with the master bearer token. Interactive `devbox login` uses +Automation may authenticate with the master or a configured principal bearer token. +Interactive `devbox login` uses the native-app Authorization Code flow with PKCE S256: the CLI binds a numeric loopback callback, opens the external browser, validates state, receives a short-lived one-time code, -and exchanges it for a scoped CLI bearer token. The browser exchanges the master token for -a signed, HTTP-only SameSite session plus a separate CSRF token. The controller never sends -the master access token back to either client. +and exchanges it for a scoped CLI bearer token. The browser exchanges the submitted operator +or principal token for a signed, HTTP-only SameSite session plus a separate CSRF token. The +controller never sends the submitted access token back to either client. Authorization codes live only in a bounded in-memory store. The controller retains a SHA-256 digest, client ID, exact loopback redirect URI, PKCE challenge, subject, and expiry. @@ -22,14 +23,63 @@ with explicit domain separation; an optional dedicated signing key can be suppli existing controller Secret. Rotating the effective signing key revokes all issued CLI tokens. There are no refresh tokens. -The controller translates lifecycle operations into native Kubernetes resources in its own namespace: +Multi-tenancy adds an identity and policy resolution step before application work: + +```text +master, principal, CLI, or browser credential + | + v + live identity lookup + | + X-Devboxes-Tenant or identity default + | + v + membership + role authorization (one tenant) + | + v + tenant namespace + effective policy + scoped operation +``` + +Principal bearer tokens are read from an existing Kubernetes Secret. Browser sessions and +CLI tokens contain a stable principal subject and a credential epoch, not an immutable role +snapshot. Every request resolves current memberships, so a configuration rollout can remove +access without waiting for a CLI token to expire. Unknown and unauthorized tenant IDs both +return the same forbidden response. + +The master token resolves to an installation operator with access to every configured tenant. +When tenancy is disabled, it resolves to a synthetic `default` tenant backed by the release +namespace, preserving the original API and resource behavior. + +The controller translates lifecycle operations into native Kubernetes resources in the +selected tenant namespace: - One `Deployment` per devbox for disposable compute. - One `Service` per devbox for SSH through `LoadBalancer` or `NodePort`. - One `PersistentVolumeClaim` per devbox, mounted at `/home/dev`. - When Insights is enabled, one scoped ingest `Secret` per devbox and one central Insights `PersistentVolumeClaim` for the controller. -Resource names are deterministic (`devbox-NAME`), and labels plus annotations carry controller ownership, creation time, expiry, preset, repository, retained storage size, and any resolved GPU or custom image allocation. +Resource names are deterministic (`devbox-NAME`) inside a namespace. Labels and annotations +carry controller ownership, tenant, creator, creation time, expiry, preset, repository, +retained storage size, and any resolved GPU or custom image allocation. + +## Namespace isolation + +The Helm release keeps one controller ServiceAccount in the release namespace. With tenancy +enabled, Helm binds that account to an identical namespaced Role in each tenant namespace. +It does not receive a ClusterRole. Each tenant gets a separate tokenless workspace +ServiceAccount; workspaces cannot use it to call the Kubernetes API. + +Optional `ResourceQuota` objects provide an aggregate namespace ceiling. Tenant TTL, +presets, GPU profiles, custom images, workspace Secret, ServiceAccount, and StorageClass are +resolved in application policy before Kubernetes admission. For per-person isolation, an +operator creates one tenant per person; a team tenant deliberately shares its inventory and +quota. + +The chart's optional NetworkPolicy selects only managed tenant workspace pods, permits the +internal SSH target port TCP `2222`, and leaves egress open. The SSH Service may expose port +`22`. This blocks other inbound pod ports without breaking SSH, repository access, package +registries, provider APIs, or Insights forwarding. NetworkPolicy enforcement depends on the +cluster CNI. ## GPU resolution path @@ -94,6 +144,12 @@ When Insights is enabled, persistence has two additional layers. A bounded SQLit A UUID instance identity is stored on both the Deployment and home PVC. Retaining and reusing the PVC preserves that identity. Purging the PVC and recreating the box creates a new identity. This separates a box name from the lifetime of the storage that produced its data. +Every central Insights record also carries a tenant ID. Reads, rollups, collector status, +exports, and purge operations require that tenant scope. A SQLite export is produced through +the online backup API and then deletes all rows owned by other tenants before it leaves the +controller. Legacy rows are assigned to the configured default tenant during the database +migration. + ## Readiness A devbox becomes `ready` only when its workspace pod reports Ready and its SSH Service has a usable endpoint. `LoadBalancer` uses the first published IP or hostname, with an optional configured fallback host. `NodePort` combines the configured node host with the port allocated by Kubernetes. @@ -102,8 +158,14 @@ The workspace entrypoint refuses to start without `SSH_AUTHORIZED_KEYS`. It prep ## Security boundaries -- Controller RBAC is a Role scoped to the release namespace; it cannot manage cluster-wide resources. +- With tenancy disabled, controller RBAC is a Role scoped to the release namespace. +- With tenancy enabled, the controller has one RoleBinding in each explicitly configured + tenant namespace, never a ClusterRole. - Workspace service accounts have no RBAC binding and do not mount Kubernetes API tokens. +- Principal credentials are loaded from a read-only Secret mount, compared without exposing + token values, and constrained by live tenant memberships and application roles. +- Tenant clients cannot select a Kubernetes namespace, Secret, ServiceAccount, StorageClass, + quota, preset outside their allowlist, GPU scheduling policy, or custom-image policy. - GPU clients can choose only a configured profile name. They cannot inject images, device resources, RuntimeClasses, supplemental groups, selectors, tolerations, privileged mode, host paths, or device paths. - Custom-image clients can choose only a configured profile name or exact configured image reference. They cannot inject a registry, command, Service, volume, port mapping, resource request, capability, host path, ServiceAccount, or scheduling field. Sidecar containers drop all Linux capabilities and cannot mount Devboxes credentials or persistent storage. - Workspace Secrets are mounted read-only with mode `0440`, scoped to the workspace group, and are not embedded in either image. @@ -111,14 +173,17 @@ The workspace entrypoint refuses to start without `SSH_AUTHORIZED_KEYS`. It prep - The controller runs as a non-root user with a read-only root filesystem and all Linux capabilities dropped. - The workspace runs as root during initialization, then exposes only the unprivileged `dev` SSH user. Password login and root login are disabled. - The trusted `dev` user has passwordless `sudo`. The pod adds only `AUDIT_WRITE`, `CHOWN`, `DAC_OVERRIDE`, `FOWNER`, `SETGID`, `SETUID`, and `SYS_CHROOT`; `AUDIT_WRITE` lets OpenSSH allocate audited PTYs, while `SYS_ADMIN` and privileged mode are not used. -- SSH host checking uses a stable alias scoped to both the Devboxes installation and box name, preventing collisions across installations. +- SSH host checking uses a stable alias scoped to the Devboxes installation, tenant, and box + name, preventing collisions across installations or same-named cross-tenant boxes. - CLI callbacks accept only exact HTTP loopback URIs with numeric loopback hosts, explicit non-privileged ports, and `/callback`; login return targets accept only the internal authorization route. The master controller token is an operator credential. Anyone holding it can create, stop, -delete, or purge every devbox in that installation. A scoped CLI token currently carries the -same lifecycle authority for that single operator, but it expires and cannot mint browser -sessions or new CLI tokens. Devboxes does not currently provide tenant isolation or per-user -authorization. +delete, or purge every devbox in every tenant. A scoped CLI token expires, cannot mint +browser sessions or new CLI tokens, and retains the live identity of the principal that +authorized it. `viewer` is read-only, `member` can manage ordinary lifecycle state, and +`admin` can also purge retained home or Insights data through the application. SSH remains +separate: a viewer whose key is present in the tenant workspace Secret can still change +files inside a running workspace. ## Scheduling and storage diff --git a/docs/cli.md b/docs/cli.md index da45f30..1df4bbd 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -26,6 +26,7 @@ Release binaries support macOS and Linux on AMD64 and ARM64. | --- | --- | | `--url URL` | Override the controller URL for this invocation | | `--token TOKEN` | Override the access token, prefer `DEVBOX_TOKEN` to avoid shell history | +| `--tenant TENANT` | Select one authorized tenant for this invocation | | `--json` | Print machine-readable JSON for supported read and lifecycle commands | | `--help` | Show command help | | `--version` | Show the CLI version | @@ -49,8 +50,8 @@ Unix. The default location is the operating system configuration directory under Values resolve in this order: -1. Command options, `--url` and `--token`. -2. Environment variables, `DEVBOX_URL` and `DEVBOX_TOKEN`. +1. Command options, `--url`, `--token`, and `--tenant`. +2. Environment variables, `DEVBOX_URL`, `DEVBOX_TOKEN`, and `DEVBOX_TENANT`. 3. The saved configuration file. Set `DEVBOX_CONFIG` to select another configuration file. This is useful for multiple clusters and automation: @@ -61,6 +62,19 @@ DEVBOX_CONFIG="$HOME/.config/devbox/lab.toml" devbox list --json Never pass tokens through shared scripts, process listings, logs, or committed files. +When tenancy is enabled, login verifies the selected or default membership and saves the +resolved tenant beside the URL and token. A principal's other memberships remain available +without another login: + +```bash +devbox tenant list +devbox tenant use research +devbox --tenant platform list +DEVBOX_TENANT=research devbox list +``` + +Tenant IDs are normalized to lowercase and validated before an HTTP request is sent. + ## Commands ### `devbox login` @@ -72,7 +86,7 @@ devbox login --url https://devboxes.example.com ``` The default flow never prompts for a token in the terminal. If the browser has no valid -Devboxes session, it first shows the existing operator token login page, then returns to an +Devboxes session, it first shows the access-token login page, then returns to an explicit Approve or Deny decision. Approval issues a scoped token that expires after the controller's configured CLI token lifetime, 30 days by default. No refresh token is issued. @@ -88,7 +102,27 @@ devbox --token "$AUTOMATION_TOKEN" login --url https://devboxes.example.com ``` Prefer `DEVBOX_TOKEN` over `--token` so the credential does not enter shell history. These -paths use the supplied master token directly and do not start a browser callback. +paths use the supplied master or principal token directly and do not start a browser +callback. + +### `devbox tenant` + +List the tenant memberships and roles available to the current identity: + +```bash +devbox tenant list +devbox tenant list --json +``` + +The current tenant is marked in human output. Persist another membership for subsequent +commands only after the server verifies it: + +```bash +devbox tenant use research +``` + +Use global `--tenant` or `DEVBOX_TENANT` for a one-command override. An unauthorized tenant +returns `403` and does not change saved configuration. ### `devbox create NAME` @@ -103,7 +137,7 @@ devbox create atlas --preset medium --ttl 24 --repo owner/project --ssh | `--preset small\|medium\|large` | Resource and storage preset, default `small` | | `--ttl HOURS` | Auto-stop interval from 1 to 168 hours, default 24 | | `--repo OWNER/REPOSITORY` | Clone a GitHub repository on first boot | -| `--gpu` | Request the operator's default GPU profile | +| `--gpu` | Request the active tenant's effective default GPU profile | | `--gpu-profile PROFILE` | Request an exact operator-approved profile; implies `--gpu` | | `--image PROFILE_OR_IMAGE` | Select an operator-approved custom image profile or its exact configured image reference | | `--no-wait` | Return after the API accepts the request | @@ -181,7 +215,9 @@ devbox ssh atlas -- -L 3000:127.0.0.1:3000 devbox ssh atlas -- -A ``` -Arguments after `--` are passed to OpenSSH before the destination. The CLI sets a host-key alias scoped to the controller installation and devbox name, uses `StrictHostKeyChecking=accept-new`, and sends keepalives every 30 seconds. +Arguments after `--` are passed to OpenSSH before the destination. The CLI sets a host-key +alias scoped to the controller installation, tenant, and devbox name, uses +`StrictHostKeyChecking=accept-new`, and sends keepalives every 30 seconds. The workspace preserves a safe, installed client `TERM` entry, including `xterm-ghostty`. Unknown future or malformed values fall back deterministically with one @@ -255,7 +291,10 @@ devbox metrics purge --box atlas devbox metrics purge --box atlas --yes ``` -JSON preserves nullable measurements and response metadata. CSV prefixes spreadsheet formula characters before writing cells. Purge permanently removes central Insights history for every retained instance of the selected box, but it does not delete workspace storage. +JSON preserves nullable measurements and response metadata. CSV prefixes spreadsheet +formula characters before writing cells. Every query and export is restricted to the active +tenant. Purge permanently removes central Insights history for every retained instance of +the selected box in that tenant, but it does not delete workspace storage. ## Output and scripting diff --git a/docs/configuration.md b/docs/configuration.md index 21a42ba..b1cd1bf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -42,6 +42,88 @@ master access token using a versioned, domain-separated HMAC. Set the field only existing controller Secret contains a dedicated key of at least 32 characters. Rotating the master token in derived mode, or rotating the dedicated key, revokes all issued CLI tokens. +## Multi-tenancy + +Multi-tenancy is disabled by default. When enabled, the release namespace holds the +controller while every configured tenant owns a separate workspace namespace. + +| Value | Default | Meaning | +| --- | --- | --- | +| `tenancy.enabled` | `false` | Enable principal authorization and namespace-per-tenant operation | +| `tenancy.defaultTenant` | `default` | Tenant selected for the master identity and legacy workspace/Insights adoption | +| `tenancy.existingPrincipalSecret` | empty | Existing controller-namespace Secret containing configured principal tokens | +| `tenancy.principalTokenMountPath` | `/var/run/secrets/devboxes/principals` | Read-only in-container Secret mount | +| `tenancy.networkPolicy.enabled` | `true` | Allow only TCP/2222 ingress to managed workspace pods in tenant namespaces | +| `tenancy.tenants[].id` | none | Stable lowercase tenant identifier used by API, CLI, labels, and telemetry | +| `tenancy.tenants[].displayName` | none | Human-readable tenant name | +| `tenancy.tenants[].namespace` | none | Dedicated Kubernetes namespace | +| `tenancy.tenants[].createNamespace` | `false` | Create a Helm-retained namespace | +| `tenancy.tenants[].namespaceLabels` | `{}` | Additional labels, including optional Pod Security labels | +| `tenancy.tenants[].workspaceSecretName` | workspace default | Tenant-local Secret mounted into its workspaces | +| `tenancy.tenants[].workspaceServiceAccountName` | workspace default | Tenant-local tokenless ServiceAccount | +| `tenancy.tenants[].storageClass` | workspace default | Tenant-specific StorageClass | +| `tenancy.tenants[].maxTtlHours` | controller maximum | Tenant TTL ceiling, never above the controller maximum | +| `tenancy.tenants[].allowedPresets` | all presets | Non-empty tenant preset allowlist | +| `tenancy.tenants[].allowedGpuProfiles` | installation catalog | GPU allowlist; `[]` permits none | +| `tenancy.tenants[].defaultGpuProfile` | installation default when allowed | Optional tenant GPU default, included in its allowlist | +| `tenancy.tenants[].allowedImageProfiles` | installation catalog | Custom-image allowlist; `[]` permits none | +| `tenancy.tenants[].resourceQuota.enabled` | `true` when present | Render a tenant ResourceQuota | +| `tenancy.tenants[].resourceQuota.hard` | none | Positive native Kubernetes quota quantities | +| `tenancy.principals[].subject` | none | Stable principal identity stored in sessions and creation metadata | +| `tenancy.principals[].displayName` | none | User-facing principal name | +| `tenancy.principals[].tokenKey` | none | Key in `existingPrincipalSecret`, not a credential value | +| `tenancy.principals[].defaultTenant` | first membership | Default selected tenant | +| `tenancy.principals[].memberships[]` | none | Unique tenant and `viewer`, `member`, or `admin` role pairs | + +Example: + +```yaml +tenancy: + enabled: true + defaultTenant: platform + existingPrincipalSecret: devboxes-principals + tenants: + - id: platform + displayName: Platform Engineering + namespace: devboxes-platform + createNamespace: true + workspaceSecretName: devboxes-platform-workspace + maxTtlHours: 72 + allowedPresets: [small, medium] + allowedGpuProfiles: [nvidia-l4] + defaultGpuProfile: nvidia-l4 + allowedImageProfiles: [nginx] + resourceQuota: + enabled: true + hard: + requests.cpu: "8" + requests.memory: 32Gi + requests.storage: 500Gi + count/deployments.apps: "10" + - id: research + displayName: Research + namespace: devboxes-research + createNamespace: true + workspaceSecretName: devboxes-research-workspace + principals: + - subject: alice@example.com + displayName: Alice + tokenKey: alice-token + defaultTenant: platform + memberships: + - tenant: platform + role: admin + - tenant: research + role: viewer +``` + +Tenant IDs, namespaces, principal subjects, Secret keys, memberships, TTL ceilings, quotas, +and profile references receive both JSON-schema and controller startup validation. Principal +tokens must each contain at least 32 characters, must be unique across identities, and are +read only from the mounted Secret. +Read [multi-tenancy](multi-tenancy.md) for provisioning, role semantics, NetworkPolicy +limits, migration, and rollback. + ## Workspace | Value | Default | Meaning | @@ -54,7 +136,10 @@ master token in derived mode, or rotating the dedicated key, revokes all issued | `workspace.imagePullSecret` | empty | Optional pull Secret for private image mirrors | | `workspace.serviceAccount.create` | `true` | Create a tokenless workspace ServiceAccount | -The namespace must permit the workspace pod's documented `sudo` capability set. Kubernetes Pod Security `baseline` is compatible; `restricted` is not. +Every workspace namespace must permit the documented `sudo` capability set. Kubernetes Pod +Security `baseline` is compatible; `restricted` is not. In multi-tenant mode, each namespace +also needs the configured workspace Secret and, when chart creation is disabled, the +configured tokenless ServiceAccount. ## Custom image profiles diff --git a/docs/credentials.md b/docs/credentials.md index fc2c51b..013266d 100644 --- a/docs/credentials.md +++ b/docs/credentials.md @@ -15,12 +15,42 @@ By default, rotating this Secret invalidates saved CLI tokens after the controll restarts. Existing browser sessions also become invalid because session signatures derive from the access token. +The controller token is always the installation-wide `operator` credential. When +multi-tenancy is enabled, reserve it for administration and break-glass recovery rather than +distributing it to tenant users. + An installation may add a dedicated signing key to the same Secret and set `controller.cliSigningKeyKey` to that field. This separates CLI-token rotation from browser session rotation without adding another Secret. The key is optional and must contain at least 32 strong random characters. Rotating whichever effective signing key is configured revokes all issued CLI tokens. +## Tenant principal tokens + +Multi-tenant identities use independent strong tokens from +`tenancy.existingPrincipalSecret` in the controller namespace. Helm values contain only the +Secret name and each principal's `tokenKey`: + +```bash +alice_token="$(openssl rand -hex 32)" +bob_token="$(openssl rand -hex 32)" +kubectl -n devboxes create secret generic devboxes-principals \ + --from-literal=alice-token="$alice_token" \ + --from-literal=bob-token="$bob_token" +unset alice_token bob_token +``` + +Each value must contain at least 32 characters, be unique across principals, and differ from +the master token. The controller mounts the Secret read-only with group-readable mode `0440` +for its dedicated uid/gid and reads tokens at startup. It stores only a short credential +epoch in browser sessions and CLI tokens, so rotating a principal value and restarting the +controller invalidates that principal's existing sessions and CLI credentials without +rotating other identities. + +Principal subjects, display names, membership roles, and Secret key references are +non-secret Helm policy. Never place the token value itself in a values file. Update the +Secret, policy, or both and restart the controller to apply a rotation or membership change. + ## Insights ingest credentials When Insights is enabled, the controller creates one namespaced Secret per workspace. It contains a write-only, HMAC-signed credential scoped to that box name and UUID instance. The workspace sidecar can submit sanitized batches but cannot read Insights, control devboxes, or mint another credential. @@ -29,7 +59,7 @@ By default, the ingest signing key is derived from the controller access token w The generated per-workspace Secret is deleted with workspace compute. Its value is referenced through `secretKeyRef`, not embedded in the Deployment. Retaining the home PVC preserves the workspace instance ID and outbox; a normal recreate issues a fresh credential for that same instance. -## Workspace Secret +## Workspace Secrets Create the minimal Secret with an SSH public key: @@ -40,6 +70,19 @@ kubectl -n devboxes create secret generic devboxes-workspace \ Multiple public keys are accepted in the same authorized-keys file. Keep private SSH keys on clients; Devboxes does not need them. +With tenancy enabled, create a separate Secret in every tenant namespace and reference it +with `tenancy.tenants[].workspaceSecretName`: + +```bash +kubectl -n devboxes-platform create secret generic devboxes-platform-workspace \ + --from-file=SSH_AUTHORIZED_KEYS="$HOME/.ssh/platform_authorized_keys" +kubectl -n devboxes-research create secret generic devboxes-research-workspace \ + --from-file=SSH_AUTHORIZED_KEYS="$HOME/.ssh/research_authorized_keys" +``` + +Do not reuse a credential-bearing workspace Secret across tenants. Namespace scoping then +prevents one tenant's pod specification from referencing another tenant's Secret. + ### GitHub and Git identity ```bash @@ -80,6 +123,10 @@ Credential formats and provider authentication policies can change. Confirm curr ## External secret managers -Point the chart at any existing Secret name. A secret operator only needs to produce the expected keys in the Devboxes namespace. Recommended controls include encryption at rest, short-lived tokens where supported, audit logging, least-privilege provider scopes, and rotation procedures tested before an incident. +Point the chart at existing Secret names. A secret operator must produce controller and +principal Secrets in the release namespace and workspace Secrets in their tenant namespaces. +Recommended controls include encryption at rest, short-lived tokens where supported, audit +logging, least-privilege provider scopes, and rotation procedures tested before an +incident. Never commit rendered Secrets, shell histories containing tokens, account JSON, or private keys to this repository. diff --git a/docs/golden-path.md b/docs/golden-path.md index 62a242f..83841ad 100644 --- a/docs/golden-path.md +++ b/docs/golden-path.md @@ -54,6 +54,15 @@ helm upgrade --install devboxes oci://ghcr.io/vicotrbb/charts/devboxes \ Before the Helm command, create `devboxes-auth` and `devboxes-workspace` through your secret manager or the commands in [credentials](credentials.md). Do not place secret values in the Helm values file. Replace the example source range with the actual trusted client network. Insights remains optional; when enabled, use a low-latency RWO volume with reliable SQLite locking. +For a shared installation, enable [multi-tenancy](multi-tenancy.md) and create one namespace +and workspace Secret per isolation or quota boundary. Give users principal credentials +instead of the master token, then verify the selected context: + +```bash +devbox tenant list +devbox tenant use platform +``` + ## 3. Warm the workspace image when startup latency matters Workspace creation uses `imagePullPolicy: IfNotPresent`. The first devbox scheduled to a node can therefore wait for the workspace image, while later boxes on that node reuse the local image. On clusters you control, pre-pull the exact released workspace image on every eligible node, or use a registry mirror close to the cluster. @@ -129,10 +138,13 @@ For a GPU box, also verify advertised device capacity, profile selectors, taints Use these checks for a slow or pending box: ```bash -kubectl -n devboxes get pod,pvc,service -l devboxes.bonalab.org/name=atlas -o wide -kubectl -n devboxes describe pod -l devboxes.bonalab.org/name=atlas -kubectl -n devboxes get events --sort-by=.lastTimestamp -kubectl top pod -n devboxes -l devboxes.bonalab.org/name=atlas +tenant_namespace=devboxes +kubectl -n "$tenant_namespace" get pod,pvc,service \ + -l devboxes.bonalab.org/name=atlas -o wide +kubectl -n "$tenant_namespace" describe pod \ + -l devboxes.bonalab.org/name=atlas +kubectl -n "$tenant_namespace" get events --sort-by=.lastTimestamp +kubectl top pod -n "$tenant_namespace" -l devboxes.bonalab.org/name=atlas ``` `kubectl top` requires Metrics Server. Storage and node-level metrics depend on the cluster platform. @@ -142,7 +154,9 @@ kubectl top pod -n devboxes -l devboxes.bonalab.org/name=atlas - Keep active repositories and tool caches under `/home/dev`, which persists across stop, delete without `--purge`, and pod replacement. - Use `devbox stop` instead of deleting a box when you plan to resume with the same environment. - Avoid `--purge` unless the home volume is intentionally disposable. -- Keep one controller replica unless you have tested the operational behavior of multiple writers. Kubernetes remains the source of truth, but the supported default is one trusted operator and one controller replica. +- Keep one controller replica unless you have tested the operational behavior of multiple + writers. Kubernetes remains the source of truth; one controller can serve the default + single-operator namespace or every explicitly configured tenant namespace. - When Insights is enabled, keep exactly one controller replica, monitor collector freshness and queue loss, and use the online backup export for its database. - Track image, chart, and CLI versions together, then read the changelog before every upgrade. diff --git a/docs/gpu.md b/docs/gpu.md index 91318e4..b68ec4e 100644 --- a/docs/gpu.md +++ b/docs/gpu.md @@ -173,7 +173,11 @@ Do not work around a Pending box by removing the GPU limit, using privileged mod ## Security model -A GPU device expands what trusted workspace code can exercise on its assigned node. The workspace user has passwordless sudo and can use every device granted to the container, so GPU profiles belong only in the existing trusted single-operator deployment model. +A GPU device expands what trusted workspace code can exercise on its assigned node. The +workspace user has passwordless sudo and can use every device granted to the container, so +GPU profiles are appropriate only for trusted operators and tenant members. A tenant can +select only profiles in the intersection of the installation catalog and its +`allowedGpuProfiles`; an empty tenant allowlist exposes no GPU profile. The GPU feature does not add privileged mode, host PID or network access, hostPath mounts, Kubernetes API credentials, or extra Linux capabilities. Clients cannot choose an image, resource name, RuntimeClass, supplemental group, selector, toleration, or device count. Only an operator changing Helm values can modify those fields. diff --git a/docs/images.md b/docs/images.md index d525ebc..0d4e2ff 100644 --- a/docs/images.md +++ b/docs/images.md @@ -105,7 +105,12 @@ A workspace profile can combine with a GPU profile only when that GPU profile do ## Security and operations -Custom image profiles are appropriate only for Devboxes' existing trusted single-operator model. The operator chooses and reviews every image, resource envelope, pull policy, and port declaration. Users can select a profile but cannot inject a registry, command, Service, volume, host path, capability, resource request, or Kubernetes scheduling field. +Custom image profiles are appropriate only for trusted operators and tenant members. The +operator chooses and reviews every image, resource envelope, pull policy, and port +declaration. A tenant sees only profiles in the intersection of the installation catalog and +its `allowedImageProfiles`; an empty tenant allowlist exposes none. Users can select an +available profile but cannot inject a registry, command, Service, volume, host path, +capability, resource request, or Kubernetes scheduling field. Treat every catalog image as part of the trusted software supply chain: diff --git a/docs/index.md b/docs/index.md index 848fea4..ccfd134 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,6 +8,8 @@ Devboxes turns Kubernetes capacity into persistent, SSH-accessible development e - [Golden path](golden-path.md), the recommended production-shaped setup and fastest daily workflow. - [Configuration](configuration.md), every supported Helm value and deployment mode. - [Credentials](credentials.md), controller authentication and optional workspace accounts. +- [Multi-tenancy](multi-tenancy.md), tenant namespaces, roles, Secrets, quotas, policy + allowlists, migration, and isolation boundaries. ## Use Devboxes @@ -34,4 +36,9 @@ Devboxes turns Kubernetes capacity into persistent, SSH-accessible development e ## Supported scope -Devboxes supports one trusted operator or trusted operator group per installation. The controller is namespaced, each workspace has persistent home storage, each SSH Service uses either `LoadBalancer` or `NodePort`, and GPU devices and custom images are optional operator-owned profiles. Multi-tenant authorization, arbitrary untrusted container execution, browser terminals, cluster-wide GPU driver installation, and automatic PVC deletion are outside the current scope. +Devboxes supports a backward-compatible single-operator mode and an opt-in +namespace-per-tenant mode. Each workspace has persistent home storage, each SSH Service uses +either `LoadBalancer` or `NodePort`, and GPU devices and custom images remain +operator-approved profiles. Arbitrary hostile container execution, browser terminals, +cluster-wide GPU driver installation, and automatic PVC deletion remain outside the +supported scope. diff --git a/docs/insights.md b/docs/insights.md index 4d90484..3d53ded 100644 --- a/docs/insights.md +++ b/docs/insights.md @@ -1,6 +1,9 @@ # Insights -Insights is an opt-in, single-operator view of locally emitted AI metrics and aggregate Git activity. It is available at `/insights`, through `/api/v1/insights/*`, and through `devbox metrics`. It does not reuse the Prometheus `/metrics` route. +Insights is an opt-in view of locally emitted AI metrics and aggregate Git activity. It is +available at `/insights`, through `/api/v1/insights/*`, and through `devbox metrics`. It +does not reuse the Prometheus `/metrics` route. In multi-tenant installations, every ingest, +query, export, and purge is restricted to one authorized tenant. ## Enable Insights @@ -71,12 +74,22 @@ Both agent and controller use an attribute allowlist for provider, model, auth m The pipeline drops email addresses, user, account, and organization identifiers, session IDs, prompts, responses, lengths, commands, paths, file names, URLs, tool inputs and outputs, error bodies, raw API bodies, logs, traces, and arbitrary attributes. -Devboxes remains a single-operator system without secure per-user attribution or tenant isolation. Metrics are self-reported by software running in a workspace controlled by the devbox user, so that user can disable, alter, or fabricate them. Use Insights for personal operational visibility, not compliance, billing, performance evaluation, or adversarial auditing. +Multi-tenancy isolates Insights storage and access by tenant, but metrics remain +self-reported by software running in a workspace controlled by the devbox user. That user +can disable, alter, or fabricate them. `created_by` is operational metadata, not proof that a +specific person produced a metric. Use Insights for operational visibility, not compliance, +billing, performance evaluation, or adversarial auditing. ## Identity, rollout, and key rotation The controller stores a UUID instance ID on both the workspace Deployment and home PVC. Deleting a workspace while retaining its PVC preserves the instance ID. Purging the PVC and recreating the workspace creates a new ID. +With multi-tenancy enabled, every newly issued workspace ingest credential and every central +database row carries the tenant ID. Same-named boxes in different tenants have independent +instance histories. Legacy v1 ingest credentials and database rows are assigned to +`tenancy.defaultTenant`, so select that tenant deliberately and back up the database before +enabling tenancy. + An upgrade does not restart active legacy workspaces. They report `restart_required` until a normal stop and start installs the sidecar. Stopped templates are reconciled immediately, and every start reconciles a stale template before scaling it up. Set `insights.signingKeyKey` to a key in `controller.existingSecret` to use a dedicated ingest signing key. Without it, the controller derives a domain-separated key from the access token. Rotating either effective key invalidates existing ingest credentials. Active workspaces become restart-required, while stopped workspaces receive the new credential immediately. @@ -98,7 +111,10 @@ curl -fsS \ "$DEVBOX_URL/api/v1/insights/export?format=sqlite" ``` -This uses the [SQLite online backup API](https://www.sqlite.org/backup.html). Never copy only a live `.db` file while WAL mode is active. +This uses the [SQLite online backup API](https://www.sqlite.org/backup.html). In a +multi-tenant installation, the returned database is a sanitized backup containing only the +active tenant; rows from all other tenants are deleted and the file is vacuumed before it is +returned. Never copy only a live `.db` file while WAL mode is active. To restore, disable or scale down the controller, replace the database on the Insights PVC with a verified snapshot using an administrative pod, preserve ownership for controller uid and gid 10001, and start the controller. Check `/ready`, the Insights page, and `devbox metrics status` before removing the previous snapshot. diff --git a/docs/multi-tenancy.md b/docs/multi-tenancy.md new file mode 100644 index 0000000..2f33676 --- /dev/null +++ b/docs/multi-tenancy.md @@ -0,0 +1,254 @@ +# Multi-tenancy + +Devboxes can isolate teams or individual users into operator-defined tenants. Multi-tenancy +is opt-in: existing installations keep their single-operator behavior until +`tenancy.enabled=true`. + +Each tenant is both an application authorization boundary and a dedicated Kubernetes +namespace. A request is resolved to exactly one tenant before any workspace, capability, or +Insights operation runs. Workspace names are unique only inside that tenant, so two tenants +may both own a box named `atlas` without sharing compute, storage, SSH identity, credentials, +or telemetry. + +## Isolation contract + +Enabling multi-tenancy provides: + +- one Kubernetes namespace, tokenless workspace ServiceAccount, namespaced controller Role, + and RoleBinding per tenant; +- tenant-local Deployments, Services, PVCs, Insights ingest Secrets, workspace Secrets, and + stable `/home/dev` volumes; +- authenticated principal memberships with `viewer`, `member`, or `admin` roles; +- tenant-filtered lifecycle, capability, dashboard, CLI, and Insights operations; +- optional tenant `ResourceQuota`, StorageClass, TTL ceiling, preset allowlist, GPU-profile + allowlist, and custom-image allowlist; +- an optional ingress NetworkPolicy that permits only the workspace SSH listener on TCP port + 2222 for managed workspace pods; +- tenant and creator metadata on controller-owned resources, plus tenant-scoped SSH host-key + aliases. + +The master controller token remains an installation-wide break-glass operator credential. +It has the synthetic `operator` role in every configured tenant. Principal tokens are +independent credentials loaded from an existing Kubernetes Secret and never stored in Helm +values or Helm release history. + +Devboxes workspaces are development machines with passwordless `sudo`; they are not a +container sandbox for hostile code. Namespace separation, RBAC, quota, and NetworkPolicy are +defense-in-depth around trusted tenant members. Kubernetes admission policy, runtime +hardening, node separation, Secret encryption, and network controls remain cluster-operator +responsibilities. + +## Roles + +| Role | Read workspaces and Insights | Create, start, stop, delete | Purge home or Insights data | +| --- | ---: | ---: | ---: | +| `viewer` | Yes | No | No | +| `member` | Yes | Yes | No | +| `admin` | Yes | Yes | Yes | +| `operator` | Yes | Yes | Yes | + +`operator` is reserved for the master controller identity and is not accepted in principal +membership configuration. Delete without `purge=true` retains the home PVC, so members may +remove compute without permanently removing data. + +Application roles govern the Devboxes API, dashboard, lifecycle, and Insights operations. +They do not grant or revoke a Linux shell: SSH authorization is separately defined by the +public keys in the tenant workspace Secret. In particular, a `viewer` who also has an +authorized SSH key can modify files inside a running workspace even though they cannot mutate +its Kubernetes lifecycle through Devboxes. + +Roles apply to the entire selected tenant. For strict per-person ownership and quota, create +one tenant namespace per person. A team tenant intentionally shares its workspace inventory +and aggregate quota among all members. + +## Configure Helm + +Create tenant namespaces and credentials before the upgrade, or set +`createNamespace: true` to let Helm create and retain the namespaces. The controller namespace +is still the Helm release namespace. + +```yaml +tenancy: + enabled: true + defaultTenant: platform + existingPrincipalSecret: devboxes-principals + networkPolicy: + enabled: true + tenants: + - id: platform + displayName: Platform Engineering + namespace: devboxes-platform + createNamespace: true + namespaceLabels: + owner: platform + pod-security.kubernetes.io/enforce: baseline + workspaceSecretName: devboxes-platform-workspace + workspaceServiceAccountName: devboxes-workspace + storageClass: fast-rwo + maxTtlHours: 72 + allowedPresets: [small, medium] + allowedGpuProfiles: [nvidia-l4] + defaultGpuProfile: nvidia-l4 + allowedImageProfiles: [nginx] + resourceQuota: + enabled: true + hard: + requests.cpu: "8" + requests.memory: 32Gi + requests.storage: 500Gi + count/deployments.apps: "10" + count/pods: "10" + - id: research + displayName: Research + namespace: devboxes-research + createNamespace: true + workspaceSecretName: devboxes-research-workspace + maxTtlHours: 168 + allowedPresets: [small, medium, large] + principals: + - subject: alice@example.com + displayName: Alice + tokenKey: alice-token + defaultTenant: platform + memberships: + - tenant: platform + role: admin + - tenant: research + role: viewer +``` + +Omitting `allowedGpuProfiles` or `allowedImageProfiles` inherits the installation catalog. +An explicit empty list allows none. `defaultGpuProfile` overrides the installation default +for that tenant and must appear in its GPU allowlist. If neither the tenant default nor the +installation default is available, users must select an explicit allowed profile. +`allowedPresets` must contain at least one preset. `maxTtlHours` may narrow, but never exceed, +`controller.maxTtlHours`. Quota quantities use native Kubernetes `ResourceQuota` keys and +syntax. + +The chart validates duplicate tenant IDs, duplicate namespaces, memberships, token keys, +defaults, TTLs, and catalog references before rendering. `createNamespace: true` namespaces +carry `helm.sh/resource-policy: keep`, so uninstalling Devboxes cannot silently delete all +tenant PVCs with their namespace. + +## Provision Secrets + +Every tenant needs its own workspace Secret in its own namespace. This keeps SSH public keys +and optional GitHub or AI-provider credentials out of other tenant pods. + +```bash +kubectl create namespace devboxes-platform +kubectl -n devboxes-platform create secret generic devboxes-platform-workspace \ + --from-file=SSH_AUTHORIZED_KEYS="$HOME/.ssh/platform_authorized_keys" +``` + +Create principal tokens in one Secret in the controller namespace. Each value must be unique, +different from the master token, and contain at least 32 strong random characters. + +```bash +alice_token="$(openssl rand -hex 32)" +kubectl -n devboxes create secret generic devboxes-principals \ + --from-literal=alice-token="$alice_token" +unset alice_token +``` + +The key must match the principal's `tokenKey`. The chart mounts the Secret read-only for +controller uid/gid 10001 with group-readable mode `0440`; the controller reads it at startup. +Restart the controller after changing token data or principal memberships. + +If `workspace.serviceAccount.create=false`, provision every configured +`workspaceServiceAccountName` in its tenant namespace. It should have no RoleBinding and +`automountServiceAccountToken: false`. + +## Authenticate and select a tenant + +Principal tokens work with the browser login page, direct bearer authentication, and +headless CLI login. The browser session and issued CLI token retain the principal identity +but do not hard-code one tenant; live membership is checked on every request. + +```bash +DEVBOX_TOKEN="$ALICE_DEVBOXES_TOKEN" \ + devbox login --url https://devboxes.example.com --tenant platform + +devbox tenant list +devbox tenant use research +devbox --tenant platform list +DEVBOX_TENANT=research devbox list +``` + +CLI tenant precedence is `--tenant`, `DEVBOX_TENANT`, then the saved configuration. A +successful login saves the server-selected tenant. `devbox tenant use` verifies membership +before changing the saved value. + +Raw API clients select a tenant with `X-Devboxes-Tenant`. Omitting it uses the identity's +configured default: + +```bash +curl --fail-with-body \ + -H "Authorization: Bearer $ALICE_DEVBOXES_TOKEN" \ + -H "X-Devboxes-Tenant: platform" \ + https://devboxes.example.com/api/v1/devboxes +``` + +An unknown or unauthorized tenant returns `403` without revealing whether the tenant exists. +The dashboard uses a tenant switcher and keeps the tenant in navigation links and API +requests. Viewer sessions render lifecycle controls as unavailable instead of relying only +on a server-side error. + +## Quota and policy behavior + +The create form, authenticated documentation, capabilities API, and CLI catalogs expose only +the selected tenant's effective presets, TTL ceiling, GPU profiles, and custom images. +Server-side validation repeats those checks before any Kubernetes write. + +Kubernetes remains authoritative for aggregate quota. A direct `ResourceQuota` admission +denial during create or start returns `409 Conflict`. Pod-level quota enforcement can occur +asynchronously in the Deployment controller; in that case the workspace becomes +`degraded` with the quota diagnostic instead of falsely reporting readiness. The dashboard +shows configured hard limits, while `kubectl describe resourcequota -n TENANT_NAMESPACE` +shows current usage. + +The default tenant NetworkPolicy applies only to controller-managed workspace pods and +allows ingress only on the workspace SSH target port, TCP `2222`, from any source that can +already reach the namespace. The external Service may still expose port `22`. This preserves +SSH behavior while blocking arbitrary pod-port ingress. Egress remains open for source +repositories, package registries, provider APIs, DNS, and the controller Insights service. +Enforcement requires a NetworkPolicy-capable CNI. Add cluster-specific source restrictions +or egress policies separately when required. + +## Insights and retained data + +Every Insights instance, metric point, Git aggregate, rollup, collector status, query, +export, and purge carries a tenant ID. JSON and CSV exports are filtered in the query. +SQLite export creates an online backup and removes every other tenant before returning it. +An administrator can purge only the active tenant. + +Upgrading an existing Insights database adds tenant ownership and assigns legacy rows to +`tenancy.defaultTenant`. Choose that default deliberately before enabling multi-tenancy. +Back up the database first and verify legacy history through that tenant after rollout. + +Workspace PVCs and SSH host keys are namespaced. The CLI host-key alias includes the +installation, tenant, and box name, preventing a same-named workspace in another tenant from +reusing the local known-host entry. + +## Rollout and rollback + +Before enabling tenancy: + +1. Back up workspace and Insights PVCs. +2. Choose the default tenant that will own existing workspace and Insights records. To adopt + release-namespace workspaces in place, keep that tenant in the current Helm release + namespace and leave `createNamespace: false`; otherwise migrate those resources explicitly. +3. Create every tenant namespace and workspace Secret, or configure Helm to create retained + namespaces. +4. Create the principal-token Secret and verify unique values. +5. Render the chart and inspect Namespace, Role, RoleBinding, ServiceAccount, + ResourceQuota, and NetworkPolicy resources. +6. Upgrade, wait for controller readiness, and verify each identity with `devbox tenant + list`. +7. Exercise viewer, member, admin, cross-tenant denial, quota rejection, SSH persistence, + and tenant-filtered Insights export. + +Do not disable multi-tenancy while tenant namespaces still contain workspaces you expect the +single-tenant controller to manage. Disabled mode intentionally returns to the release +namespace and does not merge or move resources. Inventory, back up, and migrate retained +PVCs explicitly before changing that boundary. diff --git a/docs/operations.md b/docs/operations.md index 484578e..9c0567a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,6 +1,8 @@ # Operations -This runbook covers the supported single-operator deployment model. Keep installation values, chart version, controller image, workspace image, and CLI version aligned. +This runbook covers both the default single-operator deployment and opt-in +namespace-per-tenant operation. Keep installation values, chart version, controller image, +workspace image, and CLI version aligned. ## Routine health check @@ -12,6 +14,15 @@ kubectl logs deployment/devboxes -n devboxes --tail=200 ./scripts/verify-install.sh ``` +For a multi-tenant installation, also check every configured namespace and identity: + +```bash +kubectl get resourcequota,networkpolicy,serviceaccount -n devboxes-platform +kubectl get deployment,service,pvc,pod -n devboxes-platform -o wide +devbox tenant list +devbox --tenant platform list +``` + `/health` proves that the HTTP process responds. `/ready` performs a namespaced Deployment list against the Kubernetes API and, when enabled, verifies the Insights database. It returns 503 when either required dependency is unavailable. Neither endpoint creates resources. ## Metrics and logs @@ -44,6 +55,15 @@ For custom image profiles, add each sidecar's configured CPU and memory request Profile selectors and tolerations are policy, not capacity detection. Before publishing a profile, prove that at least one node matches all selectors, tolerates the intended taints, advertises the exact resource, and can pull the profile image. Document whether each profile is dedicated or shared. +In multi-tenant mode, plan capacity from each namespace's `ResourceQuota` and the +intersection of its preset, GPU, and image allowlists. Quota is aggregate within a tenant. +Use a user-per-tenant topology when each person requires an independent ceiling. Inspect +live consumption with: + +```bash +kubectl describe resourcequota -n devboxes-platform +``` + ## Backups and restore The home PVC is the durable state. Back up important PVCs with a CSI snapshot system or storage-provider backup that is compatible with the active StorageClass. @@ -88,6 +108,15 @@ helm upgrade devboxes oci://ghcr.io/vicotrbb/charts/devboxes \ Then run `scripts/verify-install.sh`, confirm `/ready`, list existing boxes, create a disposable smoke box, connect over SSH, stop it, start it, and delete it with purge. +Before first enabling multi-tenancy, choose which tenant owns existing release-namespace +workspaces and legacy Insights rows. The database migration assigns legacy Insights records +to `tenancy.defaultTenant`, but Kubernetes workloads and PVCs are not moved between +namespaces. Keep the default tenant in the release namespace to adopt those resources in +place, or back them up and migrate them explicitly. Provision each tenant workspace Secret +and principal token before rollout, render all namespace-scoped RBAC and quota objects, then +test every role and cross-tenant denial. Follow the complete +[multi-tenancy rollout](multi-tenancy.md#rollout-and-rollback). + When GPU configuration or images change, render the profile JSON before applying, run `devbox gpu profiles` after rollout, and create a disposable box for every changed profile. Verify the vendor diagnostic inside the box, stop and start it, and confirm the same profile contract remains. Stopping releases live device capacity, so Kubernetes may select a different physical device on start. Existing GPU Deployments retain their resolved snapshots; a Helm profile edit affects only later creations. Plan migrations as explicit delete and recreate operations, with a separate PVC retention decision. When custom image profiles change, render the catalog before applying, run `devbox image profiles`, and create a disposable test box for every changed profile. For a sidecar, verify a declared non-root image user, a port from 1024 through 65535, the container list, no unexpected Secret or PVC mount, application health over pod loopback, SSH tunneling, stop and start, and cleanup. For a workspace-mode profile, also verify SSH readiness, persistent-home reuse, repository bootstrap, and Insights behavior if enabled. Existing Deployments retain their creation-time resolved snapshot; changing or disabling a Helm profile rejects new requests but does not rewrite existing boxes. Retire a profile only after inventorying `devbox list --json` and making an explicit delete or migration plan. @@ -122,6 +151,11 @@ kubectl rollout status deployment/devboxes -n devboxes Rotation invalidates browser sessions and saved CLI tokens. Log in again after the rollout. Rotate immediately after suspected disclosure, then review controller access logs available at the ingress or network boundary. +To rotate only one principal, update its key in `tenancy.existingPrincipalSecret` and restart +the controller. That principal's browser sessions and CLI tokens fail their credential-epoch +check after restart; other principals remain valid. Membership changes also require a Helm +rollout and take effect on every subsequent request. + When `insights.signingKeyKey` is empty, the same controller-token rotation also rotates the domain-separated ingest signing key. Active workspaces need a normal stop and start to receive a new scoped credential. With a dedicated Insights signing key, rotate that field independently and follow the same workspace reconciliation procedure. Rotate workspace provider tokens independently through the workspace Secret. Existing files copied into persistent homes are not overwritten automatically, so revoke compromised provider sessions at the provider and update affected homes explicitly. @@ -137,6 +171,11 @@ kubectl get deployment,service,pvc -n devboxes \ -l app.kubernetes.io/managed-by=devboxes-controller ``` +For tenancy, repeat the inventory in every configured tenant namespace. Helm-created tenant +namespaces use a keep policy and are not removed with the release. Delete a tenant namespace +only after a separate backup and retention decision, because namespace deletion also removes +its retained PVCs and Secrets. + Back up required PVCs, delete boxes through the CLI, and use `--purge` only for volumes approved for permanent removal. Verify the namespace contents before deleting the namespace. If `insights.storage.retainOnDelete=true`, the chart-created central Insights PVC remains after uninstall. Per-workspace outboxes remain on retained home PVCs. Remove either data layer only after a separate backup and retention decision. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 60cc18c..9cff862 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -26,12 +26,17 @@ kubectl describe pod -n devboxes \ Symptoms include `/ready` returning 503, CLI transport failures, or the dashboard failing to list boxes. 1. Confirm the controller pod is Ready and inspect its logs. -2. Verify the ServiceAccount, Role, and RoleBinding exist in the same namespace. +2. Verify the controller ServiceAccount exists in the release namespace and a Role plus + RoleBinding exists in every configured tenant namespace. 3. Check Kubernetes API reachability and service-account token mounting on the controller. -4. Confirm the configured namespace matches the Helm release namespace. +4. Confirm every configured tenant namespace exists and the RoleBinding references the + controller ServiceAccount in the release namespace. 5. Review NetworkPolicy or API-server authorization denials. -The controller needs namespaced access to Deployments, Services, Pods, PVCs, and its generated per-workspace Insights Secrets. It does not need cluster-wide RBAC. +The controller needs namespaced access to Deployments, Services, Pods, PVCs, and its +generated per-workspace Insights Secrets. It does not need cluster-wide RBAC. Readiness lists +Deployments in every configured tenant, so one missing RoleBinding deliberately makes the +controller not ready. If Insights is enabled, inspect the central PVC and controller logs for SQLite open, migration, or locking errors. The volume must support normal filesystem locking and WAL. Avoid NFS-backed claims with unreliable lock semantics. @@ -53,6 +58,15 @@ master token into terminal output as a workaround. For browser `403 Missing or invalid CSRF token`, clear the site cookies and log in again. Confirm that an HTTPS installation sets `controller.cookieSecure=true`. A secure cookie is not sent over plain HTTP. +For `403 Tenant access is forbidden`, list the principal's configured memberships, verify +`X-Devboxes-Tenant`, `--tenant`, or `DEVBOX_TENANT`, and restart the controller after a +principal Secret or Helm-policy change. Unknown tenants and known-but-unauthorized tenants +intentionally return the same error. + +For `403 The current tenant role is read-only`, select a membership with `member` or `admin` +authority or ask the operator to change the role. Permanent home or Insights purge requires +`admin`; the master identity has `operator` authority in every tenant. + ## TLS or URL validation fails The CLI rejects non-loopback HTTP, embedded credentials, query strings, and fragments. Use a valid certificate for the controller hostname. There is intentionally no skip-verification option. @@ -81,6 +95,24 @@ kubectl get secret devboxes-workspace -n devboxes \ If the pod is Ready but the box still starts, inspect the Service next. +## Tenant quota rejects create or start + +A direct Kubernetes `ResourceQuota` admission denial is returned as `409 Conflict`. +Pod-level enforcement may instead appear asynchronously as a `degraded` workspace with a +quota message from its Deployment. In either case, inspect the active tenant's configured +ceiling and current usage: + +```bash +devbox --tenant platform list +kubectl describe resourcequota -n devboxes-platform +kubectl get events -n devboxes-platform --sort-by=.lastTimestamp +``` + +Stopping a box releases pod CPU, memory, and GPU requests but retains its PVC and storage +quota. Delete without purge also retains the PVC. Increase quota, stop or purge an approved +box, or select a smaller allowed preset based on the constrained resource. Do not bypass +quota by moving resources manually to another tenant namespace. + ## A GPU box remains starting Start with the user-visible allocation and scheduler message: diff --git a/scripts/fixtures/tenancy-values.yaml b/scripts/fixtures/tenancy-values.yaml new file mode 100644 index 0000000..1fc8894 --- /dev/null +++ b/scripts/fixtures/tenancy-values.yaml @@ -0,0 +1,44 @@ +tenancy: + enabled: true + defaultTenant: platform + existingPrincipalSecret: devboxes-principals + tenants: + - id: platform + displayName: Platform Engineering + namespace: devboxes-platform + createNamespace: true + namespaceLabels: + owner: platform + workspaceSecretName: devboxes-platform-workspace + maxTtlHours: 72 + allowedPresets: [small, medium] + allowedGpuProfiles: [] + allowedImageProfiles: [] + resourceQuota: + enabled: true + hard: + requests.cpu: "8" + requests.memory: 32Gi + requests.storage: 500Gi + count/deployments.apps: "10" + count/pods: "10" + - id: research + displayName: Research + namespace: devboxes-research + createNamespace: true + workspaceSecretName: devboxes-research-workspace + maxTtlHours: 168 + allowedPresets: [small, medium, large] + principals: + - subject: alice@example.com + displayName: Alice + tokenKey: alice-token + defaultTenant: platform + memberships: + - tenant: platform + role: admin + - tenant: research + role: viewer + +insights: + enabled: true diff --git a/scripts/kind-e2e.sh b/scripts/kind-e2e.sh index cf1a410..6b113ee 100755 --- a/scripts/kind-e2e.sh +++ b/scripts/kind-e2e.sh @@ -3,6 +3,7 @@ set -Eeuo pipefail cluster="${DEVBOXES_E2E_CLUSTER:-devboxes-e2e}" namespace="${DEVBOXES_NAMESPACE:-devboxes}" +tenant_namespace="${DEVBOXES_E2E_TENANT_NAMESPACE:-${namespace}-research}" controller_port="${DEVBOXES_E2E_CONTROLLER_PORT:-18000}" ssh_port="${DEVBOXES_E2E_SSH_PORT:-12222}" custom_image_ssh_port="${DEVBOXES_E2E_CUSTOM_IMAGE_SSH_PORT:-12223}" @@ -13,6 +14,7 @@ released_cli="${DEVBOXES_E2E_CLI:-}" interactive_ssh="${DEVBOXES_E2E_INTERACTIVE_SSH:-1}" workspace_timeout="${DEVBOXES_E2E_WORKSPACE_TIMEOUT:-3m}" token="e2e-access-token-at-least-32-characters" +project_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" controller_port_forward="" ssh_port_forward="" @@ -43,13 +45,23 @@ if [[ "$interactive_ssh" != 0 && "$interactive_ssh" != 1 ]]; then fi previous_context="$(kubectl config current-context 2>/dev/null || true)" +report_failure() { + status="$?" + printf 'E2E command failed at line %s (exit %s): %s\n' \ + "${BASH_LINENO[0]}" "$status" "$BASH_COMMAND" >&2 + return "$status" +} +trap report_failure ERR + cleanup() { status="$1" - trap - EXIT INT TERM + trap - ERR EXIT INT TERM if [[ "$status" -ne 0 ]]; then printf '\nE2E diagnostics:\n' >&2 kubectl --context "kind-$cluster" get all,pvc -n "$namespace" -o wide >&2 || true kubectl --context "kind-$cluster" describe pods -n "$namespace" >&2 || true + kubectl --context "kind-$cluster" get all,pvc -n "$tenant_namespace" -o wide >&2 || true + kubectl --context "kind-$cluster" describe pods -n "$tenant_namespace" >&2 || true kubectl --context "kind-$cluster" logs -n "$namespace" deployment/devboxes --tail=200 >&2 || true kubectl --context "kind-$cluster" logs -n "$namespace" deployment/devbox-smoke \ -c insights-agent --tail=200 >&2 || true @@ -986,9 +998,287 @@ else fi kubectl -n "$namespace" wait --for=delete pvc/devbox-smoke-home --timeout=2m +if [[ -z "$published_version" ]]; then + alice_token="alice-viewer-token-at-least-32-characters" + bob_token="bob-admin-token-at-least-32-characters" + + api \ + -H 'Content-Type: application/json' \ + -d '{"name":"legacy-adopt","preset":"small","ttl_hours":4}' \ + "http://127.0.0.1:$controller_port/api/v1/devboxes" >/dev/null + legacy_tenant_label="$( + kubectl -n "$namespace" get deployment devbox-legacy-adopt -o json \ + | jq -r '.metadata.labels["devboxes.bonalab.org/tenant"] // ""' + )" + test -z "$legacy_tenant_label" + + kubectl create namespace "$tenant_namespace" >/dev/null + kubectl -n "$tenant_namespace" create secret generic research-workspace \ + --from-file=SSH_AUTHORIZED_KEYS="$temporary_directory/id_ed25519.pub" >/dev/null + kubectl -n "$namespace" create secret generic devboxes-principals \ + --from-literal=alice-token="$alice_token" \ + --from-literal=bob-token="$bob_token" >/dev/null + + helm upgrade devboxes "$project_directory/charts/devboxes" \ + --namespace "$namespace" \ + --reuse-values \ + --set tenancy.enabled=true \ + --set-string tenancy.defaultTenant=platform \ + --set-string tenancy.existingPrincipalSecret=devboxes-principals \ + --set-string tenancy.tenants[0].id=platform \ + --set-string 'tenancy.tenants[0].displayName=Platform' \ + --set-string "tenancy.tenants[0].namespace=$namespace" \ + --set-string tenancy.tenants[0].workspaceSecretName=devboxes-workspace \ + --set-string tenancy.tenants[1].id=research \ + --set-string 'tenancy.tenants[1].displayName=Research' \ + --set-string "tenancy.tenants[1].namespace=$tenant_namespace" \ + --set-string tenancy.tenants[1].workspaceSecretName=research-workspace \ + --set-string tenancy.principals[0].subject=alice \ + --set-string 'tenancy.principals[0].displayName=Alice' \ + --set-string tenancy.principals[0].tokenKey=alice-token \ + --set-string tenancy.principals[0].defaultTenant=platform \ + --set-string tenancy.principals[0].memberships[0].tenant=platform \ + --set-string tenancy.principals[0].memberships[0].role=viewer \ + --set-string tenancy.principals[1].subject=bob \ + --set-string 'tenancy.principals[1].displayName=Bob' \ + --set-string tenancy.principals[1].tokenKey=bob-token \ + --set-string tenancy.principals[1].defaultTenant=research \ + --set-string tenancy.principals[1].memberships[0].tenant=research \ + --set-string tenancy.principals[1].memberships[0].role=admin >/dev/null + kubectl -n "$namespace" rollout status deployment/devboxes --timeout=3m + start_controller_port_forward + + controller_deployment="$(kubectl -n "$namespace" get deployment devboxes -o json)" + jq -e ' + any(.spec.template.spec.volumes[]; + .name == "principal-tokens" + and .secret.secretName == "devboxes-principals" + and .secret.defaultMode == 288) + and .spec.template.spec.securityContext.fsGroup == 10001 + ' <<<"$controller_deployment" >/dev/null + kubectl auth can-i \ + --as "system:serviceaccount:${namespace}:devboxes" \ + get deployments.apps \ + -n "$tenant_namespace" | grep -Fx yes >/dev/null + kubectl -n "$tenant_namespace" get networkpolicy devboxes-workspace-ingress -o json \ + | jq -e ' + .spec.podSelector.matchLabels["app.kubernetes.io/managed-by"] + == "devboxes-controller" + and .spec.ingress[0].ports == [{"port": 2222, "protocol": "TCP"}] + ' >/dev/null + test "$( + kubectl -n "$tenant_namespace" get serviceaccount devboxes-workspace \ + -o jsonpath='{.automountServiceAccountToken}' + )" = false + + operator_identity="$( + api \ + -H 'X-Devboxes-Tenant: research' \ + "http://127.0.0.1:$controller_port/api/v1/whoami" + )" + jq -e ' + .tenant == "research" + and .role == "operator" + and ([.tenants[].id] | sort) == ["platform", "research"] + ' <<<"$operator_identity" >/dev/null + api \ + -H 'X-Devboxes-Tenant: platform' \ + "http://127.0.0.1:$controller_port/api/v1/devboxes/legacy-adopt" \ + | jq -e '.tenant == "platform"' >/dev/null + + tenant_api() { + local principal_token="$1" + local selected_tenant="$2" + shift 2 + curl -fsS \ + -H "Authorization: Bearer $principal_token" \ + -H "X-Devboxes-Tenant: $selected_tenant" \ + "$@" + } + + alice_identity="$( + tenant_api "$alice_token" platform \ + "http://127.0.0.1:$controller_port/api/v1/whoami" + )" + jq -e ' + .subject == "alice" + and .tenant == "platform" + and .role == "viewer" + and [.tenants[].id] == ["platform"] + ' <<<"$alice_identity" >/dev/null + bob_identity="$( + tenant_api "$bob_token" research \ + "http://127.0.0.1:$controller_port/api/v1/whoami" + )" + jq -e ' + .subject == "bob" + and .tenant == "research" + and .role == "admin" + and [.tenants[].id] == ["research"] + ' <<<"$bob_identity" >/dev/null + + unauthorized_status="$( + curl -sS \ + -o "$temporary_directory/unauthorized-tenant.json" \ + -w '%{http_code}' \ + -H "Authorization: Bearer $alice_token" \ + -H 'X-Devboxes-Tenant: research' \ + "http://127.0.0.1:$controller_port/api/v1/whoami" + )" + unknown_status="$( + curl -sS \ + -o "$temporary_directory/unknown-tenant.json" \ + -w '%{http_code}' \ + -H "Authorization: Bearer $alice_token" \ + -H 'X-Devboxes-Tenant: missing' \ + "http://127.0.0.1:$controller_port/api/v1/whoami" + )" + test "$unauthorized_status" = 403 + test "$unknown_status" = 403 + cmp -s \ + "$temporary_directory/unauthorized-tenant.json" \ + "$temporary_directory/unknown-tenant.json" + + viewer_create_status="$( + curl -sS \ + -o "$temporary_directory/viewer-create.json" \ + -w '%{http_code}' \ + -H "Authorization: Bearer $alice_token" \ + -H 'X-Devboxes-Tenant: platform' \ + -H 'Content-Type: application/json' \ + -d '{"name":"viewer-denied"}' \ + "http://127.0.0.1:$controller_port/api/v1/devboxes" + )" + test "$viewer_create_status" = 403 + + api \ + -H 'X-Devboxes-Tenant: platform' \ + -H 'Content-Type: application/json' \ + -d '{"name":"tenant-twin","preset":"small","ttl_hours":4}' \ + "http://127.0.0.1:$controller_port/api/v1/devboxes" >/dev/null + tenant_api "$bob_token" research \ + -H 'Content-Type: application/json' \ + -d '{"name":"tenant-twin","preset":"small","ttl_hours":4}' \ + "http://127.0.0.1:$controller_port/api/v1/devboxes" >/dev/null + kubectl -n "$namespace" rollout status deployment/devbox-tenant-twin \ + --timeout="$workspace_timeout" + kubectl -n "$tenant_namespace" rollout status deployment/devbox-tenant-twin \ + --timeout="$workspace_timeout" + test "$( + kubectl -n "$namespace" get deployment devbox-tenant-twin -o json \ + | jq -r '.metadata.labels["devboxes.bonalab.org/tenant"]' + )" = platform + research_deployment="$(kubectl -n "$tenant_namespace" get deployment devbox-tenant-twin -o json)" + jq -e ' + .metadata.labels["devboxes.bonalab.org/tenant"] == "research" + and .metadata.annotations["devboxes.bonalab.org/created-by"] == "bob" + and ( + .spec.template.spec.containers[0].env + | any(.name == "DEVBOX_TENANT" and .value == "research") + ) + ' <<<"$research_deployment" >/dev/null + + if [[ -n "$released_cli" ]]; then + DEVBOX_URL="http://127.0.0.1:$controller_port" \ + DEVBOX_TOKEN="$bob_token" \ + DEVBOX_CONFIG="$temporary_directory/tenant-cli-config.toml" \ + "$released_cli" --tenant research --json tenant list \ + | jq -e ' + .current == "research" + and [.tenants[].id] == ["research"] + and [.tenants[].role] == ["admin"] + ' >/dev/null + fi + + write_otlp_fixture "$temporary_directory/tenant-codex-otlp.json" codex 11 11 + for _ in {1..30}; do + if kubectl -n "$tenant_namespace" exec -i deployment/devbox-tenant-twin -c devbox -- \ + curl -fsS \ + -H 'Content-Type: application/json' \ + --data-binary @- \ + http://127.0.0.1:4318/v1/metrics \ + <"$temporary_directory/tenant-codex-otlp.json" >/dev/null; then + break + fi + sleep 1 + done + research_summary="" + for _ in {1..45}; do + research_summary="$( + tenant_api "$bob_token" research \ + "http://127.0.0.1:$controller_port/api/v1/insights/summary?since=24h&box=tenant-twin" + )" + if jq -e ' + .tenant == "research" + and .data.ai.totals.tokens == 11 + ' <<<"$research_summary" >/dev/null; then + break + fi + sleep 1 + done + jq -e '.tenant == "research" and .data.ai.totals.tokens == 11' \ + <<<"$research_summary" >/dev/null + platform_summary="$( + tenant_api "$alice_token" platform \ + "http://127.0.0.1:$controller_port/api/v1/insights/summary?since=24h&box=tenant-twin" + )" + jq -e '.tenant == "platform" and .data.ai.totals.tokens == 0' \ + <<<"$platform_summary" >/dev/null + + tenant_api "$bob_token" research \ + -o "$temporary_directory/research-insights.db" \ + "http://127.0.0.1:$controller_port/api/v1/insights/export?format=sqlite" + python3 - "$temporary_directory/research-insights.db" <<'PY' +import sqlite3 +import sys + +connection = sqlite3.connect(sys.argv[1]) +try: + assert connection.execute("PRAGMA integrity_check").fetchone()[0] == "ok" + tenants = { + row[0] + for row in connection.execute( + "SELECT DISTINCT tenant_id FROM devbox_instances" + ) + } + assert tenants == {"research"}, tenants +finally: + connection.close() +PY + + viewer_purge_status="$( + curl -sS \ + -o "$temporary_directory/viewer-purge.json" \ + -w '%{http_code}' \ + -X DELETE \ + -H "Authorization: Bearer $alice_token" \ + -H 'X-Devboxes-Tenant: platform' \ + "http://127.0.0.1:$controller_port/api/v1/insights?box=tenant-twin" + )" + test "$viewer_purge_status" = 403 + tenant_api "$bob_token" research \ + -X DELETE \ + "http://127.0.0.1:$controller_port/api/v1/insights?box=tenant-twin" >/dev/null + tenant_api "$bob_token" research \ + -X DELETE \ + "http://127.0.0.1:$controller_port/api/v1/devboxes/tenant-twin?purge=true" >/dev/null + api \ + -X DELETE \ + -H 'X-Devboxes-Tenant: platform' \ + "http://127.0.0.1:$controller_port/api/v1/devboxes/tenant-twin?purge=true" >/dev/null + api \ + -X DELETE \ + -H 'X-Devboxes-Tenant: platform' \ + "http://127.0.0.1:$controller_port/api/v1/devboxes/legacy-adopt?purge=true" >/dev/null + kubectl -n "$tenant_namespace" wait --for=delete deployment/devbox-tenant-twin \ + --timeout=2m + kubectl -n "$tenant_namespace" wait --for=delete pvc/devbox-tenant-twin-home \ + --timeout=2m +fi + if [[ -n "$published_version" ]]; then printf 'Verified published %s chart, images, and CLI through clean install, Insights ingest, Git aggregation, deduplication, durable outboxes, central restart, API, PVC, NodePort %s, SSH, retain, reuse, and explicit purge.\n' \ "$published_version" "$node_port" else - printf 'Verified clean install, Insights ingest, Git aggregation, deduplication, durable outboxes, central restart, API, PVC, NodePort %s, SSH, retain, reuse, and explicit purge.\n' "$node_port" + printf 'Verified clean install, Insights ingest, Git aggregation, deduplication, durable outboxes, central restart, API, PVC, NodePort %s, SSH, retain, reuse, explicit purge, opt-in tenancy upgrade, role enforcement, same-name isolation, CLI selection, and tenant-sanitized SQLite export.\n' "$node_port" fi diff --git a/scripts/test-helm-tenancy.sh b/scripts/test-helm-tenancy.sh new file mode 100755 index 0000000..be8214d --- /dev/null +++ b/scripts/test-helm-tenancy.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +project_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +chart="$project_directory/charts/devboxes" +values="$project_directory/scripts/fixtures/tenancy-values.yaml" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "$temporary_directory"' EXIT + +helm lint "$chart" --strict + +helm template devboxes "$chart" --namespace devboxes \ + >"$temporary_directory/disabled.yaml" +grep -Fq 'name: DEVBOXES_MULTI_TENANCY_ENABLED' "$temporary_directory/disabled.yaml" +grep -Fq 'value: "false"' "$temporary_directory/disabled.yaml" +if grep -Fq 'kind: Namespace' "$temporary_directory/disabled.yaml"; then + printf 'error: disabled tenancy rendered tenant namespaces\n' >&2 + exit 1 +fi + +helm template devboxes "$chart" --namespace devboxes -f "$values" \ + >"$temporary_directory/enabled.yaml" +grep -Fq 'name: devboxes-platform' "$temporary_directory/enabled.yaml" +grep -Fq 'name: devboxes-research' "$temporary_directory/enabled.yaml" +grep -Fq 'kind: ResourceQuota' "$temporary_directory/enabled.yaml" +grep -Fq 'kind: NetworkPolicy' "$temporary_directory/enabled.yaml" +grep -Fq 'port: 2222' "$temporary_directory/enabled.yaml" +grep -Fq 'namespace: devboxes-platform' "$temporary_directory/enabled.yaml" +grep -Fq 'namespace: devboxes-research' "$temporary_directory/enabled.yaml" +grep -Fq 'name: DEVBOXES_TENANTS' "$temporary_directory/enabled.yaml" +grep -Fq 'name: DEVBOXES_PRINCIPALS' "$temporary_directory/enabled.yaml" +grep -Fq 'secretName: devboxes-principals' "$temporary_directory/enabled.yaml" +grep -Fq 'http://devboxes.devboxes.svc:8000' "$temporary_directory/enabled.yaml" +grep -Fq 'devboxes.bonalab.org/tenant: "platform"' "$temporary_directory/enabled.yaml" + +if helm template devboxes "$chart" --namespace devboxes -f "$values" \ + --set tenancy.defaultTenant=missing \ + >"$temporary_directory/missing-default.yaml" \ + 2>"$temporary_directory/missing-default.err"; then + printf 'error: tenancy accepted an unknown default tenant\n' >&2 + exit 1 +fi +grep -Fq 'defaultTenant' "$temporary_directory/missing-default.err" + +if helm template devboxes "$chart" --namespace devboxes -f "$values" \ + --set 'tenancy.tenants[1].id=platform' \ + >"$temporary_directory/duplicate.yaml" \ + 2>"$temporary_directory/duplicate.err"; then + printf 'error: tenancy accepted a duplicate tenant ID\n' >&2 + exit 1 +fi +grep -Fq 'duplicate id' "$temporary_directory/duplicate.err" + +if helm template devboxes "$chart" --namespace devboxes -f "$values" \ + --set tenancy.existingPrincipalSecret= \ + >"$temporary_directory/missing-secret.yaml" \ + 2>"$temporary_directory/missing-secret.err"; then + printf 'error: tenancy accepted principals without a credential Secret\n' >&2 + exit 1 +fi +grep -Fq 'existingPrincipalSecret' "$temporary_directory/missing-secret.err" + +if helm template devboxes "$chart" --namespace devboxes -f "$values" \ + --set-string 'tenancy.tenants[0].namespaceLabels.kubernetes\\.io/metadata\\.name=hijack' \ + >"$temporary_directory/reserved-namespace-label.yaml" \ + 2>"$temporary_directory/reserved-namespace-label.err"; then + printf 'error: tenancy accepted a reserved Kubernetes namespace label\n' >&2 + exit 1 +fi +grep -Fq 'invalid propertyName' "$temporary_directory/reserved-namespace-label.err" + +if helm template devboxes "$chart" --namespace devboxes -f "$values" \ + --set 'tenancy.principals[0].memberships[1].tenant=missing' \ + >"$temporary_directory/missing-membership.yaml" \ + 2>"$temporary_directory/missing-membership.err"; then + printf 'error: tenancy accepted a membership for an unknown tenant\n' >&2 + exit 1 +fi +grep -Fq 'unknown tenant' "$temporary_directory/missing-membership.err" + +if helm template devboxes "$chart" --namespace devboxes -f "$values" \ + --set 'gpu.profiles[0].name=nvidia-l4' \ + --set 'gpu.profiles[0].displayName=NVIDIA L4' \ + --set-string 'gpu.profiles[0].resourceName=nvidia.com/gpu' \ + --set 'gpu.profiles[0].count=1' \ + --set 'tenancy.tenants[0].defaultGpuProfile=nvidia-l4' \ + >"$temporary_directory/disabled-gpu-default.yaml" \ + 2>"$temporary_directory/disabled-gpu-default.err"; then + printf 'error: tenancy accepted a tenant GPU default while GPU support is disabled\n' >&2 + exit 1 +fi +grep -Fq 'requires gpu.enabled=true' "$temporary_directory/disabled-gpu-default.err" + +if helm template devboxes "$chart" --namespace devboxes -f "$values" \ + --set gpu.enabled=true \ + --set gpu.defaultProfile=nvidia-l4 \ + --set 'gpu.profiles[0].name=nvidia-l4' \ + --set 'gpu.profiles[0].displayName=NVIDIA L4' \ + --set-string 'gpu.profiles[0].resourceName=nvidia.com/gpu' \ + --set 'gpu.profiles[0].count=1' \ + --set 'tenancy.tenants[0].defaultGpuProfile=missing' \ + >"$temporary_directory/missing-gpu-default.yaml" \ + 2>"$temporary_directory/missing-gpu-default.err"; then + printf 'error: tenancy accepted an unknown tenant GPU default\n' >&2 + exit 1 +fi +grep -Fq 'default GPU profile' "$temporary_directory/missing-gpu-default.err" + +printf 'Verified disabled, isolated, and invalid multi-tenancy Helm contracts.\n'