From cc9f80da30fe28ca48694b7ad384a929866a5d36 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 13:55:49 +0200 Subject: [PATCH 01/31] chore(dvcr): upgrade distribution registry 2.8.3 to 3.0.0 Migrate the DVCR build from the GOPATH-based build of distribution v2.8.3 to a Go-module build of github.com/distribution/distribution/v3 at v3.0.0. - bump core.distribution to 3.0.0 in build/components/versions.yml - drop the GO111MODULE=off GOPATH shim in images/dvcr/werf.inc.yaml and build ./cmd/registry directly from the module root - update version ldflags to the v3 package path and lowercase symbols (version.version/revision/mainpkg under github.com/distribution/distribution/v3/version) Config schema is unchanged: version, log, storage.cache.blobdescriptor, http.debug.prometheus and health.storagedriver are all still valid in v3, so templates/dvcr/configmap.yaml needs no changes. Signed-off-by: Daniil Antoshin --- build/components/versions.yml | 2 +- images/dvcr/werf.inc.yaml | 14 +++----------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/build/components/versions.yml b/build/components/versions.yml index dc4335948f..0c976a15d7 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -5,7 +5,7 @@ firmware: core: 3p-kubevirt: v1.6.2-v12n.53 3p-containerized-data-importer: v1.60.3-v12n.21 - distribution: 2.8.3 + distribution: 3.0.0 package: acl: v2.3.1 bzip2: bzip2-1.0.8 diff --git a/images/dvcr/werf.inc.yaml b/images/dvcr/werf.inc.yaml index a80724e9fc..b5f73e827a 100644 --- a/images/dvcr/werf.inc.yaml +++ b/images/dvcr/werf.inc.yaml @@ -58,19 +58,11 @@ shell: export GOOS=linux export GOARCH=amd64 export CGO_ENABLED=0 - - # Docker distribution v2.8.3 is not a go module, backoff to GOPATH build. Remove these lines on migration to distribution v3.0.0. - export GOPATH=$(go env GOPATH) - export GOROOT=$(go env GOROOT) - export GO111MODULE=off + export GO111MODULE=on mkdir -p /container-registry-binary - mkdir -p $GOPATH/src/github.com/docker - - cd $GOPATH/src/github.com/docker - mv /distribution . - cd distribution + cd /distribution export VERSION={{ $version }} {{- $_ := set $ "ProjectName" (list .ImageName "dvcr" | join "/") }} - {{- include "image-build.build" (set $ "BuildCommand" `go build -o /container-registry-binary/ -ldflags '-s -w -X registry/version.Version=v$VERSION -X registry/version.Revision=v$VERSION' ./cmd/registry`) | nindent 6 }} + {{- include "image-build.build" (set $ "BuildCommand" `go build -o /container-registry-binary/ -ldflags '-s -w -X github.com/distribution/distribution/v3/version.version=v$VERSION -X github.com/distribution/distribution/v3/version.revision=v$VERSION -X github.com/distribution/distribution/v3/version.mainpkg=github.com/distribution/distribution/v3' ./cmd/registry`) | nindent 6 }} From 728e78b3160a6df121a143c9ca3539fd0e9f9b4a Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 14:00:37 +0200 Subject: [PATCH 02/31] feat(dvcr): add multi-tenant authorization policy core Add the pure, dependency-free authorization decision logic for the DVCR in-registry AccessController: namespace-scoped access for tenant Pods, pull-only for nodes, full access for the controller, fail-closed default. The policy anchors repository-name normalization with path.Clean (not HasPrefix) so that names like vi/nsA-evil or vi/nsA/../nsB/x cannot be mistaken for another namespace, denies the registry catalog to tenants, and authorizes cross-repo blob mounts per-access (a mount from a foreign namespace is denied via its pull access on the source repo). Covered by table unit tests. The distribution auth.AccessController glue (request parsing, TokenReview) and build wiring are added separately. Signed-off-by: Daniil Antoshin --- images/dvcr/dvcr-k8s-auth/go.mod | 6 + images/dvcr/dvcr-k8s-auth/policy.go | 136 +++++++++++++++++++++++ images/dvcr/dvcr-k8s-auth/policy_test.go | 107 ++++++++++++++++++ 3 files changed, 249 insertions(+) create mode 100644 images/dvcr/dvcr-k8s-auth/go.mod create mode 100644 images/dvcr/dvcr-k8s-auth/policy.go create mode 100644 images/dvcr/dvcr-k8s-auth/policy_test.go diff --git a/images/dvcr/dvcr-k8s-auth/go.mod b/images/dvcr/dvcr-k8s-auth/go.mod new file mode 100644 index 0000000000..5ddad70210 --- /dev/null +++ b/images/dvcr/dvcr-k8s-auth/go.mod @@ -0,0 +1,6 @@ +// Local module for standalone unit-testing of the DVCR authorization policy. +// The .go sources are copied into the distribution registry module at build +// time (see images/dvcr/werf.inc.yaml); this go.mod is not used in that build. +module github.com/deckhouse/virtualization/dvcr-k8s-auth + +go 1.25 diff --git a/images/dvcr/dvcr-k8s-auth/policy.go b/images/dvcr/dvcr-k8s-auth/policy.go new file mode 100644 index 0000000000..eb4a034728 --- /dev/null +++ b/images/dvcr/dvcr-k8s-auth/policy.go @@ -0,0 +1,136 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package dvcrk8s implements the DVCR multi-tenant authorization policy. +// +// This file holds the pure, dependency-free authorization decision logic so it +// can be unit-tested without the distribution registry runtime. The distribution +// auth.AccessController glue (request parsing, TokenReview) lives in access.go +// and only translates registry types into the Subject/Access values decided here. +package dvcrk8s + +import ( + "path" + "strings" +) + +// Role is the authorization role derived from the presented credential. +type Role int + +const ( + // RoleNone denies everything (fail-closed default). + RoleNone Role = iota + // RoleAdmin grants full access. Used by the virtualization-controller. + RoleAdmin + // RolePuller grants pull-only access to any repository. Used by node containerd. + RolePuller + // RoleTenant grants namespace-scoped access. Used by importer/uploader Pods. + RoleTenant +) + +// Subject is the authenticated caller together with its authorization scope. +type Subject struct { + Role Role + // Namespace is the tenant namespace; only meaningful for RoleTenant. + Namespace string +} + +// Access mirrors the subset of distribution's auth.Access needed for a decision. +type Access struct { + // Type is the resource type: "repository" or "registry". + Type string + // Name is the repository path without tag (e.g. "vi/ns/name"), or "catalog" + // for the registry-wide catalog resource. + Name string + // Action is "pull", "push", "delete" or "*". + Action string +} + +// Repository path prefixes, kept in sync with pkg/dvcr/dvcr.go image templates: +// +// cvi/ (ClusterVirtualImage, cluster-scoped, shared read-only) +// vi// (VirtualImage, namespaced) +// vd// (VirtualDisk, namespaced) +const ( + prefixCVI = "cvi" + prefixVI = "vi" + prefixVD = "vd" +) + +// Authorize returns true only if the subject is allowed every requested access. +// A single denied access denies the whole request (fail-closed). An empty access +// list is allowed (distribution uses it for unauthenticated capability probes). +func Authorize(s Subject, accesses []Access) bool { + for i := range accesses { + if !authorizeOne(s, accesses[i]) { + return false + } + } + return true +} + +func authorizeOne(s Subject, a Access) bool { + switch s.Role { + case RoleAdmin: + return true + case RolePuller: + // Nodes only ever pull image layers; never push or delete, never enumerate. + return a.Type == "repository" && a.Action == "pull" + case RoleTenant: + return authorizeTenant(s.Namespace, a) + default: + return false + } +} + +func authorizeTenant(ns string, a Access) bool { + // Deny the registry-wide catalog and any non-repository resource: it would + // enumerate every tenant's images. + if a.Type != "repository" { + return false + } + // Empty namespace can never match a repository segment; deny. + if ns == "" { + return false + } + + seg := splitClean(a.Name) + switch { + case len(seg) == 3 && (seg[0] == prefixVI || seg[0] == prefixVD) && seg[1] == ns: + // Own-namespace VirtualImage / VirtualDisk: read and write. + return a.Action == "pull" || a.Action == "push" + case len(seg) >= 2 && seg[0] == prefixCVI: + // Cluster images are shared; tenants may read them as disk/image sources, + // but only the controller (RoleAdmin) creates them. + // ponytail: pull-only for tenants; if a future flow pushes cvi from a + // tenant Pod, grant push here and cover it with an e2e test. + return a.Action == "pull" + default: + return false + } +} + +// splitClean normalizes a repository name into path segments, neutralizing any +// "." / ".." traversal by anchoring the clean at root. Returns nil for an empty +// or root-only path. Using path.Clean (not HasPrefix) is what prevents a name +// like "vi/nsA-evil" or "vi/nsA/../nsB/x" from being mistaken for namespace nsA. +func splitClean(name string) []string { + cleaned := strings.TrimPrefix(path.Clean("/"+name), "/") + if cleaned == "" || cleaned == "." { + return nil + } + return strings.Split(cleaned, "/") +} diff --git a/images/dvcr/dvcr-k8s-auth/policy_test.go b/images/dvcr/dvcr-k8s-auth/policy_test.go new file mode 100644 index 0000000000..ac7287765f --- /dev/null +++ b/images/dvcr/dvcr-k8s-auth/policy_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Plain stdlib table tests (not ginkgo): this package is dependency-free on +// purpose so it can be compiled into the upstream distribution binary without +// dragging test frameworks into that build. +package dvcrk8s + +import "testing" + +func repo(name, action string) Access { + return Access{Type: "repository", Name: name, Action: action} +} + +func TestAuthorize(t *testing.T) { + tenantA := Subject{Role: RoleTenant, Namespace: "nsA"} + + tests := []struct { + name string + subject Subject + accesses []Access + want bool + }{ + // --- tenant: own namespace --- + {"tenant pull own vi", tenantA, []Access{repo("vi/nsA/img", "pull")}, true}, + {"tenant push own vi", tenantA, []Access{repo("vi/nsA/img", "push")}, true}, + {"tenant pull own vd", tenantA, []Access{repo("vd/nsA/disk", "pull")}, true}, + {"tenant push own vd", tenantA, []Access{repo("vd/nsA/disk", "push")}, true}, + {"tenant delete own vi denied", tenantA, []Access{repo("vi/nsA/img", "delete")}, false}, + + // --- tenant: cross-namespace (the core vulnerability) --- + {"tenant pull other-ns vi denied", tenantA, []Access{repo("vi/nsB/img", "pull")}, false}, + {"tenant push other-ns vi denied", tenantA, []Access{repo("vi/nsB/img", "push")}, false}, + {"tenant push other-ns vd denied", tenantA, []Access{repo("vd/nsB/disk", "push")}, false}, + + // --- tenant: cvi (cluster, shared read-only) --- + {"tenant pull cvi ok", tenantA, []Access{repo("cvi/ubuntu", "pull")}, true}, + {"tenant push cvi denied", tenantA, []Access{repo("cvi/ubuntu", "push")}, false}, + + // --- tenant: catalog / registry enumeration --- + {"tenant catalog denied", tenantA, []Access{{Type: "registry", Name: "catalog", Action: "*"}}, false}, + + // --- tenant: path normalization / prefix confusion --- + {"tenant prefix confusion nsA-evil denied", tenantA, []Access{repo("vi/nsA-evil/img", "push")}, false}, + {"tenant traversal to other ns denied", tenantA, []Access{repo("vi/nsA/../nsB/img", "push")}, false}, + {"tenant traversal escape denied", tenantA, []Access{repo("vi/nsA/../../etc/x", "push")}, false}, + {"tenant unknown prefix denied", tenantA, []Access{repo("foo/nsA/img", "push")}, false}, + {"tenant bare ns no name denied", tenantA, []Access{repo("vi/nsA", "push")}, false}, + + // --- tenant: cross-repo blob mount (distribution issues a pull on the from-repo) --- + {"tenant mount from own ns ok", tenantA, []Access{ + repo("vi/nsA/dst", "push"), + repo("vi/nsA/src", "pull"), + }, true}, + {"tenant mount from other ns denied", tenantA, []Access{ + repo("vi/nsA/dst", "push"), + repo("vi/nsB/src", "pull"), + }, false}, + + // --- tenant: empty namespace is never valid --- + {"tenant empty ns denied", Subject{Role: RoleTenant, Namespace: ""}, []Access{repo("vi//img", "pull")}, false}, + + // --- puller (nodes) --- + {"puller pull any ok", Subject{Role: RolePuller}, []Access{repo("vi/nsB/img", "pull")}, true}, + {"puller pull cvi ok", Subject{Role: RolePuller}, []Access{repo("cvi/ubuntu", "pull")}, true}, + {"puller push denied", Subject{Role: RolePuller}, []Access{repo("vi/nsB/img", "push")}, false}, + {"puller delete denied", Subject{Role: RolePuller}, []Access{repo("vi/nsB/img", "delete")}, false}, + {"puller catalog denied", Subject{Role: RolePuller}, []Access{{Type: "registry", Name: "catalog", Action: "*"}}, false}, + + // --- admin (controller) --- + {"admin all repo ok", Subject{Role: RoleAdmin}, []Access{repo("vi/nsB/img", "push"), repo("cvi/x", "delete")}, true}, + {"admin catalog ok", Subject{Role: RoleAdmin}, []Access{{Type: "registry", Name: "catalog", Action: "*"}}, true}, + + // --- fail-closed default --- + {"none denied", Subject{Role: RoleNone}, []Access{repo("vi/nsA/img", "pull")}, false}, + + // --- mixed list: one denied access denies all --- + {"tenant mixed one bad denies all", tenantA, []Access{ + repo("vi/nsA/ok", "pull"), + repo("vi/nsB/bad", "pull"), + }, false}, + + // --- empty access list is allowed (capability probe) --- + {"empty access allowed", tenantA, nil, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Authorize(tt.subject, tt.accesses); got != tt.want { + t.Errorf("Authorize(%+v, %+v) = %v, want %v", tt.subject, tt.accesses, got, tt.want) + } + }) + } +} From 761278730b0cf863ed6a469224e5ac57f8710071 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 14:03:51 +0200 Subject: [PATCH 03/31] feat(dvcr): add distribution v3 AccessController glue and TokenReview Wire the authorization policy into a distribution v3 auth.AccessController: - access.go registers the "dvcr-k8s" controller, parses Basic credentials, classifies them (constant-time static admin/node-puller match, else a ServiceAccount token verified via TokenReview) and maps the result to the policy Subject; denies map to a 401 challenge (bad credential) or a plain 403 (authenticated but not permitted). - tokenreview.go verifies SA tokens through the Kubernetes TokenReview API using only the standard library (no client-go), with positive/negative result caching to keep the push path off the apiserver; the registry's own SA token is re-read per call to tolerate projected-token rotation. Both files are behind the dvcr_registry build tag and compiled only inside the werf DVCR build, so the dependency-free policy stays unit-testable. The glue was type-checked against distribution v3.0.0. Signed-off-by: Daniil Antoshin --- images/dvcr/dvcr-k8s-auth/access.go | 216 +++++++++++++++++++++++ images/dvcr/dvcr-k8s-auth/policy.go | 17 ++ images/dvcr/dvcr-k8s-auth/policy_test.go | 20 +++ images/dvcr/dvcr-k8s-auth/tokenreview.go | 202 +++++++++++++++++++++ 4 files changed, 455 insertions(+) create mode 100644 images/dvcr/dvcr-k8s-auth/access.go create mode 100644 images/dvcr/dvcr-k8s-auth/tokenreview.go diff --git a/images/dvcr/dvcr-k8s-auth/access.go b/images/dvcr/dvcr-k8s-auth/access.go new file mode 100644 index 0000000000..1b8c428de0 --- /dev/null +++ b/images/dvcr/dvcr-k8s-auth/access.go @@ -0,0 +1,216 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Build tag dvcr_registry: this file depends on the distribution registry module +// and is only compiled inside the werf DVCR build (which passes -tags dvcr_registry). +// The dependency-free policy in policy.go stays unit-testable without it. + +//go:build dvcr_registry + +package dvcrk8s + +import ( + "crypto/subtle" + "errors" + "fmt" + "net/http" + "os" + "strings" + "time" + + "github.com/distribution/distribution/v3/registry/auth" + "github.com/sirupsen/logrus" +) + +func init() { + if err := auth.Register("dvcr-k8s", auth.InitFunc(newAccessController)); err != nil { + logrus.Errorf("failed to register dvcr-k8s auth: %v", err) + } +} + +type accessController struct { + realm string + + adminUsername string + adminPassword []byte + + pullerUsername string + pullerPassword []byte + + reviewer *tokenReviewer +} + +// newAccessController builds the controller from the `auth: { dvcr-k8s: {...} }` +// config block. Options: +// +// realm string (WWW-Authenticate realm) +// adminusername string +// adminpasswordfile string (path to the admin password, e.g. from dvcr-secrets) +// pullerusername string +// pullerpasswordfile string (path to the node-puller password) +// tokenreviewcachettl string (Go duration, default 45s) +// tokenreviewcachenegttl string (Go duration for failed reviews, default 5s) +func newAccessController(options map[string]interface{}) (auth.AccessController, error) { + realm, err := optString(options, "realm", "dvcr") + if err != nil { + return nil, err + } + + adminUser, err := optString(options, "adminusername", "admin") + if err != nil { + return nil, err + } + adminPass, err := readSecretFile(options, "adminpasswordfile") + if err != nil { + return nil, err + } + + pullerUser, err := optString(options, "pullerusername", "node-puller") + if err != nil { + return nil, err + } + pullerPass, err := readSecretFile(options, "pullerpasswordfile") + if err != nil { + return nil, err + } + + ttl := optDuration(options, "tokenreviewcachettl", 45*time.Second) + negTTL := optDuration(options, "tokenreviewcachenegttl", 5*time.Second) + + reviewer, err := newTokenReviewer(ttl, negTTL) + if err != nil { + return nil, fmt.Errorf("init token reviewer: %w", err) + } + + return &accessController{ + realm: realm, + adminUsername: adminUser, + adminPassword: adminPass, + pullerUsername: pullerUser, + pullerPassword: pullerPass, + reviewer: reviewer, + }, nil +} + +// Authorized implements auth.AccessController for distribution v3. +func (ac *accessController) Authorized(req *http.Request, accessRecords ...auth.Access) (*auth.Grant, error) { + username, password, ok := req.BasicAuth() + if !ok || password == "" { + return nil, &challenge{realm: ac.realm, err: auth.ErrInvalidCredential} + } + + subject, name, err := ac.classify(req, username, password) + if err != nil { + // Bad credential -> 401 challenge so the client may retry with valid creds. + return nil, &challenge{realm: ac.realm, err: err} + } + + if !Authorize(subject, toPolicyAccess(accessRecords)) { + // Authenticated but not permitted -> 403 (plain error, not a challenge). + return nil, fmt.Errorf("dvcr-k8s: access denied for %q: %w", name, auth.ErrAuthenticationFailure) + } + + return &auth.Grant{User: auth.UserInfo{Name: name}}, nil +} + +// classify maps a Basic credential to an authorization Subject. Static admin and +// node-puller passwords are matched in constant time; anything else is treated as +// a ServiceAccount token and verified via TokenReview (fail-closed on any error). +func (ac *accessController) classify(req *http.Request, username, password string) (Subject, string, error) { + if len(ac.adminPassword) > 0 && username == ac.adminUsername && + subtle.ConstantTimeCompare([]byte(password), ac.adminPassword) == 1 { + return Subject{Role: RoleAdmin}, username, nil + } + + if len(ac.pullerPassword) > 0 && username == ac.pullerUsername && + subtle.ConstantTimeCompare([]byte(password), ac.pullerPassword) == 1 { + return Subject{Role: RolePuller}, username, nil + } + + ns, err := ac.reviewer.namespaceForToken(req.Context(), password) + if err != nil { + return Subject{}, "", fmt.Errorf("token review: %w", err) + } + if ns == "" { + return Subject{}, "", errors.New("token is not a namespaced ServiceAccount") + } + return Subject{Role: RoleTenant, Namespace: ns}, "serviceaccount:" + ns, nil +} + +func toPolicyAccess(records []auth.Access) []Access { + out := make([]Access, len(records)) + for i, r := range records { + out[i] = Access{Type: r.Type, Name: r.Name, Action: r.Action} + } + return out +} + +// challenge implements auth.Challenge for a 401 Basic auth response. +type challenge struct { + realm string + err error +} + +func (ch *challenge) SetHeaders(_ *http.Request, w http.ResponseWriter) { + w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", ch.realm)) +} + +func (ch *challenge) Error() string { + return fmt.Sprintf("dvcr-k8s basic auth challenge for realm %q: %v", ch.realm, ch.err) +} + +func optString(options map[string]interface{}, key, def string) (string, error) { + v, present := options[key] + if !present { + return def, nil + } + s, ok := v.(string) + if !ok { + return "", fmt.Errorf("dvcr-k8s: option %q must be a string", key) + } + return s, nil +} + +func optDuration(options map[string]interface{}, key string, def time.Duration) time.Duration { + v, present := options[key] + if !present { + return def + } + s, ok := v.(string) + if !ok { + return def + } + d, err := time.ParseDuration(strings.TrimSpace(s)) + if err != nil { + return def + } + return d +} + +func readSecretFile(options map[string]interface{}, key string) ([]byte, error) { + path, err := optString(options, key, "") + if err != nil { + return nil, err + } + if path == "" { + return nil, nil + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("dvcr-k8s: read %q from %s: %w", key, path, err) + } + return []byte(strings.TrimSpace(string(data))), nil +} diff --git a/images/dvcr/dvcr-k8s-auth/policy.go b/images/dvcr/dvcr-k8s-auth/policy.go index eb4a034728..7b902506ef 100644 --- a/images/dvcr/dvcr-k8s-auth/policy.go +++ b/images/dvcr/dvcr-k8s-auth/policy.go @@ -70,6 +70,23 @@ const ( prefixVD = "vd" ) +// saUserPrefix is the TokenReview username prefix for a ServiceAccount. +const saUserPrefix = "system:serviceaccount:" + +// namespaceFromUsername parses "system:serviceaccount::" and returns +// . Any other username shape returns "" (denied by the caller). +func namespaceFromUsername(username string) string { + if !strings.HasPrefix(username, saUserPrefix) { + return "" + } + rest := strings.TrimPrefix(username, saUserPrefix) + ns, _, ok := strings.Cut(rest, ":") + if !ok || ns == "" { + return "" + } + return ns +} + // Authorize returns true only if the subject is allowed every requested access. // A single denied access denies the whole request (fail-closed). An empty access // list is allowed (distribution uses it for unauthenticated capability probes). diff --git a/images/dvcr/dvcr-k8s-auth/policy_test.go b/images/dvcr/dvcr-k8s-auth/policy_test.go index ac7287765f..96e69e9094 100644 --- a/images/dvcr/dvcr-k8s-auth/policy_test.go +++ b/images/dvcr/dvcr-k8s-auth/policy_test.go @@ -105,3 +105,23 @@ func TestAuthorize(t *testing.T) { }) } } + +func TestNamespaceFromUsername(t *testing.T) { + tests := []struct { + username string + want string + }{ + {"system:serviceaccount:nsA:importer", "nsA"}, + {"system:serviceaccount:d8-virtualization:dvcr", "d8-virtualization"}, + {"system:serviceaccount::name", ""}, // empty namespace + {"system:serviceaccount:nsA", ""}, // no name separator + {"system:node:worker-1", ""}, // not a service account + {"kubernetes-admin", ""}, // human user + {"", ""}, // empty + } + for _, tt := range tests { + if got := namespaceFromUsername(tt.username); got != tt.want { + t.Errorf("namespaceFromUsername(%q) = %q, want %q", tt.username, got, tt.want) + } + } +} diff --git a/images/dvcr/dvcr-k8s-auth/tokenreview.go b/images/dvcr/dvcr-k8s-auth/tokenreview.go new file mode 100644 index 0000000000..b4fe4c90ac --- /dev/null +++ b/images/dvcr/dvcr-k8s-auth/tokenreview.go @@ -0,0 +1,202 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//go:build dvcr_registry + +package dvcrk8s + +import ( + "bytes" + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" + "sync" + "time" +) + +const ( + saDir = "/var/run/secrets/kubernetes.io/serviceaccount" + saTokenPath = saDir + "/token" + saCAPath = saDir + "/ca.crt" + + tokenReviewPath = "/apis/authentication.k8s.io/v1/tokenreviews" +) + +var errTokenRejected = errors.New("token rejected by TokenReview") + +// tokenReviewer verifies ServiceAccount tokens via the Kubernetes TokenReview API +// using only the standard library (no client-go). Results are cached to keep the +// hot push path off the apiserver. +type tokenReviewer struct { + client *http.Client + apiURL string + ttl time.Duration + negTTL time.Duration + + mu sync.Mutex + cache map[string]cacheEntry +} + +type cacheEntry struct { + ns string + ok bool + expiry time.Time +} + +func newTokenReviewer(ttl, negTTL time.Duration) (*tokenReviewer, error) { + host := os.Getenv("KUBERNETES_SERVICE_HOST") + port := os.Getenv("KUBERNETES_SERVICE_PORT") + if host == "" || port == "" { + return nil, errors.New("KUBERNETES_SERVICE_HOST/PORT not set (not running in-cluster?)") + } + + caPEM, err := os.ReadFile(saCAPath) + if err != nil { + return nil, fmt.Errorf("read apiserver CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, errors.New("parse apiserver CA") + } + + return &tokenReviewer{ + client: &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + }, + }, + apiURL: fmt.Sprintf("https://%s:%s%s", host, port, tokenReviewPath), + ttl: ttl, + negTTL: negTTL, + cache: make(map[string]cacheEntry), + }, nil +} + +// namespaceForToken returns the ServiceAccount namespace for a token, or an error +// if the token is rejected. Transient errors (network/apiserver) are not cached. +func (tr *tokenReviewer) namespaceForToken(ctx context.Context, token string) (string, error) { + key := hashToken(token) + + if e, ok := tr.lookup(key); ok { + if !e.ok { + return "", errTokenRejected + } + return e.ns, nil + } + + ns, ok, err := tr.review(ctx, token) + if err != nil { + return "", err + } + if !ok { + tr.store(key, cacheEntry{ok: false, expiry: time.Now().Add(tr.negTTL)}) + return "", errTokenRejected + } + tr.store(key, cacheEntry{ok: true, ns: ns, expiry: time.Now().Add(tr.ttl)}) + return ns, nil +} + +func (tr *tokenReviewer) lookup(key string) (cacheEntry, bool) { + tr.mu.Lock() + defer tr.mu.Unlock() + e, ok := tr.cache[key] + if !ok || time.Now().After(e.expiry) { + if ok { + delete(tr.cache, key) + } + return cacheEntry{}, false + } + return e, true +} + +func (tr *tokenReviewer) store(key string, e cacheEntry) { + tr.mu.Lock() + defer tr.mu.Unlock() + tr.cache[key] = e +} + +// review performs a single TokenReview call. It authenticates to the apiserver +// with the registry's own ServiceAccount token, read fresh each call because the +// projected token rotates. +func (tr *tokenReviewer) review(ctx context.Context, token string) (ns string, authenticated bool, err error) { + reqBody, err := json.Marshal(map[string]any{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenReview", + "spec": map[string]any{"token": token}, + }) + if err != nil { + return "", false, err + } + + ownToken, err := os.ReadFile(saTokenPath) + if err != nil { + return "", false, fmt.Errorf("read own SA token: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, tr.apiURL, bytes.NewReader(reqBody)) + if err != nil { + return "", false, err + } + req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(string(ownToken))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := tr.client.Do(req) + if err != nil { + return "", false, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", false, err + } + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + return "", false, fmt.Errorf("TokenReview HTTP %d: %s", resp.StatusCode, string(body)) + } + + var out struct { + Status struct { + Authenticated bool `json:"authenticated"` + User struct { + Username string `json:"username"` + } `json:"user"` + Error string `json:"error"` + } `json:"status"` + } + if err := json.Unmarshal(body, &out); err != nil { + return "", false, fmt.Errorf("decode TokenReview: %w", err) + } + if !out.Status.Authenticated { + return "", false, nil + } + return namespaceFromUsername(out.Status.User.Username), true, nil +} + +func hashToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} From c2ec0c1b8223b519d39afa8e60e5078f64d7d204 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 14:06:46 +0200 Subject: [PATCH 04/31] feat(dvcr): build the dvcr-k8s auth plugin into the registry binary Add a custom registry entrypoint (images/dvcr/registry-main) equal to upstream cmd/registry plus a blank import of the dvcr-k8s auth plugin, and graft the plugin sources into the distribution module during the werf build. The binary is still emitted as "registry" so the existing consumers (dvcr-cleaner's `registry garbage-collect`, the final image includePaths) are unchanged. Built with -tags dvcr_registry so the distribution-dependent plugin glue is compiled in; htpasswd stays compiled in too, so the same image serves both auth modes and the feature flag only flips config. Signed-off-by: Daniil Antoshin --- images/dvcr/dvcr-k8s-auth/access.go | 4 +-- images/dvcr/registry-main/main.go | 47 +++++++++++++++++++++++++++++ images/dvcr/werf.inc.yaml | 28 ++++++++++++++++- 3 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 images/dvcr/registry-main/main.go diff --git a/images/dvcr/dvcr-k8s-auth/access.go b/images/dvcr/dvcr-k8s-auth/access.go index 1b8c428de0..bc48b2158d 100644 --- a/images/dvcr/dvcr-k8s-auth/access.go +++ b/images/dvcr/dvcr-k8s-auth/access.go @@ -26,18 +26,18 @@ import ( "crypto/subtle" "errors" "fmt" + "log" "net/http" "os" "strings" "time" "github.com/distribution/distribution/v3/registry/auth" - "github.com/sirupsen/logrus" ) func init() { if err := auth.Register("dvcr-k8s", auth.InitFunc(newAccessController)); err != nil { - logrus.Errorf("failed to register dvcr-k8s auth: %v", err) + log.Printf("failed to register dvcr-k8s auth: %v", err) } } diff --git a/images/dvcr/registry-main/main.go b/images/dvcr/registry-main/main.go new file mode 100644 index 0000000000..21363b2b70 --- /dev/null +++ b/images/dvcr/registry-main/main.go @@ -0,0 +1,47 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command dvcr-registry is the distribution registry entrypoint for DVCR. It is +// upstream cmd/registry plus a blank import of the dvcr-k8s auth plugin, so the +// same binary serves both htpasswd (feature flag off) and dvcr-k8s multi-tenant +// authorization (flag on). Built into a binary named "registry" so the existing +// consumers (dvcr-cleaner's `registry garbage-collect`, the final image) are +// unchanged. This file is copied into the distribution module at build time. +package main + +import ( + _ "net/http/pprof" + + "github.com/distribution/distribution/v3/registry" + _ "github.com/distribution/distribution/v3/registry/auth/dvcrk8s" + _ "github.com/distribution/distribution/v3/registry/auth/htpasswd" + _ "github.com/distribution/distribution/v3/registry/auth/silly" + _ "github.com/distribution/distribution/v3/registry/auth/token" + _ "github.com/distribution/distribution/v3/registry/proxy" + _ "github.com/distribution/distribution/v3/registry/storage/driver/azure" + _ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem" + _ "github.com/distribution/distribution/v3/registry/storage/driver/gcs" + _ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" + _ "github.com/distribution/distribution/v3/registry/storage/driver/middleware/cloudfront" + _ "github.com/distribution/distribution/v3/registry/storage/driver/middleware/redirect" + _ "github.com/distribution/distribution/v3/registry/storage/driver/middleware/rewrite" + _ "github.com/distribution/distribution/v3/registry/storage/driver/s3-aws" +) + +func main() { + // nolint:errcheck // RootCmd halts the program on failure of its subcommands. + registry.RootCmd.Execute() +} diff --git a/images/dvcr/werf.inc.yaml b/images/dvcr/werf.inc.yaml index b5f73e827a..2bb689001b 100644 --- a/images/dvcr/werf.inc.yaml +++ b/images/dvcr/werf.inc.yaml @@ -48,6 +48,25 @@ import: add: /src/distribution to: /distribution before: install +git: +# DVCR multi-tenant authorization plugin sources, compiled into the registry +# binary (see images/dvcr/dvcr-k8s-auth and images/dvcr/registry-main). +- add: {{ .ModuleDir }}/images/{{ .ImageName }}/dvcr-k8s-auth + to: /plugin/dvcrk8s + includePaths: + - "*.go" + excludePaths: + - "*_test.go" + stageDependencies: + install: + - "**/*.go" +- add: {{ .ModuleDir }}/images/{{ .ImageName }}/registry-main + to: /plugin/registry-main + includePaths: + - "*.go" + stageDependencies: + install: + - "**/*.go" secrets: - id: GOPROXY value: {{ .GOPROXY }} @@ -63,6 +82,13 @@ shell: mkdir -p /container-registry-binary cd /distribution + # Graft the DVCR auth plugin and its entrypoint into the distribution module. + # The plugin depends only on the distribution auth package and the standard + # library, so no go.mod changes are required. + mkdir -p /distribution/registry/auth/dvcrk8s /distribution/cmd/dvcr-registry + cp /plugin/dvcrk8s/*.go /distribution/registry/auth/dvcrk8s/ + cp /plugin/registry-main/*.go /distribution/cmd/dvcr-registry/ + export VERSION={{ $version }} {{- $_ := set $ "ProjectName" (list .ImageName "dvcr" | join "/") }} - {{- include "image-build.build" (set $ "BuildCommand" `go build -o /container-registry-binary/ -ldflags '-s -w -X github.com/distribution/distribution/v3/version.version=v$VERSION -X github.com/distribution/distribution/v3/version.revision=v$VERSION -X github.com/distribution/distribution/v3/version.mainpkg=github.com/distribution/distribution/v3' ./cmd/registry`) | nindent 6 }} + {{- include "image-build.build" (set $ "BuildCommand" `go build -tags dvcr_registry -o /container-registry-binary/registry -ldflags '-s -w -X github.com/distribution/distribution/v3/version.version=v$VERSION -X github.com/distribution/distribution/v3/version.revision=v$VERSION -X github.com/distribution/distribution/v3/version.mainpkg=github.com/distribution/distribution/v3' ./cmd/dvcr-registry`) | nindent 6 }} From 67c160fcba23ab0be3203d68e44fd2662fdf7dad Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 14:30:26 +0200 Subject: [PATCH 05/31] feat(dvcr): add node-puller credential and tenant-authz feature flag - generate a pull-only node-puller password (passwordPuller) in the generate-secret-for-dvcr hook, alongside the existing admin passwordRW; it needs no htpasswd entry since it carries no push rights - expose internal.dvcr.passwordPuller in the values schema - add the dvcr.tenantRegistryAuthorization config flag (default false) that gates per-namespace DVCR authorization, with a Russian doc-ru entry Hook unit tests updated and passing. Signed-off-by: Daniil Antoshin --- .../hooks/generate-secret-for-dvcr/hook.go | 41 ++++++++++++++----- .../generate-secret-for-dvcr/hook_test.go | 27 +++++++----- openapi/config-values.yaml | 12 ++++++ openapi/doc-ru-config-values.yaml | 11 +++++ openapi/values.yaml | 3 ++ 5 files changed, 73 insertions(+), 21 deletions(-) diff --git a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go index a64c2deb4d..07c2b8ee01 100644 --- a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go +++ b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go @@ -33,17 +33,19 @@ import ( ) const ( - dvcrSecrets = "dvcr-secrets" - passwordRWValuePath = "virtualization.internal.dvcr.passwordRW" - saltValuePath = "virtualization.internal.dvcr.salt" - htpasswdValuePath = "virtualization.internal.dvcr.htpasswd" - user = "admin" + dvcrSecrets = "dvcr-secrets" + passwordRWValuePath = "virtualization.internal.dvcr.passwordRW" + passwordPullerValuePath = "virtualization.internal.dvcr.passwordPuller" + saltValuePath = "virtualization.internal.dvcr.salt" + htpasswdValuePath = "virtualization.internal.dvcr.htpasswd" + user = "admin" ) type dvcrSecretData struct { - PasswordRW string `json:"passwordRW"` - Salt string `json:"salt"` - Htpasswd string `json:"htpasswd"` + PasswordRW string `json:"passwordRW"` + PasswordPuller string `json:"passwordPuller"` + Salt string `json:"salt"` + Htpasswd string `json:"htpasswd"` } var _ = registry.RegisterFunc(configDVCRSecrets, handlerDVCRSecrets) @@ -101,6 +103,22 @@ func handlerDVCRSecrets(ctx context.Context, input *pkg.HookInput) error { input.Values.Set(passwordRWValuePath, passwordRW) } + // passwordPuller is the pull-only node-puller credential used by the dvcr-k8s + // authorization backend. It carries no push rights, so it does not need an + // htpasswd entry (nodes fall back to admin only when the backend is htpasswd). + passwordPuller := dataFromSecret.PasswordPuller + passwordPullerBytes, err := base64.StdEncoding.DecodeString(passwordPuller) + passwordPullerDecoded := string(passwordPullerBytes) + if err != nil || passwordPullerDecoded == "" { + input.Logger.Info("Regenerate PasswordPuller") + passwordPullerDecoded = alphaNum(32) + passwordPuller = base64.StdEncoding.EncodeToString([]byte(passwordPullerDecoded)) + } + if passwordPuller != dataFromValues.PasswordPuller { + input.Logger.Info("Set PasswordPuller to values") + input.Values.Set(passwordPullerValuePath, passwordPuller) + } + salt := dataFromSecret.Salt saltBytes, err := base64.StdEncoding.DecodeString(salt) saltDecoded := string(saltBytes) @@ -150,9 +168,10 @@ func getDVCRSecretsFromSecrets(input *pkg.HookInput) (dvcrSecretData, error) { func getDVCRSecretsFromValues(input *pkg.HookInput) dvcrSecretData { return dvcrSecretData{ - PasswordRW: input.Values.Get(passwordRWValuePath).String(), - Salt: input.Values.Get(saltValuePath).String(), - Htpasswd: input.Values.Get(htpasswdValuePath).String(), + PasswordRW: input.Values.Get(passwordRWValuePath).String(), + PasswordPuller: input.Values.Get(passwordPullerValuePath).String(), + Salt: input.Values.Get(saltValuePath).String(), + Htpasswd: input.Values.Get(htpasswdValuePath).String(), } } diff --git a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go index 58abc79085..6415f8f3d1 100644 --- a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go +++ b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go @@ -41,6 +41,7 @@ func TestSetDVCRSecrets(t *testing.T) { var _ = Describe("DVCR Secrets", func() { const ( defaultPasswordBase64 = "dkREU0I1N1JXeVVFd1NwN3VubDA3SHlnWUx3MzlOTlY=" // "vDDSB57RWyUEwSp7unl07HygYLw39NNV" + defaultPullerBase64 = "bm9kZVB1bGxlclNlY3JldDAxMjM0NTY3ODlhYmNkZWY=" // "nodePullerSecret0123456789abcdef" defaultSaltBase64 = "bldlM01vZjVySlFNc3I2MDVFNDdBM1pYOU9IQ1dnVkY=" // "nWe3Mof5rJQMsr605E47A3ZX9OHCWgVF" defaultHtpasswdBase64 = "YWRtaW46JDJhJDEwJHZza21wTjVLSERKUlpNU1pWd3RZWU91YmtEcTEueEF2MXVRQkkvSzFHQ1dpNUpxSnF5amdt" // "admin:$2a$10$vskmpN5KHDJRZMSZVwtYYOubkDq1.xAv1uQBI/K1GCWi5JqJqyjgm" ) @@ -50,8 +51,9 @@ var _ = Describe("DVCR Secrets", func() { values *mock.OutputPatchableValuesCollectorMock ) - prepareValuesGet := func(passwordRW, salt, htpasswd string) { + prepareValuesGet := func(passwordRW, passwordPuller, salt, htpasswd string) { values.GetMock.When(passwordRWValuePath).Then(gjson.Result{Type: gjson.String, Str: passwordRW}) + values.GetMock.When(passwordPullerValuePath).Then(gjson.Result{Type: gjson.String, Str: passwordPuller}) values.GetMock.When(saltValuePath).Then(gjson.Result{Type: gjson.String, Str: salt}) values.GetMock.When(htpasswdValuePath).Then(gjson.Result{Type: gjson.String, Str: htpasswd}) } @@ -60,12 +62,13 @@ var _ = Describe("DVCR Secrets", func() { snapshots.GetMock.When(dvcrSecrets).Then(snaps) } - newSnapshot := func(passwordRW, salt, htpasswd string) pkg.Snapshot { + newSnapshot := func(passwordRW, passwordPuller, salt, htpasswd string) pkg.Snapshot { return mock.NewSnapshotMock(GinkgoT()).UnmarshalToMock.Set(func(v any) (err error) { data, ok := v.(*dvcrSecretData) Expect(ok).To(BeTrue()) data.PasswordRW = passwordRW + data.PasswordPuller = passwordPuller data.Salt = salt data.Htpasswd = htpasswd return nil @@ -100,8 +103,8 @@ var _ = Describe("DVCR Secrets", func() { }) It("Should set secrets from secret to values", func() { - prepareValuesGet("", "", "") - setSnapshots(newSnapshot(defaultPasswordBase64, defaultSaltBase64, defaultHtpasswdBase64)) + prepareValuesGet("", "", "", "") + setSnapshots(newSnapshot(defaultPasswordBase64, defaultPullerBase64, defaultSaltBase64, defaultHtpasswdBase64)) values.SetMock.Set(func(path string, v any) { value, ok := v.(string) @@ -110,6 +113,8 @@ var _ = Describe("DVCR Secrets", func() { switch path { case passwordRWValuePath: Expect(value).To(Equal(defaultPasswordBase64)) + case passwordPullerValuePath: + Expect(value).To(Equal(defaultPullerBase64)) case saltValuePath: Expect(value).To(Equal(defaultSaltBase64)) case htpasswdValuePath: @@ -123,8 +128,8 @@ var _ = Describe("DVCR Secrets", func() { }) It("Should regenerate all secrets", func() { - prepareValuesGet("", "", "") - setSnapshots(newSnapshot("", "", "")) + prepareValuesGet("", "", "", "") + setSnapshots(newSnapshot("", "", "", "")) var ( passwordRW string @@ -144,6 +149,8 @@ var _ = Describe("DVCR Secrets", func() { case passwordRWValuePath: passwordRW = value Expect(value).To(HaveLen(32)) + case passwordPullerValuePath: + Expect(value).To(HaveLen(32)) case saltValuePath: Expect(value).To(HaveLen(32)) case htpasswdValuePath: @@ -159,8 +166,8 @@ var _ = Describe("DVCR Secrets", func() { }) It("Should regenerate only salt", func() { - prepareValuesGet(defaultPasswordBase64, "", defaultHtpasswdBase64) - setSnapshots(newSnapshot(defaultPasswordBase64, "", defaultHtpasswdBase64)) + prepareValuesGet(defaultPasswordBase64, defaultPullerBase64, "", defaultHtpasswdBase64) + setSnapshots(newSnapshot(defaultPasswordBase64, defaultPullerBase64, "", defaultHtpasswdBase64)) values.SetMock.Set(func(path string, v any) { valueBase64, ok := v.(string) @@ -183,8 +190,8 @@ var _ = Describe("DVCR Secrets", func() { }) DescribeTable("Should regenerate only password and htpasswd", func(passwordRW, htpasswd string) { - prepareValuesGet(passwordRW, "salt", htpasswd) - setSnapshots(newSnapshot(passwordRW, "salt", htpasswd)) + prepareValuesGet(passwordRW, defaultPullerBase64, "salt", htpasswd) + setSnapshots(newSnapshot(passwordRW, defaultPullerBase64, "salt", htpasswd)) bytes, err := base64.StdEncoding.DecodeString(passwordRW) Expect(err).ToNot(HaveOccurred()) diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 2b2aa2b814..8e39217237 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -133,6 +133,18 @@ properties: Schedule to run garbage collection procedure that remove stale images for `ClusterVirtualImage`, `VirtualImage`, `VirtualDisk` resources deleted from the cluster. By default, periodic garbage collection is enabled and runs daily at 02:00. + tenantRegistryAuthorization: + type: boolean + default: false + description: | + Enable per-namespace authorization in the DVCR registry (experimental). + + When enabled, image import and upload Pods authenticate to DVCR with their own + ServiceAccount token and may access only repositories in their namespace, while + nodes get pull-only access. The shared read-write credential is no longer copied + into tenant namespaces. + + When disabled (default), DVCR uses shared htpasswd credentials. Disable to roll back. audit: type: object description: | diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index 2e29677fb7..36430b6a9f 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -75,6 +75,17 @@ properties: Расписание для запуска процедуры очистки хранилища. Очистка удалит неактуальные образы, созданные для ресурсов `ClusterVirtualImage`, `VirtualImage`, `VirtualDisk`, которых уже нет в кластере. По умолчанию периодическая очистка включена и выполняется ежедневно в 02:00. + tenantRegistryAuthorization: + description: | + Включить авторизацию по namespace в реестре DVCR (экспериментально). + + Когда включено, поды импорта и загрузки образов аутентифицируются в DVCR своим + ServiceAccount-токеном и получают доступ только к репозиториям своего namespace, а + узлы получают доступ только на чтение (pull). Общий read-write-креденшл больше не + копируется в namespace тенантов. + + Когда выключено (по умолчанию), DVCR использует общие htpasswd-учётные данные. + Выключите параметр для отката. virtualImages: type: object description: | diff --git a/openapi/values.yaml b/openapi/values.yaml index f716251736..6b229c0656 100644 --- a/openapi/values.yaml +++ b/openapi/values.yaml @@ -33,6 +33,9 @@ properties: default: "" passwordRW: type: string + passwordPuller: + type: string + default: "" htpasswd: type: string salt: From 70659ac58f446ae8699c692b8fd395c0bc264eec Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 14:34:30 +0200 Subject: [PATCH 06/31] feat(dvcr): wire dvcr-k8s auth backend behind the tenant-authz flag When dvcr.tenantRegistryAuthorization is enabled: - _helpers.tpl drops the htpasswd REGISTRY_AUTH env (auth comes from the ConfigMap instead) and mounts the passwordRW/passwordPuller files into /auth - configmap.yaml renders the auth.dvcr-k8s block (realm, admin and node-puller password files, TokenReview cache TTLs) - nodegroupconfiguration.yaml gives containerd the pull-only node-puller credential instead of admin:passwordRW - secret.yaml stores passwordPuller in dvcr-secrets - rbac-for-us.yaml binds the registry ServiceAccount to system:auth-delegator so the backend can create TokenReviews When disabled (default) the rendered output is unchanged (htpasswd). Verified with helm template for both flag states. Signed-off-by: Daniil Antoshin --- templates/dvcr/_helpers.tpl | 12 ++++++++++++ templates/dvcr/configmap.yaml | 11 +++++++++++ templates/dvcr/nodegroupconfiguration.yaml | 9 ++++++++- templates/dvcr/rbac-for-us.yaml | 16 ++++++++++++++++ templates/dvcr/secret.yaml | 1 + 5 files changed, 48 insertions(+), 1 deletion(-) diff --git a/templates/dvcr/_helpers.tpl b/templates/dvcr/_helpers.tpl index 490274a8a8..74de618c22 100644 --- a/templates/dvcr/_helpers.tpl +++ b/templates/dvcr/_helpers.tpl @@ -8,18 +8,24 @@ true {{- .Values.virtualization.internal | dig "dvcr" "garbageCollectionModeEnabled" "false" | default "false" -}} {{- end }} +{{- define "dvcr.isTenantAuthz" -}} +{{- .Values.virtualization.internal.moduleConfig | dig "dvcr" "tenantRegistryAuthorization" false -}} +{{- end }} + {{- define "dvcr.envs" -}} - name: REGISTRY_HTTP_TLS_CERTIFICATE value: /etc/ssl/docker/tls.crt - name: REGISTRY_HTTP_TLS_KEY value: /etc/ssl/docker/tls.key +{{- if ne (include "dvcr.isTenantAuthz" .) "true" }} - name: REGISTRY_AUTH value: "htpasswd" - name: REGISTRY_AUTH_HTPASSWD_REALM value: "Registry Realm" - name: REGISTRY_AUTH_HTPASSWD_PATH value: "/auth/htpasswd" +{{- end }} - name: REGISTRY_HTTP_SECRET valueFrom: @@ -91,6 +97,12 @@ true items: - key: htpasswd path: htpasswd +{{- if eq (include "dvcr.isTenantAuthz" .) "true" }} + - key: passwordRW + path: passwordRW + - key: passwordPuller + path: passwordPuller +{{- end }} {{- end -}} diff --git a/templates/dvcr/configmap.yaml b/templates/dvcr/configmap.yaml index 79e855ad0e..ca66837f46 100644 --- a/templates/dvcr/configmap.yaml +++ b/templates/dvcr/configmap.yaml @@ -18,6 +18,17 @@ data: maintenance: readonly: enabled: true +{{- end }} +{{- if eq (include "dvcr.isTenantAuthz" . ) "true" }} + auth: + dvcr-k8s: + realm: "dvcr" + adminusername: "admin" + adminpasswordfile: /auth/passwordRW + pullerusername: "node-puller" + pullerpasswordfile: /auth/passwordPuller + tokenreviewcachettl: "45s" + tokenreviewcachenegttl: "5s" {{- end }} http: addr: :5000 diff --git a/templates/dvcr/nodegroupconfiguration.yaml b/templates/dvcr/nodegroupconfiguration.yaml index c35af00c03..bf54cca683 100644 --- a/templates/dvcr/nodegroupconfiguration.yaml +++ b/templates/dvcr/nodegroupconfiguration.yaml @@ -1,7 +1,14 @@ {{- if ne (dig "dvcr" "serviceIP" "" .Values.virtualization.internal) "" }} --- {{- $ca := printf "%s" .Values.virtualization.internal.dvcr.cert.ca }} - {{- $password := printf "admin:%s" (printf "%s" .Values.virtualization.internal.dvcr.passwordRW | b64dec) | b64enc }} + {{- $authUser := "admin" }} + {{- $authPass := .Values.virtualization.internal.dvcr.passwordRW }} + {{- if eq (include "dvcr.isTenantAuthz" .) "true" }} + {{- /* Nodes only pull images; use the pull-only node-puller credential. */ -}} + {{- $authUser = "node-puller" }} + {{- $authPass = .Values.virtualization.internal.dvcr.passwordPuller }} + {{- end }} + {{- $password := printf "%s:%s" $authUser (printf "%s" $authPass | b64dec) | b64enc }} {{- $registry := include "dvcr.get_registry" (list .) }} {{- $endpoint := .Values.virtualization.internal.dvcr.serviceIP }} diff --git a/templates/dvcr/rbac-for-us.yaml b/templates/dvcr/rbac-for-us.yaml index 84e4b6deab..cbb3336555 100644 --- a/templates/dvcr/rbac-for-us.yaml +++ b/templates/dvcr/rbac-for-us.yaml @@ -84,4 +84,20 @@ subjects: - kind: ServiceAccount name: dvcr namespace: d8-{{ .Chart.Name }} +--- +# Allow the registry to verify ServiceAccount tokens via TokenReview for the +# dvcr-k8s multi-tenant authorization backend. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: d8:{{ .Chart.Name }}:dvcr:auth-delegator + {{- include "helm_lib_module_labels" (list . (dict "app" "dvcr")) | nindent 2 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: + - kind: ServiceAccount + name: dvcr + namespace: d8-{{ .Chart.Name }} {{- end }} diff --git a/templates/dvcr/secret.yaml b/templates/dvcr/secret.yaml index 15004d2184..918ab7d67c 100644 --- a/templates/dvcr/secret.yaml +++ b/templates/dvcr/secret.yaml @@ -24,6 +24,7 @@ metadata: type: Opaque data: passwordRW: {{ .Values.virtualization.internal.dvcr.passwordRW | quote }} + passwordPuller: {{ .Values.virtualization.internal.dvcr.passwordPuller | quote }} htpasswd: {{ .Values.virtualization.internal.dvcr.htpasswd | quote }} salt: {{ .Values.virtualization.internal.dvcr.salt | quote }} --- From 97b49dd02670f4733494aad2c3a98fe828efa397 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 14:46:14 +0200 Subject: [PATCH 07/31] feat(dvcr): authenticate importer/uploader Pods with SA token Cut the importer/uploader Pod path over to per-namespace DVCR authorization when dvcr.tenantRegistryAuthorization is enabled: - plumb DVCR_TENANT_AUTHZ_ENABLED into dvcr.Settings.TenantAuthzEnabled - ShouldCopyDVCRAuthSecret no longer copies the shared read-write credential into the tenant namespace when the flag is on - importer/uploader ApplyDVCRDestinationSettings mounts a projected ServiceAccount token (600s, default audience) instead of the dockerconfigjson Secret, and passes its path via *_DESTINATION_TOKEN_FILE - dvcr-artifact reads the token fresh on every request via a custom authn.Authenticator, so kubelet token rotation does not break long pushes The CDI DataVolume path (VirtualDisk provisioning via the upstream CDI importer) still uses the shared credential and is not yet covered; tracked separately. Both modules build; existing unit tests pass; helm template verified for flag on/off. Signed-off-by: Daniil Antoshin --- .../pkg/auth/token_authenticator.go | 51 +++++++++++++++++++ images/dvcr-artifact/pkg/importer/importer.go | 23 +++++++-- images/dvcr-artifact/pkg/registry/registry.go | 36 +++++++++---- images/dvcr-artifact/pkg/uploader/uploader.go | 9 +++- .../pkg/common/consts.go | 14 +++++ .../pkg/common/pod/pod.go | 22 ++++++++ .../pkg/config/load_dvcr_settings.go | 14 +++++ .../pkg/controller/importer/importer_pod.go | 21 ++++++++ .../pkg/controller/importer/settings.go | 15 ++++-- .../pkg/controller/supplements/ensure.go | 6 +++ .../pkg/controller/uploader/settings.go | 15 ++++-- .../pkg/controller/uploader/uploader_pod.go | 21 ++++++++ .../virtualization-artifact/pkg/dvcr/dvcr.go | 5 ++ .../virtualization-controller/_helpers.tpl | 2 + 14 files changed, 232 insertions(+), 22 deletions(-) create mode 100644 images/dvcr-artifact/pkg/auth/token_authenticator.go diff --git a/images/dvcr-artifact/pkg/auth/token_authenticator.go b/images/dvcr-artifact/pkg/auth/token_authenticator.go new file mode 100644 index 0000000000..fd6aa576c0 --- /dev/null +++ b/images/dvcr-artifact/pkg/auth/token_authenticator.go @@ -0,0 +1,51 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "os" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" +) + +// tokenFileAuthenticator authenticates to DVCR with a projected ServiceAccount +// token read fresh from disk on every request. Reading per-call is required +// because kubelet rotates the projected token file; a token cached once would +// break long pushes when it rotates mid-transfer. +type tokenFileAuthenticator struct { + path string +} + +// TokenFileAuthenticator returns an authn.Authenticator that presents the +// contents of the token file as the Basic auth password. The username is a +// fixed non-empty placeholder; the DVCR authorization backend only inspects the +// token in the password. +func TokenFileAuthenticator(path string) authn.Authenticator { + return &tokenFileAuthenticator{path: path} +} + +func (t *tokenFileAuthenticator) Authorization() (*authn.AuthConfig, error) { + token, err := os.ReadFile(t.path) + if err != nil { + return nil, err + } + return &authn.AuthConfig{ + Username: "sa", + Password: strings.TrimSpace(string(token)), + }, nil +} diff --git a/images/dvcr-artifact/pkg/importer/importer.go b/images/dvcr-artifact/pkg/importer/importer.go index 1c6534cb6b..dfe1510f64 100644 --- a/images/dvcr-artifact/pkg/importer/importer.go +++ b/images/dvcr-artifact/pkg/importer/importer.go @@ -50,6 +50,10 @@ const ( DVCRSource = "dvcr" BlockDeviceSource = "blockDevice" FilesystemSource = "filesystem" + + // ImporterDestinationTokenFileVar is the env var with the path to the + // projected ServiceAccount token used for per-namespace DVCR authorization. + ImporterDestinationTokenFileVar = "IMPORTER_DESTINATION_TOKEN_FILE" ) func New() *Importer { @@ -66,6 +70,7 @@ type Importer struct { destImageName string destUsername string destPassword string + destTokenFile string destInsecure bool certDir string sha256Sum string @@ -117,9 +122,10 @@ func (i *Importer) parseOptions() error { } } + i.destTokenFile = os.Getenv(ImporterDestinationTokenFileVar) i.destUsername, _ = util.ParseEnvVar(common.ImporterDestinationAccessKeyID, false) i.destPassword, _ = util.ParseEnvVar(common.ImporterDestinationSecretKey, false) - if i.destUsername == "" && i.destPassword == "" { + if i.destTokenFile == "" && i.destUsername == "" && i.destPassword == "" { destAuthConfig, _ := util.ParseEnvVar(common.ImporterDestinationAuthConfig, false) if destAuthConfig != "" { authFile, err := auth.RegistryAuthFile(destAuthConfig) @@ -164,6 +170,7 @@ func (i *Importer) runForDataSource(ctx context.Context) error { ImageName: i.destImageName, Username: i.destUsername, Password: i.destPassword, + TokenFile: i.destTokenFile, Insecure: i.destInsecure, }, i.sha256Sum, i.md5Sum) if err != nil { @@ -253,12 +260,22 @@ func (i *Importer) destRemoteOptions(ctx context.Context) []remote.Option { remoteOpts := []remote.Option{ remote.WithContext(ctx), remote.WithTransport(transport), - remote.WithAuth(&authn.Basic{Username: i.destUsername, Password: i.destPassword}), + remote.WithAuth(i.destAuthenticator()), } return remoteOpts } +// destAuthenticator returns the authenticator for pushing to DVCR. With +// per-namespace authorization the projected ServiceAccount token is read fresh +// on every request; otherwise static Basic credentials are used. +func (i *Importer) destAuthenticator() authn.Authenticator { + if i.destTokenFile != "" { + return auth.TokenFileAuthenticator(i.destTokenFile) + } + return &authn.Basic{Username: i.destUsername, Password: i.destPassword} +} + func (i *Importer) destCraneOptions(ctx context.Context) []crane.Option { tlsConfig := &tls.Config{ InsecureSkipVerify: i.destInsecure, @@ -270,7 +287,7 @@ func (i *Importer) destCraneOptions(ctx context.Context) []crane.Option { craneOpts := []crane.Option{ crane.WithContext(ctx), crane.WithTransport(transport), - crane.WithAuth(&authn.Basic{Username: i.destUsername, Password: i.destPassword}), + crane.WithAuth(i.destAuthenticator()), } return craneOpts diff --git a/images/dvcr-artifact/pkg/registry/registry.go b/images/dvcr-artifact/pkg/registry/registry.go index a8830bb7cf..d61350d9cc 100644 --- a/images/dvcr-artifact/pkg/registry/registry.go +++ b/images/dvcr-artifact/pkg/registry/registry.go @@ -39,6 +39,7 @@ import ( "golang.org/x/sync/errgroup" "k8s.io/klog/v2" + "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/auth" "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/datasource" importerrs "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/errors" "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/monitoring" @@ -73,6 +74,7 @@ type DataProcessor struct { ds datasource.DataSourceInterface destUsername string destPassword string + destTokenFile string destImageName string sha256Sum string md5Sum string @@ -83,21 +85,35 @@ type DestinationRegistry struct { ImageName string Username string Password string + // TokenFile, when set, is the path to a projected ServiceAccount token used + // for per-namespace DVCR authorization instead of static credentials. + TokenFile string Insecure bool } func NewDataProcessor(ds datasource.DataSourceInterface, dest DestinationRegistry, sha256Sum, md5Sum string) (*DataProcessor, error) { return &DataProcessor{ - ds, - dest.Username, - dest.Password, - dest.ImageName, - sha256Sum, - md5Sum, - dest.Insecure, + ds: ds, + destUsername: dest.Username, + destPassword: dest.Password, + destTokenFile: dest.TokenFile, + destImageName: dest.ImageName, + sha256Sum: sha256Sum, + md5Sum: md5Sum, + destInsecure: dest.Insecure, }, nil } +// destAuthenticator returns the authenticator for pushing to DVCR. With +// per-namespace authorization the projected ServiceAccount token is read fresh +// on every request; otherwise static Basic credentials are used. +func (p DataProcessor) destAuthenticator() authn.Authenticator { + if p.destTokenFile != "" { + return auth.TokenFileAuthenticator(p.destTokenFile) + } + return &authn.Basic{Username: p.destUsername, Password: p.destPassword} +} + func (p DataProcessor) Process(ctx context.Context) (ImportRes, error) { sourceImageFilename, err := p.ds.Filename() if err != nil { @@ -325,7 +341,7 @@ func (p DataProcessor) uploadLayersAndImage( informer *ImageInformer, ) error { nameOpts := destNameOptions(p.destInsecure) - remoteOpts := destRemoteOptions(ctx, p.destUsername, p.destPassword, p.destInsecure) + remoteOpts := destRemoteOptions(ctx, p.destAuthenticator(), p.destInsecure) image := empty.Image ref, err := name.ParseReference(p.destImageName, nameOpts...) @@ -417,7 +433,7 @@ func destNameOptions(destInsecure bool) []name.Option { return nameOpts } -func destRemoteOptions(ctx context.Context, destUsername, destPassword string, destInsecure bool) []remote.Option { +func destRemoteOptions(ctx context.Context, authenticator authn.Authenticator, destInsecure bool) []remote.Option { tlsConfig := &tls.Config{ InsecureSkipVerify: destInsecure, } @@ -428,7 +444,7 @@ func destRemoteOptions(ctx context.Context, destUsername, destPassword string, d remoteOpts := []remote.Option{ remote.WithContext(ctx), remote.WithTransport(transport), - remote.WithAuth(&authn.Basic{Username: destUsername, Password: destPassword}), + remote.WithAuth(authenticator), } return remoteOpts diff --git a/images/dvcr-artifact/pkg/uploader/uploader.go b/images/dvcr-artifact/pkg/uploader/uploader.go index 7d26dcb651..3d6a7b38de 100644 --- a/images/dvcr-artifact/pkg/uploader/uploader.go +++ b/images/dvcr-artifact/pkg/uploader/uploader.go @@ -50,6 +50,10 @@ const ( healthzPort = 8080 healthzPath = "/healthz" uploadPath = "/upload" + + // uploaderDestinationTokenFileVar is the env var with the path to the + // projected ServiceAccount token used for per-namespace DVCR authorization. + uploaderDestinationTokenFileVar = "UPLOADER_DESTINATION_TOKEN_FILE" ) // UploadServer is the interface to uploadServerApp @@ -85,6 +89,7 @@ type uploadServerApp struct { destImageName string destUsername string destPassword string + destTokenFile string destInsecure bool } @@ -125,9 +130,10 @@ func (app *uploadServerApp) parseOptions() error { app.destImageName, _ = util.ParseEnvVar(common.UploaderDestinationEndpoint, false) app.destInsecure, _ = strconv.ParseBool(os.Getenv(common.DestinationInsecureTLSVar)) + app.destTokenFile = os.Getenv(uploaderDestinationTokenFileVar) app.destUsername, _ = util.ParseEnvVar(common.UploaderDestinationAccessKeyID, false) app.destPassword, _ = util.ParseEnvVar(common.UploaderDestinationSecretKey, false) - if app.destUsername == "" && app.destPassword == "" { + if app.destTokenFile == "" && app.destUsername == "" && app.destPassword == "" { destAuthConfig, _ := util.ParseEnvVar(common.UploaderDestinationAuthConfig, false) if destAuthConfig != "" { authFile, err := auth.RegistryAuthFile(destAuthConfig) @@ -389,6 +395,7 @@ func (app *uploadServerApp) upload(stream io.ReadCloser, sourceContentType strin ImageName: app.destImageName, Username: app.destUsername, Password: app.destPassword, + TokenFile: app.destTokenFile, Insecure: app.destInsecure, }, "", "") if err != nil { diff --git a/images/virtualization-artifact/pkg/common/consts.go b/images/virtualization-artifact/pkg/common/consts.go index 41ddc972c7..522875b5dc 100644 --- a/images/virtualization-artifact/pkg/common/consts.go +++ b/images/virtualization-artifact/pkg/common/consts.go @@ -92,6 +92,13 @@ const ( ImporterDestinationAuthConfigVar = "IMPORTER_DESTINATION_AUTH_CONFIG" // ImporterDestinationAuthConfigFile is a path to auth config file in mount directory. ImporterDestinationAuthConfigFile = "/dvcr-auth/.dockerconfigjson" + // ImporterDestinationTokenFileVar is an environment variable with the path to + // the projected ServiceAccount token used for per-namespace DVCR authorization. + ImporterDestinationTokenFileVar = "IMPORTER_DESTINATION_TOKEN_FILE" + // ImporterDestinationTokenDir is a mount directory for the projected SA token. + ImporterDestinationTokenDir = "/dvcr-token" + // ImporterDestinationTokenFile is the path to the projected SA token file. + ImporterDestinationTokenFile = "/dvcr-token/token" // DestinationInsecureTLSVar is an environment variable for Importer Pod that defines whether DVCR is insecure. DestinationInsecureTLSVar = "DESTINATION_INSECURE_TLS" ImporterSHA256Sum = "IMPORTER_SHA256SUM" @@ -105,6 +112,13 @@ const ( UploaderDestinationAuthConfigVar = "UPLOADER_DESTINATION_AUTH_CONFIG" UploaderDestinationAuthConfigDir = "/dvcr-auth" UploaderDestinationAuthConfigFile = "/dvcr-auth/.dockerconfigjson" + // UploaderDestinationTokenFileVar is an environment variable with the path to + // the projected ServiceAccount token used for per-namespace DVCR authorization. + UploaderDestinationTokenFileVar = "UPLOADER_DESTINATION_TOKEN_FILE" + // UploaderDestinationTokenDir is a mount directory for the projected SA token. + UploaderDestinationTokenDir = "/dvcr-token" + // UploaderDestinationTokenFile is the path to the projected SA token file. + UploaderDestinationTokenFile = "/dvcr-token/token" DockerRegistrySchemePrefix = "docker://" diff --git a/images/virtualization-artifact/pkg/common/pod/pod.go b/images/virtualization-artifact/pkg/common/pod/pod.go index b728cd22e2..4d63fe1393 100644 --- a/images/virtualization-artifact/pkg/common/pod/pod.go +++ b/images/virtualization-artifact/pkg/common/pod/pod.go @@ -72,6 +72,28 @@ func CreateVolumeMount(volName, mountPath string) corev1.VolumeMount { } } +// CreateProjectedSATokenVolume creates a volume with a projected ServiceAccount +// token at the given relative path. The audience is left empty so the token is +// accepted by the apiserver's default TokenReview. expirationSeconds controls +// kubelet rotation of the token file. +func CreateProjectedSATokenVolume(volName, path string, expirationSeconds int64) corev1.Volume { + return corev1.Volume{ + Name: volName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Path: path, + ExpirationSeconds: ptr.To(expirationSeconds), + }, + }, + }, + }, + }, + } +} + func AddVolume(pod *corev1.Pod, container *corev1.Container, volume corev1.Volume, mount corev1.VolumeMount, envVar corev1.EnvVar) { pod.Spec.Volumes = append(pod.Spec.Volumes, volume) container.VolumeMounts = append(container.VolumeMounts, mount) diff --git a/images/virtualization-artifact/pkg/config/load_dvcr_settings.go b/images/virtualization-artifact/pkg/config/load_dvcr_settings.go index 5deadedef3..a666299399 100644 --- a/images/virtualization-artifact/pkg/config/load_dvcr_settings.go +++ b/images/virtualization-artifact/pkg/config/load_dvcr_settings.go @@ -19,6 +19,7 @@ package config import ( "fmt" "os" + "strconv" "github.com/deckhouse/virtualization-controller/pkg/dvcr" ) @@ -40,6 +41,8 @@ const ( DVCRImageMonitorScheduleVar = "DVCR_IMAGE_MONITOR_SCHEDULE" // DVCRGCScheduleVar is an env variable holds the cron schedule to run DVCR garbage collection. DVCRGCScheduleVar = "DVCR_GC_SCHEDULE" + // DVCRTenantAuthzEnabledVar is an env variable that enables per-namespace DVCR authorization. + DVCRTenantAuthzEnabledVar = "DVCR_TENANT_AUTHZ_ENABLED" // UploaderIngressHostVar is a env variable UploaderIngressHostVar = "UPLOADER_INGRESS_HOST" @@ -61,6 +64,7 @@ func LoadDVCRSettingsFromEnvs(controllerNamespace string) (*dvcr.Settings, error InsecureTLS: os.Getenv(DVCRInsecureTLSVar), ImageMonitorSchedule: os.Getenv(DVCRImageMonitorScheduleVar), GCSchedule: os.Getenv(DVCRGCScheduleVar), + TenantAuthzEnabled: parseBoolEnv(DVCRTenantAuthzEnabledVar), UploaderIngressSettings: dvcr.UploaderIngressSettings{ Host: os.Getenv(UploaderIngressHostVar), TLSSecret: os.Getenv(UploaderIngressTLSSecretVar), @@ -91,3 +95,13 @@ func LoadDVCRSettingsFromEnvs(controllerNamespace string) (*dvcr.Settings, error return dvcrSettings, nil } + +// parseBoolEnv returns the boolean value of the env variable, or false if it is +// unset or not parseable as a bool. +func parseBoolEnv(name string) bool { + v, err := strconv.ParseBool(os.Getenv(name)) + if err != nil { + return false + } + return v +} diff --git a/images/virtualization-artifact/pkg/controller/importer/importer_pod.go b/images/virtualization-artifact/pkg/controller/importer/importer_pod.go index 719a045061..04f8a40d24 100644 --- a/images/virtualization-artifact/pkg/controller/importer/importer_pod.go +++ b/images/virtualization-artifact/pkg/controller/importer/importer_pod.go @@ -48,6 +48,14 @@ const ( // destinationAuthVol is the name of the volume containing DVCR docker auth config. destinationAuthVol = "dvcr-secret-vol" + // destinationTokenVol is the name of the volume containing the projected + // ServiceAccount token used for per-namespace DVCR authorization. + destinationTokenVol = "dvcr-token-vol" + + // destinationTokenExpirationSeconds is the requested lifetime of the projected + // SA token; kubelet rotates the file well before it expires. + destinationTokenExpirationSeconds = 600 + // sourceRegistryAuthVol is the name of the volume containing source registry docker auth config. sourceRegistryAuthVol = "source-registry-secret-vol" @@ -340,6 +348,19 @@ func (imp *Importer) addVolumes(pod *corev1.Pod, container *corev1.Container) { ) } + if imp.EnvSettings.DestinationTokenAuth { + // Mount a projected ServiceAccount token so the Pod authenticates to DVCR + // with its own namespace-scoped identity. + podutil.AddVolume(pod, container, + podutil.CreateProjectedSATokenVolume(destinationTokenVol, "token", destinationTokenExpirationSeconds), + podutil.CreateVolumeMount(destinationTokenVol, common.ImporterDestinationTokenDir), + corev1.EnvVar{ + Name: common.ImporterDestinationTokenFileVar, + Value: common.ImporterDestinationTokenFile, + }, + ) + } + // Volume with CA certificates either from caBundle field or from existing ConfigMap. if imp.EnvSettings.CertConfigMap != "" { podutil.AddVolume(pod, container, diff --git a/images/virtualization-artifact/pkg/controller/importer/settings.go b/images/virtualization-artifact/pkg/controller/importer/settings.go index 3560c7884e..ad25389bf3 100644 --- a/images/virtualization-artifact/pkg/controller/importer/settings.go +++ b/images/virtualization-artifact/pkg/controller/importer/settings.go @@ -66,14 +66,21 @@ type Settings struct { DestinationEndpoint string DestinationInsecureTLS string DestinationAuthSecret string + // DestinationTokenAuth makes the Pod authenticate to DVCR with a projected + // ServiceAccount token instead of the shared read-write credential. + DestinationTokenAuth bool } func ApplyDVCRDestinationSettings(podEnvVars *Settings, dvcrSettings *dvcr.Settings, supGen supplements.Generator, dvcrImageName string) { - authSecret := dvcrSettings.AuthSecret - if supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { - authSecret = supGen.DVCRAuthSecret().Name + if dvcrSettings.TenantAuthzEnabled { + podEnvVars.DestinationTokenAuth = true + } else { + authSecret := dvcrSettings.AuthSecret + if supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { + authSecret = supGen.DVCRAuthSecret().Name + } + podEnvVars.DestinationAuthSecret = authSecret } - podEnvVars.DestinationAuthSecret = authSecret podEnvVars.DestinationInsecureTLS = dvcrSettings.InsecureTLS podEnvVars.DestinationEndpoint = dvcrImageName } diff --git a/images/virtualization-artifact/pkg/controller/supplements/ensure.go b/images/virtualization-artifact/pkg/controller/supplements/ensure.go index 8a9a62b0e0..816517ba56 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/ensure.go +++ b/images/virtualization-artifact/pkg/controller/supplements/ensure.go @@ -104,6 +104,12 @@ func ShouldCopyDVCRAuthSecret(dvcrSettings *dvcr.Settings, supGen Generator) boo if dvcrSettings.AuthSecret == "" { return false } + // With per-namespace authorization the Pod authenticates with its own + // ServiceAccount token, so the shared read-write credential must not be + // copied into the tenant namespace. + if dvcrSettings.TenantAuthzEnabled { + return false + } // Should copy if namespaces are different. return dvcrSettings.AuthSecretNamespace != supGen.Namespace() } diff --git a/images/virtualization-artifact/pkg/controller/uploader/settings.go b/images/virtualization-artifact/pkg/controller/uploader/settings.go index 13f12013c8..4bea7eb177 100644 --- a/images/virtualization-artifact/pkg/controller/uploader/settings.go +++ b/images/virtualization-artifact/pkg/controller/uploader/settings.go @@ -28,14 +28,21 @@ type Settings struct { DestinationEndpoint string DestinationInsecureTLS string DestinationAuthSecret string + // DestinationTokenAuth makes the Pod authenticate to DVCR with a projected + // ServiceAccount token instead of the shared read-write credential. + DestinationTokenAuth bool } func ApplyDVCRDestinationSettings(podEnvVars *Settings, dvcrSettings *dvcr.Settings, supGen supplements.Generator, dvcrImageName string) { - authSecret := dvcrSettings.AuthSecret - if supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { - authSecret = supGen.DVCRAuthSecret().Name + if dvcrSettings.TenantAuthzEnabled { + podEnvVars.DestinationTokenAuth = true + } else { + authSecret := dvcrSettings.AuthSecret + if supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { + authSecret = supGen.DVCRAuthSecret().Name + } + podEnvVars.DestinationAuthSecret = authSecret } - podEnvVars.DestinationAuthSecret = authSecret podEnvVars.DestinationInsecureTLS = dvcrSettings.InsecureTLS podEnvVars.DestinationEndpoint = dvcrImageName } diff --git a/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go b/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go index 570566ca57..c236397442 100644 --- a/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go +++ b/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go @@ -35,6 +35,14 @@ import ( const ( // destinationAuthVol is the name of the volume containing DVCR docker auth config. destinationAuthVol = "dvcr-secret-vol" + + // destinationTokenVol is the name of the volume containing the projected + // ServiceAccount token used for per-namespace DVCR authorization. + destinationTokenVol = "dvcr-token-vol" + + // destinationTokenExpirationSeconds is the requested lifetime of the projected + // SA token; kubelet rotates the file well before it expires. + destinationTokenExpirationSeconds = 600 ) // These constants can't be imported from "images/dvcr-artifact/pkg/uploader/uploader.go" due to conflicts with the CDI version. @@ -208,4 +216,17 @@ func (p *Pod) addVolumes(pod *corev1.Pod, container *corev1.Container) { }, ) } + + if p.Settings.DestinationTokenAuth { + // Mount a projected ServiceAccount token so the Pod authenticates to DVCR + // with its own namespace-scoped identity. + podutil.AddVolume(pod, container, + podutil.CreateProjectedSATokenVolume(destinationTokenVol, "token", destinationTokenExpirationSeconds), + podutil.CreateVolumeMount(destinationTokenVol, common.UploaderDestinationTokenDir), + corev1.EnvVar{ + Name: common.UploaderDestinationTokenFileVar, + Value: common.UploaderDestinationTokenFile, + }, + ) + } } diff --git a/images/virtualization-artifact/pkg/dvcr/dvcr.go b/images/virtualization-artifact/pkg/dvcr/dvcr.go index 561fc75980..5222620d82 100644 --- a/images/virtualization-artifact/pkg/dvcr/dvcr.go +++ b/images/virtualization-artifact/pkg/dvcr/dvcr.go @@ -42,6 +42,11 @@ type Settings struct { ImageMonitorSchedule string // GCSchedule is a cron formatted schedule to periodically run a garbage collection. GCSchedule string + // TenantAuthzEnabled turns on per-namespace DVCR authorization: importer and + // uploader Pods authenticate with their ServiceAccount token instead of the + // shared read-write credential, which is then no longer copied into tenant + // namespaces. + TenantAuthzEnabled bool } type UploaderIngressSettings struct { diff --git a/templates/virtualization-controller/_helpers.tpl b/templates/virtualization-controller/_helpers.tpl index fc6e340119..faecd31cd4 100644 --- a/templates/virtualization-controller/_helpers.tpl +++ b/templates/virtualization-controller/_helpers.tpl @@ -40,6 +40,8 @@ true value: "*/5 * * * *" - name: DVCR_GC_SCHEDULE value: "{{ .Values.virtualization.internal.moduleConfig | dig "dvcr" "gc" "schedule" "" }}" +- name: DVCR_TENANT_AUTHZ_ENABLED + value: "{{ include "dvcr.isTenantAuthz" . }}" - name: VIRTUAL_MACHINE_CIDRS value: {{ join "," .Values.virtualization.internal.moduleConfig.virtualMachineCIDRs | quote }} {{- if (hasKey .Values.virtualization.internal.moduleConfig "virtualImages") }} From c8454c4177f0983fea187ca1548a336bb080028a Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 15:07:24 +0200 Subject: [PATCH 08/31] feat(dvcr): use SA-token auth for VirtualDisk CDI imports Close the CDI/VirtualDisk gap in per-namespace DVCR authorization. When dvcr.tenantRegistryAuthorization is enabled, EnsureForDataVolume no longer copies the shared read-write credential into the tenant namespace; instead it writes a CDI-compatible Secret whose accessKeyId is the ServiceAccount-token sentinel. The patched CDI importer reads that sentinel and authenticates to DVCR with the importer Pod's own projected ServiceAccount token, which the registry authorizes for the tenant namespace only. Requires the matching CDI fork change (feat(importer): support ServiceAccount token auth for registry source). Signed-off-by: Daniil Antoshin --- .../supplements/copier/auth_secret.go | 20 +++++++++++++++++++ .../pkg/controller/supplements/ensure.go | 11 ++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go index cd461109c7..183124b497 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go +++ b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go @@ -26,6 +26,12 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/common/object" ) +// ServiceAccountTokenUsername is a sentinel accessKeyId value that instructs the +// (patched) CDI importer to authenticate to the registry with the Pod's projected +// ServiceAccount token instead of static credentials. Must match the constant in +// the CDI fork (deckhouse/3p-containerized-data-importer, pkg/importer/transport.go). +const ServiceAccountTokenUsername = "dvcr-serviceaccount-token-auth" + // AuthSecret copies auth credentials from the source Secret into // Destination Secret and ensure its data is CDI compatible: // type: Opaque @@ -39,6 +45,20 @@ type AuthSecret struct { Secret } +// CreateServiceAccountTokenAuth creates a CDI-compatible Opaque Secret whose +// accessKeyId is the ServiceAccount-token sentinel, so the importer Pod +// authenticates to DVCR with its own namespace-scoped identity instead of the +// shared read-write credential. Used for VirtualDisk imports when per-namespace +// DVCR authorization is enabled. +func (a AuthSecret) CreateServiceAccountTokenAuth(ctx context.Context, client client.Client) error { + destData := map[string][]byte{ + "accessKeyId": []byte(ServiceAccountTokenUsername), + "secretKey": []byte(""), + } + _, err := a.Create(ctx, client, destData, corev1.SecretTypeOpaque) + return err +} + // CopyCDICompatible transforms auth credentials in dockerconfigjson format into CDI compatible: // a Secret with two fields: accessKeyId and secretKey. // ref is registry url or image name. It is used to select a desired auth pair from the config. diff --git a/images/virtualization-artifact/pkg/controller/supplements/ensure.go b/images/virtualization-artifact/pkg/controller/supplements/ensure.go index 816517ba56..e48701d4a3 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/ensure.go +++ b/images/virtualization-artifact/pkg/controller/supplements/ensure.go @@ -147,8 +147,15 @@ func EnsureForDataVolume(ctx context.Context, client client.Client, supGen DataV }, } - err := authCopier.CopyCDICompatible(ctx, client, dvcrSettings.RegistryURL) - if err != nil { + // With per-namespace authorization the importer Pod authenticates to DVCR + // with its own ServiceAccount token (via the sentinel credential understood + // by the patched CDI importer), so the shared read-write credential is not + // copied into the tenant namespace. + if dvcrSettings.TenantAuthzEnabled { + if err := authCopier.CreateServiceAccountTokenAuth(ctx, client); err != nil { + return err + } + } else if err := authCopier.CopyCDICompatible(ctx, client, dvcrSettings.RegistryURL); err != nil { return err } } From 67c0a5a2ef8a5289a190a9b9e7b1b3717efa8e29 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 16:05:14 +0200 Subject: [PATCH 09/31] fix(dvcr): grant admin to ServiceAccounts in the module namespace Control-plane components run in the module namespace (d8-virtualization) as its default ServiceAccount: the controller, the ClusterVirtualImage importer and garbage collection. They legitimately read images across namespaces and write cluster-scoped cvi repositories. Classifying them as namespace-scoped tenants broke ClusterVirtualImage imports (cross-namespace source pull and cvi push returned 400). Add a privilegednamespace option to the dvcr-k8s auth backend: a ServiceAccount whose TokenReview namespace equals it is granted admin. Tenants cannot run Pods in that namespace, so this is safe. Rendered from configmap.yaml as d8-. Signed-off-by: Daniil Antoshin --- images/dvcr/dvcr-k8s-auth/access.go | 25 ++++++++++++------ images/dvcr/dvcr-k8s-auth/policy.go | 17 +++++++++++++ images/dvcr/dvcr-k8s-auth/policy_test.go | 32 ++++++++++++++++++++---- templates/dvcr/configmap.yaml | 1 + 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/images/dvcr/dvcr-k8s-auth/access.go b/images/dvcr/dvcr-k8s-auth/access.go index bc48b2158d..37bf9a6b9f 100644 --- a/images/dvcr/dvcr-k8s-auth/access.go +++ b/images/dvcr/dvcr-k8s-auth/access.go @@ -50,6 +50,8 @@ type accessController struct { pullerUsername string pullerPassword []byte + privilegedNamespace string + reviewer *tokenReviewer } @@ -87,6 +89,11 @@ func newAccessController(options map[string]interface{}) (auth.AccessController, return nil, err } + privilegedNamespace, err := optString(options, "privilegednamespace", "") + if err != nil { + return nil, err + } + ttl := optDuration(options, "tokenreviewcachettl", 45*time.Second) negTTL := optDuration(options, "tokenreviewcachenegttl", 5*time.Second) @@ -96,12 +103,13 @@ func newAccessController(options map[string]interface{}) (auth.AccessController, } return &accessController{ - realm: realm, - adminUsername: adminUser, - adminPassword: adminPass, - pullerUsername: pullerUser, - pullerPassword: pullerPass, - reviewer: reviewer, + realm: realm, + adminUsername: adminUser, + adminPassword: adminPass, + pullerUsername: pullerUser, + pullerPassword: pullerPass, + privilegedNamespace: privilegedNamespace, + reviewer: reviewer, }, nil } @@ -144,10 +152,11 @@ func (ac *accessController) classify(req *http.Request, username, password strin if err != nil { return Subject{}, "", fmt.Errorf("token review: %w", err) } - if ns == "" { + subject := SubjectForServiceAccount(ns, ac.privilegedNamespace) + if subject.Role == RoleNone { return Subject{}, "", errors.New("token is not a namespaced ServiceAccount") } - return Subject{Role: RoleTenant, Namespace: ns}, "serviceaccount:" + ns, nil + return subject, "serviceaccount:" + ns, nil } func toPolicyAccess(records []auth.Access) []Access { diff --git a/images/dvcr/dvcr-k8s-auth/policy.go b/images/dvcr/dvcr-k8s-auth/policy.go index 7b902506ef..2bb2e9670c 100644 --- a/images/dvcr/dvcr-k8s-auth/policy.go +++ b/images/dvcr/dvcr-k8s-auth/policy.go @@ -73,6 +73,23 @@ const ( // saUserPrefix is the TokenReview username prefix for a ServiceAccount. const saUserPrefix = "system:serviceaccount:" +// SubjectForServiceAccount maps an authenticated ServiceAccount namespace to an +// authorization Subject. ServiceAccounts in the module's own privileged namespace +// (e.g. d8-virtualization) are control-plane components — the controller, the +// ClusterVirtualImage importer, garbage collection — that legitimately read images +// across namespaces and write cluster-scoped cvi repositories, so they are granted +// admin. Tenants cannot run Pods in that namespace, so this is safe. Every other +// namespace is namespace-scoped. +func SubjectForServiceAccount(saNamespace, privilegedNamespace string) Subject { + if saNamespace == "" { + return Subject{Role: RoleNone} + } + if privilegedNamespace != "" && saNamespace == privilegedNamespace { + return Subject{Role: RoleAdmin} + } + return Subject{Role: RoleTenant, Namespace: saNamespace} +} + // namespaceFromUsername parses "system:serviceaccount::" and returns // . Any other username shape returns "" (denied by the caller). func namespaceFromUsername(username string) string { diff --git a/images/dvcr/dvcr-k8s-auth/policy_test.go b/images/dvcr/dvcr-k8s-auth/policy_test.go index 96e69e9094..b58d37e083 100644 --- a/images/dvcr/dvcr-k8s-auth/policy_test.go +++ b/images/dvcr/dvcr-k8s-auth/policy_test.go @@ -106,6 +106,28 @@ func TestAuthorize(t *testing.T) { } } +func TestSubjectForServiceAccount(t *testing.T) { + const priv = "d8-virtualization" + tests := []struct { + name string + saNs string + privNs string + want Subject + }{ + {"module namespace is admin", "d8-virtualization", priv, Subject{Role: RoleAdmin}}, + {"tenant namespace is scoped", "tenant-a", priv, Subject{Role: RoleTenant, Namespace: "tenant-a"}}, + {"empty namespace denied", "", priv, Subject{Role: RoleNone}}, + {"no privileged ns configured, tenant stays scoped", "d8-virtualization", "", Subject{Role: RoleTenant, Namespace: "d8-virtualization"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := SubjectForServiceAccount(tt.saNs, tt.privNs); got != tt.want { + t.Errorf("SubjectForServiceAccount(%q, %q) = %+v, want %+v", tt.saNs, tt.privNs, got, tt.want) + } + }) + } +} + func TestNamespaceFromUsername(t *testing.T) { tests := []struct { username string @@ -113,11 +135,11 @@ func TestNamespaceFromUsername(t *testing.T) { }{ {"system:serviceaccount:nsA:importer", "nsA"}, {"system:serviceaccount:d8-virtualization:dvcr", "d8-virtualization"}, - {"system:serviceaccount::name", ""}, // empty namespace - {"system:serviceaccount:nsA", ""}, // no name separator - {"system:node:worker-1", ""}, // not a service account - {"kubernetes-admin", ""}, // human user - {"", ""}, // empty + {"system:serviceaccount::name", ""}, // empty namespace + {"system:serviceaccount:nsA", ""}, // no name separator + {"system:node:worker-1", ""}, // not a service account + {"kubernetes-admin", ""}, // human user + {"", ""}, // empty } for _, tt := range tests { if got := namespaceFromUsername(tt.username); got != tt.want { diff --git a/templates/dvcr/configmap.yaml b/templates/dvcr/configmap.yaml index ca66837f46..78a0ff456f 100644 --- a/templates/dvcr/configmap.yaml +++ b/templates/dvcr/configmap.yaml @@ -27,6 +27,7 @@ data: adminpasswordfile: /auth/passwordRW pullerusername: "node-puller" pullerpasswordfile: /auth/passwordPuller + privilegednamespace: d8-{{ .Chart.Name }} tokenreviewcachettl: "45s" tokenreviewcachenegttl: "5s" {{- end }} From 3a677f8e396c3602176fa56e49da48f74d9d99fe Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 16:48:05 +0200 Subject: [PATCH 10/31] refactor(dvcr): verify scoped JWT tokens instead of TokenReview Replace the TokenReview-based per-namespace classification with signed scoped tokens: importer/uploader Pods present a JWT (minted by the controller) carrying an access claim for exactly the repository they use. The registry plugin reuses distribution's own token verification (signature, issuer, audience, nbf/exp) and authorizes each requested access against the token's grants. - policy.go: RoleScoped carrying Grants; drop tenant/TokenReview roles - access.go: verify JWT via registry/auth/token; drop namespace derivation - remove tokenreview.go (no apiserver TokenReview on the push path) Signed-off-by: Daniil Antoshin --- images/dvcr/dvcr-k8s-auth/access.go | 144 ++++++++++------ images/dvcr/dvcr-k8s-auth/policy.go | 129 +++++---------- images/dvcr/dvcr-k8s-auth/policy_test.go | 124 +++++--------- images/dvcr/dvcr-k8s-auth/tokenreview.go | 202 ----------------------- 4 files changed, 171 insertions(+), 428 deletions(-) delete mode 100644 images/dvcr/dvcr-k8s-auth/tokenreview.go diff --git a/images/dvcr/dvcr-k8s-auth/access.go b/images/dvcr/dvcr-k8s-auth/access.go index 37bf9a6b9f..05021c47c7 100644 --- a/images/dvcr/dvcr-k8s-auth/access.go +++ b/images/dvcr/dvcr-k8s-auth/access.go @@ -23,18 +23,27 @@ limitations under the License. package dvcrk8s import ( + "crypto" "crypto/subtle" + "crypto/x509" + "encoding/pem" "errors" "fmt" "log" "net/http" "os" "strings" - "time" + + jose "github.com/go-jose/go-jose/v4" "github.com/distribution/distribution/v3/registry/auth" + "github.com/distribution/distribution/v3/registry/auth/token" ) +// jwtSigningAlgs are the JWS algorithms accepted for scoped tokens. ES256 only: +// tokens are minted by the virtualization-controller with an ECDSA P-256 key. +var jwtSigningAlgs = []jose.SignatureAlgorithm{jose.ES256} + func init() { if err := auth.Register("dvcr-k8s", auth.InitFunc(newAccessController)); err != nil { log.Printf("failed to register dvcr-k8s auth: %v", err) @@ -50,21 +59,23 @@ type accessController struct { pullerUsername string pullerPassword []byte - privilegedNamespace string - - reviewer *tokenReviewer + jwtIssuer string + jwtAudience string + trustedKeys map[string]crypto.PublicKey } // newAccessController builds the controller from the `auth: { dvcr-k8s: {...} }` // config block. Options: // -// realm string (WWW-Authenticate realm) -// adminusername string -// adminpasswordfile string (path to the admin password, e.g. from dvcr-secrets) -// pullerusername string -// pullerpasswordfile string (path to the node-puller password) -// tokenreviewcachettl string (Go duration, default 45s) -// tokenreviewcachenegttl string (Go duration for failed reviews, default 5s) +// realm string (WWW-Authenticate realm) +// adminusername string +// adminpasswordfile string (path to the admin read-write password) +// pullerusername string +// pullerpasswordfile string (path to the node-puller password) +// jwtissuer string (accepted token issuer) +// jwtaudience string (accepted token audience) +// jwtpublickeyfile string (PEM PKIX public key that signs scoped tokens) +// jwtkeyid string (key id the token header must reference, default "dvcr") func newAccessController(options map[string]interface{}) (auth.AccessController, error) { realm, err := optString(options, "realm", "dvcr") if err != nil { @@ -89,27 +100,39 @@ func newAccessController(options map[string]interface{}) (auth.AccessController, return nil, err } - privilegedNamespace, err := optString(options, "privilegednamespace", "") + jwtIssuer, err := optString(options, "jwtissuer", "") if err != nil { return nil, err } - - ttl := optDuration(options, "tokenreviewcachettl", 45*time.Second) - negTTL := optDuration(options, "tokenreviewcachenegttl", 5*time.Second) - - reviewer, err := newTokenReviewer(ttl, negTTL) + jwtAudience, err := optString(options, "jwtaudience", "") + if err != nil { + return nil, err + } + keyID, err := optString(options, "jwtkeyid", "dvcr") + if err != nil { + return nil, err + } + keyPath, err := optString(options, "jwtpublickeyfile", "") + if err != nil { + return nil, err + } + if keyPath == "" { + return nil, errors.New("dvcr-k8s: jwtpublickeyfile is required") + } + pub, err := loadPublicKey(keyPath) if err != nil { - return nil, fmt.Errorf("init token reviewer: %w", err) + return nil, fmt.Errorf("dvcr-k8s: load jwt public key: %w", err) } return &accessController{ - realm: realm, - adminUsername: adminUser, - adminPassword: adminPass, - pullerUsername: pullerUser, - pullerPassword: pullerPass, - privilegedNamespace: privilegedNamespace, - reviewer: reviewer, + realm: realm, + adminUsername: adminUser, + adminPassword: adminPass, + pullerUsername: pullerUser, + pullerPassword: pullerPass, + jwtIssuer: jwtIssuer, + jwtAudience: jwtAudience, + trustedKeys: map[string]crypto.PublicKey{keyID: pub}, }, nil } @@ -120,7 +143,7 @@ func (ac *accessController) Authorized(req *http.Request, accessRecords ...auth. return nil, &challenge{realm: ac.realm, err: auth.ErrInvalidCredential} } - subject, name, err := ac.classify(req, username, password) + subject, name, err := ac.classify(username, password) if err != nil { // Bad credential -> 401 challenge so the client may retry with valid creds. return nil, &challenge{realm: ac.realm, err: err} @@ -134,10 +157,10 @@ func (ac *accessController) Authorized(req *http.Request, accessRecords ...auth. return &auth.Grant{User: auth.UserInfo{Name: name}}, nil } -// classify maps a Basic credential to an authorization Subject. Static admin and -// node-puller passwords are matched in constant time; anything else is treated as -// a ServiceAccount token and verified via TokenReview (fail-closed on any error). -func (ac *accessController) classify(req *http.Request, username, password string) (Subject, string, error) { +// classify maps a Basic credential to an authorization Subject. The static admin +// and node-puller passwords are matched in constant time; any other credential is +// treated as a signed scoped token, its password verified as a JWT (fail-closed). +func (ac *accessController) classify(username, password string) (Subject, string, error) { if len(ac.adminPassword) > 0 && username == ac.adminUsername && subtle.ConstantTimeCompare([]byte(password), ac.adminPassword) == 1 { return Subject{Role: RoleAdmin}, username, nil @@ -148,15 +171,36 @@ func (ac *accessController) classify(req *http.Request, username, password strin return Subject{Role: RolePuller}, username, nil } - ns, err := ac.reviewer.namespaceForToken(req.Context(), password) + grants, err := ac.verifyJWT(password) + if err != nil { + return Subject{}, "", fmt.Errorf("scoped token: %w", err) + } + return Subject{Role: RoleScoped, Grants: grants}, "scoped:" + username, nil +} + +// verifyJWT parses and verifies a scoped token, reusing distribution's own token +// verification (signature, issuer, audience, nbf/exp), and returns its grants. +func (ac *accessController) verifyJWT(raw string) ([]Grant, error) { + tok, err := token.NewToken(raw, jwtSigningAlgs) + if err != nil { + return nil, err + } + claims, err := tok.Verify(token.VerifyOptions{ + TrustedIssuers: []string{ac.jwtIssuer}, + AcceptedAudiences: []string{ac.jwtAudience}, + TrustedKeys: ac.trustedKeys, + }) if err != nil { - return Subject{}, "", fmt.Errorf("token review: %w", err) + return nil, err } - subject := SubjectForServiceAccount(ns, ac.privilegedNamespace) - if subject.Role == RoleNone { - return Subject{}, "", errors.New("token is not a namespaced ServiceAccount") + grants := make([]Grant, 0, len(claims.Access)) + for _, ra := range claims.Access { + if ra == nil { + continue + } + grants = append(grants, Grant{Type: ra.Type, Name: ra.Name, Actions: ra.Actions}) } - return subject, "serviceaccount:" + ns, nil + return grants, nil } func toPolicyAccess(records []auth.Access) []Access { @@ -181,32 +225,28 @@ func (ch *challenge) Error() string { return fmt.Sprintf("dvcr-k8s basic auth challenge for realm %q: %v", ch.realm, ch.err) } -func optString(options map[string]interface{}, key, def string) (string, error) { - v, present := options[key] - if !present { - return def, nil +func loadPublicKey(path string) (crypto.PublicKey, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err } - s, ok := v.(string) - if !ok { - return "", fmt.Errorf("dvcr-k8s: option %q must be a string", key) + block, _ := pem.Decode(data) + if block == nil { + return nil, errors.New("no PEM block found") } - return s, nil + return x509.ParsePKIXPublicKey(block.Bytes) } -func optDuration(options map[string]interface{}, key string, def time.Duration) time.Duration { +func optString(options map[string]interface{}, key, def string) (string, error) { v, present := options[key] if !present { - return def + return def, nil } s, ok := v.(string) if !ok { - return def - } - d, err := time.ParseDuration(strings.TrimSpace(s)) - if err != nil { - return def + return "", fmt.Errorf("dvcr-k8s: option %q must be a string", key) } - return d + return s, nil } func readSecretFile(options map[string]interface{}, key string) ([]byte, error) { diff --git a/images/dvcr/dvcr-k8s-auth/policy.go b/images/dvcr/dvcr-k8s-auth/policy.go index 2bb2e9670c..60b8ea03ad 100644 --- a/images/dvcr/dvcr-k8s-auth/policy.go +++ b/images/dvcr/dvcr-k8s-auth/policy.go @@ -18,7 +18,7 @@ limitations under the License. // // This file holds the pure, dependency-free authorization decision logic so it // can be unit-tested without the distribution registry runtime. The distribution -// auth.AccessController glue (request parsing, TokenReview) lives in access.go +// auth.AccessController glue (Basic parsing, JWT verification) lives in access.go // and only translates registry types into the Subject/Access values decided here. package dvcrk8s @@ -33,22 +33,28 @@ type Role int const ( // RoleNone denies everything (fail-closed default). RoleNone Role = iota - // RoleAdmin grants full access. Used by the virtualization-controller. + // RoleAdmin grants full access. Presented by the virtualization-controller + // with the static read-write password. RoleAdmin - // RolePuller grants pull-only access to any repository. Used by node containerd. + // RolePuller grants pull-only access to any repository. Presented by node + // containerd with the static node-puller password. RolePuller - // RoleTenant grants namespace-scoped access. Used by importer/uploader Pods. - RoleTenant + // RoleScoped grants exactly the access carried in the credential. Presented by + // importer/uploader Pods with a signed JWT the controller minted for the single + // repository that Pod reads from / writes to. + RoleScoped ) // Subject is the authenticated caller together with its authorization scope. type Subject struct { Role Role - // Namespace is the tenant namespace; only meaningful for RoleTenant. - Namespace string + // Grants is the set of allowed repository actions carried by a RoleScoped + // credential (the JWT's access claim). Ignored for other roles. + Grants []Grant } -// Access mirrors the subset of distribution's auth.Access needed for a decision. +// Access mirrors the subset of distribution's auth.Access needed for a single +// authorization decision (one requested resource + action). type Access struct { // Type is the resource type: "repository" or "registry". Type string @@ -59,49 +65,12 @@ type Access struct { Action string } -// Repository path prefixes, kept in sync with pkg/dvcr/dvcr.go image templates: -// -// cvi/ (ClusterVirtualImage, cluster-scoped, shared read-only) -// vi// (VirtualImage, namespaced) -// vd// (VirtualDisk, namespaced) -const ( - prefixCVI = "cvi" - prefixVI = "vi" - prefixVD = "vd" -) - -// saUserPrefix is the TokenReview username prefix for a ServiceAccount. -const saUserPrefix = "system:serviceaccount:" - -// SubjectForServiceAccount maps an authenticated ServiceAccount namespace to an -// authorization Subject. ServiceAccounts in the module's own privileged namespace -// (e.g. d8-virtualization) are control-plane components — the controller, the -// ClusterVirtualImage importer, garbage collection — that legitimately read images -// across namespaces and write cluster-scoped cvi repositories, so they are granted -// admin. Tenants cannot run Pods in that namespace, so this is safe. Every other -// namespace is namespace-scoped. -func SubjectForServiceAccount(saNamespace, privilegedNamespace string) Subject { - if saNamespace == "" { - return Subject{Role: RoleNone} - } - if privilegedNamespace != "" && saNamespace == privilegedNamespace { - return Subject{Role: RoleAdmin} - } - return Subject{Role: RoleTenant, Namespace: saNamespace} -} - -// namespaceFromUsername parses "system:serviceaccount::" and returns -// . Any other username shape returns "" (denied by the caller). -func namespaceFromUsername(username string) string { - if !strings.HasPrefix(username, saUserPrefix) { - return "" - } - rest := strings.TrimPrefix(username, saUserPrefix) - ns, _, ok := strings.Cut(rest, ":") - if !ok || ns == "" { - return "" - } - return ns +// Grant is one entry of a RoleScoped credential's access claim: a resource with +// the set of actions permitted on it. Mirrors distribution's token ResourceActions. +type Grant struct { + Type string + Name string + Actions []string } // Authorize returns true only if the subject is allowed every requested access. @@ -123,48 +92,34 @@ func authorizeOne(s Subject, a Access) bool { case RolePuller: // Nodes only ever pull image layers; never push or delete, never enumerate. return a.Type == "repository" && a.Action == "pull" - case RoleTenant: - return authorizeTenant(s.Namespace, a) + case RoleScoped: + return grantsCover(s.Grants, a) default: return false } } -func authorizeTenant(ns string, a Access) bool { - // Deny the registry-wide catalog and any non-repository resource: it would - // enumerate every tenant's images. - if a.Type != "repository" { - return false - } - // Empty namespace can never match a repository segment; deny. - if ns == "" { - return false - } - - seg := splitClean(a.Name) - switch { - case len(seg) == 3 && (seg[0] == prefixVI || seg[0] == prefixVD) && seg[1] == ns: - // Own-namespace VirtualImage / VirtualDisk: read and write. - return a.Action == "pull" || a.Action == "push" - case len(seg) >= 2 && seg[0] == prefixCVI: - // Cluster images are shared; tenants may read them as disk/image sources, - // but only the controller (RoleAdmin) creates them. - // ponytail: pull-only for tenants; if a future flow pushes cvi from a - // tenant Pod, grant push here and cover it with an e2e test. - return a.Action == "pull" - default: - return false +// grantsCover reports whether any grant permits the requested access. Names are +// path-cleaned on both sides so trailing slashes or "." segments cannot smuggle a +// mismatch past an exact string compare. +func grantsCover(grants []Grant, a Access) bool { + name := cleanName(a.Name) + for i := range grants { + g := grants[i] + if g.Type != a.Type || cleanName(g.Name) != name { + continue + } + for _, act := range g.Actions { + if act == a.Action || act == "*" { + return true + } + } } + return false } -// splitClean normalizes a repository name into path segments, neutralizing any -// "." / ".." traversal by anchoring the clean at root. Returns nil for an empty -// or root-only path. Using path.Clean (not HasPrefix) is what prevents a name -// like "vi/nsA-evil" or "vi/nsA/../nsB/x" from being mistaken for namespace nsA. -func splitClean(name string) []string { - cleaned := strings.TrimPrefix(path.Clean("/"+name), "/") - if cleaned == "" || cleaned == "." { - return nil - } - return strings.Split(cleaned, "/") +// cleanName normalizes a resource name by anchoring path.Clean at root, so +// "cvi/foo/", "cvi/foo" and "cvi/./foo" compare equal and no "../" can escape. +func cleanName(name string) string { + return strings.TrimPrefix(path.Clean("/"+name), "/") } diff --git a/images/dvcr/dvcr-k8s-auth/policy_test.go b/images/dvcr/dvcr-k8s-auth/policy_test.go index b58d37e083..7c7b1c9cd6 100644 --- a/images/dvcr/dvcr-k8s-auth/policy_test.go +++ b/images/dvcr/dvcr-k8s-auth/policy_test.go @@ -25,8 +25,18 @@ func repo(name, action string) Access { return Access{Type: "repository", Name: name, Action: action} } +func grantRepo(name string, actions ...string) Grant { + return Grant{Type: "repository", Name: name, Actions: actions} +} + func TestAuthorize(t *testing.T) { - tenantA := Subject{Role: RoleTenant, Namespace: "nsA"} + // A Pod scoped to its own cvi repository (the CVI-from-PVC case). + scopedCVI := Subject{Role: RoleScoped, Grants: []Grant{grantRepo("cvi/my-image", "pull", "push")}} + // A Pod scoped to a namespaced vd plus a DVCR source it pulls from. + scopedVD := Subject{Role: RoleScoped, Grants: []Grant{ + grantRepo("vd/nsA/disk", "pull", "push"), + grantRepo("cvi/base", "pull"), + }} tests := []struct { name string @@ -34,45 +44,32 @@ func TestAuthorize(t *testing.T) { accesses []Access want bool }{ - // --- tenant: own namespace --- - {"tenant pull own vi", tenantA, []Access{repo("vi/nsA/img", "pull")}, true}, - {"tenant push own vi", tenantA, []Access{repo("vi/nsA/img", "push")}, true}, - {"tenant pull own vd", tenantA, []Access{repo("vd/nsA/disk", "pull")}, true}, - {"tenant push own vd", tenantA, []Access{repo("vd/nsA/disk", "push")}, true}, - {"tenant delete own vi denied", tenantA, []Access{repo("vi/nsA/img", "delete")}, false}, - - // --- tenant: cross-namespace (the core vulnerability) --- - {"tenant pull other-ns vi denied", tenantA, []Access{repo("vi/nsB/img", "pull")}, false}, - {"tenant push other-ns vi denied", tenantA, []Access{repo("vi/nsB/img", "push")}, false}, - {"tenant push other-ns vd denied", tenantA, []Access{repo("vd/nsB/disk", "push")}, false}, - - // --- tenant: cvi (cluster, shared read-only) --- - {"tenant pull cvi ok", tenantA, []Access{repo("cvi/ubuntu", "pull")}, true}, - {"tenant push cvi denied", tenantA, []Access{repo("cvi/ubuntu", "push")}, false}, - - // --- tenant: catalog / registry enumeration --- - {"tenant catalog denied", tenantA, []Access{{Type: "registry", Name: "catalog", Action: "*"}}, false}, - - // --- tenant: path normalization / prefix confusion --- - {"tenant prefix confusion nsA-evil denied", tenantA, []Access{repo("vi/nsA-evil/img", "push")}, false}, - {"tenant traversal to other ns denied", tenantA, []Access{repo("vi/nsA/../nsB/img", "push")}, false}, - {"tenant traversal escape denied", tenantA, []Access{repo("vi/nsA/../../etc/x", "push")}, false}, - {"tenant unknown prefix denied", tenantA, []Access{repo("foo/nsA/img", "push")}, false}, - {"tenant bare ns no name denied", tenantA, []Access{repo("vi/nsA", "push")}, false}, - - // --- tenant: cross-repo blob mount (distribution issues a pull on the from-repo) --- - {"tenant mount from own ns ok", tenantA, []Access{ - repo("vi/nsA/dst", "push"), - repo("vi/nsA/src", "pull"), + // --- scoped: exactly the granted repo/action --- + {"scoped push own cvi", scopedCVI, []Access{repo("cvi/my-image", "push")}, true}, + {"scoped pull own cvi", scopedCVI, []Access{repo("cvi/my-image", "pull")}, true}, + {"scoped delete own cvi denied", scopedCVI, []Access{repo("cvi/my-image", "delete")}, false}, + + // --- scoped: any other repository is denied (the core guarantee) --- + {"scoped push other cvi denied", scopedCVI, []Access{repo("cvi/other", "push")}, false}, + {"scoped push tenant repo denied", scopedCVI, []Access{repo("vd/nsB/disk", "push")}, false}, + {"scoped prefix confusion denied", scopedCVI, []Access{repo("cvi/my-image-evil", "push")}, false}, + {"scoped traversal denied", scopedCVI, []Access{repo("cvi/my-image/../other", "push")}, false}, + {"scoped catalog denied", scopedCVI, []Access{{Type: "registry", Name: "catalog", Action: "*"}}, false}, + + // --- scoped: name normalization compares equal --- + {"scoped trailing slash ok", scopedCVI, []Access{repo("cvi/my-image/", "push")}, true}, + + // --- scoped: destination push + source pull (cross-repo, both granted) --- + {"scoped push dst pull src ok", scopedVD, []Access{ + repo("vd/nsA/disk", "push"), + repo("cvi/base", "pull"), }, true}, - {"tenant mount from other ns denied", tenantA, []Access{ - repo("vi/nsA/dst", "push"), - repo("vi/nsB/src", "pull"), + {"scoped push src denied", scopedVD, []Access{repo("cvi/base", "push")}, false}, + {"scoped mixed one bad denies all", scopedVD, []Access{ + repo("vd/nsA/disk", "push"), + repo("cvi/other", "pull"), }, false}, - // --- tenant: empty namespace is never valid --- - {"tenant empty ns denied", Subject{Role: RoleTenant, Namespace: ""}, []Access{repo("vi//img", "pull")}, false}, - // --- puller (nodes) --- {"puller pull any ok", Subject{Role: RolePuller}, []Access{repo("vi/nsB/img", "pull")}, true}, {"puller pull cvi ok", Subject{Role: RolePuller}, []Access{repo("cvi/ubuntu", "pull")}, true}, @@ -85,16 +82,11 @@ func TestAuthorize(t *testing.T) { {"admin catalog ok", Subject{Role: RoleAdmin}, []Access{{Type: "registry", Name: "catalog", Action: "*"}}, true}, // --- fail-closed default --- - {"none denied", Subject{Role: RoleNone}, []Access{repo("vi/nsA/img", "pull")}, false}, - - // --- mixed list: one denied access denies all --- - {"tenant mixed one bad denies all", tenantA, []Access{ - repo("vi/nsA/ok", "pull"), - repo("vi/nsB/bad", "pull"), - }, false}, + {"none denied", Subject{Role: RoleNone}, []Access{repo("cvi/x", "pull")}, false}, + {"scoped no grants denied", Subject{Role: RoleScoped}, []Access{repo("cvi/x", "pull")}, false}, // --- empty access list is allowed (capability probe) --- - {"empty access allowed", tenantA, nil, true}, + {"empty access allowed", scopedCVI, nil, true}, } for _, tt := range tests { @@ -105,45 +97,3 @@ func TestAuthorize(t *testing.T) { }) } } - -func TestSubjectForServiceAccount(t *testing.T) { - const priv = "d8-virtualization" - tests := []struct { - name string - saNs string - privNs string - want Subject - }{ - {"module namespace is admin", "d8-virtualization", priv, Subject{Role: RoleAdmin}}, - {"tenant namespace is scoped", "tenant-a", priv, Subject{Role: RoleTenant, Namespace: "tenant-a"}}, - {"empty namespace denied", "", priv, Subject{Role: RoleNone}}, - {"no privileged ns configured, tenant stays scoped", "d8-virtualization", "", Subject{Role: RoleTenant, Namespace: "d8-virtualization"}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := SubjectForServiceAccount(tt.saNs, tt.privNs); got != tt.want { - t.Errorf("SubjectForServiceAccount(%q, %q) = %+v, want %+v", tt.saNs, tt.privNs, got, tt.want) - } - }) - } -} - -func TestNamespaceFromUsername(t *testing.T) { - tests := []struct { - username string - want string - }{ - {"system:serviceaccount:nsA:importer", "nsA"}, - {"system:serviceaccount:d8-virtualization:dvcr", "d8-virtualization"}, - {"system:serviceaccount::name", ""}, // empty namespace - {"system:serviceaccount:nsA", ""}, // no name separator - {"system:node:worker-1", ""}, // not a service account - {"kubernetes-admin", ""}, // human user - {"", ""}, // empty - } - for _, tt := range tests { - if got := namespaceFromUsername(tt.username); got != tt.want { - t.Errorf("namespaceFromUsername(%q) = %q, want %q", tt.username, got, tt.want) - } - } -} diff --git a/images/dvcr/dvcr-k8s-auth/tokenreview.go b/images/dvcr/dvcr-k8s-auth/tokenreview.go deleted file mode 100644 index b4fe4c90ac..0000000000 --- a/images/dvcr/dvcr-k8s-auth/tokenreview.go +++ /dev/null @@ -1,202 +0,0 @@ -/* -Copyright 2026 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -//go:build dvcr_registry - -package dvcrk8s - -import ( - "bytes" - "context" - "crypto/sha256" - "crypto/tls" - "crypto/x509" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "os" - "strings" - "sync" - "time" -) - -const ( - saDir = "/var/run/secrets/kubernetes.io/serviceaccount" - saTokenPath = saDir + "/token" - saCAPath = saDir + "/ca.crt" - - tokenReviewPath = "/apis/authentication.k8s.io/v1/tokenreviews" -) - -var errTokenRejected = errors.New("token rejected by TokenReview") - -// tokenReviewer verifies ServiceAccount tokens via the Kubernetes TokenReview API -// using only the standard library (no client-go). Results are cached to keep the -// hot push path off the apiserver. -type tokenReviewer struct { - client *http.Client - apiURL string - ttl time.Duration - negTTL time.Duration - - mu sync.Mutex - cache map[string]cacheEntry -} - -type cacheEntry struct { - ns string - ok bool - expiry time.Time -} - -func newTokenReviewer(ttl, negTTL time.Duration) (*tokenReviewer, error) { - host := os.Getenv("KUBERNETES_SERVICE_HOST") - port := os.Getenv("KUBERNETES_SERVICE_PORT") - if host == "" || port == "" { - return nil, errors.New("KUBERNETES_SERVICE_HOST/PORT not set (not running in-cluster?)") - } - - caPEM, err := os.ReadFile(saCAPath) - if err != nil { - return nil, fmt.Errorf("read apiserver CA: %w", err) - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(caPEM) { - return nil, errors.New("parse apiserver CA") - } - - return &tokenReviewer{ - client: &http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, - }, - }, - apiURL: fmt.Sprintf("https://%s:%s%s", host, port, tokenReviewPath), - ttl: ttl, - negTTL: negTTL, - cache: make(map[string]cacheEntry), - }, nil -} - -// namespaceForToken returns the ServiceAccount namespace for a token, or an error -// if the token is rejected. Transient errors (network/apiserver) are not cached. -func (tr *tokenReviewer) namespaceForToken(ctx context.Context, token string) (string, error) { - key := hashToken(token) - - if e, ok := tr.lookup(key); ok { - if !e.ok { - return "", errTokenRejected - } - return e.ns, nil - } - - ns, ok, err := tr.review(ctx, token) - if err != nil { - return "", err - } - if !ok { - tr.store(key, cacheEntry{ok: false, expiry: time.Now().Add(tr.negTTL)}) - return "", errTokenRejected - } - tr.store(key, cacheEntry{ok: true, ns: ns, expiry: time.Now().Add(tr.ttl)}) - return ns, nil -} - -func (tr *tokenReviewer) lookup(key string) (cacheEntry, bool) { - tr.mu.Lock() - defer tr.mu.Unlock() - e, ok := tr.cache[key] - if !ok || time.Now().After(e.expiry) { - if ok { - delete(tr.cache, key) - } - return cacheEntry{}, false - } - return e, true -} - -func (tr *tokenReviewer) store(key string, e cacheEntry) { - tr.mu.Lock() - defer tr.mu.Unlock() - tr.cache[key] = e -} - -// review performs a single TokenReview call. It authenticates to the apiserver -// with the registry's own ServiceAccount token, read fresh each call because the -// projected token rotates. -func (tr *tokenReviewer) review(ctx context.Context, token string) (ns string, authenticated bool, err error) { - reqBody, err := json.Marshal(map[string]any{ - "apiVersion": "authentication.k8s.io/v1", - "kind": "TokenReview", - "spec": map[string]any{"token": token}, - }) - if err != nil { - return "", false, err - } - - ownToken, err := os.ReadFile(saTokenPath) - if err != nil { - return "", false, fmt.Errorf("read own SA token: %w", err) - } - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, tr.apiURL, bytes.NewReader(reqBody)) - if err != nil { - return "", false, err - } - req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(string(ownToken))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - - resp, err := tr.client.Do(req) - if err != nil { - return "", false, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) - if err != nil { - return "", false, err - } - if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { - return "", false, fmt.Errorf("TokenReview HTTP %d: %s", resp.StatusCode, string(body)) - } - - var out struct { - Status struct { - Authenticated bool `json:"authenticated"` - User struct { - Username string `json:"username"` - } `json:"user"` - Error string `json:"error"` - } `json:"status"` - } - if err := json.Unmarshal(body, &out); err != nil { - return "", false, fmt.Errorf("decode TokenReview: %w", err) - } - if !out.Status.Authenticated { - return "", false, nil - } - return namespaceFromUsername(out.Status.User.Username), true, nil -} - -func hashToken(token string) string { - sum := sha256.Sum256([]byte(token)) - return hex.EncodeToString(sum[:]) -} From 6f86f524414f82930e063c2282d1e68ef4ab0e8c Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 16:51:18 +0200 Subject: [PATCH 11/31] feat(controller): add scoped DVCR token signer Mint ES256-signed JWTs whose access claim (matching distribution's ClaimSet/ResourceActions) grants exactly the DVCR repositories a Pod uses. Promote golang-jwt/v5 to a direct dependency. Signed-off-by: Daniil Antoshin --- images/virtualization-artifact/go.mod | 2 + .../pkg/dvcr/registrytoken/signer.go | 105 ++++++++++++++++++ .../pkg/dvcr/registrytoken/signer_test.go | 98 ++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go create mode 100644 images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go diff --git a/images/virtualization-artifact/go.mod b/images/virtualization-artifact/go.mod index b12e2ad771..84fcea18bf 100644 --- a/images/virtualization-artifact/go.mod +++ b/images/virtualization-artifact/go.mod @@ -40,6 +40,8 @@ require ( sigs.k8s.io/yaml v1.6.0 ) +require github.com/golang-jwt/jwt/v5 v5.2.2 + require ( cel.dev/expr v0.25.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect diff --git a/images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go new file mode 100644 index 0000000000..b3264bd0d3 --- /dev/null +++ b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go @@ -0,0 +1,105 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package registrytoken mints scoped JWT credentials for DVCR. An importer or +// uploader Pod presents such a token instead of the shared read-write password; +// the token authorizes exactly the repositories that Pod reads from and writes to. +// The DVCR registry verifies it with distribution's own token backend (the format +// matches its ClaimSet / ResourceActions), so no shared read-write credential is +// copied into tenant namespaces. +package registrytoken + +import ( + "crypto/ecdsa" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +const ( + // Issuer is the token issuer; the registry accepts only this value. + Issuer = "virtualization-controller" + // Audience is the token audience; the registry accepts only this value. + Audience = "dvcr" + // KeyID is the JWS `kid` header the registry maps to the trusted public key. + KeyID = "dvcr" + // DefaultTTL is the scoped token lifetime. Generous on purpose: an import may + // run for hours, and the token is scoped to a single repository, so a long + // validity window is a small exposure. + // ponytail: fixed TTL; bind to the Pod's activeDeadlineSeconds if imports ever + // need to outlive this or exposure must be tightened further. + DefaultTTL = 24 * time.Hour +) + +// Access is one entry of a token's access claim: a resource and the actions +// permitted on it. JSON tags match distribution's token.ResourceActions. +type Access struct { + Type string `json:"type"` + Name string `json:"name"` + Actions []string `json:"actions"` +} + +// Signer mints ES256-signed scoped tokens with an ECDSA P-256 private key. +type Signer struct { + key *ecdsa.PrivateKey +} + +// NewSignerFromPEM builds a Signer from a PKCS8 (or SEC1) PEM-encoded ECDSA key. +func NewSignerFromPEM(keyPEM []byte) (*Signer, error) { + block, _ := pem.Decode(keyPEM) + if block == nil { + return nil, errors.New("registrytoken: no PEM block in private key") + } + key, err := parseECKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("registrytoken: parse private key: %w", err) + } + return &Signer{key: key}, nil +} + +func parseECKey(der []byte) (*ecdsa.PrivateKey, error) { + if k, err := x509.ParsePKCS8PrivateKey(der); err == nil { + ec, ok := k.(*ecdsa.PrivateKey) + if !ok { + return nil, errors.New("private key is not ECDSA") + } + return ec, nil + } + return x509.ParseECPrivateKey(der) +} + +// Sign issues a token granting access, valid for ttl. A non-positive ttl uses +// DefaultTTL. now is passed explicitly to keep the function testable. +func (s *Signer) Sign(access []Access, ttl time.Duration, now time.Time) (string, error) { + if ttl <= 0 { + ttl = DefaultTTL + } + claims := jwt.MapClaims{ + "iss": Issuer, + "aud": Audience, + "iat": now.Unix(), + "nbf": now.Add(-30 * time.Second).Unix(), + "exp": now.Add(ttl).Unix(), + "access": access, + } + tok := jwt.NewWithClaims(jwt.SigningMethodES256, claims) + tok.Header["kid"] = KeyID + return tok.SignedString(s.key) +} diff --git a/images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go new file mode 100644 index 0000000000..f1854ccb6b --- /dev/null +++ b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registrytoken + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +func newSignerPEM(t *testing.T) ([]byte, *ecdsa.PublicKey) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), &key.PublicKey +} + +func TestSignRoundTrip(t *testing.T) { + keyPEM, pub := newSignerPEM(t) + signer, err := NewSignerFromPEM(keyPEM) + if err != nil { + t.Fatal(err) + } + + access := []Access{{Type: "repository", Name: "cvi/my-image", Actions: []string{"pull", "push"}}} + now := time.Unix(1_700_000_000, 0) + raw, err := signer.Sign(access, time.Hour, now) + if err != nil { + t.Fatal(err) + } + + // Verify signature and claims the way the registry will. + parsed, err := jwt.Parse(raw, func(*jwt.Token) (interface{}, error) { return pub, nil }, + jwt.WithValidMethods([]string{"ES256"}), + jwt.WithTimeFunc(func() time.Time { return now.Add(time.Minute) })) + if err != nil { + t.Fatalf("parse: %v", err) + } + if kid := parsed.Header["kid"]; kid != KeyID { + t.Errorf("kid = %v, want %q", kid, KeyID) + } + claims := parsed.Claims.(jwt.MapClaims) + if claims["iss"] != Issuer { + t.Errorf("iss = %v, want %q", claims["iss"], Issuer) + } + if claims["aud"] != Audience { + t.Errorf("aud = %v, want %q", claims["aud"], Audience) + } + acc, ok := claims["access"].([]interface{}) + if !ok || len(acc) != 1 { + t.Fatalf("access claim = %v", claims["access"]) + } + entry := acc[0].(map[string]interface{}) + if entry["name"] != "cvi/my-image" || entry["type"] != "repository" { + t.Errorf("access entry = %v", entry) + } +} + +func TestSignExpired(t *testing.T) { + keyPEM, pub := newSignerPEM(t) + signer, _ := NewSignerFromPEM(keyPEM) + now := time.Unix(1_700_000_000, 0) + raw, err := signer.Sign(nil, time.Hour, now) + if err != nil { + t.Fatal(err) + } + _, err = jwt.Parse(raw, func(*jwt.Token) (interface{}, error) { return pub, nil }, + jwt.WithTimeFunc(func() time.Time { return now.Add(2 * time.Hour) })) + if err == nil { + t.Fatal("expected expired token to fail verification") + } +} From 72cf7832dac5ef6abbb89bcab278f888d360facc Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 17:02:23 +0200 Subject: [PATCH 12/31] feat(controller): mint scoped DVCR tokens for importer/uploader/CDI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When per-namespace authorization is on, the controller mints a token scoped to exactly the repositories each Pod uses and delivers it through the existing destination-auth-secret path — a dockerconfigjson for the dvcr-artifact importer/uploader, an Opaque accessKeyId/secretKey for the CDI DataVolume. No shared read-write credential is copied into tenant namespaces, and no projected ServiceAccount token / TokenReview is needed. - dvcr.Settings: TokenSigner + RepoPath helper; load private key from env - supplements: EnsureForPod/EnsureForDataVolume take the token scope - service: compute per-Pod scope (dest push+pull, source pull, CDI pull) - drop projected SA token volumes and their consts Signed-off-by: Daniil Antoshin --- .../pkg/common/consts.go | 14 ---- .../pkg/common/pod/pod.go | 22 ------- .../pkg/config/load_dvcr_settings.go | 16 +++++ .../pkg/controller/importer/importer_pod.go | 21 ------ .../pkg/controller/importer/settings.go | 18 ++---- .../pkg/controller/service/disk_service.go | 4 +- .../controller/service/dvcr_token_scope.go | 63 ++++++++++++++++++ .../controller/service/importer_service.go | 4 +- .../controller/service/uploader_service.go | 2 +- .../supplements/copier/auth_secret.go | 64 +++++++++++++++---- .../pkg/controller/supplements/ensure.go | 35 ++++++---- .../pkg/controller/uploader/settings.go | 18 ++---- .../pkg/controller/uploader/uploader_pod.go | 21 ------ .../virtualization-artifact/pkg/dvcr/dvcr.go | 27 +++++++- 14 files changed, 193 insertions(+), 136 deletions(-) create mode 100644 images/virtualization-artifact/pkg/controller/service/dvcr_token_scope.go diff --git a/images/virtualization-artifact/pkg/common/consts.go b/images/virtualization-artifact/pkg/common/consts.go index 522875b5dc..41ddc972c7 100644 --- a/images/virtualization-artifact/pkg/common/consts.go +++ b/images/virtualization-artifact/pkg/common/consts.go @@ -92,13 +92,6 @@ const ( ImporterDestinationAuthConfigVar = "IMPORTER_DESTINATION_AUTH_CONFIG" // ImporterDestinationAuthConfigFile is a path to auth config file in mount directory. ImporterDestinationAuthConfigFile = "/dvcr-auth/.dockerconfigjson" - // ImporterDestinationTokenFileVar is an environment variable with the path to - // the projected ServiceAccount token used for per-namespace DVCR authorization. - ImporterDestinationTokenFileVar = "IMPORTER_DESTINATION_TOKEN_FILE" - // ImporterDestinationTokenDir is a mount directory for the projected SA token. - ImporterDestinationTokenDir = "/dvcr-token" - // ImporterDestinationTokenFile is the path to the projected SA token file. - ImporterDestinationTokenFile = "/dvcr-token/token" // DestinationInsecureTLSVar is an environment variable for Importer Pod that defines whether DVCR is insecure. DestinationInsecureTLSVar = "DESTINATION_INSECURE_TLS" ImporterSHA256Sum = "IMPORTER_SHA256SUM" @@ -112,13 +105,6 @@ const ( UploaderDestinationAuthConfigVar = "UPLOADER_DESTINATION_AUTH_CONFIG" UploaderDestinationAuthConfigDir = "/dvcr-auth" UploaderDestinationAuthConfigFile = "/dvcr-auth/.dockerconfigjson" - // UploaderDestinationTokenFileVar is an environment variable with the path to - // the projected ServiceAccount token used for per-namespace DVCR authorization. - UploaderDestinationTokenFileVar = "UPLOADER_DESTINATION_TOKEN_FILE" - // UploaderDestinationTokenDir is a mount directory for the projected SA token. - UploaderDestinationTokenDir = "/dvcr-token" - // UploaderDestinationTokenFile is the path to the projected SA token file. - UploaderDestinationTokenFile = "/dvcr-token/token" DockerRegistrySchemePrefix = "docker://" diff --git a/images/virtualization-artifact/pkg/common/pod/pod.go b/images/virtualization-artifact/pkg/common/pod/pod.go index 4d63fe1393..b728cd22e2 100644 --- a/images/virtualization-artifact/pkg/common/pod/pod.go +++ b/images/virtualization-artifact/pkg/common/pod/pod.go @@ -72,28 +72,6 @@ func CreateVolumeMount(volName, mountPath string) corev1.VolumeMount { } } -// CreateProjectedSATokenVolume creates a volume with a projected ServiceAccount -// token at the given relative path. The audience is left empty so the token is -// accepted by the apiserver's default TokenReview. expirationSeconds controls -// kubelet rotation of the token file. -func CreateProjectedSATokenVolume(volName, path string, expirationSeconds int64) corev1.Volume { - return corev1.Volume{ - Name: volName, - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{ - { - ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ - Path: path, - ExpirationSeconds: ptr.To(expirationSeconds), - }, - }, - }, - }, - }, - } -} - func AddVolume(pod *corev1.Pod, container *corev1.Container, volume corev1.Volume, mount corev1.VolumeMount, envVar corev1.EnvVar) { pod.Spec.Volumes = append(pod.Spec.Volumes, volume) container.VolumeMounts = append(container.VolumeMounts, mount) diff --git a/images/virtualization-artifact/pkg/config/load_dvcr_settings.go b/images/virtualization-artifact/pkg/config/load_dvcr_settings.go index a666299399..4f863dbc62 100644 --- a/images/virtualization-artifact/pkg/config/load_dvcr_settings.go +++ b/images/virtualization-artifact/pkg/config/load_dvcr_settings.go @@ -22,6 +22,7 @@ import ( "strconv" "github.com/deckhouse/virtualization-controller/pkg/dvcr" + "github.com/deckhouse/virtualization-controller/pkg/dvcr/registrytoken" ) const ( @@ -43,6 +44,9 @@ const ( DVCRGCScheduleVar = "DVCR_GC_SCHEDULE" // DVCRTenantAuthzEnabledVar is an env variable that enables per-namespace DVCR authorization. DVCRTenantAuthzEnabledVar = "DVCR_TENANT_AUTHZ_ENABLED" + // DVCRTokenPrivateKeyVar holds the PEM ECDSA private key used to mint scoped + // DVCR tokens. Required when DVCR_TENANT_AUTHZ_ENABLED is true. + DVCRTokenPrivateKeyVar = "DVCR_TOKEN_PRIVATE_KEY" // UploaderIngressHostVar is a env variable UploaderIngressHostVar = "UPLOADER_INGRESS_HOST" @@ -93,6 +97,18 @@ func LoadDVCRSettingsFromEnvs(controllerNamespace string) (*dvcr.Settings, error dvcrSettings.GCSchedule = dvcr.DefaultGCSchedule } + if dvcrSettings.TenantAuthzEnabled { + keyPEM := os.Getenv(DVCRTokenPrivateKeyVar) + if keyPEM == "" { + return nil, fmt.Errorf("environment variable %q undefined, required when %s is true", DVCRTokenPrivateKeyVar, DVCRTenantAuthzEnabledVar) + } + signer, err := registrytoken.NewSignerFromPEM([]byte(keyPEM)) + if err != nil { + return nil, fmt.Errorf("init DVCR token signer: %w", err) + } + dvcrSettings.TokenSigner = signer + } + return dvcrSettings, nil } diff --git a/images/virtualization-artifact/pkg/controller/importer/importer_pod.go b/images/virtualization-artifact/pkg/controller/importer/importer_pod.go index 04f8a40d24..719a045061 100644 --- a/images/virtualization-artifact/pkg/controller/importer/importer_pod.go +++ b/images/virtualization-artifact/pkg/controller/importer/importer_pod.go @@ -48,14 +48,6 @@ const ( // destinationAuthVol is the name of the volume containing DVCR docker auth config. destinationAuthVol = "dvcr-secret-vol" - // destinationTokenVol is the name of the volume containing the projected - // ServiceAccount token used for per-namespace DVCR authorization. - destinationTokenVol = "dvcr-token-vol" - - // destinationTokenExpirationSeconds is the requested lifetime of the projected - // SA token; kubelet rotates the file well before it expires. - destinationTokenExpirationSeconds = 600 - // sourceRegistryAuthVol is the name of the volume containing source registry docker auth config. sourceRegistryAuthVol = "source-registry-secret-vol" @@ -348,19 +340,6 @@ func (imp *Importer) addVolumes(pod *corev1.Pod, container *corev1.Container) { ) } - if imp.EnvSettings.DestinationTokenAuth { - // Mount a projected ServiceAccount token so the Pod authenticates to DVCR - // with its own namespace-scoped identity. - podutil.AddVolume(pod, container, - podutil.CreateProjectedSATokenVolume(destinationTokenVol, "token", destinationTokenExpirationSeconds), - podutil.CreateVolumeMount(destinationTokenVol, common.ImporterDestinationTokenDir), - corev1.EnvVar{ - Name: common.ImporterDestinationTokenFileVar, - Value: common.ImporterDestinationTokenFile, - }, - ) - } - // Volume with CA certificates either from caBundle field or from existing ConfigMap. if imp.EnvSettings.CertConfigMap != "" { podutil.AddVolume(pod, container, diff --git a/images/virtualization-artifact/pkg/controller/importer/settings.go b/images/virtualization-artifact/pkg/controller/importer/settings.go index ad25389bf3..a5fd939463 100644 --- a/images/virtualization-artifact/pkg/controller/importer/settings.go +++ b/images/virtualization-artifact/pkg/controller/importer/settings.go @@ -66,21 +66,17 @@ type Settings struct { DestinationEndpoint string DestinationInsecureTLS string DestinationAuthSecret string - // DestinationTokenAuth makes the Pod authenticate to DVCR with a projected - // ServiceAccount token instead of the shared read-write credential. - DestinationTokenAuth bool } func ApplyDVCRDestinationSettings(podEnvVars *Settings, dvcrSettings *dvcr.Settings, supGen supplements.Generator, dvcrImageName string) { - if dvcrSettings.TenantAuthzEnabled { - podEnvVars.DestinationTokenAuth = true - } else { - authSecret := dvcrSettings.AuthSecret - if supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { - authSecret = supGen.DVCRAuthSecret().Name - } - podEnvVars.DestinationAuthSecret = authSecret + authSecret := dvcrSettings.AuthSecret + // With per-namespace authorization every Pod gets a per-import scoped token + // Secret (created by supplements.EnsureForPod); otherwise the shared credential + // is used, copied into the namespace only when they differ. + if dvcrSettings.TenantAuthzEnabled || supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { + authSecret = supGen.DVCRAuthSecret().Name } + podEnvVars.DestinationAuthSecret = authSecret podEnvVars.DestinationInsecureTLS = dvcrSettings.InsecureTLS podEnvVars.DestinationEndpoint = dvcrImageName } diff --git a/images/virtualization-artifact/pkg/controller/service/disk_service.go b/images/virtualization-artifact/pkg/controller/service/disk_service.go index eae8af0931..ee6651540b 100644 --- a/images/virtualization-artifact/pkg/controller/service/disk_service.go +++ b/images/virtualization-artifact/pkg/controller/service/disk_service.go @@ -123,7 +123,7 @@ func (s DiskService) Start( return nil } - return supplements.EnsureForDataVolume(ctx, s.client, sup, dvBuilder.GetResource(), s.dvcrSettings) + return supplements.EnsureForDataVolume(ctx, s.client, sup, dvBuilder.GetResource(), s.dvcrSettings, dataVolumeTokenScope(s.dvcrSettings, source)) } func (s DiskService) GetVolumeAndAccessModes(ctx context.Context, obj client.Object, sc *storagev1.StorageClass) (corev1.PersistentVolumeMode, corev1.PersistentVolumeAccessMode, error) { @@ -163,7 +163,7 @@ func (s DiskService) StartImmediate( return nil } - return supplements.EnsureForDataVolume(ctx, s.client, dataVolumeSupplement, dvBuilder.GetResource(), s.dvcrSettings) + return supplements.EnsureForDataVolume(ctx, s.client, dataVolumeSupplement, dvBuilder.GetResource(), s.dvcrSettings, dataVolumeTokenScope(s.dvcrSettings, source)) } func (s DiskService) CheckProvisioning(ctx context.Context, pvc *corev1.PersistentVolumeClaim) error { diff --git a/images/virtualization-artifact/pkg/controller/service/dvcr_token_scope.go b/images/virtualization-artifact/pkg/controller/service/dvcr_token_scope.go new file mode 100644 index 0000000000..0b16cc598a --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/service/dvcr_token_scope.go @@ -0,0 +1,63 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + + "github.com/deckhouse/virtualization-controller/pkg/controller/importer" + "github.com/deckhouse/virtualization-controller/pkg/controller/uploader" + "github.com/deckhouse/virtualization-controller/pkg/dvcr" + "github.com/deckhouse/virtualization-controller/pkg/dvcr/registrytoken" +) + +// importerTokenScope is the DVCR access an importer Pod needs: push+pull on its +// destination repository, plus pull on the source repository when the source is +// itself a DVCR image (dvcr-artifact copies both through the same credential). +// Returns nil when per-namespace authorization is off (no token is minted). +func importerTokenScope(s *dvcr.Settings, settings *importer.Settings) []registrytoken.Access { + if !s.TenantAuthzEnabled { + return nil + } + access := []registrytoken.Access{repoAccess(s.RepoPath(settings.DestinationEndpoint), "pull", "push")} + if settings.Source == importer.SourceDVCR && settings.Endpoint != "" { + access = append(access, repoAccess(s.RepoPath(settings.Endpoint), "pull")) + } + return access +} + +// uploaderTokenScope is the DVCR access an uploader Pod needs: push+pull on its +// destination repository (an uploader has no DVCR source). +func uploaderTokenScope(s *dvcr.Settings, settings *uploader.Settings) []registrytoken.Access { + if !s.TenantAuthzEnabled { + return nil + } + return []registrytoken.Access{repoAccess(s.RepoPath(settings.DestinationEndpoint), "pull", "push")} +} + +// dataVolumeTokenScope is the DVCR access a CDI DataVolume needs: pull-only on the +// source repository it imports from (it writes to a PVC, not to DVCR). +func dataVolumeTokenScope(s *dvcr.Settings, source *cdiv1.DataVolumeSource) []registrytoken.Access { + if !s.TenantAuthzEnabled || source == nil || source.Registry == nil || source.Registry.URL == nil { + return nil + } + return []registrytoken.Access{repoAccess(s.RepoPath(*source.Registry.URL), "pull")} +} + +func repoAccess(name string, actions ...string) registrytoken.Access { + return registrytoken.Access{Type: "repository", Name: name, Actions: actions} +} diff --git a/images/virtualization-artifact/pkg/controller/service/importer_service.go b/images/virtualization-artifact/pkg/controller/service/importer_service.go index a8478837d3..13e23b09bc 100644 --- a/images/virtualization-artifact/pkg/controller/service/importer_service.go +++ b/images/virtualization-artifact/pkg/controller/service/importer_service.go @@ -94,7 +94,7 @@ func (s ImporterService) Start( return fmt.Errorf("failed to create NetworkPolicy: %w", err) } - return supplements.EnsureForPod(ctx, s.client, sup, pod, caBundle, s.dvcrSettings) + return supplements.EnsureForPod(ctx, s.client, sup, pod, caBundle, s.dvcrSettings, importerTokenScope(s.dvcrSettings, settings)) } func (s ImporterService) StartWithPodSetting( @@ -118,7 +118,7 @@ func (s ImporterService) StartWithPodSetting( return err } - return supplements.EnsureForPod(ctx, s.client, sup, pod, caBundle, s.dvcrSettings) + return supplements.EnsureForPod(ctx, s.client, sup, pod, caBundle, s.dvcrSettings, importerTokenScope(s.dvcrSettings, settings)) } func (s ImporterService) CleanUp(ctx context.Context, sup supplements.Generator) (bool, error) { diff --git a/images/virtualization-artifact/pkg/controller/service/uploader_service.go b/images/virtualization-artifact/pkg/controller/service/uploader_service.go index 461f37e513..73335d983f 100644 --- a/images/virtualization-artifact/pkg/controller/service/uploader_service.go +++ b/images/virtualization-artifact/pkg/controller/service/uploader_service.go @@ -96,7 +96,7 @@ func (s UploaderService) Start( return fmt.Errorf("failed to create NetworkPolicy: %w", err) } - err = supplements.EnsureForPod(ctx, s.client, sup, pod, caBundle, s.dvcrSettings) + err = supplements.EnsureForPod(ctx, s.client, sup, pod, caBundle, s.dvcrSettings, uploaderTokenScope(s.dvcrSettings, settings)) if err != nil { return err } diff --git a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go index 183124b497..67d8c26ae3 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go +++ b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go @@ -18,19 +18,23 @@ package copier import ( "context" + "encoding/base64" + "encoding/json" + "fmt" + "time" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/deckhouse/virtualization-controller/pkg/auth" "github.com/deckhouse/virtualization-controller/pkg/common/object" + "github.com/deckhouse/virtualization-controller/pkg/dvcr/registrytoken" ) -// ServiceAccountTokenUsername is a sentinel accessKeyId value that instructs the -// (patched) CDI importer to authenticate to the registry with the Pod's projected -// ServiceAccount token instead of static credentials. Must match the constant in -// the CDI fork (deckhouse/3p-containerized-data-importer, pkg/importer/transport.go). -const ServiceAccountTokenUsername = "dvcr-serviceaccount-token-auth" +// ScopedTokenUsername is the Basic-auth username presented alongside a scoped +// DVCR token. The registry authorizes on the token in the password; the username +// is a fixed, human-readable marker. +const ScopedTokenUsername = "dvcr-jwt" // AuthSecret copies auth credentials from the source Secret into // Destination Secret and ensure its data is CDI compatible: @@ -45,20 +49,52 @@ type AuthSecret struct { Secret } -// CreateServiceAccountTokenAuth creates a CDI-compatible Opaque Secret whose -// accessKeyId is the ServiceAccount-token sentinel, so the importer Pod -// authenticates to DVCR with its own namespace-scoped identity instead of the -// shared read-write credential. Used for VirtualDisk imports when per-namespace -// DVCR authorization is enabled. -func (a AuthSecret) CreateServiceAccountTokenAuth(ctx context.Context, client client.Client) error { +// CreateScopedTokenCDI mints a scoped DVCR token for access and stores it in a +// CDI-compatible Opaque Secret (accessKeyId/secretKey). The importer Pod +// authenticates to DVCR with a credential scoped to exactly the repositories in +// access, instead of the shared read-write credential. +func (a AuthSecret) CreateScopedTokenCDI(ctx context.Context, client client.Client, signer *registrytoken.Signer, access []registrytoken.Access) error { + raw, err := signer.Sign(access, 0, time.Now()) + if err != nil { + return fmt.Errorf("mint scoped DVCR token: %w", err) + } destData := map[string][]byte{ - "accessKeyId": []byte(ServiceAccountTokenUsername), - "secretKey": []byte(""), + "accessKeyId": []byte(ScopedTokenUsername), + "secretKey": []byte(raw), } - _, err := a.Create(ctx, client, destData, corev1.SecretTypeOpaque) + _, err = a.Create(ctx, client, destData, corev1.SecretTypeOpaque) return err } +// CreateScopedTokenDockerConfig mints a scoped DVCR token for access and stores +// it as a dockerconfigjson Secret keyed by registryURL, so the dvcr-artifact +// importer/uploader Pod reads it through the standard destination auth config. +func (a AuthSecret) CreateScopedTokenDockerConfig(ctx context.Context, client client.Client, signer *registrytoken.Signer, access []registrytoken.Access, registryURL string) error { + raw, err := signer.Sign(access, 0, time.Now()) + if err != nil { + return fmt.Errorf("mint scoped DVCR token: %w", err) + } + cfg, err := dockerConfigJSON(registryURL, ScopedTokenUsername, raw) + if err != nil { + return err + } + destData := map[string][]byte{corev1.DockerConfigJsonKey: cfg} + _, err = a.Create(ctx, client, destData, corev1.SecretTypeDockerConfigJson) + return err +} + +// dockerConfigJSON builds a minimal ~/.docker/config.json for a single registry. +func dockerConfigJSON(registryURL, username, password string) ([]byte, error) { + entry := map[string]string{ + "username": username, + "password": password, + "auth": base64.StdEncoding.EncodeToString([]byte(username + ":" + password)), + } + return json.Marshal(map[string]any{ + "auths": map[string]any{registryURL: entry}, + }) +} + // CopyCDICompatible transforms auth credentials in dockerconfigjson format into CDI compatible: // a Secret with two fields: accessKeyId and secretKey. // ref is registry url or image name. It is used to select a desired auth pair from the config. diff --git a/images/virtualization-artifact/pkg/controller/supplements/ensure.go b/images/virtualization-artifact/pkg/controller/supplements/ensure.go index e48701d4a3..a66520b210 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/ensure.go +++ b/images/virtualization-artifact/pkg/controller/supplements/ensure.go @@ -34,6 +34,7 @@ import ( podutil "github.com/deckhouse/virtualization-controller/pkg/common/pod" "github.com/deckhouse/virtualization-controller/pkg/controller/supplements/copier" "github.com/deckhouse/virtualization-controller/pkg/dvcr" + "github.com/deckhouse/virtualization-controller/pkg/dvcr/registrytoken" ) type DataSource interface { @@ -45,7 +46,7 @@ type DataSource interface { // EnsureForPod make supplements for importer or uploader Pod: // - It creates ConfigMap with caBundle for http and containerImage data sources. // - It copies DVCR auth Secret to use DVCR as destination. -func EnsureForPod(ctx context.Context, client client.Client, supGen Generator, pod *corev1.Pod, ds DataSource, dvcrSettings *dvcr.Settings) error { +func EnsureForPod(ctx context.Context, client client.Client, supGen Generator, pod *corev1.Pod, ds DataSource, dvcrSettings *dvcr.Settings, scope []registrytoken.Access) error { // Create ConfigMap with caBundle. if ds.HasCABundle() { caBundleCM := supGen.CABundleConfigMap() @@ -60,7 +61,20 @@ func EnsureForPod(ctx context.Context, client client.Client, supGen Generator, p } // Create Secret with auth config to use DVCR as destination. - if ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { + switch { + case dvcrSettings.TenantAuthzEnabled: + // Mint a token scoped to this Pod's repositories instead of copying the + // shared read-write credential into the (possibly tenant) namespace. + authCopier := copier.AuthSecret{ + Secret: copier.Secret{ + Destination: supGen.DVCRAuthSecret(), + OwnerReference: podutil.MakeOwnerReference(pod), + }, + } + if err := authCopier.CreateScopedTokenDockerConfig(ctx, client, dvcrSettings.TokenSigner, scope, dvcrSettings.RegistryURL); err != nil { + return err + } + case ShouldCopyDVCRAuthSecret(dvcrSettings, supGen): authSecret := supGen.DVCRAuthSecret() authCopier := copier.AuthSecret{ Secret: copier.Secret{ @@ -104,12 +118,6 @@ func ShouldCopyDVCRAuthSecret(dvcrSettings *dvcr.Settings, supGen Generator) boo if dvcrSettings.AuthSecret == "" { return false } - // With per-namespace authorization the Pod authenticates with its own - // ServiceAccount token, so the shared read-write credential must not be - // copied into the tenant namespace. - if dvcrSettings.TenantAuthzEnabled { - return false - } // Should copy if namespaces are different. return dvcrSettings.AuthSecretNamespace != supGen.Namespace() } @@ -133,7 +141,7 @@ func ShouldCopyImagePullSecret(ctrImg *datasource.ContainerRegistry, targetNS st return imgPullNS != "" && imgPullNS != targetNS } -func EnsureForDataVolume(ctx context.Context, client client.Client, supGen DataVolumeSupplement, dv *cdiv1.DataVolume, dvcrSettings *dvcr.Settings) error { +func EnsureForDataVolume(ctx context.Context, client client.Client, supGen DataVolumeSupplement, dv *cdiv1.DataVolume, dvcrSettings *dvcr.Settings, scope []registrytoken.Access) error { if dvcrSettings.AuthSecret != "" { authSecret := supGen.DVCRAuthSecretForDV() authCopier := copier.AuthSecret{ @@ -147,12 +155,11 @@ func EnsureForDataVolume(ctx context.Context, client client.Client, supGen DataV }, } - // With per-namespace authorization the importer Pod authenticates to DVCR - // with its own ServiceAccount token (via the sentinel credential understood - // by the patched CDI importer), so the shared read-write credential is not - // copied into the tenant namespace. + // With per-namespace authorization the CDI importer authenticates to DVCR + // with a token scoped to the source repository it pulls from, so the shared + // read-write credential is not copied into the tenant namespace. if dvcrSettings.TenantAuthzEnabled { - if err := authCopier.CreateServiceAccountTokenAuth(ctx, client); err != nil { + if err := authCopier.CreateScopedTokenCDI(ctx, client, dvcrSettings.TokenSigner, scope); err != nil { return err } } else if err := authCopier.CopyCDICompatible(ctx, client, dvcrSettings.RegistryURL); err != nil { diff --git a/images/virtualization-artifact/pkg/controller/uploader/settings.go b/images/virtualization-artifact/pkg/controller/uploader/settings.go index 4bea7eb177..546c637f32 100644 --- a/images/virtualization-artifact/pkg/controller/uploader/settings.go +++ b/images/virtualization-artifact/pkg/controller/uploader/settings.go @@ -28,21 +28,17 @@ type Settings struct { DestinationEndpoint string DestinationInsecureTLS string DestinationAuthSecret string - // DestinationTokenAuth makes the Pod authenticate to DVCR with a projected - // ServiceAccount token instead of the shared read-write credential. - DestinationTokenAuth bool } func ApplyDVCRDestinationSettings(podEnvVars *Settings, dvcrSettings *dvcr.Settings, supGen supplements.Generator, dvcrImageName string) { - if dvcrSettings.TenantAuthzEnabled { - podEnvVars.DestinationTokenAuth = true - } else { - authSecret := dvcrSettings.AuthSecret - if supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { - authSecret = supGen.DVCRAuthSecret().Name - } - podEnvVars.DestinationAuthSecret = authSecret + authSecret := dvcrSettings.AuthSecret + // With per-namespace authorization every Pod gets a per-import scoped token + // Secret (created by supplements.EnsureForPod); otherwise the shared credential + // is used, copied into the namespace only when they differ. + if dvcrSettings.TenantAuthzEnabled || supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { + authSecret = supGen.DVCRAuthSecret().Name } + podEnvVars.DestinationAuthSecret = authSecret podEnvVars.DestinationInsecureTLS = dvcrSettings.InsecureTLS podEnvVars.DestinationEndpoint = dvcrImageName } diff --git a/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go b/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go index c236397442..570566ca57 100644 --- a/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go +++ b/images/virtualization-artifact/pkg/controller/uploader/uploader_pod.go @@ -35,14 +35,6 @@ import ( const ( // destinationAuthVol is the name of the volume containing DVCR docker auth config. destinationAuthVol = "dvcr-secret-vol" - - // destinationTokenVol is the name of the volume containing the projected - // ServiceAccount token used for per-namespace DVCR authorization. - destinationTokenVol = "dvcr-token-vol" - - // destinationTokenExpirationSeconds is the requested lifetime of the projected - // SA token; kubelet rotates the file well before it expires. - destinationTokenExpirationSeconds = 600 ) // These constants can't be imported from "images/dvcr-artifact/pkg/uploader/uploader.go" due to conflicts with the CDI version. @@ -216,17 +208,4 @@ func (p *Pod) addVolumes(pod *corev1.Pod, container *corev1.Container) { }, ) } - - if p.Settings.DestinationTokenAuth { - // Mount a projected ServiceAccount token so the Pod authenticates to DVCR - // with its own namespace-scoped identity. - podutil.AddVolume(pod, container, - podutil.CreateProjectedSATokenVolume(destinationTokenVol, "token", destinationTokenExpirationSeconds), - podutil.CreateVolumeMount(destinationTokenVol, common.UploaderDestinationTokenDir), - corev1.EnvVar{ - Name: common.UploaderDestinationTokenFileVar, - Value: common.UploaderDestinationTokenFile, - }, - ) - } } diff --git a/images/virtualization-artifact/pkg/dvcr/dvcr.go b/images/virtualization-artifact/pkg/dvcr/dvcr.go index 5222620d82..7df634b9da 100644 --- a/images/virtualization-artifact/pkg/dvcr/dvcr.go +++ b/images/virtualization-artifact/pkg/dvcr/dvcr.go @@ -19,8 +19,11 @@ package dvcr import ( "fmt" "path" + "strings" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/deckhouse/virtualization-controller/pkg/dvcr/registrytoken" ) type Settings struct { @@ -43,10 +46,12 @@ type Settings struct { // GCSchedule is a cron formatted schedule to periodically run a garbage collection. GCSchedule string // TenantAuthzEnabled turns on per-namespace DVCR authorization: importer and - // uploader Pods authenticate with their ServiceAccount token instead of the - // shared read-write credential, which is then no longer copied into tenant - // namespaces. + // uploader Pods authenticate with a scoped token minted for the single + // repository they use, instead of the shared read-write credential, which is + // then no longer copied into tenant namespaces. TenantAuthzEnabled bool + // TokenSigner mints the scoped tokens. Non-nil only when TenantAuthzEnabled. + TokenSigner *registrytoken.Signer } type UploaderIngressSettings struct { @@ -80,3 +85,19 @@ func (s *Settings) RegistryImageForVD(obj client.Object) string { imgPath := path.Clean(fmt.Sprintf(VMDImageTmpl, obj.GetNamespace(), obj.GetName(), obj.GetUID())) return path.Join(s.RegistryURL, imgPath) } + +// RepoPath extracts the repository path (e.g. "vi/ns/name", "cvi/name") from a +// DVCR image reference by stripping the optional docker:// scheme, the registry +// host and the tag/digest. It is the name used in a scoped token's access claim. +func (s *Settings) RepoPath(imageRef string) string { + ref := strings.TrimPrefix(imageRef, "docker://") + ref = strings.TrimPrefix(ref, s.RegistryURL) + ref = strings.TrimPrefix(ref, "/") + if i := strings.Index(ref, "@"); i >= 0 { + ref = ref[:i] + } + if slash := strings.LastIndex(ref, "/"); strings.LastIndex(ref, ":") > slash { + ref = ref[:strings.LastIndex(ref, ":")] + } + return path.Clean(ref) +} From 5c723b6468122f6f48cc935a1af3412dea3af119 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 17:04:18 +0200 Subject: [PATCH 13/31] refactor(dvcr-artifact): drop projected SA token auth The scoped DVCR token now arrives as ordinary Basic credentials in the destination auth config, so the per-request ServiceAccount token file reader is no longer needed. Destination auth is plain Basic again. Signed-off-by: Daniil Antoshin --- .../pkg/auth/token_authenticator.go | 51 ------------------- images/dvcr-artifact/pkg/importer/importer.go | 20 ++------ images/dvcr-artifact/pkg/registry/registry.go | 15 ++---- images/dvcr-artifact/pkg/uploader/uploader.go | 9 +--- 4 files changed, 9 insertions(+), 86 deletions(-) delete mode 100644 images/dvcr-artifact/pkg/auth/token_authenticator.go diff --git a/images/dvcr-artifact/pkg/auth/token_authenticator.go b/images/dvcr-artifact/pkg/auth/token_authenticator.go deleted file mode 100644 index fd6aa576c0..0000000000 --- a/images/dvcr-artifact/pkg/auth/token_authenticator.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2026 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package auth - -import ( - "os" - "strings" - - "github.com/google/go-containerregistry/pkg/authn" -) - -// tokenFileAuthenticator authenticates to DVCR with a projected ServiceAccount -// token read fresh from disk on every request. Reading per-call is required -// because kubelet rotates the projected token file; a token cached once would -// break long pushes when it rotates mid-transfer. -type tokenFileAuthenticator struct { - path string -} - -// TokenFileAuthenticator returns an authn.Authenticator that presents the -// contents of the token file as the Basic auth password. The username is a -// fixed non-empty placeholder; the DVCR authorization backend only inspects the -// token in the password. -func TokenFileAuthenticator(path string) authn.Authenticator { - return &tokenFileAuthenticator{path: path} -} - -func (t *tokenFileAuthenticator) Authorization() (*authn.AuthConfig, error) { - token, err := os.ReadFile(t.path) - if err != nil { - return nil, err - } - return &authn.AuthConfig{ - Username: "sa", - Password: strings.TrimSpace(string(token)), - }, nil -} diff --git a/images/dvcr-artifact/pkg/importer/importer.go b/images/dvcr-artifact/pkg/importer/importer.go index dfe1510f64..a874d7ab91 100644 --- a/images/dvcr-artifact/pkg/importer/importer.go +++ b/images/dvcr-artifact/pkg/importer/importer.go @@ -49,11 +49,7 @@ const ( DockerRegistrySchemePrefix = "docker://" DVCRSource = "dvcr" BlockDeviceSource = "blockDevice" - FilesystemSource = "filesystem" - - // ImporterDestinationTokenFileVar is the env var with the path to the - // projected ServiceAccount token used for per-namespace DVCR authorization. - ImporterDestinationTokenFileVar = "IMPORTER_DESTINATION_TOKEN_FILE" + FilesystemSource = "filesystem" ) func New() *Importer { @@ -70,7 +66,6 @@ type Importer struct { destImageName string destUsername string destPassword string - destTokenFile string destInsecure bool certDir string sha256Sum string @@ -122,10 +117,9 @@ func (i *Importer) parseOptions() error { } } - i.destTokenFile = os.Getenv(ImporterDestinationTokenFileVar) i.destUsername, _ = util.ParseEnvVar(common.ImporterDestinationAccessKeyID, false) i.destPassword, _ = util.ParseEnvVar(common.ImporterDestinationSecretKey, false) - if i.destTokenFile == "" && i.destUsername == "" && i.destPassword == "" { + if i.destUsername == "" && i.destPassword == "" { destAuthConfig, _ := util.ParseEnvVar(common.ImporterDestinationAuthConfig, false) if destAuthConfig != "" { authFile, err := auth.RegistryAuthFile(destAuthConfig) @@ -170,7 +164,6 @@ func (i *Importer) runForDataSource(ctx context.Context) error { ImageName: i.destImageName, Username: i.destUsername, Password: i.destPassword, - TokenFile: i.destTokenFile, Insecure: i.destInsecure, }, i.sha256Sum, i.md5Sum) if err != nil { @@ -266,13 +259,10 @@ func (i *Importer) destRemoteOptions(ctx context.Context) []remote.Option { return remoteOpts } -// destAuthenticator returns the authenticator for pushing to DVCR. With -// per-namespace authorization the projected ServiceAccount token is read fresh -// on every request; otherwise static Basic credentials are used. +// destAuthenticator returns the Basic authenticator for pushing to DVCR. The +// credentials are a scoped token (per-namespace authorization) or the shared +// read-write credential; both are presented as Basic username/password. func (i *Importer) destAuthenticator() authn.Authenticator { - if i.destTokenFile != "" { - return auth.TokenFileAuthenticator(i.destTokenFile) - } return &authn.Basic{Username: i.destUsername, Password: i.destPassword} } diff --git a/images/dvcr-artifact/pkg/registry/registry.go b/images/dvcr-artifact/pkg/registry/registry.go index d61350d9cc..bd7777c02b 100644 --- a/images/dvcr-artifact/pkg/registry/registry.go +++ b/images/dvcr-artifact/pkg/registry/registry.go @@ -39,7 +39,6 @@ import ( "golang.org/x/sync/errgroup" "k8s.io/klog/v2" - "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/auth" "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/datasource" importerrs "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/errors" "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/monitoring" @@ -74,7 +73,6 @@ type DataProcessor struct { ds datasource.DataSourceInterface destUsername string destPassword string - destTokenFile string destImageName string sha256Sum string md5Sum string @@ -85,9 +83,6 @@ type DestinationRegistry struct { ImageName string Username string Password string - // TokenFile, when set, is the path to a projected ServiceAccount token used - // for per-namespace DVCR authorization instead of static credentials. - TokenFile string Insecure bool } @@ -96,7 +91,6 @@ func NewDataProcessor(ds datasource.DataSourceInterface, dest DestinationRegistr ds: ds, destUsername: dest.Username, destPassword: dest.Password, - destTokenFile: dest.TokenFile, destImageName: dest.ImageName, sha256Sum: sha256Sum, md5Sum: md5Sum, @@ -104,13 +98,10 @@ func NewDataProcessor(ds datasource.DataSourceInterface, dest DestinationRegistr }, nil } -// destAuthenticator returns the authenticator for pushing to DVCR. With -// per-namespace authorization the projected ServiceAccount token is read fresh -// on every request; otherwise static Basic credentials are used. +// destAuthenticator returns the Basic authenticator for pushing to DVCR. The +// credentials are a scoped token (per-namespace authorization) or the shared +// read-write credential; both are presented as Basic username/password. func (p DataProcessor) destAuthenticator() authn.Authenticator { - if p.destTokenFile != "" { - return auth.TokenFileAuthenticator(p.destTokenFile) - } return &authn.Basic{Username: p.destUsername, Password: p.destPassword} } diff --git a/images/dvcr-artifact/pkg/uploader/uploader.go b/images/dvcr-artifact/pkg/uploader/uploader.go index 3d6a7b38de..7d26dcb651 100644 --- a/images/dvcr-artifact/pkg/uploader/uploader.go +++ b/images/dvcr-artifact/pkg/uploader/uploader.go @@ -50,10 +50,6 @@ const ( healthzPort = 8080 healthzPath = "/healthz" uploadPath = "/upload" - - // uploaderDestinationTokenFileVar is the env var with the path to the - // projected ServiceAccount token used for per-namespace DVCR authorization. - uploaderDestinationTokenFileVar = "UPLOADER_DESTINATION_TOKEN_FILE" ) // UploadServer is the interface to uploadServerApp @@ -89,7 +85,6 @@ type uploadServerApp struct { destImageName string destUsername string destPassword string - destTokenFile string destInsecure bool } @@ -130,10 +125,9 @@ func (app *uploadServerApp) parseOptions() error { app.destImageName, _ = util.ParseEnvVar(common.UploaderDestinationEndpoint, false) app.destInsecure, _ = strconv.ParseBool(os.Getenv(common.DestinationInsecureTLSVar)) - app.destTokenFile = os.Getenv(uploaderDestinationTokenFileVar) app.destUsername, _ = util.ParseEnvVar(common.UploaderDestinationAccessKeyID, false) app.destPassword, _ = util.ParseEnvVar(common.UploaderDestinationSecretKey, false) - if app.destTokenFile == "" && app.destUsername == "" && app.destPassword == "" { + if app.destUsername == "" && app.destPassword == "" { destAuthConfig, _ := util.ParseEnvVar(common.UploaderDestinationAuthConfig, false) if destAuthConfig != "" { authFile, err := auth.RegistryAuthFile(destAuthConfig) @@ -395,7 +389,6 @@ func (app *uploadServerApp) upload(stream io.ReadCloser, sourceContentType strin ImageName: app.destImageName, Username: app.destUsername, Password: app.destPassword, - TokenFile: app.destTokenFile, Insecure: app.destInsecure, }, "", "") if err != nil { From 5665f18dbef1fdbbb6bdaf649fbd5dc5e365b6ca Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 17:08:08 +0200 Subject: [PATCH 14/31] feat(hooks): generate ECDSA keypair for scoped DVCR tokens Add tokenPrivateKey/tokenPublicKey (ECDSA P-256, PKCS8/PKIX PEM) to dvcr-secrets. The controller signs scoped tokens with the private key; the registry verifies them with the public key. Regenerated only when the private key is missing or unparseable. Signed-off-by: Daniil Antoshin --- images/hooks/go.mod | 1 + images/hooks/go.sum | 2 + .../hooks/generate-secret-for-dvcr/hook.go | 98 ++++++++++++++++--- .../generate-secret-for-dvcr/hook_test.go | 20 +++- 4 files changed, 104 insertions(+), 17 deletions(-) diff --git a/images/hooks/go.mod b/images/hooks/go.mod index e88c332ea2..da2d916f61 100644 --- a/images/hooks/go.mod +++ b/images/hooks/go.mod @@ -44,6 +44,7 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/gojuno/minimock/v3 v3.4.5 // indirect + github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/certificate-transparency-go v1.1.7 // indirect github.com/google/gnostic-models v0.7.0 // indirect diff --git a/images/hooks/go.sum b/images/hooks/go.sum index fcb300fc87..55b1a059bf 100644 --- a/images/hooks/go.sum +++ b/images/hooks/go.sum @@ -110,6 +110,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gojuno/minimock/v3 v3.4.5 h1:Jcb0tEYZvVlQNtAAYpg3jCOoSwss2c1/rNugYTzj304= github.com/gojuno/minimock/v3 v3.4.5/go.mod h1:o9F8i2IT8v3yirA7mmdpNGzh1WNesm6iQakMtQV6KiE= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= diff --git a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go index 07c2b8ee01..38d0bb8d87 100644 --- a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go +++ b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go @@ -18,7 +18,12 @@ package generate_secret_for_dvcr import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + crand "crypto/rand" + "crypto/x509" "encoding/base64" + "encoding/pem" "fmt" "math/rand/v2" "strings" @@ -33,19 +38,23 @@ import ( ) const ( - dvcrSecrets = "dvcr-secrets" - passwordRWValuePath = "virtualization.internal.dvcr.passwordRW" - passwordPullerValuePath = "virtualization.internal.dvcr.passwordPuller" - saltValuePath = "virtualization.internal.dvcr.salt" - htpasswdValuePath = "virtualization.internal.dvcr.htpasswd" - user = "admin" + dvcrSecrets = "dvcr-secrets" + passwordRWValuePath = "virtualization.internal.dvcr.passwordRW" + passwordPullerValuePath = "virtualization.internal.dvcr.passwordPuller" + saltValuePath = "virtualization.internal.dvcr.salt" + htpasswdValuePath = "virtualization.internal.dvcr.htpasswd" + tokenPrivateKeyValuePath = "virtualization.internal.dvcr.tokenPrivateKey" + tokenPublicKeyValuePath = "virtualization.internal.dvcr.tokenPublicKey" + user = "admin" ) type dvcrSecretData struct { - PasswordRW string `json:"passwordRW"` - PasswordPuller string `json:"passwordPuller"` - Salt string `json:"salt"` - Htpasswd string `json:"htpasswd"` + PasswordRW string `json:"passwordRW"` + PasswordPuller string `json:"passwordPuller"` + Salt string `json:"salt"` + Htpasswd string `json:"htpasswd"` + TokenPrivateKey string `json:"tokenPrivateKey"` + TokenPublicKey string `json:"tokenPublicKey"` } var _ = registry.RegisterFunc(configDVCRSecrets, handlerDVCRSecrets) @@ -148,6 +157,30 @@ func handlerDVCRSecrets(ctx context.Context, input *pkg.HookInput) error { input.Values.Set(htpasswdValuePath, htpasswd) } + // tokenPrivateKey/tokenPublicKey is the ECDSA keypair used to mint and verify + // scoped DVCR tokens when per-namespace authorization is on. Regenerate the + // pair whenever the private key is missing or unparseable. + tokenPrivateKey := dataFromSecret.TokenPrivateKey + tokenPublicKey := dataFromSecret.TokenPublicKey + privBytes, err := base64.StdEncoding.DecodeString(tokenPrivateKey) + if err != nil || !validateECPrivateKey(privBytes) { + input.Logger.Info("Regenerate DVCR token keypair") + privPEM, pubPEM, genErr := generateECKeypair() + if genErr != nil { + return fmt.Errorf("generate DVCR token keypair: %w", genErr) + } + tokenPrivateKey = base64.StdEncoding.EncodeToString(privPEM) + tokenPublicKey = base64.StdEncoding.EncodeToString(pubPEM) + } + if tokenPrivateKey != dataFromValues.TokenPrivateKey { + input.Logger.Info("Set DVCR token private key to values") + input.Values.Set(tokenPrivateKeyValuePath, tokenPrivateKey) + } + if tokenPublicKey != dataFromValues.TokenPublicKey { + input.Logger.Info("Set DVCR token public key to values") + input.Values.Set(tokenPublicKeyValuePath, tokenPublicKey) + } + return nil } @@ -168,13 +201,50 @@ func getDVCRSecretsFromSecrets(input *pkg.HookInput) (dvcrSecretData, error) { func getDVCRSecretsFromValues(input *pkg.HookInput) dvcrSecretData { return dvcrSecretData{ - PasswordRW: input.Values.Get(passwordRWValuePath).String(), - PasswordPuller: input.Values.Get(passwordPullerValuePath).String(), - Salt: input.Values.Get(saltValuePath).String(), - Htpasswd: input.Values.Get(htpasswdValuePath).String(), + PasswordRW: input.Values.Get(passwordRWValuePath).String(), + PasswordPuller: input.Values.Get(passwordPullerValuePath).String(), + Salt: input.Values.Get(saltValuePath).String(), + Htpasswd: input.Values.Get(htpasswdValuePath).String(), + TokenPrivateKey: input.Values.Get(tokenPrivateKeyValuePath).String(), + TokenPublicKey: input.Values.Get(tokenPublicKeyValuePath).String(), } } +// generateECKeypair returns a fresh ECDSA P-256 keypair as PKCS8/PKIX PEM. The +// private key mints scoped DVCR tokens (controller); the public key verifies them +// (registry). +func generateECKeypair() (privPEM, pubPEM []byte, err error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), crand.Reader) + if err != nil { + return nil, nil, err + } + privDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, nil, err + } + pubDER, err := x509.MarshalPKIXPublicKey(&key.PublicKey) + if err != nil { + return nil, nil, err + } + privPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privDER}) + pubPEM = pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: pubDER}) + return privPEM, pubPEM, nil +} + +// validateECPrivateKey reports whether pemBytes is a parseable ECDSA private key. +func validateECPrivateKey(pemBytes []byte) bool { + block, _ := pem.Decode(pemBytes) + if block == nil { + return false + } + if k, err := x509.ParsePKCS8PrivateKey(block.Bytes); err == nil { + _, ok := k.(*ecdsa.PrivateKey) + return ok + } + _, err := x509.ParseECPrivateKey(block.Bytes) + return err == nil +} + func generateHtpasswd(password string) (string, error) { hashed, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) if err != nil { diff --git a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go index 6415f8f3d1..a2010390bc 100644 --- a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go +++ b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go @@ -46,16 +46,23 @@ var _ = Describe("DVCR Secrets", func() { defaultHtpasswdBase64 = "YWRtaW46JDJhJDEwJHZza21wTjVLSERKUlpNU1pWd3RZWU91YmtEcTEueEF2MXVRQkkvSzFHQ1dpNUpxSnF5amdt" // "admin:$2a$10$vskmpN5KHDJRZMSZVwtYYOubkDq1.xAv1uQBI/K1GCWi5JqJqyjgm" ) var ( - dc *mock.DependencyContainerMock - snapshots *mock.SnapshotsMock - values *mock.OutputPatchableValuesCollectorMock + dc *mock.DependencyContainerMock + snapshots *mock.SnapshotsMock + values *mock.OutputPatchableValuesCollectorMock + validTokenKey string + validTokenPub string ) + // prepareValuesGet mocks the values-side reads. The token keypair is always a + // valid pair matching the snapshot, so these tests never exercise keypair + // regeneration (its own coverage lives in signer_test.go and the hook helpers). prepareValuesGet := func(passwordRW, passwordPuller, salt, htpasswd string) { values.GetMock.When(passwordRWValuePath).Then(gjson.Result{Type: gjson.String, Str: passwordRW}) values.GetMock.When(passwordPullerValuePath).Then(gjson.Result{Type: gjson.String, Str: passwordPuller}) values.GetMock.When(saltValuePath).Then(gjson.Result{Type: gjson.String, Str: salt}) values.GetMock.When(htpasswdValuePath).Then(gjson.Result{Type: gjson.String, Str: htpasswd}) + values.GetMock.When(tokenPrivateKeyValuePath).Then(gjson.Result{Type: gjson.String, Str: validTokenKey}) + values.GetMock.When(tokenPublicKeyValuePath).Then(gjson.Result{Type: gjson.String, Str: validTokenPub}) } setSnapshots := func(snaps ...pkg.Snapshot) { @@ -71,6 +78,8 @@ var _ = Describe("DVCR Secrets", func() { data.PasswordPuller = passwordPuller data.Salt = salt data.Htpasswd = htpasswd + data.TokenPrivateKey = validTokenKey + data.TokenPublicKey = validTokenPub return nil }) } @@ -94,6 +103,11 @@ var _ = Describe("DVCR Secrets", func() { snapshots = mock.NewSnapshotsMock(GinkgoT()) values = mock.NewPatchableValuesCollectorMock(GinkgoT()) values.GetMock.When(settings.InternalValuesConfigCopyPath).Then(gjson.Result{}) + + priv, pub, err := generateECKeypair() + Expect(err).ToNot(HaveOccurred()) + validTokenKey = base64.StdEncoding.EncodeToString(priv) + validTokenPub = base64.StdEncoding.EncodeToString(pub) }) AfterEach(func() { From 27c7ce0f0ff549e0d6f84b480bc7326263dcc8b8 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 17:10:03 +0200 Subject: [PATCH 15/31] feat(dvcr): wire scoped-token JWT auth into manifests - dvcr-secrets: ship the ECDSA keypair when tenant authz is on - registry configmap: dvcr-k8s auth verifies JWT (issuer/audience/kid, public key file) instead of TokenReview - mount tokenPublicKey into the registry; inject tokenPrivateKey into the controller - drop the system:auth-delegator binding (no TokenReview) - revert the CDI fork pin to upstream (no fork needed) Signed-off-by: Daniil Antoshin --- openapi/values.yaml | 6 ++++++ templates/dvcr/_helpers.tpl | 2 ++ templates/dvcr/configmap.yaml | 7 ++++--- templates/dvcr/rbac-for-us.yaml | 16 ---------------- templates/dvcr/secret.yaml | 4 ++++ templates/virtualization-controller/_helpers.tpl | 7 +++++++ 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/openapi/values.yaml b/openapi/values.yaml index 6b229c0656..0087d66969 100644 --- a/openapi/values.yaml +++ b/openapi/values.yaml @@ -36,6 +36,12 @@ properties: passwordPuller: type: string default: "" + tokenPrivateKey: + type: string + default: "" + tokenPublicKey: + type: string + default: "" htpasswd: type: string salt: diff --git a/templates/dvcr/_helpers.tpl b/templates/dvcr/_helpers.tpl index 74de618c22..8259289594 100644 --- a/templates/dvcr/_helpers.tpl +++ b/templates/dvcr/_helpers.tpl @@ -102,6 +102,8 @@ true path: passwordRW - key: passwordPuller path: passwordPuller + - key: tokenPublicKey + path: tokenPublicKey {{- end }} {{- end -}} diff --git a/templates/dvcr/configmap.yaml b/templates/dvcr/configmap.yaml index 78a0ff456f..c54b353877 100644 --- a/templates/dvcr/configmap.yaml +++ b/templates/dvcr/configmap.yaml @@ -27,9 +27,10 @@ data: adminpasswordfile: /auth/passwordRW pullerusername: "node-puller" pullerpasswordfile: /auth/passwordPuller - privilegednamespace: d8-{{ .Chart.Name }} - tokenreviewcachettl: "45s" - tokenreviewcachenegttl: "5s" + jwtissuer: "virtualization-controller" + jwtaudience: "dvcr" + jwtkeyid: "dvcr" + jwtpublickeyfile: /auth/tokenPublicKey {{- end }} http: addr: :5000 diff --git a/templates/dvcr/rbac-for-us.yaml b/templates/dvcr/rbac-for-us.yaml index cbb3336555..84e4b6deab 100644 --- a/templates/dvcr/rbac-for-us.yaml +++ b/templates/dvcr/rbac-for-us.yaml @@ -84,20 +84,4 @@ subjects: - kind: ServiceAccount name: dvcr namespace: d8-{{ .Chart.Name }} ---- -# Allow the registry to verify ServiceAccount tokens via TokenReview for the -# dvcr-k8s multi-tenant authorization backend. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: d8:{{ .Chart.Name }}:dvcr:auth-delegator - {{- include "helm_lib_module_labels" (list . (dict "app" "dvcr")) | nindent 2 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: - - kind: ServiceAccount - name: dvcr - namespace: d8-{{ .Chart.Name }} {{- end }} diff --git a/templates/dvcr/secret.yaml b/templates/dvcr/secret.yaml index 918ab7d67c..6a0141842a 100644 --- a/templates/dvcr/secret.yaml +++ b/templates/dvcr/secret.yaml @@ -27,6 +27,10 @@ data: passwordPuller: {{ .Values.virtualization.internal.dvcr.passwordPuller | quote }} htpasswd: {{ .Values.virtualization.internal.dvcr.htpasswd | quote }} salt: {{ .Values.virtualization.internal.dvcr.salt | quote }} +{{- if eq (include "dvcr.isTenantAuthz" .) "true" }} + tokenPrivateKey: {{ .Values.virtualization.internal.dvcr.tokenPrivateKey | quote }} + tokenPublicKey: {{ .Values.virtualization.internal.dvcr.tokenPublicKey | quote }} +{{- end }} --- {{- /* TODO: delete this secret, because containerd has credentials to auth to the dvcr. This secret was needed to create virtual machines from dvcr images */ -}} diff --git a/templates/virtualization-controller/_helpers.tpl b/templates/virtualization-controller/_helpers.tpl index faecd31cd4..c76eb13c1d 100644 --- a/templates/virtualization-controller/_helpers.tpl +++ b/templates/virtualization-controller/_helpers.tpl @@ -42,6 +42,13 @@ true value: "{{ .Values.virtualization.internal.moduleConfig | dig "dvcr" "gc" "schedule" "" }}" - name: DVCR_TENANT_AUTHZ_ENABLED value: "{{ include "dvcr.isTenantAuthz" . }}" +{{- if eq (include "dvcr.isTenantAuthz" .) "true" }} +- name: DVCR_TOKEN_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: dvcr-secrets + key: tokenPrivateKey +{{- end }} - name: VIRTUAL_MACHINE_CIDRS value: {{ join "," .Values.virtualization.internal.moduleConfig.virtualMachineCIDRs | quote }} {{- if (hasKey .Values.virtualization.internal.moduleConfig "virtualImages") }} From ff2b03a54db663a0f1787ff5255d520710f26036 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 17:26:58 +0200 Subject: [PATCH 16/31] refactor(dvcr): simplify scoped-token plumbing - revert dvcr-artifact to upstream: the scoped token arrives as plain Basic creds, so the destAuthenticator indirection added earlier is gone (the package is now byte-identical to main) - drop the unused ttl parameter from Signer.Sign (always DefaultTTL) - correct the feature-flag docs: scoped token, not ServiceAccount token Signed-off-by: Daniil Antoshin --- images/dvcr-artifact/pkg/importer/importer.go | 13 +++------ images/dvcr-artifact/pkg/registry/registry.go | 27 +++++++------------ .../supplements/copier/auth_secret.go | 4 +-- .../pkg/dvcr/registrytoken/signer.go | 11 +++----- .../pkg/dvcr/registrytoken/signer_test.go | 7 ++--- openapi/config-values.yaml | 8 +++--- openapi/doc-ru-config-values.yaml | 6 ++--- 7 files changed, 30 insertions(+), 46 deletions(-) diff --git a/images/dvcr-artifact/pkg/importer/importer.go b/images/dvcr-artifact/pkg/importer/importer.go index a874d7ab91..1c6534cb6b 100644 --- a/images/dvcr-artifact/pkg/importer/importer.go +++ b/images/dvcr-artifact/pkg/importer/importer.go @@ -49,7 +49,7 @@ const ( DockerRegistrySchemePrefix = "docker://" DVCRSource = "dvcr" BlockDeviceSource = "blockDevice" - FilesystemSource = "filesystem" + FilesystemSource = "filesystem" ) func New() *Importer { @@ -253,19 +253,12 @@ func (i *Importer) destRemoteOptions(ctx context.Context) []remote.Option { remoteOpts := []remote.Option{ remote.WithContext(ctx), remote.WithTransport(transport), - remote.WithAuth(i.destAuthenticator()), + remote.WithAuth(&authn.Basic{Username: i.destUsername, Password: i.destPassword}), } return remoteOpts } -// destAuthenticator returns the Basic authenticator for pushing to DVCR. The -// credentials are a scoped token (per-namespace authorization) or the shared -// read-write credential; both are presented as Basic username/password. -func (i *Importer) destAuthenticator() authn.Authenticator { - return &authn.Basic{Username: i.destUsername, Password: i.destPassword} -} - func (i *Importer) destCraneOptions(ctx context.Context) []crane.Option { tlsConfig := &tls.Config{ InsecureSkipVerify: i.destInsecure, @@ -277,7 +270,7 @@ func (i *Importer) destCraneOptions(ctx context.Context) []crane.Option { craneOpts := []crane.Option{ crane.WithContext(ctx), crane.WithTransport(transport), - crane.WithAuth(i.destAuthenticator()), + crane.WithAuth(&authn.Basic{Username: i.destUsername, Password: i.destPassword}), } return craneOpts diff --git a/images/dvcr-artifact/pkg/registry/registry.go b/images/dvcr-artifact/pkg/registry/registry.go index bd7777c02b..a8830bb7cf 100644 --- a/images/dvcr-artifact/pkg/registry/registry.go +++ b/images/dvcr-artifact/pkg/registry/registry.go @@ -88,23 +88,16 @@ type DestinationRegistry struct { func NewDataProcessor(ds datasource.DataSourceInterface, dest DestinationRegistry, sha256Sum, md5Sum string) (*DataProcessor, error) { return &DataProcessor{ - ds: ds, - destUsername: dest.Username, - destPassword: dest.Password, - destImageName: dest.ImageName, - sha256Sum: sha256Sum, - md5Sum: md5Sum, - destInsecure: dest.Insecure, + ds, + dest.Username, + dest.Password, + dest.ImageName, + sha256Sum, + md5Sum, + dest.Insecure, }, nil } -// destAuthenticator returns the Basic authenticator for pushing to DVCR. The -// credentials are a scoped token (per-namespace authorization) or the shared -// read-write credential; both are presented as Basic username/password. -func (p DataProcessor) destAuthenticator() authn.Authenticator { - return &authn.Basic{Username: p.destUsername, Password: p.destPassword} -} - func (p DataProcessor) Process(ctx context.Context) (ImportRes, error) { sourceImageFilename, err := p.ds.Filename() if err != nil { @@ -332,7 +325,7 @@ func (p DataProcessor) uploadLayersAndImage( informer *ImageInformer, ) error { nameOpts := destNameOptions(p.destInsecure) - remoteOpts := destRemoteOptions(ctx, p.destAuthenticator(), p.destInsecure) + remoteOpts := destRemoteOptions(ctx, p.destUsername, p.destPassword, p.destInsecure) image := empty.Image ref, err := name.ParseReference(p.destImageName, nameOpts...) @@ -424,7 +417,7 @@ func destNameOptions(destInsecure bool) []name.Option { return nameOpts } -func destRemoteOptions(ctx context.Context, authenticator authn.Authenticator, destInsecure bool) []remote.Option { +func destRemoteOptions(ctx context.Context, destUsername, destPassword string, destInsecure bool) []remote.Option { tlsConfig := &tls.Config{ InsecureSkipVerify: destInsecure, } @@ -435,7 +428,7 @@ func destRemoteOptions(ctx context.Context, authenticator authn.Authenticator, d remoteOpts := []remote.Option{ remote.WithContext(ctx), remote.WithTransport(transport), - remote.WithAuth(authenticator), + remote.WithAuth(&authn.Basic{Username: destUsername, Password: destPassword}), } return remoteOpts diff --git a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go index 67d8c26ae3..48c7756929 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go +++ b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go @@ -54,7 +54,7 @@ type AuthSecret struct { // authenticates to DVCR with a credential scoped to exactly the repositories in // access, instead of the shared read-write credential. func (a AuthSecret) CreateScopedTokenCDI(ctx context.Context, client client.Client, signer *registrytoken.Signer, access []registrytoken.Access) error { - raw, err := signer.Sign(access, 0, time.Now()) + raw, err := signer.Sign(access, time.Now()) if err != nil { return fmt.Errorf("mint scoped DVCR token: %w", err) } @@ -70,7 +70,7 @@ func (a AuthSecret) CreateScopedTokenCDI(ctx context.Context, client client.Clie // it as a dockerconfigjson Secret keyed by registryURL, so the dvcr-artifact // importer/uploader Pod reads it through the standard destination auth config. func (a AuthSecret) CreateScopedTokenDockerConfig(ctx context.Context, client client.Client, signer *registrytoken.Signer, access []registrytoken.Access, registryURL string) error { - raw, err := signer.Sign(access, 0, time.Now()) + raw, err := signer.Sign(access, time.Now()) if err != nil { return fmt.Errorf("mint scoped DVCR token: %w", err) } diff --git a/images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go index b3264bd0d3..bf6837304a 100644 --- a/images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go +++ b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go @@ -85,18 +85,15 @@ func parseECKey(der []byte) (*ecdsa.PrivateKey, error) { return x509.ParseECPrivateKey(der) } -// Sign issues a token granting access, valid for ttl. A non-positive ttl uses -// DefaultTTL. now is passed explicitly to keep the function testable. -func (s *Signer) Sign(access []Access, ttl time.Duration, now time.Time) (string, error) { - if ttl <= 0 { - ttl = DefaultTTL - } +// Sign issues a token granting access, valid for DefaultTTL. now is passed +// explicitly to keep the function testable. +func (s *Signer) Sign(access []Access, now time.Time) (string, error) { claims := jwt.MapClaims{ "iss": Issuer, "aud": Audience, "iat": now.Unix(), "nbf": now.Add(-30 * time.Second).Unix(), - "exp": now.Add(ttl).Unix(), + "exp": now.Add(DefaultTTL).Unix(), "access": access, } tok := jwt.NewWithClaims(jwt.SigningMethodES256, claims) diff --git a/images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go index f1854ccb6b..381e2a40b1 100644 --- a/images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go +++ b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go @@ -50,7 +50,7 @@ func TestSignRoundTrip(t *testing.T) { access := []Access{{Type: "repository", Name: "cvi/my-image", Actions: []string{"pull", "push"}}} now := time.Unix(1_700_000_000, 0) - raw, err := signer.Sign(access, time.Hour, now) + raw, err := signer.Sign(access, now) if err != nil { t.Fatal(err) } @@ -86,12 +86,13 @@ func TestSignExpired(t *testing.T) { keyPEM, pub := newSignerPEM(t) signer, _ := NewSignerFromPEM(keyPEM) now := time.Unix(1_700_000_000, 0) - raw, err := signer.Sign(nil, time.Hour, now) + raw, err := signer.Sign(nil, now) if err != nil { t.Fatal(err) } + // Past DefaultTTL the token must no longer verify. _, err = jwt.Parse(raw, func(*jwt.Token) (interface{}, error) { return pub, nil }, - jwt.WithTimeFunc(func() time.Time { return now.Add(2 * time.Hour) })) + jwt.WithTimeFunc(func() time.Time { return now.Add(DefaultTTL + time.Hour) })) if err == nil { t.Fatal("expected expired token to fail verification") } diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 8e39217237..852be4b154 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -139,10 +139,10 @@ properties: description: | Enable per-namespace authorization in the DVCR registry (experimental). - When enabled, image import and upload Pods authenticate to DVCR with their own - ServiceAccount token and may access only repositories in their namespace, while - nodes get pull-only access. The shared read-write credential is no longer copied - into tenant namespaces. + When enabled, image import and upload Pods authenticate to DVCR with a scoped + token that grants access only to the repositories they use, while nodes get + pull-only access. The shared read-write credential is no longer copied into + tenant namespaces. When disabled (default), DVCR uses shared htpasswd credentials. Disable to roll back. audit: diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index 36430b6a9f..500ff4efa3 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -79,9 +79,9 @@ properties: description: | Включить авторизацию по namespace в реестре DVCR (экспериментально). - Когда включено, поды импорта и загрузки образов аутентифицируются в DVCR своим - ServiceAccount-токеном и получают доступ только к репозиториям своего namespace, а - узлы получают доступ только на чтение (pull). Общий read-write-креденшл больше не + Когда включено, поды импорта и загрузки образов аутентифицируются в DVCR + scoped-токеном, дающим доступ только к используемым ими репозиториям, а узлы + получают доступ только на чтение (pull). Общий read-write-креденшл больше не копируется в namespace тенантов. Когда выключено (по умолчанию), DVCR использует общие htpasswd-учётные данные. From fbb4c01564ebe61ec35a796b46becb6a6c332f79 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 17:30:01 +0200 Subject: [PATCH 17/31] docs(dvcr): reword token TTL comment Signed-off-by: Daniil Antoshin --- .../virtualization-artifact/pkg/dvcr/registrytoken/signer.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go index bf6837304a..f5223e8224 100644 --- a/images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go +++ b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer.go @@ -42,9 +42,8 @@ const ( KeyID = "dvcr" // DefaultTTL is the scoped token lifetime. Generous on purpose: an import may // run for hours, and the token is scoped to a single repository, so a long - // validity window is a small exposure. - // ponytail: fixed TTL; bind to the Pod's activeDeadlineSeconds if imports ever - // need to outlive this or exposure must be tightened further. + // validity window is a small exposure. If imports ever outlive this, bind the + // TTL to the Pod's activeDeadlineSeconds instead. DefaultTTL = 24 * time.Hour ) From 59abbd0b54ce9dad6b8cee13ccffc98aa239dfcf Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 17:32:24 +0200 Subject: [PATCH 18/31] refactor(dvcr): drop redundant comments Signed-off-by: Daniil Antoshin --- .../pkg/controller/importer/settings.go | 3 --- .../pkg/controller/supplements/ensure.go | 5 ----- .../pkg/controller/uploader/settings.go | 3 --- 3 files changed, 11 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/importer/settings.go b/images/virtualization-artifact/pkg/controller/importer/settings.go index a5fd939463..c1f5b4d1c7 100644 --- a/images/virtualization-artifact/pkg/controller/importer/settings.go +++ b/images/virtualization-artifact/pkg/controller/importer/settings.go @@ -70,9 +70,6 @@ type Settings struct { func ApplyDVCRDestinationSettings(podEnvVars *Settings, dvcrSettings *dvcr.Settings, supGen supplements.Generator, dvcrImageName string) { authSecret := dvcrSettings.AuthSecret - // With per-namespace authorization every Pod gets a per-import scoped token - // Secret (created by supplements.EnsureForPod); otherwise the shared credential - // is used, copied into the namespace only when they differ. if dvcrSettings.TenantAuthzEnabled || supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { authSecret = supGen.DVCRAuthSecret().Name } diff --git a/images/virtualization-artifact/pkg/controller/supplements/ensure.go b/images/virtualization-artifact/pkg/controller/supplements/ensure.go index a66520b210..9a798cf961 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/ensure.go +++ b/images/virtualization-artifact/pkg/controller/supplements/ensure.go @@ -63,8 +63,6 @@ func EnsureForPod(ctx context.Context, client client.Client, supGen Generator, p // Create Secret with auth config to use DVCR as destination. switch { case dvcrSettings.TenantAuthzEnabled: - // Mint a token scoped to this Pod's repositories instead of copying the - // shared read-write credential into the (possibly tenant) namespace. authCopier := copier.AuthSecret{ Secret: copier.Secret{ Destination: supGen.DVCRAuthSecret(), @@ -155,9 +153,6 @@ func EnsureForDataVolume(ctx context.Context, client client.Client, supGen DataV }, } - // With per-namespace authorization the CDI importer authenticates to DVCR - // with a token scoped to the source repository it pulls from, so the shared - // read-write credential is not copied into the tenant namespace. if dvcrSettings.TenantAuthzEnabled { if err := authCopier.CreateScopedTokenCDI(ctx, client, dvcrSettings.TokenSigner, scope); err != nil { return err diff --git a/images/virtualization-artifact/pkg/controller/uploader/settings.go b/images/virtualization-artifact/pkg/controller/uploader/settings.go index 546c637f32..e72b2e0903 100644 --- a/images/virtualization-artifact/pkg/controller/uploader/settings.go +++ b/images/virtualization-artifact/pkg/controller/uploader/settings.go @@ -32,9 +32,6 @@ type Settings struct { func ApplyDVCRDestinationSettings(podEnvVars *Settings, dvcrSettings *dvcr.Settings, supGen supplements.Generator, dvcrImageName string) { authSecret := dvcrSettings.AuthSecret - // With per-namespace authorization every Pod gets a per-import scoped token - // Secret (created by supplements.EnsureForPod); otherwise the shared credential - // is used, copied into the namespace only when they differ. if dvcrSettings.TenantAuthzEnabled || supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { authSecret = supGen.DVCRAuthSecret().Name } From 64a4b627aa346cfa877c70a7691be1754a5ad05c Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 17:50:02 +0200 Subject: [PATCH 19/31] chore(dvcr): bump distribution to v3.1.1 Signed-off-by: Daniil Antoshin --- build/components/versions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/components/versions.yml b/build/components/versions.yml index 0c976a15d7..b466a63b68 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -5,7 +5,7 @@ firmware: core: 3p-kubevirt: v1.6.2-v12n.53 3p-containerized-data-importer: v1.60.3-v12n.21 - distribution: 3.0.0 + distribution: 3.1.1 package: acl: v2.3.1 bzip2: bzip2-1.0.8 From 37c51f57d509d57636b7e41d05c05db3b1d4cdd7 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 18:56:25 +0200 Subject: [PATCH 20/31] fix(dvcr): reject x5c/jwk-signed scoped tokens in registry authz distribution's token.Verify tries a token's embedded x5c/jwk certificate chain before the pinned kid->TrustedKeys lookup. With VerifyOptions.Roots unset the chain validates against the system CA pool, so any publicly trusted ECDSA leaf certificate could forge a token and bypass per-namespace authorization. Pass a non-nil empty cert pool so no external chain verifies and only the pinned public key (matched by kid) is trusted. Signed-off-by: Daniil Antoshin --- images/dvcr/dvcr-k8s-auth/access.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/images/dvcr/dvcr-k8s-auth/access.go b/images/dvcr/dvcr-k8s-auth/access.go index 05021c47c7..538b3b9518 100644 --- a/images/dvcr/dvcr-k8s-auth/access.go +++ b/images/dvcr/dvcr-k8s-auth/access.go @@ -189,6 +189,12 @@ func (ac *accessController) verifyJWT(raw string) ([]Grant, error) { TrustedIssuers: []string{ac.jwtIssuer}, AcceptedAudiences: []string{ac.jwtAudience}, TrustedKeys: ac.trustedKeys, + // Roots must be a non-nil empty pool. distribution verifies a token's + // x5c/jwk certificate chain before the pinned kid->TrustedKeys lookup; + // with Roots unset (nil) that chain validates against the system CA pool, + // letting any publicly-trusted leaf cert forge a token. An empty pool + // fails every chain, so only our pinned key (matched by kid) is trusted. + Roots: x509.NewCertPool(), }) if err != nil { return nil, err From 1851b57eb34ee0b8994a6b807ca12e7309ceffb2 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 18:56:50 +0200 Subject: [PATCH 21/31] fix(dvcr): mint scoped DataVolume token independent of AuthSecret CreateScopedTokenCDI signs from TokenSigner and does not read the source AuthSecret, but it was nested under `if AuthSecret != ""`. With tenant authorization on and AuthSecret empty, the DataVolume referenced an auth secret that was never created, stalling the import. Switch on TenantAuthzEnabled first, matching EnsureForPod. Signed-off-by: Daniil Antoshin --- .../pkg/controller/supplements/ensure.go | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/supplements/ensure.go b/images/virtualization-artifact/pkg/controller/supplements/ensure.go index 9a798cf961..b4d572528b 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/ensure.go +++ b/images/virtualization-artifact/pkg/controller/supplements/ensure.go @@ -140,24 +140,29 @@ func ShouldCopyImagePullSecret(ctrImg *datasource.ContainerRegistry, targetNS st } func EnsureForDataVolume(ctx context.Context, client client.Client, supGen DataVolumeSupplement, dv *cdiv1.DataVolume, dvcrSettings *dvcr.Settings, scope []registrytoken.Access) error { - if dvcrSettings.AuthSecret != "" { - authSecret := supGen.DVCRAuthSecretForDV() + switch { + case dvcrSettings.TenantAuthzEnabled: + authCopier := copier.AuthSecret{ + Secret: copier.Secret{ + Destination: supGen.DVCRAuthSecretForDV(), + OwnerReference: dvutil.MakeOwnerReference(dv), + }, + } + if err := authCopier.CreateScopedTokenCDI(ctx, client, dvcrSettings.TokenSigner, scope); err != nil { + return err + } + case dvcrSettings.AuthSecret != "": authCopier := copier.AuthSecret{ Secret: copier.Secret{ Source: types.NamespacedName{ Name: dvcrSettings.AuthSecret, Namespace: dvcrSettings.AuthSecretNamespace, }, - Destination: authSecret, + Destination: supGen.DVCRAuthSecretForDV(), OwnerReference: dvutil.MakeOwnerReference(dv), }, } - - if dvcrSettings.TenantAuthzEnabled { - if err := authCopier.CreateScopedTokenCDI(ctx, client, dvcrSettings.TokenSigner, scope); err != nil { - return err - } - } else if err := authCopier.CopyCDICompatible(ctx, client, dvcrSettings.RegistryURL); err != nil { + if err := authCopier.CopyCDICompatible(ctx, client, dvcrSettings.RegistryURL); err != nil { return err } } From 432b848dbd01b272012da3e25d6d11e72ceb43c0 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 18:57:18 +0200 Subject: [PATCH 22/31] fix(dvcr): generate registry passwords with crypto/rand alphaNum seeded math/rand/v2 with the wall-clock time, a non-cryptographic PRNG, to produce the shared registry passwordRW and the new node-puller password. Use crypto/rand so these long-lived credentials are not derivable from the generation time. Signed-off-by: Daniil Antoshin --- .../hooks/pkg/hooks/generate-secret-for-dvcr/hook.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go index 38d0bb8d87..a0dc7825b6 100644 --- a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go +++ b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go @@ -25,9 +25,8 @@ import ( "encoding/base64" "encoding/pem" "fmt" - "math/rand/v2" + "math/big" "strings" - "time" "golang.org/x/crypto/bcrypt" "k8s.io/utils/ptr" @@ -273,11 +272,14 @@ const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456 //nolint:unparam // we need to pass the length func alphaNum(length int) string { - rnd := rand.New(rand.NewPCG(uint64(time.Now().UnixNano()), 0)) - + max := big.NewInt(int64(len(letterBytes))) b := make([]byte, length) for i := range b { - b[i] = letterBytes[rnd.IntN(len(letterBytes))] + n, err := crand.Int(crand.Reader, max) + if err != nil { + panic(fmt.Sprintf("generate random password: %v", err)) + } + b[i] = letterBytes[n.Int64()] } return string(b) } From d096ab06a8a7bab89c2af95a14d006738ef15336 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 18:57:54 +0200 Subject: [PATCH 23/31] fix(dvcr): regenerate token keypair when public key is bad or mismatched Keypair regeneration was gated only on the private key parsing. A corrupt, missing or mismatched tokenPublicKey (shipped to the registry) would then be kept, making the registry reject every scoped token with no self-heal. Validate that the public key parses and matches the private key, else regenerate the pair. Signed-off-by: Daniil Antoshin --- .../hooks/generate-secret-for-dvcr/hook.go | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go index a0dc7825b6..7b5d1afdff 100644 --- a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go +++ b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go @@ -162,7 +162,8 @@ func handlerDVCRSecrets(ctx context.Context, input *pkg.HookInput) error { tokenPrivateKey := dataFromSecret.TokenPrivateKey tokenPublicKey := dataFromSecret.TokenPublicKey privBytes, err := base64.StdEncoding.DecodeString(tokenPrivateKey) - if err != nil || !validateECPrivateKey(privBytes) { + pubBytes, pubErr := base64.StdEncoding.DecodeString(tokenPublicKey) + if err != nil || pubErr != nil || !validateECKeypair(privBytes, pubBytes) { input.Logger.Info("Regenerate DVCR token keypair") privPEM, pubPEM, genErr := generateECKeypair() if genErr != nil { @@ -230,18 +231,43 @@ func generateECKeypair() (privPEM, pubPEM []byte, err error) { return privPEM, pubPEM, nil } -// validateECPrivateKey reports whether pemBytes is a parseable ECDSA private key. -func validateECPrivateKey(pemBytes []byte) bool { - block, _ := pem.Decode(pemBytes) +// validateECKeypair reports whether privPEM is a parseable ECDSA private key and +// pubPEM is its matching public key. The public key is shipped to the registry as +// /auth/tokenPublicKey; a missing, corrupt or mismatched one would make it reject +// every scoped token, so any of those must force regeneration of the whole pair. +func validateECKeypair(privPEM, pubPEM []byte) bool { + priv := parseECPrivateKey(privPEM) + if priv == nil { + return false + } + block, _ := pem.Decode(pubPEM) if block == nil { return false } + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return false + } + ecPub, ok := pub.(*ecdsa.PublicKey) + return ok && ecPub.Equal(&priv.PublicKey) +} + +func parseECPrivateKey(pemBytes []byte) *ecdsa.PrivateKey { + block, _ := pem.Decode(pemBytes) + if block == nil { + return nil + } if k, err := x509.ParsePKCS8PrivateKey(block.Bytes); err == nil { - _, ok := k.(*ecdsa.PrivateKey) - return ok + if ec, ok := k.(*ecdsa.PrivateKey); ok { + return ec + } + return nil } - _, err := x509.ParseECPrivateKey(block.Bytes) - return err == nil + ec, err := x509.ParseECPrivateKey(block.Bytes) + if err != nil { + return nil + } + return ec } func generateHtpasswd(password string) (string, error) { From f0cdae19279b96cc3448a750ea14ff819446a5f9 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 18:58:08 +0200 Subject: [PATCH 24/31] docs(dvcr): note go-jose dependency in registry graft comment Signed-off-by: Daniil Antoshin --- images/dvcr/werf.inc.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/images/dvcr/werf.inc.yaml b/images/dvcr/werf.inc.yaml index 2bb689001b..35b50c45fe 100644 --- a/images/dvcr/werf.inc.yaml +++ b/images/dvcr/werf.inc.yaml @@ -83,8 +83,9 @@ shell: cd /distribution # Graft the DVCR auth plugin and its entrypoint into the distribution module. - # The plugin depends only on the distribution auth package and the standard - # library, so no go.mod changes are required. + # The plugin depends only on the distribution auth package, go-jose/v4 (already + # a direct dependency of distribution v3) and the standard library, so no go.mod + # changes are required. mkdir -p /distribution/registry/auth/dvcrk8s /distribution/cmd/dvcr-registry cp /plugin/dvcrk8s/*.go /distribution/registry/auth/dvcrk8s/ cp /plugin/registry-main/*.go /distribution/cmd/dvcr-registry/ From a2122afd7d91cfa0db4ebe38c82c6d562d9a6bfd Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 19:45:05 +0200 Subject: [PATCH 25/31] fix(dvcr): refresh scoped token secret before it expires The scoped token was minted once and stored with a create-only Secret, so a reconcile never re-issued it. An import that outlived the 24h token TTL, or a Pod rescheduled past it, read an expired token and got a permanent 403 with no self-heal. Record the token expiry in an annotation and re-mint the Secret on reconcile only when less than half the TTL remains, keeping churn low while guaranteeing a valid token for long-running imports. Signed-off-by: Daniil Antoshin --- .../supplements/copier/auth_secret.go | 86 +++++++++--- .../supplements/copier/auth_secret_test.go | 123 ++++++++++++++++++ 2 files changed, 194 insertions(+), 15 deletions(-) create mode 100644 images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret_test.go diff --git a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go index 48c7756929..5733ff573d 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go +++ b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go @@ -24,6 +24,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/deckhouse/virtualization-controller/pkg/auth" @@ -36,6 +37,11 @@ import ( // is a fixed, human-readable marker. const ScopedTokenUsername = "dvcr-jwt" +// scopedTokenExpiryAnnotation records the scoped token's expiry (RFC3339) so a +// reconcile can re-mint the Secret only when the token is close to expiring, +// instead of writing a fresh token on every pass. +const scopedTokenExpiryAnnotation = "dvcr.deckhouse.io/scoped-token-expires-at" + // AuthSecret copies auth credentials from the source Secret into // Destination Secret and ensure its data is CDI compatible: // type: Opaque @@ -54,33 +60,83 @@ type AuthSecret struct { // authenticates to DVCR with a credential scoped to exactly the repositories in // access, instead of the shared read-write credential. func (a AuthSecret) CreateScopedTokenCDI(ctx context.Context, client client.Client, signer *registrytoken.Signer, access []registrytoken.Access) error { - raw, err := signer.Sign(access, time.Now()) - if err != nil { - return fmt.Errorf("mint scoped DVCR token: %w", err) - } - destData := map[string][]byte{ - "accessKeyId": []byte(ScopedTokenUsername), - "secretKey": []byte(raw), - } - _, err = a.Create(ctx, client, destData, corev1.SecretTypeOpaque) - return err + return a.ensureScopedToken(ctx, client, signer, access, func(raw string) (map[string][]byte, corev1.SecretType, error) { + return map[string][]byte{ + "accessKeyId": []byte(ScopedTokenUsername), + "secretKey": []byte(raw), + }, corev1.SecretTypeOpaque, nil + }) } // CreateScopedTokenDockerConfig mints a scoped DVCR token for access and stores // it as a dockerconfigjson Secret keyed by registryURL, so the dvcr-artifact // importer/uploader Pod reads it through the standard destination auth config. func (a AuthSecret) CreateScopedTokenDockerConfig(ctx context.Context, client client.Client, signer *registrytoken.Signer, access []registrytoken.Access, registryURL string) error { - raw, err := signer.Sign(access, time.Now()) + return a.ensureScopedToken(ctx, client, signer, access, func(raw string) (map[string][]byte, corev1.SecretType, error) { + cfg, err := dockerConfigJSON(registryURL, ScopedTokenUsername, raw) + if err != nil { + return nil, "", err + } + return map[string][]byte{corev1.DockerConfigJsonKey: cfg}, corev1.SecretTypeDockerConfigJson, nil + }) +} + +// ensureScopedToken mints a scoped token and writes the destination Secret only +// when it is missing or within half its TTL of expiring. Reusing a still-fresh +// Secret keeps re-mint churn down; refreshing before expiry means an import that +// outlives one token gets a valid one on a later reconcile instead of a stale 403. +func (a AuthSecret) ensureScopedToken(ctx context.Context, client client.Client, signer *registrytoken.Signer, access []registrytoken.Access, build func(raw string) (map[string][]byte, corev1.SecretType, error)) error { + now := time.Now() + + existing := &corev1.Secret{} + err := client.Get(ctx, a.Destination, existing) + switch { + case err == nil: + if scopedTokenValidFor(existing, now) > registrytoken.DefaultTTL/2 { + return nil + } + case !k8serrors.IsNotFound(err): + return err + } + + raw, err := signer.Sign(access, now) if err != nil { return fmt.Errorf("mint scoped DVCR token: %w", err) } - cfg, err := dockerConfigJSON(registryURL, ScopedTokenUsername, raw) + data, secretType, err := build(raw) if err != nil { return err } - destData := map[string][]byte{corev1.DockerConfigJsonKey: cfg} - _, err = a.Create(ctx, client, destData, corev1.SecretTypeDockerConfigJson) - return err + + secret := a.makeSecret(data, secretType) + secret.Annotations[scopedTokenExpiryAnnotation] = now.Add(registrytoken.DefaultTTL).Format(time.RFC3339) + + if createErr := client.Create(ctx, secret); createErr != nil { + if !k8serrors.IsAlreadyExists(createErr) { + return createErr + } + existing.Data = secret.Data + if existing.Annotations == nil { + existing.Annotations = map[string]string{} + } + existing.Annotations[scopedTokenExpiryAnnotation] = secret.Annotations[scopedTokenExpiryAnnotation] + return client.Update(ctx, existing) + } + return nil +} + +// scopedTokenValidFor returns how long the Secret's scoped token stays valid, or +// zero if the expiry annotation is missing, unparseable, or already past. +func scopedTokenValidFor(secret *corev1.Secret, now time.Time) time.Duration { + raw := secret.Annotations[scopedTokenExpiryAnnotation] + if raw == "" { + return 0 + } + exp, err := time.Parse(time.RFC3339, raw) + if err != nil || exp.Before(now) { + return 0 + } + return exp.Sub(now) } // dockerConfigJSON builds a minimal ~/.docker/config.json for a single registry. diff --git a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret_test.go b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret_test.go new file mode 100644 index 0000000000..2a5cc4674f --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret_test.go @@ -0,0 +1,123 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package copier + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/deckhouse/virtualization-controller/pkg/dvcr/registrytoken" +) + +func TestCopier(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Copier Suite") +} + +func newTestSigner() *registrytoken.Signer { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + Expect(err).ToNot(HaveOccurred()) + der, err := x509.MarshalPKCS8PrivateKey(key) + Expect(err).ToNot(HaveOccurred()) + signer, err := registrytoken.NewSignerFromPEM(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})) + Expect(err).ToNot(HaveOccurred()) + return signer +} + +var _ = Describe("Scoped token secret refresh", func() { + var ( + ctx context.Context + scheme *runtime.Scheme + signer *registrytoken.Signer + dest types.NamespacedName + access []registrytoken.Access + ) + + BeforeEach(func() { + ctx = context.Background() + scheme = runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + signer = newTestSigner() + dest = types.NamespacedName{Name: "dvcr-auth", Namespace: "default"} + access = []registrytoken.Access{{Type: "repository", Name: "cvi/img", Actions: []string{"pull", "push"}}} + }) + + authSecret := func() AuthSecret { + return AuthSecret{Secret: Secret{Destination: dest}} + } + + It("creates the secret with an expiry annotation", func() { + c := fake.NewClientBuilder().WithScheme(scheme).Build() + Expect(authSecret().CreateScopedTokenCDI(ctx, c, signer, access)).To(Succeed()) + + got := &corev1.Secret{} + Expect(c.Get(ctx, dest, got)).To(Succeed()) + Expect(got.Data).To(HaveKey("secretKey")) + Expect(got.Annotations).To(HaveKey(scopedTokenExpiryAnnotation)) + }) + + It("does not re-mint while the token is still fresh", func() { + c := fake.NewClientBuilder().WithScheme(scheme).Build() + Expect(authSecret().CreateScopedTokenCDI(ctx, c, signer, access)).To(Succeed()) + first := &corev1.Secret{} + Expect(c.Get(ctx, dest, first)).To(Succeed()) + + Expect(authSecret().CreateScopedTokenCDI(ctx, c, signer, access)).To(Succeed()) + second := &corev1.Secret{} + Expect(c.Get(ctx, dest, second)).To(Succeed()) + Expect(second.Data["secretKey"]).To(Equal(first.Data["secretKey"])) + }) + + It("re-mints when the token is within half its TTL of expiring", func() { + c := fake.NewClientBuilder().WithScheme(scheme).Build() + Expect(authSecret().CreateScopedTokenCDI(ctx, c, signer, access)).To(Succeed()) + stale := &corev1.Secret{} + Expect(c.Get(ctx, dest, stale)).To(Succeed()) + before := stale.Data["secretKey"] + + stale.Annotations[scopedTokenExpiryAnnotation] = time.Now().Add(time.Minute).Format(time.RFC3339) + Expect(c.Update(ctx, stale)).To(Succeed()) + + Expect(authSecret().CreateScopedTokenCDI(ctx, c, signer, access)).To(Succeed()) + after := &corev1.Secret{} + Expect(c.Get(ctx, dest, after)).To(Succeed()) + Expect(after.Data["secretKey"]).ToNot(Equal(before)) + }) + + It("scopedTokenValidFor returns zero for missing, unparseable or past expiry", func() { + now := time.Now() + Expect(scopedTokenValidFor(&corev1.Secret{}, now)).To(BeZero()) + bad := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{scopedTokenExpiryAnnotation: "garbage"}}} + Expect(scopedTokenValidFor(bad, now)).To(BeZero()) + past := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{scopedTokenExpiryAnnotation: now.Add(-time.Hour).Format(time.RFC3339)}}} + Expect(scopedTokenValidFor(past, now)).To(BeZero()) + }) +}) From c4c3d3101db0027f6919a764912b05ee425c813f Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 19:48:05 +0200 Subject: [PATCH 26/31] test(dvcr): cover JWT verifier against real distribution backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verifyJWT path (signature, issuer, audience, expiry, and the empty-Roots x5c/jwk defense) had no test — access.go sits behind the dvcr_registry build tag with distribution imports absent from the standalone module. Add access_test.go under the same tag plus the distribution/go-jose/golang-jwt test deps, minting tokens as the controller does and as an x5c attacker would, then verifying through distribution's own token backend. Asserts the x5c-signed token is rejected, guarding the load-bearing Roots line against regression. Signed-off-by: Daniil Antoshin --- images/dvcr/dvcr-k8s-auth/access_test.go | 223 +++++++++++++++++++++++ images/dvcr/dvcr-k8s-auth/go.mod | 23 ++- images/dvcr/dvcr-k8s-auth/go.sum | 22 +++ 3 files changed, 264 insertions(+), 4 deletions(-) create mode 100644 images/dvcr/dvcr-k8s-auth/access_test.go create mode 100644 images/dvcr/dvcr-k8s-auth/go.sum diff --git a/images/dvcr/dvcr-k8s-auth/access_test.go b/images/dvcr/dvcr-k8s-auth/access_test.go new file mode 100644 index 0000000000..6d74ec0f74 --- /dev/null +++ b/images/dvcr/dvcr-k8s-auth/access_test.go @@ -0,0 +1,223 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//go:build dvcr_registry + +// Integration tests for the JWT verification path. They mint tokens the way the +// virtualization-controller does (golang-jwt/v5) and an attacker would (go-jose +// x5c header), then verify them through the real distribution token backend the +// registry uses. This is the only place the load-bearing empty-Roots defense in +// verifyJWT is exercised end to end; it is excluded from the werf build (which +// copies *.go but not *_test.go). + +package dvcrk8s + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/json" + "math/big" + "testing" + "time" + + golangjwt "github.com/golang-jwt/jwt/v5" + + jose "github.com/go-jose/go-jose/v4" +) + +const ( + testIssuer = "virtualization-controller" + testAudience = "dvcr" + testKeyID = "dvcr" +) + +func newController(t *testing.T, pub crypto.PublicKey) *accessController { + t.Helper() + return &accessController{ + realm: "dvcr", + jwtIssuer: testIssuer, + jwtAudience: testAudience, + trustedKeys: map[string]crypto.PublicKey{testKeyID: pub}, + } +} + +func testClaims(now time.Time) map[string]interface{} { + return map[string]interface{}{ + "iss": testIssuer, + "aud": testAudience, + "iat": now.Unix(), + "nbf": now.Add(-30 * time.Second).Unix(), + "exp": now.Add(time.Hour).Unix(), + "access": []map[string]interface{}{ + {"type": "repository", "name": "cvi/img", "actions": []string{"pull", "push"}}, + }, + } +} + +// mintKidToken signs a token the way the controller does: golang-jwt/v5, ES256, +// kid header, no embedded key material. +func mintKidToken(t *testing.T, key *ecdsa.PrivateKey, claims map[string]interface{}) string { + t.Helper() + tok := golangjwt.NewWithClaims(golangjwt.SigningMethodES256, golangjwt.MapClaims(claims)) + tok.Header["kid"] = testKeyID + raw, err := tok.SignedString(key) + if err != nil { + t.Fatalf("sign kid token: %v", err) + } + return raw +} + +// mintX5cToken forges a token carrying its own self-signed certificate in the +// x5c header — the attack the empty-Roots pool must defeat. +func mintX5cToken(t *testing.T, claims map[string]interface{}) string { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "attacker"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + } + certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + opts := (&jose.SignerOptions{}).WithHeader("x5c", []string{base64.StdEncoding.EncodeToString(certDER)}) + signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.ES256, Key: key}, opts) + if err != nil { + t.Fatal(err) + } + payload, err := json.Marshal(claims) + if err != nil { + t.Fatal(err) + } + jws, err := signer.Sign(payload) + if err != nil { + t.Fatal(err) + } + raw, err := jws.CompactSerialize() + if err != nil { + t.Fatal(err) + } + return raw +} + +func newKey(t *testing.T) *ecdsa.PrivateKey { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + return key +} + +func TestVerifyJWT_ValidKidToken(t *testing.T) { + key := newKey(t) + ac := newController(t, &key.PublicKey) + + grants, err := ac.verifyJWT(mintKidToken(t, key, testClaims(time.Now()))) + if err != nil { + t.Fatalf("valid token rejected: %v", err) + } + if len(grants) != 1 || grants[0].Name != "cvi/img" || grants[0].Type != "repository" { + t.Fatalf("unexpected grants: %+v", grants) + } +} + +// The core security guarantee: a token whose signing key is smuggled in via x5c +// must not be trusted, even though its self-signed cert is internally valid. +func TestVerifyJWT_X5cTokenRejected(t *testing.T) { + key := newKey(t) + ac := newController(t, &key.PublicKey) + + if _, err := ac.verifyJWT(mintX5cToken(t, testClaims(time.Now()))); err == nil { + t.Fatal("x5c-signed token must be rejected") + } +} + +func TestVerifyJWT_Rejections(t *testing.T) { + key := newKey(t) + ac := newController(t, &key.PublicKey) + now := time.Now() + + t.Run("expired", func(t *testing.T) { + claims := testClaims(now) + claims["exp"] = now.Add(-time.Hour).Unix() + if _, err := ac.verifyJWT(mintKidToken(t, key, claims)); err == nil { + t.Fatal("expired token accepted") + } + }) + + t.Run("wrong issuer", func(t *testing.T) { + claims := testClaims(now) + claims["iss"] = "someone-else" + if _, err := ac.verifyJWT(mintKidToken(t, key, claims)); err == nil { + t.Fatal("token with wrong issuer accepted") + } + }) + + t.Run("wrong audience", func(t *testing.T) { + claims := testClaims(now) + claims["aud"] = "not-dvcr" + if _, err := ac.verifyJWT(mintKidToken(t, key, claims)); err == nil { + t.Fatal("token with wrong audience accepted") + } + }) + + t.Run("wrong signing key", func(t *testing.T) { + attacker := newKey(t) + if _, err := ac.verifyJWT(mintKidToken(t, attacker, testClaims(now))); err == nil { + t.Fatal("token signed by untrusted key accepted") + } + }) + + t.Run("tampered payload", func(t *testing.T) { + raw := mintKidToken(t, key, testClaims(now)) + tampered := raw[:len(raw)-3] + "AAA" + if _, err := ac.verifyJWT(tampered); err == nil { + t.Fatal("tampered token accepted") + } + }) +} + +func TestClassify_StaticCredentials(t *testing.T) { + key := newKey(t) + ac := newController(t, &key.PublicKey) + ac.adminUsername = "admin" + ac.adminPassword = []byte("admin-pass") + ac.pullerUsername = "node-puller" + ac.pullerPassword = []byte("puller-pass") + + if s, _, err := ac.classify("admin", "admin-pass"); err != nil || s.Role != RoleAdmin { + t.Fatalf("admin credential: role=%v err=%v", s.Role, err) + } + if s, _, err := ac.classify("node-puller", "puller-pass"); err != nil || s.Role != RolePuller { + t.Fatalf("puller credential: role=%v err=%v", s.Role, err) + } + // A scoped token presented under the admin username must not become admin. + scoped := mintKidToken(t, key, testClaims(time.Now())) + if s, _, err := ac.classify("admin", scoped); err != nil || s.Role != RoleScoped { + t.Fatalf("scoped token as admin username: role=%v err=%v", s.Role, err) + } +} diff --git a/images/dvcr/dvcr-k8s-auth/go.mod b/images/dvcr/dvcr-k8s-auth/go.mod index 5ddad70210..bcc7bbc760 100644 --- a/images/dvcr/dvcr-k8s-auth/go.mod +++ b/images/dvcr/dvcr-k8s-auth/go.mod @@ -1,6 +1,21 @@ -// Local module for standalone unit-testing of the DVCR authorization policy. -// The .go sources are copied into the distribution registry module at build -// time (see images/dvcr/werf.inc.yaml); this go.mod is not used in that build. +// Local module for standalone unit-testing of the DVCR authorization code. +// policy_test.go tests the dependency-free policy without build tags; the +// distribution/go-jose/golang-jwt requires below exist only so access_test.go +// (build tag dvcr_registry) can exercise the JWT verifier against the real +// distribution token backend. The .go sources are copied into the distribution +// registry module at build time (see images/dvcr/werf.inc.yaml); this go.mod and +// the *_test.go files are not used in that build. module github.com/deckhouse/virtualization/dvcr-k8s-auth -go 1.25 +go 1.25.0 + +require ( + github.com/distribution/distribution/v3 v3.1.1 + github.com/go-jose/go-jose/v4 v4.1.4 + github.com/golang-jwt/jwt/v5 v5.3.0 +) + +require ( + github.com/sirupsen/logrus v1.9.4 // indirect + golang.org/x/sys v0.42.0 // indirect +) diff --git a/images/dvcr/dvcr-k8s-auth/go.sum b/images/dvcr/dvcr-k8s-auth/go.sum new file mode 100644 index 0000000000..be48551c47 --- /dev/null +++ b/images/dvcr/dvcr-k8s-auth/go.sum @@ -0,0 +1,22 @@ +github.com/AdaLogics/go-fuzz-headers v0.0.0-20221103172237-443f56ff4ba8 h1:d+pBUmsteW5tM87xmVXHZ4+LibHRFn40SPAoZJOg2ak= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20221103172237-443f56ff4ba8/go.mod h1:i9fr2JpcEcY/IHEvzCM3qXUZYOQHgR89dt4es1CgMhc= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/distribution/v3 v3.1.1 h1:KUbk7C8CfaLXy8kbf/hGq9cad/wCoLB6dbWH6DMbmX0= +github.com/distribution/distribution/v3 v3.1.1/go.mod h1:d7lXwZpph0bVcOj4Aqn0nMrWHIwRQGdiV5TLeI+/w6Y= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 03b7e6c612a2f87d032a8c24446f629231cff27d Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 19:48:50 +0200 Subject: [PATCH 27/31] test(dvcr): cover token keypair validation on mismatch hook_test.go claimed keypair-regeneration coverage lived elsewhere, but validateECKeypair was never exercised. Add cases for a matching pair, a mismatched public key, and corrupt/missing keys. Signed-off-by: Daniil Antoshin --- .../generate-secret-for-dvcr/hook_test.go | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go index a2010390bc..ea39b009fd 100644 --- a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go +++ b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go @@ -38,6 +38,36 @@ func TestSetDVCRSecrets(t *testing.T) { RunSpecs(t, "DVCR Secrets Suite") } +var _ = Describe("validateECKeypair", func() { + It("accepts a matching pair", func() { + priv, pub, err := generateECKeypair() + Expect(err).ToNot(HaveOccurred()) + Expect(validateECKeypair(priv, pub)).To(BeTrue()) + }) + + It("rejects a public key from a different pair", func() { + priv, _, err := generateECKeypair() + Expect(err).ToNot(HaveOccurred()) + _, otherPub, err := generateECKeypair() + Expect(err).ToNot(HaveOccurred()) + Expect(validateECKeypair(priv, otherPub)).To(BeFalse()) + }) + + It("rejects a missing or corrupt public key", func() { + priv, _, err := generateECKeypair() + Expect(err).ToNot(HaveOccurred()) + Expect(validateECKeypair(priv, nil)).To(BeFalse()) + Expect(validateECKeypair(priv, []byte("not a pem"))).To(BeFalse()) + }) + + It("rejects a missing or corrupt private key", func() { + _, pub, err := generateECKeypair() + Expect(err).ToNot(HaveOccurred()) + Expect(validateECKeypair(nil, pub)).To(BeFalse()) + Expect(validateECKeypair([]byte("not a pem"), pub)).To(BeFalse()) + }) +}) + var _ = Describe("DVCR Secrets", func() { const ( defaultPasswordBase64 = "dkREU0I1N1JXeVVFd1NwN3VubDA3SHlnWUx3MzlOTlY=" // "vDDSB57RWyUEwSp7unl07HygYLw39NNV" From 2c8b7a662c3020d5a5bded354b87db2e96c94e53 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 19:51:46 +0200 Subject: [PATCH 28/31] refactor(dvcr): refresh scoped token in its last hour, not at half TTL Reconciles run far more often than an hour, so waiting until under an hour of validity remains keeps a valid token available while cutting re-mint churn roughly in half versus the TTL/2 threshold. Signed-off-by: Daniil Antoshin --- .../pkg/controller/supplements/copier/auth_secret.go | 7 ++++++- .../pkg/controller/supplements/copier/auth_secret_test.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go index 5733ff573d..30bee65042 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go +++ b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go @@ -42,6 +42,11 @@ const ScopedTokenUsername = "dvcr-jwt" // instead of writing a fresh token on every pass. const scopedTokenExpiryAnnotation = "dvcr.deckhouse.io/scoped-token-expires-at" +// scopedTokenRefreshThreshold is how much validity must remain before a reconcile +// re-mints the token. Reconciles run far more often than this, so refreshing in +// the last hour keeps a valid token available while minimizing re-mint churn. +const scopedTokenRefreshThreshold = time.Hour + // AuthSecret copies auth credentials from the source Secret into // Destination Secret and ensure its data is CDI compatible: // type: Opaque @@ -92,7 +97,7 @@ func (a AuthSecret) ensureScopedToken(ctx context.Context, client client.Client, err := client.Get(ctx, a.Destination, existing) switch { case err == nil: - if scopedTokenValidFor(existing, now) > registrytoken.DefaultTTL/2 { + if scopedTokenValidFor(existing, now) > scopedTokenRefreshThreshold { return nil } case !k8serrors.IsNotFound(err): diff --git a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret_test.go b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret_test.go index 2a5cc4674f..a32ffaae37 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret_test.go +++ b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret_test.go @@ -96,7 +96,7 @@ var _ = Describe("Scoped token secret refresh", func() { Expect(second.Data["secretKey"]).To(Equal(first.Data["secretKey"])) }) - It("re-mints when the token is within half its TTL of expiring", func() { + It("re-mints when the token is close to expiring", func() { c := fake.NewClientBuilder().WithScheme(scheme).Build() Expect(authSecret().CreateScopedTokenCDI(ctx, c, signer, access)).To(Succeed()) stale := &corev1.Secret{} From 5703ea657c4a14c048255cb387c669a9c6b7328f Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 3 Jul 2026 19:55:47 +0200 Subject: [PATCH 29/31] refactor(dvcr): tighten comments on scoped-token code Trim the verbose comments added with the token refresh and JWT verifier work to concise why-only notes; also drop a stale "half its TTL" reference. Signed-off-by: Daniil Antoshin --- images/dvcr/dvcr-k8s-auth/access.go | 8 +++----- images/dvcr/dvcr-k8s-auth/access_test.go | 9 +++------ images/dvcr/dvcr-k8s-auth/go.mod | 12 +++++------- .../pkg/hooks/generate-secret-for-dvcr/hook.go | 7 +++---- .../supplements/copier/auth_secret.go | 17 +++++++---------- 5 files changed, 21 insertions(+), 32 deletions(-) diff --git a/images/dvcr/dvcr-k8s-auth/access.go b/images/dvcr/dvcr-k8s-auth/access.go index 538b3b9518..bca02337e3 100644 --- a/images/dvcr/dvcr-k8s-auth/access.go +++ b/images/dvcr/dvcr-k8s-auth/access.go @@ -189,11 +189,9 @@ func (ac *accessController) verifyJWT(raw string) ([]Grant, error) { TrustedIssuers: []string{ac.jwtIssuer}, AcceptedAudiences: []string{ac.jwtAudience}, TrustedKeys: ac.trustedKeys, - // Roots must be a non-nil empty pool. distribution verifies a token's - // x5c/jwk certificate chain before the pinned kid->TrustedKeys lookup; - // with Roots unset (nil) that chain validates against the system CA pool, - // letting any publicly-trusted leaf cert forge a token. An empty pool - // fails every chain, so only our pinned key (matched by kid) is trusted. + // Non-nil empty pool: distribution tries a token's x5c/jwk chain before the + // pinned kid, and with nil Roots that chain validates against the system CAs. + // An empty pool rejects every chain, leaving only the pinned key. Roots: x509.NewCertPool(), }) if err != nil { diff --git a/images/dvcr/dvcr-k8s-auth/access_test.go b/images/dvcr/dvcr-k8s-auth/access_test.go index 6d74ec0f74..7150ed464f 100644 --- a/images/dvcr/dvcr-k8s-auth/access_test.go +++ b/images/dvcr/dvcr-k8s-auth/access_test.go @@ -16,12 +16,9 @@ limitations under the License. //go:build dvcr_registry -// Integration tests for the JWT verification path. They mint tokens the way the -// virtualization-controller does (golang-jwt/v5) and an attacker would (go-jose -// x5c header), then verify them through the real distribution token backend the -// registry uses. This is the only place the load-bearing empty-Roots defense in -// verifyJWT is exercised end to end; it is excluded from the werf build (which -// copies *.go but not *_test.go). +// Integration tests for verifyJWT: mint tokens as the controller (golang-jwt) and +// as an x5c attacker (go-jose), verify through the real distribution backend. The +// only end-to-end check of the empty-Roots x5c defense. package dvcrk8s diff --git a/images/dvcr/dvcr-k8s-auth/go.mod b/images/dvcr/dvcr-k8s-auth/go.mod index bcc7bbc760..0101d3869e 100644 --- a/images/dvcr/dvcr-k8s-auth/go.mod +++ b/images/dvcr/dvcr-k8s-auth/go.mod @@ -1,10 +1,8 @@ -// Local module for standalone unit-testing of the DVCR authorization code. -// policy_test.go tests the dependency-free policy without build tags; the -// distribution/go-jose/golang-jwt requires below exist only so access_test.go -// (build tag dvcr_registry) can exercise the JWT verifier against the real -// distribution token backend. The .go sources are copied into the distribution -// registry module at build time (see images/dvcr/werf.inc.yaml); this go.mod and -// the *_test.go files are not used in that build. +// Local module for unit-testing the DVCR authorization code. The requires below +// exist only for access_test.go (build tag dvcr_registry) to verify tokens +// against the real distribution backend. The .go sources are copied into the +// distribution module at build time (see images/dvcr/werf.inc.yaml); this go.mod +// and the *_test.go files are not used in that build. module github.com/deckhouse/virtualization/dvcr-k8s-auth go 1.25.0 diff --git a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go index 7b5d1afdff..fa46d671dd 100644 --- a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go +++ b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go @@ -231,10 +231,9 @@ func generateECKeypair() (privPEM, pubPEM []byte, err error) { return privPEM, pubPEM, nil } -// validateECKeypair reports whether privPEM is a parseable ECDSA private key and -// pubPEM is its matching public key. The public key is shipped to the registry as -// /auth/tokenPublicKey; a missing, corrupt or mismatched one would make it reject -// every scoped token, so any of those must force regeneration of the whole pair. +// validateECKeypair reports whether privPEM parses as ECDSA and pubPEM is its +// matching public key. A bad public key (shipped to the registry) would reject +// every scoped token, so it must force regenerating the pair too. func validateECKeypair(privPEM, pubPEM []byte) bool { priv := parseECPrivateKey(privPEM) if priv == nil { diff --git a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go index 30bee65042..775b235ecf 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go +++ b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go @@ -37,14 +37,12 @@ import ( // is a fixed, human-readable marker. const ScopedTokenUsername = "dvcr-jwt" -// scopedTokenExpiryAnnotation records the scoped token's expiry (RFC3339) so a -// reconcile can re-mint the Secret only when the token is close to expiring, -// instead of writing a fresh token on every pass. +// scopedTokenExpiryAnnotation records the token expiry (RFC3339) so a reconcile +// re-mints only when it is close to expiring, not on every pass. const scopedTokenExpiryAnnotation = "dvcr.deckhouse.io/scoped-token-expires-at" -// scopedTokenRefreshThreshold is how much validity must remain before a reconcile -// re-mints the token. Reconciles run far more often than this, so refreshing in -// the last hour keeps a valid token available while minimizing re-mint churn. +// scopedTokenRefreshThreshold: re-mint once validity drops below this. Reconciles +// run far more often than an hour, so a valid token stays available cheaply. const scopedTokenRefreshThreshold = time.Hour // AuthSecret copies auth credentials from the source Secret into @@ -86,10 +84,9 @@ func (a AuthSecret) CreateScopedTokenDockerConfig(ctx context.Context, client cl }) } -// ensureScopedToken mints a scoped token and writes the destination Secret only -// when it is missing or within half its TTL of expiring. Reusing a still-fresh -// Secret keeps re-mint churn down; refreshing before expiry means an import that -// outlives one token gets a valid one on a later reconcile instead of a stale 403. +// ensureScopedToken writes the destination Secret with a freshly minted token +// only when it is missing or near expiry, so an import that outlives one token +// gets a valid one on a later reconcile instead of a stale 403. func (a AuthSecret) ensureScopedToken(ctx context.Context, client client.Client, signer *registrytoken.Signer, access []registrytoken.Access, build func(raw string) (map[string][]byte, corev1.SecretType, error)) error { now := time.Now() From 7f0ed50d2366dca783fe8250fdf294d296bec6a1 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Mon, 6 Jul 2026 11:14:40 +0000 Subject: [PATCH 30/31] refactor(core, dvcr): always enable per-namespace registry authorization Remove the tenantRegistryAuthorization module config toggle: per-namespace DVCR authorization is now always on. Import and upload Pods authenticate with a scoped token instead of the shared read-write credential, and the dead shared-htpasswd fallback paths are dropped. Rename the pull-only credential passwordPuller to passwordRO for consistency with passwordRW, and convert the signer and access-controller tests to ginkgo/gomega. Signed-off-by: Daniil Antoshin --- images/dvcr/dvcr-k8s-auth/access_test.go | 212 ++++++++---------- images/dvcr/dvcr-k8s-auth/go.mod | 10 + images/dvcr/dvcr-k8s-auth/go.sum | 19 ++ .../hooks/generate-secret-for-dvcr/hook.go | 28 +-- .../generate-secret-for-dvcr/hook_test.go | 12 +- .../pkg/config/load_dvcr_settings.go | 34 +-- .../pkg/controller/importer/settings.go | 6 +- .../controller/service/dvcr_token_scope.go | 9 +- .../supplements/copier/auth_secret.go | 33 --- .../pkg/controller/supplements/ensure.go | 78 ++----- .../pkg/controller/uploader/settings.go | 6 +- .../virtualization-artifact/pkg/dvcr/dvcr.go | 10 +- .../pkg/dvcr/registrytoken/signer_test.go | 176 ++++++++++----- openapi/config-values.yaml | 12 - openapi/doc-ru-config-values.yaml | 11 - openapi/values.yaml | 2 +- templates/dvcr/_helpers.tpl | 19 +- templates/dvcr/configmap.yaml | 4 +- templates/dvcr/nodegroupconfiguration.yaml | 8 +- templates/dvcr/secret.yaml | 4 +- .../virtualization-controller/_helpers.tpl | 4 - 21 files changed, 299 insertions(+), 398 deletions(-) diff --git a/images/dvcr/dvcr-k8s-auth/access_test.go b/images/dvcr/dvcr-k8s-auth/access_test.go index 7150ed464f..575de37eef 100644 --- a/images/dvcr/dvcr-k8s-auth/access_test.go +++ b/images/dvcr/dvcr-k8s-auth/access_test.go @@ -38,16 +38,22 @@ import ( golangjwt "github.com/golang-jwt/jwt/v5" jose "github.com/go-jose/go-jose/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) +func TestAccessController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DVCR Auth Suite") +} + const ( testIssuer = "virtualization-controller" testAudience = "dvcr" testKeyID = "dvcr" ) -func newController(t *testing.T, pub crypto.PublicKey) *accessController { - t.Helper() +func newController(pub crypto.PublicKey) *accessController { return &accessController{ realm: "dvcr", jwtIssuer: testIssuer, @@ -56,14 +62,14 @@ func newController(t *testing.T, pub crypto.PublicKey) *accessController { } } -func testClaims(now time.Time) map[string]interface{} { - return map[string]interface{}{ +func testClaims(now time.Time) map[string]any { + return map[string]any{ "iss": testIssuer, "aud": testAudience, "iat": now.Unix(), "nbf": now.Add(-30 * time.Second).Unix(), "exp": now.Add(time.Hour).Unix(), - "access": []map[string]interface{}{ + "access": []map[string]any{ {"type": "repository", "name": "cvi/img", "actions": []string{"pull", "push"}}, }, } @@ -71,25 +77,21 @@ func testClaims(now time.Time) map[string]interface{} { // mintKidToken signs a token the way the controller does: golang-jwt/v5, ES256, // kid header, no embedded key material. -func mintKidToken(t *testing.T, key *ecdsa.PrivateKey, claims map[string]interface{}) string { - t.Helper() +func mintKidToken(key *ecdsa.PrivateKey, claims map[string]any) string { + GinkgoHelper() tok := golangjwt.NewWithClaims(golangjwt.SigningMethodES256, golangjwt.MapClaims(claims)) tok.Header["kid"] = testKeyID raw, err := tok.SignedString(key) - if err != nil { - t.Fatalf("sign kid token: %v", err) - } + Expect(err).NotTo(HaveOccurred()) return raw } // mintX5cToken forges a token carrying its own self-signed certificate in the // x5c header — the attack the empty-Roots pool must defeat. -func mintX5cToken(t *testing.T, claims map[string]interface{}) string { - t.Helper() +func mintX5cToken(claims map[string]any) string { + GinkgoHelper() key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - t.Fatal(err) - } + Expect(err).NotTo(HaveOccurred()) tmpl := &x509.Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "attacker"}, @@ -97,124 +99,102 @@ func mintX5cToken(t *testing.T, claims map[string]interface{}) string { NotAfter: time.Now().Add(time.Hour), } certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) - if err != nil { - t.Fatal(err) - } + Expect(err).NotTo(HaveOccurred()) opts := (&jose.SignerOptions{}).WithHeader("x5c", []string{base64.StdEncoding.EncodeToString(certDER)}) signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.ES256, Key: key}, opts) - if err != nil { - t.Fatal(err) - } + Expect(err).NotTo(HaveOccurred()) payload, err := json.Marshal(claims) - if err != nil { - t.Fatal(err) - } + Expect(err).NotTo(HaveOccurred()) jws, err := signer.Sign(payload) - if err != nil { - t.Fatal(err) - } + Expect(err).NotTo(HaveOccurred()) raw, err := jws.CompactSerialize() - if err != nil { - t.Fatal(err) - } + Expect(err).NotTo(HaveOccurred()) return raw } -func newKey(t *testing.T) *ecdsa.PrivateKey { - t.Helper() +func newKey() *ecdsa.PrivateKey { + GinkgoHelper() key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - t.Fatal(err) - } + Expect(err).NotTo(HaveOccurred()) return key } -func TestVerifyJWT_ValidKidToken(t *testing.T) { - key := newKey(t) - ac := newController(t, &key.PublicKey) - - grants, err := ac.verifyJWT(mintKidToken(t, key, testClaims(time.Now()))) - if err != nil { - t.Fatalf("valid token rejected: %v", err) - } - if len(grants) != 1 || grants[0].Name != "cvi/img" || grants[0].Type != "repository" { - t.Fatalf("unexpected grants: %+v", grants) - } -} - -// The core security guarantee: a token whose signing key is smuggled in via x5c -// must not be trusted, even though its self-signed cert is internally valid. -func TestVerifyJWT_X5cTokenRejected(t *testing.T) { - key := newKey(t) - ac := newController(t, &key.PublicKey) - - if _, err := ac.verifyJWT(mintX5cToken(t, testClaims(time.Now()))); err == nil { - t.Fatal("x5c-signed token must be rejected") - } -} +var _ = Describe("verifyJWT", func() { + It("accepts a valid kid-signed token and returns its access grants", func() { + key := newKey() + ac := newController(&key.PublicKey) -func TestVerifyJWT_Rejections(t *testing.T) { - key := newKey(t) - ac := newController(t, &key.PublicKey) - now := time.Now() - - t.Run("expired", func(t *testing.T) { - claims := testClaims(now) - claims["exp"] = now.Add(-time.Hour).Unix() - if _, err := ac.verifyJWT(mintKidToken(t, key, claims)); err == nil { - t.Fatal("expired token accepted") - } + grants, err := ac.verifyJWT(mintKidToken(key, testClaims(time.Now()))) + Expect(err).NotTo(HaveOccurred()) + Expect(grants).To(HaveLen(1)) + Expect(grants[0].Name).To(Equal("cvi/img")) + Expect(grants[0].Type).To(Equal("repository")) }) - t.Run("wrong issuer", func(t *testing.T) { - claims := testClaims(now) - claims["iss"] = "someone-else" - if _, err := ac.verifyJWT(mintKidToken(t, key, claims)); err == nil { - t.Fatal("token with wrong issuer accepted") - } - }) + // The core security guarantee: a token whose signing key is smuggled in via + // x5c must not be trusted, even though its self-signed cert is internally valid. + It("rejects an x5c-smuggled token", func() { + key := newKey() + ac := newController(&key.PublicKey) - t.Run("wrong audience", func(t *testing.T) { - claims := testClaims(now) - claims["aud"] = "not-dvcr" - if _, err := ac.verifyJWT(mintKidToken(t, key, claims)); err == nil { - t.Fatal("token with wrong audience accepted") - } + _, err := ac.verifyJWT(mintX5cToken(testClaims(time.Now()))) + Expect(err).To(HaveOccurred()) }) - t.Run("wrong signing key", func(t *testing.T) { - attacker := newKey(t) - if _, err := ac.verifyJWT(mintKidToken(t, attacker, testClaims(now))); err == nil { - t.Fatal("token signed by untrusted key accepted") - } - }) + DescribeTable("rejects an invalid token", + func(makeToken func(key *ecdsa.PrivateKey) string) { + key := newKey() + ac := newController(&key.PublicKey) - t.Run("tampered payload", func(t *testing.T) { - raw := mintKidToken(t, key, testClaims(now)) - tampered := raw[:len(raw)-3] + "AAA" - if _, err := ac.verifyJWT(tampered); err == nil { - t.Fatal("tampered token accepted") - } + _, err := ac.verifyJWT(makeToken(key)) + Expect(err).To(HaveOccurred()) + }, + Entry("expired", func(key *ecdsa.PrivateKey) string { + claims := testClaims(time.Now()) + claims["exp"] = time.Now().Add(-time.Hour).Unix() + return mintKidToken(key, claims) + }), + Entry("wrong issuer", func(key *ecdsa.PrivateKey) string { + claims := testClaims(time.Now()) + claims["iss"] = "someone-else" + return mintKidToken(key, claims) + }), + Entry("wrong audience", func(key *ecdsa.PrivateKey) string { + claims := testClaims(time.Now()) + claims["aud"] = "not-dvcr" + return mintKidToken(key, claims) + }), + Entry("untrusted signing key", func(*ecdsa.PrivateKey) string { + return mintKidToken(newKey(), testClaims(time.Now())) + }), + Entry("tampered payload", func(key *ecdsa.PrivateKey) string { + raw := mintKidToken(key, testClaims(time.Now())) + return raw[:len(raw)-3] + "AAA" + }), + ) +}) + +var _ = Describe("classify", func() { + It("maps static credentials and scoped tokens to roles", func() { + key := newKey() + ac := newController(&key.PublicKey) + ac.adminUsername = "admin" + ac.adminPassword = []byte("admin-pass") + ac.pullerUsername = "node-puller" + ac.pullerPassword = []byte("puller-pass") + + s, _, err := ac.classify("admin", "admin-pass") + Expect(err).NotTo(HaveOccurred()) + Expect(s.Role).To(Equal(RoleAdmin)) + + s, _, err = ac.classify("node-puller", "puller-pass") + Expect(err).NotTo(HaveOccurred()) + Expect(s.Role).To(Equal(RolePuller)) + + // A scoped token presented under the admin username must not become admin. + scoped := mintKidToken(key, testClaims(time.Now())) + s, _, err = ac.classify("admin", scoped) + Expect(err).NotTo(HaveOccurred()) + Expect(s.Role).To(Equal(RoleScoped)) }) -} - -func TestClassify_StaticCredentials(t *testing.T) { - key := newKey(t) - ac := newController(t, &key.PublicKey) - ac.adminUsername = "admin" - ac.adminPassword = []byte("admin-pass") - ac.pullerUsername = "node-puller" - ac.pullerPassword = []byte("puller-pass") - - if s, _, err := ac.classify("admin", "admin-pass"); err != nil || s.Role != RoleAdmin { - t.Fatalf("admin credential: role=%v err=%v", s.Role, err) - } - if s, _, err := ac.classify("node-puller", "puller-pass"); err != nil || s.Role != RolePuller { - t.Fatalf("puller credential: role=%v err=%v", s.Role, err) - } - // A scoped token presented under the admin username must not become admin. - scoped := mintKidToken(t, key, testClaims(time.Now())) - if s, _, err := ac.classify("admin", scoped); err != nil || s.Role != RoleScoped { - t.Fatalf("scoped token as admin username: role=%v err=%v", s.Role, err) - } -} +}) diff --git a/images/dvcr/dvcr-k8s-auth/go.mod b/images/dvcr/dvcr-k8s-auth/go.mod index 0101d3869e..33e89c08c2 100644 --- a/images/dvcr/dvcr-k8s-auth/go.mod +++ b/images/dvcr/dvcr-k8s-auth/go.mod @@ -11,9 +11,19 @@ require ( github.com/distribution/distribution/v3 v3.1.1 github.com/go-jose/go-jose/v4 v4.1.4 github.com/golang-jwt/jwt/v5 v5.3.0 + github.com/onsi/ginkgo/v2 v2.23.3 + github.com/onsi/gomega v1.37.0 ) require ( + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect github.com/sirupsen/logrus v1.9.4 // indirect + golang.org/x/net v0.52.0 // indirect golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect + golang.org/x/tools v0.42.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/images/dvcr/dvcr-k8s-auth/go.sum b/images/dvcr/dvcr-k8s-auth/go.sum index be48551c47..6ef30bb181 100644 --- a/images/dvcr/dvcr-k8s-auth/go.sum +++ b/images/dvcr/dvcr-k8s-auth/go.sum @@ -8,15 +8,34 @@ github.com/distribution/distribution/v3 v3.1.1 h1:KUbk7C8CfaLXy8kbf/hGq9cad/wCoL github.com/distribution/distribution/v3 v3.1.1/go.mod h1:d7lXwZpph0bVcOj4Aqn0nMrWHIwRQGdiV5TLeI+/w6Y= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/onsi/ginkgo/v2 v2.23.3 h1:edHxnszytJ4lD9D5Jjc4tiDkPBZ3siDeJJkUZJJVkp0= +github.com/onsi/ginkgo/v2 v2.23.3/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGmaSRvxnM= +github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= +github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go index fa46d671dd..536a254421 100644 --- a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go +++ b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook.go @@ -39,7 +39,7 @@ import ( const ( dvcrSecrets = "dvcr-secrets" passwordRWValuePath = "virtualization.internal.dvcr.passwordRW" - passwordPullerValuePath = "virtualization.internal.dvcr.passwordPuller" + passwordROValuePath = "virtualization.internal.dvcr.passwordRO" saltValuePath = "virtualization.internal.dvcr.salt" htpasswdValuePath = "virtualization.internal.dvcr.htpasswd" tokenPrivateKeyValuePath = "virtualization.internal.dvcr.tokenPrivateKey" @@ -49,7 +49,7 @@ const ( type dvcrSecretData struct { PasswordRW string `json:"passwordRW"` - PasswordPuller string `json:"passwordPuller"` + PasswordRO string `json:"passwordRO"` Salt string `json:"salt"` Htpasswd string `json:"htpasswd"` TokenPrivateKey string `json:"tokenPrivateKey"` @@ -111,20 +111,20 @@ func handlerDVCRSecrets(ctx context.Context, input *pkg.HookInput) error { input.Values.Set(passwordRWValuePath, passwordRW) } - // passwordPuller is the pull-only node-puller credential used by the dvcr-k8s + // passwordRO is the pull-only node-puller credential used by the dvcr-k8s // authorization backend. It carries no push rights, so it does not need an // htpasswd entry (nodes fall back to admin only when the backend is htpasswd). - passwordPuller := dataFromSecret.PasswordPuller - passwordPullerBytes, err := base64.StdEncoding.DecodeString(passwordPuller) - passwordPullerDecoded := string(passwordPullerBytes) - if err != nil || passwordPullerDecoded == "" { - input.Logger.Info("Regenerate PasswordPuller") - passwordPullerDecoded = alphaNum(32) - passwordPuller = base64.StdEncoding.EncodeToString([]byte(passwordPullerDecoded)) + passwordRO := dataFromSecret.PasswordRO + passwordROBytes, err := base64.StdEncoding.DecodeString(passwordRO) + passwordRODecoded := string(passwordROBytes) + if err != nil || passwordRODecoded == "" { + input.Logger.Info("Regenerate PasswordRO") + passwordRODecoded = alphaNum(32) + passwordRO = base64.StdEncoding.EncodeToString([]byte(passwordRODecoded)) } - if passwordPuller != dataFromValues.PasswordPuller { - input.Logger.Info("Set PasswordPuller to values") - input.Values.Set(passwordPullerValuePath, passwordPuller) + if passwordRO != dataFromValues.PasswordRO { + input.Logger.Info("Set PasswordRO to values") + input.Values.Set(passwordROValuePath, passwordRO) } salt := dataFromSecret.Salt @@ -202,7 +202,7 @@ func getDVCRSecretsFromSecrets(input *pkg.HookInput) (dvcrSecretData, error) { func getDVCRSecretsFromValues(input *pkg.HookInput) dvcrSecretData { return dvcrSecretData{ PasswordRW: input.Values.Get(passwordRWValuePath).String(), - PasswordPuller: input.Values.Get(passwordPullerValuePath).String(), + PasswordRO: input.Values.Get(passwordROValuePath).String(), Salt: input.Values.Get(saltValuePath).String(), Htpasswd: input.Values.Get(htpasswdValuePath).String(), TokenPrivateKey: input.Values.Get(tokenPrivateKeyValuePath).String(), diff --git a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go index ea39b009fd..61cc63294c 100644 --- a/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go +++ b/images/hooks/pkg/hooks/generate-secret-for-dvcr/hook_test.go @@ -86,9 +86,9 @@ var _ = Describe("DVCR Secrets", func() { // prepareValuesGet mocks the values-side reads. The token keypair is always a // valid pair matching the snapshot, so these tests never exercise keypair // regeneration (its own coverage lives in signer_test.go and the hook helpers). - prepareValuesGet := func(passwordRW, passwordPuller, salt, htpasswd string) { + prepareValuesGet := func(passwordRW, passwordRO, salt, htpasswd string) { values.GetMock.When(passwordRWValuePath).Then(gjson.Result{Type: gjson.String, Str: passwordRW}) - values.GetMock.When(passwordPullerValuePath).Then(gjson.Result{Type: gjson.String, Str: passwordPuller}) + values.GetMock.When(passwordROValuePath).Then(gjson.Result{Type: gjson.String, Str: passwordRO}) values.GetMock.When(saltValuePath).Then(gjson.Result{Type: gjson.String, Str: salt}) values.GetMock.When(htpasswdValuePath).Then(gjson.Result{Type: gjson.String, Str: htpasswd}) values.GetMock.When(tokenPrivateKeyValuePath).Then(gjson.Result{Type: gjson.String, Str: validTokenKey}) @@ -99,13 +99,13 @@ var _ = Describe("DVCR Secrets", func() { snapshots.GetMock.When(dvcrSecrets).Then(snaps) } - newSnapshot := func(passwordRW, passwordPuller, salt, htpasswd string) pkg.Snapshot { + newSnapshot := func(passwordRW, passwordRO, salt, htpasswd string) pkg.Snapshot { return mock.NewSnapshotMock(GinkgoT()).UnmarshalToMock.Set(func(v any) (err error) { data, ok := v.(*dvcrSecretData) Expect(ok).To(BeTrue()) data.PasswordRW = passwordRW - data.PasswordPuller = passwordPuller + data.PasswordRO = passwordRO data.Salt = salt data.Htpasswd = htpasswd data.TokenPrivateKey = validTokenKey @@ -157,7 +157,7 @@ var _ = Describe("DVCR Secrets", func() { switch path { case passwordRWValuePath: Expect(value).To(Equal(defaultPasswordBase64)) - case passwordPullerValuePath: + case passwordROValuePath: Expect(value).To(Equal(defaultPullerBase64)) case saltValuePath: Expect(value).To(Equal(defaultSaltBase64)) @@ -193,7 +193,7 @@ var _ = Describe("DVCR Secrets", func() { case passwordRWValuePath: passwordRW = value Expect(value).To(HaveLen(32)) - case passwordPullerValuePath: + case passwordROValuePath: Expect(value).To(HaveLen(32)) case saltValuePath: Expect(value).To(HaveLen(32)) diff --git a/images/virtualization-artifact/pkg/config/load_dvcr_settings.go b/images/virtualization-artifact/pkg/config/load_dvcr_settings.go index 4f863dbc62..2d5fc2bc74 100644 --- a/images/virtualization-artifact/pkg/config/load_dvcr_settings.go +++ b/images/virtualization-artifact/pkg/config/load_dvcr_settings.go @@ -19,7 +19,6 @@ package config import ( "fmt" "os" - "strconv" "github.com/deckhouse/virtualization-controller/pkg/dvcr" "github.com/deckhouse/virtualization-controller/pkg/dvcr/registrytoken" @@ -42,10 +41,8 @@ const ( DVCRImageMonitorScheduleVar = "DVCR_IMAGE_MONITOR_SCHEDULE" // DVCRGCScheduleVar is an env variable holds the cron schedule to run DVCR garbage collection. DVCRGCScheduleVar = "DVCR_GC_SCHEDULE" - // DVCRTenantAuthzEnabledVar is an env variable that enables per-namespace DVCR authorization. - DVCRTenantAuthzEnabledVar = "DVCR_TENANT_AUTHZ_ENABLED" // DVCRTokenPrivateKeyVar holds the PEM ECDSA private key used to mint scoped - // DVCR tokens. Required when DVCR_TENANT_AUTHZ_ENABLED is true. + // per-namespace DVCR tokens. DVCRTokenPrivateKeyVar = "DVCR_TOKEN_PRIVATE_KEY" // UploaderIngressHostVar is a env variable @@ -68,7 +65,6 @@ func LoadDVCRSettingsFromEnvs(controllerNamespace string) (*dvcr.Settings, error InsecureTLS: os.Getenv(DVCRInsecureTLSVar), ImageMonitorSchedule: os.Getenv(DVCRImageMonitorScheduleVar), GCSchedule: os.Getenv(DVCRGCScheduleVar), - TenantAuthzEnabled: parseBoolEnv(DVCRTenantAuthzEnabledVar), UploaderIngressSettings: dvcr.UploaderIngressSettings{ Host: os.Getenv(UploaderIngressHostVar), TLSSecret: os.Getenv(UploaderIngressTLSSecretVar), @@ -97,27 +93,15 @@ func LoadDVCRSettingsFromEnvs(controllerNamespace string) (*dvcr.Settings, error dvcrSettings.GCSchedule = dvcr.DefaultGCSchedule } - if dvcrSettings.TenantAuthzEnabled { - keyPEM := os.Getenv(DVCRTokenPrivateKeyVar) - if keyPEM == "" { - return nil, fmt.Errorf("environment variable %q undefined, required when %s is true", DVCRTokenPrivateKeyVar, DVCRTenantAuthzEnabledVar) - } - signer, err := registrytoken.NewSignerFromPEM([]byte(keyPEM)) - if err != nil { - return nil, fmt.Errorf("init DVCR token signer: %w", err) - } - dvcrSettings.TokenSigner = signer + keyPEM := os.Getenv(DVCRTokenPrivateKeyVar) + if keyPEM == "" { + return nil, fmt.Errorf("environment variable %q undefined, specify DVCR settings", DVCRTokenPrivateKeyVar) } - - return dvcrSettings, nil -} - -// parseBoolEnv returns the boolean value of the env variable, or false if it is -// unset or not parseable as a bool. -func parseBoolEnv(name string) bool { - v, err := strconv.ParseBool(os.Getenv(name)) + signer, err := registrytoken.NewSignerFromPEM([]byte(keyPEM)) if err != nil { - return false + return nil, fmt.Errorf("init DVCR token signer: %w", err) } - return v + dvcrSettings.TokenSigner = signer + + return dvcrSettings, nil } diff --git a/images/virtualization-artifact/pkg/controller/importer/settings.go b/images/virtualization-artifact/pkg/controller/importer/settings.go index c1f5b4d1c7..98605ea1c7 100644 --- a/images/virtualization-artifact/pkg/controller/importer/settings.go +++ b/images/virtualization-artifact/pkg/controller/importer/settings.go @@ -69,11 +69,7 @@ type Settings struct { } func ApplyDVCRDestinationSettings(podEnvVars *Settings, dvcrSettings *dvcr.Settings, supGen supplements.Generator, dvcrImageName string) { - authSecret := dvcrSettings.AuthSecret - if dvcrSettings.TenantAuthzEnabled || supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { - authSecret = supGen.DVCRAuthSecret().Name - } - podEnvVars.DestinationAuthSecret = authSecret + podEnvVars.DestinationAuthSecret = supGen.DVCRAuthSecret().Name podEnvVars.DestinationInsecureTLS = dvcrSettings.InsecureTLS podEnvVars.DestinationEndpoint = dvcrImageName } diff --git a/images/virtualization-artifact/pkg/controller/service/dvcr_token_scope.go b/images/virtualization-artifact/pkg/controller/service/dvcr_token_scope.go index 0b16cc598a..c3ec5b0d4e 100644 --- a/images/virtualization-artifact/pkg/controller/service/dvcr_token_scope.go +++ b/images/virtualization-artifact/pkg/controller/service/dvcr_token_scope.go @@ -28,11 +28,7 @@ import ( // importerTokenScope is the DVCR access an importer Pod needs: push+pull on its // destination repository, plus pull on the source repository when the source is // itself a DVCR image (dvcr-artifact copies both through the same credential). -// Returns nil when per-namespace authorization is off (no token is minted). func importerTokenScope(s *dvcr.Settings, settings *importer.Settings) []registrytoken.Access { - if !s.TenantAuthzEnabled { - return nil - } access := []registrytoken.Access{repoAccess(s.RepoPath(settings.DestinationEndpoint), "pull", "push")} if settings.Source == importer.SourceDVCR && settings.Endpoint != "" { access = append(access, repoAccess(s.RepoPath(settings.Endpoint), "pull")) @@ -43,16 +39,13 @@ func importerTokenScope(s *dvcr.Settings, settings *importer.Settings) []registr // uploaderTokenScope is the DVCR access an uploader Pod needs: push+pull on its // destination repository (an uploader has no DVCR source). func uploaderTokenScope(s *dvcr.Settings, settings *uploader.Settings) []registrytoken.Access { - if !s.TenantAuthzEnabled { - return nil - } return []registrytoken.Access{repoAccess(s.RepoPath(settings.DestinationEndpoint), "pull", "push")} } // dataVolumeTokenScope is the DVCR access a CDI DataVolume needs: pull-only on the // source repository it imports from (it writes to a PVC, not to DVCR). func dataVolumeTokenScope(s *dvcr.Settings, source *cdiv1.DataVolumeSource) []registrytoken.Access { - if !s.TenantAuthzEnabled || source == nil || source.Registry == nil || source.Registry.URL == nil { + if source == nil || source.Registry == nil || source.Registry.URL == nil { return nil } return []registrytoken.Access{repoAccess(s.RepoPath(*source.Registry.URL), "pull")} diff --git a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go index 775b235ecf..ccbc873d06 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go +++ b/images/virtualization-artifact/pkg/controller/supplements/copier/auth_secret.go @@ -27,8 +27,6 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/deckhouse/virtualization-controller/pkg/auth" - "github.com/deckhouse/virtualization-controller/pkg/common/object" "github.com/deckhouse/virtualization-controller/pkg/dvcr/registrytoken" ) @@ -152,34 +150,3 @@ func dockerConfigJSON(registryURL, username, password string) ([]byte, error) { "auths": map[string]any{registryURL: entry}, }) } - -// CopyCDICompatible transforms auth credentials in dockerconfigjson format into CDI compatible: -// a Secret with two fields: accessKeyId and secretKey. -// ref is registry url or image name. It is used to select a desired auth pair from the config. -func (a AuthSecret) CopyCDICompatible(ctx context.Context, client client.Client, ref string) error { - srcObj, err := object.FetchObject(ctx, a.Source, client, &corev1.Secret{}) - if err != nil { - return err - } - - destData := srcObj.Data - destType := srcObj.Type - if srcObj.Type == corev1.SecretTypeDockerConfigJson { - cfg, err := auth.ReadDockerConfigJSON(srcObj.Data[corev1.DockerConfigJsonKey]) - if err != nil { - return err - } - username, password, err := auth.CredsFromRegistryAuthFile(cfg, ref) - if err != nil { - return err - } - destData = map[string][]byte{ - "accessKeyId": []byte(username), - "secretKey": []byte(password), - } - destType = corev1.SecretTypeOpaque - } - - _, err = a.Create(ctx, client, destData, destType) - return err -} diff --git a/images/virtualization-artifact/pkg/controller/supplements/ensure.go b/images/virtualization-artifact/pkg/controller/supplements/ensure.go index b4d572528b..66c11833c4 100644 --- a/images/virtualization-artifact/pkg/controller/supplements/ensure.go +++ b/images/virtualization-artifact/pkg/controller/supplements/ensure.go @@ -60,34 +60,15 @@ func EnsureForPod(ctx context.Context, client client.Client, supGen Generator, p } } - // Create Secret with auth config to use DVCR as destination. - switch { - case dvcrSettings.TenantAuthzEnabled: - authCopier := copier.AuthSecret{ - Secret: copier.Secret{ - Destination: supGen.DVCRAuthSecret(), - OwnerReference: podutil.MakeOwnerReference(pod), - }, - } - if err := authCopier.CreateScopedTokenDockerConfig(ctx, client, dvcrSettings.TokenSigner, scope, dvcrSettings.RegistryURL); err != nil { - return err - } - case ShouldCopyDVCRAuthSecret(dvcrSettings, supGen): - authSecret := supGen.DVCRAuthSecret() - authCopier := copier.AuthSecret{ - Secret: copier.Secret{ - Source: types.NamespacedName{ - Name: dvcrSettings.AuthSecret, - Namespace: dvcrSettings.AuthSecretNamespace, - }, - Destination: authSecret, - OwnerReference: podutil.MakeOwnerReference(pod), - }, - } - err := authCopier.Copy(ctx, client) - if err != nil { - return err - } + // Create Secret with a scoped token to use DVCR as destination. + authCopier := copier.AuthSecret{ + Secret: copier.Secret{ + Destination: supGen.DVCRAuthSecret(), + OwnerReference: podutil.MakeOwnerReference(pod), + }, + } + if err := authCopier.CreateScopedTokenDockerConfig(ctx, client, dvcrSettings.TokenSigner, scope, dvcrSettings.RegistryURL); err != nil { + return err } // Copy imagePullSecret if namespaces are differ (e.g. CVMI). @@ -112,14 +93,6 @@ func EnsureForPod(ctx context.Context, client client.Client, supGen Generator, p return nil } -func ShouldCopyDVCRAuthSecret(dvcrSettings *dvcr.Settings, supGen Generator) bool { - if dvcrSettings.AuthSecret == "" { - return false - } - // Should copy if namespaces are different. - return dvcrSettings.AuthSecretNamespace != supGen.Namespace() -} - func ShouldCopyUploaderTLSSecret(dvcrSettings *dvcr.Settings, supGen Generator) bool { if dvcrSettings.UploaderIngressSettings.TLSSecret == "" { return false @@ -140,31 +113,14 @@ func ShouldCopyImagePullSecret(ctrImg *datasource.ContainerRegistry, targetNS st } func EnsureForDataVolume(ctx context.Context, client client.Client, supGen DataVolumeSupplement, dv *cdiv1.DataVolume, dvcrSettings *dvcr.Settings, scope []registrytoken.Access) error { - switch { - case dvcrSettings.TenantAuthzEnabled: - authCopier := copier.AuthSecret{ - Secret: copier.Secret{ - Destination: supGen.DVCRAuthSecretForDV(), - OwnerReference: dvutil.MakeOwnerReference(dv), - }, - } - if err := authCopier.CreateScopedTokenCDI(ctx, client, dvcrSettings.TokenSigner, scope); err != nil { - return err - } - case dvcrSettings.AuthSecret != "": - authCopier := copier.AuthSecret{ - Secret: copier.Secret{ - Source: types.NamespacedName{ - Name: dvcrSettings.AuthSecret, - Namespace: dvcrSettings.AuthSecretNamespace, - }, - Destination: supGen.DVCRAuthSecretForDV(), - OwnerReference: dvutil.MakeOwnerReference(dv), - }, - } - if err := authCopier.CopyCDICompatible(ctx, client, dvcrSettings.RegistryURL); err != nil { - return err - } + authCopier := copier.AuthSecret{ + Secret: copier.Secret{ + Destination: supGen.DVCRAuthSecretForDV(), + OwnerReference: dvutil.MakeOwnerReference(dv), + }, + } + if err := authCopier.CreateScopedTokenCDI(ctx, client, dvcrSettings.TokenSigner, scope); err != nil { + return err } // CABundle needs transformation, so it always copied. diff --git a/images/virtualization-artifact/pkg/controller/uploader/settings.go b/images/virtualization-artifact/pkg/controller/uploader/settings.go index e72b2e0903..cca5395f5d 100644 --- a/images/virtualization-artifact/pkg/controller/uploader/settings.go +++ b/images/virtualization-artifact/pkg/controller/uploader/settings.go @@ -31,11 +31,7 @@ type Settings struct { } func ApplyDVCRDestinationSettings(podEnvVars *Settings, dvcrSettings *dvcr.Settings, supGen supplements.Generator, dvcrImageName string) { - authSecret := dvcrSettings.AuthSecret - if dvcrSettings.TenantAuthzEnabled || supplements.ShouldCopyDVCRAuthSecret(dvcrSettings, supGen) { - authSecret = supGen.DVCRAuthSecret().Name - } - podEnvVars.DestinationAuthSecret = authSecret + podEnvVars.DestinationAuthSecret = supGen.DVCRAuthSecret().Name podEnvVars.DestinationInsecureTLS = dvcrSettings.InsecureTLS podEnvVars.DestinationEndpoint = dvcrImageName } diff --git a/images/virtualization-artifact/pkg/dvcr/dvcr.go b/images/virtualization-artifact/pkg/dvcr/dvcr.go index 7df634b9da..12d0a18b80 100644 --- a/images/virtualization-artifact/pkg/dvcr/dvcr.go +++ b/images/virtualization-artifact/pkg/dvcr/dvcr.go @@ -45,12 +45,10 @@ type Settings struct { ImageMonitorSchedule string // GCSchedule is a cron formatted schedule to periodically run a garbage collection. GCSchedule string - // TenantAuthzEnabled turns on per-namespace DVCR authorization: importer and - // uploader Pods authenticate with a scoped token minted for the single - // repository they use, instead of the shared read-write credential, which is - // then no longer copied into tenant namespaces. - TenantAuthzEnabled bool - // TokenSigner mints the scoped tokens. Non-nil only when TenantAuthzEnabled. + // TokenSigner mints the scoped per-namespace DVCR tokens: importer and uploader + // Pods authenticate with a token minted for the single repository they use, + // instead of the shared read-write credential, which is then no longer copied + // into tenant namespaces. TokenSigner *registrytoken.Signer } diff --git a/images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go index 381e2a40b1..c1e14ad557 100644 --- a/images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go +++ b/images/virtualization-artifact/pkg/dvcr/registrytoken/signer_test.go @@ -20,80 +20,132 @@ import ( "crypto/ecdsa" "crypto/elliptic" "crypto/rand" + "crypto/rsa" "crypto/x509" "encoding/pem" "testing" "time" "github.com/golang-jwt/jwt/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) -func newSignerPEM(t *testing.T) ([]byte, *ecdsa.PublicKey) { - t.Helper() +func TestRegistryToken(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "RegistryToken Suite") +} + +// newSignerPEM returns a fresh PKCS8-encoded ECDSA private key and its public key. +func newSignerPEM() ([]byte, *ecdsa.PublicKey) { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - t.Fatal(err) - } + Expect(err).NotTo(HaveOccurred()) der, err := x509.MarshalPKCS8PrivateKey(key) - if err != nil { - t.Fatal(err) - } + Expect(err).NotTo(HaveOccurred()) return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), &key.PublicKey } -func TestSignRoundTrip(t *testing.T) { - keyPEM, pub := newSignerPEM(t) - signer, err := NewSignerFromPEM(keyPEM) - if err != nil { - t.Fatal(err) - } - - access := []Access{{Type: "repository", Name: "cvi/my-image", Actions: []string{"pull", "push"}}} +var _ = Describe("Signer", func() { now := time.Unix(1_700_000_000, 0) - raw, err := signer.Sign(access, now) - if err != nil { - t.Fatal(err) - } - - // Verify signature and claims the way the registry will. - parsed, err := jwt.Parse(raw, func(*jwt.Token) (interface{}, error) { return pub, nil }, - jwt.WithValidMethods([]string{"ES256"}), - jwt.WithTimeFunc(func() time.Time { return now.Add(time.Minute) })) - if err != nil { - t.Fatalf("parse: %v", err) - } - if kid := parsed.Header["kid"]; kid != KeyID { - t.Errorf("kid = %v, want %q", kid, KeyID) - } - claims := parsed.Claims.(jwt.MapClaims) - if claims["iss"] != Issuer { - t.Errorf("iss = %v, want %q", claims["iss"], Issuer) - } - if claims["aud"] != Audience { - t.Errorf("aud = %v, want %q", claims["aud"], Audience) - } - acc, ok := claims["access"].([]interface{}) - if !ok || len(acc) != 1 { - t.Fatalf("access claim = %v", claims["access"]) - } - entry := acc[0].(map[string]interface{}) - if entry["name"] != "cvi/my-image" || entry["type"] != "repository" { - t.Errorf("access entry = %v", entry) - } -} -func TestSignExpired(t *testing.T) { - keyPEM, pub := newSignerPEM(t) - signer, _ := NewSignerFromPEM(keyPEM) - now := time.Unix(1_700_000_000, 0) - raw, err := signer.Sign(nil, now) - if err != nil { - t.Fatal(err) - } - // Past DefaultTTL the token must no longer verify. - _, err = jwt.Parse(raw, func(*jwt.Token) (interface{}, error) { return pub, nil }, - jwt.WithTimeFunc(func() time.Time { return now.Add(DefaultTTL + time.Hour) })) - if err == nil { - t.Fatal("expected expired token to fail verification") - } -} + It("mints a token the registry accepts, with the expected claims", func() { + keyPEM, pub := newSignerPEM() + signer, err := NewSignerFromPEM(keyPEM) + Expect(err).NotTo(HaveOccurred()) + + access := []Access{{Type: "repository", Name: "cvi/my-image", Actions: []string{"pull", "push"}}} + raw, err := signer.Sign(access, now) + Expect(err).NotTo(HaveOccurred()) + + // Verify signature and claims the way the registry will. + parsed, err := jwt.Parse(raw, func(*jwt.Token) (any, error) { return pub, nil }, + jwt.WithValidMethods([]string{"ES256"}), + jwt.WithTimeFunc(func() time.Time { return now.Add(time.Minute) })) + Expect(err).NotTo(HaveOccurred()) + + Expect(parsed.Header["kid"]).To(Equal(KeyID)) + claims := parsed.Claims.(jwt.MapClaims) + Expect(claims["iss"]).To(Equal(Issuer)) + Expect(claims["aud"]).To(Equal(Audience)) + + acc, ok := claims["access"].([]any) + Expect(ok).To(BeTrue()) + Expect(acc).To(HaveLen(1)) + entry := acc[0].(map[string]any) + Expect(entry["name"]).To(Equal("cvi/my-image")) + Expect(entry["type"]).To(Equal("repository")) + + // Time claims must frame exactly [iat-30s, iat+DefaultTTL] so the registry + // accepts a token immediately and rejects it past its lifetime. + Expect(int64(claims["iat"].(float64))).To(Equal(now.Unix())) + Expect(int64(claims["nbf"].(float64))).To(Equal(now.Add(-30 * time.Second).Unix())) + Expect(int64(claims["exp"].(float64))).To(Equal(now.Add(DefaultTTL).Unix())) + }) + + It("binds the token to its signing key: a wrong public key must not verify", func() { + keyPEM, _ := newSignerPEM() + signer, err := NewSignerFromPEM(keyPEM) + Expect(err).NotTo(HaveOccurred()) + _, otherPub := newSignerPEM() + + raw, err := signer.Sign(nil, now) + Expect(err).NotTo(HaveOccurred()) + + _, err = jwt.Parse(raw, func(*jwt.Token) (any, error) { return otherPub, nil }, + jwt.WithValidMethods([]string{"ES256"}), + jwt.WithTimeFunc(func() time.Time { return now.Add(time.Minute) })) + Expect(err).To(HaveOccurred()) + }) + + It("mints a token that no longer verifies past DefaultTTL", func() { + keyPEM, pub := newSignerPEM() + signer, err := NewSignerFromPEM(keyPEM) + Expect(err).NotTo(HaveOccurred()) + + raw, err := signer.Sign(nil, now) + Expect(err).NotTo(HaveOccurred()) + + _, err = jwt.Parse(raw, func(*jwt.Token) (any, error) { return pub, nil }, + jwt.WithTimeFunc(func() time.Time { return now.Add(DefaultTTL + time.Hour) })) + Expect(err).To(HaveOccurred()) + }) + + // Key-parsing branches: both accepted EC formats (PKCS8, SEC1) and the + // rejected inputs (non-PEM bytes, a non-ECDSA key). + DescribeTable("NewSignerFromPEM", + func(pemFn func() []byte, wantErr bool) { + signer, err := NewSignerFromPEM(pemFn()) + if wantErr { + Expect(err).To(HaveOccurred()) + return + } + Expect(err).NotTo(HaveOccurred()) + _, err = signer.Sign(nil, now) + Expect(err).NotTo(HaveOccurred()) + }, + Entry("PKCS8 EC key", func() []byte { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + Expect(err).NotTo(HaveOccurred()) + der, err := x509.MarshalPKCS8PrivateKey(key) + Expect(err).NotTo(HaveOccurred()) + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}) + }, false), + Entry("SEC1 EC key", func() []byte { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + Expect(err).NotTo(HaveOccurred()) + der, err := x509.MarshalECPrivateKey(key) + Expect(err).NotTo(HaveOccurred()) + return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der}) + }, false), + Entry("not a PEM block", func() []byte { + return []byte("definitely not pem") + }, true), + Entry("RSA key is not ECDSA", func() []byte { + key, err := rsa.GenerateKey(rand.Reader, 2048) + Expect(err).NotTo(HaveOccurred()) + der, err := x509.MarshalPKCS8PrivateKey(key) + Expect(err).NotTo(HaveOccurred()) + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}) + }, true), + ) +}) diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 852be4b154..2b2aa2b814 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -133,18 +133,6 @@ properties: Schedule to run garbage collection procedure that remove stale images for `ClusterVirtualImage`, `VirtualImage`, `VirtualDisk` resources deleted from the cluster. By default, periodic garbage collection is enabled and runs daily at 02:00. - tenantRegistryAuthorization: - type: boolean - default: false - description: | - Enable per-namespace authorization in the DVCR registry (experimental). - - When enabled, image import and upload Pods authenticate to DVCR with a scoped - token that grants access only to the repositories they use, while nodes get - pull-only access. The shared read-write credential is no longer copied into - tenant namespaces. - - When disabled (default), DVCR uses shared htpasswd credentials. Disable to roll back. audit: type: object description: | diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index 500ff4efa3..2e29677fb7 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -75,17 +75,6 @@ properties: Расписание для запуска процедуры очистки хранилища. Очистка удалит неактуальные образы, созданные для ресурсов `ClusterVirtualImage`, `VirtualImage`, `VirtualDisk`, которых уже нет в кластере. По умолчанию периодическая очистка включена и выполняется ежедневно в 02:00. - tenantRegistryAuthorization: - description: | - Включить авторизацию по namespace в реестре DVCR (экспериментально). - - Когда включено, поды импорта и загрузки образов аутентифицируются в DVCR - scoped-токеном, дающим доступ только к используемым ими репозиториям, а узлы - получают доступ только на чтение (pull). Общий read-write-креденшл больше не - копируется в namespace тенантов. - - Когда выключено (по умолчанию), DVCR использует общие htpasswd-учётные данные. - Выключите параметр для отката. virtualImages: type: object description: | diff --git a/openapi/values.yaml b/openapi/values.yaml index 0087d66969..e727b02fec 100644 --- a/openapi/values.yaml +++ b/openapi/values.yaml @@ -33,7 +33,7 @@ properties: default: "" passwordRW: type: string - passwordPuller: + passwordRO: type: string default: "" tokenPrivateKey: diff --git a/templates/dvcr/_helpers.tpl b/templates/dvcr/_helpers.tpl index 8259289594..cb4ce81061 100644 --- a/templates/dvcr/_helpers.tpl +++ b/templates/dvcr/_helpers.tpl @@ -8,25 +8,12 @@ true {{- .Values.virtualization.internal | dig "dvcr" "garbageCollectionModeEnabled" "false" | default "false" -}} {{- end }} -{{- define "dvcr.isTenantAuthz" -}} -{{- .Values.virtualization.internal.moduleConfig | dig "dvcr" "tenantRegistryAuthorization" false -}} -{{- end }} - {{- define "dvcr.envs" -}} - name: REGISTRY_HTTP_TLS_CERTIFICATE value: /etc/ssl/docker/tls.crt - name: REGISTRY_HTTP_TLS_KEY value: /etc/ssl/docker/tls.key -{{- if ne (include "dvcr.isTenantAuthz" .) "true" }} -- name: REGISTRY_AUTH - value: "htpasswd" -- name: REGISTRY_AUTH_HTPASSWD_REALM - value: "Registry Realm" -- name: REGISTRY_AUTH_HTPASSWD_PATH - value: "/auth/htpasswd" -{{- end }} - - name: REGISTRY_HTTP_SECRET valueFrom: secretKeyRef: @@ -97,14 +84,12 @@ true items: - key: htpasswd path: htpasswd -{{- if eq (include "dvcr.isTenantAuthz" .) "true" }} - key: passwordRW path: passwordRW - - key: passwordPuller - path: passwordPuller + - key: passwordRO + path: passwordRO - key: tokenPublicKey path: tokenPublicKey -{{- end }} {{- end -}} diff --git a/templates/dvcr/configmap.yaml b/templates/dvcr/configmap.yaml index c54b353877..91420293c6 100644 --- a/templates/dvcr/configmap.yaml +++ b/templates/dvcr/configmap.yaml @@ -19,19 +19,17 @@ data: readonly: enabled: true {{- end }} -{{- if eq (include "dvcr.isTenantAuthz" . ) "true" }} auth: dvcr-k8s: realm: "dvcr" adminusername: "admin" adminpasswordfile: /auth/passwordRW pullerusername: "node-puller" - pullerpasswordfile: /auth/passwordPuller + pullerpasswordfile: /auth/passwordRO jwtissuer: "virtualization-controller" jwtaudience: "dvcr" jwtkeyid: "dvcr" jwtpublickeyfile: /auth/tokenPublicKey -{{- end }} http: addr: :5000 headers: diff --git a/templates/dvcr/nodegroupconfiguration.yaml b/templates/dvcr/nodegroupconfiguration.yaml index bf54cca683..eb84c43bfe 100644 --- a/templates/dvcr/nodegroupconfiguration.yaml +++ b/templates/dvcr/nodegroupconfiguration.yaml @@ -1,13 +1,9 @@ {{- if ne (dig "dvcr" "serviceIP" "" .Values.virtualization.internal) "" }} --- {{- $ca := printf "%s" .Values.virtualization.internal.dvcr.cert.ca }} - {{- $authUser := "admin" }} - {{- $authPass := .Values.virtualization.internal.dvcr.passwordRW }} - {{- if eq (include "dvcr.isTenantAuthz" .) "true" }} {{- /* Nodes only pull images; use the pull-only node-puller credential. */ -}} - {{- $authUser = "node-puller" }} - {{- $authPass = .Values.virtualization.internal.dvcr.passwordPuller }} - {{- end }} + {{- $authUser := "node-puller" }} + {{- $authPass := .Values.virtualization.internal.dvcr.passwordRO }} {{- $password := printf "%s:%s" $authUser (printf "%s" $authPass | b64dec) | b64enc }} {{- $registry := include "dvcr.get_registry" (list .) }} {{- $endpoint := .Values.virtualization.internal.dvcr.serviceIP }} diff --git a/templates/dvcr/secret.yaml b/templates/dvcr/secret.yaml index 6a0141842a..eefe322cbb 100644 --- a/templates/dvcr/secret.yaml +++ b/templates/dvcr/secret.yaml @@ -24,13 +24,11 @@ metadata: type: Opaque data: passwordRW: {{ .Values.virtualization.internal.dvcr.passwordRW | quote }} - passwordPuller: {{ .Values.virtualization.internal.dvcr.passwordPuller | quote }} + passwordRO: {{ .Values.virtualization.internal.dvcr.passwordRO | quote }} htpasswd: {{ .Values.virtualization.internal.dvcr.htpasswd | quote }} salt: {{ .Values.virtualization.internal.dvcr.salt | quote }} -{{- if eq (include "dvcr.isTenantAuthz" .) "true" }} tokenPrivateKey: {{ .Values.virtualization.internal.dvcr.tokenPrivateKey | quote }} tokenPublicKey: {{ .Values.virtualization.internal.dvcr.tokenPublicKey | quote }} -{{- end }} --- {{- /* TODO: delete this secret, because containerd has credentials to auth to the dvcr. This secret was needed to create virtual machines from dvcr images */ -}} diff --git a/templates/virtualization-controller/_helpers.tpl b/templates/virtualization-controller/_helpers.tpl index c76eb13c1d..a883bdc746 100644 --- a/templates/virtualization-controller/_helpers.tpl +++ b/templates/virtualization-controller/_helpers.tpl @@ -40,15 +40,11 @@ true value: "*/5 * * * *" - name: DVCR_GC_SCHEDULE value: "{{ .Values.virtualization.internal.moduleConfig | dig "dvcr" "gc" "schedule" "" }}" -- name: DVCR_TENANT_AUTHZ_ENABLED - value: "{{ include "dvcr.isTenantAuthz" . }}" -{{- if eq (include "dvcr.isTenantAuthz" .) "true" }} - name: DVCR_TOKEN_PRIVATE_KEY valueFrom: secretKeyRef: name: dvcr-secrets key: tokenPrivateKey -{{- end }} - name: VIRTUAL_MACHINE_CIDRS value: {{ join "," .Values.virtualization.internal.moduleConfig.virtualMachineCIDRs | quote }} {{- if (hasKey .Values.virtualization.internal.moduleConfig "virtualImages") }} From 5a279985601937b64af2dfb76c488da75a1f3c69 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Mon, 6 Jul 2026 12:26:47 +0000 Subject: [PATCH 31/31] test(images/vi): expect per-namespace dvcr auth secret name The registry authorization refactor makes DestinationAuthSecret come from the generated per-namespace secret name instead of dvcrSettings.AuthSecret; update the CreatePodStep expectations accordingly. Signed-off-by: Daniil Antoshin --- .../vi/internal/source/step/create_pod_step_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vi/internal/source/step/create_pod_step_test.go b/images/virtualization-artifact/pkg/controller/vi/internal/source/step/create_pod_step_test.go index a8ffb8bb9c..e79cd60ef7 100644 --- a/images/virtualization-artifact/pkg/controller/vi/internal/source/step/create_pod_step_test.go +++ b/images/virtualization-artifact/pkg/controller/vi/internal/source/step/create_pod_step_test.go @@ -288,7 +288,7 @@ var _ = Describe("CreatePodStep", func() { envSettings := step.getEnvSettings(vi, supplements.NewGenerator("vi", vi.Name, vi.Namespace, vi.UID), volumeMode) Expect(envSettings.Source).To(Equal(expectedSource)) - Expect(envSettings.DestinationAuthSecret).To(Equal("dvcr-secret")) + Expect(envSettings.DestinationAuthSecret).To(Equal("d8v-vi-dvcr-auth-vi-vi-uid")) Expect(envSettings.DestinationEndpoint).To(Equal("registry.example.com/vi/default/vi:vi-uid")) }, Entry("filesystem by default", nil, importer.SourceFilesystem), @@ -317,7 +317,7 @@ var _ = Describe("CreatePodStep", func() { Expect(importerStub.startCalls).To(Equal(1)) Expect(importerStub.settings).ToNot(BeNil()) Expect(importerStub.settings.Source).To(Equal(importer.SourceBlockDevice)) - Expect(importerStub.settings.DestinationAuthSecret).To(Equal("dvcr-secret")) + Expect(importerStub.settings.DestinationAuthSecret).To(Equal("d8v-vi-dvcr-auth-vi-vi-uid")) Expect(importerStub.settings.DestinationEndpoint).To(Equal("registry.example.com/vi/default/vi:vi-uid")) Expect(vi.Status.Progress).To(Equal("0%")) Expect(vi.Status.Target.RegistryURL).To(Equal("registry.example.com/custom:tag"))