Skip to content

[Perf] parallelization for snapshots downloads and secrets creation & deletion #21421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mohammedahmed18 wants to merge 2 commits into from
+115 −43
Closed

[Perf] parallelization for snapshots downloads and secrets creation & deletion #21421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2b319f0
perf: parallelize K8s secret creation and deletion in workspace lifec…
mohammedahmed18 Apr 30, 2026
efb3d85
perf: parallelize SignDownload calls in CollectRemoteContent
mohammedahmed18 Apr 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 74 additions & 24 deletions components/ws-daemon/pkg/content/initializer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,15 +68,6 @@ var (
func CollectRemoteContent(ctx context.Context, rs storage.DirectAccess, ps storage.PresignedAccess, workspaceOwner string, initializer *csapi.WorkspaceInitializer) (rc map[string]storage.DownloadInfo, err error) {
rc = make(map[string]storage.DownloadInfo)

backup, err := ps.SignDownload(ctx, rs.Bucket(workspaceOwner), rs.BackupObject(storage.DefaultBackup), &storage.SignedURLOptions{})
if err == storage.ErrNotFound {
// no backup found - that's fine
} else if err != nil {
return nil, err
} else {
rc[storage.DefaultBackup] = *backup
}

si := initializer.GetSnapshot()
pi := initializer.GetPrebuild()
if ci := initializer.GetComposite(); ci != nil {
Expand All @@ -89,33 +80,92 @@ func CollectRemoteContent(ctx context.Context, rs storage.DirectAccess, ps stora
}
}
}

// Parse snapshot/prebuild names before launching goroutines so we can fail fast
// on invalid input without wasting network calls.
var snapshotBkt, snapshotObj string
if si != nil {
bkt, obj, err := storage.ParseSnapshotName(si.Snapshot)
snapshotBkt, snapshotObj, err = storage.ParseSnapshotName(si.Snapshot)
if err != nil {
return nil, err
}
info, err := ps.SignDownload(ctx, bkt, obj, &storage.SignedURLOptions{})
if err == storage.ErrNotFound {
return nil, errCannotFindSnapshot
}
if err != nil {
return nil, xerrors.Errorf("cannot find snapshot: %w", err)
}

rc[si.Snapshot] = *info
}
var prebuildBkt, prebuildObj string
if pi != nil && pi.Prebuild != nil && pi.Prebuild.Snapshot != "" {
bkt, obj, err := storage.ParseSnapshotName(pi.Prebuild.Snapshot)
prebuildBkt, prebuildObj, err = storage.ParseSnapshotName(pi.Prebuild.Snapshot)
if err != nil {
return nil, err
}
info, err := ps.SignDownload(ctx, bkt, obj, &storage.SignedURLOptions{})
}

// Fire all SignDownload calls in parallel since they are independent network
// operations (each signs a different storage object URL).
type result struct {
key string
info *storage.DownloadInfo
err error
}

// Count how many downloads we need to perform.
numDownloads := 1 // always check backup
if si != nil {
numDownloads++
}
if pi != nil && pi.Prebuild != nil && pi.Prebuild.Snapshot != "" {
numDownloads++
}

results := make(chan result, numDownloads)

// 1. Backup download
go func() {
backup, err := ps.SignDownload(ctx, rs.Bucket(workspaceOwner), rs.BackupObject(storage.DefaultBackup), &storage.SignedURLOptions{})
if err == storage.ErrNotFound {
// no prebuild found - that's fine
results <- result{key: storage.DefaultBackup, info: nil, err: nil}
} else if err != nil {
return nil, xerrors.Errorf("cannot find prebuild: %w", err)
results <- result{key: storage.DefaultBackup, err: err}
} else {
rc[pi.Prebuild.Snapshot] = *info
results <- result{key: storage.DefaultBackup, info: backup}
}
}()

// 2. Snapshot download (if needed)
if si != nil {
go func() {
info, err := ps.SignDownload(ctx, snapshotBkt, snapshotObj, &storage.SignedURLOptions{})
if err == storage.ErrNotFound {
results <- result{key: si.Snapshot, err: errCannotFindSnapshot}
} else if err != nil {
results <- result{key: si.Snapshot, err: xerrors.Errorf("cannot find snapshot: %w", err)}
} else {
results <- result{key: si.Snapshot, info: info}
}
}()
}

// 3. Prebuild download (if needed)
if pi != nil && pi.Prebuild != nil && pi.Prebuild.Snapshot != "" {
go func() {
info, err := ps.SignDownload(ctx, prebuildBkt, prebuildObj, &storage.SignedURLOptions{})
if err == storage.ErrNotFound {
// no prebuild found - that's fine
results <- result{key: pi.Prebuild.Snapshot, info: nil, err: nil}
} else if err != nil {
results <- result{key: pi.Prebuild.Snapshot, err: xerrors.Errorf("cannot find prebuild: %w", err)}
} else {
results <- result{key: pi.Prebuild.Snapshot, info: info}
}
}()
}

// Collect results
for i := 0; i < numDownloads; i++ {
r := <-results
if r.err != nil {
return nil, r.err
}
if r.info != nil {
rc[r.key] = *r.info
}
}

Expand Down
36 changes: 24 additions & 12 deletions components/ws-manager-mk2/controllers/workspace_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -511,21 +511,33 @@ func (r *WorkspaceReconciler) deleteWorkspaceSecrets(ctx context.Context, ws *wo
defer tracing.FinishSpan(span, &err)
log := log.FromContext(ctx).WithValues("owi", ws.OWI())

// if a secret cannot be deleted we do not return early because we want to attempt
// the deletion of the remaining secrets
// Delete env and token secrets in parallel since they are independent operations.
var envErr, tokenErr error
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
if e := r.deleteSecret(ctx, fmt.Sprintf("%s-%s", ws.Name, "env"), r.Config.Namespace); e != nil {
envErr = e
log.Error(e, "could not delete environment secret", "workspace", ws.Name)
}
}()
go func() {
defer wg.Done()
if e := r.deleteSecret(ctx, fmt.Sprintf("%s-%s", ws.Name, "tokens"), r.Config.SecretsNamespace); e != nil {
tokenErr = e
log.Error(e, "could not delete token secret", "workspace", ws.Name)
}
}()
wg.Wait()

var errs []string
err = r.deleteSecret(ctx, fmt.Sprintf("%s-%s", ws.Name, "env"), r.Config.Namespace)
if err != nil {
errs = append(errs, err.Error())
log.Error(err, "could not delete environment secret", "workspace", ws.Name)
if envErr != nil {
errs = append(errs, envErr.Error())
}

err = r.deleteSecret(ctx, fmt.Sprintf("%s-%s", ws.Name, "tokens"), r.Config.SecretsNamespace)
if err != nil {
errs = append(errs, err.Error())
log.Error(err, "could not delete token secret", "workspace", ws.Name)
if tokenErr != nil {
errs = append(errs, tokenErr.Error())
}

if len(errs) != 0 {
return fmt.Errorf("%s", strings.Join(errs, ":"))
}
Expand Down
24 changes: 17 additions & 7 deletions components/ws-manager-mk2/service/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,14 +298,24 @@ func (wsm *WorkspaceManagerServer) StartWorkspace(ctx context.Context, req *wsma
return nil, status.Errorf(codes.AlreadyExists, "workspace %s already exists", req.Metadata.MetaId)
}

err = wsm.createWorkspaceSecret(ctx, &ws, envSecretName, wsm.Config.Namespace, envData)
if err != nil {
return nil, fmt.Errorf("cannot create env secret for workspace %s: %w", req.Id, err)
// Create env and token secrets in parallel since they are independent operations.
var envSecretErr, tokenSecretErr error
var secretWg sync.WaitGroup
secretWg.Add(2)
go func() {
defer secretWg.Done()
envSecretErr = wsm.createWorkspaceSecret(ctx, &ws, envSecretName, wsm.Config.Namespace, envData)
}()
go func() {
defer secretWg.Done()
tokenSecretErr = wsm.createWorkspaceSecret(ctx, &ws, fmt.Sprintf("%s-%s", req.Id, "tokens"), wsm.Config.SecretsNamespace, tokenData)
}()
secretWg.Wait()
if envSecretErr != nil {
return nil, fmt.Errorf("cannot create env secret for workspace %s: %w", req.Id, envSecretErr)
}

err = wsm.createWorkspaceSecret(ctx, &ws, fmt.Sprintf("%s-%s", req.Id, "tokens"), wsm.Config.SecretsNamespace, tokenData)
if err != nil {
return nil, fmt.Errorf("cannot create token secret for workspace %s: %w", req.Id, err)
if tokenSecretErr != nil {
return nil, fmt.Errorf("cannot create token secret for workspace %s: %w", req.Id, tokenSecretErr)
}

wsm.metrics.recordWorkspaceStart(&ws)
Expand Down