From eb26cc20a086c58163b02c7c7aa38ff9a0487b40 Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Thu, 13 Aug 2026 12:47:54 -0400 Subject: [PATCH] split controlapi functional tests into their own package functional_test.go was also split into one file per resource, to match what we're doing for the RPC handlers too. See #891 I also needed to make some changes to move the tests to a separate package: * `NewAteletDialer` now takes `options`, and `WithDialCredentials` lets a test build its own transport credentials. The fake atelet is reached over insecure transport. Didn't change existing callers. This is the only non-test change. --- cmd/ateapi/internal/controlapi/actor_test.go | 393 -- cmd/ateapi/internal/controlapi/common_test.go | 48 + cmd/ateapi/internal/controlapi/dialer.go | 18 +- .../functionaltest/actor_snapshot_test.go | 115 + .../actor_test.go} | 3884 +++++++---------- .../functionaltest/atespace_test.go | 247 ++ .../controlapi/functionaltest/common_test.go | 691 +++ .../controlapi/functionaltest/main_test.go | 229 + .../controlapi/functionaltest/worker_test.go | 82 + .../internal/controlapi/span_identity_test.go | 3 +- 10 files changed, 2974 insertions(+), 2736 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/common_test.go create mode 100644 cmd/ateapi/internal/controlapi/functionaltest/actor_snapshot_test.go rename cmd/ateapi/internal/controlapi/{functional_test.go => functionaltest/actor_test.go} (70%) create mode 100644 cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go create mode 100644 cmd/ateapi/internal/controlapi/functionaltest/common_test.go create mode 100644 cmd/ateapi/internal/controlapi/functionaltest/main_test.go create mode 100644 cmd/ateapi/internal/controlapi/functionaltest/worker_test.go diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index 809d7e25b3..2fe78fee9a 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -16,63 +16,19 @@ package controlapi import ( "context" - "fmt" - "strings" "testing" - "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" - "github.com/agent-substrate/substrate/internal/ateattr" "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/volume" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" - "go.opentelemetry.io/otel/attribute" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/fieldmaskpb" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/apimachinery/pkg/util/wait" ) -// CreateActor is the only lifecycle op with the full identity (incl. version) -// available in the request, so the whole ate.* set should land on its span. -func TestCreateActor_StampsFullSpanIdentity(t *testing.T) { - ns := namespaceForTest("ns-span-create") - tc := setupTest(t, ns) - defer tc.cleanup() - createTemplate(t, tc, ns) - - attrs := recordRootSpanAttrs(t, func(ctx context.Context) { - if _, err := tc.service.CreateActor(ctx, &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }, - }); err != nil { - t.Fatalf("CreateActor: %v", err) - } - }) - - assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) - assertSpanStr(t, attrs, ateattr.ActorNameKey, testActorID) - assertSpanStr(t, attrs, ateattr.TemplateNameKey, "tmpl1") - assertSpanStr(t, attrs, ateattr.TemplateNamespaceKey, ns) - // uid is server-assigned on create, so assert it is present and non-empty - // rather than a fixed value. - if v, ok := attrs[ateattr.ActorUIDKey]; !ok || v.Type() != attribute.STRING || v.AsString() == "" { - t.Errorf("%s = %v, want non-empty server-assigned uid", ateattr.ActorUIDKey, v.Emit()) - } - if v, ok := attrs[ateattr.ActorVersionKey]; !ok || v.Type() != attribute.INT64 || v.AsInt64() != 1 { - t.Errorf("%s = %v, want int64 1", ateattr.ActorVersionKey, v.Emit()) - } -} - func TestValidateCreateActorRequest(t *testing.T) { validActor := func(mutate func(*ateapipb.Actor)) *ateapipb.CreateActorRequest { a := &ateapipb.Actor{ @@ -172,104 +128,6 @@ func TestValidateCreateActorRequest(t *testing.T) { } } -func TestCreateActor_RejectsDifferentTemplateForDataSnapshot(t *testing.T) { - ns := namespaceForTest("ns-data-snapshot-template") - tc := setupTest(t, ns) - defer tc.cleanup() - createTemplate(t, tc, ns) - createTemplateWithSelector(t, tc, ns, "tmpl2", nil) - - tmpl, err := tc.actorTemplateLister.ActorTemplates(ns).Get("tmpl1") - if err != nil { - t.Fatalf("Get source ActorTemplate: %v", err) - } - snapshot, err := tc.persistence.CreateActorSnapshot(context.Background(), &ateapipb.ActorSnapshot{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "data-snapshot"}, - SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "source"}, - ActorTemplateUid: string(tmpl.GetUID()), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, - SnapshotUri: "gs://snapshots/snapshots/" + testAtespace + "/data-snapshot", - }) - if err != nil { - t.Fatalf("CreateActorSnapshot: %v", err) - } - if _, err := tc.persistence.CreateActorSnapshotTag(context.Background(), testAtespace, snapshot.GetMetadata().GetName(), &ateapipb.ActorSnapshotTag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "data-snapshot"}, - Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE, - }); err != nil { - t.Fatalf("CreateActorSnapshotTag: %v", err) - } - - _, err = tc.service.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "clone"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl2", - SourceSnapshot: &ateapipb.ActorSnapshotSource{Tag: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "data-snapshot"}}, - }, - }) - if status.Code(err) != codes.FailedPrecondition { - t.Fatalf("CreateActor status = %v, want FailedPrecondition", status.Code(err)) - } -} - -func TestCreateActor_RejectsSnapshotWithExternalVolumes(t *testing.T) { - ns := namespaceForTest("ns-snapshot-external-volume") - tc := setupTest(t, ns) - defer tc.cleanup() - template, err := tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).Create(context.Background(), &atev1alpha1.ActorTemplate{ - ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: ns}, - Spec: atev1alpha1.ActorTemplateSpec{ - SnapshotsConfig: atev1alpha1.SnapshotsConfig{Location: "gs://snapshots"}, - Containers: []atev1alpha1.Container{{ - Name: "main", Image: "main@sha256:abc", VolumeMounts: []atev1alpha1.VolumeMount{{Name: "data", MountPath: "/data"}}, - }}, - Volumes: []atev1alpha1.Volume{{ - Name: "data", - VolumeSource: atev1alpha1.VolumeSource{ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ - Capacity: resource.MustParse("1Gi"), StorageClassName: "standard", - }}, - }}, - }, - }, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Create ActorTemplate: %v", err) - } - if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - got, err := tc.actorTemplateLister.ActorTemplates(ns).Get("tmpl1") - return err == nil && len(got.Spec.Volumes) == 1, nil - }); err != nil { - t.Fatalf("wait for ActorTemplate update: %v", err) - } - snapshot, err := tc.persistence.CreateActorSnapshot(context.Background(), &ateapipb.ActorSnapshot{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "external-volume-snapshot"}, - ActorTemplateUid: string(template.GetUID()), - SnapshotUri: "gs://snapshots/snapshots/" + testAtespace + "/external-volume-snapshot", - }) - if err != nil { - t.Fatalf("CreateActorSnapshot: %v", err) - } - tagRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "external-volume-snapshot"} - if _, err := tc.persistence.CreateActorSnapshotTag(context.Background(), testAtespace, snapshot.GetMetadata().GetName(), &ateapipb.ActorSnapshotTag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: tagRef.GetAtespace(), Name: tagRef.GetName()}, - Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE, - }); err != nil { - t.Fatalf("CreateActorSnapshotTag: %v", err) - } - - _, err = tc.service.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "clone"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - SourceSnapshot: &ateapipb.ActorSnapshotSource{Tag: tagRef}, - }, - }) - if status.Code(err) != codes.FailedPrecondition { - t.Fatalf("CreateActor status = %v, want FailedPrecondition", status.Code(err)) - } -} - func TestValidateGetActorRequest(t *testing.T) { tests := []struct { name string @@ -515,71 +373,6 @@ func TestUpdateActor_FieldMasks(t *testing.T) { } } -func TestUpdateActor_StampsFullSpanIdentity(t *testing.T) { - ns := namespaceForTest("ns-span-update") - tc := setupTest(t, ns) - defer tc.cleanup() - createTemplate(t, tc, ns) - - if _, err := tc.service.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }, - }); err != nil { - t.Fatalf("seed CreateActor: %v", err) - } - - attrs := recordRootSpanAttrs(t, func(ctx context.Context) { - if _, err := tc.service.UpdateActor(ctx, &ateapipb.UpdateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}, - WorkerSelector: &ateapipb.Selector{ - MatchLabels: map[string]string{"env": "prod"}, - }, - }, - UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"worker_selector"}}, - }); err != nil { - t.Fatalf("UpdateActor: %v", err) - } - }) - - assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) - assertSpanStr(t, attrs, ateattr.ActorNameKey, testActorID) - assertSpanStr(t, attrs, ateattr.TemplateNameKey, "tmpl1") - assertSpanStr(t, attrs, ateattr.TemplateNamespaceKey, ns) - if v, ok := attrs[ateattr.ActorUIDKey]; !ok || v.Type() != attribute.STRING || v.AsString() == "" { - t.Errorf("%s = %v, want non-empty server-assigned uid", ateattr.ActorUIDKey, v.Emit()) - } - if v, ok := attrs[ateattr.ActorVersionKey]; !ok || v.Type() != attribute.INT64 || v.AsInt64() != 2 { - t.Errorf("%s = %v, want int64 2 (updated version)", ateattr.ActorVersionKey, v.Emit()) - } -} - -func TestUpdateActor_FailedLookupStampsRefIdentityOnly(t *testing.T) { - ns := namespaceForTest("ns-span-update-err") - tc := setupTest(t, ns) - defer tc.cleanup() - - attrs := recordRootSpanAttrs(t, func(ctx context.Context) { - if _, err := tc.service.UpdateActor(ctx, &ateapipb.UpdateActorRequest{ - Actor: &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}}, - UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"worker_selector"}}, - }); status.Code(err) != codes.NotFound { - t.Fatalf("UpdateActor(missing) error = %v, want code NotFound", err) - } - }) - - assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) - assertSpanStr(t, attrs, ateattr.ActorNameKey, testActorID) - for _, k := range []attribute.Key{ateattr.ActorUIDKey, ateattr.TemplateNameKey, ateattr.TemplateNamespaceKey, ateattr.ActorVersionKey} { - if _, ok := attrs[k]; ok { - t.Errorf("unexpected %s on failed-update span", k) - } - } -} - // TestUpdateActor_DeleteRecreateRace checks that an update is not applied // if an actor was deleted and recreated during the update operation. func TestUpdateActor_DeleteRecreateRace(t *testing.T) { @@ -767,35 +560,6 @@ func serviceWithActor(t *testing.T, actor *ateapipb.Actor) (*Service, *ateapipb. return &Service{persistence: persistence}, created } -// Delete addresses the actor by ref (atespace + id) and does not resolve the -// template/version, so only the ref identity is stamped. -func TestDeleteActor_StampsRefSpanIdentity(t *testing.T) { - ns := namespaceForTest("ns-span-delete") - tc := setupTest(t, ns) - defer tc.cleanup() - createTemplate(t, tc, ns) - if _, err := tc.service.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }, - }); err != nil { - t.Fatalf("seed CreateActor: %v", err) - } - - attrs := recordRootSpanAttrs(t, func(ctx context.Context) { - if _, err := tc.service.DeleteActor(ctx, &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: testActorID}, - }); err != nil { - t.Fatalf("DeleteActor: %v", err) - } - }) - - assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) - assertSpanStr(t, attrs, ateattr.ActorNameKey, testActorID) -} - func TestValidateDeleteActorRequest(t *testing.T) { tests := []struct { name string @@ -833,144 +597,6 @@ func TestValidateDeleteActorRequest(t *testing.T) { } } -func TestDeleteActor_StatusDeleting(t *testing.T) { - ns := namespaceForTest("ns-delete-deleting") - tc := setupTest(t, ns) - defer tc.cleanup() - createTemplate(t, tc, ns) - - deletingActor := &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{ - Atespace: testAtespace, - Name: "deleting-actor", - }, - Status: ateapipb.Actor_STATUS_DELETING, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - } - if _, err := tc.persistence.CreateActor(context.Background(), deletingActor); err != nil { - t.Fatalf("CreateActor: %v", err) - } - - if _, err := tc.service.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "deleting-actor"}, - }); err != nil { - t.Fatalf("DeleteActor on STATUS_DELETING actor failed: %v", err) - } - - if _, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: "deleting-actor"}); err == nil { - t.Errorf("expected actor to be deleted, but it still exists") - } -} - -func TestDeleteActor_WrongStatus(t *testing.T) { - ns := namespaceForTest("ns-delete-wrong-status") - tc := setupTest(t, ns) - defer tc.cleanup() - createTemplate(t, tc, ns) - - runningActor := &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{ - Atespace: testAtespace, - Name: "running-actor", - }, - Status: ateapipb.Actor_STATUS_RUNNING, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - } - if _, err := tc.persistence.CreateActor(context.Background(), runningActor); err != nil { - t.Fatalf("CreateActor: %v", err) - } - - _, err := tc.service.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "running-actor"}, - }) - if err == nil { - t.Fatalf("expected DeleteActor on STATUS_RUNNING actor to fail, but it succeeded") - } -} - -type failingVolumePlugin struct { - volume.VolumePluginControlPlane - deletedIDs []string -} - -func (f *failingVolumePlugin) DeleteVolume(ctx context.Context, volumeID string) error { - f.deletedIDs = append(f.deletedIDs, volumeID) - return fmt.Errorf("simulated delete error for %s", volumeID) -} - -func TestDeleteActor_MultipleVolumeDeletionFailures(t *testing.T) { - ns := namespaceForTest("ns-delete-multivol-fail") - tc := setupTest(t, ns) - defer tc.cleanup() - createTemplate(t, tc, ns) - - plugin := &failingVolumePlugin{} - tc.service.volumePlugins = map[string]volume.VolumePluginControlPlane{ - "substrate.io/mock": plugin, - } - - actor := &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{ - Atespace: testAtespace, - Name: "multi-vol-actor", - }, - Status: ateapipb.Actor_STATUS_SUSPENDED, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - ActorVolumes: []*ateapipb.ExternalVolume{ - {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, - {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, - }, - } - if _, err := tc.persistence.CreateActor(context.Background(), actor); err != nil { - t.Fatalf("CreateActor: %v", err) - } - - _, err := tc.service.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "multi-vol-actor"}, - }) - if err == nil { - t.Fatalf("expected DeleteActor to fail when volume deletion fails, but it succeeded") - } - - wantDeleted := []string{"storage-vol-1", "storage-vol-2"} - if diff := cmp.Diff(wantDeleted, plugin.deletedIDs); diff != "" { - t.Errorf("deletedIDs mismatch (-want +got):\n%s", diff) - } - - errMsg := err.Error() - if !strings.Contains(errMsg, "storage-vol-1") || !strings.Contains(errMsg, "storage-vol-2") { - t.Errorf("expected error message to contain both volume failure details, got: %v", errMsg) - } -} - -// Pause stamps the ref identity before resolving the Actor record, so a failed -// lookup still carries who/where; it must not invent template/version, which are -// known only once the record resolves (and stamped on success). -func TestPauseActor_FailedLookupStampsRefIdentityOnly(t *testing.T) { - ns := namespaceForTest("ns-span-pause-err") - tc := setupTest(t, ns) - defer tc.cleanup() - - attrs := recordRootSpanAttrs(t, func(ctx context.Context) { - if _, err := tc.service.PauseActor(ctx, &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: testActorID}, - }); status.Code(err) != codes.NotFound { - t.Fatalf("PauseActor(missing) error = %v, want code NotFound", err) - } - }) - - assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) - assertSpanStr(t, attrs, ateattr.ActorNameKey, testActorID) - for _, k := range []attribute.Key{ateattr.ActorUIDKey, ateattr.TemplateNameKey, ateattr.TemplateNamespaceKey, ateattr.ActorVersionKey} { - if _, ok := attrs[k]; ok { - t.Errorf("unexpected %s on failed-pause span", k) - } - } -} - func TestValidatePauseActorRequest(t *testing.T) { tests := []struct { name string @@ -1008,25 +634,6 @@ func TestValidatePauseActorRequest(t *testing.T) { } } -// The early ref stamp must land on the span even when the op fails, so a failed -// resume is still attributable to who/where. -func TestResumeActor_ErrorStillStampsRefSpanIdentity(t *testing.T) { - ns := namespaceForTest("ns-span-resume-err") - tc := setupTest(t, ns) - defer tc.cleanup() - - attrs := recordRootSpanAttrs(t, func(ctx context.Context) { - if _, err := tc.service.ResumeActor(ctx, &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "missing"}, - }); err == nil { - t.Fatal("expected error resuming missing actor") - } - }) - - assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) - assertSpanStr(t, attrs, ateattr.ActorNameKey, "missing") -} - func TestValidateResumeActorRequest(t *testing.T) { tests := []struct { name string diff --git a/cmd/ateapi/internal/controlapi/common_test.go b/cmd/ateapi/internal/controlapi/common_test.go new file mode 100644 index 0000000000..70d878baf2 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/common_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "fmt" + "testing" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/testing/protocmp" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// Helpers shared by the unit tests in this package. +const ( + testAtespace = "test-atespace" + testActorID = "id1" +) + +var ( + ignoreUID = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "uid") + ignoreTimestamps = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "create_time", "update_time") +) + +func selectorLabelsOfSize(n int) map[string]string { + labels := make(map[string]string, n) + for i := 0; i < n; i++ { + labels[fmt.Sprintf("k%d", i)] = "v" + } + return labels +} + +func assertValidateErr(t *testing.T, got field.ErrorList, want field.ErrorList) { + t.Helper() + field.ErrorMatcher{}.ByType().ByField().ByValue().Test(t, want, got) +} diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index 4944a2364b..43d6c4b865 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -64,10 +64,20 @@ type AteletDialer struct { dialCredentials func(expectedPodUID string) (credentials.TransportCredentials, error) } +// DialerOption customizes an AteletDialer built by NewAteletDialer. +type DialerOption func(*AteletDialer) + +// WithDialCredentials overrides how transport credentials are built for a given +// atelet pod UID. Tests use it to reach a fake atelet over insecure transport +// while still exercising the real lookup, dial and connection-cache path. +func WithDialCredentials(build func(expectedPodUID string) (credentials.TransportCredentials, error)) DialerOption { + return func(d *AteletDialer) { d.dialCredentials = build } +} + // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet connection. -func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string) *AteletDialer { - return &AteletDialer{ +func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, clientBundlePath, serverCAPath string, opts ...DialerOption) *AteletDialer { + d := &AteletDialer{ workerIndexer: workerIndexer, ateletIndexer: ateletIndexer, ateletConns: lru.New(1024), @@ -79,6 +89,10 @@ func NewAteletDialer(workerIndexer cache.Indexer, ateletIndexer cache.Indexer, c return credentials.NewTLS(tlsConfig), nil }, } + for _, opt := range opts { + opt(d) + } + return d } // DialForWorker returns a gRPC connection to the Atelet running on the same node as the specified worker pod. diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_snapshot_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_snapshot_test.go new file mode 100644 index 0000000000..b523e1f8f4 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_snapshot_test.go @@ -0,0 +1,115 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package functionaltest + +import ( + "context" + "fmt" + "testing" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/fieldmaskpb" +) + +// TestUpdateActorSnapshotTag_Preconditions verifies the optional version and uid +// guards carried in the tag's metadata. +func TestUpdateActorSnapshotTag_Preconditions(t *testing.T) { + ns := namespaceForTest("ns-update-tag-preconditions") + tc := setupTest(t, ns) + defer tc.cleanup() + + createTemplate(t, tc, ns) + + ctx := context.Background() + const snapshotName, tagName = "snapshot-1", "before-upgrade" + snapshotRef := createActorSnapshot(t, tc, snapshotName) + + // Each call to update() flips the scope, so every accepted update is an + // observable write that bumps the version. + update := func(meta *ateapipb.ResourceMetadata, scope ateapipb.ActorSnapshotTagScope) (*ateapipb.ActorSnapshotTag, error) { + return updateActorSnapshotTagScope(tc, tagName, meta, scope) + } + + // Delete and recreate the same atespace/name tag, so the first lifecycle's + // uid becomes stale. + staleUID := tagActorSnapshot(t, tc, snapshotRef, tagName).GetMetadata().GetUid() + if _, err := tc.client.DeleteActorSnapshotTag(ctx, &ateapipb.DeleteActorSnapshotTagRequest{ + Tag: &ateapipb.ObjectRef{Atespace: testAtespace, Name: tagName}, + }); err != nil { + t.Fatalf("DeleteActorSnapshotTag failed: %v", err) + } + + tagged := tagActorSnapshot(t, tc, snapshotRef, tagName) + staleVersion := tagged.GetMetadata().GetVersion() + uid := tagged.GetMetadata().GetUid() + if uid == staleUID { + t.Fatalf("recreated tag reused uid %s, want a fresh one", uid) + } + // The uid from the deleted lifecycle must be rejected, even though the + // atespace/name it was observed under still resolves. + _, err := update(&ateapipb.ResourceMetadata{Uid: staleUID}, ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED) + assertGrpcError(t, err, codes.Aborted, fmt.Sprintf("ActorSnapshot tag %s/%s not found with uid %s", testAtespace, tagName, staleUID)) + + // An unguarded update is last-writer-wins, and moves the tag past the + // version observed above. + unguarded, err := update(&ateapipb.ResourceMetadata{}, ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED) + if err != nil { + t.Fatalf("UpdateActorSnapshotTag(no guards) failed: %v", err) + } + currentVersion := unguarded.GetMetadata().GetVersion() + if currentVersion <= staleVersion { + t.Fatalf("version = %d, want greater than %d after an update", currentVersion, staleVersion) + } + if got, want := unguarded.GetScope(), ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED; got != want { + t.Errorf("scope = %v, want %v", got, want) + } + + // The version observed before that write is now stale: rejected rather than + // silently overwriting the concurrent change. + _, err = update(&ateapipb.ResourceMetadata{Version: staleVersion}, ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE) + assertGrpcError(t, err, codes.Aborted, "concurrent update conflict, please retry") + + // Both uid and version matching the observed state: the update goes through. + updated, err := update(&ateapipb.ResourceMetadata{Uid: uid, Version: currentVersion}, ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE) + if err != nil { + t.Fatalf("UpdateActorSnapshotTag(matching guards) failed: %v", err) + } + if got, want := updated.GetScope(), ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE; got != want { + t.Errorf("scope = %v, want %v", got, want) + } + if updated.GetMetadata().GetVersion() <= currentVersion { + t.Errorf("version = %d, want greater than %d", updated.GetMetadata().GetVersion(), currentVersion) + } + + // The guard the client just satisfied is now stale in turn. + _, err = update(&ateapipb.ResourceMetadata{Version: currentVersion}, ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED) + assertGrpcError(t, err, codes.Aborted, "concurrent update conflict, please retry") +} + +func TestUpdateActorSnapshotTag_NotFound(t *testing.T) { + ns := namespaceForTest("ns-update-tag-notfound") + tc := setupTest(t, ns) + defer tc.cleanup() + + _, err := tc.client.UpdateActorSnapshotTag(context.Background(), &ateapipb.UpdateActorSnapshotTagRequest{ + Tag: &ateapipb.ActorSnapshotTag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "does-not-exist"}, + Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED, + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"scope"}}, + }) + assertGrpcError(t, err, codes.NotFound, "ActorSnapshot tag test-atespace/does-not-exist not found") +} diff --git a/cmd/ateapi/internal/controlapi/functional_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go similarity index 70% rename from cmd/ateapi/internal/controlapi/functional_test.go rename to cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 0087b0da1d..d57a1871a3 100644 --- a/cmd/ateapi/internal/controlapi/functional_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -12,1683 +12,1535 @@ // See the License for the specific language governing permissions and // limitations under the License. -package controlapi +package functionaltest import ( "context" "fmt" - "log" - "net" - "os" - "regexp" "strings" "sync" "testing" "time" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/ateredis" - "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" - "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/ateattr" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/resources" - "github.com/agent-substrate/substrate/internal/testenv" "github.com/agent-substrate/substrate/internal/volume" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" - "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" - listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" - "github.com/alicebob/miniredis/v2" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/redis/go-redis/v9" - sdkmetric "go.opentelemetry.io/otel/sdk/metric" - "google.golang.org/grpc" + "go.opentelemetry.io/otel/attribute" "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/fieldmaskpb" "google.golang.org/protobuf/types/known/timestamppb" - corev1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/utils/ptr" ) -var ( - cfg *rest.Config - fakeAtelet = &FakeAteletServer{} -) - -const ( - testAtespace = "test-atespace" - testActorID = "id1" -) - -var ( - ignoreUID = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "uid") - ignoreVersion = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "version") - ignoreTimestamps = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "create_time", "update_time") -) +// TestCreateActor_Success tests the happy path for creating an actor. +// Workflow: +// 1. Creates a mock ActorTemplate in the test namespace. +// 2. Calls CreateActor RPC. +// 3. Verifies that the actor is successfully created and returned in the response with a generated ID. +func TestCreateActor_Success(t *testing.T) { + ns := namespaceForTest("ns-create-success") + tc := setupTest(t, ns) + defer tc.cleanup() -func TestMain(m *testing.M) { - var stopEnv func() - cfg, stopEnv = testenv.Start() + createTemplate(t, tc, ns) - // Create ate-system namespace - k8sClient, err := kubernetes.NewForConfig(cfg) + createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: testAtespace, + Name: "id1", + Uid: "caller-supplied-uid", + Version: 999, + CreateTime: timestamppb.New(time.Unix(1, 0)), + UpdateTime: timestamppb.New(time.Unix(1, 0)), + }, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": "free"}}, + Status: ateapipb.Actor_STATUS_RUNNING, + }}) if err != nil { - log.Fatalf("kubernetes.NewForConfig: %v", err) - } - _, err = k8sClient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "ate-system"}, - }, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - log.Fatalf("create ate-system namespace: %v", err) + t.Fatalf("CreateActor failed: %v", err) } - // Create StorageClasses for volume tests - _, err = k8sClient.StorageV1().StorageClasses().Create(context.Background(), &storagev1.StorageClass{ - ObjectMeta: metav1.ObjectMeta{Name: "standard"}, - Provisioner: "substrate.io/mock", - }, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - log.Fatalf("create standard storage class: %v", err) - } - _, err = k8sClient.StorageV1().StorageClasses().Create(context.Background(), &storagev1.StorageClass{ - ObjectMeta: metav1.ObjectMeta{Name: "fast"}, - Provisioner: "substrate.io/mock", - }, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - log.Fatalf("create fast storage class: %v", err) + want := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 1}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + Status: ateapipb.Actor_STATUS_SUSPENDED, + WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": "free"}}, } - // Create shared Atelet Pod - ateletPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "atelet-shared", - Namespace: "ate-system", - Labels: map[string]string{ - "app": "atelet", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node1", - Containers: []corev1.Container{ - {Name: "main", Image: "nginx"}, - }, - }, + // The diff below ignores the server-assigned uid/timestamps (non-deterministic), + // so assert they are populated separately — and that uid is server-generated, + // not the caller-supplied value. + md := createResp.GetMetadata() + if md.GetUid() == "" { + t.Errorf("CreateActor response missing server-assigned uid") } - createdAtelet, err := k8sClient.CoreV1().Pods("ate-system").Create(context.Background(), ateletPod, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - log.Fatalf("create atelet pod: %v", err) + if md.GetUid() == "caller-supplied-uid" { + t.Errorf("CreateActor echoed caller-supplied uid instead of generating one") } - if err == nil { - createdAtelet.Status.PodIPs = []corev1.PodIP{{IP: "127.0.0.1"}} - createdAtelet.Status.Phase = corev1.PodRunning - _, err = k8sClient.CoreV1().Pods("ate-system").UpdateStatus(context.Background(), createdAtelet, metav1.UpdateOptions{}) - if err != nil { - log.Fatalf("update atelet pod status: %v", err) - } + if md.GetCreateTime() == nil { + t.Errorf("CreateActor response missing create_time") } - - // Start Fake Atelet Server on port 8085 - ateletGrpcServer := grpc.NewServer() - ateletpb.RegisterAteomHerderServer(ateletGrpcServer, fakeAtelet) - ateletLis, err := net.Listen("tcp", "127.0.0.1:8085") - if err != nil { - log.Fatalf("listen on 127.0.0.1:8085: %v", err) + if md.GetUpdateTime() == nil { + t.Errorf("CreateActor response missing update_time") } - go func() { - if err := ateletGrpcServer.Serve(ateletLis); err != nil { - fmt.Printf("atelet grpc server exited: %v\n", err) - } - }() - - code := m.Run() - - ateletGrpcServer.Stop() - - stopEnv() - os.Exit(code) + if diff := cmp.Diff(want, createResp, protocmp.Transform(), ignoreUID, ignoreTimestamps); diff != "" { + t.Errorf("CreateActor response mismatch (-want +got):\n%s", diff) + } } -// FakeAteletServer implements ateletpb.WorkersServer -type FakeAteletServer struct { - ateletpb.UnimplementedAteomHerderServer - - Lock sync.Mutex +func TestCreateActor_WithExternalVolumes(t *testing.T) { + ns := namespaceForTest("ns-create-ext-vols") + tc := setupTest(t, ns) + defer tc.cleanup() - RunCalled bool - RunRequest *ateletpb.RunRequest - FailRun error + volumes := []atev1alpha1.Volume{ + { + Name: "ext-vol-1", + VolumeSource: atev1alpha1.VolumeSource{ + ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ + StorageClassName: "standard", + Capacity: resource.MustParse("10Gi"), + }, + }, + }, + } + mounts := []atev1alpha1.VolumeMount{ + { + Name: "ext-vol-1", + MountPath: "/data", + }, + } + createTemplateWithVolumes(t, tc, ns, volumes, mounts) - CheckpointCalled bool - CheckpointRequest *ateletpb.CheckpointRequest + createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "vol-actor-1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }, + }) + if err != nil { + t.Fatalf("CreateActor failed: %v", err) + } - RestoreCalled bool - RestoreRequest *ateletpb.RestoreRequest - FailRestore error - RestoreDelay time.Duration + if len(createResp.GetActorVolumes()) != 1 { + t.Fatalf("expected 1 volume in CreateActor response, got %d", len(createResp.GetActorVolumes())) + } + vol := createResp.GetActorVolumes()[0] + if vol.GetVolumeName() != "ext-vol-1" { + t.Errorf("volume name = %q, want %q", vol.GetVolumeName(), "ext-vol-1") + } + if vol.GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { + t.Errorf("volume status = %v, want %v", vol.GetStatus(), ateapipb.ExternalVolume_STATUS_PENDING) + } + if vol.GetStorageVolumeId() != "" { + t.Errorf("expected empty storageVolumeId before resume, got %q", vol.GetStorageVolumeId()) + } - UploadCalled bool - UploadRequest *ateletpb.UploadPausedCheckpointRequest - FailUpload error + // Verify GetActor returns the same external volume state + getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "vol-actor-1"}, + }) + if err != nil { + t.Fatalf("GetActor failed: %v", err) + } + if len(getResp.GetActorVolumes()) != 1 { + t.Fatalf("expected 1 volume in GetActor response, got %d", len(getResp.GetActorVolumes())) + } + if getResp.GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { + t.Errorf("GetActor status = %v, want %v", getResp.GetActorVolumes()[0].GetStatus(), ateapipb.ExternalVolume_STATUS_PENDING) + } } -func (f *FakeAteletServer) Reset() { - f.Lock.Lock() - defer f.Lock.Unlock() - - f.RunCalled = false - f.RunRequest = nil - f.FailRun = nil - - f.CheckpointCalled = false - f.CheckpointRequest = nil - - f.RestoreCalled = false - f.RestoreRequest = nil - f.FailRestore = nil - f.RestoreDelay = 0 +// TestCreateActor_TemplateNotFound tests that creating an actor with a non-existent template fails with FailedPrecondition. +func TestCreateActor_TemplateNotFound(t *testing.T) { + ns := namespaceForTest("ns-create-notfound") + tc := setupTest(t, ns) + defer tc.cleanup() - f.UploadCalled = false - f.UploadRequest = nil - f.FailUpload = nil + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "non-existent", + }}) + assertGrpcError(t, err, codes.FailedPrecondition, fmt.Sprintf("ActorTemplate %s/non-existent not found", ns)) } -func (f *FakeAteletServer) UploadPausedCheckpoint(ctx context.Context, req *ateletpb.UploadPausedCheckpointRequest) (*ateletpb.UploadPausedCheckpointResponse, error) { - f.Lock.Lock() - defer f.Lock.Unlock() - - f.UploadCalled = true - f.UploadRequest = proto.Clone(req).(*ateletpb.UploadPausedCheckpointRequest) - if f.FailUpload != nil { - return nil, f.FailUpload - } - return &ateletpb.UploadPausedCheckpointResponse{}, nil -} +// TestCreateActor_Duplicate tests that creating an actor with an existing ID fails. +func TestCreateActor_Duplicate(t *testing.T) { + ns := namespaceForTest("ns-create-dup") + tc := setupTest(t, ns) + defer tc.cleanup() -func (f *FakeAteletServer) Run(ctx context.Context, req *ateletpb.RunRequest) (*ateletpb.RunResponse, error) { - f.Lock.Lock() - defer f.Lock.Unlock() + createTemplate(t, tc, ns) - f.RunCalled = true - f.RunRequest = proto.Clone(req).(*ateletpb.RunRequest) - if f.FailRun != nil { - return nil, f.FailRun + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}) + if err != nil { + t.Fatalf("first CreateActor failed: %v", err) } - return &ateletpb.RunResponse{}, nil + _, err = tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}) + assertGrpcError(t, err, codes.AlreadyExists, "Actor id1 already exists") } -func (f *FakeAteletServer) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest) (*ateletpb.CheckpointResponse, error) { - f.Lock.Lock() - defer f.Lock.Unlock() - - f.CheckpointCalled = true - f.CheckpointRequest = proto.Clone(req).(*ateletpb.CheckpointRequest) - - return &ateletpb.CheckpointResponse{}, nil -} +// CreateActor is the only lifecycle op with the full identity (incl. version) +// available in the request, so the whole ate.* set should land on its span. +func TestCreateActor_StampsFullSpanIdentity(t *testing.T) { + ns := namespaceForTest("ns-span-create") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) -func (f *FakeAteletServer) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (*ateletpb.RestoreResponse, error) { - f.Lock.Lock() - defer f.Lock.Unlock() + attrs := recordRootSpanAttrs(t, func(ctx context.Context) { + if _, err := tc.service.CreateActor(ctx, &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }, + }); err != nil { + t.Fatalf("CreateActor: %v", err) + } + }) - f.RestoreCalled = true - f.RestoreRequest = proto.Clone(req).(*ateletpb.RestoreRequest) - if f.RestoreDelay > 0 { - time.Sleep(f.RestoreDelay) + assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) + assertSpanStr(t, attrs, ateattr.ActorNameKey, testActorID) + assertSpanStr(t, attrs, ateattr.TemplateNameKey, "tmpl1") + assertSpanStr(t, attrs, ateattr.TemplateNamespaceKey, ns) + // uid is server-assigned on create, so assert it is present and non-empty + // rather than a fixed value. + if v, ok := attrs[ateattr.ActorUIDKey]; !ok || v.Type() != attribute.STRING || v.AsString() == "" { + t.Errorf("%s = %v, want non-empty server-assigned uid", ateattr.ActorUIDKey, v.Emit()) } - if f.FailRestore != nil { - return nil, f.FailRestore + if v, ok := attrs[ateattr.ActorVersionKey]; !ok || v.Type() != attribute.INT64 || v.AsInt64() != 1 { + t.Errorf("%s = %v, want int64 1", ateattr.ActorVersionKey, v.Emit()) } - return &ateletpb.RestoreResponse{}, nil } -func (f *FakeAteletServer) lastRestoreRequest() *ateletpb.RestoreRequest { - f.Lock.Lock() - defer f.Lock.Unlock() - - if f.RestoreRequest == nil { - return nil - } - return proto.Clone(f.RestoreRequest).(*ateletpb.RestoreRequest) -} - -type testContext struct { - mr *miniredis.Miniredis - service *Service - client ateapipb.ControlClient - k8sClient kubernetes.Interface - substrateClient versioned.Interface - persistence *ateredis.Persistence - workerCache *workercache.Cache - fakeAtelet *FakeAteletServer - cleanup func() - actorTemplateLister listersv1alpha1.ActorTemplateLister - workerPoolLister listersv1alpha1.WorkerPoolLister - sandboxConfigLister listersv1alpha1.SandboxConfigLister -} +func TestCreateActor_RejectsDifferentTemplateForDataSnapshot(t *testing.T) { + ns := namespaceForTest("ns-data-snapshot-template") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) + createTemplateWithSelector(t, tc, ns, "tmpl2", nil) -// setupTest sets up a fully isolated test environment. -func setupTest(t *testing.T, ns string) *testContext { - t.Helper() - // 1. Start Miniredis - mr, err := miniredis.Run() + tmpl, err := tc.actorTemplateLister.ActorTemplates(ns).Get("tmpl1") if err != nil { - t.Fatalf("failed to start miniredis: %v", err) + t.Fatalf("Get source ActorTemplate: %v", err) } - - rdb := redis.NewClusterClient(&redis.ClusterOptions{ - Addrs: []string{mr.Addr()}, + snapshot, err := tc.persistence.CreateActorSnapshot(context.Background(), &ateapipb.ActorSnapshot{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "data-snapshot"}, + SourceActor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "source"}, + ActorTemplateUid: string(tmpl.GetUID()), + ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + SnapshotUri: "gs://snapshots/snapshots/" + testAtespace + "/data-snapshot", }) - persistence := ateredis.NewPersistence(rdb) - - // 2. Initialize Clientsets using global cfg - k8sClient, err := kubernetes.NewForConfig(cfg) - if err != nil { - mr.Close() - t.Fatalf("failed to create k8s clientset: %v", err) - } - - substrateClient, err := versioned.NewForConfig(cfg) if err != nil { - mr.Close() - t.Fatalf("failed to create substrate clientset: %v", err) + t.Fatalf("CreateActorSnapshot: %v", err) } - - // 3. Initialize Informers - workerFactory, workerInformer := WorkerPodInformer(k8sClient) - ateletFactory, ateletInformer := AteletInformer(k8sClient) - scFactory := informers.NewSharedInformerFactory(k8sClient, 0) - scLister := scFactory.Storage().V1().StorageClasses().Lister() - - substrateInformerFactory := externalversions.NewSharedInformerFactory(substrateClient, 0) - actorTemplateLister := substrateInformerFactory.Api().V1alpha1().ActorTemplates().Lister() - workerPoolLister := substrateInformerFactory.Api().V1alpha1().WorkerPools().Lister() - sandboxConfigLister := substrateInformerFactory.Api().V1alpha1().SandboxConfigs().Lister() - csiDriverConfigLister := substrateInformerFactory.Api().V1alpha1().CSIDriverConfigs().Lister() - - ctx, cancel := context.WithCancel(context.Background()) - - syncer := NewWorkerPoolSyncer(persistence, workerInformer, workerPoolLister) - syncer.Start(ctx) - - workerFactory.Start(ctx.Done()) - ateletFactory.Start(ctx.Done()) - substrateInformerFactory.Start(ctx.Done()) - scFactory.Start(ctx.Done()) - - workerFactory.WaitForCacheSync(ctx.Done()) - ateletFactory.WaitForCacheSync(ctx.Done()) - substrateInformerFactory.WaitForCacheSync(ctx.Done()) - scFactory.WaitForCacheSync(ctx.Done()) - - // 4. Initialize Service - wc := workercache.New(persistence, 5*time.Minute) - if err := wc.Start(ctx); err != nil { - cancel() - mr.Close() - t.Fatalf("failed to start worker cache: %v", err) + if _, err := tc.persistence.CreateActorSnapshotTag(context.Background(), testAtespace, snapshot.GetMetadata().GetName(), &ateapipb.ActorSnapshotTag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "data-snapshot"}, + Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE, + }); err != nil { + t.Fatalf("CreateActorSnapshotTag: %v", err) } - dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), "", "") - // Dial the fake atelet over insecure transport instead of per-atelet mTLS, - // so DialForWorker's real lookup/dial/cache path is exercised under test. - dialer.dialCredentials = func(_ string) (credentials.TransportCredentials, error) { - return insecure.NewCredentials(), nil + _, err = tc.service.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "clone"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl2", + SourceSnapshot: &ateapipb.ActorSnapshotSource{Tag: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "data-snapshot"}}, + }, + }) + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("CreateActor status = %v, want FailedPrecondition", status.Code(err)) } +} - instruments, err := NewInstruments(sdkmetric.NewMeterProvider(sdkmetric.WithReader(sdkmetric.NewManualReader())).Meter("ateapi")) +func TestCreateActor_RejectsSnapshotWithExternalVolumes(t *testing.T) { + ns := namespaceForTest("ns-snapshot-external-volume") + tc := setupTest(t, ns) + defer tc.cleanup() + template, err := tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).Create(context.Background(), &atev1alpha1.ActorTemplate{ + ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: ns}, + Spec: atev1alpha1.ActorTemplateSpec{ + SnapshotsConfig: atev1alpha1.SnapshotsConfig{Location: "gs://snapshots"}, + Containers: []atev1alpha1.Container{{ + Name: "main", Image: "main@sha256:abc", VolumeMounts: []atev1alpha1.VolumeMount{{Name: "data", MountPath: "/data"}}, + }}, + Volumes: []atev1alpha1.Volume{{ + Name: "data", + VolumeSource: atev1alpha1.VolumeSource{ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ + Capacity: resource.MustParse("1Gi"), StorageClassName: "standard", + }}, + }}, + }, + }, metav1.CreateOptions{}) if err != nil { - cancel() - mr.Close() - t.Fatalf("failed to create metric instruments: %v", err) + t.Fatalf("Create ActorTemplate: %v", err) + } + if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { + got, err := tc.actorTemplateLister.ActorTemplates(ns).Get("tmpl1") + return err == nil && len(got.Spec.Volumes) == 1, nil + }); err != nil { + t.Fatalf("wait for ActorTemplate update: %v", err) } - mockPlugin := volume.NewMockVolumePlugin() - mockDriverName, err := mockPlugin.DriverName(ctx) + snapshot, err := tc.persistence.CreateActorSnapshot(context.Background(), &ateapipb.ActorSnapshot{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "external-volume-snapshot"}, + ActorTemplateUid: string(template.GetUID()), + SnapshotUri: "gs://snapshots/snapshots/" + testAtespace + "/external-volume-snapshot", + }) if err != nil { - t.Fatalf("failed to get mock driver name: %v", err) + t.Fatalf("CreateActorSnapshot: %v", err) } - volPlugins := map[string]volume.VolumePluginControlPlane{ - mockDriverName: mockPlugin, + tagRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "external-volume-snapshot"} + if _, err := tc.persistence.CreateActorSnapshotTag(context.Background(), testAtespace, snapshot.GetMetadata().GetName(), &ateapipb.ActorSnapshotTag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: tagRef.GetAtespace(), Name: tagRef.GetName()}, + Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE, + }); err != nil { + t.Fatalf("CreateActorSnapshotTag: %v", err) } - service := NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, scLister, dialer, instruments, "", volPlugins) - - // 5. Start REAL gRPC Server for ATE API - grpcServer := grpc.NewServer(grpc.UnaryInterceptor(ateinterceptors.ServerUnaryInterceptor)) - ateapipb.RegisterControlServer(grpcServer, service) - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - cancel() - mr.Close() - t.Fatalf("failed to listen: %v", err) + _, err = tc.service.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "clone"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + SourceSnapshot: &ateapipb.ActorSnapshotSource{Tag: tagRef}, + }, + }) + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("CreateActor status = %v, want FailedPrecondition", status.Code(err)) } +} - go func() { - if err := grpcServer.Serve(lis); err != nil { - t.Logf("grpc server exited: %v", err) - } - }() +// TestGetActor_Found tests that an existing actor can be retrieved. +func TestGetActor_Found(t *testing.T) { + ns := namespaceForTest("ns-get-found") + tc := setupTest(t, ns) + defer tc.cleanup() - conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) - if err != nil { - grpcServer.Stop() - cancel() - mr.Close() - t.Fatalf("failed to connect: %v", err) - } + createTemplate(t, tc, ns) - client := ateapipb.NewControlClient(conn) + name := "id1" - // Call Reset on global mock - fakeAtelet.Reset() + createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}) + if err != nil { + t.Fatalf("CreateActor failed: %v", err) + } - // Create namespace - _, err = k8sClient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: ns}, - }, metav1.CreateOptions{}) + getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) if err != nil { - conn.Close() - grpcServer.Stop() - cancel() - mr.Close() - t.Fatalf("failed to create namespace %s: %v", ns, err) - } - - // CreateActor now requires the atespace to exist first. - if _, err := client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: testAtespace}}}); err != nil { - conn.Close() - grpcServer.Stop() - cancel() - mr.Close() - t.Fatalf("failed to seed test atespace %q: %v", testAtespace, err) - } - - cleanup := func() { - conn.Close() - grpcServer.Stop() - cancel() - rdb.Close() - mr.Close() - } - - return &testContext{ - mr: mr, - service: service, - client: client, - k8sClient: k8sClient, - substrateClient: substrateClient, - persistence: persistence, - workerCache: wc, - fakeAtelet: fakeAtelet, - cleanup: cleanup, - actorTemplateLister: actorTemplateLister, - workerPoolLister: workerPoolLister, - sandboxConfigLister: sandboxConfigLister, + t.Fatalf("GetActor failed: %v", err) } -} -func namespaceForTest(baseName string) string { - return fmt.Sprintf("%s-%d", baseName, time.Now().UnixNano()) -} + want := createResp -func selectorLabelsOfSize(n int) map[string]string { - labels := make(map[string]string, n) - for i := 0; i < n; i++ { - labels[fmt.Sprintf("k%d", i)] = "v" + if diff := cmp.Diff(want, getResp, protocmp.Transform()); diff != "" { + t.Errorf("GetActor response mismatch (-want +got):\n%s", diff) } - return labels } -func createTemplate(t *testing.T, tc *testContext, ns string) { - t.Helper() - createTemplateWithContainers(t, tc, ns, []atev1alpha1.Container{ - { - Name: "main", - Image: "main@sha256:abc", - Command: []string{"/main"}, - }, - }) -} - -// createAtespace creates an atespace via the API. -func createAtespace(t *testing.T, tc *testContext, name string) { - t.Helper() - if _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: name}}}); err != nil { - t.Fatalf("CreateAtespace(%s) failed: %v", name, err) - } -} - -// createActorSnapshot seeds an ActorSnapshot in testAtespace directly through -// the store, so tag tests do not need a full resume/suspend lifecycle. -func createActorSnapshot(t *testing.T, tc *testContext, name string) *ateapipb.ObjectRef { - t.Helper() - if _, err := tc.persistence.CreateActorSnapshot(context.Background(), &ateapipb.ActorSnapshot{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, - SnapshotUri: "gs://my-bucket/snapshots/" + testAtespace + "/" + name, - }); err != nil { - t.Fatalf("CreateActorSnapshot(%s) failed: %v", name, err) - } - return &ateapipb.ObjectRef{Atespace: testAtespace, Name: name} -} - -// tagActorSnapshot points tagName at snapshotRef with atespace scope. -func tagActorSnapshot(t *testing.T, tc *testContext, snapshotRef *ateapipb.ObjectRef, tagName string) *ateapipb.ActorSnapshotTag { - t.Helper() - tag, err := tc.client.CreateActorSnapshotTag(context.Background(), &ateapipb.CreateActorSnapshotTagRequest{ - ActorSnapshotTag: &ateapipb.ActorSnapshotTag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: tagName}, - Snapshot: snapshotRef, - Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE, - }, - }) - if err != nil { - t.Fatalf("CreateActorSnapshotTag(%s) failed: %v", tagName, err) - } - return tag -} +// TestGetActor_NotFound tests that retrieving a non-existent actor fails. +// Workflow: +// 1. Calls GetActor RPC with a non-existent ID. +// 2. Verifies that it returns an error (NotFound). +func TestGetActor_NotFound(t *testing.T) { + ns := namespaceForTest("ns-get-notfound") + tc := setupTest(t, ns) + defer tc.cleanup() -// updateActorSnapshotTagScope sets tagName's scope, carrying meta as the -// optional uid/version preconditions. -func updateActorSnapshotTagScope(tc *testContext, tagName string, meta *ateapipb.ResourceMetadata, scope ateapipb.ActorSnapshotTagScope) (*ateapipb.ActorSnapshotTag, error) { - meta.Atespace, meta.Name = testAtespace, tagName - return tc.client.UpdateActorSnapshotTag(context.Background(), &ateapipb.UpdateActorSnapshotTagRequest{ - Tag: &ateapipb.ActorSnapshotTag{Metadata: meta, Scope: scope}, - UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"scope"}}, + _, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "non-existent"}, }) + assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/non-existent not found") } -const poolLabelKey = "pool" - -func createTemplateWithContainers(t *testing.T, tc *testContext, ns string, containers []atev1alpha1.Container) { - createTemplateWithContainersAndVolumes(t, tc, ns, containers, nil) -} - -func createTemplateWithVolumes(t *testing.T, tc *testContext, ns string, volumes []atev1alpha1.Volume, mounts []atev1alpha1.VolumeMount) { - createTemplateWithContainersAndVolumes(t, tc, ns, []atev1alpha1.Container{ - { - Name: "main", - Image: "main@sha256:abc", - Command: []string{"/main"}, - VolumeMounts: mounts, - }, - }, volumes) -} - -func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns string, containers []atev1alpha1.Container, volumes []atev1alpha1.Volume) { - t.Helper() +// TestListActors tests that all created actors can be listed. +// Workflow: +// 1. Creates a mock ActorTemplate. +// 2. Calls CreateActor twice to create two actors. +// 3. Calls ListActors RPC. +// 4. Verifies that both actors are returned in the list. +func TestListActors(t *testing.T) { + ns := namespaceForTest("ns-list-actors") + tc := setupTest(t, ns) + defer tc.cleanup() - // Sandbox binaries now live on a (cluster-scoped) SandboxConfig resolved via - // the actor's WorkerPool, not on the ActorTemplate. Create a default gvisor - // SandboxConfig so a boot-from-spec Run can resolve its assets. - ensureDefaultGvisorSandboxConfig(t, tc) - createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns}) + createTemplate(t, tc, ns) - actorTemplate := &atev1alpha1.ActorTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tmpl1", - Namespace: ns, - }, - Spec: atev1alpha1.ActorTemplateSpec{ - SnapshotsConfig: atev1alpha1.SnapshotsConfig{ - Location: "gs://fake-fake-fake", - }, - Containers: containers, - Volumes: volumes, - WorkerSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{poolLabelKey: ns}, - }, - }, - } - createdTemplate, err := tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).Create(context.Background(), actorTemplate, metav1.CreateOptions{}) + resp1, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}) if err != nil { - t.Fatalf("failed to create actor template: %v", err) + t.Fatalf("CreateActor 1 failed: %v", err) } - - const goldenSnapshot = "golden" - if _, err := tc.persistence.CreateActorSnapshot(context.Background(), &ateapipb.ActorSnapshot{ - Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: goldenSnapshot}, + resp2, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id2"}, ActorTemplateNamespace: ns, - ActorTemplateName: createdTemplate.GetName(), - ActorTemplateUid: string(createdTemplate.GetUID()), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - SnapshotUri: "gs://fake-fake-fake/snapshots/" + resources.GoldenActorAtespace + "/" + goldenSnapshot, - }); err != nil { - t.Fatalf("failed to create golden ActorSnapshot: %v", err) - } - createdTemplate.Status = atev1alpha1.ActorTemplateStatus{ - GoldenSnapshot: goldenSnapshot, - } - - _, err = tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).UpdateStatus(context.Background(), createdTemplate, metav1.UpdateOptions{}) + ActorTemplateName: "tmpl1", + }}) if err != nil { - t.Fatalf("failed to update status: %v", err) + t.Fatalf("CreateActor 2 failed: %v", err) } - // Wait for Informer cache to sync - err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - tmpl, err := tc.actorTemplateLister.ActorTemplates(ns).Get("tmpl1") - if err != nil { - return false, nil // Retry if not found in cache yet - } - return tmpl.Status.GoldenSnapshot != "", nil - }) + listResp, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{Atespace: testAtespace}) if err != nil { - t.Fatalf("failed to wait for template status update in informer: %v", err) + t.Fatalf("ListActors failed: %v", err) } -} -// testPauseImage is the pause image the default test SandboxConfig carries; -// it is what a resolved WorkloadSpec's sandbox assets should name. -const testPauseImage = "pause@sha256:abc" - -// ensureDefaultGvisorSandboxConfig creates the cluster-scoped default gvisor -// SandboxConfig (idempotently) and waits for it to appear in the lister. -func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) { - t.Helper() - const name = "gvisor-default" - sc := &atev1alpha1.SandboxConfig{ - ObjectMeta: metav1.ObjectMeta{Name: name}, - Spec: atev1alpha1.SandboxConfigSpec{ - SandboxClass: atev1alpha1.SandboxClassGvisor, - Default: true, - PauseImage: testPauseImage, - Assets: map[string]map[string]atev1alpha1.AssetFile{ - "amd64": {"runsc": { - URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc", - SHA256: "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea842b63", - }}, - "arm64": {"runsc": { - URL: "gs://gvisor/releases/nightly/2026-05-19/aarch64/runsc", - SHA256: "1ba2366ae2efceba166046f51a4104f9261c9cb72c6db8f5b3fe2dc57dea86b9", - }}, - }, - }, - } - if _, err := tc.substrateClient.ApiV1alpha1().SandboxConfigs().Create(context.Background(), sc, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { - t.Fatalf("failed to create default SandboxConfig: %v", err) - } - if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - _, err := tc.sandboxConfigLister.Get(name) - return err == nil, nil - }); err != nil { - t.Fatalf("default SandboxConfig not synced into lister: %v", err) + if len(listResp.Actors) != 2 { + t.Fatalf("expected 2 actors, got %d", len(listResp.Actors)) } -} -func createWorkerPool(t *testing.T, tc *testContext, ns string, name string, labels map[string]string) { - t.Helper() - wp := &atev1alpha1.WorkerPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ns, - Labels: labels, - }, - Spec: atev1alpha1.WorkerPoolSpec{ - Replicas: 1, - AteomImage: "ateom@sha256:abc", - }, + want := []*ateapipb.Actor{ + resp1, + resp2, } - _, err := tc.substrateClient.ApiV1alpha1().WorkerPools(ns).Create(context.Background(), wp, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("failed to create WorkerPool: %v", err) + + opts := []cmp.Option{ + protocmp.Transform(), + cmpopts.SortSlices(func(a, b *ateapipb.Actor) bool { + return a.GetMetadata().GetName() < b.GetMetadata().GetName() + }), } - err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - _, err := tc.workerPoolLister.WorkerPools(ns).Get(name) - return err == nil, nil - }) - if err != nil { - t.Fatalf("failed to wait for WorkerPool %s/%s in informer: %v", ns, name, err) + if diff := cmp.Diff(want, listResp.Actors, opts...); diff != "" { + t.Errorf("ListActors response mismatch (-want +got):\n%s", diff) } } -func createTemplateWithSelector(t *testing.T, tc *testContext, ns string, name string, selector *metav1.LabelSelector) { - t.Helper() - ensureDefaultGvisorSandboxConfig(t, tc) - actorTemplate := &atev1alpha1.ActorTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ns, - }, - Spec: atev1alpha1.ActorTemplateSpec{ - SnapshotsConfig: atev1alpha1.SnapshotsConfig{ - Location: "gs://fake-fake-fake", - }, - Containers: []atev1alpha1.Container{ - {Name: "main", Image: "main@sha256:abc", Command: []string{"/main"}}, - }, - WorkerSelector: selector, - }, - } - _, err := tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).Create(context.Background(), actorTemplate, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("failed to create actor template: %v", err) - } +// TestListActors_ByAtespace verifies create + list are scoped by atespace end to +// end through the RPC surface: an actor created with a given atespace is only +// returned by ListActors(atespace=X) and only fetched by GetActor(atespace=X). +func TestListActors_ByAtespace(t *testing.T) { + ns := namespaceForTest("ns-list-by-atespace") + tc := setupTest(t, ns) + defer tc.cleanup() - err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - _, err := tc.actorTemplateLister.ActorTemplates(ns).Get(name) - return err == nil, nil - }) - if err != nil { - t.Fatalf("failed to wait for template %s/%s in informer: %v", ns, name, err) + createTemplate(t, tc, ns) + createAtespace(t, tc, "team-a") + createAtespace(t, tc, "team-b") + + create := func(atespace, name string) *ateapipb.Actor { + resp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}) + if err != nil { + t.Fatalf("CreateActor(%s, atespace=%q) failed: %v", name, atespace, err) + } + return resp } -} + a1 := create("team-a", "id1") + a2 := create("team-a", "id2") + b1 := create("team-b", "id3") -func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, nodeName string, poolName string) { - t.Helper() - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ns, - UID: "08675309-4a65-6e6e-7973-6e756d626572", - Labels: map[string]string{ - "ate.dev/worker-pool": poolName, - }, - }, - Spec: corev1.PodSpec{ - NodeName: nodeName, - Containers: []corev1.Container{ - {Name: "main", Image: "nginx"}, - }, - }, + sortByID := []cmp.Option{ + protocmp.Transform(), + cmpopts.SortSlices(func(a, b *ateapipb.Actor) bool { return a.GetMetadata().GetName() < b.GetMetadata().GetName() }), } - /* - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: ns, - UID: "08675309-4a65-6e6e-7973-6e756d626572", - Labels: map[string]string{ - workerPodLabel: poolName, - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node1", - Containers: []corev1.Container{{Name: "main", Image: "nginx"}}, - }, - } - */ - createdPod, err := tc.k8sClient.CoreV1().Pods(ns).Create(context.Background(), pod, metav1.CreateOptions{}) + // List scoped to team-a returns only its actors. + listA, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{Atespace: "team-a"}) if err != nil { - t.Fatalf("failed to create worker pod: %v", err) + t.Fatalf("ListActors(team-a) failed: %v", err) } - createdPod.Status.PodIPs = []corev1.PodIP{{IP: "127.0.0.1"}} - createdPod.Status.Phase = corev1.PodRunning - _, err = tc.k8sClient.CoreV1().Pods(ns).UpdateStatus(context.Background(), createdPod, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("failed to update worker pod status: %v", err) + if diff := cmp.Diff([]*ateapipb.Actor{a1, a2}, listA.GetActors(), sortByID...); diff != "" { + t.Errorf("ListActors(team-a) mismatch (-want +got):\n%s", diff) } - // Wait for worker to be registered via API - err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - resp, err := tc.client.ListWorkers(ctx, &ateapipb.ListWorkersRequest{}) - if err != nil { - return false, nil // Retry on API error - } - for _, w := range resp.GetWorkers() { - if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { - return true, nil - } - } - return false, nil - }) + // List scoped to team-b returns only its actor. + listB, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{Atespace: "team-b"}) if err != nil { - t.Fatalf("failed to wait for worker to be registered: %v", err) + t.Fatalf("ListActors(team-b) failed: %v", err) + } + if diff := cmp.Diff([]*ateapipb.Actor{b1}, listB.GetActors(), sortByID...); diff != "" { + t.Errorf("ListActors(team-b) mismatch (-want +got):\n%s", diff) } - // Wait for the worker to appear in worker cache. - err = wait.PollUntilContextTimeout(context.Background(), 10*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - workers, err := tc.workerCache.Workers() - if err != nil { - return false, nil // Cache not ready yet; retry. - } - for _, w := range workers { - if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { - return true, nil - } - } - return false, nil - }) - if err != nil { - t.Fatalf("failed to wait for worker to appear in worker cache: %v", err) + // Get is scoped: the right atespace hits, the empty atespace misses (deny-across by key). + if _, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "id1"}}); err != nil { + t.Errorf("GetActor(id1, team-a) failed: %v", err) } + _, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}) + assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found") } -func deleteWorkerPod(t *testing.T, tc *testContext, ns string, name string) { - t.Helper() - err := tc.k8sClient.CoreV1().Pods(ns).Delete(context.Background(), name, metav1.DeleteOptions{ - GracePeriodSeconds: ptr.To[int64](0), - }) - if err != nil { - t.Fatalf("failed to delete worker pod %s: %v", name, err) - } +// TestListActors_AllAtespaces verifies that an empty atespace lists actors across +// all atespaces (the `-A` / admin view), unlike the scoped single-atespace listing. +func TestListActors_AllAtespaces(t *testing.T) { + ns := namespaceForTest("ns-list-all-atespaces") + tc := setupTest(t, ns) + defer tc.cleanup() - // Wait for worker to be removed from API - err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - resp, err := tc.client.ListWorkers(ctx, &ateapipb.ListWorkersRequest{}) - if err != nil { - return false, nil // Retry on API error - } - for _, w := range resp.GetWorkers() { - if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { - return false, nil // Still there - } + createTemplate(t, tc, ns) + createAtespace(t, tc, "team-a") + createAtespace(t, tc, "team-b") + + create := func(atespace, name string) { + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}); err != nil { + t.Fatalf("CreateActor(%s, atespace=%q) failed: %v", name, atespace, err) } - return true, nil // Gone! - }) - if err != nil { - t.Fatalf("failed to wait for worker to be removed: %v", err) } + create("team-a", "id1") + create("team-b", "id2") - err = wait.PollUntilContextTimeout(context.Background(), 10*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { - workers, err := tc.workerCache.Workers() - if err != nil { - return false, nil // Cache not ready yet; retry. - } - for _, w := range workers { - if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { - return false, nil // Still there - } - } - return true, nil - }) + // Empty atespace lists across all atespaces; returned actors carry their atespace. + resp, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{}) if err != nil { - t.Fatalf("failed to wait for worker to be removed from worker cache: %v", err) + t.Fatalf("ListActors(all) failed: %v", err) + } + got := map[string]string{} + for _, a := range resp.GetActors() { + got[a.GetMetadata().GetName()] = a.GetMetadata().GetAtespace() + } + if got["id1"] != "team-a" { + t.Errorf("ListActors(all): got[id1]=%q, want team-a", got["id1"]) + } + if got["id2"] != "team-b" { + t.Errorf("ListActors(all): got[id2]=%q, want team-b", got["id2"]) } } -// TestCreateActor_Success tests the happy path for creating an actor. -// Workflow: -// 1. Creates a mock ActorTemplate in the test namespace. -// 2. Calls CreateActor RPC. -// 3. Verifies that the actor is successfully created and returned in the response with a generated ID. -func TestCreateActor_Success(t *testing.T) { - ns := namespaceForTest("ns-create-success") +// TestListActors_Pagination tests that ListActors correctly paginates results. +func TestListActors_Pagination(t *testing.T) { + ns := namespaceForTest("ns-list-actors-pagination") tc := setupTest(t, ns) defer tc.cleanup() createTemplate(t, tc, ns) - createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{ - Atespace: testAtespace, - Name: "id1", - Uid: "caller-supplied-uid", - Version: 999, - CreateTime: timestamppb.New(time.Unix(1, 0)), - UpdateTime: timestamppb.New(time.Unix(1, 0)), - }, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": "free"}}, - Status: ateapipb.Actor_STATUS_RUNNING, - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - - want := &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 1}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - Status: ateapipb.Actor_STATUS_SUSPENDED, - WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": "free"}}, + var want []*ateapipb.Actor + for i := 0; i < 5; i++ { + resp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: fmt.Sprintf("name%d", i)}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}) + if err != nil { + t.Fatalf("CreateActor %d failed: %v", i, err) + } + want = append(want, resp) } - // The diff below ignores the server-assigned uid/timestamps (non-deterministic), - // so assert they are populated separately — and that uid is server-generated, - // not the caller-supplied value. - md := createResp.GetMetadata() - if md.GetUid() == "" { - t.Errorf("CreateActor response missing server-assigned uid") - } - if md.GetUid() == "caller-supplied-uid" { - t.Errorf("CreateActor echoed caller-supplied uid instead of generating one") + var allActors []*ateapipb.Actor + pageToken := "" + + for { + listResp, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{ + Atespace: testAtespace, + PageSize: 2, + PageToken: pageToken, + }) + if err != nil { + t.Fatalf("ListActors failed: %v", err) + } + + allActors = append(allActors, listResp.Actors...) + pageToken = listResp.GetNextPageToken() + if pageToken == "" { + break + } } - if md.GetCreateTime() == nil { - t.Errorf("CreateActor response missing create_time") + + if len(allActors) != 5 { + t.Fatalf("expected 5 actors total, got %d", len(allActors)) } - if md.GetUpdateTime() == nil { - t.Errorf("CreateActor response missing update_time") + + opts := []cmp.Option{ + protocmp.Transform(), + cmpopts.SortSlices(func(a, b *ateapipb.Actor) bool { + return a.GetMetadata().GetName() < b.GetMetadata().GetName() + }), } - if diff := cmp.Diff(want, createResp, protocmp.Transform(), ignoreUID, ignoreTimestamps); diff != "" { - t.Errorf("CreateActor response mismatch (-want +got):\n%s", diff) + if diff := cmp.Diff(want, allActors, opts...); diff != "" { + t.Errorf("ListActors pagination response mismatch (-want +got):\n%s", diff) } } -func TestCreateActor_WithExternalVolumes(t *testing.T) { - ns := namespaceForTest("ns-create-ext-vols") +// TestUpdateActor_Success verifies UpdateActor replaces the actor's +// worker_selector and that the change is durably persisted. +func TestUpdateActor_Success(t *testing.T) { + ns := namespaceForTest("ns-update-actor") tc := setupTest(t, ns) defer tc.cleanup() - volumes := []atev1alpha1.Volume{ - { - Name: "ext-vol-1", - VolumeSource: atev1alpha1.VolumeSource{ - ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ - StorageClassName: "standard", - Capacity: resource.MustParse("10Gi"), - }, - }, - }, - } - mounts := []atev1alpha1.VolumeMount{ - { - Name: "ext-vol-1", - MountPath: "/data", + createTemplate(t, tc, ns) + + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + WorkerSelector: &ateapipb.Selector{ + MatchLabels: map[string]string{"tier": "free"}, }, + }}) + if err != nil { + t.Fatalf("CreateActor failed: %v", err) } - createTemplateWithVolumes(t, tc, ns, volumes, mounts) - createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + updateResp, err := tc.client.UpdateActor(context.Background(), &ateapipb.UpdateActorRequest{ Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "vol-actor-1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + WorkerSelector: &ateapipb.Selector{ + MatchLabels: map[string]string{"tier": "paid"}, + }, + // Output-only fields outside the mask are ignored. + Status: ateapipb.Actor_STATUS_RUNNING, }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"worker_selector"}}, }) if err != nil { - t.Fatalf("CreateActor failed: %v", err) + t.Fatalf("UpdateActor failed: %v", err) } - if len(createResp.GetActorVolumes()) != 1 { - t.Fatalf("expected 1 volume in CreateActor response, got %d", len(createResp.GetActorVolumes())) - } - vol := createResp.GetActorVolumes()[0] - if vol.GetVolumeName() != "ext-vol-1" { - t.Errorf("volume name = %q, want %q", vol.GetVolumeName(), "ext-vol-1") - } - if vol.GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { - t.Errorf("volume status = %v, want %v", vol.GetStatus(), ateapipb.ExternalVolume_STATUS_PENDING) + wantActor := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + Status: ateapipb.Actor_STATUS_SUSPENDED, + WorkerSelector: &ateapipb.Selector{ + MatchLabels: map[string]string{"tier": "paid"}, + }, } - if vol.GetStorageVolumeId() != "" { - t.Errorf("expected empty storageVolumeId before resume, got %q", vol.GetStorageVolumeId()) + if diff := cmp.Diff(wantActor, updateResp, protocmp.Transform(), ignoreUID, ignoreTimestamps); diff != "" { + t.Errorf("UpdateActor response mismatch (-want +got):\n%s", diff) } - // Verify GetActor returns the same external volume state - getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "vol-actor-1"}, - }) + getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}) if err != nil { t.Fatalf("GetActor failed: %v", err) } - if len(getResp.GetActorVolumes()) != 1 { - t.Fatalf("expected 1 volume in GetActor response, got %d", len(getResp.GetActorVolumes())) - } - if getResp.GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { - t.Errorf("GetActor status = %v, want %v", getResp.GetActorVolumes()[0].GetStatus(), ateapipb.ExternalVolume_STATUS_PENDING) + wantGetResp := wantActor + if diff := cmp.Diff(wantGetResp, getResp, protocmp.Transform(), ignoreUID, ignoreTimestamps); diff != "" { + t.Errorf("GetActor response mismatch after UpdateActor (-want +got):\n%s", diff) } } -func TestActorLifecycle_WithExternalVolumes(t *testing.T) { - ns := namespaceForTest("ns-lifecycle-ext-vols") +// TestUpdateActor_Preconditions verifies the optional version and uid guards +// carried in the embedded resource's metadata. +func TestUpdateActor_Preconditions(t *testing.T) { + ns := namespaceForTest("ns-update-preconditions") tc := setupTest(t, ns) defer tc.cleanup() - volumes := []atev1alpha1.Volume{ - { - Name: "data-vol", - VolumeSource: atev1alpha1.VolumeSource{ - ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ - StorageClassName: "fast", - Capacity: resource.MustParse("20Gi"), - }, - }, - }, - } - mounts := []atev1alpha1.VolumeMount{ - { - Name: "data-vol", - MountPath: "/mnt/data", - }, - } - createTemplateWithVolumes(t, tc, ns, volumes, mounts) - createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + createTemplate(t, tc, ns) - // 1. CreateActor - createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "actor-vol-lc"}, + ctx := context.Background() + createActor := func() *ateapipb.Actor { + t.Helper() + actor, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}, ActorTemplateNamespace: ns, ActorTemplateName: "tmpl1", - }, - }) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) + }}) + if err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + return actor } - if createResp.GetStatus() != ateapipb.Actor_STATUS_SUSPENDED { - t.Fatalf("expected initial status STATUS_SUSPENDED, got %v", createResp.GetStatus()) + + update := func(meta *ateapipb.ResourceMetadata, tier string) (*ateapipb.Actor, error) { + meta.Atespace, meta.Name = testAtespace, testActorID + return tc.client.UpdateActor(ctx, &ateapipb.UpdateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: meta, + WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": tier}}, + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"worker_selector"}}, + }) } - if len(createResp.GetActorVolumes()) != 1 || createResp.GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { - t.Fatalf("expected 1 pending volume after CreateActor, got %v", createResp.GetActorVolumes()) + + // Delete and recreate the same atespace/name actor, so the first lifecycle's uid + // becomes stale. + staleUID := createActor().GetMetadata().GetUid() + if _, err := tc.client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: testActorID}, + }); err != nil { + t.Fatalf("DeleteActor failed: %v", err) } - // 2. ResumeActor - resumeResp, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, - }) - if err != nil { - t.Fatalf("ResumeActor failed: %v", err) + created := createActor() + staleVersion := created.GetMetadata().GetVersion() + uid := created.GetMetadata().GetUid() + if uid == staleUID { + t.Fatalf("recreated actor reused uid %s, want a fresh one", uid) } - if resumeResp.GetActor().GetStatus() != ateapipb.Actor_STATUS_RUNNING { - t.Fatalf("expected status STATUS_RUNNING after resume, got %v", resumeResp.GetActor().GetStatus()) + // The uid from the deleted lifecycle must be rejected, even though the + // atespace/name it was observed under still resolves. + _, err := update(&ateapipb.ResourceMetadata{Uid: staleUID}, "other-lifecycle") + assertGrpcError(t, err, codes.Aborted, fmt.Sprintf("actor %s/%s not found with uid %s", testAtespace, testActorID, staleUID)) + + // An unguarded update is last-writer-wins, and moves the resource past the + // version observed above. + unguarded, err := update(&ateapipb.ResourceMetadata{}, "free") + if err != nil { + t.Fatalf("UpdateActor(no guards) failed: %v", err) } - if len(resumeResp.GetActor().GetActorVolumes()) != 1 || resumeResp.GetActor().GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED { - t.Fatalf("expected 1 created volume after ResumeActor, got %v", resumeResp.GetActor().GetActorVolumes()) + currentVersion := unguarded.GetMetadata().GetVersion() + if currentVersion <= staleVersion { + t.Fatalf("version = %d, want greater than %d after an update", currentVersion, staleVersion) } - if resumeResp.GetActor().GetActorVolumes()[0].GetStorageVolumeId() == "" { - t.Fatalf("expected non-empty storageVolumeId after ResumeActor") + if got := unguarded.GetWorkerSelector().GetMatchLabels()["tier"]; got != "free" { + t.Errorf("worker_selector[tier] = %q, want free", got) } - // 3. PauseActor - pauseResp, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, - }) + // The version observed before that write is now stale: rejected rather than + // silently overwriting the concurrent change. + _, err = update(&ateapipb.ResourceMetadata{Version: staleVersion}, "stale") + assertGrpcError(t, err, codes.Aborted, "concurrent update conflict, please retry") + + // Both uid and version matching the observed state: the update goes through. + updated, err := update(&ateapipb.ResourceMetadata{Uid: uid, Version: currentVersion}, "paid") if err != nil { - t.Fatalf("PauseActor failed: %v", err) + t.Fatalf("UpdateActor(matching guards) failed: %v", err) } - if pauseResp.GetActor().GetStatus() != ateapipb.Actor_STATUS_PAUSED { - t.Fatalf("expected status STATUS_PAUSED after pause, got %v", pauseResp.GetActor().GetStatus()) + if got := updated.GetWorkerSelector().GetMatchLabels()["tier"]; got != "paid" { + t.Errorf("worker_selector[tier] = %q, want paid", got) + } + if updated.GetMetadata().GetVersion() <= currentVersion { + t.Errorf("version = %d, want greater than %d", updated.GetMetadata().GetVersion(), currentVersion) } - // 4. ResumeActor from paused - resumeResp2, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, + // The guard the client just satisfied is now stale in turn. + _, err = update(&ateapipb.ResourceMetadata{Version: currentVersion}, "free") + assertGrpcError(t, err, codes.Aborted, "concurrent update conflict, please retry") +} + +func TestUpdateActor_NotFound(t *testing.T) { + ns := namespaceForTest("ns-update-actor-notfound") + tc := setupTest(t, ns) + defer tc.cleanup() + + _, err := tc.client.UpdateActor(context.Background(), &ateapipb.UpdateActorRequest{ + Actor: &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "does-not-exist"}}, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"worker_selector"}}, }) - if err != nil { - t.Fatalf("ResumeActor from paused failed: %v", err) + assertGrpcError(t, err, codes.NotFound, "actor test-atespace/does-not-exist not found") +} + +func TestUpdateActor_StampsFullSpanIdentity(t *testing.T) { + ns := namespaceForTest("ns-span-update") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) + + if _, err := tc.service.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }, + }); err != nil { + t.Fatalf("seed CreateActor: %v", err) } - if resumeResp2.GetActor().GetStatus() != ateapipb.Actor_STATUS_RUNNING { - t.Fatalf("expected status STATUS_RUNNING after second resume, got %v", resumeResp2.GetActor().GetStatus()) + + attrs := recordRootSpanAttrs(t, func(ctx context.Context) { + if _, err := tc.service.UpdateActor(ctx, &ateapipb.UpdateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}, + WorkerSelector: &ateapipb.Selector{ + MatchLabels: map[string]string{"env": "prod"}, + }, + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"worker_selector"}}, + }); err != nil { + t.Fatalf("UpdateActor: %v", err) + } + }) + + assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) + assertSpanStr(t, attrs, ateattr.ActorNameKey, testActorID) + assertSpanStr(t, attrs, ateattr.TemplateNameKey, "tmpl1") + assertSpanStr(t, attrs, ateattr.TemplateNamespaceKey, ns) + if v, ok := attrs[ateattr.ActorUIDKey]; !ok || v.Type() != attribute.STRING || v.AsString() == "" { + t.Errorf("%s = %v, want non-empty server-assigned uid", ateattr.ActorUIDKey, v.Emit()) } + if v, ok := attrs[ateattr.ActorVersionKey]; !ok || v.Type() != attribute.INT64 || v.AsInt64() != 2 { + t.Errorf("%s = %v, want int64 2 (updated version)", ateattr.ActorVersionKey, v.Emit()) + } +} - // 5. SuspendActor - suspendResp, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, +func TestUpdateActor_FailedLookupStampsRefIdentityOnly(t *testing.T) { + ns := namespaceForTest("ns-span-update-err") + tc := setupTest(t, ns) + defer tc.cleanup() + + attrs := recordRootSpanAttrs(t, func(ctx context.Context) { + if _, err := tc.service.UpdateActor(ctx, &ateapipb.UpdateActorRequest{ + Actor: &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}}, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"worker_selector"}}, + }); status.Code(err) != codes.NotFound { + t.Fatalf("UpdateActor(missing) error = %v, want code NotFound", err) + } }) - if err != nil { - t.Fatalf("SuspendActor failed: %v", err) + + assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) + assertSpanStr(t, attrs, ateattr.ActorNameKey, testActorID) + for _, k := range []attribute.Key{ateattr.ActorUIDKey, ateattr.TemplateNameKey, ateattr.TemplateNamespaceKey, ateattr.ActorVersionKey} { + if _, ok := attrs[k]; ok { + t.Errorf("unexpected %s on failed-update span", k) + } } - if suspendResp.GetActor().GetStatus() != ateapipb.Actor_STATUS_SUSPENDED { - t.Fatalf("expected status STATUS_SUSPENDED after suspend, got %v", suspendResp.GetActor().GetStatus()) +} + +func TestDeleteActor_Success(t *testing.T) { + ns := namespaceForTest("ns-delete-success") + tc := setupTest(t, ns) + defer tc.cleanup() + + createTemplate(t, tc, ns) + + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}) + if err != nil { + t.Fatalf("CreateActor failed: %v", err) } - // 6. DeleteActor - deleteResp, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, + deleted, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, }) if err != nil { t.Fatalf("DeleteActor failed: %v", err) } - if deleteResp.GetMetadata().GetName() != "actor-vol-lc" { - t.Errorf("deleted actor name = %q, want %q", deleteResp.GetMetadata().GetName(), "actor-vol-lc") + // DeleteActor returns the deleted resource. + if got := deleted.GetMetadata().GetName(); got != "id1" { + t.Errorf("deleted actor name = %q, want id1", got) + } + if got := deleted.GetMetadata().GetAtespace(); got != testAtespace { + t.Errorf("deleted actor atespace = %q, want %q", got, testAtespace) } - // Confirm GetActor returns NotFound after deletion _, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, }) - if status.Code(err) != codes.NotFound { - t.Errorf("GetActor after delete err = %v, want NotFound", err) - } + assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found") } -type partialFailVolumePlugin struct { - volume.VolumePluginControlPlane - deleted []string -} +func TestDeleteActor_NotSuspended(t *testing.T) { + ns := namespaceForTest("ns-delete-notsuspended") + tc := setupTest(t, ns) + defer tc.cleanup() -func (f *partialFailVolumePlugin) CreateVolume(ctx context.Context, name, capacity, driverName string, parameters map[string]string) (string, map[string]string, error) { - if strings.HasSuffix(name, "fail-vol2") { - return "", nil, fmt.Errorf("simulated volume creation failure") - } - return "storage-" + name, parameters, nil -} + createTemplate(t, tc, ns) + createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") -func (f *partialFailVolumePlugin) AttachVolume(ctx context.Context, volumeID, node string) error { - return nil -} + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}) + if err != nil { + t.Fatalf("CreateActor failed: %v", err) + } -func (f *partialFailVolumePlugin) DetachVolume(ctx context.Context, volumeID, node string) error { - return nil -} + _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, + }) + if err != nil { + t.Fatalf("ResumeActor failed: %v", err) + } -func (f *partialFailVolumePlugin) DeleteVolume(ctx context.Context, volumeID string) error { - f.deleted = append(f.deleted, volumeID) - return nil + _, err = tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, + }) + assertGrpcError(t, err, codes.FailedPrecondition, "Actor test-atespace/id1 is not in a deletable status (status: STATUS_RUNNING)") } -// TestResumeActor_VolumeCreationFailure tests that when volume provisioning fails during ResumeActor, -// successfully created volumes are saved, the actor remains in STATUS_SUSPENDED, -// and that calling DeleteActor on the suspended actor cleans up all partially created volumes. -func TestResumeActor_VolumeCreationFailure(t *testing.T) { - ns := namespaceForTest("ns-resume-vol-fail") +func TestDeleteActor_Crashed(t *testing.T) { + ns := namespaceForTest("ns-delete-crashed") tc := setupTest(t, ns) defer tc.cleanup() - volumes := []atev1alpha1.Volume{ - { - Name: "succ-vol1", - VolumeSource: atev1alpha1.VolumeSource{ - ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ - StorageClassName: "standard", - Capacity: resource.MustParse("10Gi"), - }, - }, - }, - { - Name: "fail-vol2", - VolumeSource: atev1alpha1.VolumeSource{ - ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ - StorageClassName: "standard", - Capacity: resource.MustParse("10Gi"), - }, - }, - }, + createTemplate(t, tc, ns) + + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) } - mounts := []atev1alpha1.VolumeMount{ - {Name: "succ-vol1", MountPath: "/mnt/vol1"}, - {Name: "fail-vol2", MountPath: "/mnt/vol2"}, + + actorRef := resources.ActorRef{Atespace: testAtespace, Name: "id1"} + if _, err := tc.persistence.UpdateActor(context.Background(), actorRef, func(toUpdate *ateapipb.Actor) error { + toUpdate.Status = ateapipb.Actor_STATUS_CRASHED + return nil + }); err != nil { + t.Fatalf("UpdateActor failed: %v", err) } - createTemplateWithVolumes(t, tc, ns, volumes, mounts) - // Inject a custom partial-failing VolumePlugin into global scope - // TODO this doesn't support parallelism of test cases - plugin := &partialFailVolumePlugin{} - tc.service.volumePlugins = map[string]volume.VolumePluginControlPlane{ - "substrate.io/mock": plugin, + deleted, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, + }) + if err != nil { + t.Fatalf("DeleteActor of crashed actor failed: %v", err) + } + if got := deleted.GetStatus(); got != ateapipb.Actor_STATUS_DELETING { + t.Errorf("deleted actor status = %v, want %v", got, ateapipb.Actor_STATUS_DELETING) } - // Call CreateActor RPC directly - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + _, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, + }) + assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found") +} + +func TestDeleteActor_NotFound(t *testing.T) { + ns := namespaceForTest("ns-delete-notfound") + tc := setupTest(t, ns) + defer tc.cleanup() + + _, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "non-existent"}, + }) + assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/non-existent not found") +} + +// Delete addresses the actor by ref (atespace + id) and does not resolve the +// template/version, so only the ref identity is stamped. +func TestDeleteActor_StampsRefSpanIdentity(t *testing.T) { + ns := namespaceForTest("ns-span-delete") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) + if _, err := tc.service.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "fail-actor"}, + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}, ActorTemplateNamespace: ns, ActorTemplateName: "tmpl1", }, - }) - if err != nil { - t.Fatalf("expected CreateActor to succeed, got: %v", err) + }); err != nil { + t.Fatalf("seed CreateActor: %v", err) } - // Call ResumeActor RPC, which should trigger volume provisioning and fail on fail-vol2 - _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "fail-actor"}, + attrs := recordRootSpanAttrs(t, func(ctx context.Context) { + if _, err := tc.service.DeleteActor(ctx, &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: testActorID}, + }); err != nil { + t.Fatalf("DeleteActor: %v", err) + } }) - if err == nil { - t.Fatalf("expected ResumeActor to fail due to volume creation error, but it succeeded") - } - // Verify GetActor returns the actor in STATUS_SUSPENDED status - getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "fail-actor"}, - }) - if err != nil { - t.Fatalf("GetActor failed: %v", err) + assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) + assertSpanStr(t, attrs, ateattr.ActorNameKey, testActorID) +} + +func TestDeleteActor_StatusDeleting(t *testing.T) { + ns := namespaceForTest("ns-delete-deleting") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) + + deletingActor := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: testAtespace, + Name: "deleting-actor", + }, + Status: ateapipb.Actor_STATUS_DELETING, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", } - if getResp.GetStatus() != ateapipb.Actor_STATUS_SUSPENDED { - t.Errorf("actor status = %v, want %v", getResp.GetStatus(), ateapipb.Actor_STATUS_SUSPENDED) + if _, err := tc.persistence.CreateActor(context.Background(), deletingActor); err != nil { + t.Fatalf("CreateActor: %v", err) } - actorUID := getResp.GetMetadata().GetUid() - if actorUID == "" { - t.Fatalf("expected non-empty UID on actor") + if _, err := tc.service.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "deleting-actor"}, + }); err != nil { + t.Fatalf("DeleteActor on STATUS_DELETING actor failed: %v", err) } - // Verify that succ-vol1 was updated to CREATED with a storageVolumeId, and fail-vol2 is still PENDING - if len(getResp.GetActorVolumes()) != 2 { - t.Fatalf("expected 2 volumes on actor, got %d", len(getResp.GetActorVolumes())) + if _, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: "deleting-actor"}); err == nil { + t.Errorf("expected actor to be deleted, but it still exists") } - volsByName := make(map[string]*ateapipb.ExternalVolume) - for _, v := range getResp.GetActorVolumes() { - volsByName[v.GetVolumeName()] = v - } - if v1, ok := volsByName["succ-vol1"]; !ok || v1.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v1.GetStorageVolumeId() == "" { - t.Errorf("succ-vol1 unexpected state: %v", v1) +} + +func TestDeleteActor_WrongStatus(t *testing.T) { + ns := namespaceForTest("ns-delete-wrong-status") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) + + runningActor := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: testAtespace, + Name: "running-actor", + }, + Status: ateapipb.Actor_STATUS_RUNNING, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", } - if v2, ok := volsByName["fail-vol2"]; !ok || v2.GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { - t.Errorf("fail-vol2 unexpected state: %v", v2) + if _, err := tc.persistence.CreateActor(context.Background(), runningActor); err != nil { + t.Fatalf("CreateActor: %v", err) } - // Call DeleteActor on the actor in STATUS_SUSPENDED - _, err = tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "fail-actor"}, + _, err := tc.service.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "running-actor"}, }) - if err != nil { - t.Fatalf("DeleteActor failed: %v", err) + if err == nil { + t.Fatalf("expected DeleteActor on STATUS_RUNNING actor to fail, but it succeeded") } +} - // Verify both volumes were deleted (succ-vol1 via storageID, fail-vol2 via fallback actorVolumeID) - wantDeleted := []string{ - "storage-substrate-" + actorUID + "-succ-vol1", - "substrate-" + actorUID + "-fail-vol2", +type failingVolumePlugin struct { + volume.VolumePluginControlPlane + deletedIDs []string +} + +func (f *failingVolumePlugin) DeleteVolume(ctx context.Context, volumeID string) error { + f.deletedIDs = append(f.deletedIDs, volumeID) + return fmt.Errorf("simulated delete error for %s", volumeID) +} + +func TestDeleteActor_MultipleVolumeDeletionFailures(t *testing.T) { + ns := namespaceForTest("ns-delete-multivol-fail") + plugin := &failingVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() + createTemplate(t, tc, ns) + + actor := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: testAtespace, + Name: "multi-vol-actor", + }, + Status: ateapipb.Actor_STATUS_SUSPENDED, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + ActorVolumes: []*ateapipb.ExternalVolume{ + {VolumeName: "vol1", StorageVolumeId: "storage-vol-1", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, + {VolumeName: "vol2", StorageVolumeId: "storage-vol-2", Status: ateapipb.ExternalVolume_STATUS_CREATED, VolumeType: "substrate.io/mock"}, + }, } - if diff := cmp.Diff(wantDeleted, plugin.deleted); diff != "" { - t.Errorf("deleted volume IDs mismatch (-want +got):\n%s", diff) + if _, err := tc.persistence.CreateActor(context.Background(), actor); err != nil { + t.Fatalf("CreateActor: %v", err) } - // Confirm GetActor returns NotFound after deletion - _, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "fail-actor"}, + _, err := tc.service.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "multi-vol-actor"}, }) - if status.Code(err) != codes.NotFound { - t.Errorf("GetActor after DeleteActor err = %v, want NotFound", err) + if err == nil { + t.Fatalf("expected DeleteActor to fail when volume deletion fails, but it succeeded") } -} -type retrySuccessVolumePlugin struct { - volume.VolumePluginControlPlane - mu sync.Mutex - attempts int - deleted []string -} + wantDeleted := []string{"storage-vol-1", "storage-vol-2"} + if diff := cmp.Diff(wantDeleted, plugin.deletedIDs); diff != "" { + t.Errorf("deletedIDs mismatch (-want +got):\n%s", diff) + } -func (r *retrySuccessVolumePlugin) CreateVolume(ctx context.Context, name, capacity, driverName string, parameters map[string]string) (string, map[string]string, error) { - r.mu.Lock() - defer r.mu.Unlock() - if strings.HasSuffix(name, "retry-vol2") { - r.attempts++ - if r.attempts == 1 { - return "", nil, fmt.Errorf("simulated temporary volume creation failure") - } + errMsg := err.Error() + if !strings.Contains(errMsg, "storage-vol-1") || !strings.Contains(errMsg, "storage-vol-2") { + t.Errorf("expected error message to contain both volume failure details, got: %v", errMsg) } - return "storage-" + name, parameters, nil } -func (r *retrySuccessVolumePlugin) AttachVolume(ctx context.Context, volumeID, node string) error { - return nil -} +func TestCreateActor_AtespaceNotFound(t *testing.T) { + ns := namespaceForTest("ns-create-actor-no-atespace") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) -func (r *retrySuccessVolumePlugin) DetachVolume(ctx context.Context, volumeID, node string) error { - return nil + // The template exists, but "missing-as" was never created. The template + // check fires first, so reaching this error proves the atespace check ran. + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "missing-as", Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}) + assertGrpcError(t, err, codes.FailedPrecondition, "Atespace missing-as not found") } -func (r *retrySuccessVolumePlugin) DeleteVolume(ctx context.Context, volumeID string) error { - r.mu.Lock() - defer r.mu.Unlock() - r.deleted = append(r.deleted, volumeID) - return nil +func TestValidation_Actor(t *testing.T) { + ns := namespaceForTest("ns-validation-actor") + tc := setupTest(t, ns) + defer tc.cleanup() + + t.Run("CreateActor", func(t *testing.T) { + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") + }) + + t.Run("GetActor", func(t *testing.T) { + _, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") + }) + + t.Run("ResumeActor", func(t *testing.T) { + _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") + }) + + t.Run("PauseActor", func(t *testing.T) { + _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") + }) + + t.Run("SuspendActor", func(t *testing.T) { + _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") + }) + + t.Run("UpdateActor", func(t *testing.T) { + _, err := tc.client.UpdateActor(context.Background(), &ateapipb.UpdateActorRequest{}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") + }) + + t.Run("DeleteActor", func(t *testing.T) { + _, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") + }) + + t.Run("ListActors", func(t *testing.T) { + _, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{PageSize: -1}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "page_size: Invalid value") + }) } -// TestResumeActor_VolumeCreationRetrySuccess tests that when volume provisioning fails on the first ResumeActor call, -// a subsequent call to ResumeActor retries provisioning only the pending volumes and succeeds. -func TestResumeActor_VolumeCreationRetrySuccess(t *testing.T) { - ns := namespaceForTest("ns-resume-vol-retry") +func TestActorLifecycle_WithExternalVolumes(t *testing.T) { + ns := namespaceForTest("ns-lifecycle-ext-vols") tc := setupTest(t, ns) defer tc.cleanup() volumes := []atev1alpha1.Volume{ { - Name: "succ-vol1", + Name: "data-vol", VolumeSource: atev1alpha1.VolumeSource{ ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ - StorageClassName: "standard", - Capacity: resource.MustParse("10Gi"), + StorageClassName: "fast", + Capacity: resource.MustParse("20Gi"), }, }, }, + } + mounts := []atev1alpha1.VolumeMount{ { - Name: "retry-vol2", - VolumeSource: atev1alpha1.VolumeSource{ - ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ - StorageClassName: "standard", - Capacity: resource.MustParse("10Gi"), - }, - }, + Name: "data-vol", + MountPath: "/mnt/data", }, } - retryMounts := []atev1alpha1.VolumeMount{ - {Name: "succ-vol1", MountPath: "/mnt/vol1"}, - {Name: "retry-vol2", MountPath: "/mnt/vol2"}, - } - createTemplateWithVolumes(t, tc, ns, volumes, retryMounts) + createTemplateWithVolumes(t, tc, ns, volumes, mounts) createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - plugin := &retrySuccessVolumePlugin{} - tc.service.volumePlugins = map[string]volume.VolumePluginControlPlane{ - "substrate.io/mock": plugin, - } - - // Call CreateActor RPC directly - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + // 1. CreateActor + createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "retry-actor"}, + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "actor-vol-lc"}, ActorTemplateNamespace: ns, ActorTemplateName: "tmpl1", }, }) if err != nil { - t.Fatalf("expected CreateActor to succeed, got: %v", err) + t.Fatalf("CreateActor failed: %v", err) } - - // First call to ResumeActor RPC, which should fail on retry-vol2 (attempt 1) - _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, - }) - if err == nil { - t.Fatalf("expected first ResumeActor to fail due to temporary volume creation error, but it succeeded") + if createResp.GetStatus() != ateapipb.Actor_STATUS_SUSPENDED { + t.Fatalf("expected initial status STATUS_SUSPENDED, got %v", createResp.GetStatus()) + } + if len(createResp.GetActorVolumes()) != 1 || createResp.GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { + t.Fatalf("expected 1 pending volume after CreateActor, got %v", createResp.GetActorVolumes()) } - // Verify GetActor returns the actor in STATUS_SUSPENDED status with succ-vol1 created and retry-vol2 pending - getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, + // 2. ResumeActor + resumeResp, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, }) if err != nil { - t.Fatalf("GetActor after first resume failed: %v", err) - } - if getResp.GetStatus() != ateapipb.Actor_STATUS_SUSPENDED { - t.Errorf("actor status after first resume = %v, want %v", getResp.GetStatus(), ateapipb.Actor_STATUS_SUSPENDED) + t.Fatalf("ResumeActor failed: %v", err) } - - volsByName := make(map[string]*ateapipb.ExternalVolume) - for _, v := range getResp.GetActorVolumes() { - volsByName[v.GetVolumeName()] = v + if resumeResp.GetActor().GetStatus() != ateapipb.Actor_STATUS_RUNNING { + t.Fatalf("expected status STATUS_RUNNING after resume, got %v", resumeResp.GetActor().GetStatus()) } - if v1, ok := volsByName["succ-vol1"]; !ok || v1.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v1.GetStorageVolumeId() == "" { - t.Errorf("succ-vol1 unexpected state after first resume: %v", v1) + if len(resumeResp.GetActor().GetActorVolumes()) != 1 || resumeResp.GetActor().GetActorVolumes()[0].GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED { + t.Fatalf("expected 1 created volume after ResumeActor, got %v", resumeResp.GetActor().GetActorVolumes()) } - if v2, ok := volsByName["retry-vol2"]; !ok || v2.GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { - t.Errorf("retry-vol2 unexpected state after first resume: %v", v2) + if resumeResp.GetActor().GetActorVolumes()[0].GetStorageVolumeId() == "" { + t.Fatalf("expected non-empty storageVolumeId after ResumeActor") } - // Second call to ResumeActor RPC, which should succeed on retry-vol2 (attempt 2) - _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, + // 3. PauseActor + pauseResp, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, }) if err != nil { - t.Fatalf("expected second ResumeActor to succeed, got: %v", err) + t.Fatalf("PauseActor failed: %v", err) + } + if pauseResp.GetActor().GetStatus() != ateapipb.Actor_STATUS_PAUSED { + t.Fatalf("expected status STATUS_PAUSED after pause, got %v", pauseResp.GetActor().GetStatus()) } - // Verify GetActor returns the actor in STATUS_RUNNING status with both volumes CREATED - getResp, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, + // 4. ResumeActor from paused + resumeResp2, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, }) if err != nil { - t.Fatalf("GetActor after second resume failed: %v", err) - } - if getResp.GetStatus() != ateapipb.Actor_STATUS_RUNNING { - t.Errorf("actor status after second resume = %v, want %v", getResp.GetStatus(), ateapipb.Actor_STATUS_RUNNING) + t.Fatalf("ResumeActor from paused failed: %v", err) } - for _, v := range getResp.GetActorVolumes() { - if v.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v.GetStorageVolumeId() == "" { - t.Errorf("volume %s unexpected state after second resume: %v", v.GetVolumeName(), v) - } + if resumeResp2.GetActor().GetStatus() != ateapipb.Actor_STATUS_RUNNING { + t.Fatalf("expected status STATUS_RUNNING after second resume, got %v", resumeResp2.GetActor().GetStatus()) } - // Clean up by suspending and deleting the actor - _, err = tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, + // 5. SuspendActor + suspendResp, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, }) if err != nil { t.Fatalf("SuspendActor failed: %v", err) } - _, err = tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, - }) - if err != nil { - t.Fatalf("DeleteActor failed: %v", err) + if suspendResp.GetActor().GetStatus() != ateapipb.Actor_STATUS_SUSPENDED { + t.Fatalf("expected status STATUS_SUSPENDED after suspend, got %v", suspendResp.GetActor().GetStatus()) } -} -// TestCreateActor_TemplateNotFound tests that creating an actor with a non-existent template fails with FailedPrecondition. -func TestCreateActor_TemplateNotFound(t *testing.T) { - ns := namespaceForTest("ns-create-notfound") - tc := setupTest(t, ns) - defer tc.cleanup() - - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "non-existent", - }}) - assertGrpcError(t, err, codes.FailedPrecondition, fmt.Sprintf("ActorTemplate %s/non-existent not found", ns)) -} - -// TestCreateActor_Duplicate tests that creating an actor with an existing ID fails. -func TestCreateActor_Duplicate(t *testing.T) { - ns := namespaceForTest("ns-create-dup") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}) + // 6. DeleteActor + deleteResp, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, + }) if err != nil { - t.Fatalf("first CreateActor failed: %v", err) + t.Fatalf("DeleteActor failed: %v", err) } - - _, err = tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}) - assertGrpcError(t, err, codes.AlreadyExists, "Actor id1 already exists") -} - -// TestGetActor_Found tests that an existing actor can be retrieved. -func TestGetActor_Found(t *testing.T) { - ns := namespaceForTest("ns-get-found") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - - name := "id1" - - createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) + if deleteResp.GetMetadata().GetName() != "actor-vol-lc" { + t.Errorf("deleted actor name = %q, want %q", deleteResp.GetMetadata().GetName(), "actor-vol-lc") } - getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + // Confirm GetActor returns NotFound after deletion + _, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, }) - if err != nil { - t.Fatalf("GetActor failed: %v", err) + if status.Code(err) != codes.NotFound { + t.Errorf("GetActor after delete err = %v, want NotFound", err) } +} - want := createResp +type partialFailVolumePlugin struct { + volume.VolumePluginControlPlane + deleted []string +} - if diff := cmp.Diff(want, getResp, protocmp.Transform()); diff != "" { - t.Errorf("GetActor response mismatch (-want +got):\n%s", diff) +func (f *partialFailVolumePlugin) CreateVolume(ctx context.Context, name, capacity, driverName string, parameters map[string]string) (string, map[string]string, error) { + if strings.HasSuffix(name, "fail-vol2") { + return "", nil, fmt.Errorf("simulated volume creation failure") } + return "storage-" + name, parameters, nil } -// TestGetActor_NotFound tests that retrieving a non-existent actor fails. -// Workflow: -// 1. Calls GetActor RPC with a non-existent ID. -// 2. Verifies that it returns an error (NotFound). -func TestGetActor_NotFound(t *testing.T) { - ns := namespaceForTest("ns-get-notfound") - tc := setupTest(t, ns) - defer tc.cleanup() +func (f *partialFailVolumePlugin) AttachVolume(ctx context.Context, volumeID, node string) error { + return nil +} - _, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "non-existent"}, - }) - assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/non-existent not found") +func (f *partialFailVolumePlugin) DetachVolume(ctx context.Context, volumeID, node string) error { + return nil } -// TestListActors tests that all created actors can be listed. -// Workflow: -// 1. Creates a mock ActorTemplate. -// 2. Calls CreateActor twice to create two actors. -// 3. Calls ListActors RPC. -// 4. Verifies that both actors are returned in the list. -func TestListActors(t *testing.T) { - ns := namespaceForTest("ns-list-actors") - tc := setupTest(t, ns) - defer tc.cleanup() +func (f *partialFailVolumePlugin) DeleteVolume(ctx context.Context, volumeID string) error { + f.deleted = append(f.deleted, volumeID) + return nil +} - createTemplate(t, tc, ns) +// TestResumeActor_VolumeCreationFailure tests that when volume provisioning fails during ResumeActor, +// successfully created volumes are saved, the actor remains in STATUS_SUSPENDED, +// and that calling DeleteActor on the suspended actor cleans up all partially created volumes. +func TestResumeActor_VolumeCreationFailure(t *testing.T) { + ns := namespaceForTest("ns-resume-vol-fail") + plugin := &partialFailVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) + defer tc.cleanup() - resp1, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}) - if err != nil { - t.Fatalf("CreateActor 1 failed: %v", err) + volumes := []atev1alpha1.Volume{ + { + Name: "succ-vol1", + VolumeSource: atev1alpha1.VolumeSource{ + ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ + StorageClassName: "standard", + Capacity: resource.MustParse("10Gi"), + }, + }, + }, + { + Name: "fail-vol2", + VolumeSource: atev1alpha1.VolumeSource{ + ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ + StorageClassName: "standard", + Capacity: resource.MustParse("10Gi"), + }, + }, + }, } - resp2, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id2"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}) - if err != nil { - t.Fatalf("CreateActor 2 failed: %v", err) + mounts := []atev1alpha1.VolumeMount{ + {Name: "succ-vol1", MountPath: "/mnt/vol1"}, + {Name: "fail-vol2", MountPath: "/mnt/vol2"}, } + createTemplateWithVolumes(t, tc, ns, volumes, mounts) - listResp, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{Atespace: testAtespace}) + // Call CreateActor RPC directly + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "fail-actor"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }, + }) if err != nil { - t.Fatalf("ListActors failed: %v", err) + t.Fatalf("expected CreateActor to succeed, got: %v", err) } - if len(listResp.Actors) != 2 { - t.Fatalf("expected 2 actors, got %d", len(listResp.Actors)) + // Call ResumeActor RPC, which should trigger volume provisioning and fail on fail-vol2 + _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "fail-actor"}, + }) + if err == nil { + t.Fatalf("expected ResumeActor to fail due to volume creation error, but it succeeded") } - want := []*ateapipb.Actor{ - resp1, - resp2, + // Verify GetActor returns the actor in STATUS_SUSPENDED status + getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "fail-actor"}, + }) + if err != nil { + t.Fatalf("GetActor failed: %v", err) } - - opts := []cmp.Option{ - protocmp.Transform(), - cmpopts.SortSlices(func(a, b *ateapipb.Actor) bool { - return a.GetMetadata().GetName() < b.GetMetadata().GetName() - }), + if getResp.GetStatus() != ateapipb.Actor_STATUS_SUSPENDED { + t.Errorf("actor status = %v, want %v", getResp.GetStatus(), ateapipb.Actor_STATUS_SUSPENDED) } - if diff := cmp.Diff(want, listResp.Actors, opts...); diff != "" { - t.Errorf("ListActors response mismatch (-want +got):\n%s", diff) + actorUID := getResp.GetMetadata().GetUid() + if actorUID == "" { + t.Fatalf("expected non-empty UID on actor") } -} -// TestListActors_ByAtespace verifies create + list are scoped by atespace end to -// end through the RPC surface: an actor created with a given atespace is only -// returned by ListActors(atespace=X) and only fetched by GetActor(atespace=X). -func TestListActors_ByAtespace(t *testing.T) { - ns := namespaceForTest("ns-list-by-atespace") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - createAtespace(t, tc, "team-a") - createAtespace(t, tc, "team-b") - - create := func(atespace, name string) *ateapipb.Actor { - resp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}) - if err != nil { - t.Fatalf("CreateActor(%s, atespace=%q) failed: %v", name, atespace, err) - } - return resp + // Verify that succ-vol1 was updated to CREATED with a storageVolumeId, and fail-vol2 is still PENDING + if len(getResp.GetActorVolumes()) != 2 { + t.Fatalf("expected 2 volumes on actor, got %d", len(getResp.GetActorVolumes())) } - a1 := create("team-a", "id1") - a2 := create("team-a", "id2") - b1 := create("team-b", "id3") - - sortByID := []cmp.Option{ - protocmp.Transform(), - cmpopts.SortSlices(func(a, b *ateapipb.Actor) bool { return a.GetMetadata().GetName() < b.GetMetadata().GetName() }), + volsByName := make(map[string]*ateapipb.ExternalVolume) + for _, v := range getResp.GetActorVolumes() { + volsByName[v.GetVolumeName()] = v } - - // List scoped to team-a returns only its actors. - listA, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{Atespace: "team-a"}) - if err != nil { - t.Fatalf("ListActors(team-a) failed: %v", err) + if v1, ok := volsByName["succ-vol1"]; !ok || v1.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v1.GetStorageVolumeId() == "" { + t.Errorf("succ-vol1 unexpected state: %v", v1) } - if diff := cmp.Diff([]*ateapipb.Actor{a1, a2}, listA.GetActors(), sortByID...); diff != "" { - t.Errorf("ListActors(team-a) mismatch (-want +got):\n%s", diff) + if v2, ok := volsByName["fail-vol2"]; !ok || v2.GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { + t.Errorf("fail-vol2 unexpected state: %v", v2) } - // List scoped to team-b returns only its actor. - listB, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{Atespace: "team-b"}) + // Call DeleteActor on the actor in STATUS_SUSPENDED + _, err = tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "fail-actor"}, + }) if err != nil { - t.Fatalf("ListActors(team-b) failed: %v", err) + t.Fatalf("DeleteActor failed: %v", err) } - if diff := cmp.Diff([]*ateapipb.Actor{b1}, listB.GetActors(), sortByID...); diff != "" { - t.Errorf("ListActors(team-b) mismatch (-want +got):\n%s", diff) + + // Verify both volumes were deleted (succ-vol1 via storageID, fail-vol2 via fallback actorVolumeID) + wantDeleted := []string{ + "storage-substrate-" + actorUID + "-succ-vol1", + "substrate-" + actorUID + "-fail-vol2", + } + if diff := cmp.Diff(wantDeleted, plugin.deleted); diff != "" { + t.Errorf("deleted volume IDs mismatch (-want +got):\n%s", diff) } - // Get is scoped: the right atespace hits, the empty atespace misses (deny-across by key). - if _, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "id1"}}); err != nil { - t.Errorf("GetActor(id1, team-a) failed: %v", err) + // Confirm GetActor returns NotFound after deletion + _, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "fail-actor"}, + }) + if status.Code(err) != codes.NotFound { + t.Errorf("GetActor after DeleteActor err = %v, want NotFound", err) } - _, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}) - assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found") } -// TestListActors_AllAtespaces verifies that an empty atespace lists actors across -// all atespaces (the `-A` / admin view), unlike the scoped single-atespace listing. -func TestListActors_AllAtespaces(t *testing.T) { - ns := namespaceForTest("ns-list-all-atespaces") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - createAtespace(t, tc, "team-a") - createAtespace(t, tc, "team-b") +type retrySuccessVolumePlugin struct { + volume.VolumePluginControlPlane + mu sync.Mutex + attempts int + deleted []string +} - create := func(atespace, name string) { - if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}); err != nil { - t.Fatalf("CreateActor(%s, atespace=%q) failed: %v", name, atespace, err) +func (r *retrySuccessVolumePlugin) CreateVolume(ctx context.Context, name, capacity, driverName string, parameters map[string]string) (string, map[string]string, error) { + r.mu.Lock() + defer r.mu.Unlock() + if strings.HasSuffix(name, "retry-vol2") { + r.attempts++ + if r.attempts == 1 { + return "", nil, fmt.Errorf("simulated temporary volume creation failure") } } - create("team-a", "id1") - create("team-b", "id2") + return "storage-" + name, parameters, nil +} - // Empty atespace lists across all atespaces; returned actors carry their atespace. - resp, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{}) - if err != nil { - t.Fatalf("ListActors(all) failed: %v", err) - } - got := map[string]string{} - for _, a := range resp.GetActors() { - got[a.GetMetadata().GetName()] = a.GetMetadata().GetAtespace() - } - if got["id1"] != "team-a" { - t.Errorf("ListActors(all): got[id1]=%q, want team-a", got["id1"]) - } - if got["id2"] != "team-b" { - t.Errorf("ListActors(all): got[id2]=%q, want team-b", got["id2"]) - } +func (r *retrySuccessVolumePlugin) AttachVolume(ctx context.Context, volumeID, node string) error { + return nil } -// TestListActors_Pagination tests that ListActors correctly paginates results. -func TestListActors_Pagination(t *testing.T) { - ns := namespaceForTest("ns-list-actors-pagination") - tc := setupTest(t, ns) +func (r *retrySuccessVolumePlugin) DetachVolume(ctx context.Context, volumeID, node string) error { + return nil +} + +func (r *retrySuccessVolumePlugin) DeleteVolume(ctx context.Context, volumeID string) error { + r.mu.Lock() + defer r.mu.Unlock() + r.deleted = append(r.deleted, volumeID) + return nil +} + +// TestResumeActor_VolumeCreationRetrySuccess tests that when volume provisioning fails on the first ResumeActor call, +// a subsequent call to ResumeActor retries provisioning only the pending volumes and succeeds. +func TestResumeActor_VolumeCreationRetrySuccess(t *testing.T) { + ns := namespaceForTest("ns-resume-vol-retry") + plugin := &retrySuccessVolumePlugin{} + tc := setupTestWithVolumePlugins(t, ns, map[string]volume.VolumePluginControlPlane{ + "substrate.io/mock": plugin, + }) defer tc.cleanup() - createTemplate(t, tc, ns) + volumes := []atev1alpha1.Volume{ + { + Name: "succ-vol1", + VolumeSource: atev1alpha1.VolumeSource{ + ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ + StorageClassName: "standard", + Capacity: resource.MustParse("10Gi"), + }, + }, + }, + { + Name: "retry-vol2", + VolumeSource: atev1alpha1.VolumeSource{ + ExternalVolumeTemplate: &atev1alpha1.ExternalVolumeTemplate{ + StorageClassName: "standard", + Capacity: resource.MustParse("10Gi"), + }, + }, + }, + } + retryMounts := []atev1alpha1.VolumeMount{ + {Name: "succ-vol1", MountPath: "/mnt/vol1"}, + {Name: "retry-vol2", MountPath: "/mnt/vol2"}, + } + createTemplateWithVolumes(t, tc, ns, volumes, retryMounts) + createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - var want []*ateapipb.Actor - for i := 0; i < 5; i++ { - resp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: fmt.Sprintf("name%d", i)}, + // Call CreateActor RPC directly + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "retry-actor"}, ActorTemplateNamespace: ns, ActorTemplateName: "tmpl1", - }}) - if err != nil { - t.Fatalf("CreateActor %d failed: %v", i, err) - } - want = append(want, resp) + }, + }) + if err != nil { + t.Fatalf("expected CreateActor to succeed, got: %v", err) } - var allActors []*ateapipb.Actor - pageToken := "" - - for { - listResp, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{ - Atespace: testAtespace, - PageSize: 2, - PageToken: pageToken, - }) - if err != nil { - t.Fatalf("ListActors failed: %v", err) - } + // First call to ResumeActor RPC, which should fail on retry-vol2 (attempt 1) + _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, + }) + if err == nil { + t.Fatalf("expected first ResumeActor to fail due to temporary volume creation error, but it succeeded") + } - allActors = append(allActors, listResp.Actors...) - pageToken = listResp.GetNextPageToken() - if pageToken == "" { - break - } + // Verify GetActor returns the actor in STATUS_SUSPENDED status with succ-vol1 created and retry-vol2 pending + getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, + }) + if err != nil { + t.Fatalf("GetActor after first resume failed: %v", err) + } + if getResp.GetStatus() != ateapipb.Actor_STATUS_SUSPENDED { + t.Errorf("actor status after first resume = %v, want %v", getResp.GetStatus(), ateapipb.Actor_STATUS_SUSPENDED) } - if len(allActors) != 5 { - t.Fatalf("expected 5 actors total, got %d", len(allActors)) + volsByName := make(map[string]*ateapipb.ExternalVolume) + for _, v := range getResp.GetActorVolumes() { + volsByName[v.GetVolumeName()] = v + } + if v1, ok := volsByName["succ-vol1"]; !ok || v1.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v1.GetStorageVolumeId() == "" { + t.Errorf("succ-vol1 unexpected state after first resume: %v", v1) } - - opts := []cmp.Option{ - protocmp.Transform(), - cmpopts.SortSlices(func(a, b *ateapipb.Actor) bool { - return a.GetMetadata().GetName() < b.GetMetadata().GetName() - }), + if v2, ok := volsByName["retry-vol2"]; !ok || v2.GetStatus() != ateapipb.ExternalVolume_STATUS_PENDING { + t.Errorf("retry-vol2 unexpected state after first resume: %v", v2) } - if diff := cmp.Diff(want, allActors, opts...); diff != "" { - t.Errorf("ListActors pagination response mismatch (-want +got):\n%s", diff) + // Second call to ResumeActor RPC, which should succeed on retry-vol2 (attempt 2) + _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, + }) + if err != nil { + t.Fatalf("expected second ResumeActor to succeed, got: %v", err) } -} - -// TestListWorkers tests that workers mirrored to Redis are listed. -// Workflow: -// 1. Creates a mock WorkerPool in Kubernetes. -// 2. Creates a mock worker Pod in Kubernetes belonging to that pool. -// 3. Waits for the background WorkerPoolSyncer to mirror it to Redis. -// 4. Calls ListWorkers RPC. -// 5. Verifies that the worker appears in the response. -func TestListWorkers(t *testing.T) { - ns := namespaceForTest("ns-list-workers") - tc := setupTest(t, ns) - defer tc.cleanup() - - createWorkerPool(t, tc, ns, "pool1", map[string]string{"foo": "bar"}) - createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - listResp, err := tc.client.ListWorkers(context.Background(), &ateapipb.ListWorkersRequest{}) + // Verify GetActor returns the actor in STATUS_RUNNING status with both volumes CREATED + getResp, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, + }) if err != nil { - t.Fatalf("ListWorkers failed: %v", err) + t.Fatalf("GetActor after second resume failed: %v", err) } - - var filteredWorkers []*ateapipb.Worker - for _, w := range listResp.GetWorkers() { - if w.GetWorkerNamespace() == ns { - filteredWorkers = append(filteredWorkers, w) + if getResp.GetStatus() != ateapipb.Actor_STATUS_RUNNING { + t.Errorf("actor status after second resume = %v, want %v", getResp.GetStatus(), ateapipb.Actor_STATUS_RUNNING) + } + for _, v := range getResp.GetActorVolumes() { + if v.GetStatus() != ateapipb.ExternalVolume_STATUS_CREATED || v.GetStorageVolumeId() == "" { + t.Errorf("volume %s unexpected state after second resume: %v", v.GetVolumeName(), v) } } - want := []*ateapipb.Worker{ - { - WorkerNamespace: ns, - WorkerPool: "pool1", - WorkerPod: "worker-1", - NodeName: "node1", - Ip: "127.0.0.1", - Version: 1, - SandboxClass: "gvisor", - Labels: map[string]string{"foo": "bar"}, - State: ateapipb.Worker_STATE_ACTIVE, - }, + // Clean up by suspending and deleting the actor + _, err = tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, + }) + if err != nil { + t.Fatalf("SuspendActor failed: %v", err) } - - if diff := cmp.Diff(want, filteredWorkers, protocmp.Transform(), protocmp.IgnoreFields(&ateapipb.Worker{}, "worker_pod_uid")); diff != "" { - t.Errorf("ListWorkers response mismatch (-want +got):\n%s", diff) + _, err = tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "retry-actor"}, + }) + if err != nil { + t.Fatalf("DeleteActor failed: %v", err) } } @@ -1841,241 +1693,37 @@ func TestResumeActorPassesLiteralEnv(t *testing.T) { } gotEnv := map[string]string{} for _, env := range restoreReq.GetSpec().GetContainers()[0].GetEnv() { - gotEnv[env.GetName()] = env.GetValue() - } - wantEnv := map[string]string{ - "LITERAL": "plain", - } - if diff := cmp.Diff(wantEnv, gotEnv); diff != "" { - t.Errorf("env mismatch (-want +got):\n%s", diff) - } -} - -// TestResumeActor_NoWorkers tests that resuming an actor fails when no free workers are available. -// Workflow: -// 1. Creates a mock ActorTemplate. -// 2. Creates an actor. -// 3. Calls ResumeActor RPC without creating any workers. -// 4. Verifies that ResumeActor fails with FailedPrecondition status. -// TestResumeActor_NoWorkers tests that resuming an actor fails when no free workers are available. -// Workflow: -// 1. Creates a mock ActorTemplate. -// 2. Creates an actor. -// 3. Calls ResumeActor RPC without creating any workers. -// 4. Verifies that ResumeActor fails with FailedPrecondition status. -func TestResumeActor_NoWorkers(t *testing.T) { - ns := namespaceForTest("ns-resume-no-workers") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - - createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - - name := createResp.GetMetadata().GetName() - - _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - assertGrpcError(t, err, codes.FailedPrecondition, "no free workers available") -} - -// TestResumeActor_MultiPoolSelector exercises the AND-of-two-selectors path -// end to end: a template's WorkerSelector gates two pools, and the actor's -// worker_selector narrows to just one of them. -func TestResumeActor_MultiPoolSelector(t *testing.T) { - ns := namespaceForTest("ns-multi-pool") - tc := setupTest(t, ns) - defer tc.cleanup() - - createWorkerPool(t, tc, ns, "pool-a", map[string]string{"group": ns, "tier": "a"}) - createWorkerPool(t, tc, ns, "pool-b", map[string]string{"group": ns, "tier": "b"}) - createTemplateWithSelector(t, tc, ns, "tmpl1", &metav1.LabelSelector{ - MatchLabels: map[string]string{"group": ns}, - }) - - createWorkerPod(t, tc, ns, "worker-a", "node1", "pool-a") - createWorkerPod(t, tc, ns, "worker-b", "node1", "pool-b") - - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - WorkerSelector: &ateapipb.Selector{ - MatchLabels: map[string]string{"tier": "b"}, - }, - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - - _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}) - if err != nil { - t.Fatalf("ResumeActor failed: %v", err) - } - - getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}) - if err != nil { - t.Fatalf("GetActor failed: %v", err) - } - if got := getResp.GetWorkerAssignment().GetWorkerPod(); got != "worker-b" { - t.Errorf("expected actor to be assigned to worker-b (pool-b, matching narrowed selector), got %q", got) - } - if got := getResp.GetWorkerAssignment().GetWorkerPool(); got != "pool-b" { - t.Errorf("expected actor's worker_assignment.worker_pool to be pool-b, got %q", got) - } -} - -// TestResumeActor_RequiresBothSelectorsToMatch proves eligibility is the AND -// of the template's WorkerSelector and the actor's worker_selector, not -// either one alone: a pool matching only the template selector and a pool -// matching only the actor selector must both be rejected, end to end -// through CreateActor/ResumeActor (not just the eligibleWorkerPools unit -// test), while a pool matching both is the one actually used. -func TestResumeActor_RequiresBothSelectorsToMatch(t *testing.T) { - ns := namespaceForTest("ns-resume-and-selectors") - tc := setupTest(t, ns) - defer tc.cleanup() - - createWorkerPool(t, tc, ns, "pool-both", map[string]string{"group": ns, "tier": "b"}) - createWorkerPool(t, tc, ns, "pool-template-only", map[string]string{"group": ns, "tier": "a"}) - createWorkerPool(t, tc, ns, "pool-actor-only", map[string]string{"tier": "b"}) - createTemplateWithSelector(t, tc, ns, "tmpl1", &metav1.LabelSelector{ - MatchLabels: map[string]string{"group": ns}, - }) - - createWorkerPod(t, tc, ns, "worker-both", "node1", "pool-both") - createWorkerPod(t, tc, ns, "worker-template-only", "node1", "pool-template-only") - createWorkerPod(t, tc, ns, "worker-actor-only", "node1", "pool-actor-only") - - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - WorkerSelector: &ateapipb.Selector{ - MatchLabels: map[string]string{"tier": "b"}, - }, - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - - if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}); err != nil { - t.Fatalf("ResumeActor failed: %v", err) - } - - getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}) - if err != nil { - t.Fatalf("GetActor failed: %v", err) - } - if got := getResp.GetWorkerAssignment().GetWorkerPool(); got != "pool-both" { - t.Errorf("expected actor to be assigned to pool-both (the only pool matching both selectors), got worker_assignment.worker_pool=%q", got) - } -} - -// TestResumeActor_Reentrancy tests the failure recovery and re-entrancy of ResumeActor. -// Workflow: -// 1. Creates a mock ActorTemplate. -// 2. Creates a mock Atelet Pod and a mock Worker Pod. -// 3. Waits for the WorkerPoolSyncer to mirror the worker to store. -// 4. Creates an actor in SUSPENDED state. -// 5. Configures fake Atelet to FAIL on Restore. -// 6. Calls ResumeActor and verifies it fails, but actor status becomes RESUMING. -// 7. Configures fake Atelet to SUCCEED on Restore. -// 8. Calls ResumeActor again and verifies it succeeds and actor status becomes RUNNING. -func TestResumeActor_Reentrancy(t *testing.T) { - ns := namespaceForTest("ns-resume-reentrancy") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - - // Create Worker Pod - createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - - name := "id1" - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - - // STEP 1: Make Atelet FAIL on Restore! - tc.fakeAtelet.FailRestore = fmt.Errorf("mock atelet failure") - - _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - if err == nil { - t.Fatalf("expected ResumeActor to fail due to atelet error") - } - - // Verify actor state is RESUMING in Redis! - actor, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: name}) - if err != nil { - t.Fatalf("failed to get actor from store: %v", err) - } - if actor.GetStatus() != ateapipb.Actor_STATUS_RESUMING { - t.Errorf("expected status RESUMING, got %v", actor.GetStatus()) - } - - // STEP 2: Make Atelet SUCCEED! - tc.fakeAtelet.FailRestore = nil - tc.fakeAtelet.RestoreCalled = false // reset for verification - - _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - if err != nil { - t.Fatalf("ResumeActor failed on retry: %v", err) - } - - if !tc.fakeAtelet.RestoreCalled { - t.Errorf("expected Restore to be called on retry") + gotEnv[env.GetName()] = env.GetValue() } - - // Verify actor state is RUNNING! - actor, err = tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: name}) - if err != nil { - t.Fatalf("failed to get actor from store: %v", err) + wantEnv := map[string]string{ + "LITERAL": "plain", } - if actor.GetStatus() != ateapipb.Actor_STATUS_RUNNING { - t.Errorf("expected status RUNNING, got %v", actor.GetStatus()) + if diff := cmp.Diff(wantEnv, gotEnv); diff != "" { + t.Errorf("env mismatch (-want +got):\n%s", diff) } } -// TestSuspendActor tests the full workflow of suspending a running actor. +// TestResumeActor_NoWorkers tests that resuming an actor fails when no free workers are available. // Workflow: // 1. Creates a mock ActorTemplate. -// 2. Creates a mock Atelet Pod on 'node1'. -// 3. Creates a mock worker Pod on 'node1'. -// 4. Waits for the WorkerPoolSyncer to mirror the worker to Redis. -// 5. Creates an actor. -// 6. Calls ResumeActor to transition it to RUNNING. -// 7. Calls SuspendActor RPC. -// 8. Verifies that the fake Atelet received the Suspend call. -func TestSuspendActor(t *testing.T) { - ns := namespaceForTest("ns-suspend") +// 2. Creates an actor. +// 3. Calls ResumeActor RPC without creating any workers. +// 4. Verifies that ResumeActor fails with FailedPrecondition status. +// TestResumeActor_NoWorkers tests that resuming an actor fails when no free workers are available. +// Workflow: +// 1. Creates a mock ActorTemplate. +// 2. Creates an actor. +// 3. Calls ResumeActor RPC without creating any workers. +// 4. Verifies that ResumeActor fails with FailedPrecondition status. +func TestResumeActor_NoWorkers(t *testing.T) { + ns := namespaceForTest("ns-resume-no-workers") tc := setupTest(t, ns) defer tc.cleanup() createTemplate(t, tc, ns) - createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - name := "id1" - - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + createResp, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, ActorTemplateNamespace: ns, ActorTemplateName: "tmpl1", }}) @@ -2083,187 +1731,125 @@ func TestSuspendActor(t *testing.T) { t.Fatalf("CreateActor failed: %v", err) } - // Resume first to make it running - running, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) - if err != nil { - t.Fatalf("ResumeActor failed: %v", err) - } + name := createResp.GetMetadata().GetName() - // Suspend - suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, }) - if err != nil { - t.Fatalf("SuspendActor failed: %v", err) - } + assertGrpcError(t, err, codes.FailedPrecondition, "no free workers available") +} - if !tc.fakeAtelet.CheckpointCalled { - t.Errorf("expected atelet Checkpoint to be called") - } - ref := suspended.GetActor().GetLatestSnapshot() - if ref.GetName() == "" { - t.Fatalf("SuspendActor returned no ActorSnapshot reference: %v", suspended) - } - snapshotRef := ref - snapshot, err := tc.client.GetActorSnapshot(context.Background(), &ateapipb.GetActorSnapshotRequest{Snapshot: snapshotRef}) - if err != nil { - t.Fatalf("GetActorSnapshot failed: %v", err) - } - if got := snapshot.GetSourceActorVersion(); got != running.GetActor().GetMetadata().GetVersion() { - t.Errorf("snapshot source version = %d, want %d", got, running.GetActor().GetMetadata().GetVersion()) - } - listed, err := tc.client.ListActorSnapshots(context.Background(), &ateapipb.ListActorSnapshotsRequest{Atespace: testAtespace, PageSize: 1}) - if err != nil || len(listed.GetSnapshots()) != 1 { - t.Fatalf("ListActorSnapshots = (%v, %v), want one", listed, err) - } - tagRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "before-upgrade"} - tagged, err := tc.client.CreateActorSnapshotTag(context.Background(), &ateapipb.CreateActorSnapshotTagRequest{ - ActorSnapshotTag: &ateapipb.ActorSnapshotTag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "before-upgrade"}, - Snapshot: snapshotRef, - Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE, - }, - }) - if err != nil || !proto.Equal(tagged.GetSnapshot(), ref) { - t.Fatalf("CreateActorSnapshotTag = (%v, %v), want tag for snapshot", tagged, err) - } - if _, err := tc.client.CreateActorSnapshotTag(context.Background(), &ateapipb.CreateActorSnapshotTagRequest{ - ActorSnapshotTag: &ateapipb.ActorSnapshotTag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "other", Name: "cross-atespace"}, - Snapshot: snapshotRef, - Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE, - }, - }); status.Code(err) != codes.FailedPrecondition { - t.Fatalf("cross-atespace CreateActorSnapshotTag status = %v, want FailedPrecondition", status.Code(err)) - } - if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "other", Name: "cross-atespace"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - SourceSnapshot: &ateapipb.ActorSnapshotSource{Tag: tagRef}, - }, - }); status.Code(err) != codes.FailedPrecondition { - t.Fatalf("cross-atespace CreateActor status = %v, want FailedPrecondition", status.Code(err)) - } - updated, err := tc.client.UpdateActorSnapshotTag(context.Background(), &ateapipb.UpdateActorSnapshotTagRequest{ - Tag: &ateapipb.ActorSnapshotTag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: tagRef.GetAtespace(), Name: tagRef.GetName()}, - Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED, - }, - UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"scope"}}, +// TestResumeActor_MultiPoolSelector exercises the AND-of-two-selectors path +// end to end: a template's WorkerSelector gates two pools, and the actor's +// worker_selector narrows to just one of them. +func TestResumeActor_MultiPoolSelector(t *testing.T) { + ns := namespaceForTest("ns-multi-pool") + tc := setupTest(t, ns) + defer tc.cleanup() + + createWorkerPool(t, tc, ns, "pool-a", map[string]string{"group": ns, "tier": "a"}) + createWorkerPool(t, tc, ns, "pool-b", map[string]string{"group": ns, "tier": "b"}) + createTemplateWithSelector(t, tc, ns, "tmpl1", &metav1.LabelSelector{ + MatchLabels: map[string]string{"group": ns}, }) - if err != nil || updated.GetScope() != ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED { - t.Fatalf("UpdateActorSnapshotTag = (%v, %v), want published", updated, err) - } - if got, err := tc.client.GetActorSnapshotTag(context.Background(), &ateapipb.GetActorSnapshotTagRequest{Tag: tagRef}); err != nil || !proto.Equal(got.GetSnapshot(), ref) { - t.Fatalf("tag after publication = (%v, %v), want same address", got, err) - } - createAtespace(t, tc, "other") - if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "other", Name: "cross-atespace"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - SourceSnapshot: &ateapipb.ActorSnapshotSource{Tag: tagRef}, - }, - }); err != nil { - t.Fatalf("CreateActor from published tag failed: %v", err) - } - clone, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "clone"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - SourceSnapshot: &ateapipb.ActorSnapshotSource{Tag: tagRef}, + createWorkerPod(t, tc, ns, "worker-a", "node1", "pool-a") + createWorkerPod(t, tc, ns, "worker-b", "node1", "pool-b") + + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + WorkerSelector: &ateapipb.Selector{ + MatchLabels: map[string]string{"tier": "b"}, }, - }) + }}) if err != nil { - t.Fatalf("CreateActor from snapshot failed: %v", err) - } - if !proto.Equal(clone.GetLatestSnapshot(), ref) { - t.Fatalf("clone latest snapshot = %v, want %v", clone.GetLatestSnapshot(), ref) - } - if got := clone.GetSourceSnapshot(); !proto.Equal(got.GetTag(), tagRef) || !proto.Equal(got.GetSnapshot(), ref) || got.GetSnapshotUid() != snapshot.GetMetadata().GetUid() { - t.Fatalf("clone source snapshot = %v, want tag %v, snapshot %v, uid %v", got, tagRef, ref, snapshot.GetMetadata().GetUid()) - } - if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "clone"}}); err != nil { - t.Fatalf("ResumeActor clone failed: %v", err) + t.Fatalf("CreateActor failed: %v", err) } - if !tc.fakeAtelet.RestoreCalled { - t.Error("resuming clone did not restore its source ActorSnapshot") + + _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}) + if err != nil { + t.Fatalf("ResumeActor failed: %v", err) } - cloneSuspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "clone"}}) + + getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}) if err != nil { - t.Fatalf("SuspendActor clone failed: %v", err) + t.Fatalf("GetActor failed: %v", err) } - if cloneSuspended.GetActor().GetLatestSnapshot().GetName() == ref.GetName() { - t.Fatal("clone suspension reused its source snapshot") + if got := getResp.GetWorkerAssignment().GetWorkerPod(); got != "worker-b" { + t.Errorf("expected actor to be assigned to worker-b (pool-b, matching narrowed selector), got %q", got) } - listed, err = tc.client.ListActorSnapshots(context.Background(), &ateapipb.ListActorSnapshotsRequest{Atespace: testAtespace}) - if err != nil || len(listed.GetSnapshots()) != 2 { - t.Fatalf("ListActorSnapshots after clone suspension = (%v, %v), want two", listed, err) + if got := getResp.GetWorkerAssignment().GetWorkerPool(); got != "pool-b" { + t.Errorf("expected actor's worker_assignment.worker_pool to be pool-b, got %q", got) } +} - getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, +// TestResumeActor_RequiresBothSelectorsToMatch proves eligibility is the AND +// of the template's WorkerSelector and the actor's worker_selector, not +// either one alone: a pool matching only the template selector and a pool +// matching only the actor selector must both be rejected, end to end +// through CreateActor/ResumeActor (not just the eligibleWorkerPools unit +// test), while a pool matching both is the one actually used. +func TestResumeActor_RequiresBothSelectorsToMatch(t *testing.T) { + ns := namespaceForTest("ns-resume-and-selectors") + tc := setupTest(t, ns) + defer tc.cleanup() + + createWorkerPool(t, tc, ns, "pool-both", map[string]string{"group": ns, "tier": "b"}) + createWorkerPool(t, tc, ns, "pool-template-only", map[string]string{"group": ns, "tier": "a"}) + createWorkerPool(t, tc, ns, "pool-actor-only", map[string]string{"tier": "b"}) + createTemplateWithSelector(t, tc, ns, "tmpl1", &metav1.LabelSelector{ + MatchLabels: map[string]string{"group": ns}, }) - if err != nil { - t.Fatalf("GetActor failed: %v", err) - } - want := &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, + + createWorkerPod(t, tc, ns, "worker-both", "node1", "pool-both") + createWorkerPod(t, tc, ns, "worker-template-only", "node1", "pool-template-only") + createWorkerPod(t, tc, ns, "worker-actor-only", "node1", "pool-actor-only") + + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, ActorTemplateNamespace: ns, ActorTemplateName: "tmpl1", - Status: ateapipb.Actor_STATUS_SUSPENDED, + WorkerSelector: &ateapipb.Selector{ + MatchLabels: map[string]string{"tier": "b"}, + }, + }}) + if err != nil { + t.Fatalf("CreateActor failed: %v", err) } - if diff := cmp.Diff(want, getResp, - protocmp.Transform(), - ignoreUID, - ignoreVersion, - ignoreTimestamps, - protocmp.IgnoreFields(&ateapipb.Actor{}, "latest_snapshot"), - ); diff != "" { - t.Errorf("GetActor response mismatch (-want +got):\n%s", diff) - } - if _, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}}); err != nil { - t.Fatalf("DeleteActor source failed: %v", err) - } - if _, err := tc.client.GetActorSnapshotTag(context.Background(), &ateapipb.GetActorSnapshotTagRequest{Tag: tagRef}); err != nil { - t.Fatalf("source snapshot tag disappeared with source Actor: %v", err) - } - if deleted, err := tc.client.DeleteActorSnapshotTag(context.Background(), &ateapipb.DeleteActorSnapshotTagRequest{Tag: tagRef}); err != nil || deleted.GetMetadata().GetName() != tagRef.GetName() { - t.Fatalf("DeleteActorSnapshotTag = (%v, %v)", deleted, err) + if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}); err != nil { + t.Fatalf("ResumeActor failed: %v", err) } - if _, err := tc.client.GetActorSnapshotTag(context.Background(), &ateapipb.GetActorSnapshotTagRequest{Tag: tagRef}); status.Code(err) != codes.NotFound { - t.Fatalf("deleted tag status = %v, want NotFound", status.Code(err)) + + getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}) + if err != nil { + t.Fatalf("GetActor failed: %v", err) } - if _, err := tc.client.GetActorSnapshot(context.Background(), &ateapipb.GetActorSnapshotRequest{Snapshot: snapshotRef}); err != nil { - t.Fatalf("snapshot metadata disappeared after tag deletion: %v", err) + if got := getResp.GetWorkerAssignment().GetWorkerPool(); got != "pool-both" { + t.Errorf("expected actor to be assigned to pool-both (the only pool matching both selectors), got worker_assignment.worker_pool=%q", got) } } -// TestPauseActor tests the full workflow of pausing a running actor. +// TestResumeActor_Reentrancy tests the failure recovery and re-entrancy of ResumeActor. // Workflow: // 1. Creates a mock ActorTemplate. -// 2. Creates a mock Atelet Pod on 'node1'. -// 3. Creates a mock worker Pod on 'node1'. -// 4. Waits for the WorkerPoolSyncer to mirror the worker to Redis. -// 5. Creates an actor. -// 6. Calls ResumeActor to transition it to RUNNING. -// 7. Calls PauseActor RPC. -// 8. Verifies that the fake Atelet received the Pause call. -func TestPauseActor(t *testing.T) { - ns := namespaceForTest("ns-pause") +// 2. Creates a mock Atelet Pod and a mock Worker Pod. +// 3. Waits for the WorkerPoolSyncer to mirror the worker to store. +// 4. Creates an actor in SUSPENDED state. +// 5. Configures fake Atelet to FAIL on Restore. +// 6. Calls ResumeActor and verifies it fails, but actor status becomes RESUMING. +// 7. Configures fake Atelet to SUCCEED on Restore. +// 8. Calls ResumeActor again and verifies it succeeds and actor status becomes RUNNING. +func TestResumeActor_Reentrancy(t *testing.T) { + ns := namespaceForTest("ns-resume-reentrancy") tc := setupTest(t, ns) defer tc.cleanup() createTemplate(t, tc, ns) + // Create Worker Pod createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") name := "id1" @@ -2276,307 +1862,366 @@ func TestPauseActor(t *testing.T) { t.Fatalf("CreateActor failed: %v", err) } - // Resume first to make it running + // STEP 1: Make Atelet FAIL on Restore! + tc.fakeAtelet.FailRestore = fmt.Errorf("mock atelet failure") + _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, }) - if err != nil { - t.Fatalf("ResumeActor failed: %v", err) + if err == nil { + t.Fatalf("expected ResumeActor to fail due to atelet error") } - // Pause - _, err = tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, - }) + // Verify actor state is RESUMING in Redis! + actor, err := tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: name}) if err != nil { - t.Fatalf("PauseActor failed: %v", err) + t.Fatalf("failed to get actor from store: %v", err) } - - if !tc.fakeAtelet.CheckpointCalled { - t.Errorf("expected atelet Checkpoint to be called") + if actor.GetStatus() != ateapipb.Actor_STATUS_RESUMING { + t.Errorf("expected status RESUMING, got %v", actor.GetStatus()) } - getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + // STEP 2: Make Atelet SUCCEED! + tc.fakeAtelet.FailRestore = nil + tc.fakeAtelet.RestoreCalled = false // reset for verification + + _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, }) if err != nil { - t.Fatalf("GetActor failed: %v", err) + t.Fatalf("ResumeActor failed on retry: %v", err) } - want := &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - Status: ateapipb.Actor_STATUS_PAUSED, - LocalSnapshotInfo: &ateapipb.LocalSnapshotInfo{ - NodeVmsWithLocalSnapshots: []string{"node1"}, - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - }, + + if !tc.fakeAtelet.RestoreCalled { + t.Errorf("expected Restore to be called on retry") } - if diff := cmp.Diff(want, getResp, - protocmp.Transform(), - ignoreUID, - ignoreVersion, - ignoreTimestamps, - protocmp.IgnoreFields(&ateapipb.WorkerAssignment{}, "worker_pod_uid"), - protocmp.IgnoreFields(&ateapipb.LocalSnapshotInfo{}, "snapshot_name"), - ); diff != "" { - t.Errorf("GetActor response mismatch (-want +got):\n%s", diff) + // Verify actor state is RUNNING! + actor, err = tc.persistence.GetActor(context.Background(), resources.ActorRef{Atespace: testAtespace, Name: name}) + if err != nil { + t.Fatalf("failed to get actor from store: %v", err) } - if getResp.GetLocalSnapshotInfo().GetSnapshotName() == "" { - t.Error("LocalSnapshotInfo.SnapshotName is empty, want the name the pause checkpointed under") + if actor.GetStatus() != ateapipb.Actor_STATUS_RUNNING { + t.Errorf("expected status RUNNING, got %v", actor.GetStatus()) } } -// TestUpdateActor_Success verifies UpdateActor replaces the actor's -// worker_selector and that the change is durably persisted. -func TestUpdateActor_Success(t *testing.T) { - ns := namespaceForTest("ns-update-actor") +// The early ref stamp must land on the span even when the op fails, so a failed +// resume is still attributable to who/where. +func TestResumeActor_ErrorStillStampsRefSpanIdentity(t *testing.T) { + ns := namespaceForTest("ns-span-resume-err") + tc := setupTest(t, ns) + defer tc.cleanup() + + attrs := recordRootSpanAttrs(t, func(ctx context.Context) { + if _, err := tc.service.ResumeActor(ctx, &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "missing"}, + }); err == nil { + t.Fatal("expected error resuming missing actor") + } + }) + + assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) + assertSpanStr(t, attrs, ateattr.ActorNameKey, "missing") +} + +// TestSuspendActor tests the full workflow of suspending a running actor. +// Workflow: +// 1. Creates a mock ActorTemplate. +// 2. Creates a mock Atelet Pod on 'node1'. +// 3. Creates a mock worker Pod on 'node1'. +// 4. Waits for the WorkerPoolSyncer to mirror the worker to Redis. +// 5. Creates an actor. +// 6. Calls ResumeActor to transition it to RUNNING. +// 7. Calls SuspendActor RPC. +// 8. Verifies that the fake Atelet received the Suspend call. +func TestSuspendActor(t *testing.T) { + ns := namespaceForTest("ns-suspend") tc := setupTest(t, ns) defer tc.cleanup() createTemplate(t, tc, ns) + createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + name := "id1" + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, ActorTemplateNamespace: ns, ActorTemplateName: "tmpl1", - WorkerSelector: &ateapipb.Selector{ - MatchLabels: map[string]string{"tier": "free"}, - }, }}) if err != nil { t.Fatalf("CreateActor failed: %v", err) } - updateResp, err := tc.client.UpdateActor(context.Background(), &ateapipb.UpdateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, - WorkerSelector: &ateapipb.Selector{ - MatchLabels: map[string]string{"tier": "paid"}, - }, - // Output-only fields outside the mask are ignored. - Status: ateapipb.Actor_STATUS_RUNNING, - }, - UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"worker_selector"}}, + // Resume first to make it running + running, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, }) if err != nil { - t.Fatalf("UpdateActor failed: %v", err) + t.Fatalf("ResumeActor failed: %v", err) } - wantActor := &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Name: "id1", Atespace: testAtespace, Version: 2}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - Status: ateapipb.Actor_STATUS_SUSPENDED, - WorkerSelector: &ateapipb.Selector{ - MatchLabels: map[string]string{"tier": "paid"}, - }, - } - if diff := cmp.Diff(wantActor, updateResp, protocmp.Transform(), ignoreUID, ignoreTimestamps); diff != "" { - t.Errorf("UpdateActor response mismatch (-want +got):\n%s", diff) + // Suspend + suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) + if err != nil { + t.Fatalf("SuspendActor failed: %v", err) } - getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}}) + if !tc.fakeAtelet.CheckpointCalled { + t.Errorf("expected atelet Checkpoint to be called") + } + ref := suspended.GetActor().GetLatestSnapshot() + if ref.GetName() == "" { + t.Fatalf("SuspendActor returned no ActorSnapshot reference: %v", suspended) + } + snapshotRef := ref + snapshot, err := tc.client.GetActorSnapshot(context.Background(), &ateapipb.GetActorSnapshotRequest{Snapshot: snapshotRef}) if err != nil { - t.Fatalf("GetActor failed: %v", err) + t.Fatalf("GetActorSnapshot failed: %v", err) } - wantGetResp := wantActor - if diff := cmp.Diff(wantGetResp, getResp, protocmp.Transform(), ignoreUID, ignoreTimestamps); diff != "" { - t.Errorf("GetActor response mismatch after UpdateActor (-want +got):\n%s", diff) + if got := snapshot.GetSourceActorVersion(); got != running.GetActor().GetMetadata().GetVersion() { + t.Errorf("snapshot source version = %d, want %d", got, running.GetActor().GetMetadata().GetVersion()) } -} - -// TestUpdateActor_Preconditions verifies the optional version and uid guards -// carried in the embedded resource's metadata. -func TestUpdateActor_Preconditions(t *testing.T) { - ns := namespaceForTest("ns-update-preconditions") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - - ctx := context.Background() - createActor := func() *ateapipb.Actor { - t.Helper() - actor, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: testActorID}, + listed, err := tc.client.ListActorSnapshots(context.Background(), &ateapipb.ListActorSnapshotsRequest{Atespace: testAtespace, PageSize: 1}) + if err != nil || len(listed.GetSnapshots()) != 1 { + t.Fatalf("ListActorSnapshots = (%v, %v), want one", listed, err) + } + tagRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "before-upgrade"} + tagged, err := tc.client.CreateActorSnapshotTag(context.Background(), &ateapipb.CreateActorSnapshotTagRequest{ + ActorSnapshotTag: &ateapipb.ActorSnapshotTag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "before-upgrade"}, + Snapshot: snapshotRef, + Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE, + }, + }) + if err != nil || !proto.Equal(tagged.GetSnapshot(), ref) { + t.Fatalf("CreateActorSnapshotTag = (%v, %v), want tag for snapshot", tagged, err) + } + if _, err := tc.client.CreateActorSnapshotTag(context.Background(), &ateapipb.CreateActorSnapshotTagRequest{ + ActorSnapshotTag: &ateapipb.ActorSnapshotTag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "other", Name: "cross-atespace"}, + Snapshot: snapshotRef, + Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE, + }, + }); status.Code(err) != codes.FailedPrecondition { + t.Fatalf("cross-atespace CreateActorSnapshotTag status = %v, want FailedPrecondition", status.Code(err)) + } + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "other", Name: "cross-atespace"}, ActorTemplateNamespace: ns, ActorTemplateName: "tmpl1", - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - return actor + SourceSnapshot: &ateapipb.ActorSnapshotSource{Tag: tagRef}, + }, + }); status.Code(err) != codes.FailedPrecondition { + t.Fatalf("cross-atespace CreateActor status = %v, want FailedPrecondition", status.Code(err)) } - - update := func(meta *ateapipb.ResourceMetadata, tier string) (*ateapipb.Actor, error) { - meta.Atespace, meta.Name = testAtespace, testActorID - return tc.client.UpdateActor(ctx, &ateapipb.UpdateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: meta, - WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"tier": tier}}, - }, - UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"worker_selector"}}, - }) + updated, err := tc.client.UpdateActorSnapshotTag(context.Background(), &ateapipb.UpdateActorSnapshotTagRequest{ + Tag: &ateapipb.ActorSnapshotTag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: tagRef.GetAtespace(), Name: tagRef.GetName()}, + Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED, + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"scope"}}, + }) + if err != nil || updated.GetScope() != ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED { + t.Fatalf("UpdateActorSnapshotTag = (%v, %v), want published", updated, err) } - - // Delete and recreate the same atespace/name actor, so the first lifecycle's uid - // becomes stale. - staleUID := createActor().GetMetadata().GetUid() - if _, err := tc.client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: testActorID}, - }); err != nil { - t.Fatalf("DeleteActor failed: %v", err) + if got, err := tc.client.GetActorSnapshotTag(context.Background(), &ateapipb.GetActorSnapshotTagRequest{Tag: tagRef}); err != nil || !proto.Equal(got.GetSnapshot(), ref) { + t.Fatalf("tag after publication = (%v, %v), want same address", got, err) } - - created := createActor() - staleVersion := created.GetMetadata().GetVersion() - uid := created.GetMetadata().GetUid() - if uid == staleUID { - t.Fatalf("recreated actor reused uid %s, want a fresh one", uid) + createAtespace(t, tc, "other") + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "other", Name: "cross-atespace"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + SourceSnapshot: &ateapipb.ActorSnapshotSource{Tag: tagRef}, + }, + }); err != nil { + t.Fatalf("CreateActor from published tag failed: %v", err) } - // The uid from the deleted lifecycle must be rejected, even though the - // atespace/name it was observed under still resolves. - _, err := update(&ateapipb.ResourceMetadata{Uid: staleUID}, "other-lifecycle") - assertGrpcError(t, err, codes.Aborted, fmt.Sprintf("actor %s/%s not found with uid %s", testAtespace, testActorID, staleUID)) - // An unguarded update is last-writer-wins, and moves the resource past the - // version observed above. - unguarded, err := update(&ateapipb.ResourceMetadata{}, "free") + clone, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "clone"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + SourceSnapshot: &ateapipb.ActorSnapshotSource{Tag: tagRef}, + }, + }) if err != nil { - t.Fatalf("UpdateActor(no guards) failed: %v", err) + t.Fatalf("CreateActor from snapshot failed: %v", err) } - currentVersion := unguarded.GetMetadata().GetVersion() - if currentVersion <= staleVersion { - t.Fatalf("version = %d, want greater than %d after an update", currentVersion, staleVersion) + if !proto.Equal(clone.GetLatestSnapshot(), ref) { + t.Fatalf("clone latest snapshot = %v, want %v", clone.GetLatestSnapshot(), ref) } - if got := unguarded.GetWorkerSelector().GetMatchLabels()["tier"]; got != "free" { - t.Errorf("worker_selector[tier] = %q, want free", got) + if got := clone.GetSourceSnapshot(); !proto.Equal(got.GetTag(), tagRef) || !proto.Equal(got.GetSnapshot(), ref) || got.GetSnapshotUid() != snapshot.GetMetadata().GetUid() { + t.Fatalf("clone source snapshot = %v, want tag %v, snapshot %v, uid %v", got, tagRef, ref, snapshot.GetMetadata().GetUid()) } - - // The version observed before that write is now stale: rejected rather than - // silently overwriting the concurrent change. - _, err = update(&ateapipb.ResourceMetadata{Version: staleVersion}, "stale") - assertGrpcError(t, err, codes.Aborted, "concurrent update conflict, please retry") - - // Both uid and version matching the observed state: the update goes through. - updated, err := update(&ateapipb.ResourceMetadata{Uid: uid, Version: currentVersion}, "paid") + if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "clone"}}); err != nil { + t.Fatalf("ResumeActor clone failed: %v", err) + } + if !tc.fakeAtelet.RestoreCalled { + t.Error("resuming clone did not restore its source ActorSnapshot") + } + cloneSuspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "clone"}}) if err != nil { - t.Fatalf("UpdateActor(matching guards) failed: %v", err) + t.Fatalf("SuspendActor clone failed: %v", err) } - if got := updated.GetWorkerSelector().GetMatchLabels()["tier"]; got != "paid" { - t.Errorf("worker_selector[tier] = %q, want paid", got) + if cloneSuspended.GetActor().GetLatestSnapshot().GetName() == ref.GetName() { + t.Fatal("clone suspension reused its source snapshot") } - if updated.GetMetadata().GetVersion() <= currentVersion { - t.Errorf("version = %d, want greater than %d", updated.GetMetadata().GetVersion(), currentVersion) + listed, err = tc.client.ListActorSnapshots(context.Background(), &ateapipb.ListActorSnapshotsRequest{Atespace: testAtespace}) + if err != nil || len(listed.GetSnapshots()) != 2 { + t.Fatalf("ListActorSnapshots after clone suspension = (%v, %v), want two", listed, err) } - // The guard the client just satisfied is now stale in turn. - _, err = update(&ateapipb.ResourceMetadata{Version: currentVersion}, "free") - assertGrpcError(t, err, codes.Aborted, "concurrent update conflict, please retry") -} - -func TestUpdateActor_NotFound(t *testing.T) { - ns := namespaceForTest("ns-update-actor-notfound") - tc := setupTest(t, ns) - defer tc.cleanup() - - _, err := tc.client.UpdateActor(context.Background(), &ateapipb.UpdateActorRequest{ - Actor: &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "does-not-exist"}}, - UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"worker_selector"}}, + getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, }) - assertGrpcError(t, err, codes.NotFound, "actor test-atespace/does-not-exist not found") + if err != nil { + t.Fatalf("GetActor failed: %v", err) + } + want := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + Status: ateapipb.Actor_STATUS_SUSPENDED, + } + + if diff := cmp.Diff(want, getResp, + protocmp.Transform(), + ignoreUID, + ignoreVersion, + ignoreTimestamps, + protocmp.IgnoreFields(&ateapipb.Actor{}, "latest_snapshot"), + ); diff != "" { + t.Errorf("GetActor response mismatch (-want +got):\n%s", diff) + } + if _, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}}); err != nil { + t.Fatalf("DeleteActor source failed: %v", err) + } + if _, err := tc.client.GetActorSnapshotTag(context.Background(), &ateapipb.GetActorSnapshotTagRequest{Tag: tagRef}); err != nil { + t.Fatalf("source snapshot tag disappeared with source Actor: %v", err) + } + if deleted, err := tc.client.DeleteActorSnapshotTag(context.Background(), &ateapipb.DeleteActorSnapshotTagRequest{Tag: tagRef}); err != nil || deleted.GetMetadata().GetName() != tagRef.GetName() { + t.Fatalf("DeleteActorSnapshotTag = (%v, %v)", deleted, err) + } + if _, err := tc.client.GetActorSnapshotTag(context.Background(), &ateapipb.GetActorSnapshotTagRequest{Tag: tagRef}); status.Code(err) != codes.NotFound { + t.Fatalf("deleted tag status = %v, want NotFound", status.Code(err)) + } + if _, err := tc.client.GetActorSnapshot(context.Background(), &ateapipb.GetActorSnapshotRequest{Snapshot: snapshotRef}); err != nil { + t.Fatalf("snapshot metadata disappeared after tag deletion: %v", err) + } } -// TestUpdateActorSnapshotTag_Preconditions verifies the optional version and uid -// guards carried in the tag's metadata. -func TestUpdateActorSnapshotTag_Preconditions(t *testing.T) { - ns := namespaceForTest("ns-update-tag-preconditions") +// TestPauseActor tests the full workflow of pausing a running actor. +// Workflow: +// 1. Creates a mock ActorTemplate. +// 2. Creates a mock Atelet Pod on 'node1'. +// 3. Creates a mock worker Pod on 'node1'. +// 4. Waits for the WorkerPoolSyncer to mirror the worker to Redis. +// 5. Creates an actor. +// 6. Calls ResumeActor to transition it to RUNNING. +// 7. Calls PauseActor RPC. +// 8. Verifies that the fake Atelet received the Pause call. +func TestPauseActor(t *testing.T) { + ns := namespaceForTest("ns-pause") tc := setupTest(t, ns) defer tc.cleanup() createTemplate(t, tc, ns) - ctx := context.Background() - const snapshotName, tagName = "snapshot-1", "before-upgrade" - snapshotRef := createActorSnapshot(t, tc, snapshotName) - - // Each call to update() flips the scope, so every accepted update is an - // observable write that bumps the version. - update := func(meta *ateapipb.ResourceMetadata, scope ateapipb.ActorSnapshotTagScope) (*ateapipb.ActorSnapshotTag, error) { - return updateActorSnapshotTagScope(tc, tagName, meta, scope) - } + createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - // Delete and recreate the same atespace/name tag, so the first lifecycle's - // uid becomes stale. - staleUID := tagActorSnapshot(t, tc, snapshotRef, tagName).GetMetadata().GetUid() - if _, err := tc.client.DeleteActorSnapshotTag(ctx, &ateapipb.DeleteActorSnapshotTagRequest{ - Tag: &ateapipb.ObjectRef{Atespace: testAtespace, Name: tagName}, - }); err != nil { - t.Fatalf("DeleteActorSnapshotTag failed: %v", err) + name := "id1" + _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}) + if err != nil { + t.Fatalf("CreateActor failed: %v", err) } - tagged := tagActorSnapshot(t, tc, snapshotRef, tagName) - staleVersion := tagged.GetMetadata().GetVersion() - uid := tagged.GetMetadata().GetUid() - if uid == staleUID { - t.Fatalf("recreated tag reused uid %s, want a fresh one", uid) + // Resume first to make it running + _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) + if err != nil { + t.Fatalf("ResumeActor failed: %v", err) } - // The uid from the deleted lifecycle must be rejected, even though the - // atespace/name it was observed under still resolves. - _, err := update(&ateapipb.ResourceMetadata{Uid: staleUID}, ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED) - assertGrpcError(t, err, codes.Aborted, fmt.Sprintf("ActorSnapshot tag %s/%s not found with uid %s", testAtespace, tagName, staleUID)) - // An unguarded update is last-writer-wins, and moves the tag past the - // version observed above. - unguarded, err := update(&ateapipb.ResourceMetadata{}, ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED) + // Pause + _, err = tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) if err != nil { - t.Fatalf("UpdateActorSnapshotTag(no guards) failed: %v", err) - } - currentVersion := unguarded.GetMetadata().GetVersion() - if currentVersion <= staleVersion { - t.Fatalf("version = %d, want greater than %d after an update", currentVersion, staleVersion) - } - if got, want := unguarded.GetScope(), ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED; got != want { - t.Errorf("scope = %v, want %v", got, want) + t.Fatalf("PauseActor failed: %v", err) } - // The version observed before that write is now stale: rejected rather than - // silently overwriting the concurrent change. - _, err = update(&ateapipb.ResourceMetadata{Version: staleVersion}, ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE) - assertGrpcError(t, err, codes.Aborted, "concurrent update conflict, please retry") + if !tc.fakeAtelet.CheckpointCalled { + t.Errorf("expected atelet Checkpoint to be called") + } - // Both uid and version matching the observed state: the update goes through. - updated, err := update(&ateapipb.ResourceMetadata{Uid: uid, Version: currentVersion}, ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE) + getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}, + }) if err != nil { - t.Fatalf("UpdateActorSnapshotTag(matching guards) failed: %v", err) - } - if got, want := updated.GetScope(), ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE; got != want { - t.Errorf("scope = %v, want %v", got, want) + t.Fatalf("GetActor failed: %v", err) } - if updated.GetMetadata().GetVersion() <= currentVersion { - t.Errorf("version = %d, want greater than %d", updated.GetMetadata().GetVersion(), currentVersion) + want := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: testAtespace}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + Status: ateapipb.Actor_STATUS_PAUSED, + LocalSnapshotInfo: &ateapipb.LocalSnapshotInfo{ + NodeVmsWithLocalSnapshots: []string{"node1"}, + ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + }, } - // The guard the client just satisfied is now stale in turn. - _, err = update(&ateapipb.ResourceMetadata{Version: currentVersion}, ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED) - assertGrpcError(t, err, codes.Aborted, "concurrent update conflict, please retry") + if diff := cmp.Diff(want, getResp, + protocmp.Transform(), + ignoreUID, + ignoreVersion, + ignoreTimestamps, + protocmp.IgnoreFields(&ateapipb.WorkerAssignment{}, "worker_pod_uid"), + protocmp.IgnoreFields(&ateapipb.LocalSnapshotInfo{}, "snapshot_name"), + ); diff != "" { + t.Errorf("GetActor response mismatch (-want +got):\n%s", diff) + } + if getResp.GetLocalSnapshotInfo().GetSnapshotName() == "" { + t.Error("LocalSnapshotInfo.SnapshotName is empty, want the name the pause checkpointed under") + } } -func TestUpdateActorSnapshotTag_NotFound(t *testing.T) { - ns := namespaceForTest("ns-update-tag-notfound") +// Pause stamps the ref identity before resolving the Actor record, so a failed +// lookup still carries who/where; it must not invent template/version, which are +// known only once the record resolves (and stamped on success). +func TestPauseActor_FailedLookupStampsRefIdentityOnly(t *testing.T) { + ns := namespaceForTest("ns-span-pause-err") tc := setupTest(t, ns) defer tc.cleanup() - _, err := tc.client.UpdateActorSnapshotTag(context.Background(), &ateapipb.UpdateActorSnapshotTagRequest{ - Tag: &ateapipb.ActorSnapshotTag{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "does-not-exist"}, - Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED, - }, - UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"scope"}}, + attrs := recordRootSpanAttrs(t, func(ctx context.Context) { + if _, err := tc.service.PauseActor(ctx, &ateapipb.PauseActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: testActorID}, + }); status.Code(err) != codes.NotFound { + t.Fatalf("PauseActor(missing) error = %v, want code NotFound", err) + } }) - assertGrpcError(t, err, codes.NotFound, "ActorSnapshot tag test-atespace/does-not-exist not found") + + assertSpanStr(t, attrs, ateattr.AtespaceKey, testAtespace) + assertSpanStr(t, attrs, ateattr.ActorNameKey, testActorID) + for _, k := range []attribute.Key{ateattr.ActorUIDKey, ateattr.TemplateNameKey, ateattr.TemplateNamespaceKey, ateattr.ActorVersionKey} { + if _, ok := attrs[k]; ok { + t.Errorf("unexpected %s on failed-pause span", k) + } + } } // TestResumeActor_ReleasesStaleWorkerWhenPoolBecomesIneligible verifies that @@ -2869,83 +2514,12 @@ func TestUpdateActor_ReassignsPoolAcrossSuspendResume(t *testing.T) { if got := getResp.GetWorkerAssignment().GetWorkerPool(); got != "pool-b" { t.Errorf("expected actor to resume onto pool-b after selector update, got worker_assignment.worker_pool=%q", got) } - if got := getResp.GetWorkerAssignment().GetWorkerPod(); got != "worker-b" { - t.Errorf("expected actor to resume onto worker-b after selector update, got worker_assignment.worker_pod=%q", got) - } - if got := getResp.GetStatus(); got != ateapipb.Actor_STATUS_RUNNING { - t.Errorf("expected actor status RUNNING after second resume, got %v", got) - } -} - -func TestValidation(t *testing.T) { - ns := namespaceForTest("ns-validation") - tc := setupTest(t, ns) - defer tc.cleanup() - - t.Run("CreateActor", func(t *testing.T) { - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") - }) - - t.Run("GetActor", func(t *testing.T) { - _, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") - }) - - t.Run("ResumeActor", func(t *testing.T) { - _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") - }) - - t.Run("PauseActor", func(t *testing.T) { - _, err := tc.client.PauseActor(context.Background(), &ateapipb.PauseActorRequest{}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") - }) - - t.Run("SuspendActor", func(t *testing.T) { - _, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") - }) - - t.Run("UpdateActor", func(t *testing.T) { - _, err := tc.client.UpdateActor(context.Background(), &ateapipb.UpdateActorRequest{}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") - }) - - t.Run("DeleteActor", func(t *testing.T) { - _, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "actor: Required value") - }) - - t.Run("ListActors", func(t *testing.T) { - _, err := tc.client.ListActors(context.Background(), &ateapipb.ListActorsRequest{PageSize: -1}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "page_size: Invalid value") - }) - - t.Run("ListWorkers", func(t *testing.T) { - _, err := tc.client.ListWorkers(context.Background(), &ateapipb.ListWorkersRequest{PageSize: -1}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "page_size: Invalid value") - }) - - t.Run("ListAtespaces", func(t *testing.T) { - _, err := tc.client.ListAtespaces(context.Background(), &ateapipb.ListAtespacesRequest{PageSize: -1}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "page_size: Invalid value") - }) - - t.Run("CreateAtespace", func(t *testing.T) { - _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "atespace: Required value") - }) - - t.Run("GetAtespace", func(t *testing.T) { - _, err := tc.client.GetAtespace(context.Background(), &ateapipb.GetAtespaceRequest{}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "atespace: Required value") - }) - - t.Run("DeleteAtespace", func(t *testing.T) { - _, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{}) - assertGrpcErrorRegex(t, err, codes.InvalidArgument, "atespace: Required value") - }) + if got := getResp.GetWorkerAssignment().GetWorkerPod(); got != "worker-b" { + t.Errorf("expected actor to resume onto worker-b after selector update, got worker_assignment.worker_pod=%q", got) + } + if got := getResp.GetStatus(); got != ateapipb.Actor_STATUS_RUNNING { + t.Errorf("expected actor status RUNNING after second resume, got %v", got) + } } func TestResumeActor_LockConflict(t *testing.T) { @@ -3129,374 +2703,6 @@ func TestSuspendActor_DanglingWorker(t *testing.T) { } } -func TestDeleteActor_Success(t *testing.T) { - ns := namespaceForTest("ns-delete-success") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - - deleted, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, - }) - if err != nil { - t.Fatalf("DeleteActor failed: %v", err) - } - // DeleteActor returns the deleted resource. - if got := deleted.GetMetadata().GetName(); got != "id1" { - t.Errorf("deleted actor name = %q, want id1", got) - } - if got := deleted.GetMetadata().GetAtespace(); got != testAtespace { - t.Errorf("deleted actor atespace = %q, want %q", got, testAtespace) - } - - _, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, - }) - assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found") -} - -func TestDeleteActor_NotSuspended(t *testing.T) { - ns := namespaceForTest("ns-delete-notsuspended") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") - - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}) - if err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - - _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, - }) - if err != nil { - t.Fatalf("ResumeActor failed: %v", err) - } - - _, err = tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, - }) - assertGrpcError(t, err, codes.FailedPrecondition, "Actor test-atespace/id1 is not in a deletable status (status: STATUS_RUNNING)") -} - -func TestDeleteActor_Crashed(t *testing.T) { - ns := namespaceForTest("ns-delete-crashed") - tc := setupTest(t, ns) - defer tc.cleanup() - - createTemplate(t, tc, ns) - - if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}); err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - - actorRef := resources.ActorRef{Atespace: testAtespace, Name: "id1"} - if _, err := tc.persistence.UpdateActor(context.Background(), actorRef, func(toUpdate *ateapipb.Actor) error { - toUpdate.Status = ateapipb.Actor_STATUS_CRASHED - return nil - }); err != nil { - t.Fatalf("UpdateActor failed: %v", err) - } - - deleted, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, - }) - if err != nil { - t.Fatalf("DeleteActor of crashed actor failed: %v", err) - } - if got := deleted.GetStatus(); got != ateapipb.Actor_STATUS_DELETING { - t.Errorf("deleted actor status = %v, want %v", got, ateapipb.Actor_STATUS_DELETING) - } - - _, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"}, - }) - assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/id1 not found") -} - -func TestDeleteActor_NotFound(t *testing.T) { - ns := namespaceForTest("ns-delete-notfound") - tc := setupTest(t, ns) - defer tc.cleanup() - - _, err := tc.client.DeleteActor(context.Background(), &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "non-existent"}, - }) - assertGrpcError(t, err, codes.NotFound, "Actor test-atespace/non-existent not found") -} - -func assertGrpcErrorRegex(t *testing.T, err error, wantCode codes.Code, wantMsg string) { - t.Helper() - fn := func(got string) (string, bool) { - matched, matchErr := regexp.MatchString(wantMsg, got) - if matchErr != nil { - t.Fatalf("failed to compile regex %q: %v", wantMsg, matchErr) - } - return wantMsg, matched - } - assertGrpcErrorImpl(t, err, wantCode, fn) -} - -func assertGrpcError(t *testing.T, err error, wantCode codes.Code, wantMsg string) { - t.Helper() - fn := func(got string) (string, bool) { - return wantMsg, got == wantMsg - } - assertGrpcErrorImpl(t, err, wantCode, fn) -} - -func assertGrpcErrorImpl(t *testing.T, err error, wantCode codes.Code, msgMatches func(got string) (string, bool)) { - t.Helper() - if err == nil { - t.Fatalf("expected error, got nil") - } - st, ok := status.FromError(err) - if !ok { - t.Fatalf("expected gRPC status error, got: %v", err) - } - if st.Code() != wantCode { - t.Errorf("expected status %v, got %v", wantCode, st.Code()) - } - if want, ok := msgMatches(st.Message()); !ok { - t.Errorf("expected message %q, got %q", want, st.Message()) - } -} - -func TestCreateActor_AtespaceNotFound(t *testing.T) { - ns := namespaceForTest("ns-create-actor-no-atespace") - tc := setupTest(t, ns) - defer tc.cleanup() - createTemplate(t, tc, ns) - - // The template exists, but "missing-as" was never created. The template - // check fires first, so reaching this error proves the atespace check ran. - _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "missing-as", Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}) - assertGrpcError(t, err, codes.FailedPrecondition, "Atespace missing-as not found") -} - -func TestCreateAtespace_Success(t *testing.T) { - ns := namespaceForTest("ns-create-atespace") - tc := setupTest(t, ns) - defer tc.cleanup() - createTemplate(t, tc, ns) - - resp, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{ - Atespace: &ateapipb.Atespace{ - Metadata: &ateapipb.ResourceMetadata{ - Name: "team-a", - Uid: "caller-supplied-uid", - Version: 999, - CreateTime: timestamppb.New(time.Unix(1, 0)), - UpdateTime: timestamppb.New(time.Unix(1, 0)), - }, - }, - }) - if err != nil { - t.Fatalf("CreateAtespace failed: %v", err) - } - md := resp.GetMetadata() - if md.GetName() != "team-a" { - t.Errorf("Name = %q, want team-a", md.GetName()) - } - if md.GetAtespace() != "" { - t.Errorf("Atespace = %q, want empty (global-scoped)", md.GetAtespace()) - } - if md.GetVersion() != 1 { - t.Errorf("Version = %d, want 1 (caller-set 999 must be ignored)", md.GetVersion()) - } - if md.GetUid() == "" || md.GetUid() == "caller-supplied-uid" { - t.Errorf("uid = %q, want a server-generated value", md.GetUid()) - } - - if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{ - Atespace: "team-a", - Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}); err != nil { - t.Errorf("CreateActor into freshly created atespace failed: %v", err) - } -} - -func TestCreateAtespace_AlreadyExists(t *testing.T) { - ns := namespaceForTest("ns-create-atespace-dup") - tc := setupTest(t, ns) - defer tc.cleanup() - - if _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team-a"}}}); err != nil { - t.Fatalf("first CreateAtespace failed: %v", err) - } - _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team-a"}}}) - assertGrpcError(t, err, codes.AlreadyExists, "Atespace team-a already exists") -} - -func TestGetAtespace_Found(t *testing.T) { - ns := namespaceForTest("ns-get-atespace") - tc := setupTest(t, ns) - defer tc.cleanup() - - created, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team-a"}}}) - if err != nil { - t.Fatalf("CreateAtespace failed: %v", err) - } - resp, err := tc.client.GetAtespace(context.Background(), &ateapipb.GetAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}) - if err != nil { - t.Fatalf("GetAtespace failed: %v", err) - } - if diff := cmp.Diff(created, resp, protocmp.Transform()); diff != "" { - t.Errorf("GetAtespace mismatch (-created +got):\n%s", diff) - } -} - -func TestGetAtespace_NotFound(t *testing.T) { - ns := namespaceForTest("ns-get-atespace-missing") - tc := setupTest(t, ns) - defer tc.cleanup() - - _, err := tc.client.GetAtespace(context.Background(), &ateapipb.GetAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "nope"}}) - assertGrpcError(t, err, codes.NotFound, "Atespace nope not found") -} - -func TestListAtespaces(t *testing.T) { - ns := namespaceForTest("ns-list-atespaces") - tc := setupTest(t, ns) - defer tc.cleanup() - - for _, n := range []string{"team-a", "team-b"} { - if _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: n}}}); err != nil { - t.Fatalf("CreateAtespace(%s) failed: %v", n, err) - } - } - resp, err := tc.client.ListAtespaces(context.Background(), &ateapipb.ListAtespacesRequest{}) - if err != nil { - t.Fatalf("ListAtespaces failed: %v", err) - } - got := map[string]bool{} - for _, a := range resp.GetAtespaces() { - got[a.GetMetadata().GetName()] = true - } - // setupTest seeds testAtespace; team-a and team-b were created above. - for _, n := range []string{testAtespace, "team-a", "team-b"} { - if !got[n] { - t.Errorf("ListAtespaces missing %q; got %v", n, got) - } - } -} - -func TestDeleteAtespace_Empty_Success(t *testing.T) { - ns := namespaceForTest("ns-delete-atespace-empty") - tc := setupTest(t, ns) - defer tc.cleanup() - - if _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team-a"}}}); err != nil { - t.Fatalf("CreateAtespace failed: %v", err) - } - deleted, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}) - if err != nil { - t.Fatalf("DeleteAtespace failed: %v", err) - } - // DeleteAtespace returns the deleted resource. - if got := deleted.GetMetadata().GetName(); got != "team-a" { - t.Errorf("deleted atespace name = %q, want team-a", got) - } - - _, err = tc.client.GetAtespace(context.Background(), &ateapipb.GetAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}) - assertGrpcError(t, err, codes.NotFound, "Atespace team-a not found") -} - -func TestDeleteAtespace_NonEmpty_Rejected(t *testing.T) { - ns := namespaceForTest("ns-delete-atespace-nonempty") - tc := setupTest(t, ns) - defer tc.cleanup() - createTemplate(t, tc, ns) - - if _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team-a"}}}); err != nil { - t.Fatalf("CreateAtespace failed: %v", err) - } - if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}); err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - _, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}) - assertGrpcError(t, err, codes.FailedPrecondition, "Atespace team-a is not empty") - // The atespace must survive a rejected delete. - if _, err := tc.client.GetAtespace(context.Background(), &ateapipb.GetAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}); err != nil { - t.Errorf("atespace should survive a rejected delete, got %v", err) - } -} - -// TestDeleteAtespace_ScopedToTargetAtespace pins (at the RPC layer) that the -// emptiness check is scoped to the target atespace: deleting an empty atespace -// succeeds even when a different atespace holds actors. -func TestDeleteAtespace_ScopedToTargetAtespace(t *testing.T) { - ns := namespaceForTest("ns-delete-atespace-scoped") - tc := setupTest(t, ns) - defer tc.cleanup() - createTemplate(t, tc, ns) - createAtespace(t, tc, "team-a") - createAtespace(t, tc, "team-b") - - // Actor only in team-b. - if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "team-b", Name: "id1"}, - ActorTemplateNamespace: ns, - ActorTemplateName: "tmpl1", - }}); err != nil { - t.Fatalf("CreateActor failed: %v", err) - } - - // Empty team-a deletes fine despite team-b holding an actor. - if _, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}); err != nil { - t.Errorf("DeleteAtespace(team-a, empty) failed: %v", err) - } - // team-b is still non-empty → rejected. - _, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-b"}}) - assertGrpcError(t, err, codes.FailedPrecondition, "Atespace team-b is not empty") -} - -func TestDeleteAtespace_NotFound(t *testing.T) { - ns := namespaceForTest("ns-delete-atespace-missing") - tc := setupTest(t, ns) - defer tc.cleanup() - - _, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "nope"}}) - assertGrpcError(t, err, codes.NotFound, "Atespace nope not found") -} - -func assertValidateErr(t *testing.T, got field.ErrorList, want field.ErrorList) { - t.Helper() - field.ErrorMatcher{}.ByType().ByField().ByValue().Test(t, want, got) -} - // TestSuspendActor_FromPaused suspends a PAUSED actor end-to-end: instead of // checkpointing a running workload, ateapi asks the atelet on the node // holding the pause snapshot to upload it, then finalizes the actor with a diff --git a/cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go b/cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go new file mode 100644 index 0000000000..8fb79e3dbc --- /dev/null +++ b/cmd/ateapi/internal/controlapi/functionaltest/atespace_test.go @@ -0,0 +1,247 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package functionaltest + +import ( + "context" + "testing" + "time" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestCreateAtespace_Success(t *testing.T) { + ns := namespaceForTest("ns-create-atespace") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) + + resp, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{ + Atespace: &ateapipb.Atespace{ + Metadata: &ateapipb.ResourceMetadata{ + Name: "team-a", + Uid: "caller-supplied-uid", + Version: 999, + CreateTime: timestamppb.New(time.Unix(1, 0)), + UpdateTime: timestamppb.New(time.Unix(1, 0)), + }, + }, + }) + if err != nil { + t.Fatalf("CreateAtespace failed: %v", err) + } + md := resp.GetMetadata() + if md.GetName() != "team-a" { + t.Errorf("Name = %q, want team-a", md.GetName()) + } + if md.GetAtespace() != "" { + t.Errorf("Atespace = %q, want empty (global-scoped)", md.GetAtespace()) + } + if md.GetVersion() != 1 { + t.Errorf("Version = %d, want 1 (caller-set 999 must be ignored)", md.GetVersion()) + } + if md.GetUid() == "" || md.GetUid() == "caller-supplied-uid" { + t.Errorf("uid = %q, want a server-generated value", md.GetUid()) + } + + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: "team-a", + Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}); err != nil { + t.Errorf("CreateActor into freshly created atespace failed: %v", err) + } +} + +func TestCreateAtespace_AlreadyExists(t *testing.T) { + ns := namespaceForTest("ns-create-atespace-dup") + tc := setupTest(t, ns) + defer tc.cleanup() + + if _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team-a"}}}); err != nil { + t.Fatalf("first CreateAtespace failed: %v", err) + } + _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team-a"}}}) + assertGrpcError(t, err, codes.AlreadyExists, "Atespace team-a already exists") +} + +func TestGetAtespace_Found(t *testing.T) { + ns := namespaceForTest("ns-get-atespace") + tc := setupTest(t, ns) + defer tc.cleanup() + + created, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team-a"}}}) + if err != nil { + t.Fatalf("CreateAtespace failed: %v", err) + } + resp, err := tc.client.GetAtespace(context.Background(), &ateapipb.GetAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}) + if err != nil { + t.Fatalf("GetAtespace failed: %v", err) + } + if diff := cmp.Diff(created, resp, protocmp.Transform()); diff != "" { + t.Errorf("GetAtespace mismatch (-created +got):\n%s", diff) + } +} + +func TestGetAtespace_NotFound(t *testing.T) { + ns := namespaceForTest("ns-get-atespace-missing") + tc := setupTest(t, ns) + defer tc.cleanup() + + _, err := tc.client.GetAtespace(context.Background(), &ateapipb.GetAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "nope"}}) + assertGrpcError(t, err, codes.NotFound, "Atespace nope not found") +} + +func TestListAtespaces(t *testing.T) { + ns := namespaceForTest("ns-list-atespaces") + tc := setupTest(t, ns) + defer tc.cleanup() + + for _, n := range []string{"team-a", "team-b"} { + if _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: n}}}); err != nil { + t.Fatalf("CreateAtespace(%s) failed: %v", n, err) + } + } + resp, err := tc.client.ListAtespaces(context.Background(), &ateapipb.ListAtespacesRequest{}) + if err != nil { + t.Fatalf("ListAtespaces failed: %v", err) + } + got := map[string]bool{} + for _, a := range resp.GetAtespaces() { + got[a.GetMetadata().GetName()] = true + } + // setupTest seeds testAtespace; team-a and team-b were created above. + for _, n := range []string{testAtespace, "team-a", "team-b"} { + if !got[n] { + t.Errorf("ListAtespaces missing %q; got %v", n, got) + } + } +} + +func TestDeleteAtespace_Empty_Success(t *testing.T) { + ns := namespaceForTest("ns-delete-atespace-empty") + tc := setupTest(t, ns) + defer tc.cleanup() + + if _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team-a"}}}); err != nil { + t.Fatalf("CreateAtespace failed: %v", err) + } + deleted, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}) + if err != nil { + t.Fatalf("DeleteAtespace failed: %v", err) + } + // DeleteAtespace returns the deleted resource. + if got := deleted.GetMetadata().GetName(); got != "team-a" { + t.Errorf("deleted atespace name = %q, want team-a", got) + } + + _, err = tc.client.GetAtespace(context.Background(), &ateapipb.GetAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}) + assertGrpcError(t, err, codes.NotFound, "Atespace team-a not found") +} + +func TestDeleteAtespace_NonEmpty_Rejected(t *testing.T) { + ns := namespaceForTest("ns-delete-atespace-nonempty") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) + + if _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "team-a"}}}); err != nil { + t.Fatalf("CreateAtespace failed: %v", err) + } + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + _, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}) + assertGrpcError(t, err, codes.FailedPrecondition, "Atespace team-a is not empty") + // The atespace must survive a rejected delete. + if _, err := tc.client.GetAtespace(context.Background(), &ateapipb.GetAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}); err != nil { + t.Errorf("atespace should survive a rejected delete, got %v", err) + } +} + +// TestDeleteAtespace_ScopedToTargetAtespace pins (at the RPC layer) that the +// emptiness check is scoped to the target atespace: deleting an empty atespace +// succeeds even when a different atespace holds actors. +func TestDeleteAtespace_ScopedToTargetAtespace(t *testing.T) { + ns := namespaceForTest("ns-delete-atespace-scoped") + tc := setupTest(t, ns) + defer tc.cleanup() + createTemplate(t, tc, ns) + createAtespace(t, tc, "team-a") + createAtespace(t, tc, "team-b") + + // Actor only in team-b. + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-b", Name: "id1"}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + }}); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + + // Empty team-a deletes fine despite team-b holding an actor. + if _, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-a"}}); err != nil { + t.Errorf("DeleteAtespace(team-a, empty) failed: %v", err) + } + // team-b is still non-empty → rejected. + _, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "team-b"}}) + assertGrpcError(t, err, codes.FailedPrecondition, "Atespace team-b is not empty") +} + +func TestDeleteAtespace_NotFound(t *testing.T) { + ns := namespaceForTest("ns-delete-atespace-missing") + tc := setupTest(t, ns) + defer tc.cleanup() + + _, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{Atespace: &ateapipb.ObjectRef{Name: "nope"}}) + assertGrpcError(t, err, codes.NotFound, "Atespace nope not found") +} + +func TestValidation_Atespace(t *testing.T) { + ns := namespaceForTest("ns-validation-atespace") + tc := setupTest(t, ns) + defer tc.cleanup() + + t.Run("ListAtespaces", func(t *testing.T) { + _, err := tc.client.ListAtespaces(context.Background(), &ateapipb.ListAtespacesRequest{PageSize: -1}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "page_size: Invalid value") + }) + + t.Run("CreateAtespace", func(t *testing.T) { + _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "atespace: Required value") + }) + + t.Run("GetAtespace", func(t *testing.T) { + _, err := tc.client.GetAtespace(context.Background(), &ateapipb.GetAtespaceRequest{}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "atespace: Required value") + }) + + t.Run("DeleteAtespace", func(t *testing.T) { + _, err := tc.client.DeleteAtespace(context.Background(), &ateapipb.DeleteAtespaceRequest{}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "atespace: Required value") + }) +} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go new file mode 100644 index 0000000000..971289718a --- /dev/null +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -0,0 +1,691 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package functionaltest + +import ( + "context" + "fmt" + "net" + "regexp" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/ateredis" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" + "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/internal/volume" + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "github.com/agent-substrate/substrate/pkg/client/clientset/versioned" + "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" + listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/fieldmaskpb" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" +) + +const ( + testAtespace = "test-atespace" + testActorID = "id1" +) + +var ( + ignoreUID = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "uid") + ignoreVersion = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "version") + ignoreTimestamps = protocmp.IgnoreFields(&ateapipb.ResourceMetadata{}, "create_time", "update_time") +) + +type testContext struct { + mr *miniredis.Miniredis + service *controlapi.Service + client ateapipb.ControlClient + k8sClient kubernetes.Interface + substrateClient versioned.Interface + persistence *ateredis.Persistence + workerCache *workercache.Cache + fakeAtelet *FakeAteletServer + cleanup func() + actorTemplateLister listersv1alpha1.ActorTemplateLister + workerPoolLister listersv1alpha1.WorkerPoolLister + sandboxConfigLister listersv1alpha1.SandboxConfigLister +} + +// setupTest sets up a fully isolated test environment. +func setupTest(t *testing.T, ns string) *testContext { + t.Helper() + return setupTestWithVolumePlugins(t, ns, nil) +} + +// setupTestWithVolumePlugins is setupTest with the default mock volume plugin +// replaced by plugins, keyed by driver name. Tests that need a failure-injecting +// plugin pass it here rather than swapping it into the running Service, so each +// test owns its own plugin set. +func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volume.VolumePluginControlPlane) *testContext { + t.Helper() + // 1. Start Miniredis + mr, err := miniredis.Run() + if err != nil { + t.Fatalf("failed to start miniredis: %v", err) + } + + rdb := redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: []string{mr.Addr()}, + }) + persistence := ateredis.NewPersistence(rdb) + + // 2. Initialize Clientsets using global cfg + k8sClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + mr.Close() + t.Fatalf("failed to create k8s clientset: %v", err) + } + + substrateClient, err := versioned.NewForConfig(cfg) + if err != nil { + mr.Close() + t.Fatalf("failed to create substrate clientset: %v", err) + } + + // 3. Initialize Informers + workerFactory, workerInformer := controlapi.WorkerPodInformer(k8sClient) + ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient) + scFactory := informers.NewSharedInformerFactory(k8sClient, 0) + scLister := scFactory.Storage().V1().StorageClasses().Lister() + + substrateInformerFactory := externalversions.NewSharedInformerFactory(substrateClient, 0) + actorTemplateLister := substrateInformerFactory.Api().V1alpha1().ActorTemplates().Lister() + workerPoolLister := substrateInformerFactory.Api().V1alpha1().WorkerPools().Lister() + sandboxConfigLister := substrateInformerFactory.Api().V1alpha1().SandboxConfigs().Lister() + csiDriverConfigLister := substrateInformerFactory.Api().V1alpha1().CSIDriverConfigs().Lister() + + ctx, cancel := context.WithCancel(context.Background()) + + syncer := controlapi.NewWorkerPoolSyncer(persistence, workerInformer, workerPoolLister) + syncer.Start(ctx) + + workerFactory.Start(ctx.Done()) + ateletFactory.Start(ctx.Done()) + substrateInformerFactory.Start(ctx.Done()) + scFactory.Start(ctx.Done()) + + workerFactory.WaitForCacheSync(ctx.Done()) + ateletFactory.WaitForCacheSync(ctx.Done()) + substrateInformerFactory.WaitForCacheSync(ctx.Done()) + scFactory.WaitForCacheSync(ctx.Done()) + + // 4. Initialize Service + wc := workercache.New(persistence, 5*time.Minute) + if err := wc.Start(ctx); err != nil { + cancel() + mr.Close() + t.Fatalf("failed to start worker cache: %v", err) + } + + // Dial the fake atelet over insecure transport instead of per-atelet mTLS, + // so DialForWorker's real lookup/dial/cache path is exercised under test. + dialer := controlapi.NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer(), "", "", + controlapi.WithDialCredentials(func(_ string) (credentials.TransportCredentials, error) { + return insecure.NewCredentials(), nil + })) + + instruments, err := controlapi.NewInstruments(sdkmetric.NewMeterProvider(sdkmetric.WithReader(sdkmetric.NewManualReader())).Meter("ateapi")) + if err != nil { + cancel() + mr.Close() + t.Fatalf("failed to create metric instruments: %v", err) + } + volPlugins := plugins + if volPlugins == nil { + mockPlugin := volume.NewMockVolumePlugin() + mockDriverName, err := mockPlugin.DriverName(ctx) + if err != nil { + t.Fatalf("failed to get mock driver name: %v", err) + } + volPlugins = map[string]volume.VolumePluginControlPlane{ + mockDriverName: mockPlugin, + } + } + service := controlapi.NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, scLister, dialer, instruments, "", volPlugins) + + // 5. Start REAL gRPC Server for ATE API + grpcServer := grpc.NewServer(grpc.UnaryInterceptor(ateinterceptors.ServerUnaryInterceptor)) + ateapipb.RegisterControlServer(grpcServer, service) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + cancel() + mr.Close() + t.Fatalf("failed to listen: %v", err) + } + + go func() { + if err := grpcServer.Serve(lis); err != nil { + t.Logf("grpc server exited: %v", err) + } + }() + + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + grpcServer.Stop() + cancel() + mr.Close() + t.Fatalf("failed to connect: %v", err) + } + + client := ateapipb.NewControlClient(conn) + + // Call Reset on global mock + fakeAtelet.Reset() + + // Create namespace + _, err = k8sClient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + }, metav1.CreateOptions{}) + if err != nil { + conn.Close() + grpcServer.Stop() + cancel() + mr.Close() + t.Fatalf("failed to create namespace %s: %v", ns, err) + } + + // CreateActor now requires the atespace to exist first. + if _, err := client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: testAtespace}}}); err != nil { + conn.Close() + grpcServer.Stop() + cancel() + mr.Close() + t.Fatalf("failed to seed test atespace %q: %v", testAtespace, err) + } + + cleanup := func() { + conn.Close() + grpcServer.Stop() + cancel() + rdb.Close() + mr.Close() + } + + return &testContext{ + mr: mr, + service: service, + client: client, + k8sClient: k8sClient, + substrateClient: substrateClient, + persistence: persistence, + workerCache: wc, + fakeAtelet: fakeAtelet, + cleanup: cleanup, + actorTemplateLister: actorTemplateLister, + workerPoolLister: workerPoolLister, + sandboxConfigLister: sandboxConfigLister, + } +} + +func namespaceForTest(baseName string) string { + return fmt.Sprintf("%s-%d", baseName, time.Now().UnixNano()) +} + +func createTemplate(t *testing.T, tc *testContext, ns string) { + t.Helper() + createTemplateWithContainers(t, tc, ns, []atev1alpha1.Container{ + { + Name: "main", + Image: "main@sha256:abc", + Command: []string{"/main"}, + }, + }) +} + +// createAtespace creates an atespace via the API. +func createAtespace(t *testing.T, tc *testContext, name string) { + t.Helper() + if _, err := tc.client.CreateAtespace(context.Background(), &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: name}}}); err != nil { + t.Fatalf("CreateAtespace(%s) failed: %v", name, err) + } +} + +// createActorSnapshot seeds an ActorSnapshot in testAtespace directly through +// the store, so tag tests do not need a full resume/suspend lifecycle. +func createActorSnapshot(t *testing.T, tc *testContext, name string) *ateapipb.ObjectRef { + t.Helper() + if _, err := tc.persistence.CreateActorSnapshot(context.Background(), &ateapipb.ActorSnapshot{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, + SnapshotUri: "gs://my-bucket/snapshots/" + testAtespace + "/" + name, + }); err != nil { + t.Fatalf("CreateActorSnapshot(%s) failed: %v", name, err) + } + return &ateapipb.ObjectRef{Atespace: testAtespace, Name: name} +} + +// tagActorSnapshot points tagName at snapshotRef with atespace scope. +func tagActorSnapshot(t *testing.T, tc *testContext, snapshotRef *ateapipb.ObjectRef, tagName string) *ateapipb.ActorSnapshotTag { + t.Helper() + tag, err := tc.client.CreateActorSnapshotTag(context.Background(), &ateapipb.CreateActorSnapshotTagRequest{ + ActorSnapshotTag: &ateapipb.ActorSnapshotTag{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: tagName}, + Snapshot: snapshotRef, + Scope: ateapipb.ActorSnapshotTagScope_ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE, + }, + }) + if err != nil { + t.Fatalf("CreateActorSnapshotTag(%s) failed: %v", tagName, err) + } + return tag +} + +// updateActorSnapshotTagScope sets tagName's scope, carrying meta as the +// optional uid/version preconditions. +func updateActorSnapshotTagScope(tc *testContext, tagName string, meta *ateapipb.ResourceMetadata, scope ateapipb.ActorSnapshotTagScope) (*ateapipb.ActorSnapshotTag, error) { + meta.Atespace, meta.Name = testAtespace, tagName + return tc.client.UpdateActorSnapshotTag(context.Background(), &ateapipb.UpdateActorSnapshotTagRequest{ + Tag: &ateapipb.ActorSnapshotTag{Metadata: meta, Scope: scope}, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"scope"}}, + }) +} + +const poolLabelKey = "pool" + +func createTemplateWithContainers(t *testing.T, tc *testContext, ns string, containers []atev1alpha1.Container) { + createTemplateWithContainersAndVolumes(t, tc, ns, containers, nil) +} + +func createTemplateWithVolumes(t *testing.T, tc *testContext, ns string, volumes []atev1alpha1.Volume, mounts []atev1alpha1.VolumeMount) { + createTemplateWithContainersAndVolumes(t, tc, ns, []atev1alpha1.Container{ + { + Name: "main", + Image: "main@sha256:abc", + Command: []string{"/main"}, + VolumeMounts: mounts, + }, + }, volumes) +} + +func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns string, containers []atev1alpha1.Container, volumes []atev1alpha1.Volume) { + t.Helper() + + // Sandbox binaries now live on a (cluster-scoped) SandboxConfig resolved via + // the actor's WorkerPool, not on the ActorTemplate. Create a default gvisor + // SandboxConfig so a boot-from-spec Run can resolve its assets. + ensureDefaultGvisorSandboxConfig(t, tc) + createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns}) + + actorTemplate := &atev1alpha1.ActorTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tmpl1", + Namespace: ns, + }, + Spec: atev1alpha1.ActorTemplateSpec{ + SnapshotsConfig: atev1alpha1.SnapshotsConfig{ + Location: "gs://fake-fake-fake", + }, + Containers: containers, + Volumes: volumes, + WorkerSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{poolLabelKey: ns}, + }, + }, + } + createdTemplate, err := tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).Create(context.Background(), actorTemplate, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create actor template: %v", err) + } + + const goldenSnapshot = "golden" + if _, err := tc.persistence.CreateActorSnapshot(context.Background(), &ateapipb.ActorSnapshot{ + Metadata: &ateapipb.ResourceMetadata{Atespace: resources.GoldenActorAtespace, Name: goldenSnapshot}, + ActorTemplateNamespace: ns, + ActorTemplateName: createdTemplate.GetName(), + ActorTemplateUid: string(createdTemplate.GetUID()), + ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + SnapshotUri: "gs://fake-fake-fake/snapshots/" + resources.GoldenActorAtespace + "/" + goldenSnapshot, + }); err != nil { + t.Fatalf("failed to create golden ActorSnapshot: %v", err) + } + createdTemplate.Status = atev1alpha1.ActorTemplateStatus{ + GoldenSnapshot: goldenSnapshot, + } + + _, err = tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).UpdateStatus(context.Background(), createdTemplate, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("failed to update status: %v", err) + } + + // Wait for Informer cache to sync + err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { + tmpl, err := tc.actorTemplateLister.ActorTemplates(ns).Get("tmpl1") + if err != nil { + return false, nil // Retry if not found in cache yet + } + return tmpl.Status.GoldenSnapshot != "", nil + }) + if err != nil { + t.Fatalf("failed to wait for template status update in informer: %v", err) + } +} + +// testPauseImage is the pause image the default test SandboxConfig carries; +// it is what a resolved WorkloadSpec's sandbox assets should name. +const testPauseImage = "pause@sha256:abc" + +// ensureDefaultGvisorSandboxConfig creates the cluster-scoped default gvisor +// SandboxConfig (idempotently) and waits for it to appear in the lister. +func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) { + t.Helper() + const name = "gvisor-default" + sc := &atev1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: atev1alpha1.SandboxConfigSpec{ + SandboxClass: atev1alpha1.SandboxClassGvisor, + Default: true, + PauseImage: testPauseImage, + Assets: map[string]map[string]atev1alpha1.AssetFile{ + "amd64": {"runsc": { + URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc", + SHA256: "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea842b63", + }}, + "arm64": {"runsc": { + URL: "gs://gvisor/releases/nightly/2026-05-19/aarch64/runsc", + SHA256: "1ba2366ae2efceba166046f51a4104f9261c9cb72c6db8f5b3fe2dc57dea86b9", + }}, + }, + }, + } + if _, err := tc.substrateClient.ApiV1alpha1().SandboxConfigs().Create(context.Background(), sc, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + t.Fatalf("failed to create default SandboxConfig: %v", err) + } + if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { + _, err := tc.sandboxConfigLister.Get(name) + return err == nil, nil + }); err != nil { + t.Fatalf("default SandboxConfig not synced into lister: %v", err) + } +} + +func createWorkerPool(t *testing.T, tc *testContext, ns string, name string, labels map[string]string) { + t.Helper() + wp := &atev1alpha1.WorkerPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: labels, + }, + Spec: atev1alpha1.WorkerPoolSpec{ + Replicas: 1, + AteomImage: "ateom@sha256:abc", + }, + } + _, err := tc.substrateClient.ApiV1alpha1().WorkerPools(ns).Create(context.Background(), wp, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create WorkerPool: %v", err) + } + + err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { + _, err := tc.workerPoolLister.WorkerPools(ns).Get(name) + return err == nil, nil + }) + if err != nil { + t.Fatalf("failed to wait for WorkerPool %s/%s in informer: %v", ns, name, err) + } +} + +func createTemplateWithSelector(t *testing.T, tc *testContext, ns string, name string, selector *metav1.LabelSelector) { + t.Helper() + ensureDefaultGvisorSandboxConfig(t, tc) + actorTemplate := &atev1alpha1.ActorTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: atev1alpha1.ActorTemplateSpec{ + SnapshotsConfig: atev1alpha1.SnapshotsConfig{ + Location: "gs://fake-fake-fake", + }, + Containers: []atev1alpha1.Container{ + {Name: "main", Image: "main@sha256:abc", Command: []string{"/main"}}, + }, + WorkerSelector: selector, + }, + } + _, err := tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).Create(context.Background(), actorTemplate, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create actor template: %v", err) + } + + err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { + _, err := tc.actorTemplateLister.ActorTemplates(ns).Get(name) + return err == nil, nil + }) + if err != nil { + t.Fatalf("failed to wait for template %s/%s in informer: %v", ns, name, err) + } +} + +func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, nodeName string, poolName string) { + t.Helper() + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + UID: "08675309-4a65-6e6e-7973-6e756d626572", + Labels: map[string]string{ + "ate.dev/worker-pool": poolName, + }, + }, + Spec: corev1.PodSpec{ + NodeName: nodeName, + Containers: []corev1.Container{ + {Name: "main", Image: "nginx"}, + }, + }, + } + /* + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: ns, + UID: "08675309-4a65-6e6e-7973-6e756d626572", + Labels: map[string]string{ + workerPodLabel: poolName, + }, + }, + Spec: corev1.PodSpec{ + NodeName: "node1", + Containers: []corev1.Container{{Name: "main", Image: "nginx"}}, + }, + } + + */ + createdPod, err := tc.k8sClient.CoreV1().Pods(ns).Create(context.Background(), pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create worker pod: %v", err) + } + createdPod.Status.PodIPs = []corev1.PodIP{{IP: "127.0.0.1"}} + createdPod.Status.Phase = corev1.PodRunning + _, err = tc.k8sClient.CoreV1().Pods(ns).UpdateStatus(context.Background(), createdPod, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("failed to update worker pod status: %v", err) + } + + // Wait for worker to be registered via API + err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { + resp, err := tc.client.ListWorkers(ctx, &ateapipb.ListWorkersRequest{}) + if err != nil { + return false, nil // Retry on API error + } + for _, w := range resp.GetWorkers() { + if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { + return true, nil + } + } + return false, nil + }) + if err != nil { + t.Fatalf("failed to wait for worker to be registered: %v", err) + } + + // Wait for the worker to appear in worker cache. + err = wait.PollUntilContextTimeout(context.Background(), 10*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { + workers, err := tc.workerCache.Workers() + if err != nil { + return false, nil // Cache not ready yet; retry. + } + for _, w := range workers { + if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { + return true, nil + } + } + return false, nil + }) + if err != nil { + t.Fatalf("failed to wait for worker to appear in worker cache: %v", err) + } +} + +func deleteWorkerPod(t *testing.T, tc *testContext, ns string, name string) { + t.Helper() + err := tc.k8sClient.CoreV1().Pods(ns).Delete(context.Background(), name, metav1.DeleteOptions{ + GracePeriodSeconds: ptr.To[int64](0), + }) + if err != nil { + t.Fatalf("failed to delete worker pod %s: %v", name, err) + } + + // Wait for worker to be removed from API + err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { + resp, err := tc.client.ListWorkers(ctx, &ateapipb.ListWorkersRequest{}) + if err != nil { + return false, nil // Retry on API error + } + for _, w := range resp.GetWorkers() { + if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { + return false, nil // Still there + } + } + return true, nil // Gone! + }) + if err != nil { + t.Fatalf("failed to wait for worker to be removed: %v", err) + } + + err = wait.PollUntilContextTimeout(context.Background(), 10*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { + workers, err := tc.workerCache.Workers() + if err != nil { + return false, nil // Cache not ready yet; retry. + } + for _, w := range workers { + if w.GetWorkerNamespace() == ns && w.GetWorkerPod() == name { + return false, nil // Still there + } + } + return true, nil + }) + if err != nil { + t.Fatalf("failed to wait for worker to be removed from worker cache: %v", err) + } +} + +func assertGrpcErrorRegex(t *testing.T, err error, wantCode codes.Code, wantMsg string) { + t.Helper() + fn := func(got string) (string, bool) { + matched, matchErr := regexp.MatchString(wantMsg, got) + if matchErr != nil { + t.Fatalf("failed to compile regex %q: %v", wantMsg, matchErr) + } + return wantMsg, matched + } + assertGrpcErrorImpl(t, err, wantCode, fn) +} + +func assertGrpcError(t *testing.T, err error, wantCode codes.Code, wantMsg string) { + t.Helper() + fn := func(got string) (string, bool) { + return wantMsg, got == wantMsg + } + assertGrpcErrorImpl(t, err, wantCode, fn) +} + +func assertGrpcErrorImpl(t *testing.T, err error, wantCode codes.Code, msgMatches func(got string) (string, bool)) { + t.Helper() + if err == nil { + t.Fatalf("expected error, got nil") + } + st, ok := status.FromError(err) + if !ok { + t.Fatalf("expected gRPC status error, got: %v", err) + } + if st.Code() != wantCode { + t.Errorf("expected status %v, got %v", wantCode, st.Code()) + } + if want, ok := msgMatches(st.Message()); !ok { + t.Errorf("expected message %q, got %q", want, st.Message()) + } +} + +// recordRootSpanAttrs runs fn under a fresh recording root span from a local +// TracerProvider and returns that span's attributes, so a test can observe what +// the code under test stamps on the span carried in ctx. It never swaps the +// global provider (the code under test reads its span via trace.SpanFromContext, +// not the global provider), so span tests stay parallel-safe. +func recordRootSpanAttrs(t *testing.T, fn func(ctx context.Context)) map[attribute.Key]attribute.Value { + t.Helper() + sr := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr)) + ctx, root := tp.Tracer("test").Start(context.Background(), "root") + fn(ctx) + root.End() + for _, s := range sr.Ended() { + if s.Name() == "root" { + m := make(map[attribute.Key]attribute.Value, len(s.Attributes())) + for _, kv := range s.Attributes() { + m[kv.Key] = kv.Value + } + return m + } + } + t.Fatal("root span not recorded") + return nil +} + +func assertSpanStr(t *testing.T, attrs map[attribute.Key]attribute.Value, key attribute.Key, want string) { + t.Helper() + v, ok := attrs[key] + if !ok { + t.Errorf("missing %s", key) + return + } + if v.AsString() != want { + t.Errorf("%s = %q, want %q", key, v.AsString(), want) + } +} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/main_test.go b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go new file mode 100644 index 0000000000..385bdcf599 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/functionaltest/main_test.go @@ -0,0 +1,229 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package functionaltest + +import ( + "context" + "fmt" + "log" + "net" + "os" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/internal/testenv" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +var ( + cfg *rest.Config + fakeAtelet = &FakeAteletServer{} +) + +func TestMain(m *testing.M) { + var stopEnv func() + cfg, stopEnv = testenv.Start() + + // Create ate-system namespace + k8sClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + log.Fatalf("kubernetes.NewForConfig: %v", err) + } + _, err = k8sClient.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "ate-system"}, + }, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + log.Fatalf("create ate-system namespace: %v", err) + } + + // Create StorageClasses for volume tests + _, err = k8sClient.StorageV1().StorageClasses().Create(context.Background(), &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "standard"}, + Provisioner: "substrate.io/mock", + }, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + log.Fatalf("create standard storage class: %v", err) + } + _, err = k8sClient.StorageV1().StorageClasses().Create(context.Background(), &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "fast"}, + Provisioner: "substrate.io/mock", + }, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + log.Fatalf("create fast storage class: %v", err) + } + + // Create shared Atelet Pod + ateletPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "atelet-shared", + Namespace: "ate-system", + Labels: map[string]string{ + "app": "atelet", + }, + }, + Spec: corev1.PodSpec{ + NodeName: "node1", + Containers: []corev1.Container{ + {Name: "main", Image: "nginx"}, + }, + }, + } + createdAtelet, err := k8sClient.CoreV1().Pods("ate-system").Create(context.Background(), ateletPod, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + log.Fatalf("create atelet pod: %v", err) + } + if err == nil { + createdAtelet.Status.PodIPs = []corev1.PodIP{{IP: "127.0.0.1"}} + createdAtelet.Status.Phase = corev1.PodRunning + _, err = k8sClient.CoreV1().Pods("ate-system").UpdateStatus(context.Background(), createdAtelet, metav1.UpdateOptions{}) + if err != nil { + log.Fatalf("update atelet pod status: %v", err) + } + } + + // Start Fake Atelet Server on port 8085 + ateletGrpcServer := grpc.NewServer() + ateletpb.RegisterAteomHerderServer(ateletGrpcServer, fakeAtelet) + ateletLis, err := net.Listen("tcp", "127.0.0.1:8085") + if err != nil { + log.Fatalf("listen on 127.0.0.1:8085: %v", err) + } + go func() { + if err := ateletGrpcServer.Serve(ateletLis); err != nil { + fmt.Printf("atelet grpc server exited: %v\n", err) + } + }() + + code := m.Run() + + ateletGrpcServer.Stop() + + stopEnv() + + os.Exit(code) +} + +// FakeAteletServer implements ateletpb.WorkersServer +type FakeAteletServer struct { + ateletpb.UnimplementedAteomHerderServer + + Lock sync.Mutex + + RunCalled bool + RunRequest *ateletpb.RunRequest + FailRun error + + CheckpointCalled bool + CheckpointRequest *ateletpb.CheckpointRequest + + RestoreCalled bool + RestoreRequest *ateletpb.RestoreRequest + FailRestore error + RestoreDelay time.Duration + + UploadCalled bool + UploadRequest *ateletpb.UploadPausedCheckpointRequest + FailUpload error +} + +func (f *FakeAteletServer) Reset() { + f.Lock.Lock() + defer f.Lock.Unlock() + + f.RunCalled = false + f.RunRequest = nil + f.FailRun = nil + + f.CheckpointCalled = false + f.CheckpointRequest = nil + + f.RestoreCalled = false + f.RestoreRequest = nil + f.FailRestore = nil + f.RestoreDelay = 0 + + f.UploadCalled = false + f.UploadRequest = nil + f.FailUpload = nil +} + +func (f *FakeAteletServer) UploadPausedCheckpoint(ctx context.Context, req *ateletpb.UploadPausedCheckpointRequest) (*ateletpb.UploadPausedCheckpointResponse, error) { + f.Lock.Lock() + defer f.Lock.Unlock() + + f.UploadCalled = true + f.UploadRequest = proto.Clone(req).(*ateletpb.UploadPausedCheckpointRequest) + if f.FailUpload != nil { + return nil, f.FailUpload + } + return &ateletpb.UploadPausedCheckpointResponse{}, nil +} + +func (f *FakeAteletServer) Run(ctx context.Context, req *ateletpb.RunRequest) (*ateletpb.RunResponse, error) { + f.Lock.Lock() + defer f.Lock.Unlock() + + f.RunCalled = true + f.RunRequest = proto.Clone(req).(*ateletpb.RunRequest) + if f.FailRun != nil { + return nil, f.FailRun + } + + return &ateletpb.RunResponse{}, nil +} + +func (f *FakeAteletServer) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRequest) (*ateletpb.CheckpointResponse, error) { + f.Lock.Lock() + defer f.Lock.Unlock() + + f.CheckpointCalled = true + f.CheckpointRequest = proto.Clone(req).(*ateletpb.CheckpointRequest) + + return &ateletpb.CheckpointResponse{}, nil +} + +func (f *FakeAteletServer) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (*ateletpb.RestoreResponse, error) { + f.Lock.Lock() + defer f.Lock.Unlock() + + f.RestoreCalled = true + f.RestoreRequest = proto.Clone(req).(*ateletpb.RestoreRequest) + if f.RestoreDelay > 0 { + time.Sleep(f.RestoreDelay) + } + if f.FailRestore != nil { + return nil, f.FailRestore + } + return &ateletpb.RestoreResponse{}, nil +} + +func (f *FakeAteletServer) lastRestoreRequest() *ateletpb.RestoreRequest { + f.Lock.Lock() + defer f.Lock.Unlock() + + if f.RestoreRequest == nil { + return nil + } + return proto.Clone(f.RestoreRequest).(*ateletpb.RestoreRequest) +} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go new file mode 100644 index 0000000000..62d45edbc0 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go @@ -0,0 +1,82 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package functionaltest + +import ( + "context" + "testing" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/testing/protocmp" +) + +// TestListWorkers tests that workers mirrored to Redis are listed. +// Workflow: +// 1. Creates a mock WorkerPool in Kubernetes. +// 2. Creates a mock worker Pod in Kubernetes belonging to that pool. +// 3. Waits for the background WorkerPoolSyncer to mirror it to Redis. +// 4. Calls ListWorkers RPC. +// 5. Verifies that the worker appears in the response. +func TestListWorkers(t *testing.T) { + ns := namespaceForTest("ns-list-workers") + tc := setupTest(t, ns) + defer tc.cleanup() + + createWorkerPool(t, tc, ns, "pool1", map[string]string{"foo": "bar"}) + createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + + listResp, err := tc.client.ListWorkers(context.Background(), &ateapipb.ListWorkersRequest{}) + if err != nil { + t.Fatalf("ListWorkers failed: %v", err) + } + + var filteredWorkers []*ateapipb.Worker + for _, w := range listResp.GetWorkers() { + if w.GetWorkerNamespace() == ns { + filteredWorkers = append(filteredWorkers, w) + } + } + + want := []*ateapipb.Worker{ + { + WorkerNamespace: ns, + WorkerPool: "pool1", + WorkerPod: "worker-1", + NodeName: "node1", + Ip: "127.0.0.1", + Version: 1, + SandboxClass: "gvisor", + Labels: map[string]string{"foo": "bar"}, + State: ateapipb.Worker_STATE_ACTIVE, + }, + } + + if diff := cmp.Diff(want, filteredWorkers, protocmp.Transform(), protocmp.IgnoreFields(&ateapipb.Worker{}, "worker_pod_uid")); diff != "" { + t.Errorf("ListWorkers response mismatch (-want +got):\n%s", diff) + } +} + +func TestValidation_Worker(t *testing.T) { + ns := namespaceForTest("ns-validation-worker") + tc := setupTest(t, ns) + defer tc.cleanup() + + t.Run("ListWorkers", func(t *testing.T) { + _, err := tc.client.ListWorkers(context.Background(), &ateapipb.ListWorkersRequest{PageSize: -1}) + assertGrpcErrorRegex(t, err, codes.InvalidArgument, "page_size: Invalid value") + }) +} diff --git a/cmd/ateapi/internal/controlapi/span_identity_test.go b/cmd/ateapi/internal/controlapi/span_identity_test.go index 615d94d0e8..ccebfe293b 100644 --- a/cmd/ateapi/internal/controlapi/span_identity_test.go +++ b/cmd/ateapi/internal/controlapi/span_identity_test.go @@ -31,8 +31,7 @@ import ( // TracerProvider and returns that span's attributes, so a test can observe what // the code under test stamps on the span carried in ctx. It never swaps the // global provider (the code under test reads its span via trace.SpanFromContext, -// not the global provider), so span tests stay parallel-safe. Shared by the -// per-method span tests (create/delete/resume/pause_actor_test.go). +// not the global provider), so span tests stay parallel-safe. func recordRootSpanAttrs(t *testing.T, fn func(ctx context.Context)) map[attribute.Key]attribute.Value { t.Helper() sr := tracetest.NewSpanRecorder()