From ee6c2d2e99ab36c649146e87bf237edd200ec41a Mon Sep 17 00:00:00 2001 From: Rafael Benevides Date: Thu, 3 Sep 2026 13:34:23 -0300 Subject: [PATCH] HYPERFLEET-1574 - feat: OCI CI compartment, quota, sweep, and budget --- .gitignore | 6 + Makefile | 17 + functions/oci-ci-sweep/Dockerfile | 20 + functions/oci-ci-sweep/README.md | 101 ++++ functions/oci-ci-sweep/func.yaml | 6 + functions/oci-ci-sweep/go.mod | 16 + functions/oci-ci-sweep/go.sum | 24 + .../oci-ci-sweep/internal/sweep/decision.go | 86 +++ .../internal/sweep/decision_test.go | 120 ++++ functions/oci-ci-sweep/main.go | 571 ++++++++++++++++++ functions/oci-ci-sweep/main_test.go | 58 ++ terraform/README.md | 7 +- terraform/modules/budget/oci/main.tf | 38 ++ terraform/modules/budget/oci/outputs.tf | 4 + terraform/modules/budget/oci/variables.tf | 52 ++ terraform/modules/compartment/oci/main.tf | 8 + terraform/modules/compartment/oci/outputs.tf | 9 + .../modules/compartment/oci/variables.tf | 28 + terraform/modules/lifecycle/oci/functions.tf | 40 ++ terraform/modules/lifecycle/oci/iam.tf | 76 +++ terraform/modules/lifecycle/oci/network.tf | 77 +++ terraform/modules/lifecycle/oci/outputs.tf | 19 + terraform/modules/lifecycle/oci/scheduler.tf | 15 + terraform/modules/lifecycle/oci/variables.tf | 70 +++ terraform/modules/quota/oci/main.tf | 7 + terraform/modules/quota/oci/outputs.tf | 4 + terraform/modules/quota/oci/variables.tf | 62 ++ terraform/oci/README.md | 227 +++++++ terraform/oci/backend.tf | 6 + terraform/oci/ci.tfbackend.example | 2 + terraform/oci/ci.tfvars.example | 63 ++ terraform/oci/main.tf | 45 ++ terraform/oci/outputs.tf | 14 + terraform/oci/providers.tf | 15 + terraform/oci/variables.tf | 123 ++++ terraform/oci/versions.tf | 10 + 36 files changed, 2045 insertions(+), 1 deletion(-) create mode 100644 functions/oci-ci-sweep/Dockerfile create mode 100644 functions/oci-ci-sweep/README.md create mode 100644 functions/oci-ci-sweep/func.yaml create mode 100644 functions/oci-ci-sweep/go.mod create mode 100644 functions/oci-ci-sweep/go.sum create mode 100644 functions/oci-ci-sweep/internal/sweep/decision.go create mode 100644 functions/oci-ci-sweep/internal/sweep/decision_test.go create mode 100644 functions/oci-ci-sweep/main.go create mode 100644 functions/oci-ci-sweep/main_test.go create mode 100644 terraform/modules/budget/oci/main.tf create mode 100644 terraform/modules/budget/oci/outputs.tf create mode 100644 terraform/modules/budget/oci/variables.tf create mode 100644 terraform/modules/compartment/oci/main.tf create mode 100644 terraform/modules/compartment/oci/outputs.tf create mode 100644 terraform/modules/compartment/oci/variables.tf create mode 100644 terraform/modules/lifecycle/oci/functions.tf create mode 100644 terraform/modules/lifecycle/oci/iam.tf create mode 100644 terraform/modules/lifecycle/oci/network.tf create mode 100644 terraform/modules/lifecycle/oci/outputs.tf create mode 100644 terraform/modules/lifecycle/oci/scheduler.tf create mode 100644 terraform/modules/lifecycle/oci/variables.tf create mode 100644 terraform/modules/quota/oci/main.tf create mode 100644 terraform/modules/quota/oci/outputs.tf create mode 100644 terraform/modules/quota/oci/variables.tf create mode 100644 terraform/oci/README.md create mode 100644 terraform/oci/backend.tf create mode 100644 terraform/oci/ci.tfbackend.example create mode 100644 terraform/oci/ci.tfvars.example create mode 100644 terraform/oci/main.tf create mode 100644 terraform/oci/outputs.tf create mode 100644 terraform/oci/providers.tf create mode 100644 terraform/oci/variables.tf create mode 100644 terraform/oci/versions.tf diff --git a/.gitignore b/.gitignore index 61c8e7d..1f4b285 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,12 @@ !**/dev-prow.tfvars !**/dev-prow.tfbackend +# OCI CI stack personal config (keep the examples) +terraform/oci/ci.tfvars +terraform/oci/ci.tfbackend +*.tfvars.bak +*.tfbackend.bak + # Crash logs crash.log crash.*.log diff --git a/Makefile b/Makefile index 10259a8..cd22394 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,7 @@ AUTHORINO_OPERATOR_MANIFEST ?= https://raw.githubusercontent.com/Kuadrant/ AUTHORINO_OPERATOR_MANIFEST_SHA256 ?= ce2bef459d1456cbe462754cad571f87150fc1ad8bee4f1d010eb0db5b0aabdd LIFECYCLE_DIR ?= functions/lifecycle-enforcer +OCI_SWEEP_DIR ?= functions/oci-ci-sweep CLEANER_NAMESPACE ?= $(NAMESPACE) CLEANER_SCHEDULE ?= 0 * * * * @@ -373,6 +374,22 @@ lint-lifecycle-function: ## Lint the lifecycle enforcer function @command -v go >/dev/null 2>&1 || { echo "ERROR: go is not installed"; exit 1; } cd "$(LIFECYCLE_DIR)" && go vet ./... +# ==== OCI CI Sweep Function Targets ==== +.PHONY: test-oci-sweep-function +test-oci-sweep-function: ## Run unit tests for the OCI CI compartment sweep function + @command -v go >/dev/null 2>&1 || { echo "ERROR: go is not installed"; exit 1; } + cd "$(OCI_SWEEP_DIR)" && go test ./... -v + +.PHONY: build-oci-sweep-function +build-oci-sweep-function: ## Build the OCI CI compartment sweep function + @command -v go >/dev/null 2>&1 || { echo "ERROR: go is not installed"; exit 1; } + cd "$(OCI_SWEEP_DIR)" && go build ./... + +.PHONY: lint-oci-sweep-function +lint-oci-sweep-function: ## Lint the OCI CI compartment sweep function + @command -v go >/dev/null 2>&1 || { echo "ERROR: go is not installed"; exit 1; } + cd "$(OCI_SWEEP_DIR)" && go vet ./... + .PHONY: add-ttl-labels add-ttl-labels: ## Add TTL labels to existing GKE clusters (DRY_RUN=true by default) ./scripts/add-ttl-labels.sh diff --git a/functions/oci-ci-sweep/Dockerfile b/functions/oci-ci-sweep/Dockerfile new file mode 100644 index 0000000..8d8edd1 --- /dev/null +++ b/functions/oci-ci-sweep/Dockerfile @@ -0,0 +1,20 @@ +FROM golang:1.26-alpine@sha256:ce864e7223ac17b1775e6fd0b4c0db580c2eb50e7953a427916379e4b92a1628 AS build +WORKDIR /go/src/func +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -o func . + +# Plain alpine, not an Fn/OCI-specific runtime image: the compiled binary is +# static (CGO_ENABLED=0) and fdk-go embeds the OCI Functions invocation +# protocol directly, so no Go toolchain is needed at runtime. fnproject/go +# (the officially documented Fn Go runtime image) tops out at Go 1.24, which +# would force downgrading go.mod; this trades that official-but-stale image +# for staying on a current Go release. +FROM alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b +RUN adduser -D -u 10001 func +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +WORKDIR /function +COPY --from=build /go/src/func/func /function/func +USER func +ENTRYPOINT ["./func"] diff --git a/functions/oci-ci-sweep/README.md b/functions/oci-ci-sweep/README.md new file mode 100644 index 0000000..0180a59 --- /dev/null +++ b/functions/oci-ci-sweep/README.md @@ -0,0 +1,101 @@ +# OCI CI Compartment Sweep + +An OCI Function (Go) that sweeps the `hyperfleet-ci` compartment on a schedule, +deleting anything older than the run window: OKE clusters, load balancers, +block volumes, and DB systems. It is the backstop for +[HYPERFLEET-1563](https://redhat.atlassian.net/browse/HYPERFLEET-1563)'s +per-run teardown — resources that survive a failed or killed CI run still get +cleaned up here. + +Deployed and scheduled via Terraform: [`terraform/modules/lifecycle/oci/`](../../terraform/modules/lifecycle/oci/). + +## How it works + +1. OCI Resource Scheduler invokes the function on a cron schedule (default + hourly). +2. The function authenticates as a resource principal (no embedded + credentials) and lists clusters, load balancers, block volumes, and DB + systems in `COMPARTMENT_ID`. +3. `EvaluateResource()` (in `internal/sweep`) decides, per resource, whether + its age exceeds `RUN_WINDOW_HOURS`. A resource tagged + `hyperfleet-keep=true` is held regardless of age, for manual debugging. +4. Resources marked for deletion are deleted, unless `DRY_RUN=true`, in which + case the action is only logged. +5. A JSON summary (per-resource action, reason, and outcome) is returned and + logged — this is what satisfies the "sweep runs on a schedule and its log + shows what it removed" acceptance criterion. + +The decision logic (`internal/sweep`) has no OCI SDK dependency and is +independently unit-tested; `main.go` wires it to the OCI SDK and the +[`fdk-go`](https://github.com/fnproject/fdk-go) Functions runtime. + +### Scope + +The sweep lists and deletes exactly four resource types: OKE clusters, +(classic) load balancers, block volumes, and DB systems. It deliberately +does **not** touch: + +- Network load balancers (the newer NLB service — a different API from the + classic load balancers it does sweep) +- Standalone compute instances and node pools +- VCNs, subnets, and other network resources +- Object storage buckets + +Anything a CI run creates outside those four types has to be cleaned up by +the run itself; the compartment quota and budget are the backstops for the +rest. A delete that comes back as a 409/`IncorrectState` (the resource is +already terminating, typically from the CI job's own teardown) is recorded +as a skip, not a failure — the next run re-evaluates it. + +## Configuration (function config / environment variables) + +| Variable | Default | Description | +| ------------------ | ---------- | ------------------------------------------------- | +| `COMPARTMENT_ID` | *(required)* | OCID of the compartment to sweep | +| `RUN_WINDOW_HOURS` | `8` | Age past which a resource is swept | +| `DRY_RUN` | `true` | Set to `false` to actually delete resources | + +## Development + +```bash +make test-oci-sweep-function +make build-oci-sweep-function +make lint-oci-sweep-function +``` + +## Deployment + +Build and push the image, then point Terraform at it **by immutable +@sha256 digest**. For the rhelcert tenancy in us-sanjose-1, that's region key +`sjc` and namespace `axpiwif30tzw` (confirmed 2026-09-02). Repository +immutability isn't supported by the Artifacts API in us-sanjose-1 (see +`terraform/modules/lifecycle/oci/functions.tf`), so a digest — not a tag — +is what guards against the deployed content being silently swapped. A tag is +still handy for the push (readability), but `sweep_function_image` must be +set to the digest, and Terraform validation rejects a tag-only reference: + +```bash +cd functions/oci-ci-sweep +TAG=$(git rev-parse --short HEAD) +REPO=sjc.ocir.io/axpiwif30tzw/oci-ci-sweep +docker build --platform linux/amd64 -t "$REPO:$TAG" . +docker push "$REPO:$TAG" + +# Read back the digest the push produced and pin Terraform to it: +DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "$REPO:$TAG") +echo "$DIGEST" # e.g. sjc.ocir.io/axpiwif30tzw/oci-ci-sweep@sha256:<64-hex> +# set sweep_function_image = "" in ci.tfvars +``` + +Terraform creates the OCIR repository (see the +`sweep_container_repository_path` output of `terraform/oci/`) but does not +build or push the image — that stays a CI/manual step, same division of +labor as the rest of this repo's container images. + +### Code structure + +| File | Purpose | +| --------------------------- | ---------------------------------------------------------------- | +| `internal/sweep/decision.go` | Pure sweep decision logic — no OCI SDK dependency, unit-testable | +| `internal/sweep/decision_test.go` | Table-driven tests covering all decision scenarios | +| `main.go` | Function entry point, OCI SDK clients, action executor | diff --git a/functions/oci-ci-sweep/func.yaml b/functions/oci-ci-sweep/func.yaml new file mode 100644 index 0000000..6144988 --- /dev/null +++ b/functions/oci-ci-sweep/func.yaml @@ -0,0 +1,6 @@ +schema_version: 20180708 +name: oci-ci-sweep +version: 0.0.1 +runtime: docker +timeout: 300 +memory: 256 diff --git a/functions/oci-ci-sweep/go.mod b/functions/oci-ci-sweep/go.mod new file mode 100644 index 0000000..2392869 --- /dev/null +++ b/functions/oci-ci-sweep/go.mod @@ -0,0 +1,16 @@ +module github.com/openshift-hyperfleet/hyperfleet-infra/functions/oci-ci-sweep + +go 1.26.5 + +require ( + github.com/fnproject/fdk-go v0.1.17 + github.com/oracle/oci-go-sdk/v65 v65.124.1 +) + +require ( + github.com/gofrs/flock v0.10.0 // indirect + github.com/sony/gobreaker/v2 v2.4.0 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/sys v0.45.0 // indirect +) diff --git a/functions/oci-ci-sweep/go.sum b/functions/oci-ci-sweep/go.sum new file mode 100644 index 0000000..a81c4b3 --- /dev/null +++ b/functions/oci-ci-sweep/go.sum @@ -0,0 +1,24 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fnproject/fdk-go v0.1.17 h1:ejFcjuN31J+AqC+J5vohQN441bg5U9GyWHBNpVfwnlc= +github.com/fnproject/fdk-go v0.1.17/go.mod h1:txRgA8HlRMZJE1f/xCIXthIbui6/GMFdb1q6bN5/HHE= +github.com/gofrs/flock v0.10.0 h1:SHMXenfaB03KbroETaCMtbBg3Yn29v4w1r+tgy4ff4k= +github.com/gofrs/flock v0.10.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc= +github.com/oracle/oci-go-sdk/v65 v65.124.1 h1:Wuos4/Ru9PRI83ObOfAmaEu+m+LcmjG7f17c0p6YzwM= +github.com/oracle/oci-go-sdk/v65 v65.124.1/go.mod h1:Pzy+BpgkDesvGZXEHgslwhIYobHCPHg6wRta1mWnlqQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg= +github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/functions/oci-ci-sweep/internal/sweep/decision.go b/functions/oci-ci-sweep/internal/sweep/decision.go new file mode 100644 index 0000000..322b7c0 --- /dev/null +++ b/functions/oci-ci-sweep/internal/sweep/decision.go @@ -0,0 +1,86 @@ +package sweep + +import "time" + +const ( + // TagKeep, when set to "true" as a freeform tag, exempts a resource from + // the sweep regardless of age. Used to hold a resource during manual debugging. + TagKeep = "hyperfleet-keep" +) + +type ResourceType int + +const ( + ResourceCluster ResourceType = iota + ResourceLoadBalancer + ResourceBlockVolume + ResourceDBSystem +) + +func (t ResourceType) String() string { + switch t { + case ResourceCluster: + return "cluster" + case ResourceLoadBalancer: + return "load-balancer" + case ResourceBlockVolume: + return "block-volume" + case ResourceDBSystem: + return "db-system" + default: + return "unknown" + } +} + +// Resource describes an OCI resource found in the CI compartment, in +// cloud-agnostic terms so EvaluateResource has no OCI SDK dependency. +type Resource struct { + OCID string + Name string + Type ResourceType + TimeCreated time.Time + FreeformTags map[string]string +} + +type ActionType int + +const ( + ActionSkip ActionType = iota + ActionDelete +) + +func (a ActionType) String() string { + switch a { + case ActionSkip: + return "skip" + case ActionDelete: + return "delete" + default: + return "unknown" + } +} + +type Decision struct { + Action ActionType + Reason string +} + +// EvaluateResource decides whether a resource should be deleted by the sweep. +// A resource is deleted once it is older than runWindow, unless it carries the +// TagKeep freeform tag set to "true". +func EvaluateResource(r Resource, now time.Time, runWindow time.Duration) Decision { + if r.FreeformTags[TagKeep] == "true" { + return Decision{Action: ActionSkip, Reason: "held by " + TagKeep + " tag"} + } + + if r.TimeCreated.IsZero() { + return Decision{Action: ActionSkip, Reason: "no creation timestamp available"} + } + + age := now.Sub(r.TimeCreated) + if age <= runWindow { + return Decision{Action: ActionSkip, Reason: "within run window"} + } + + return Decision{Action: ActionDelete, Reason: "older than run window (age=" + age.Round(time.Minute).String() + ")"} +} diff --git a/functions/oci-ci-sweep/internal/sweep/decision_test.go b/functions/oci-ci-sweep/internal/sweep/decision_test.go new file mode 100644 index 0000000..b8cac9b --- /dev/null +++ b/functions/oci-ci-sweep/internal/sweep/decision_test.go @@ -0,0 +1,120 @@ +package sweep + +import ( + "testing" + "time" +) + +func TestEvaluateResource(t *testing.T) { + now := time.Date(2026, 9, 2, 12, 0, 0, 0, time.UTC) + const runWindow = 24 * time.Hour + + tests := []struct { + name string + resource Resource + expectedAction ActionType + }{ + { + name: "skip: cluster created within run window", + resource: Resource{ + Name: "hyperfleet-ci-e2e-1", + Type: ResourceCluster, + TimeCreated: now.Add(-1 * time.Hour), + }, + expectedAction: ActionSkip, + }, + { + name: "skip: resource created exactly at run window boundary", + resource: Resource{ + Name: "hyperfleet-ci-lb-1", + Type: ResourceLoadBalancer, + TimeCreated: now.Add(-runWindow), + }, + expectedAction: ActionSkip, + }, + { + name: "delete: cluster older than run window", + resource: Resource{ + Name: "hyperfleet-ci-e2e-2", + Type: ResourceCluster, + TimeCreated: now.Add(-25 * time.Hour), + }, + expectedAction: ActionDelete, + }, + { + name: "delete: block volume older than run window", + resource: Resource{ + Name: "hyperfleet-ci-vol-1", + Type: ResourceBlockVolume, + TimeCreated: now.Add(-48 * time.Hour), + }, + expectedAction: ActionDelete, + }, + { + name: "delete: db system older than run window", + resource: Resource{ + Name: "hyperfleet-ci-db-1", + Type: ResourceDBSystem, + TimeCreated: now.Add(-72 * time.Hour), + }, + expectedAction: ActionDelete, + }, + { + name: "skip: held by keep tag despite being old", + resource: Resource{ + Name: "hyperfleet-ci-e2e-3", + Type: ResourceCluster, + TimeCreated: now.Add(-72 * time.Hour), + FreeformTags: map[string]string{TagKeep: "true"}, + }, + expectedAction: ActionSkip, + }, + { + name: "delete: keep tag set to non-true value does not exempt", + resource: Resource{ + Name: "hyperfleet-ci-e2e-4", + Type: ResourceCluster, + TimeCreated: now.Add(-72 * time.Hour), + FreeformTags: map[string]string{TagKeep: "false"}, + }, + expectedAction: ActionDelete, + }, + { + name: "skip: zero-value creation timestamp is never deleted", + resource: Resource{ + Name: "hyperfleet-ci-e2e-5", + Type: ResourceCluster, + // TimeCreated left at its zero value, as if the API omitted it. + }, + expectedAction: ActionSkip, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + decision := EvaluateResource(tt.resource, now, runWindow) + if decision.Action != tt.expectedAction { + t.Errorf("EvaluateResource() action = %v, want %v (reason: %s)", decision.Action, tt.expectedAction, decision.Reason) + } + }) + } +} + +func TestResourceTypeString(t *testing.T) { + tests := []struct { + rt ResourceType + want string + }{ + {ResourceCluster, "cluster"}, + {ResourceLoadBalancer, "load-balancer"}, + {ResourceBlockVolume, "block-volume"}, + {ResourceDBSystem, "db-system"}, + {ResourceType(99), "unknown"}, + } + + for _, tt := range tests { + if got := tt.rt.String(); got != tt.want { + t.Errorf("ResourceType(%d).String() = %q, want %q", tt.rt, got, tt.want) + } + } +} diff --git a/functions/oci-ci-sweep/main.go b/functions/oci-ci-sweep/main.go new file mode 100644 index 0000000..da7fd6a --- /dev/null +++ b/functions/oci-ci-sweep/main.go @@ -0,0 +1,571 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/fnproject/fdk-go" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/common/auth" + "github.com/oracle/oci-go-sdk/v65/containerengine" + "github.com/oracle/oci-go-sdk/v65/core" + "github.com/oracle/oci-go-sdk/v65/database" + "github.com/oracle/oci-go-sdk/v65/loadbalancer" + + "github.com/openshift-hyperfleet/hyperfleet-infra/functions/oci-ci-sweep/internal/sweep" +) + +const ( + defaultRunWindowHours = 8 + // maxRunWindowHours bounds RUN_WINDOW_HOURS so a misconfiguration (a + // negative value, or one large enough to overflow the Duration + // conversion) can't silently turn into "delete everything". + maxRunWindowHours = 8760 // 1 year +) + +func main() { + fdk.Handle(fdk.HandlerFunc(handleSweep)) +} + +type sweepResult struct { + Type string `json:"type"` + Name string `json:"name"` + OCID string `json:"ocid"` + Action string `json:"action"` + Reason string `json:"reason"` + Executed bool `json:"executed"` + Error string `json:"error,omitempty"` +} + +type ociClients struct { + containerEngine containerengine.ContainerEngineClient + loadBalancer loadbalancer.LoadBalancerClient + blockStorage core.BlockstorageClient + database database.DatabaseClient +} + +func handleSweep(ctx context.Context, in io.Reader, out io.Writer) { + logger := slog.Default() + now := time.Now().UTC() + + compartmentID := os.Getenv("COMPARTMENT_ID") + if compartmentID == "" { + logger.Error("COMPARTMENT_ID environment variable is required") + writeError(out, "COMPARTMENT_ID environment variable is required") + return + } + + runWindow := defaultRunWindowHours * time.Hour + if v := os.Getenv("RUN_WINDOW_HOURS"); v != "" { + hours, err := strconv.Atoi(v) + if err != nil { + logger.Error("invalid RUN_WINDOW_HOURS", "value", v, "error", err) + writeError(out, "invalid RUN_WINDOW_HOURS: "+err.Error()) + return + } + if hours <= 0 || hours > maxRunWindowHours { + msg := fmt.Sprintf("RUN_WINDOW_HOURS must be between 1 and %d, got %d", maxRunWindowHours, hours) + logger.Error(msg) + writeError(out, msg) + return + } + runWindow = time.Duration(hours) * time.Hour + } + + dryRun := os.Getenv("DRY_RUN") != "false" + + logger.Info("starting CI compartment sweep", + "compartment", compartmentID, + "run_window", runWindow.String(), + "dry_run", dryRun, + "timestamp", now.Format(time.RFC3339), + ) + + provider, err := auth.ResourcePrincipalConfigurationProvider() + if err != nil { + logger.Error("failed to create resource principal provider", "error", err) + writeError(out, "failed to create resource principal provider") + return + } + + clients, err := initOCIClients(provider) + if err != nil { + logger.Error("failed to initialize OCI clients", "error", err) + writeError(out, "failed to initialize OCI clients: "+err.Error()) + return + } + + resources, listErrs := listAllResources(ctx, provider, compartmentID) + for _, e := range listErrs { + logger.Error("failed to list resources", "error", e) + } + + logger.Info("found resources", "count", len(resources)) + + results := make([]sweepResult, 0, len(resources)) + hadFailure := len(listErrs) > 0 + + for _, r := range resources { + decision := sweep.EvaluateResource(r, now, runWindow) + + logger.Info("evaluated resource", + "type", r.Type.String(), + "name", r.Name, + "ocid", r.OCID, + "action", decision.Action.String(), + "reason", decision.Reason, + ) + + res := sweepResult{ + Type: r.Type.String(), + Name: r.Name, + OCID: r.OCID, + Action: decision.Action.String(), + Reason: decision.Reason, + } + + if decision.Action != sweep.ActionDelete { + results = append(results, res) + continue + } + + if dryRun { + logger.Info("DRY RUN: would delete resource", "type", r.Type.String(), "name", r.Name, "ocid", r.OCID) + results = append(results, res) + continue + } + + // Refetch to guard against TOCTOU: re-evaluate current state before deleting + refetched, err := refetchResource(ctx, clients, r) + if err != nil { + hadFailure = true + res.Error = fmt.Sprintf("refetch failed: %v", err) + logger.Error("failed to refetch resource before deletion", "type", r.Type.String(), "name", r.Name, "ocid", r.OCID, "error", err) + results = append(results, res) + continue + } + if refetched == nil { + // Resource was deleted by someone else in the meantime. + res.Executed = true + logger.Info("resource already deleted", "type", r.Type.String(), "name", r.Name, "ocid", r.OCID) + results = append(results, res) + continue + } + + // Re-evaluate: if it no longer qualifies for deletion, skip it. + updatedDecision := sweep.EvaluateResource(*refetched, now, runWindow) + if updatedDecision.Action != sweep.ActionDelete { + logger.Info("resource no longer qualifies for deletion after refetch", "type", r.Type.String(), "name", r.Name, "ocid", r.OCID, "reason", updatedDecision.Reason) + res.Action = updatedDecision.Action.String() + res.Reason = updatedDecision.Reason + results = append(results, res) + continue + } + + if err := deleteResource(ctx, clients, *refetched); err != nil { + if isConflictError(err) { + // The resource is mid-transition (typically already + // terminating from the CI job's own teardown, or otherwise in + // a state that blocks deletion right now). This isn't a + // failure: the next scheduled run re-evaluates it and finds it + // either gone or ready to delete. Record it as a skip so it + // doesn't fail the whole run. + res.Action = sweep.ActionSkip.String() + res.Reason = fmt.Sprintf("deletion deferred (resource busy or already terminating): %v", err) + logger.Info("deletion deferred; will retry next run", "type", r.Type.String(), "name", r.Name, "ocid", r.OCID, "error", err) + } else { + hadFailure = true + res.Error = err.Error() + logger.Error("failed to delete resource", "type", r.Type.String(), "name", r.Name, "ocid", r.OCID, "error", err) + } + } else { + res.Executed = true + logger.Info("deleted resource", "type", r.Type.String(), "name", r.Name, "ocid", r.OCID) + } + results = append(results, res) + } + + response := map[string]any{ + "timestamp": now.Format(time.RFC3339), + "compartment": compartmentID, + "dry_run": dryRun, + "results": results, + } + if hadFailure { + fdk.WriteStatus(out, 500) + } + if err := json.NewEncoder(out).Encode(response); err != nil { + logger.Error("failed to encode response", "error", err) + } +} + +func writeError(out io.Writer, msg string) { + logger := slog.Default() + fdk.WriteStatus(out, 500) + if err := json.NewEncoder(out).Encode(map[string]string{"error": msg}); err != nil { + logger.Error("failed to encode error response", "error", err) + } +} + +func listAllResources(ctx context.Context, provider common.ConfigurationProvider, compartmentID string) ([]sweep.Resource, []error) { + var resources []sweep.Resource + var errs []error + + if r, err := listClusters(ctx, provider, compartmentID); err != nil { + errs = append(errs, fmt.Errorf("listing clusters: %w", err)) + } else { + resources = append(resources, r...) + } + + if r, err := listLoadBalancers(ctx, provider, compartmentID); err != nil { + errs = append(errs, fmt.Errorf("listing load balancers: %w", err)) + } else { + resources = append(resources, r...) + } + + if r, err := listBlockVolumes(ctx, provider, compartmentID); err != nil { + errs = append(errs, fmt.Errorf("listing block volumes: %w", err)) + } else { + resources = append(resources, r...) + } + + if r, err := listDBSystems(ctx, provider, compartmentID); err != nil { + errs = append(errs, fmt.Errorf("listing db systems: %w", err)) + } else { + resources = append(resources, r...) + } + + return resources, errs +} + +func listClusters(ctx context.Context, provider common.ConfigurationProvider, compartmentID string) ([]sweep.Resource, error) { + client, err := containerengine.NewContainerEngineClientWithConfigurationProvider(provider) + if err != nil { + return nil, fmt.Errorf("creating container engine client: %w", err) + } + + var resources []sweep.Resource + var page *string + for { + resp, err := client.ListClusters(ctx, containerengine.ListClustersRequest{ + CompartmentId: &compartmentID, + LifecycleState: []containerengine.ClusterLifecycleStateEnum{ + containerengine.ClusterLifecycleStateActive, + containerengine.ClusterLifecycleStateCreating, + containerengine.ClusterLifecycleStateFailed, + }, + Page: page, + }) + if err != nil { + return nil, err + } + + for _, c := range resp.Items { + res := sweep.Resource{ + OCID: valueOrEmpty(c.Id), + Name: valueOrEmpty(c.Name), + Type: sweep.ResourceCluster, + FreeformTags: c.FreeformTags, + } + if c.Metadata != nil && c.Metadata.TimeCreated != nil { + res.TimeCreated = c.Metadata.TimeCreated.Time + } + resources = append(resources, res) + } + + if resp.OpcNextPage == nil { + break + } + page = resp.OpcNextPage + } + return resources, nil +} + +func listLoadBalancers(ctx context.Context, provider common.ConfigurationProvider, compartmentID string) ([]sweep.Resource, error) { + client, err := loadbalancer.NewLoadBalancerClientWithConfigurationProvider(provider) + if err != nil { + return nil, fmt.Errorf("creating load balancer client: %w", err) + } + + var resources []sweep.Resource + var page *string + for { + resp, err := client.ListLoadBalancers(ctx, loadbalancer.ListLoadBalancersRequest{ + CompartmentId: &compartmentID, + Page: page, + }) + if err != nil { + return nil, err + } + + for _, lb := range resp.Items { + if lb.LifecycleState == loadbalancer.LoadBalancerLifecycleStateDeleting || + lb.LifecycleState == loadbalancer.LoadBalancerLifecycleStateDeleted { + continue + } + res := sweep.Resource{ + OCID: valueOrEmpty(lb.Id), + Name: valueOrEmpty(lb.DisplayName), + Type: sweep.ResourceLoadBalancer, + FreeformTags: lb.FreeformTags, + } + if lb.TimeCreated != nil { + res.TimeCreated = lb.TimeCreated.Time + } + resources = append(resources, res) + } + + if resp.OpcNextPage == nil { + break + } + page = resp.OpcNextPage + } + return resources, nil +} + +func listBlockVolumes(ctx context.Context, provider common.ConfigurationProvider, compartmentID string) ([]sweep.Resource, error) { + client, err := core.NewBlockstorageClientWithConfigurationProvider(provider) + if err != nil { + return nil, fmt.Errorf("creating blockstorage client: %w", err) + } + + var resources []sweep.Resource + var page *string + for { + resp, err := client.ListVolumes(ctx, core.ListVolumesRequest{ + CompartmentId: &compartmentID, + Page: page, + }) + if err != nil { + return nil, err + } + + for _, v := range resp.Items { + if v.LifecycleState == core.VolumeLifecycleStateTerminating || + v.LifecycleState == core.VolumeLifecycleStateTerminated { + continue + } + res := sweep.Resource{ + OCID: valueOrEmpty(v.Id), + Name: valueOrEmpty(v.DisplayName), + Type: sweep.ResourceBlockVolume, + FreeformTags: v.FreeformTags, + } + if v.TimeCreated != nil { + res.TimeCreated = v.TimeCreated.Time + } + resources = append(resources, res) + } + + if resp.OpcNextPage == nil { + break + } + page = resp.OpcNextPage + } + return resources, nil +} + +func listDBSystems(ctx context.Context, provider common.ConfigurationProvider, compartmentID string) ([]sweep.Resource, error) { + client, err := database.NewDatabaseClientWithConfigurationProvider(provider) + if err != nil { + return nil, fmt.Errorf("creating database client: %w", err) + } + + var resources []sweep.Resource + var page *string + for { + resp, err := client.ListDbSystems(ctx, database.ListDbSystemsRequest{ + CompartmentId: &compartmentID, + Page: page, + }) + if err != nil { + return nil, err + } + + for _, db := range resp.Items { + if db.LifecycleState == database.DbSystemSummaryLifecycleStateTerminating || + db.LifecycleState == database.DbSystemSummaryLifecycleStateTerminated { + continue + } + res := sweep.Resource{ + OCID: valueOrEmpty(db.Id), + Name: valueOrEmpty(db.DisplayName), + Type: sweep.ResourceDBSystem, + FreeformTags: db.FreeformTags, + } + if db.TimeCreated != nil { + res.TimeCreated = db.TimeCreated.Time + } + resources = append(resources, res) + } + + if resp.OpcNextPage == nil { + break + } + page = resp.OpcNextPage + } + return resources, nil +} + +func initOCIClients(provider common.ConfigurationProvider) (*ociClients, error) { + ceClient, err := containerengine.NewContainerEngineClientWithConfigurationProvider(provider) + if err != nil { + return nil, fmt.Errorf("creating container engine client: %w", err) + } + lbClient, err := loadbalancer.NewLoadBalancerClientWithConfigurationProvider(provider) + if err != nil { + return nil, fmt.Errorf("creating load balancer client: %w", err) + } + bsClient, err := core.NewBlockstorageClientWithConfigurationProvider(provider) + if err != nil { + return nil, fmt.Errorf("creating blockstore client: %w", err) + } + dbClient, err := database.NewDatabaseClientWithConfigurationProvider(provider) + if err != nil { + return nil, fmt.Errorf("creating database client: %w", err) + } + return &ociClients{ + containerEngine: ceClient, + loadBalancer: lbClient, + blockStorage: bsClient, + database: dbClient, + }, nil +} + +// refetchResource checks if a resource still exists and retrieves current tags. +// Returns nil if the resource no longer exists (already deleted). It keeps the +// original TimeCreated: the age doesn't change between the list and this GET, so +// there's no need to re-read it (only the freeform tags, which gate the +// hyperfleet-keep exemption, can have changed). +func refetchResource(ctx context.Context, clients *ociClients, r sweep.Resource) (*sweep.Resource, error) { + switch r.Type { + case sweep.ResourceCluster: + resp, err := clients.containerEngine.GetCluster(ctx, containerengine.GetClusterRequest{ClusterId: &r.OCID}) + if err != nil { + if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "NotFound") { + return nil, nil + } + return nil, fmt.Errorf("fetching cluster: %w", err) + } + result := r + result.FreeformTags = resp.Cluster.FreeformTags + return &result, nil + + case sweep.ResourceLoadBalancer: + resp, err := clients.loadBalancer.GetLoadBalancer(ctx, loadbalancer.GetLoadBalancerRequest{LoadBalancerId: &r.OCID}) + if err != nil { + if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "NotFound") { + return nil, nil + } + return nil, fmt.Errorf("fetching load balancer: %w", err) + } + result := r + result.FreeformTags = resp.LoadBalancer.FreeformTags + return &result, nil + + case sweep.ResourceBlockVolume: + resp, err := clients.blockStorage.GetVolume(ctx, core.GetVolumeRequest{VolumeId: &r.OCID}) + if err != nil { + if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "NotFound") { + return nil, nil + } + return nil, fmt.Errorf("fetching block volume: %w", err) + } + result := r + result.FreeformTags = resp.Volume.FreeformTags + return &result, nil + + case sweep.ResourceDBSystem: + resp, err := clients.database.GetDbSystem(ctx, database.GetDbSystemRequest{DbSystemId: &r.OCID}) + if err != nil { + if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "NotFound") { + return nil, nil + } + return nil, fmt.Errorf("fetching db system: %w", err) + } + result := r + result.FreeformTags = resp.DbSystem.FreeformTags + return &result, nil + + default: + return nil, fmt.Errorf("unsupported resource type %q", r.Type.String()) + } +} + +// isConflictError reports whether err is an OCI 409/IncorrectState response, +// meaning the resource can't be deleted right now because it's mid-transition +// (e.g. already terminating). The sweep treats these as a skip, not a failure: +// the next scheduled run re-evaluates the resource. It checks the structured +// ServiceError first and falls back to string matching for errors that don't +// unwrap to one. +func isConflictError(err error) bool { + if err == nil { + return false + } + // If it unwraps to a structured ServiceError, trust the status/code and + // return immediately — do not fall through to string matching. Otherwise + // an unrelated error (e.g. a 500) whose message, OPC request ID, or OCID + // happens to contain "409" would be misread as a conflict and quietly + // skipped, when that's exactly the case we want to fail the run so a human + // looks at it. + var svcErr common.ServiceError + if errors.As(err, &svcErr) { + return svcErr.GetHTTPStatusCode() == http.StatusConflict || + strings.EqualFold(svcErr.GetCode(), "IncorrectState") + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "409") || + strings.Contains(msg, "incorrectstate") || + strings.Contains(msg, "already terminating") +} + +func deleteResource(ctx context.Context, clients *ociClients, r sweep.Resource) error { + switch r.Type { + case sweep.ResourceCluster: + _, err := clients.containerEngine.DeleteCluster(ctx, containerengine.DeleteClusterRequest{ClusterId: &r.OCID}) + if err != nil { + return fmt.Errorf("deleting cluster: %w", err) + } + return nil + + case sweep.ResourceLoadBalancer: + _, err := clients.loadBalancer.DeleteLoadBalancer(ctx, loadbalancer.DeleteLoadBalancerRequest{LoadBalancerId: &r.OCID}) + if err != nil { + return fmt.Errorf("deleting load balancer: %w", err) + } + return nil + + case sweep.ResourceBlockVolume: + _, err := clients.blockStorage.DeleteVolume(ctx, core.DeleteVolumeRequest{VolumeId: &r.OCID}) + if err != nil { + return fmt.Errorf("deleting block volume: %w", err) + } + return nil + + case sweep.ResourceDBSystem: + _, err := clients.database.TerminateDbSystem(ctx, database.TerminateDbSystemRequest{DbSystemId: &r.OCID}) + if err != nil { + return fmt.Errorf("terminating db system: %w", err) + } + return nil + + default: + return fmt.Errorf("unsupported resource type %q", r.Type.String()) + } +} + +func valueOrEmpty(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/functions/oci-ci-sweep/main_test.go b/functions/oci-ci-sweep/main_test.go new file mode 100644 index 0000000..e1a3495 --- /dev/null +++ b/functions/oci-ci-sweep/main_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "errors" + "fmt" + "net/http" + "testing" +) + +// fakeServiceError implements common.ServiceError so isConflictError can be +// tested without a live OCI response. msg is what Error() returns, kept +// separate from status/code so a test can exercise the structured checks +// (GetHTTPStatusCode / GetCode) without the fallback string match also firing. +type fakeServiceError struct { + status int + code string + msg string +} + +func (e fakeServiceError) Error() string { return e.msg } +func (e fakeServiceError) GetHTTPStatusCode() int { return e.status } +func (e fakeServiceError) GetMessage() string { return e.msg } +func (e fakeServiceError) GetCode() string { return e.code } +func (e fakeServiceError) GetOpcRequestID() string { return "" } + +func TestIsConflictError(t *testing.T) { + // msg deliberately avoids the fallback substrings so these two isolate the + // structured ServiceError path. + structured409 := fakeServiceError{status: http.StatusConflict, code: "LimitExceeded", msg: "request rejected"} + structuredIncorrectState := fakeServiceError{status: http.StatusBadRequest, code: "IncorrectState", msg: "request rejected"} + + tests := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"structured 409", structured409, true}, + {"structured IncorrectState", structuredIncorrectState, true}, + {"wrapped structured 409", fmt.Errorf("deleting cluster: %w", structured409), true}, + {"string 409", errors.New("Error returned by Service. Http Status Code: 409"), true}, + {"string already terminating", errors.New("resource is already terminating"), true}, + {"string IncorrectState", errors.New("IncorrectState: cannot delete"), true}, + {"unrelated 404", fakeServiceError{status: http.StatusNotFound, code: "NotFound", msg: "not found"}, false}, + // A structured non-conflict error must not be reclassified by the string + // fallback just because its message/OPC request ID contains "409". + {"structured 500 with 409 in message", fakeServiceError{status: http.StatusInternalServerError, code: "InternalServerError", msg: "opc-request-id 409ab/..."}, false}, + {"unrelated string", errors.New("connection refused"), false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isConflictError(tt.err); got != tt.want { + t.Errorf("isConflictError(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} diff --git a/terraform/README.md b/terraform/README.md index c4ac2e6..1ed2a21 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -484,8 +484,13 @@ terraform/ ├── modules/ │ ├── cluster/ │ │ └── gke/ # GKE cluster module -│ ├── lifecycle/ # Lifecycle enforcer Cloud Function module +│ ├── lifecycle/ # Lifecycle enforcer Cloud Function module (GCP) +│ │ └── oci/ # OCI CI compartment sweep function module +│ ├── compartment/oci/ # OCI compartment module +│ ├── quota/oci/ # OCI compartment quota policy module +│ ├── budget/oci/ # OCI budget + email alert-rule module │ └── pubsub/ # Google Pub/Sub module +├── oci/ # OCI CI compartment stack (see oci/README.md) └── envs/ └── gke/ └── dev.tfvars.example diff --git a/terraform/modules/budget/oci/main.tf b/terraform/modules/budget/oci/main.tf new file mode 100644 index 0000000..e637953 --- /dev/null +++ b/terraform/modules/budget/oci/main.tf @@ -0,0 +1,38 @@ +resource "oci_budget_budget" "ci" { + compartment_id = var.tenancy_ocid + amount = var.amount + reset_period = "MONTHLY" + display_name = var.display_name + description = "Monthly spend cap for the hyperfleet-ci compartment (ephemeral e2e cluster runs)." + + target_type = "COMPARTMENT" + targets = [var.target_compartment_id] + + freeform_tags = var.freeform_tags +} + +locals { + recipients = join(",", var.alert_recipients) +} + +resource "oci_budget_alert_rule" "actual" { + for_each = toset([for t in var.alert_thresholds_percent : tostring(t)]) + + budget_id = oci_budget_budget.ci.id + type = "ACTUAL" + threshold_type = "PERCENTAGE" + threshold = tonumber(each.value) + display_name = "hyperfleet-ci-actual-${each.value}pct" + message = "hyperfleet-ci compartment spend has reached ${each.value}% of its $$${var.amount}/month budget." + recipients = local.recipients +} + +resource "oci_budget_alert_rule" "forecast" { + budget_id = oci_budget_budget.ci.id + type = "FORECAST" + threshold_type = "PERCENTAGE" + threshold = var.forecast_threshold_percent + display_name = "hyperfleet-ci-forecast-${var.forecast_threshold_percent}pct" + message = "hyperfleet-ci compartment is forecast to reach ${var.forecast_threshold_percent}% of its $$${var.amount}/month budget before period end." + recipients = local.recipients +} diff --git a/terraform/modules/budget/oci/outputs.tf b/terraform/modules/budget/oci/outputs.tf new file mode 100644 index 0000000..b06ef02 --- /dev/null +++ b/terraform/modules/budget/oci/outputs.tf @@ -0,0 +1,4 @@ +output "budget_id" { + description = "OCID of the budget." + value = oci_budget_budget.ci.id +} diff --git a/terraform/modules/budget/oci/variables.tf b/terraform/modules/budget/oci/variables.tf new file mode 100644 index 0000000..603de8a --- /dev/null +++ b/terraform/modules/budget/oci/variables.tf @@ -0,0 +1,52 @@ +variable "tenancy_ocid" { + description = "Tenancy OCID. The budget's own compartment_id must be the tenancy; target_compartment_id scopes what spend it tracks." + type = string +} + +variable "target_compartment_id" { + description = "OCID of the compartment whose spend this budget tracks (the CI compartment)." + type = string +} + +variable "display_name" { + description = "Display name for the budget." + type = string + default = "hyperfleet-ci-budget" +} + +variable "amount" { + description = "Monthly budget amount, in the tenancy's currency." + type = number + default = 150 +} + +variable "alert_recipients" { + description = <<-EOT + Email addresses that receive budget alerts, via the alert rule's built-in + email delivery (no Slack app/webhook setup required). + EOT + type = list(string) + + validation { + condition = length(var.alert_recipients) > 0 + error_message = "At least one alert recipient email is required." + } +} + +variable "alert_thresholds_percent" { + description = "Percent-of-budget thresholds (of ACTUAL spend) that trigger an alert." + type = list(number) + default = [50, 80, 100] +} + +variable "forecast_threshold_percent" { + description = "Percent-of-budget threshold on FORECASTED spend that triggers an early-warning alert." + type = number + default = 100 +} + +variable "freeform_tags" { + description = "Freeform tags applied to the budget." + type = map(string) + default = {} +} diff --git a/terraform/modules/compartment/oci/main.tf b/terraform/modules/compartment/oci/main.tf new file mode 100644 index 0000000..f3b5227 --- /dev/null +++ b/terraform/modules/compartment/oci/main.tf @@ -0,0 +1,8 @@ +resource "oci_identity_compartment" "this" { + compartment_id = var.parent_compartment_id + name = var.name + description = var.description + freeform_tags = var.freeform_tags + + enable_delete = true +} diff --git a/terraform/modules/compartment/oci/outputs.tf b/terraform/modules/compartment/oci/outputs.tf new file mode 100644 index 0000000..de64ca0 --- /dev/null +++ b/terraform/modules/compartment/oci/outputs.tf @@ -0,0 +1,9 @@ +output "id" { + description = "OCID of the CI compartment." + value = oci_identity_compartment.this.id +} + +output "name" { + description = "Name of the CI compartment." + value = oci_identity_compartment.this.name +} diff --git a/terraform/modules/compartment/oci/variables.tf b/terraform/modules/compartment/oci/variables.tf new file mode 100644 index 0000000..2a78006 --- /dev/null +++ b/terraform/modules/compartment/oci/variables.tf @@ -0,0 +1,28 @@ +variable "parent_compartment_id" { + description = <<-EOT + OCID of the HyperFleet team compartment. hyperfleet-ci is created directly + under it, as a sibling of hyperfleet-sandbox/hyperfleet-poc/hyperfleet-demos + — never point this at one of those sub-compartments, and never create + resources directly in the team compartment itself (team convention: every + resource lives in a sub-compartment so spend is attributable). + EOT + type = string +} + +variable "name" { + description = "Name of the compartment." + type = string + default = "hyperfleet-ci" +} + +variable "description" { + description = "Description of the compartment." + type = string + default = "HyperFleet OCI CI compartment: ephemeral e2e cluster runs, swept on a schedule." +} + +variable "freeform_tags" { + description = "Freeform tags applied to the compartment." + type = map(string) + default = {} +} diff --git a/terraform/modules/lifecycle/oci/functions.tf b/terraform/modules/lifecycle/oci/functions.tf new file mode 100644 index 0000000..bd5de7c --- /dev/null +++ b/terraform/modules/lifecycle/oci/functions.tf @@ -0,0 +1,40 @@ +# Immutable repositories would keep whoever can push to OCIR from silently +# swapping the image a scheduled, already-deployed function runs (the +# function's IAM grants it manage access on cluster/LB/volume/DB resources in +# this compartment). However, the Artifacts API in us-sanjose-1 rejects +# isImmutable with "400-BAD_REQUEST, Setting isImmutable is not currently +# supported" (confirmed live 2026-09-04), so we cannot set it here. Instead +# var.function_image is required to be pinned by an immutable @sha256 digest +# (enforced by variable validation): a digest is content-addressed, so unlike +# a tag it cannot be repointed to different content on a mutable repository. +resource "oci_artifacts_container_repository" "sweep" { + compartment_id = var.compartment_id + display_name = "oci-ci-sweep" + + freeform_tags = var.freeform_tags +} + +resource "oci_functions_application" "sweep" { + compartment_id = var.compartment_id + display_name = "oci-ci-sweep" + subnet_ids = [oci_core_subnet.sweep.id] + + freeform_tags = var.freeform_tags +} + +resource "oci_functions_function" "sweep" { + application_id = oci_functions_application.sweep.id + display_name = "oci-ci-sweep" + image = var.function_image + memory_in_mbs = "256" + + timeout_in_seconds = 300 + + config = { + COMPARTMENT_ID = var.compartment_id + RUN_WINDOW_HOURS = tostring(var.run_window_hours) + DRY_RUN = tostring(var.dry_run) + } + + freeform_tags = var.freeform_tags +} diff --git a/terraform/modules/lifecycle/oci/iam.tf b/terraform/modules/lifecycle/oci/iam.tf new file mode 100644 index 0000000..72ecd2c --- /dev/null +++ b/terraform/modules/lifecycle/oci/iam.tf @@ -0,0 +1,76 @@ +# Dynamic group + policy #1: the sweep function's own runtime identity +# (resource principal). Scoped to this one function by OCID — not to every +# fnfunc in the compartment — so an unrelated function created here later +# doesn't silently inherit the sweep's manage grants. The function doesn't +# reference this group, so there's no dependency cycle. +resource "oci_identity_dynamic_group" "sweep_function" { + compartment_id = var.tenancy_ocid + name = "hyperfleet-ci-sweep-fn" + description = "Matches the oci-ci-sweep function so it can authenticate as a resource principal." + matching_rule = "ALL {resource.type = 'fnfunc', resource.id = '${oci_functions_function.sweep.id}'}" + + freeform_tags = var.freeform_tags +} + +resource "oci_identity_policy" "sweep_function" { + compartment_id = var.compartment_id + name = "hyperfleet-ci-sweep-fn-policy" + description = "Lets the oci-ci-sweep function manage swept resource types, only in the CI compartment." + + statements = [ + "allow dynamic-group ${oci_identity_dynamic_group.sweep_function.name} to manage cluster-family in compartment id ${var.compartment_id}", + "allow dynamic-group ${oci_identity_dynamic_group.sweep_function.name} to manage load-balancers in compartment id ${var.compartment_id}", + "allow dynamic-group ${oci_identity_dynamic_group.sweep_function.name} to manage volume-family in compartment id ${var.compartment_id}", + "allow dynamic-group ${oci_identity_dynamic_group.sweep_function.name} to manage database-family in compartment id ${var.compartment_id}", + ] + + freeform_tags = var.freeform_tags +} + +# Dynamic group + policy #2: the Resource Scheduler schedule itself, which +# needs permission to invoke the function on the sweep's cron cadence. +resource "oci_identity_dynamic_group" "scheduler" { + compartment_id = var.tenancy_ocid + name = "hyperfleet-ci-sweep-scheduler" + description = "Matches the sweep's Resource Scheduler schedule so it can invoke the function." + matching_rule = "ALL {resource.type = 'resourceschedule', resource.id = '${oci_resource_scheduler_schedule.sweep.id}'}" + + freeform_tags = var.freeform_tags +} + +resource "oci_identity_policy" "scheduler" { + compartment_id = var.compartment_id + name = "hyperfleet-ci-sweep-scheduler-policy" + description = "Lets the sweep's Resource Scheduler schedule invoke the oci-ci-sweep function." + + statements = [ + "allow dynamic-group ${oci_identity_dynamic_group.scheduler.name} to manage functions-family in compartment id ${var.compartment_id}", + ] + + freeform_tags = var.freeform_tags +} + +# Policy #3: the FaaS (Functions) service itself, not a dynamic group. When a +# function runs, the service needs to plumb it into the VCN subnet and pull +# the image from OCIR on the function's behalf. Without these grants the +# function fails to invoke (network attach or image pull denied), regardless +# of what the function's own resource-principal policy allows. +# +# This policy is attached at the tenancy root because OCI requires the OCIR +# read grant for the FaaS service to be tenancy-scoped ("read repos in +# tenancy") — a compartment-attached policy cannot express "in tenancy", and +# scoping repos to a single compartment does not authorize the pull. The +# virtual-network-family statement is still scoped down to the CI compartment, +# where the subnet lives. +resource "oci_identity_policy" "faas_service" { + compartment_id = var.tenancy_ocid + name = "hyperfleet-ci-faas-service-policy" + description = "Lets the FaaS service attach the sweep function to its subnet and pull its image." + + statements = [ + "allow service FaaS to use virtual-network-family in compartment id ${var.compartment_id}", + "allow service FaaS to read repos in tenancy", + ] + + freeform_tags = var.freeform_tags +} diff --git a/terraform/modules/lifecycle/oci/network.tf b/terraform/modules/lifecycle/oci/network.tf new file mode 100644 index 0000000..39f80e5 --- /dev/null +++ b/terraform/modules/lifecycle/oci/network.tf @@ -0,0 +1,77 @@ +# Dedicated, minimal network for the sweep function's Application. OCI +# Functions always run inside a VCN subnet even though they're serverless. +# The subnet is private: the function only needs to reach other OCI services +# (Container Engine, Load Balancer, Block Storage, Database APIs), which it +# does over the OCI backbone via a service gateway, so no NAT/internet egress +# is required. + +# The service gateway must target the "all OCI services in the region" +# destination so the function can reach every API it sweeps (Container Engine, +# Load Balancer, Block Storage, Database). oci_core_services returns several +# entries in a non-deterministic order, so filter for the aggregate one by +# name instead of indexing an arbitrary element. +data "oci_core_services" "all_oci_services" { + filter { + name = "name" + values = ["All .* Services In Oracle Services Network"] + regex = true + } +} + +resource "oci_core_vcn" "sweep" { + compartment_id = var.compartment_id + display_name = "hyperfleet-ci-sweep-vcn" + cidr_block = var.vcn_cidr_block + freeform_tags = var.freeform_tags +} + +resource "oci_core_service_gateway" "sweep" { + compartment_id = var.compartment_id + vcn_id = oci_core_vcn.sweep.id + display_name = "hyperfleet-ci-sweep-sgw" + + services { + service_id = data.oci_core_services.all_oci_services.services[0].id + } + + freeform_tags = var.freeform_tags +} + +resource "oci_core_route_table" "sweep" { + compartment_id = var.compartment_id + vcn_id = oci_core_vcn.sweep.id + display_name = "hyperfleet-ci-sweep-rt" + + route_rules { + destination = data.oci_core_services.all_oci_services.services[0].cidr_block + destination_type = "SERVICE_CIDR_BLOCK" + network_entity_id = oci_core_service_gateway.sweep.id + } + + freeform_tags = var.freeform_tags +} + +resource "oci_core_security_list" "sweep" { + compartment_id = var.compartment_id + vcn_id = oci_core_vcn.sweep.id + display_name = "hyperfleet-ci-sweep-sl" + + egress_security_rules { + destination = "0.0.0.0/0" + protocol = "all" + } + + freeform_tags = var.freeform_tags +} + +resource "oci_core_subnet" "sweep" { + compartment_id = var.compartment_id + vcn_id = oci_core_vcn.sweep.id + display_name = "hyperfleet-ci-sweep-subnet" + cidr_block = var.subnet_cidr_block + prohibit_public_ip_on_vnic = true + route_table_id = oci_core_route_table.sweep.id + security_list_ids = [oci_core_security_list.sweep.id] + + freeform_tags = var.freeform_tags +} diff --git a/terraform/modules/lifecycle/oci/outputs.tf b/terraform/modules/lifecycle/oci/outputs.tf new file mode 100644 index 0000000..f33d05c --- /dev/null +++ b/terraform/modules/lifecycle/oci/outputs.tf @@ -0,0 +1,19 @@ +output "application_id" { + description = "OCID of the Functions Application hosting the sweep function." + value = oci_functions_application.sweep.id +} + +output "function_id" { + description = "OCID of the sweep function." + value = oci_functions_function.sweep.id +} + +output "container_repository_path" { + description = "OCIR repository path to push the sweep function's image to. After pushing, reference the image by its immutable @sha256 digest (not a tag) in function_image — repository immutability isn't supported in us-sanjose-1, so the digest is the only guard against the deployed content being silently swapped." + value = "${oci_artifacts_container_repository.sweep.namespace}/${oci_artifacts_container_repository.sweep.display_name}" +} + +output "schedule_id" { + description = "OCID of the Resource Scheduler schedule invoking the sweep function." + value = oci_resource_scheduler_schedule.sweep.id +} diff --git a/terraform/modules/lifecycle/oci/scheduler.tf b/terraform/modules/lifecycle/oci/scheduler.tf new file mode 100644 index 0000000..dc90216 --- /dev/null +++ b/terraform/modules/lifecycle/oci/scheduler.tf @@ -0,0 +1,15 @@ +resource "oci_resource_scheduler_schedule" "sweep" { + compartment_id = var.compartment_id + display_name = "hyperfleet-ci-sweep-schedule" + description = "Invokes the oci-ci-sweep function on a cron schedule." + + action = "START_RESOURCE" + recurrence_type = "CRON" + recurrence_details = var.schedule_recurrence + + resources { + id = oci_functions_function.sweep.id + } + + freeform_tags = var.freeform_tags +} diff --git a/terraform/modules/lifecycle/oci/variables.tf b/terraform/modules/lifecycle/oci/variables.tf new file mode 100644 index 0000000..827f631 --- /dev/null +++ b/terraform/modules/lifecycle/oci/variables.tf @@ -0,0 +1,70 @@ +variable "tenancy_ocid" { + description = "Tenancy OCID. Dynamic groups must be created at the tenancy root." + type = string +} + +variable "compartment_id" { + description = "OCID of the CI compartment the sweep function watches and cleans up." + type = string +} + +variable "function_image" { + description = <<-EOT + Full OCIR image reference for the sweep function, pinned by an immutable + @sha256 digest, e.g. + ".ocir.io//oci-ci-sweep@sha256:<64-hex>", built and + pushed out of band by CI from functions/oci-ci-sweep. A digest (not a + tag) is required: repository immutability isn't supported by the Artifacts + API in us-sanjose-1, so digest-pinning is the only guard against the image + a deployed function runs being silently swapped for different content. The + container_repository_path output gives the repository half of this + reference. + EOT + type = string + + validation { + condition = can(regex("@sha256:[0-9a-f]{64}$", var.function_image)) + error_message = "function_image must be pinned by an immutable digest ending in @sha256:<64 hex chars>, not a mutable tag." + } +} + +variable "run_window_hours" { + description = "Age, in hours, past which a resource in the CI compartment is swept." + type = number + default = 8 + + validation { + condition = var.run_window_hours >= 1 && var.run_window_hours <= 8760 && floor(var.run_window_hours) == var.run_window_hours + error_message = "run_window_hours must be a whole number between 1 and 8760 (1 year)." + } +} + +variable "dry_run" { + description = "When true, the sweep logs what it would delete without deleting anything. Flip to false once verified." + type = bool + default = true +} + +variable "schedule_recurrence" { + description = "Cron expression controlling how often the sweep runs." + type = string + default = "0 * * * *" +} + +variable "vcn_cidr_block" { + description = "CIDR block for the dedicated VCN the sweep function's Application runs in." + type = string + default = "10.99.0.0/24" +} + +variable "subnet_cidr_block" { + description = "CIDR block for the private subnet the sweep function's Application runs in." + type = string + default = "10.99.0.0/25" +} + +variable "freeform_tags" { + description = "Freeform tags applied to created resources." + type = map(string) + default = {} +} diff --git a/terraform/modules/quota/oci/main.tf b/terraform/modules/quota/oci/main.tf new file mode 100644 index 0000000..8f8ebc8 --- /dev/null +++ b/terraform/modules/quota/oci/main.tf @@ -0,0 +1,7 @@ +resource "oci_limits_quota" "this" { + compartment_id = var.tenancy_ocid + name = var.name + description = var.description + statements = var.statements + freeform_tags = var.freeform_tags +} diff --git a/terraform/modules/quota/oci/outputs.tf b/terraform/modules/quota/oci/outputs.tf new file mode 100644 index 0000000..900286b --- /dev/null +++ b/terraform/modules/quota/oci/outputs.tf @@ -0,0 +1,4 @@ +output "id" { + description = "OCID of the quota policy." + value = oci_limits_quota.this.id +} diff --git a/terraform/modules/quota/oci/variables.tf b/terraform/modules/quota/oci/variables.tf new file mode 100644 index 0000000..42f3e62 --- /dev/null +++ b/terraform/modules/quota/oci/variables.tf @@ -0,0 +1,62 @@ +variable "tenancy_ocid" { + description = "Tenancy OCID. Quota policies must be created at the tenancy root compartment." + type = string +} + +variable "name" { + description = "Name of the quota policy." + type = string + default = "hyperfleet-ci-quota" +} + +variable "description" { + description = "Description of the quota policy." + type = string + default = "Caps compute cores and OKE clusters in the hyperfleet-ci compartment." +} + +variable "statements" { + description = <<-EOT + Quota policy statements (OCI quota policy language), e.g. + ["set compute-core quota standard-e4-core-count to 16 in compartment HyperFleet:hyperfleet-ci"]. + + There is no hardcoded default here because the module is tenancy-agnostic: + the exact quota-family and quota-name strings depend on the worker node + shape chosen for the CI compartment's OKE clusters, and whether Container + Engine exposes a compartment-quota-manageable cluster count for this + tenancy. Discover the authoritative values before setting this: + + oci limits definition list --compartment-id --service-name compute + oci limits definition list --compartment-id --service-name container-engine + + The statement's quota family must be the limit's "supported-quota-families" + value, NOT its "service-name": standard-e4-core-count belongs to service + "compute" but quota family "compute-core", so the statement reads "set + compute-core quota standard-e4-core-count ...". Check "are-quotas-supported" + (true) rather than "is-quota-managed" (unpopulated in this API version). + + Reference a nested compartment by its path from the tenancy root, e.g. + "in compartment HyperFleet:hyperfleet-ci" — a bare compartment name only + resolves for direct children of the tenancy root. + + If container-engine does not expose a cluster-count quota, enforce the + concurrent-cluster cap as a soft check in the sweep function instead + (see functions/oci-ci-sweep) and leave this list to compute cores only. + + For the rhelcert tenancy specifically, both are already confirmed (see + terraform/oci/ci.tfvars.example and terraform/oci/README.md) — Container + Engine does expose "container-engine" / "cluster-count" there. + EOT + type = list(string) + + validation { + condition = length(var.statements) > 0 + error_message = "At least one quota statement is required; see the variable description for how to derive it." + } +} + +variable "freeform_tags" { + description = "Freeform tags applied to the quota policy." + type = map(string) + default = {} +} diff --git a/terraform/oci/README.md b/terraform/oci/README.md new file mode 100644 index 0000000..34813a5 --- /dev/null +++ b/terraform/oci/README.md @@ -0,0 +1,227 @@ +# HyperFleet OCI CI Infrastructure + +Terraform for the `hyperfleet-ci` compartment: a quota fence, a scheduled +teardown sweep, and a budget. This is the foundation +[HYPERFLEET-1563](https://redhat.atlassian.net/browse/HYPERFLEET-1563)'s e2e +CI job runs against. + +## What this creates + +| Resource | Purpose | +| --------------------------- | ------------------------------------------------------------------------ | +| `hyperfleet-ci` compartment | Isolates CI-created OKE clusters and their dependents from everything else in the team compartment | +| Compartment quota policy | Caps compute cores and (if the tenancy exposes it) concurrent OKE clusters | +| Budget + alert rules | $150/month budget on the compartment's spend, alerting at 50/80/100% actual and 100% forecast | +| `oci-ci-sweep` function | Scheduled (hourly by default) sweep that deletes clusters, load balancers, block volumes, and DB systems older than the run window | + +`hyperfleet-ci` is a sibling of the team's existing `hyperfleet-sandbox`, +`hyperfleet-poc`, and `hyperfleet-demos` compartments under the `HyperFleet` +team compartment — never directly inside `HyperFleet` itself. Set +`team_compartment_id` to `HyperFleet`'s own OCID, not one of those +sub-compartments'. This compartment is dedicated to the OCI e2e CI suite, +not a general-purpose sandbox — for ad hoc experiments, use +`hyperfleet-sandbox` instead. + +## Sweep policy + +A scheduled OCI Function sweeps the compartment and deletes anything older +than the run window: OKE clusters, load balancers, block volumes, and DB +systems. + +| Setting | Value | +|---|---| +| Run window | `sweep_run_window_hours` (default 8h) | +| Schedule | `sweep_schedule_recurrence` (default hourly, `0 * * * *`) | +| Exemption | Freeform tag `hyperfleet-keep=true` holds a resource regardless of age, for manual debugging | + +The sweep only handles those four resource types. It does **not** delete +network load balancers (the newer NLB service, distinct from the classic +load balancers it does sweep), standalone compute instances, node pools, +VCNs/subnets, or object storage buckets. Anything the e2e job creates +outside those four types must be torn down by the job itself; the +compartment quota and budget are the backstops for those. + +The `hyperfleet-keep=true` exemption is evaluated per resource, against that +resource's own freeform tags — it is **not** inherited. OKE doesn't +propagate a cluster's tags to the block volumes or load balancers it +provisions, so tagging a cluster `hyperfleet-keep=true` holds only the +cluster object; the sweep can still delete its dependent volumes and load +balancers once they age out. To pin a whole cluster's footprint for +debugging, tag each of its resources. + +The sweep is the **backstop**, not the primary cleanup mechanism: the e2e CI +job is expected to tear down every billable resource at the end of each run +(including failed runs). The sweep only catches what that per-run teardown +misses — a leaked resource costs at most a few hours of spend before the +sweep removes it, rather than sitting forgotten indefinitely. That bound +only holds for a non-exempt resource once the sweep actually runs and +deletes successfully: a resource tagged `hyperfleet-keep=true` is held +indefinitely by design, and a sweep left in `DRY_RUN` or otherwise not +running normally won't delete anything at all. + +The sweep does not send notifications of any kind; it only logs its +decisions (Logging service, `oci-ci-sweep` function). + +## Quota policy + +`quota_statements` (see `ci.tfvars.example`) has no hardcoded default in the +module — confirmed live against the rhelcert tenancy on 2026-09-04 instead of +guessed: + +- `compute-core` quota family, `standard-e4-core-count` name (region limit + 11111 in us-sanjose-1 — plenty of headroom for a compartment cap of 16). + The quota family is `compute-core`, **not** the `compute` service name: + the statement family must match the limit's `supported-quota-families`, or + the API rejects it with "not a valid quota name for service compute". +- `container-engine` family, `cluster-count` name — Container Engine *does* + expose a compartment-quota-manageable cluster count in this tenancy + (region limit 15, 0 used), so the 2-concurrent-cluster cap is a real IAM + guarantee, not a soft check in the sweep function. + +`hyperfleet-ci` is nested under `HyperFleet`, so the statements reference it +by path (`in compartment HyperFleet:hyperfleet-ci`) — a bare compartment name +only resolves for direct children of the tenancy root. + +At list price, a running OKE cluster with three small nodes costs about +$7/day. The quota is a sanity cap on concurrency, not a mathematical +guarantee that spend stays under budget — a bug that kept both quota slots +occupied for a full month would still cost roughly $420, well over budget. +The budget alerts (below) are the real early-warning mechanism: at 2 +concurrent clusters, the 50/80/100% thresholds fire after roughly 11, 17, +and 21 cluster-days of usage respectively, out of the ~60 cluster-days +theoretically possible in a 30-day month — comfortably before the worst +case is reached. + +If the worker node shape changes, re-derive the compute quota name with the +command below, and read `supported-quota-families` on the matching limit for +the statement's quota family (check `are-quotas-supported`, not the +unpopulated `is-quota-managed`): + +```bash +oci limits definition list --compartment-id "$TENANCY_OCID" --service-name compute +``` + +Applying a `quota_statements` change requires an identity with the `quota` +resource-type manage permission at the tenancy level (`allow group + to manage quota in tenancy`) — not full tenancy +administration. `oci_limits_quota` is created at the tenancy root even +though its statements target `hyperfleet-ci` specifically, so a +compartment-scoped identity is not sufficient. + +The same applies to the sweep function's IAM: dynamic groups live at the +tenancy root, so applying this stack also needs `manage dynamic-groups in +tenancy` (and `manage policies` in the CI compartment for the policies that +reference them). A purely compartment-scoped identity can create the +compartment, VCN, budget, and function, but will fail on the dynamic +groups. + +## Notifications + +**Owner:** `#hcm-hyperfleet-team`. + +Budget alerts (50/80/100% actual spend, 100% forecast) are delivered by +email, using `oci_budget_alert_rule`'s built-in `recipients` field — no +Slack app, webhook, or Notifications-topic wiring needed. Current +recipients (`budget_alert_recipients` in `ci.tfvars`): + +- `mbrudnoy@redhat.com` +- `croche@redhat.com` +- `rbenevid@redhat.com` + +## The sweep function's network + +OCI Functions always run inside a VCN subnet even though they're +serverless. `terraform/modules/lifecycle/oci/network.tf` creates a small, +dedicated VCN with a private subnet — the function reaches other OCI service +APIs (Container Engine, Load Balancer, Block Storage, Database) over the OCI +backbone via a service gateway, so no NAT gateway or internet egress is +needed. + +## Usage + +```bash +cd terraform/oci +cp ci.tfbackend.example ci.tfbackend # edit prefix if needed +cp ci.tfvars.example ci.tfvars # fill in tenancy/compartment/quota/recipients + +terraform init -backend-config=ci.tfbackend +terraform plan -var-file=ci.tfvars +terraform apply -var-file=ci.tfvars +``` + +The sweep function's image is built and pushed separately (see +[`functions/oci-ci-sweep/README.md`](../../functions/oci-ci-sweep/README.md)) +— Terraform creates the OCIR repository but does not build or push images. +`sweep_function_image` must be pinned by an immutable `@sha256` digest, not a +tag (Terraform validation enforces this); the function README shows how to +read the digest back after pushing. `sweep_dry_run` defaults to `true`; flip +it to `false` once you've confirmed the sweep's dry-run log output looks +right. + +### View compartment contents (read-only, any team member) + +```bash +oci iam compartment list --compartment-id "$HYPERFLEET_COMPARTMENT_OCID" \ + --query "data[?name=='hyperfleet-ci']" + +oci ce cluster list --compartment-id "$HYPERFLEET_CI_OCID" +``` + +### View Terraform state and outputs + +```bash +cd terraform/oci +terraform init -backend-config=ci.tfbackend +terraform state list +terraform output +``` + +## Auth + +Set `oci_auth` to match how you're authenticated: + +- `"ApiKey"` (default): durable API key, needs `oci_user_ocid`, + `oci_fingerprint`, `oci_private_key_path`. +- `"SecurityToken"`: browser SSO session token from `oci session + authenticate`, needs `oci_config_file_profile`. + +## Remote state + +Same GCS backend as the GKE stacks (`hyperfleet-terraform-state`), under +`terraform/state/oci-ci`. See the top-level [`terraform/README.md`](../README.md) +for backend setup and team access. + +## Key configuration files + +| File | Purpose | +|---|---| +| `terraform/oci/ci.tfvars.example` | Compartment, quota, budget, and sweep configuration | +| `terraform/oci/ci.tfbackend.example` | Remote state configuration | +| `terraform/oci/main.tf` | Root module wiring the compartment, quota, budget, and sweep modules | +| `terraform/modules/{compartment,quota,budget,lifecycle}/oci/` | Individual resource modules | +| `functions/oci-ci-sweep/` | The sweep function's Go source | + +## Troubleshooting + +### Sweep isn't removing an expected resource + +Check the function's logs (Logging service, `oci-ci-sweep` function) for +the per-resource action and reason. Common causes: the resource carries +`hyperfleet-keep=true`, it's younger than the run window, or `DRY_RUN` is +still `true` on the function. + +### Quota apply fails with a permissions error + +`oci_limits_quota` requires `manage quota in tenancy` permissions (see +[Quota policy](#quota-policy)) — a compartment-scoped identity is not +sufficient. + +### Budget alert emails never arrived + +Check `budget_alert_recipients` in `ci.tfvars` for typos, and check +spam/junk folders — the alert rule's email delivery is OCI's built-in +mechanism, not a separate subscription to confirm. + +## Additional documentation + +- **Sweep function internals**: [`functions/oci-ci-sweep/README.md`](../../functions/oci-ci-sweep/README.md) diff --git a/terraform/oci/backend.tf b/terraform/oci/backend.tf new file mode 100644 index 0000000..d4b4507 --- /dev/null +++ b/terraform/oci/backend.tf @@ -0,0 +1,6 @@ +# Remote state on the same GCS backend as the GKE stacks (see +# terraform/README.md), under its own prefix. Configure with: +# terraform init -backend-config=ci.tfbackend +terraform { + backend "gcs" {} +} diff --git a/terraform/oci/ci.tfbackend.example b/terraform/oci/ci.tfbackend.example new file mode 100644 index 0000000..ccdd095 --- /dev/null +++ b/terraform/oci/ci.tfbackend.example @@ -0,0 +1,2 @@ +bucket = "hyperfleet-terraform-state" +prefix = "terraform/state/oci-ci" diff --git a/terraform/oci/ci.tfvars.example b/terraform/oci/ci.tfvars.example new file mode 100644 index 0000000..9e68ca3 --- /dev/null +++ b/terraform/oci/ci.tfvars.example @@ -0,0 +1,63 @@ +# Copy: cp ci.tfvars.example ci.tfvars +# Usage: terraform init -backend-config=ci.tfbackend +# terraform apply -var-file=ci.tfvars + +# Confirmed live against the rhelcert tenancy on 2026-09-02. +tenancy_ocid = "ocid1.tenancy.oc1..aaaaaaaayikwwnfeirfik6fwqdt5rfjfajmwjuj5u34vkbpao5u6hohucnsa" + +# rhelcert's home region. IAM/quota/budget resources are tenancy-global +# regardless of this setting; OKE/VCN/Functions/ONS/Events are regional and +# will be created here. Change it if the team runs OCI workloads elsewhere. +region = "us-sanjose-1" + +# The "HyperFleet" team compartment (parent of hyperfleet-sandbox/poc/demos). +# hyperfleet-ci is created as a sibling of those, never inside one of them. +team_compartment_id = "ocid1.compartment.oc1..aaaaaaaaa53uyu26gzto2ox5znqygzvhm7cemt6kznjkvrx7dzae6g2g4nda" + +# Auth: default is a durable API key. For browser SSO instead, set +# oci_auth = "SecurityToken" and run `oci session authenticate` first. +# oci_user_ocid = "ocid1.user.oc1..REPLACE_ME" +# oci_fingerprint = "xx:xx:xx:..." +# oci_private_key_path = "~/.oci/oci_api_key.pem" + +# Confirmed against the rhelcert tenancy on 2026-09-04: +# - "standard-e4-core-count" is quota-manageable (are-quotas-supported=true). +# Its quota family is "compute-core", NOT "compute" — the statement family +# must match the limit's supported-quota-families, or the API rejects it +# with "not a valid quota name for service compute". +# - container-engine exposes a compartment-quota-manageable "cluster-count" +# (family "container-engine"; region limit 15, 0 used) — no soft-check +# fallback needed. +# - hyperfleet-ci is nested under HyperFleet, so quota statements must use +# the compartment path "HyperFleet:hyperfleet-ci" — a bare "hyperfleet-ci" +# only resolves for direct children of the tenancy root. +# Re-verify with `oci limits definition list --compartment-id +# --service-name ` (check supported-quota-families) +# if the worker shape changes. +quota_statements = [ + "set compute-core quota standard-e4-core-count to 16 in compartment HyperFleet:hyperfleet-ci", + "set container-engine quota cluster-count to 2 in compartment HyperFleet:hyperfleet-ci", +] + +budget_amount = 150 +budget_alert_recipients = [ + "mbrudnoy@redhat.com", + "croche@redhat.com", + "rbenevid@redhat.com", +] + +# Build and push functions/oci-ci-sweep first (see its README), then set this +# to the pushed image's immutable @sha256 digest (read it back with +# `docker inspect --format='{{index .RepoDigests 0}}'`). Terraform creates the +# OCIR repository (sweep_container_repository_path output) but does not build +# or push images. "sjc" is us-sanjose-1's OCIR region key; "axpiwif30tzw" is +# the rhelcert tenancy's Object Storage/Registry namespace (confirmed +# 2026-09-02). Repository immutability (isImmutable) is not supported by the +# Artifacts API in us-sanjose-1 (confirmed 2026-09-04), so a digest — not a +# tag — is what guards against an already-deployed function's image being +# silently swapped; variable validation rejects a tag-only reference. +sweep_function_image = "sjc.ocir.io/axpiwif30tzw/oci-ci-sweep@sha256:REPLACE_ME_IMAGE_DIGEST" + +sweep_run_window_hours = 8 +sweep_dry_run = true # flip to false once verified end to end +sweep_schedule_recurrence = "0 * * * *" diff --git a/terraform/oci/main.tf b/terraform/oci/main.tf new file mode 100644 index 0000000..a178689 --- /dev/null +++ b/terraform/oci/main.tf @@ -0,0 +1,45 @@ +module "ci_compartment" { + source = "../modules/compartment/oci" + parent_compartment_id = var.team_compartment_id + freeform_tags = local.tags +} + +module "ci_quota" { + source = "../modules/quota/oci" + tenancy_ocid = var.tenancy_ocid + statements = var.quota_statements + + freeform_tags = local.tags + + depends_on = [module.ci_compartment] +} + +module "ci_budget" { + source = "../modules/budget/oci" + tenancy_ocid = var.tenancy_ocid + target_compartment_id = module.ci_compartment.id + amount = var.budget_amount + alert_recipients = var.budget_alert_recipients + + freeform_tags = local.tags +} + +module "ci_sweep" { + source = "../modules/lifecycle/oci" + tenancy_ocid = var.tenancy_ocid + compartment_id = module.ci_compartment.id + function_image = var.sweep_function_image + + run_window_hours = var.sweep_run_window_hours + dry_run = var.sweep_dry_run + schedule_recurrence = var.sweep_schedule_recurrence + + freeform_tags = local.tags +} + +locals { + tags = { + "hyperfleet-managed-by" = "terraform" + "hyperfleet-purpose" = "ci" + } +} diff --git a/terraform/oci/outputs.tf b/terraform/oci/outputs.tf new file mode 100644 index 0000000..4eb87d0 --- /dev/null +++ b/terraform/oci/outputs.tf @@ -0,0 +1,14 @@ +output "ci_compartment_id" { + description = "OCID of the hyperfleet-ci compartment." + value = module.ci_compartment.id +} + +output "sweep_container_repository_path" { + description = "OCIR repository path to push the sweep function's image to." + value = module.ci_sweep.container_repository_path +} + +output "sweep_function_id" { + description = "OCID of the deployed sweep function." + value = module.ci_sweep.function_id +} diff --git a/terraform/oci/providers.tf b/terraform/oci/providers.tf new file mode 100644 index 0000000..9a3a24c --- /dev/null +++ b/terraform/oci/providers.tf @@ -0,0 +1,15 @@ +# Supports two auth modes, matching whatever the operator already has set up +# via the oci CLI: +# - auth = "ApiKey" (default): durable API key, reads fingerprint/private_key +# from var.oci_fingerprint / var.oci_private_key_path. +# - auth = "SecurityToken": browser SSO session token created via +# `oci session authenticate`, reads var.oci_config_file_profile. +provider "oci" { + auth = var.oci_auth + tenancy_ocid = var.tenancy_ocid + region = var.region + config_file_profile = var.oci_auth == "SecurityToken" ? var.oci_config_file_profile : null + user_ocid = var.oci_auth == "ApiKey" ? var.oci_user_ocid : null + fingerprint = var.oci_auth == "ApiKey" ? var.oci_fingerprint : null + private_key_path = var.oci_auth == "ApiKey" ? var.oci_private_key_path : null +} diff --git a/terraform/oci/variables.tf b/terraform/oci/variables.tf new file mode 100644 index 0000000..715ec06 --- /dev/null +++ b/terraform/oci/variables.tf @@ -0,0 +1,123 @@ +variable "tenancy_ocid" { + description = "OCID of the rhelcert tenancy." + type = string +} + +variable "region" { + description = "OCI region to create resources in." + type = string + default = "us-sanjose-1" +} + +variable "oci_auth" { + description = "OCI provider auth mode: \"ApiKey\" (durable) or \"SecurityToken\" (browser SSO session, via `oci session authenticate`)." + type = string + default = "ApiKey" + + validation { + condition = contains(["ApiKey", "SecurityToken"], var.oci_auth) + error_message = "oci_auth must be \"ApiKey\" or \"SecurityToken\"." + } +} + +variable "oci_config_file_profile" { + description = "Profile name in ~/.oci/config to use when oci_auth = \"SecurityToken\"." + type = string + default = "DEFAULT" +} + +variable "oci_user_ocid" { + description = "User OCID, when oci_auth = \"ApiKey\"." + type = string + default = null +} + +variable "oci_fingerprint" { + description = "API key fingerprint, when oci_auth = \"ApiKey\"." + type = string + default = null +} + +variable "oci_private_key_path" { + description = "Path to the API private key, when oci_auth = \"ApiKey\"." + type = string + default = null +} + +variable "team_compartment_id" { + description = <<-EOT + OCID of the "HyperFleet" team compartment (rhelcert tenancy). This is the + parent, not one of its existing sub-compartments — hyperfleet-ci is + created as a sibling of hyperfleet-sandbox/hyperfleet-poc/hyperfleet-demos, + per team convention: never create resources directly in the team + compartment, always in a sub-compartment. + EOT + type = string +} + +variable "quota_statements" { + description = <<-EOT + Quota policy statements. See terraform/modules/quota/oci/variables.tf for + how to derive the correct compute-core and container-engine values for + this tenancy before setting this. + EOT + type = list(string) +} + +variable "budget_amount" { + description = "Monthly budget amount (USD) for the hyperfleet-ci compartment." + type = number + default = 150 +} + +variable "budget_alert_recipients" { + description = "Email addresses that receive hyperfleet-ci budget alerts." + type = list(string) +} + +variable "sweep_function_image" { + description = <<-EOT + OCIR image reference for the oci-ci-sweep function, pinned by an immutable + @sha256 digest (e.g. "sjc.ocir.io//oci-ci-sweep@sha256:<64-hex>"). + Push the image first, read back its digest, then set this. Repository + immutability isn't supported by the Artifacts API in us-sanjose-1, so a + digest — not a tag — is what guarantees the deployed function keeps running + the exact content that was reviewed. + EOT + type = string + + validation { + condition = can(regex("@sha256:[0-9a-f]{64}$", var.sweep_function_image)) + error_message = "sweep_function_image must be pinned by an immutable digest ending in @sha256:<64 hex chars>, not a mutable tag." + } +} + +variable "sweep_run_window_hours" { + description = <<-EOT + Age, in hours, past which the sweep deletes a resource. Keep this close + to how long a normal e2e run actually takes plus a small buffer, not a + full day: at $7/day per 3-node OKE cluster, a shorter window shrinks how + much a single leaked/orphaned resource can cost before the sweep catches + it (the sweep is the backstop for HYPERFLEET-1563's per-run teardown, + not the primary cleanup path). + EOT + type = number + default = 8 + + validation { + condition = var.sweep_run_window_hours >= 1 && var.sweep_run_window_hours <= 8760 && floor(var.sweep_run_window_hours) == var.sweep_run_window_hours + error_message = "sweep_run_window_hours must be a whole number between 1 and 8760 (1 year)." + } +} + +variable "sweep_dry_run" { + description = "When true, the sweep only logs what it would delete. Flip to false once verified end to end." + type = bool + default = true +} + +variable "sweep_schedule_recurrence" { + description = "Cron expression for how often the sweep runs." + type = string + default = "0 * * * *" +} diff --git a/terraform/oci/versions.tf b/terraform/oci/versions.tf new file mode 100644 index 0000000..90a75be --- /dev/null +++ b/terraform/oci/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + oci = { + source = "oracle/oci" + version = "~> 7.0" + } + } +}