HYPERFLEET-1574 - feat: OCI CI compartment, quota, sweep, and budget

Makefile

@@ -52,6 +52,7 @@ AUTHORINO_OPERATOR_MANIFEST       ?= https://raw.githubusercontent.com/Kuadrant/
 AUTHORINO_OPERATOR_MANIFEST_SHA256 ?= ce2bef459d1456cbe462754cad571f87150fc1ad8bee4f1d010eb0db5b0aabdd
 
 LIFECYCLE_DIR        ?= functions/lifecycle-enforcer
+OCI_SWEEP_DIR        ?= functions/oci-ci-sweep
 
 CLEANER_NAMESPACE    ?= $(NAMESPACE)
 CLEANER_SCHEDULE     ?= 0 * * * *
@@ -373,6 +374,22 @@ lint-lifecycle-function: ## Lint the lifecycle enforcer function
 	@command -v go >/dev/null 2>&1 || { echo "ERROR: go is not installed"; exit 1; }
 	cd "$(LIFECYCLE_DIR)" && go vet ./...
 
+# ==== OCI CI Sweep Function Targets ====
+.PHONY: test-oci-sweep-function
+test-oci-sweep-function: ## Run unit tests for the OCI CI compartment sweep function
+	@command -v go >/dev/null 2>&1 || { echo "ERROR: go is not installed"; exit 1; }
+	cd "$(OCI_SWEEP_DIR)" && go test ./... -v
+
+.PHONY: build-oci-sweep-function
+build-oci-sweep-function: ## Build the OCI CI compartment sweep function
+	@command -v go >/dev/null 2>&1 || { echo "ERROR: go is not installed"; exit 1; }
+	cd "$(OCI_SWEEP_DIR)" && go build ./...
+
+.PHONY: lint-oci-sweep-function
+lint-oci-sweep-function: ## Lint the OCI CI compartment sweep function
+	@command -v go >/dev/null 2>&1 || { echo "ERROR: go is not installed"; exit 1; }
+	cd "$(OCI_SWEEP_DIR)" && go vet ./...
+
 .PHONY: add-ttl-labels
 add-ttl-labels: ## Add TTL labels to existing GKE clusters (DRY_RUN=true by default)
 	./scripts/add-ttl-labels.sh

functions/oci-ci-sweep/Dockerfile

@@ -0,0 +1,20 @@
+FROM golang:1.26-alpine@sha256:ce864e7223ac17b1775e6fd0b4c0db580c2eb50e7953a427916379e4b92a1628 AS build
+WORKDIR /go/src/func
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 go build -o func .
+
+# Plain alpine, not an Fn/OCI-specific runtime image: the compiled binary is
+# static (CGO_ENABLED=0) and fdk-go embeds the OCI Functions invocation
+# protocol directly, so no Go toolchain is needed at runtime. fnproject/go
+# (the officially documented Fn Go runtime image) tops out at Go 1.24, which
+# would force downgrading go.mod; this trades that official-but-stale image
+# for staying on a current Go release.
+FROM alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b
+RUN adduser -D -u 10001 func
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+WORKDIR /function
+COPY --from=build /go/src/func/func /function/func
+USER func
+ENTRYPOINT ["./func"]

functions/oci-ci-sweep/README.md

@@ -0,0 +1,75 @@
+# OCI CI Compartment Sweep
+
+An OCI Function (Go) that sweeps the `hyperfleet-ci` compartment on a schedule,
+deleting anything older than the run window: OKE clusters, load balancers,
+block volumes, and DB systems. It is the backstop for
+[HYPERFLEET-1563](https://redhat.atlassian.net/browse/HYPERFLEET-1563)'s
+per-run teardown — resources that survive a failed or killed CI run still get
+cleaned up here.
+
+Deployed and scheduled via Terraform: [`terraform/modules/lifecycle/oci/`](../../terraform/modules/lifecycle/oci/).
+
+## How it works
+
+1. OCI Resource Scheduler invokes the function on a cron schedule (default
+   hourly).
+2. The function authenticates as a resource principal (no embedded
+   credentials) and lists clusters, load balancers, block volumes, and DB
+   systems in `COMPARTMENT_ID`.
+3. `EvaluateResource()` (in `internal/sweep`) decides, per resource, whether
+   its age exceeds `RUN_WINDOW_HOURS`. A resource tagged
+   `hyperfleet-keep=true` is held regardless of age, for manual debugging.
+4. Resources marked for deletion are deleted, unless `DRY_RUN=true`, in which
+   case the action is only logged.
+5. A JSON summary (per-resource action, reason, and outcome) is returned and
+   logged — this is what satisfies the "sweep runs on a schedule and its log
+   shows what it removed" acceptance criterion.
+
+The decision logic (`internal/sweep`) has no OCI SDK dependency and is
+independently unit-tested; `main.go` wires it to the OCI SDK and the
+[`fdk-go`](https://github.com/fnproject/fdk-go) Functions runtime.
+
+## Configuration (function config / environment variables)
+
+| Variable           | Default    | Description                                      |
+| ------------------ | ---------- | ------------------------------------------------- |
+| `COMPARTMENT_ID`   | *(required)* | OCID of the compartment to sweep                |
+| `RUN_WINDOW_HOURS`  | `8`        | Age past which a resource is swept                |
+| `DRY_RUN`           | `true`     | Set to `false` to actually delete resources       |
+
+## Development
+
+```bash
+make test-oci-sweep-function
+make build-oci-sweep-function
+make lint-oci-sweep-function
+```
+
+## Deployment
+
+Build and push the image, then point Terraform at it. For the rhelcert
+tenancy in us-sanjose-1, that's region key `sjc` and namespace
+`axpiwif30tzw` (confirmed 2026-09-02). The OCIR repository is immutable
+(see `terraform/modules/lifecycle/oci/functions.tf`), so tag each build
+uniquely — e.g. the git commit SHA — instead of `:latest`, which can't be
+re-pushed once used:
+
+```bash
+cd functions/oci-ci-sweep
+TAG=$(git rev-parse --short HEAD)
+docker build -t sjc.ocir.io/axpiwif30tzw/oci-ci-sweep:"$TAG" .
+docker push sjc.ocir.io/axpiwif30tzw/oci-ci-sweep:"$TAG"
+```
+
+Terraform creates the OCIR repository (see the
+`sweep_container_repository_path` output of `terraform/oci/`) but does not
+build or push the image — that stays a CI/manual step, same division of
+labor as the rest of this repo's container images.
+
+### Code structure
+
+| File                        | Purpose                                                        |
+| --------------------------- | ---------------------------------------------------------------- |
+| `internal/sweep/decision.go` | Pure sweep decision logic — no OCI SDK dependency, unit-testable |
+| `internal/sweep/decision_test.go` | Table-driven tests covering all decision scenarios          |
+| `main.go`                   | Function entry point, OCI SDK clients, action executor           |

functions/oci-ci-sweep/func.yaml

@@ -0,0 +1,6 @@
+schema_version: 20180708
+name: oci-ci-sweep
+version: 0.0.1
+runtime: docker
+timeout: 300
+memory: 256

functions/oci-ci-sweep/go.mod

@@ -0,0 +1,16 @@
+module github.com/openshift-hyperfleet/hyperfleet-infra/functions/oci-ci-sweep
+
+go 1.26.5
+
+require (
+	github.com/fnproject/fdk-go v0.1.17
+	github.com/oracle/oci-go-sdk/v65 v65.124.1
+)
+
+require (
+	github.com/gofrs/flock v0.10.0 // indirect
+	github.com/sony/gobreaker/v2 v2.4.0 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
+	golang.org/x/crypto v0.52.0 // indirect
+	golang.org/x/sys v0.45.0 // indirect
+)

functions/oci-ci-sweep/go.sum

@@ -0,0 +1,24 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fnproject/fdk-go v0.1.17 h1:ejFcjuN31J+AqC+J5vohQN441bg5U9GyWHBNpVfwnlc=
+github.com/fnproject/fdk-go v0.1.17/go.mod h1:txRgA8HlRMZJE1f/xCIXthIbui6/GMFdb1q6bN5/HHE=
+github.com/gofrs/flock v0.10.0 h1:SHMXenfaB03KbroETaCMtbBg3Yn29v4w1r+tgy4ff4k=
+github.com/gofrs/flock v0.10.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc=
+github.com/oracle/oci-go-sdk/v65 v65.124.1 h1:Wuos4/Ru9PRI83ObOfAmaEu+m+LcmjG7f17c0p6YzwM=
+github.com/oracle/oci-go-sdk/v65 v65.124.1/go.mod h1:Pzy+BpgkDesvGZXEHgslwhIYobHCPHg6wRta1mWnlqQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg=
+github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
+golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
+golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

functions/oci-ci-sweep/internal/sweep/decision.go

@@ -0,0 +1,86 @@
+package sweep
+
+import "time"
+
+const (
+	// TagKeep, when set to "true" as a freeform tag, exempts a resource from
+	// the sweep regardless of age. Used to hold a resource during manual debugging.
+	TagKeep = "hyperfleet-keep"
+)
+
+type ResourceType int
+
+const (
+	ResourceCluster ResourceType = iota
+	ResourceLoadBalancer
+	ResourceBlockVolume
+	ResourceDBSystem
+)
+
+func (t ResourceType) String() string {
+	switch t {
+	case ResourceCluster:
+		return "cluster"
+	case ResourceLoadBalancer:
+		return "load-balancer"
+	case ResourceBlockVolume:
+		return "block-volume"
+	case ResourceDBSystem:
+		return "db-system"
+	default:
+		return "unknown"
+	}
+}
+
+// Resource describes an OCI resource found in the CI compartment, in
+// cloud-agnostic terms so EvaluateResource has no OCI SDK dependency.
+type Resource struct {
+	OCID         string
+	Name         string
+	Type         ResourceType
+	TimeCreated  time.Time
+	FreeformTags map[string]string
+}
+
+type ActionType int
+
+const (
+	ActionSkip ActionType = iota
+	ActionDelete
+)
+
+func (a ActionType) String() string {
+	switch a {
+	case ActionSkip:
+		return "skip"
+	case ActionDelete:
+		return "delete"
+	default:
+		return "unknown"
+	}
+}
+
+type Decision struct {
+	Action ActionType
+	Reason string
+}
+
+// EvaluateResource decides whether a resource should be deleted by the sweep.
+// A resource is deleted once it is older than runWindow, unless it carries the
+// TagKeep freeform tag set to "true".
+func EvaluateResource(r Resource, now time.Time, runWindow time.Duration) Decision {
+	if r.FreeformTags[TagKeep] == "true" {
+		return Decision{Action: ActionSkip, Reason: "held by " + TagKeep + " tag"}
+	}
+
+	if r.TimeCreated.IsZero() {
+		return Decision{Action: ActionSkip, Reason: "no creation timestamp available"}
+	}
+
+	age := now.Sub(r.TimeCreated)
+	if age <= runWindow {
+		return Decision{Action: ActionSkip, Reason: "within run window"}
+	}
+
+	return Decision{Action: ActionDelete, Reason: "older than run window (age=" + age.Round(time.Minute).String() + ")"}
+}

functions/oci-ci-sweep/internal/sweep/decision_test.go

@@ -0,0 +1,120 @@
+package sweep
+
+import (
+	"testing"
+	"time"
+)
+
+func TestEvaluateResource(t *testing.T) {
+	now := time.Date(2026, 9, 2, 12, 0, 0, 0, time.UTC)
+	const runWindow = 24 * time.Hour
+
+	tests := []struct {
+		name           string
+		resource       Resource
+		expectedAction ActionType
+	}{
+		{
+			name: "skip: cluster created within run window",
+			resource: Resource{
+				Name:        "hyperfleet-ci-e2e-1",
+				Type:        ResourceCluster,
+				TimeCreated: now.Add(-1 * time.Hour),
+			},
+			expectedAction: ActionSkip,
+		},
+		{
+			name: "skip: resource created exactly at run window boundary",
+			resource: Resource{
+				Name:        "hyperfleet-ci-lb-1",
+				Type:        ResourceLoadBalancer,
+				TimeCreated: now.Add(-runWindow),
+			},
+			expectedAction: ActionSkip,
+		},
+		{
+			name: "delete: cluster older than run window",
+			resource: Resource{
+				Name:        "hyperfleet-ci-e2e-2",
+				Type:        ResourceCluster,
+				TimeCreated: now.Add(-25 * time.Hour),
+			},
+			expectedAction: ActionDelete,
+		},
+		{
+			name: "delete: block volume older than run window",
+			resource: Resource{
+				Name:        "hyperfleet-ci-vol-1",
+				Type:        ResourceBlockVolume,
+				TimeCreated: now.Add(-48 * time.Hour),
+			},
+			expectedAction: ActionDelete,
+		},
+		{
+			name: "delete: db system older than run window",
+			resource: Resource{
+				Name:        "hyperfleet-ci-db-1",
+				Type:        ResourceDBSystem,
+				TimeCreated: now.Add(-72 * time.Hour),
+			},
+			expectedAction: ActionDelete,
+		},
+		{
+			name: "skip: held by keep tag despite being old",
+			resource: Resource{
+				Name:         "hyperfleet-ci-e2e-3",
+				Type:         ResourceCluster,
+				TimeCreated:  now.Add(-72 * time.Hour),
+				FreeformTags: map[string]string{TagKeep: "true"},
+			},
+			expectedAction: ActionSkip,
+		},
+		{
+			name: "delete: keep tag set to non-true value does not exempt",
+			resource: Resource{
+				Name:         "hyperfleet-ci-e2e-4",
+				Type:         ResourceCluster,
+				TimeCreated:  now.Add(-72 * time.Hour),
+				FreeformTags: map[string]string{TagKeep: "false"},
+			},
+			expectedAction: ActionDelete,
+		},
+		{
+			name: "skip: zero-value creation timestamp is never deleted",
+			resource: Resource{
+				Name: "hyperfleet-ci-e2e-5",
+				Type: ResourceCluster,
+				// TimeCreated left at its zero value, as if the API omitted it.
+			},
+			expectedAction: ActionSkip,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			decision := EvaluateResource(tt.resource, now, runWindow)
+			if decision.Action != tt.expectedAction {
+				t.Errorf("EvaluateResource() action = %v, want %v (reason: %s)", decision.Action, tt.expectedAction, decision.Reason)
+			}
+		})
+	}
+}
+
+func TestResourceTypeString(t *testing.T) {
+	tests := []struct {
+		rt   ResourceType
+		want string
+	}{
+		{ResourceCluster, "cluster"},
+		{ResourceLoadBalancer, "load-balancer"},
+		{ResourceBlockVolume, "block-volume"},
+		{ResourceDBSystem, "db-system"},
+		{ResourceType(99), "unknown"},
+	}
+
+	for _, tt := range tests {
+		if got := tt.rt.String(); got != tt.want {
+			t.Errorf("ResourceType(%d).String() = %q, want %q", tt.rt, got, tt.want)
+		}
+	}
+}