diff --git a/.claude/commands/list-e2e-steps.md b/.claude/commands/list-e2e-steps.md index 9d8bf0b821..8db4f2feed 100644 --- a/.claude/commands/list-e2e-steps.md +++ b/.claude/commands/list-e2e-steps.md @@ -16,13 +16,13 @@ When this command is invoked, you MUST: Run: ```bash - go test ./test/e2e/features_test.go -d || true 2>&1 + KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" go test ./test/e2e/operator-controller -args --godog.definitions 2>&1 || true ``` - The `-d` flag prints all registered step definitions without running tests. Parse each step regex pattern from the output. + Set `KUBECONFIG` for suite initialization; the `--godog.definitions` flag prints all registered step definitions without contacting a cluster or running tests. Godog returns status 2 after listing definitions, so `go test` reports `FAIL`. Verify that registered patterns were printed, then parse each step regex pattern from the output. 2. **Read the step registration and handler implementations** - Read `test/e2e/steps/steps.go` and locate the `RegisterSteps()` function. For each regex pattern from step 1, find its corresponding `sc.Step()` call and the handler function it references. + Read `test/internal/e2e/steps/steps.go` and locate the `RegisterSteps()` function. For each regex pattern from step 1, find its corresponding `sc.Step()` call and the handler function it references. Then read each handler function's signature to determine: - What parameters it accepts (string, int, `*godog.DocString`, etc.) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index e7fb5d0f1a..9f675c351b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -69,7 +69,7 @@ jobs: run: | if kubectl cluster-info &>/dev/null; then mkdir -p /tmp/artifacts - support-bundle --interactive=false -o /tmp/artifacts/support-bundle test/e2e/support-bundle.yaml + support-bundle --interactive=false -o /tmp/artifacts/support-bundle test/internal/e2e/support-bundle.yaml else echo "No cluster available, skipping support-bundle collection" fi diff --git a/.github/workflows/update-demos.yaml b/.github/workflows/update-demos.yaml index 222801a27d..6dc13ebb54 100644 --- a/.github/workflows/update-demos.yaml +++ b/.github/workflows/update-demos.yaml @@ -7,17 +7,17 @@ on: paths: - 'api/*' - 'config/*' - - 'test/e2e/features/demos.feature' - - 'test/e2e/steps/demo_steps.go' - - 'test/e2e/steps/asciicast_*.go' + - 'test/e2e/operator-controller/features/demos.feature' + - 'test/internal/e2e/steps/demo_steps.go' + - 'test/internal/e2e/steps/asciicast_*.go' - '.github/workflows/update-demos.yaml' pull_request: paths: - 'api/*' - 'config/*' - - 'test/e2e/features/demos.feature' - - 'test/e2e/steps/demo_steps.go' - - 'test/e2e/steps/asciicast_*.go' + - 'test/e2e/operator-controller/features/demos.feature' + - 'test/internal/e2e/steps/demo_steps.go' + - 'test/internal/e2e/steps/asciicast_*.go' - '.github/workflows/update-demos.yaml' jobs: diff --git a/Makefile b/Makefile index 19bafb600f..ce033648e4 100644 --- a/Makefile +++ b/Makefile @@ -439,11 +439,11 @@ ifeq ($(strip $(GODOG_ARGS)),) set +e; \ KUBECONFIG=$(E2E_KUBECONFIG) \ PROMETHEUS_URL=http://localhost:$$E2E_PROMETHEUS_PORT \ - go test -count=1 -v ./test/e2e/features_test.go -timeout $(or $(E2E_TIMEOUT),20m) -args --godog.tags="~@Serial && ~@demo" --godog.concurrency=100; \ + go test -count=1 -v ./test/e2e/... -timeout $(or $(E2E_TIMEOUT),20m) -args --godog.tags="~@Serial && ~@demo" --godog.concurrency=100; \ parallelExit=$$?; \ KUBECONFIG=$(E2E_KUBECONFIG) \ PROMETHEUS_URL=http://localhost:$$E2E_PROMETHEUS_PORT \ - go test -count=1 -v ./test/e2e/features_test.go -timeout $(or $(E2E_TIMEOUT),20m) -args --godog.tags="@Serial && ~@demo" --godog.concurrency=1; \ + go test -p 1 -count=1 -v ./test/e2e/... -timeout $(or $(E2E_TIMEOUT),20m) -args --godog.tags="@Serial && ~@demo" --godog.concurrency=1; \ serialExit=$$?; \ if [[ $$parallelExit -ne 0 ]] || [[ $$serialExit -ne 0 ]]; then \ echo "e2e tests failed: parallel=$$parallelExit serial=$$serialExit"; \ @@ -454,7 +454,7 @@ else if [[ -z "$$E2E_PROMETHEUS_PORT" ]]; then echo "error: failed to extract prometheus hostPort from $(KIND_CONFIG)" >&2; exit 1; fi; \ KUBECONFIG=$(E2E_KUBECONFIG) \ PROMETHEUS_URL=http://localhost:$$E2E_PROMETHEUS_PORT \ - go test -count=1 -v ./test/e2e/features_test.go -timeout=$(or $(E2E_TIMEOUT),20m) -args $(GODOG_ARGS) + go test -count=1 -v ./test/e2e/... -timeout=$(or $(E2E_TIMEOUT),20m) -args $(GODOG_ARGS) endif .PHONY: e2e-coverage-% @@ -487,7 +487,7 @@ e2e/%: #EXHELP Run e2e scenario against a cluster created by e2e-setup or experi @feature=$$(echo "$*" | cut -d/ -f1); \ scenario=$$(echo "$*" | cut -d/ -f2-); \ if [ "$$scenario" = "$$feature" ]; then scenario=""; fi; \ - KUBECONFIG=$(KUBECONFIG) go test -count=1 -v ./test/e2e/features_test.go \ + KUBECONFIG=$(KUBECONFIG) go test -count=1 -v ./test/e2e/operator-controller/... \ -timeout $(E2E_TIMEOUT) \ -args --godog.concurrency=1 --e2e.scenario="$$scenario" "features/$$feature.feature" @@ -748,7 +748,7 @@ demo-e2e: @command -v jq >/dev/null 2>&1 || { echo "Error: jq not found in PATH."; exit 1; } @mkdir -p $(DEMO_OUTPUT_DIR) KUBECONFIG=$(KUBECONFIG_DIR)/operator-controller-experimental-e2e.kubeconfig \ - DEMO_OUTPUT_DIR=$(DEMO_OUTPUT_DIR) go test -count=1 -v ./test/e2e/features_test.go -timeout 30m \ + DEMO_OUTPUT_DIR=$(DEMO_OUTPUT_DIR) go test -p 1 -count=1 -v ./test/e2e/... -timeout 30m \ -args --godog.tags="@demo" --godog.concurrency=1 .PHONY: demo-svg diff --git a/api/v1/suite_test.go b/api/v1/suite_test.go index e0828f7d04..d45632470c 100644 --- a/api/v1/suite_test.go +++ b/api/v1/suite_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/operator-framework/operator-controller/test" + test "github.com/operator-framework/operator-controller/internal/testutil" ) func newScheme(t *testing.T) *apimachineryruntime.Scheme { diff --git a/cmd/object-controller/main_test.go b/cmd/object-controller/main_test.go index 6fea75f96b..3497e6dd39 100644 --- a/cmd/object-controller/main_test.go +++ b/cmd/object-controller/main_test.go @@ -22,7 +22,7 @@ import ( ocv1 "github.com/operator-framework/operator-controller/api/v1" "github.com/operator-framework/operator-controller/internal/object-controller/scheme" - "github.com/operator-framework/operator-controller/test" + test "github.com/operator-framework/operator-controller/internal/testutil" ) func TestValidateMetricsFlags(t *testing.T) { diff --git a/docs/designs/e2e-per-component.md b/docs/designs/e2e-per-component.md index d04b11abc7..5b586dc238 100644 --- a/docs/designs/e2e-per-component.md +++ b/docs/designs/e2e-per-component.md @@ -27,7 +27,8 @@ Today all e2e tests live in a single Godog (Gherkin/BDD) suite at `test/e2e`: - Each component has its own `features/` directory and its own suite entrypoint, runnable in isolation. - A single command still runs the entire suite across all components. -- A future repo split is mechanical: move a component directory out; the shared library becomes a vendored/published dependency. +- Keep test helpers internal to this repository. A future repo split must explicitly + extract any reusable helpers into a separate library or copy them with the tests. - No change to test behavior during the reorganization — moves are behavior-preserving. ### Non-Goals @@ -43,27 +44,30 @@ Today all e2e tests live in a single Godog (Gherkin/BDD) suite at `test/e2e`: ### Target layout ``` -test/e2e/ - shared/ # extractable library: everything reusable - world/hooks/helpers # world state, hooks, "OLM is available", registry + catalog helpers - runner # TestMain boilerplate: options, --e2e.scenario filter, summary wiring - steps/ # (post-PR3) truly-shared step definitions - support-bundle.yaml # shared diagnostics (moved from test/e2e/) - operator-controller/ - features/ # install, update, uninstall, status, namespace, recover, vap-access-control - suite_test.go # entrypoint: shared runner + shared steps + OC-specific steps - steps/ # (post-PR3) ClusterExtension-specific steps - catalogd/ - features/ # ha, proxy, tls - suite_test.go - steps/ # (post-PR3) catalog-specific steps - object-controller/ - features/ # revision, user-managed-fields - suite_test.go - steps/ # (post-PR3) COS/COD-specific steps - platform/ # genuinely cross-component features - features/ # metrics (OC + catalogd), demos - suite_test.go +test/ + internal/ + e2e/ + runner.go # TestMain boilerplate: options, --e2e.scenario filter, summary wiring + steps/ # (post-PR3) truly-shared step definitions, world state, hooks and helpers + operator-controller/steps/ # (post-PR3) ClusterExtension-specific steps + catalogd/steps/ # (post-PR3) catalog-specific steps + object-controller/steps/ # (post-PR3) COS/COD-specific steps + platform/steps/ # (post-PR3) cross-component and demo steps + support-bundle.yaml # shared diagnostics (moved from test/e2e/) + summary/ # summary generation and embedded templates + e2e/ + operator-controller/ + features/ # install, update, uninstall, status, namespace, recover, vap-access-control + suite_test.go # entrypoint: shared runner + shared steps + OC-specific steps + catalogd/ + features/ # ha, proxy, tls + suite_test.go + object-controller/ + features/ # revision, user-managed-fields + suite_test.go + platform/ # cross-component features + features/ # metrics (OC + catalogd), demos + suite_test.go ``` Each `*/suite_test.go` is a thin Go test package that calls the shared runner, pointing it at its own `features/` directory and registering the shared step set plus any component-specific steps. @@ -79,7 +83,7 @@ Each `*/suite_test.go` is a thin Go test package that calls the shared runner, p - **Tag-only on a single suite (rejected):** smallest change, but leaves everything in one package/dir — it does *not* create the clean boundary a repo split needs, and features are not physically owned. Weakest match to the driver. - **Hybrid single parameterized runner (rejected):** physically separates features/steps but keeps one runner selected by flag. The runner becomes a shared choke point that accretes `if component == …` conditionals the moment components need different suite-level behavior, and the conversion to per-entrypoint has to happen at split time anyway. We pay the cost later without the clean boundary now. -The expensive work — partitioning features and (eventually) factoring the step library — is identical across these options. Approach A is the only one that makes the repo split mechanical and avoids the shared-runner-conditionals risk, so we do it directly. +The expensive work — partitioning features and (eventually) factoring the step library — is identical across these options. Approach A gives each component its own entrypoint without adding component conditionals to the shared runner. Helpers remain internal until a repository split requires an explicit library boundary. --- @@ -101,14 +105,25 @@ Some features are not cleanly owned by a single component. Assignment is by the ## The Shared Library Boundary -`test/e2e/shared` is the seam a repo split will cut along, so it must hold everything reusable and nothing component-specific: +`test/internal/e2e` holds the runner and test utilities shared by the component and +upgrade suites. Go's `internal` visibility lets packages under `test/` reuse them +without treating exported test functions and types as public Go APIs. Component-specific +step packages also stay under this internal tree when they are separated in PR3. - **World/state + hooks** (`hooks.go`): scenario setup/teardown, namespace lifecycle. - **Common Given steps:** "OLM is available", "an image registry is available", "a catalog … with packages", generic apply/assert helpers. These are prerequisites every component needs. -- **Runner boilerplate:** the Godog `Options`, the `--e2e.scenario` prefix filter (`findScenarioFirstLineNumberByPrefix`), and summary wiring (`test/internal/summary`) — currently inlined in `features_test.go:36-134`. Lifted into a shared helper so each `suite_test.go` is a few lines. **Caveat:** `test/internal/summary` is a Go `internal` package, importable only within this repo's tree; the shared runner can import it today, but a repo split would make that import illegal. Either move the summary code under `shared/` as part of the extraction, or keep the summary callback in each repo's `suite_test.go` so the shared runner carries no `internal` dependency. +- **Runner boilerplate:** the Godog `Options`, the `--e2e.scenario` prefix filter (`findScenarioFirstLineNumberByPrefix`), and summary wiring (`test/internal/summary`) — currently inlined in `features_test.go:36-134`. Lifted into a shared helper so each `suite_test.go` is a few lines. Summary templates are embedded so generation works from any suite directory. - **Diagnostics:** `support-bundle.yaml`. -`test/upgrade-e2e` already imports `test/e2e/steps`; after the move it becomes another consumer of `test/e2e/shared/steps`. This is expected and reinforces the shared-library design. +`test/upgrade-e2e` already imports `test/e2e/steps`; after the move it becomes another +consumer of `test/internal/e2e/steps`. Envtest helpers used outside `test/` move from +`test/utils.go` to `internal/testutil`, where API and controller unit tests can import them. + +A future repository split must extract or copy the required internal helpers with +the suites. These helpers do not provide a cross-repository compatibility guarantee. +Moving the existing public packages still reports API removals during this migration; +the initial transition requires maintainer review and the existing `go-apidiff-override` +label. Subsequent internal-helper changes should no longer need that override. --- @@ -116,13 +131,17 @@ Some features are not cleanly owned by a single component. Assignment is by the The ordering is deliberate: **PR2 is safe precisely because PR3 has not happened yet.** Through PR1–PR2 the step definitions remain one monolithic package, so every component suite can register the *entire* shared step set. A feature moved into `catalogd/features/` still resolves all its steps. Godog `Strict: true` fails only on *undefined/pending* steps, never on *registered-but-unused* ones, so over-registering is free. The step split (PR3) therefore blocks nothing. -### PR1 — Reroot layout + establish `shared` (mechanical, behavior-preserving) +### PR1 — Reroot layout + establish internal helpers (mechanical, behavior-preserving) - `git mv test/e2e/features/` → `test/e2e/operator-controller/features/` (all features stay together for now). -- `git mv test/e2e/steps/` → `test/e2e/shared/steps/`. -- Lift runner boilerplate (`TestMain`, `--e2e.scenario` filter, summary wiring) from `features_test.go` into `test/e2e/shared` so later suites reuse it. +- `git mv test/e2e/steps/` → `test/internal/e2e/steps/`. +- Lift runner boilerplate (`TestMain`, `--e2e.scenario` filter, summary wiring) from `features_test.go` into `test/internal/e2e` so later suites reuse it. - Create the operator-controller `suite_test.go` calling the shared runner. -- Move `support-bundle.yaml` into `shared/`. +- Reserve `catalogd/features/`, `object-controller/features/`, and `platform/features/` + with `.gitkeep` files. Their suite entry points and feature moves follow in PR2. +- Move `support-bundle.yaml` into `test/internal/e2e/`. +- Keep summary generation under `test/internal/summary` and embed its templates. +- Move `test/utils.go` to `internal/testutil` and update unit-test imports and project-root discovery. - **Update all references in the same PR:** - `test/upgrade-e2e/upgrade_test.go` import path for the steps package. - Makefile targets referencing `./test/e2e/features_test.go` (lines ~431, 435, 446, 479, 740) and the `e2e/%` scenario path. @@ -132,7 +151,7 @@ The ordering is deliberate: **PR2 is safe precisely because PR3 has not happened ### PR2 — Sort features into component suites (may split per component) -- Create `catalogd/`, `object-controller/`, and `platform/` suites, each with a thin `suite_test.go` that calls the shared runner and registers the full shared step set. +- Add a thin `suite_test.go` in each reserved `catalogd/`, `object-controller/`, and `platform/` directory, calling the shared runner and registering the full shared step set. Remove each `.gitkeep` when populating its feature directory. - `git mv` features to their owners per the ownership table. Moving `demos.feature` into `platform/features/` shifts a path tracked by `.github/workflows/update-demos.yaml`; update that workflow in the same PR (maintainer sign-off, as in PR1). - The operator-controller suite shrinks as features leave; `go test ./test/e2e/...` now runs four suites. - Can be landed as separate PRs per component (e.g. object-controller first, then catalogd, then platform). @@ -140,7 +159,8 @@ The ordering is deliberate: **PR2 is safe precisely because PR3 has not happened ### PR3 — Split the step library per component (judgment-heavy, incremental) -- Partition `shared/steps` into truly-shared vs. component-specific. Rough starting map: +- Partition `test/internal/e2e/steps` into truly-shared vs. component-specific packages + under `test/internal/e2e`. Rough starting map: - **Shared:** `hooks.go`, world setup, "OLM is available"/registry/catalog Given steps, generic apply/assert (the shared portion of `steps.go`). - **operator-controller:** `ClusterExtension`-specific steps from `steps.go`. - **catalogd:** `ha_steps.go`, `proxy_steps.go`, `tls_steps.go`. @@ -148,13 +168,13 @@ The ordering is deliberate: **PR2 is safe precisely because PR3 has not happened - **upgrade-e2e:** `upgrade_steps.go` (consider relocating to the upgrade suite). - Moving the demo step files (`demo_steps.go`, `asciicast_*.go`) again shifts paths tracked by `.github/workflows/update-demos.yaml`; update that workflow in the same PR. - Each suite registers shared + its own steps. -- Can land one component at a time; nothing depends on it completing. This is where a future repo split becomes trivial. +- Can land one component at a time; nothing depends on it completing. A future repo split can then select the required internal helpers explicitly. ### Sequencing summary | PR | Scope | Risk | Splittable | |---|---|---|---| -| PR1 | Reroot + `shared`, update refs | Low (mechanical) | No | +| PR1 | Reroot + internal helpers, update refs | Low (mechanical) | No | | PR2 | Sort features into suites | Low (moves; over-registered steps) | Up to 3–4 PRs | | PR3 | Split step library | Medium (judgment) | Up to 3–4 PRs, incremental | @@ -175,4 +195,4 @@ The ordering is deliberate: **PR2 is safe precisely because PR3 has not happened 1. **`platform/` bucket vs. splitting `metrics`/`demos` per component.** Recommended: keep `platform/`. Alternative: duplicate the metrics scenario outline per component and drop `platform/`. 2. **Per-component Makefile targets.** Whether to add `test-e2e-` convenience targets, or rely on `go test ./test/e2e//...` directly. (Leaning: add thin targets mirroring the existing `test-e2e` env setup, since cluster/manifest env wiring is non-trivial.) -3. **`upgrade_steps.go` home.** Relocate into the upgrade suite during PR3, or keep in `shared/steps`. +3. **`upgrade_steps.go` home.** Relocate into the upgrade suite during PR3, or keep in `test/internal/e2e/steps`. diff --git a/internal/object-controller/controllers/suite_test.go b/internal/object-controller/controllers/suite_test.go index e1c6636e14..1aee97a3c5 100644 --- a/internal/object-controller/controllers/suite_test.go +++ b/internal/object-controller/controllers/suite_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/client-go/rest" ocv1 "github.com/operator-framework/operator-controller/api/v1" - "github.com/operator-framework/operator-controller/test" + test "github.com/operator-framework/operator-controller/internal/testutil" ) func newScheme(t *testing.T) *apimachineryruntime.Scheme { diff --git a/internal/operator-controller/action/action_test.go b/internal/operator-controller/action/action_test.go index 46eefe8cec..84f770b618 100644 --- a/internal/operator-controller/action/action_test.go +++ b/internal/operator-controller/action/action_test.go @@ -9,7 +9,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/rest" - "github.com/operator-framework/operator-controller/test" + test "github.com/operator-framework/operator-controller/internal/testutil" ) var cfg *rest.Config diff --git a/internal/operator-controller/applier/boxcutter_test.go b/internal/operator-controller/applier/boxcutter_test.go index 7f51e4feb9..3fafcfe2c4 100644 --- a/internal/operator-controller/applier/boxcutter_test.go +++ b/internal/operator-controller/applier/boxcutter_test.go @@ -33,7 +33,6 @@ import ( "github.com/operator-framework/operator-controller/internal/testutil" mockapplier "github.com/operator-framework/operator-controller/internal/testutil/mock/applier" mockctrlclient "github.com/operator-framework/operator-controller/internal/testutil/mock/ctrlclient" - "github.com/operator-framework/operator-controller/test" ) var ( @@ -1040,10 +1039,10 @@ func TestBoxcutterApply_Adoption(t *testing.T) { func TestBoxcutterApply_RecreatesDeploymentAfterDeletion(t *testing.T) { t.Parallel() - testEnv := test.NewEnv() + testEnv := testutil.NewEnv() config, err := testEnv.Start() require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, test.StopWithRetry(testEnv, time.Minute, time.Second)) }) + t.Cleanup(func() { require.NoError(t, testutil.StopWithRetry(testEnv, time.Minute, time.Second)) }) scheme := codTestScheme(t) cl, err := client.New(config, client.Options{Scheme: scheme}) require.NoError(t, err) diff --git a/internal/operator-controller/controllers/suite_test.go b/internal/operator-controller/controllers/suite_test.go index ba2b926f42..ad8deb784d 100644 --- a/internal/operator-controller/controllers/suite_test.go +++ b/internal/operator-controller/controllers/suite_test.go @@ -37,8 +37,8 @@ import ( "github.com/operator-framework/operator-controller/internal/operator-controller/controllers" "github.com/operator-framework/operator-controller/internal/operator-controller/resolve" "github.com/operator-framework/operator-controller/internal/shared/util/image" + test "github.com/operator-framework/operator-controller/internal/testutil" mockcontrollers "github.com/operator-framework/operator-controller/internal/testutil/mock/controllers" - "github.com/operator-framework/operator-controller/test" ) func newScheme(t *testing.T) *apimachineryruntime.Scheme { diff --git a/test/utils.go b/internal/testutil/envtest.go similarity index 97% rename from test/utils.go rename to internal/testutil/envtest.go index 441aa7d689..5a3cf631e0 100644 --- a/test/utils.go +++ b/internal/testutil/envtest.go @@ -1,4 +1,4 @@ -package test +package testutil import ( "bufio" @@ -47,7 +47,7 @@ func NewEnv() *envtest.Environment { // pathFromProjectRoot returns the absolute path to the given relative path from the project root. func pathFromProjectRoot(relativePath string) string { _, filename, _, _ := runtime.Caller(0) - p := path.Join(path.Dir(path.Dir(filename)), relativePath) + p := path.Join(path.Dir(path.Dir(path.Dir(filename))), relativePath) return p } diff --git a/test/e2e/README.md b/test/e2e/README.md index 954787f5fb..1e369bded0 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -20,34 +20,49 @@ non-technical stakeholders. ## Project Structure ``` -test/e2e/ -├── README.md # This file -├── features_test.go # Test runner and suite initialization -├── features/ # Gherkin feature files -│ ├── install.feature # ClusterExtension installation scenarios -│ ├── update.feature # ClusterExtension update scenarios -│ ├── recover.feature # Recovery scenarios -│ ├── status.feature # ClusterExtension status scenarios -│ └── metrics.feature # Metrics endpoint scenarios -└── steps/ # Step definitions and test utilities - ├── steps.go # Step definition implementations - ├── hooks.go # Test hooks and scenario context - └── testdata/ # Test data (RBAC templates) - ├── serviceaccount-template.yaml - ├── olm-sa-helm-rbac-template.yaml - ├── olm-sa-boxcutter-rbac-template.yaml - ├── pvc-probe-sa-boxcutter-rbac-template.yaml - ├── cluster-admin-rbac-template.yaml - └── metrics-reader-rbac-template.yaml +test/ +├── e2e/ +│ ├── README.md +│ ├── features_test.go # Legacy file entry point +│ ├── operator-controller/ +│ │ ├── suite_test.go # Component suite entry point +│ │ └── features/ # All existing features during the first migration stage +│ ├── catalogd/ +│ │ └── features/ # Placeholder for the catalogd suite +│ ├── object-controller/ +│ │ └── features/ # Placeholder for the object-controller suite +│ └── platform/ +│ └── features/ # Placeholder for the cross-component suite +└── internal/ + ├── e2e/ + │ ├── runner.go # Godog options, scenario selection, summary wiring + │ ├── steps/ # All existing step definitions and hooks + │ │ └── testdata/ # RBAC and resource templates + │ └── support-bundle.yaml # Shared diagnostics + └── summary/ # Summary generation and embedded templates ``` ## Architecture -### 1. Test Runner (`features_test.go`) +### 1. Test Runner (`operator-controller/suite_test.go`) -The main test entry point that configures and runs the Godog test suite. +The component entry point registers the full shared step set and hooks, then calls +`e2e.Run` from `test/internal/e2e` with its feature directory. The shared runner configures Godog options, +scenario selection, and summary generation. -### 2. Feature Files (`features/*.feature`) +Test helpers live under `test/internal` so their exported functions can be shared +by test suites without becoming public Go APIs. Envtest helpers used by API and +controller unit tests live under `internal/testutil`, which is accessible throughout +the repository. + +This is the first stage of the [per-component suite design](../../docs/designs/e2e-per-component.md). +All existing features remain together under `operator-controller/features/` during +this stage. The catalogd, object-controller, and platform feature directories contain +only `.gitkeep` placeholders. Later branches will add their suite entry points and +move features to their owners, then separate component-specific steps. Feature +contents and step behavior are unchanged. + +### 2. Feature Files (`operator-controller/features/*.feature`) Gherkin files that describe test scenarios in natural language. @@ -101,7 +116,7 @@ Feature: Install ClusterExtension And ClusterExtension is available ``` -### 3. Step Definitions (`steps/steps.go`) +### 3. Step Definitions (`test/internal/e2e/steps/steps.go`) Go functions that implement the steps defined in feature files. Each step is registered with a regex pattern that matches the Gherkin text. @@ -153,7 +168,7 @@ func BundleInstalled(ctx context.Context, name, version string) error { } ``` -### 4. Hooks and Context (`steps/hooks.go`) +### 4. Hooks and Context (`test/internal/e2e/steps/hooks.go`) Manages test lifecycle and scenario-specific context. @@ -171,7 +186,7 @@ Replaces `${TEST_NAMESPACE}`, `${NAME}`, `${SCENARIO_ID}`, `${PACKAGE:}`, ### 1. Create a Feature File -Create a new `.feature` file in `test/e2e/features/`: +Create a new `.feature` file in `test/e2e/operator-controller/features/`: ```gherkin Feature: Your Feature Name @@ -188,7 +203,7 @@ Feature: Your Feature Name ### 2. Implement Step Definitions -Add step implementations in `steps/steps.go`: +Add step implementations in `test/internal/e2e/steps/steps.go`: ```go func RegisterSteps(sc *godog.ScenarioContext) { @@ -269,6 +284,22 @@ make test-e2e # Standard features make test-experimental-e2e # Experimental features ``` +### Run Suites Against an Existing Cluster + +The aggregate package pattern discovers component suites automatically. Preserve +the two-pass tag policy and disable Go test caching when running it directly: + +```bash +go test -count=1 -v ./test/e2e/... -args --godog.tags="~@Serial && ~@demo" --godog.concurrency=100 +go test -p 1 -count=1 -v ./test/e2e/... -args --godog.tags="@Serial && ~@demo" --godog.concurrency=1 +``` + +`-p 1` also serializes packages, keeping serial scenarios exclusive as more suites +are added. Demo scenarios run separately through `make update-demos`. + +To select the current component suite, use `./test/e2e/operator-controller/...` +with the same flags. Additional component suites will be added in the next stage. + ### Run Standard and Experimental in Parallel Both suites can run simultaneously on separate KIND clusters (see the Makefile @@ -340,27 +371,27 @@ make experimental-e2e-teardown # Experimental cluster ### Run Specific Feature ```bash -go test test/e2e/features_test.go -- features/install.feature +go test -count=1 ./test/e2e/operator-controller/... -args features/install.feature ``` ### Run Specific Scenario by Tag ```bash -go test test/e2e/features_test.go --godog.tags="@WebhookProviderCertManager" +go test -count=1 ./test/e2e/operator-controller/... -args --godog.tags="@WebhookProviderCertManager" ``` ### Run with Debug Logging ```bash -go test -v test/e2e/features_test.go --log.debug +go test -count=1 -v ./test/e2e/operator-controller/... -args --log.debug ``` ### CLI Options -Godog options can be passed after `--`: +Godog options can be passed after `-args`: ```bash -go test test/e2e/features_test.go \ +go test -count=1 ./test/e2e/operator-controller/... -args \ --godog.format=pretty \ --godog.tags="@WebhookProviderCertManager" ``` @@ -378,7 +409,7 @@ Available formats: `pretty`, `cucumber`, `progress`, `junit` **Example:** ```bash -go test test/e2e/features_test.go --log.debug --k8s.cli=oc +go test -count=1 ./test/e2e/operator-controller/... -args --log.debug --k8s.cli=oc ``` ### Environment Variables @@ -459,7 +490,7 @@ features. A list of available, implemented steps can be obtained by running: ```shell -go test test/e2e/features_test.go -d +go test -count=1 ./test/e2e/operator-controller/... -args --godog.definitions ``` When working with [Claude Code](https://claude.com/claude-code), run the `/list-e2e-steps` command to get a categorized diff --git a/test/e2e/catalogd/features/.gitkeep b/test/e2e/catalogd/features/.gitkeep new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/test/e2e/catalogd/features/.gitkeep @@ -0,0 +1 @@ + diff --git a/test/e2e/features_test.go b/test/e2e/features_test.go index 65eaebe255..5cdee2cb92 100644 --- a/test/e2e/features_test.go +++ b/test/e2e/features_test.go @@ -1,144 +1,30 @@ +//go:build e2e_legacy + +// This compatibility entry point is invoked explicitly as a file by downstream +// Godog jobs. The build tag excludes it from ./test/e2e/... to avoid duplicate runs. package e2e import ( - "fmt" "log" "os" - "strings" "testing" - gherkin "github.com/cucumber/gherkin/go/v26" "github.com/cucumber/godog" - "github.com/cucumber/godog/colors" - messages "github.com/cucumber/messages/go/v21" - "github.com/spf13/pflag" - "github.com/operator-framework/operator-controller/test/e2e/steps" - "github.com/operator-framework/operator-controller/test/internal/summary" + "github.com/operator-framework/operator-controller/test/internal/e2e" + "github.com/operator-framework/operator-controller/test/internal/e2e/steps" ) -var opts = godog.Options{ - Format: "pretty", - Paths: []string{"features"}, - Output: colors.Colored(os.Stdout), - Concurrency: 1, - NoColors: true, - Strict: true, -} - -var scenarioFilter string - -func init() { - godog.BindCommandLineFlags("godog.", &opts) - pflag.StringVar(&scenarioFilter, "e2e.scenario", "", "scenario name prefix (case-insensitive)") -} - func TestMain(m *testing.M) { - // parse CLI arguments - pflag.Parse() - opts.Paths = pflag.Args() - - if scenarioFilter != "" { - if len(opts.Paths) != 1 { - log.Fatalf("--e2e.scenario requires exactly one feature file path, got %d", len(opts.Paths)) - } - lines, err := findScenarioFirstLineNumberByPrefix(opts.Paths[0], scenarioFilter) - if err != nil { - log.Fatal(err) - } - basePath := opts.Paths[0] - opts.Paths = make([]string, len(lines)) - for i, line := range lines { - opts.Paths[i] = fmt.Sprintf("%s:%d", basePath, line) - } - } - - // run tests - sc := godog.TestSuite{ - TestSuiteInitializer: InitializeSuite, - ScenarioInitializer: InitializeScenario, - Options: &opts, - }.Run() - - switch sc { - // 0 - success - case 0: - - path := os.Getenv("E2E_SUMMARY_OUTPUT") - if path == "" { - fmt.Println("Note: E2E_SUMMARY_OUTPUT is unset; skipping summary generation") - } else { - if err := summary.PrintSummary(path); err != nil { - // Alert but do not fail the run if alerts are found - fmt.Printf("%v", err) - } - } - return - - // 1 - failed - // 2 - command line usage error - // 128 - or higher, os signal related error exit codes - default: - log.Fatalf("non-zero status returned (%d), failed to run feature tests", sc) - } -} - -func findScenarioFirstLineNumberByPrefix(featurePath, prefix string) ([]int, error) { - f, err := os.Open(featurePath) - if err != nil { - return nil, fmt.Errorf("failed to open %s: %w", featurePath, err) - } - defer f.Close() - - doc, err := gherkin.ParseGherkinDocument(f, (&messages.Incrementing{}).NewId) - if err != nil { - return nil, fmt.Errorf("failed to parse %s: %w", featurePath, err) - } - - if doc.Feature == nil { - return nil, fmt.Errorf("no Feature found in %s", featurePath) - } - - prefix = strings.TrimSpace(prefix) - if prefix == "" { - return nil, fmt.Errorf("scenario prefix must not be empty") - } - prefix = strings.ToLower(prefix) - var matches []int - var allNames []string - - matchScenario := func(sc *messages.Scenario) { - allNames = append(allNames, sc.Name) - if strings.HasPrefix(strings.ToLower(sc.Name), prefix) { - matches = append(matches, int(sc.Location.Line)) - } - } - for _, child := range doc.Feature.Children { - if child.Scenario != nil { - matchScenario(child.Scenario) - } - if child.Rule != nil { - for _, rc := range child.Rule.Children { - if rc.Scenario != nil { - matchScenario(rc.Scenario) - } - } - } - } - - if len(matches) == 0 { - return nil, fmt.Errorf("no scenario matching prefix %q in %s\navailable scenarios:\n %s", - prefix, featurePath, strings.Join(allNames, "\n ")) - } - return matches, nil -} - -func InitializeSuite(tc *godog.TestSuiteContext) { - tc.BeforeSuite(steps.BeforeSuite) -} - -func InitializeScenario(sc *godog.ScenarioContext) { - steps.RegisterSteps(sc) - steps.RegisterDemoSteps(sc) - steps.RegisterHooks(sc) + // Keep legacy paths such as features/install.feature relative to the suite. + if err := os.Chdir("operator-controller"); err != nil { + log.Fatal(err) + } + os.Exit(e2e.Run("features", func(tc *godog.TestSuiteContext) { + tc.BeforeSuite(steps.BeforeSuite) + }, func(sc *godog.ScenarioContext) { + steps.RegisterSteps(sc) + steps.RegisterDemoSteps(sc) + steps.RegisterHooks(sc) + })) } diff --git a/test/e2e/object-controller/features/.gitkeep b/test/e2e/object-controller/features/.gitkeep new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/test/e2e/object-controller/features/.gitkeep @@ -0,0 +1 @@ + diff --git a/test/e2e/features/cod_adoption_conflict.feature b/test/e2e/operator-controller/features/cod_adoption_conflict.feature similarity index 100% rename from test/e2e/features/cod_adoption_conflict.feature rename to test/e2e/operator-controller/features/cod_adoption_conflict.feature diff --git a/test/e2e/features/cod_progressing_reasons.feature b/test/e2e/operator-controller/features/cod_progressing_reasons.feature similarity index 100% rename from test/e2e/features/cod_progressing_reasons.feature rename to test/e2e/operator-controller/features/cod_progressing_reasons.feature diff --git a/test/e2e/features/cod_revision_history.feature b/test/e2e/operator-controller/features/cod_revision_history.feature similarity index 100% rename from test/e2e/features/cod_revision_history.feature rename to test/e2e/operator-controller/features/cod_revision_history.feature diff --git a/test/e2e/features/cod_rollout.feature b/test/e2e/operator-controller/features/cod_rollout.feature similarity index 100% rename from test/e2e/features/cod_rollout.feature rename to test/e2e/operator-controller/features/cod_rollout.feature diff --git a/test/e2e/features/cod_template_metadata.feature b/test/e2e/operator-controller/features/cod_template_metadata.feature similarity index 100% rename from test/e2e/features/cod_template_metadata.feature rename to test/e2e/operator-controller/features/cod_template_metadata.feature diff --git a/test/e2e/features/cos_object_collision_and_adoption.feature b/test/e2e/operator-controller/features/cos_object_collision_and_adoption.feature similarity index 100% rename from test/e2e/features/cos_object_collision_and_adoption.feature rename to test/e2e/operator-controller/features/cos_object_collision_and_adoption.feature diff --git a/test/e2e/features/cos_preflight.feature b/test/e2e/operator-controller/features/cos_preflight.feature similarity index 100% rename from test/e2e/features/cos_preflight.feature rename to test/e2e/operator-controller/features/cos_preflight.feature diff --git a/test/e2e/features/cos_revision_identity.feature b/test/e2e/operator-controller/features/cos_revision_identity.feature similarity index 100% rename from test/e2e/features/cos_revision_identity.feature rename to test/e2e/operator-controller/features/cos_revision_identity.feature diff --git a/test/e2e/features/cos_static_validation.feature b/test/e2e/operator-controller/features/cos_static_validation.feature similarity index 100% rename from test/e2e/features/cos_static_validation.feature rename to test/e2e/operator-controller/features/cos_static_validation.feature diff --git a/test/e2e/features/demos.feature b/test/e2e/operator-controller/features/demos.feature similarity index 100% rename from test/e2e/features/demos.feature rename to test/e2e/operator-controller/features/demos.feature diff --git a/test/e2e/features/group.feature b/test/e2e/operator-controller/features/group.feature similarity index 100% rename from test/e2e/features/group.feature rename to test/e2e/operator-controller/features/group.feature diff --git a/test/e2e/features/ha.feature b/test/e2e/operator-controller/features/ha.feature similarity index 100% rename from test/e2e/features/ha.feature rename to test/e2e/operator-controller/features/ha.feature diff --git a/test/e2e/features/install.feature b/test/e2e/operator-controller/features/install.feature similarity index 100% rename from test/e2e/features/install.feature rename to test/e2e/operator-controller/features/install.feature diff --git a/test/e2e/features/metrics.feature b/test/e2e/operator-controller/features/metrics.feature similarity index 100% rename from test/e2e/features/metrics.feature rename to test/e2e/operator-controller/features/metrics.feature diff --git a/test/e2e/features/namespace.feature b/test/e2e/operator-controller/features/namespace.feature similarity index 100% rename from test/e2e/features/namespace.feature rename to test/e2e/operator-controller/features/namespace.feature diff --git a/test/e2e/features/proxy.feature b/test/e2e/operator-controller/features/proxy.feature similarity index 100% rename from test/e2e/features/proxy.feature rename to test/e2e/operator-controller/features/proxy.feature diff --git a/test/e2e/features/recover.feature b/test/e2e/operator-controller/features/recover.feature similarity index 100% rename from test/e2e/features/recover.feature rename to test/e2e/operator-controller/features/recover.feature diff --git a/test/e2e/features/revision.feature b/test/e2e/operator-controller/features/revision.feature similarity index 100% rename from test/e2e/features/revision.feature rename to test/e2e/operator-controller/features/revision.feature diff --git a/test/e2e/features/status.feature b/test/e2e/operator-controller/features/status.feature similarity index 100% rename from test/e2e/features/status.feature rename to test/e2e/operator-controller/features/status.feature diff --git a/test/e2e/features/tls.feature b/test/e2e/operator-controller/features/tls.feature similarity index 100% rename from test/e2e/features/tls.feature rename to test/e2e/operator-controller/features/tls.feature diff --git a/test/e2e/features/uninstall.feature b/test/e2e/operator-controller/features/uninstall.feature similarity index 100% rename from test/e2e/features/uninstall.feature rename to test/e2e/operator-controller/features/uninstall.feature diff --git a/test/e2e/features/update.feature b/test/e2e/operator-controller/features/update.feature similarity index 100% rename from test/e2e/features/update.feature rename to test/e2e/operator-controller/features/update.feature diff --git a/test/e2e/features/user-managed-fields.feature b/test/e2e/operator-controller/features/user-managed-fields.feature similarity index 100% rename from test/e2e/features/user-managed-fields.feature rename to test/e2e/operator-controller/features/user-managed-fields.feature diff --git a/test/e2e/features/vap-access-control.feature b/test/e2e/operator-controller/features/vap-access-control.feature similarity index 100% rename from test/e2e/features/vap-access-control.feature rename to test/e2e/operator-controller/features/vap-access-control.feature diff --git a/test/e2e/operator-controller/suite_test.go b/test/e2e/operator-controller/suite_test.go new file mode 100644 index 0000000000..721ee038df --- /dev/null +++ b/test/e2e/operator-controller/suite_test.go @@ -0,0 +1,21 @@ +package e2e + +import ( + "os" + "testing" + + "github.com/cucumber/godog" + + "github.com/operator-framework/operator-controller/test/internal/e2e" + "github.com/operator-framework/operator-controller/test/internal/e2e/steps" +) + +func TestMain(m *testing.M) { + os.Exit(e2e.Run("features", func(tc *godog.TestSuiteContext) { + tc.BeforeSuite(steps.BeforeSuite) + }, func(sc *godog.ScenarioContext) { + steps.RegisterSteps(sc) + steps.RegisterDemoSteps(sc) + steps.RegisterHooks(sc) + })) +} diff --git a/test/e2e/platform/features/.gitkeep b/test/e2e/platform/features/.gitkeep new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/test/e2e/platform/features/.gitkeep @@ -0,0 +1 @@ + diff --git a/test/internal/e2e/runner.go b/test/internal/e2e/runner.go new file mode 100644 index 0000000000..90a6a12863 --- /dev/null +++ b/test/internal/e2e/runner.go @@ -0,0 +1,135 @@ +// Package e2e provides the runner used by component E2E suites. +package e2e + +import ( + "fmt" + "log" + "os" + "strings" + + gherkin "github.com/cucumber/gherkin/go/v26" + "github.com/cucumber/godog" + "github.com/cucumber/godog/colors" + messages "github.com/cucumber/messages/go/v21" + "github.com/spf13/pflag" + + "github.com/operator-framework/operator-controller/test/internal/summary" +) + +var opts = godog.Options{ + Format: "pretty", + Output: colors.Colored(os.Stdout), + Concurrency: 1, + NoColors: true, + Strict: true, +} + +var scenarioFilter string + +func init() { + godog.BindCommandLineFlags("godog.", &opts) + pflag.StringVar(&scenarioFilter, "e2e.scenario", "", "scenario name prefix (case-insensitive)") +} + +// Run executes a component suite using its feature directory and initializers. +// Call it from TestMain and pass the returned status to os.Exit. +func Run(featureDir string, suiteInitializer func(*godog.TestSuiteContext), scenarioInitializer func(*godog.ScenarioContext)) int { + pflag.Parse() + options := opts + paths, err := featurePaths(featureDir, pflag.Args(), scenarioFilter) + if err != nil { + log.Print(err) + return 1 + } + options.Paths = paths + + status := godog.TestSuite{ + TestSuiteInitializer: suiteInitializer, + ScenarioInitializer: scenarioInitializer, + Options: &options, + }.Run() + if status != 0 { + log.Printf("non-zero status returned (%d), failed to run feature tests", status) + return 1 + } + + path := os.Getenv("E2E_SUMMARY_OUTPUT") + if path == "" { + fmt.Println("Note: E2E_SUMMARY_OUTPUT is unset; skipping summary generation") + } else if err := summary.PrintSummary(path); err != nil { + // Alert but do not fail the run if alerts are found. + fmt.Printf("%v", err) + } + return 0 +} + +func featurePaths(featureDir string, paths []string, prefix string) ([]string, error) { + if prefix == "" { + if len(paths) == 0 { + return []string{featureDir}, nil + } + return paths, nil + } + if len(paths) != 1 { + return nil, fmt.Errorf("--e2e.scenario requires exactly one feature file path, got %d", len(paths)) + } + lines, err := findScenarioFirstLineNumberByPrefix(paths[0], prefix) + if err != nil { + return nil, err + } + selected := make([]string, len(lines)) + for i, line := range lines { + selected[i] = fmt.Sprintf("%s:%d", paths[0], line) + } + return selected, nil +} + +func findScenarioFirstLineNumberByPrefix(featurePath, prefix string) ([]int, error) { + f, err := os.Open(featurePath) + if err != nil { + return nil, fmt.Errorf("failed to open %s: %w", featurePath, err) + } + defer f.Close() + + doc, err := gherkin.ParseGherkinDocument(f, (&messages.Incrementing{}).NewId) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", featurePath, err) + } + + if doc.Feature == nil { + return nil, fmt.Errorf("no Feature found in %s", featurePath) + } + + prefix = strings.TrimSpace(prefix) + if prefix == "" { + return nil, fmt.Errorf("scenario prefix must not be empty") + } + prefix = strings.ToLower(prefix) + var matches []int + var allNames []string + + matchScenario := func(sc *messages.Scenario) { + allNames = append(allNames, sc.Name) + if strings.HasPrefix(strings.ToLower(sc.Name), prefix) { + matches = append(matches, int(sc.Location.Line)) + } + } + for _, child := range doc.Feature.Children { + if child.Scenario != nil { + matchScenario(child.Scenario) + } + if child.Rule != nil { + for _, rc := range child.Rule.Children { + if rc.Scenario != nil { + matchScenario(rc.Scenario) + } + } + } + } + + if len(matches) == 0 { + return nil, fmt.Errorf("no scenario matching prefix %q in %s\navailable scenarios:\n %s", + prefix, featurePath, strings.Join(allNames, "\n ")) + } + return matches, nil +} diff --git a/test/internal/e2e/runner_test.go b/test/internal/e2e/runner_test.go new file mode 100644 index 0000000000..6b2a509b8d --- /dev/null +++ b/test/internal/e2e/runner_test.go @@ -0,0 +1,156 @@ +package e2e_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/cucumber/godog" + "github.com/stretchr/testify/require" + + "github.com/operator-framework/operator-controller/test/internal/e2e" +) + +func TestRunner(t *testing.T) { + // The runner parses process-wide command-line flags, so these cases are serial. + originalArgs := os.Args + t.Cleanup(func() { os.Args = originalArgs }) + t.Setenv("E2E_SUMMARY_OUTPUT", "") + + featureDir := t.TempDir() + featurePath := filepath.Join(featureDir, "install.feature") + require.NoError(t, os.WriteFile(featurePath, []byte(`Feature: Install + Scenario: Install latest + Given a passing step + + @Serial + Scenario Outline: Install version + Given version "" + Examples: + | version | + | 1.0.0 | + | 2.0.0 | + + Rule: Recovery + Scenario: Install after failure + Given a passing step + + Scenario: Recover existing + Given a passing step + + @Undefined + Scenario: Undefined step + Given an undefined step +`), 0o600)) + + for _, tt := range []struct { + name string + paths []string + prefix string + tags string + want []string + wantError bool + }{ + { + name: "default feature directory", + want: []string{"Install latest", "Install version", "Install version", "Install after failure", "Recover existing"}, + }, + { + name: "explicit line selector", + paths: []string{featurePath + ":2"}, + want: []string{"Install latest"}, + }, + { + name: "case insensitive prefix includes outlines and rules", + paths: []string{featurePath}, + prefix: " INSTALL ", + want: []string{"Install latest", "Install version", "Install version", "Install after failure"}, + }, + { + name: "single scenario prefix", + paths: []string{featurePath}, + prefix: "Install latest", + want: []string{"Install latest"}, + }, + { + name: "serial tag selection", + tags: "@Serial", + want: []string{"Install version", "Install version"}, + }, + { + name: "parallel tag selection", + tags: "~@Serial", + want: []string{"Install latest", "Install after failure", "Recover existing"}, + }, + { + name: "missing feature path", + prefix: "Install", + wantError: true, + }, + { + name: "multiple feature paths", + paths: []string{featurePath, featurePath}, + prefix: "Install", + wantError: true, + }, + { + name: "no matching scenario", + paths: []string{featurePath}, + prefix: "Uninstall", + wantError: true, + }, + { + name: "empty prefix", + paths: []string{featurePath}, + prefix: " ", + wantError: true, + }, + { + name: "missing file", + paths: []string{filepath.Join(featureDir, "missing.feature")}, + prefix: "Install", + wantError: true, + }, + { + name: "strict mode rejects undefined steps", + tags: "@Undefined", + want: []string{"Undefined step"}, + wantError: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + tags := "~@Undefined" + if tt.tags != "" { + tags += " && " + tt.tags + } + if tt.tags == "@Undefined" { + tags = tt.tags + } + os.Args = append([]string{ + "e2e.test", "--godog.format=progress", "--godog.concurrency=1", + "--godog.tags=" + tags, "--e2e.scenario=" + tt.prefix, + }, tt.paths...) + + var scenarios []string + beforeSuite := false + status := e2e.Run(featureDir, func(tc *godog.TestSuiteContext) { + tc.BeforeSuite(func() { beforeSuite = true }) + }, func(sc *godog.ScenarioContext) { + sc.Step(`^a passing step$`, func() {}) + sc.Step(`^version "([^"]+)"$`, func(string) {}) + sc.Before(func(ctx context.Context, scenario *godog.Scenario) (context.Context, error) { + scenarios = append(scenarios, scenario.Name) + return ctx, nil + }) + }) + if tt.wantError { + require.NotZero(t, status) + } else { + require.Zero(t, status) + require.True(t, beforeSuite) + } + require.Equal(t, tt.want, scenarios) + }) + } +} diff --git a/test/e2e/steps/asciicast_hooks.go b/test/internal/e2e/steps/asciicast_hooks.go similarity index 100% rename from test/e2e/steps/asciicast_hooks.go rename to test/internal/e2e/steps/asciicast_hooks.go diff --git a/test/e2e/steps/asciicast_recorder.go b/test/internal/e2e/steps/asciicast_recorder.go similarity index 100% rename from test/e2e/steps/asciicast_recorder.go rename to test/internal/e2e/steps/asciicast_recorder.go diff --git a/test/e2e/steps/conditions.go b/test/internal/e2e/steps/conditions.go similarity index 100% rename from test/e2e/steps/conditions.go rename to test/internal/e2e/steps/conditions.go diff --git a/test/e2e/steps/demo_steps.go b/test/internal/e2e/steps/demo_steps.go similarity index 100% rename from test/e2e/steps/demo_steps.go rename to test/internal/e2e/steps/demo_steps.go diff --git a/test/e2e/steps/group_steps.go b/test/internal/e2e/steps/group_steps.go similarity index 100% rename from test/e2e/steps/group_steps.go rename to test/internal/e2e/steps/group_steps.go diff --git a/test/e2e/steps/ha_steps.go b/test/internal/e2e/steps/ha_steps.go similarity index 100% rename from test/e2e/steps/ha_steps.go rename to test/internal/e2e/steps/ha_steps.go diff --git a/test/e2e/steps/hooks.go b/test/internal/e2e/steps/hooks.go similarity index 100% rename from test/e2e/steps/hooks.go rename to test/internal/e2e/steps/hooks.go diff --git a/test/e2e/steps/proxy_steps.go b/test/internal/e2e/steps/proxy_steps.go similarity index 100% rename from test/e2e/steps/proxy_steps.go rename to test/internal/e2e/steps/proxy_steps.go diff --git a/test/e2e/steps/steps.go b/test/internal/e2e/steps/steps.go similarity index 100% rename from test/e2e/steps/steps.go rename to test/internal/e2e/steps/steps.go diff --git a/test/e2e/steps/testdata/cluster-admin-rbac-template.yaml b/test/internal/e2e/steps/testdata/cluster-admin-rbac-template.yaml similarity index 100% rename from test/e2e/steps/testdata/cluster-admin-rbac-template.yaml rename to test/internal/e2e/steps/testdata/cluster-admin-rbac-template.yaml diff --git a/test/e2e/steps/testdata/metrics-reader-rbac-template.yaml b/test/internal/e2e/steps/testdata/metrics-reader-rbac-template.yaml similarity index 100% rename from test/e2e/steps/testdata/metrics-reader-rbac-template.yaml rename to test/internal/e2e/steps/testdata/metrics-reader-rbac-template.yaml diff --git a/test/e2e/steps/testdata/namespace-template.yaml b/test/internal/e2e/steps/testdata/namespace-template.yaml similarity index 100% rename from test/e2e/steps/testdata/namespace-template.yaml rename to test/internal/e2e/steps/testdata/namespace-template.yaml diff --git a/test/e2e/steps/testdata/pvc-probe-sa-boxcutter-rbac-template.yaml b/test/internal/e2e/steps/testdata/pvc-probe-sa-boxcutter-rbac-template.yaml similarity index 100% rename from test/e2e/steps/testdata/pvc-probe-sa-boxcutter-rbac-template.yaml rename to test/internal/e2e/steps/testdata/pvc-probe-sa-boxcutter-rbac-template.yaml diff --git a/test/e2e/steps/testdata/rbac-template-for-all-extensions.yaml b/test/internal/e2e/steps/testdata/rbac-template-for-all-extensions.yaml similarity index 100% rename from test/e2e/steps/testdata/rbac-template-for-all-extensions.yaml rename to test/internal/e2e/steps/testdata/rbac-template-for-all-extensions.yaml diff --git a/test/e2e/steps/testdata/serviceaccount-template.yaml b/test/internal/e2e/steps/testdata/serviceaccount-template.yaml similarity index 100% rename from test/e2e/steps/testdata/serviceaccount-template.yaml rename to test/internal/e2e/steps/testdata/serviceaccount-template.yaml diff --git a/test/e2e/steps/tls_steps.go b/test/internal/e2e/steps/tls_steps.go similarity index 100% rename from test/e2e/steps/tls_steps.go rename to test/internal/e2e/steps/tls_steps.go diff --git a/test/e2e/steps/upgrade_steps.go b/test/internal/e2e/steps/upgrade_steps.go similarity index 100% rename from test/e2e/steps/upgrade_steps.go rename to test/internal/e2e/steps/upgrade_steps.go diff --git a/test/internal/e2e/summary_test.go b/test/internal/e2e/summary_test.go new file mode 100644 index 0000000000..fb13abb2d6 --- /dev/null +++ b/test/internal/e2e/summary_test.go @@ -0,0 +1,40 @@ +package e2e_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/operator-framework/operator-controller/test/internal/summary" +) + +func TestSummaryOutsideSuiteDirectory(t *testing.T) { + t.Chdir(t.TempDir()) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v1/alerts": + fmt.Fprint(w, `{"status":"success","data":{"alerts":[]}}`) + case "/api/v1/query": + fmt.Fprint(w, `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{},"values":[[1,"1"]]}]}}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + t.Setenv("PROMETHEUS_URL", server.URL) + + path := filepath.Join(t.TempDir(), "summary.md") + require.NoError(t, summary.PrintSummary(path)) + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Contains(t, string(content), "# E2E Summary") + require.Contains(t, string(content), "operator-controller") + require.Contains(t, string(content), "catalogd") + require.Contains(t, string(content), "xychart-beta") +} diff --git a/test/e2e/support-bundle.yaml b/test/internal/e2e/support-bundle.yaml similarity index 100% rename from test/e2e/support-bundle.yaml rename to test/internal/e2e/support-bundle.yaml diff --git a/test/internal/summary/summary.go b/test/internal/summary/summary.go index 9e47aa3fc0..a764be4ee6 100644 --- a/test/internal/summary/summary.go +++ b/test/internal/summary/summary.go @@ -2,10 +2,10 @@ package summary import ( "context" + "embed" "fmt" "math" "os" - "path/filepath" "strings" "text/template" "time" @@ -21,6 +21,9 @@ var ( chartTemplate = "mermaid_chart.md.tmpl" ) +//go:embed templates/*.tmpl +var templates embed.FS + type summaryAlerts struct { FiringAlerts []summaryAlert PendingAlerts []summaryAlert @@ -151,11 +154,7 @@ func (s *githubSummary) Alerts() (string, error) { } func executeTemplate(templateFile string, obj any) (string, error) { - wd, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("failed to get working directory: %w", err) - } - tmpl, err := template.New(templateFile).ParseGlob(filepath.Join(wd, "../internal/summary/templates", templateFile)) + tmpl, err := template.New(templateFile).ParseFS(templates, "templates/"+templateFile) if err != nil { return "", err } diff --git a/test/upgrade-e2e/README.md b/test/upgrade-e2e/README.md index d2edf87cbe..9a9844b14c 100644 --- a/test/upgrade-e2e/README.md +++ b/test/upgrade-e2e/README.md @@ -3,14 +3,14 @@ End-to-end tests that verify OLM upgrades don't break pre-existing ClusterCatalogs and ClusterExtensions. These tests use the [Godog](https://github.com/cucumber/godog) BDD framework with shared step definitions -from `test/e2e/steps/`. The upgrade-specific steps live in `test/e2e/steps/upgrade_steps.go`. +from `test/internal/e2e/steps/`. The upgrade-specific steps live in `test/internal/e2e/steps/upgrade_steps.go`. See [test/e2e/README.md](../e2e/README.md) for more information on the Godog framework and writing tests. ## Structure ``` test/upgrade-e2e/ -├── upgrade_test.go # Test runner (reuses shared steps from test/e2e/steps/) +├── upgrade_test.go # Test runner (reuses shared steps from test/internal/e2e/steps/) └── features/ └── operator-upgrade.feature # Upgrade verification scenarios ``` diff --git a/test/upgrade-e2e/upgrade_test.go b/test/upgrade-e2e/upgrade_test.go index 5991dd81ae..857cd369ee 100644 --- a/test/upgrade-e2e/upgrade_test.go +++ b/test/upgrade-e2e/upgrade_test.go @@ -9,7 +9,7 @@ import ( "github.com/cucumber/godog/colors" "github.com/spf13/pflag" - "github.com/operator-framework/operator-controller/test/e2e/steps" + "github.com/operator-framework/operator-controller/test/internal/e2e/steps" ) var opts = godog.Options{