From 0f3fcbf1992e47ec02d2608c69b5f1de6f0fae98 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Wed, 19 Aug 2026 16:14:34 +0800 Subject: [PATCH 1/4] test: make the Gateway API conformance report honest and reproducible (backport apache/apisix-ingress-controller#2843) Signed-off-by: AlinsRan (cherry picked from commit 4d0015c0da41e8c37b0ceff0f1d76fd4ee1e0d69) --- .github/workflows/apisix-conformance-test.yml | 94 +++++++++++-- .gitignore | 2 +- Makefile | 82 +++++++++++- test/conformance/README.md | 123 ++++++++++++++++++ test/conformance/api7ee/conformance_test.go | 10 +- test/conformance/conformance_test.go | 10 +- test/e2e/framework/apisix_consts.go | 4 + test/e2e/framework/ingress.go | 5 + test/e2e/framework/manifests/apisix.yaml | 4 +- test/e2e/framework/manifests/ingress.yaml | 4 +- test/e2e/scaffold/apisix_deployer.go | 3 + 11 files changed, 304 insertions(+), 37 deletions(-) create mode 100644 test/conformance/README.md diff --git a/.github/workflows/apisix-conformance-test.yml b/.github/workflows/apisix-conformance-test.yml index dbff2fba8..907c152c8 100644 --- a/.github/workflows/apisix-conformance-test.yml +++ b/.github/workflows/apisix-conformance-test.yml @@ -24,6 +24,16 @@ on: pull_request: branches: - master + # The report submitted to the Gateway API conformance reports repository is + # produced here, with image_tag naming the release to test. It cannot key off + # the tag push itself: push-docker.yaml publishes the images on that same + # event, so they do not exist yet when this would start pulling them. + workflow_dispatch: + inputs: + image_tag: + description: "Released tag to test, for example 2.2.0. Run it from that same tag, since the skip list and the manifests come from the checked-out ref. Leave empty to build and test the checked-out commit." + required: false + default: "" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -35,13 +45,18 @@ permissions: jobs: conformance-test: env: - CONFORMANCE_TEST_REPORT_OUTPUT: /tmp/apisix-ingress-controller-conformance-report.yaml + # The report name encodes the mode, so each provider writes its own file + # and the two can be submitted side by side. apisix is the controller's + # default provider, the standalone run declares a named mode. + CONFORMANCE_MODE: ${{ matrix.conformance_mode }} timeout-minutes: 60 strategy: matrix: - provider_type: - - apisix-standalone - - apisix + include: + - provider_type: apisix-standalone + conformance_mode: apisix-standalone + - provider_type: apisix + conformance_mode: default runs-on: ubuntu-latest steps: - name: Checkout @@ -64,7 +79,11 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + # A release run must exercise the published images for that release, + # otherwise the report would describe a local build while claiming the + # release version. Every other run tests what was just built. - name: Build images + if: ${{ !inputs.image_tag }} env: TAG: dev ARCH: amd64 @@ -81,48 +100,101 @@ jobs: - name: Install And Run Cloud Provider KIND run: | - go install sigs.k8s.io/cloud-provider-kind@v0.8.0 - nohup cloud-provider-kind > /tmp/kind-loadbalancer.log 2>&1 & + make kind-lb - name: Install Gateway API And CRDs run: | make install - name: Loading Docker Image to Kind Cluster + if: ${{ !inputs.image_tag }} run: | make kind-load-adc-image make kind-load-ingress-image + # A release run pulls the published images instead, and the report names + # the release rather than the commit. + - name: Resolve Released Images + if: ${{ inputs.image_tag }} + shell: bash + run: | + # The report names image_tag, but the skip list, the supported feature + # list and the manifests come from the ref this runs on, so only a run + # from that same tag is submittable. It stays a warning rather than an + # error because running from another ref is how this path gets + # rehearsed against an already published release, and the workflow a + # tag run executes is the one inside that tag, which cannot be fixed + # afterwards without a new tag. + if [ "${{ github.ref_name }}" != "${{ inputs.image_tag }}" ]; then + echo "::warning::running from ${{ github.ref_name }} while reporting ${{ inputs.image_tag }}, this report is a rehearsal and must not be submitted" + fi + echo "CONFORMANCE_IMAGE_TAG=${{ inputs.image_tag }}" >> $GITHUB_ENV + - name: Run Conformance Test + id: conformance shell: bash continue-on-error: true env: PROVIDER_TYPE: ${{ matrix.provider_type }} run: | make conformance-test - + - name: Show Conformance Report + if: always() shell: bash run: | - cat ${CONFORMANCE_TEST_REPORT_OUTPUT} + cat ./*-report.yaml || echo "no report was produced" + + # The suite runs most tests with t.Parallel, and those report their result + # after the report has been written, so a parallel failure leaves the + # report at Failed: 0 while the suite exits non-zero. The step outcome is + # the only place such a failure shows, which is why the run above is + # continue-on-error and the verdict is reached here instead. + - name: Check Conformance Result + if: always() + shell: bash + env: + STRICT: ${{ inputs.image_tag != '' }} + RUN_OUTCOME: ${{ steps.conformance.outcome }} + run: | + problem="" + if [ "${RUN_OUTCOME}" != "success" ]; then + problem="the suite exited non-zero, see the failures above" + elif ! ls ./*-report.yaml >/dev/null 2>&1; then + problem="no report was produced" + elif grep -qE '^[[:space:]]+result: failure' ./*-report.yaml; then + problem="the report contains a failing profile" + fi + if [ -n "${problem}" ]; then + echo "::error::${problem}" + grep -nE '^[[:space:]]+result:' ./*-report.yaml 2>/dev/null || true + if [ "${STRICT}" = "true" ]; then + exit 1 + fi + echo "::warning::not a release run, not failing the job" + fi - name: Get Logs from apisix-ingress-controller + if: always() shell: bash run: | kubectl logs -n apisix-conformance-test -l app=apisix-ingress-controller - name: Upload Gateway API Conformance Report + if: always() uses: actions/upload-artifact@v4 with: - name: apisix-ingress-controller-conformance-report-${{ matrix.provider_type }}.yaml - path: ${{ env.CONFORMANCE_TEST_REPORT_OUTPUT }} + # The artifact keeps the upstream report file name, so a release run + # can be submitted straight from the download without renaming it. + name: conformance-report-${{ matrix.provider_type }} + path: ./*-report.yaml - name: Format Conformance Test Report if: ${{ github.event_name == 'pull_request' }} run: | echo '# conformance test report - ${{ matrix.provider_type }} mode' > report.md echo '```yaml' >> report.md - cat ${CONFORMANCE_TEST_REPORT_OUTPUT} >> report.md + cat ./*-report.yaml >> report.md echo '```' >> report.md - name: Report Conformance Test Result to PR Comment diff --git a/.gitignore b/.gitignore index 21bcb4140..759157348 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,7 @@ go.work dist .tmp apisix-ingress-controller -apisix-ingress-controller-conformance-report.yaml +/*-report.yaml *.mdx .cursor/ diff --git a/Makefile b/Makefile index 39a1f7ae0..681b211a7 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,8 @@ IMG ?= api7/api7-ingress-controller:$(IMAGE_TAG) ENVTEST_K8S_VERSION = 1.30.0 KIND_NAME ?= apisix-ingress-cluster KIND_NODE_IMAGE ?= kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f +CLOUD_PROVIDER_KIND_VERSION ?= v0.8.0 +CLOUD_PROVIDER_KIND_PID ?= /tmp/cloud-provider-kind.pid DASHBOARD_VERSION ?= dev ADC_VERSION ?= 0.27.1 @@ -57,12 +59,43 @@ MIN_K8S_VERSION ?= 1.31.0 GO_LDFLAGS ?= "-X=$(VERSYM)=$(VERSION) -X=$(GITSHASYM)=$(GITSHA) -X=$(BUILDOSSYM)=$(OSNAME)/$(OSARCH) -X=$(MINK8SVERSYM)=$(MIN_K8S_VERSION)" # gateway-api -GATEAY_API_VERSION ?= v1.6.0 +GATEWAY_API_VERSION ?= v1.6.0 ## https://github.com/kubernetes-sigs/gateway-api/blob/v1.6.0/pkg/features/httproute.go SUPPORTED_EXTENDED_FEATURES = "HTTPRouteDestinationPortMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteRequestMirror,HTTPRouteSchemeRedirect,GatewayAddressEmpty,HTTPRouteResponseHeaderModification,GatewayPort8080,HTTPRouteHostRewrite,HTTPRouteQueryParamMatching,HTTPRoutePathRewrite,HTTPRouteBackendProtocolWebSocket,TLSRouteModeTerminate" -CONFORMANCE_TEST_REPORT_OUTPUT ?= $(DIR)/apisix-ingress-controller-conformance-report.yaml ## https://github.com/kubernetes-sigs/gateway-api/blob/v1.6.0/conformance/utils/suite/profiles.go CONFORMANCE_PROFILES ?= GATEWAY-HTTP,GATEWAY-GRPC,GATEWAY-TLS +# Report metadata, filled into the report's implementation block by the suite. +# https://github.com/kubernetes-sigs/gateway-api/blob/main/conformance/reports/README.md +CONFORMANCE_ORGANIZATION ?= api7 +CONFORMANCE_PROJECT ?= api7-ingress-controller +CONFORMANCE_URL ?= https://github.com/api7/api7-ingress-controller +CONFORMANCE_CONTACT ?= https://github.com/api7/api7-ingress-controller/issues +# The channel install-gateway-api installs from. +CONFORMANCE_CHANNEL ?= experimental +# A non-default mode must map to a specific setup and be documented in the +# report's Reproduce section. +CONFORMANCE_MODE ?= default +# The data plane a release report is produced against. apisix:dev is a floating +# tag, so a report meant to be reproducible has to name a released one. +CONFORMANCE_DATAPLANE_VERSION ?= 3.17.0-debian +# What the run deploys and what the report declares, following the checked-out +# state: a release tag pulls the published images for that release, anything +# else uses the dev images. Upstream rejects a floating name as the version, so +# a dev run declares the commit instead. +CONFORMANCE_IMAGE_TAG ?= $(shell git describe --tags --exact-match 2>/dev/null || echo dev) +override CONFORMANCE_IMAGE_TAG := $(or $(strip $(CONFORMANCE_IMAGE_TAG)),dev) +ifeq ($(CONFORMANCE_IMAGE_TAG),dev) +CONFORMANCE_VERSION ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) +CONFORMANCE_INGRESS_IMAGE ?= api7/api7-ingress-controller:dev +CONFORMANCE_ADC_IMAGE ?= ghcr.io/api7/adc:dev +CONFORMANCE_DATAPLANE_IMAGE ?= apache/apisix:dev +else +CONFORMANCE_VERSION ?= $(CONFORMANCE_IMAGE_TAG) +CONFORMANCE_INGRESS_IMAGE ?= api7/api7-ingress-controller:$(CONFORMANCE_IMAGE_TAG) +CONFORMANCE_ADC_IMAGE ?= ghcr.io/api7/adc:$(ADC_VERSION) +CONFORMANCE_DATAPLANE_IMAGE ?= apache/apisix:$(CONFORMANCE_DATAPLANE_VERSION) +endif +CONFORMANCE_TEST_REPORT_OUTPUT ?= $(DIR)/$(CONFORMANCE_CHANNEL)-$(CONFORMANCE_VERSION)-$(CONFORMANCE_MODE)-report.yaml # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -167,21 +200,49 @@ ginkgo-api7ee-e2e-test: adc install-ginkgo: @go install github.com/onsi/ginkgo/v2/ginkgo@v$(GINKGO_VERSION) +.PHONY: conformance-report-path +conformance-report-path: ## Print the path conformance-test writes the report to. + @echo $(CONFORMANCE_TEST_REPORT_OUTPUT) + .PHONY: conformance-test +conformance-test: export INGRESS_IMAGE=$(CONFORMANCE_INGRESS_IMAGE) +conformance-test: export ADC_IMAGE=$(CONFORMANCE_ADC_IMAGE) +conformance-test: export DATAPLANE_IMAGE=$(CONFORMANCE_DATAPLANE_IMAGE) conformance-test: go test -v ./test/conformance -tags conformance,experimental -timeout 60m \ --supported-features=$(SUPPORTED_EXTENDED_FEATURES) \ --conformance-profiles=$(CONFORMANCE_PROFILES) \ + --organization="$(CONFORMANCE_ORGANIZATION)" \ + --project="$(CONFORMANCE_PROJECT)" \ + --url="$(CONFORMANCE_URL)" \ + --version="$(CONFORMANCE_VERSION)" \ + --contact="$(CONFORMANCE_CONTACT)" \ + --mode="$(CONFORMANCE_MODE)" \ --report-output=$(CONFORMANCE_TEST_REPORT_OUTPUT) .PHONY: conformance-test-api7ee +conformance-test-api7ee: export INGRESS_IMAGE=$(CONFORMANCE_INGRESS_IMAGE) +conformance-test-api7ee: export ADC_IMAGE=$(CONFORMANCE_ADC_IMAGE) +conformance-test-api7ee: CONFORMANCE_MODE=api7ee conformance-test-api7ee: DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test -v ./test/conformance/api7ee -tags conformance,experimental -timeout 60m \ --supported-features=$(SUPPORTED_EXTENDED_FEATURES) \ --conformance-profiles=$(CONFORMANCE_PROFILES) \ + --organization="$(CONFORMANCE_ORGANIZATION)" \ + --project="$(CONFORMANCE_PROJECT)" \ + --url="$(CONFORMANCE_URL)" \ + --version="$(CONFORMANCE_VERSION)" \ + --contact="$(CONFORMANCE_CONTACT)" \ + --mode="$(CONFORMANCE_MODE)" \ --report-output=$(CONFORMANCE_TEST_REPORT_OUTPUT) +.PHONY: conformance-images +conformance-images: ## Print the images the conformance run deploys. + @echo $(CONFORMANCE_INGRESS_IMAGE) + @echo $(CONFORMANCE_ADC_IMAGE) + @echo $(CONFORMANCE_DATAPLANE_IMAGE) + .PHONY: lint lint: sort-import golangci-lint ## Run golangci-lint linter $(GOLANGCI_LINT) run @@ -196,6 +257,17 @@ kind-up: && kind create cluster --name $(KIND_NAME) --image $(KIND_NODE_IMAGE) \ || echo "kind cluster already exists" +.PHONY: kind-lb +kind-lb: ## Run cloud-provider-kind so LoadBalancer Services in kind get an address. + @if [ -f $(CLOUD_PROVIDER_KIND_PID) ] && kill -0 "$$(cat $(CLOUD_PROVIDER_KIND_PID))" 2>/dev/null; then \ + echo "cloud-provider-kind already running"; \ + else \ + go install sigs.k8s.io/cloud-provider-kind@$(CLOUD_PROVIDER_KIND_VERSION); \ + echo "starting cloud-provider-kind, logs in /tmp/cloud-provider-kind.log"; \ + nohup $(GOBIN)/cloud-provider-kind > /tmp/cloud-provider-kind.log 2>&1 & \ + echo $$! > $(CLOUD_PROVIDER_KIND_PID); \ + fi + .PHONY: kind-down kind-down: @kind get clusters 2>&1 | grep $(KIND_NAME) \ @@ -332,11 +404,11 @@ endif install-gateway-api: ## Install Gateway API CRDs into the K8s cluster specified in ~/.kube/config. # Server-side apply: the v1.6 CRDs exceed the 262144-byte annotation limit of # client-side apply (last-applied-configuration). - kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/gateway-api/releases/download/$(GATEAY_API_VERSION)/experimental-install.yaml + kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/gateway-api/releases/download/$(GATEWAY_API_VERSION)/experimental-install.yaml .PHONY: uninstall-gateway-api uninstall-gateway-api: ## Uninstall Gateway API CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/$(GATEAY_API_VERSION)/experimental-install.yaml + kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/$(GATEWAY_API_VERSION)/experimental-install.yaml .PHONY: install install: manifests kustomize install-gateway-api ## Install CRDs into the K8s cluster specified in ~/.kube/config. @@ -447,7 +519,7 @@ endef helm-build-crds: @echo "build gateway-api standard crds" - $(KUSTOMIZE) build github.com/kubernetes-sigs/gateway-api/config/crd\?ref=${GATEAY_API_VERSION} > charts/crds/gwapi-crds.yaml + $(KUSTOMIZE) build github.com/kubernetes-sigs/gateway-api/config/crd\?ref=${GATEWAY_API_VERSION} > charts/crds/gwapi-crds.yaml @echo "build apisix ic crds" $(KUSTOMIZE) build config/crd > charts/crds/apisixic-crds.yaml diff --git a/test/conformance/README.md b/test/conformance/README.md new file mode 100644 index 000000000..f590dede7 --- /dev/null +++ b/test/conformance/README.md @@ -0,0 +1,123 @@ + + +# Gateway API conformance + +This directory holds the [Gateway API conformance](https://gateway-api.sigs.k8s.io/docs/concepts/conformance/) +suite configuration. Running it produces a report that can be submitted to the +[Gateway API conformance reports](https://github.com/kubernetes-sigs/gateway-api/tree/main/conformance/reports) +repository, which requires the run to be reproducible by a third party. + +## Prerequisites + +The following binaries are assumed to be installed: + +- [docker](https://docs.docker.com/get-started/get-docker/) +- [kubectl](https://kubernetes.io/docs/tasks/tools/) +- [kind](https://github.com/kubernetes-sigs/kind) +- [go](https://go.dev/learn/) + +Tested on Linux. Any cluster works as long as it supports LoadBalancer +Services: the suite reaches the gateway through the data plane Service's +external address, and the controller publishes that address in every Gateway's +`status.addresses`. Steps 2 and 3 below only exist to give a local kind cluster +that capability. + +## Reproduce + +1. Clone the repository and check out the release to reproduce + + ```shell + git clone https://github.com/api7/api7-ingress-controller.git && cd api7-ingress-controller + git checkout + ``` + + Checking out the release tag is what selects the published images for it and + makes the report name that release, so the steps below are available from the + first release that carries them. `make conformance-images` prints the three + images the run will deploy; do not run `make build-image` here, it would + replace a published image locally with a build of your own. + +2. Create the cluster + + ```shell + make kind-up + ``` + +3. Run a local LoadBalancer provider + + ```shell + make kind-lb + ``` + + This runs [cloud-provider-kind](https://kind.sigs.k8s.io/docs/user/loadbalancer) + in the background, logging to `/tmp/cloud-provider-kind.log`. It needs access + to the Docker socket. Skip this step on a cluster that already has + LoadBalancer support. + +4. Install the Gateway API and the controller's CRDs + + ```shell + make install + ``` + + On a commit that is not a release, the run uses the `dev` images and the + report names the commit. Build and load them into the cluster first: + + ```shell + make build-image && make kind-load-images + ``` + +5. Run the suite + + ```shell + make conformance-test + ``` + + For the standalone data plane mode, run it with the provider type and the + matching report mode: + + ```shell + PROVIDER_TYPE=apisix-standalone make conformance-test CONFORMANCE_MODE=apisix-standalone + ``` + + The API7 EE data plane has its own target, which already declares the + `api7ee` mode: + + ```shell + make conformance-test-api7ee + ``` + +6. Read the report + + ```shell + cat "$(make -s conformance-report-path)" + ``` + + Pass the same `CONFORMANCE_MODE` to get the path of a standalone run. + + The file is named `---report.yaml`, which is the name + the upstream repository expects, so it can be submitted as produced. Reports + must be uploaded unmodified. + + A run that fails before writing the report leaves the previous one in place, + so delete `*-report.yaml` before rerunning. Note also that most of the suite + runs in parallel and those tests report after the report is written, so a + report can read `Failed: 0` while `make conformance-test` exited non-zero; + the exit status is what to trust. diff --git a/test/conformance/api7ee/conformance_test.go b/test/conformance/api7ee/conformance_test.go index a9572a9f2..deebb733f 100644 --- a/test/conformance/api7ee/conformance_test.go +++ b/test/conformance/api7ee/conformance_test.go @@ -21,7 +21,6 @@ import ( "testing" "sigs.k8s.io/gateway-api/conformance" - conformancev1 "sigs.k8s.io/gateway-api/conformance/apis/v1" "sigs.k8s.io/gateway-api/conformance/tests" ) @@ -39,13 +38,8 @@ func TestGatewayAPIConformance(t *testing.T) { opts.CleanupBaseResources = true opts.GatewayClassName = gatewayClassName opts.SkipTests = append(opts.SkipTests, skippedTestsForSSL...) - opts.Implementation = conformancev1.Implementation{ - Organization: "APISIX", - Project: "apisix-ingress-controller", - URL: "https://github.com/apache/apisix-ingress-controller.git", - Version: "v2.0.0", - Contact: []string{"https://github.com/apache/apisix-ingress-controller/issues"}, - } + // Implementation is left to the flags DefaultOptions already applied. + // Assigning it here would override them and pin the report to a stale version. conformance.RunConformanceWithOptions(t, opts) } diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index cf606fc60..b1a360355 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -21,7 +21,6 @@ import ( "testing" "sigs.k8s.io/gateway-api/conformance" - conformancev1 "sigs.k8s.io/gateway-api/conformance/apis/v1" "sigs.k8s.io/gateway-api/conformance/tests" ) @@ -79,13 +78,8 @@ func TestGatewayAPIConformance(t *testing.T) { opts.SkipTests = append(opts.SkipTests, skippedTestsForSSL...) opts.SkipTests = append(opts.SkipTests, skippedTestsForTLSPassthrough...) opts.SkipTests = append(opts.SkipTests, skippedTestsForKnownGaps...) - opts.Implementation = conformancev1.Implementation{ - Organization: "APISIX", - Project: "apisix-ingress-controller", - URL: "https://github.com/apache/apisix-ingress-controller.git", - Version: "v2.0.0", - Contact: []string{"https://github.com/apache/apisix-ingress-controller/issues"}, - } + // Implementation is left to the flags DefaultOptions already applied. + // Assigning it here would override them and pin the report to a stale version. conformance.RunConformanceWithOptions(t, opts) } diff --git a/test/e2e/framework/apisix_consts.go b/test/e2e/framework/apisix_consts.go index d2fdaa3d7..5b3197860 100644 --- a/test/e2e/framework/apisix_consts.go +++ b/test/e2e/framework/apisix_consts.go @@ -28,6 +28,10 @@ import ( var ( ProviderType = cmp.Or(os.Getenv("PROVIDER_TYPE"), ProviderTypeAPISIXStandalone) + + IngressImage = cmp.Or(os.Getenv("INGRESS_IMAGE"), "api7/api7-ingress-controller:dev") + ADCImage = cmp.Or(os.Getenv("ADC_IMAGE"), "ghcr.io/api7/adc:dev") + DataplaneImage = cmp.Or(os.Getenv("DATAPLANE_IMAGE"), "apache/apisix:dev") ) const ( diff --git a/test/e2e/framework/ingress.go b/test/e2e/framework/ingress.go index 9c2e71998..36a503ecb 100644 --- a/test/e2e/framework/ingress.go +++ b/test/e2e/framework/ingress.go @@ -55,6 +55,11 @@ type IngressDeployOpts struct { WebhookPort int } +// Methods rather than fields, so a caller that executes the template directly +// cannot end up rendering an empty image. +func (IngressDeployOpts) ControllerImage() string { return IngressImage } +func (IngressDeployOpts) ADCImage() string { return ADCImage } + func (f *Framework) DeployIngress(opts IngressDeployOpts) { buf := bytes.NewBuffer(nil) diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index 302891948..55a334b40 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -78,7 +78,7 @@ spec: spec: initContainers: - name: config-setup - image: apache/apisix:dev + image: {{ .Image }} command: - sh - -c @@ -96,7 +96,7 @@ spec: mountPath: /tmp/apisix-conf containers: - name: apisix - image: apache/apisix:dev + image: {{ .Image }} ports: - name: http containerPort: 9080 diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index a4e244a4c..7b0bef18f 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -358,7 +358,7 @@ spec: securityContext: fsGroup: 2000 containers: - - image: api7/api7-ingress-controller:dev + - image: {{ .ControllerImage }} env: - name: POD_NAMESPACE valueFrom: @@ -401,7 +401,7 @@ spec: port: 8081 initialDelaySeconds: 5 periodSeconds: 10 - - image: ghcr.io/api7/adc:dev + - image: {{ .ADCImage }} env: - name: ADC_RUNNING_MODE value: ingress diff --git a/test/e2e/scaffold/apisix_deployer.go b/test/e2e/scaffold/apisix_deployer.go index aa6d8fad3..d6736da45 100644 --- a/test/e2e/scaffold/apisix_deployer.go +++ b/test/e2e/scaffold/apisix_deployer.go @@ -48,6 +48,9 @@ type APISIXDeployOptions struct { Replicas *int } +// See IngressDeployOpts.ControllerImage. +func (APISIXDeployOptions) Image() string { return framework.DataplaneImage } + type APISIXDeployer struct { *Scaffold adminTunnel *k8s.Tunnel From 81eb78e869c48ab2a5ebdb5f8b80a3e9aded49da Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Thu, 20 Aug 2026 15:02:04 +0800 Subject: [PATCH 2/4] test(conformance): adapt the api7ee suite so its run is a real verdict Signed-off-by: AlinsRan --- .github/workflows/conformance-test.yml | 45 ++++++++++++++++----- test/conformance/api7ee/conformance_test.go | 34 +++++++++++++++- 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/.github/workflows/conformance-test.yml b/.github/workflows/conformance-test.yml index fe890388b..768993fa5 100644 --- a/.github/workflows/conformance-test.yml +++ b/.github/workflows/conformance-test.yml @@ -31,8 +31,6 @@ concurrency: jobs: conformance-test: - env: - CONFORMANCE_TEST_REPORT_OUTPUT: /tmp/api7-ingress-controller-conformance-report.yaml timeout-minutes: 60 runs-on: ubuntu-latest permissions: @@ -82,8 +80,7 @@ jobs: - name: Install And Run Cloud Provider KIND run: | - go install sigs.k8s.io/cloud-provider-kind@v0.8.0 - nohup cloud-provider-kind > /tmp/kind-loadbalancer.log 2>&1 & + make kind-lb - name: Install Gateway API And CRDs run: | @@ -98,6 +95,7 @@ jobs: make download-api7ee3-chart - name: Run Conformance Test + id: conformance shell: bash env: API7_EE_LICENSE: ${{ secrets.API7_EE_LICENSE }} @@ -106,28 +104,57 @@ jobs: make conformance-test-api7ee - name: Show Conformance Report + if: always() shell: bash run: | - cat ${CONFORMANCE_TEST_REPORT_OUTPUT} + cat ./*-report.yaml || echo "no report was produced" + + # The suite runs most tests with t.Parallel, and those report their result + # after the report has been written, so a parallel failure leaves the + # report at Failed: 0 while the suite exits non-zero. The step outcome is + # the only place such a failure shows, which is why the run above is + # continue-on-error and the verdict is reached here instead. + - name: Check Conformance Result + if: always() + shell: bash + env: + RUN_OUTCOME: ${{ steps.conformance.outcome }} + run: | + problem="" + if [ "${RUN_OUTCOME}" != "success" ]; then + problem="the suite exited non-zero, see the failures above" + elif ! ls ./*-report.yaml >/dev/null 2>&1; then + problem="no report was produced" + elif grep -qE '^[[:space:]]+result: failure' ./*-report.yaml; then + problem="the report contains a failing profile" + fi + if [ -n "${problem}" ]; then + echo "::error::${problem}" + grep -nE '^[[:space:]]+result:' ./*-report.yaml 2>/dev/null || true + exit 1 + fi - name: Get Logs from api7-ingress-controller + if: always() shell: bash run: | kubectl logs -n apisix-conformance-test -l app=apisix-ingress-controller - name: Upload Gateway API Conformance Report - if: ${{ github.event_name == 'push' }} + if: always() uses: actions/upload-artifact@v4 with: - name: api7-ingress-controller-conformance-report.yaml - path: ${{ env.CONFORMANCE_TEST_REPORT_OUTPUT }} + # The artifact keeps the upstream report file name, so it can be read + # straight from the download without renaming it. + name: conformance-report-api7ee + path: ./*-report.yaml - name: Format Conformance Test Report if: ${{ github.event_name == 'pull_request' }} run: | echo '# conformance test report' > report.md echo '```yaml' >> report.md - cat ${CONFORMANCE_TEST_REPORT_OUTPUT} >> report.md + cat ./*-report.yaml >> report.md echo '```' >> report.md - name: Report Conformance Test Result to PR Comment diff --git a/test/conformance/api7ee/conformance_test.go b/test/conformance/api7ee/conformance_test.go index deebb733f..fcbab972e 100644 --- a/test/conformance/api7ee/conformance_test.go +++ b/test/conformance/api7ee/conformance_test.go @@ -30,7 +30,37 @@ var skippedTestsForSSL = []string{ tests.HTTPRouteRedirectPortAndScheme.ShortName, } -// TODO: HTTPRoute hostname intersection and listener hostname matching +// These fixtures create a Gateway of their own, and that Gateway carries no +// GatewayProxy, so it never leaves "gateway proxy not found" and no traffic +// flows. TLSRoute itself is exercised by the e2e suite against this provider; +// what is missing here is a way to attach the proxy to a Gateway the test +// creates, not the feature. +var skippedTestsForStandaloneGateway = []string{ + tests.TLSRouteSimpleSameNamespace.ShortName, + tests.TLSRouteHostnameIntersection.ShortName, + tests.TLSRouteInvalidBackendRefNonexistent.ShortName, + tests.TLSRouteInvalidBackendRefUnknownKind.ShortName, + tests.TLSRouteTerminateSimpleSameNamespace.ShortName, +} + +// Known gaps tracked for follow-up. These are genuine gaps rather than +// architectural limits, so they are expected to shrink over time. +var skippedTestsForKnownGaps = []string{ + // The control plane rejects a second SSL that claims an SNI another one + // already holds ("responded with status 400 Bad Request, error_msg: SNI + // already exists"), which APISIX accepts. Adding the HTTPS listener this + // test asks for therefore fails the sync and the Gateway never reaches + // Accepted. The controller has to reconcile SSLs by SNI for this provider. + tests.GatewayModifyListeners.ShortName, + + // A single HTTPRoute attached to several Gateways is not served from each + // parent independently. + tests.HTTPRouteMultipleGateways.ShortName, + + // A backendRef that cannot be resolved must still produce a route that + // answers 500; today no route is generated at all, so the request 404s. + tests.HTTPRouteNoBackendRefs.ShortName, +} func TestGatewayAPIConformance(t *testing.T) { opts := conformance.DefaultOptions(t) @@ -38,6 +68,8 @@ func TestGatewayAPIConformance(t *testing.T) { opts.CleanupBaseResources = true opts.GatewayClassName = gatewayClassName opts.SkipTests = append(opts.SkipTests, skippedTestsForSSL...) + opts.SkipTests = append(opts.SkipTests, skippedTestsForStandaloneGateway...) + opts.SkipTests = append(opts.SkipTests, skippedTestsForKnownGaps...) // Implementation is left to the flags DefaultOptions already applied. // Assigning it here would override them and pin the report to a stale version. From 851efd48e4de34ce64fe132a1de9b2e2c25d7def Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Thu, 20 Aug 2026 15:28:49 +0800 Subject: [PATCH 3/4] test(conformance): address review on the conformance workflows and Makefile Signed-off-by: AlinsRan --- .github/workflows/apisix-conformance-test.yml | 24 +++++++++++++++---- .github/workflows/conformance-test.yml | 7 +++++- Makefile | 14 +++++++++-- test/conformance/README.md | 3 +++ test/e2e/framework/api7_consts.go | 2 ++ test/e2e/framework/api7_gateway.go | 3 +++ test/e2e/framework/manifests/dp.yaml | 2 +- 7 files changed, 47 insertions(+), 8 deletions(-) diff --git a/.github/workflows/apisix-conformance-test.yml b/.github/workflows/apisix-conformance-test.yml index 907c152c8..fcd7f7bf9 100644 --- a/.github/workflows/apisix-conformance-test.yml +++ b/.github/workflows/apisix-conformance-test.yml @@ -117,7 +117,17 @@ jobs: - name: Resolve Released Images if: ${{ inputs.image_tag }} shell: bash + env: + # A dispatch input is text someone typed, so it is read from the + # environment rather than expanded into the script, and checked + # against the tag grammar before it is used or written anywhere. + IMAGE_TAG: ${{ inputs.image_tag }} + REF_NAME: ${{ github.ref_name }} run: | + if ! [[ "${IMAGE_TAG}" =~ ^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$ ]]; then + echo "::error::image_tag is not a valid image tag: ${IMAGE_TAG}" + exit 1 + fi # The report names image_tag, but the skip list, the supported feature # list and the manifests come from the ref this runs on, so only a run # from that same tag is submittable. It stays a warning rather than an @@ -125,10 +135,10 @@ jobs: # rehearsed against an already published release, and the workflow a # tag run executes is the one inside that tag, which cannot be fixed # afterwards without a new tag. - if [ "${{ github.ref_name }}" != "${{ inputs.image_tag }}" ]; then - echo "::warning::running from ${{ github.ref_name }} while reporting ${{ inputs.image_tag }}, this report is a rehearsal and must not be submitted" + if [ "${REF_NAME}" != "${IMAGE_TAG}" ]; then + echo "::warning::running from ${REF_NAME} while reporting ${IMAGE_TAG}, this report is a rehearsal and must not be submitted" fi - echo "CONFORMANCE_IMAGE_TAG=${{ inputs.image_tag }}" >> $GITHUB_ENV + printf 'CONFORMANCE_IMAGE_TAG=%s\n' "${IMAGE_TAG}" >> "$GITHUB_ENV" - name: Run Conformance Test id: conformance @@ -194,7 +204,13 @@ jobs: run: | echo '# conformance test report - ${{ matrix.provider_type }} mode' > report.md echo '```yaml' >> report.md - cat ./*-report.yaml >> report.md + # An unmatched glob would make cat fail and take the job with it, + # which would turn a warned-about missing report into a hard failure. + if ls ./*-report.yaml >/dev/null 2>&1; then + cat ./*-report.yaml >> report.md + else + echo 'no report was produced' >> report.md + fi echo '```' >> report.md - name: Report Conformance Test Result to PR Comment diff --git a/.github/workflows/conformance-test.yml b/.github/workflows/conformance-test.yml index 768993fa5..59f3f49c3 100644 --- a/.github/workflows/conformance-test.yml +++ b/.github/workflows/conformance-test.yml @@ -154,7 +154,12 @@ jobs: run: | echo '# conformance test report' > report.md echo '```yaml' >> report.md - cat ./*-report.yaml >> report.md + # An unmatched glob would make cat fail and take the job with it. + if ls ./*-report.yaml >/dev/null 2>&1; then + cat ./*-report.yaml >> report.md + else + echo 'no report was produced' >> report.md + fi echo '```' >> report.md - name: Report Conformance Test Result to PR Comment diff --git a/Makefile b/Makefile index 681b211a7..7880e7775 100644 --- a/Makefile +++ b/Makefile @@ -95,6 +95,9 @@ CONFORMANCE_INGRESS_IMAGE ?= api7/api7-ingress-controller:$(CONFORMANCE_IMAGE_TA CONFORMANCE_ADC_IMAGE ?= ghcr.io/api7/adc:$(ADC_VERSION) CONFORMANCE_DATAPLANE_IMAGE ?= apache/apisix:$(CONFORMANCE_DATAPLANE_VERSION) endif +# The API7EE run deploys its own data plane, not apache/apisix, so it needs its +# own declaration for the report to name what actually served the traffic. +CONFORMANCE_API7EE_DATAPLANE_IMAGE ?= ghcr.io/api7/api7-ee-3-gateway:$(DASHBOARD_VERSION) CONFORMANCE_TEST_REPORT_OUTPUT ?= $(DIR)/$(CONFORMANCE_CHANNEL)-$(CONFORMANCE_VERSION)-$(CONFORMANCE_MODE)-report.yaml # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) @@ -224,7 +227,10 @@ conformance-test: .PHONY: conformance-test-api7ee conformance-test-api7ee: export INGRESS_IMAGE=$(CONFORMANCE_INGRESS_IMAGE) conformance-test-api7ee: export ADC_IMAGE=$(CONFORMANCE_ADC_IMAGE) -conformance-test-api7ee: CONFORMANCE_MODE=api7ee +conformance-test-api7ee: export API7EE_DATAPLANE_IMAGE=$(CONFORMANCE_API7EE_DATAPLANE_IMAGE) +# override, so a command-line CONFORMANCE_MODE cannot label an API7EE run +# as some other mode in the report it writes. +conformance-test-api7ee: override CONFORMANCE_MODE := api7ee conformance-test-api7ee: DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test -v ./test/conformance/api7ee -tags conformance,experimental -timeout 60m \ --supported-features=$(SUPPORTED_EXTENDED_FEATURES) \ @@ -242,6 +248,8 @@ conformance-images: ## Print the images the conformance run deploys. @echo $(CONFORMANCE_INGRESS_IMAGE) @echo $(CONFORMANCE_ADC_IMAGE) @echo $(CONFORMANCE_DATAPLANE_IMAGE) + # the api7ee mode deploys this one instead of the APISIX data plane + @echo $(CONFORMANCE_API7EE_DATAPLANE_IMAGE) .PHONY: lint lint: sort-import golangci-lint ## Run golangci-lint linter @@ -259,9 +267,11 @@ kind-up: .PHONY: kind-lb kind-lb: ## Run cloud-provider-kind so LoadBalancer Services in kind get an address. - @if [ -f $(CLOUD_PROVIDER_KIND_PID) ] && kill -0 "$$(cat $(CLOUD_PROVIDER_KIND_PID))" 2>/dev/null; then \ + @pid=$$(cat $(CLOUD_PROVIDER_KIND_PID) 2>/dev/null); \ + if [ -n "$$pid" ] && ps -p $$pid -o args= 2>/dev/null | grep -q cloud-provider-kind; then \ echo "cloud-provider-kind already running"; \ else \ + rm -f $(CLOUD_PROVIDER_KIND_PID); \ go install sigs.k8s.io/cloud-provider-kind@$(CLOUD_PROVIDER_KIND_VERSION); \ echo "starting cloud-provider-kind, logs in /tmp/cloud-provider-kind.log"; \ nohup $(GOBIN)/cloud-provider-kind > /tmp/cloud-provider-kind.log 2>&1 & \ diff --git a/test/conformance/README.md b/test/conformance/README.md index f590dede7..eeb312269 100644 --- a/test/conformance/README.md +++ b/test/conformance/README.md @@ -104,6 +104,9 @@ that capability. make conformance-test-api7ee ``` + That run deploys `CONFORMANCE_API7EE_DATAPLANE_IMAGE` rather than the APISIX + data plane, which is the last line `make conformance-images` prints. + 6. Read the report ```shell diff --git a/test/e2e/framework/api7_consts.go b/test/e2e/framework/api7_consts.go index e18116e61..2bd5dae51 100644 --- a/test/e2e/framework/api7_consts.go +++ b/test/e2e/framework/api7_consts.go @@ -44,4 +44,6 @@ const ( var ( IngressVersion = cmp.Or(os.Getenv("INGRESS_VERSION"), "v1") + + API7EEDataplaneImage = cmp.Or(os.Getenv("API7EE_DATAPLANE_IMAGE"), "ghcr.io/api7/api7-ee-3-gateway:dev") ) diff --git a/test/e2e/framework/api7_gateway.go b/test/e2e/framework/api7_gateway.go index fc5ccdb12..fbdd0b0de 100644 --- a/test/e2e/framework/api7_gateway.go +++ b/test/e2e/framework/api7_gateway.go @@ -62,6 +62,9 @@ type API7DeployOptions struct { Replicas *int } +// See IngressDeployOpts.ControllerImage. +func (API7DeployOptions) Image() string { return API7EEDataplaneImage } + func (f *Framework) DeployGateway(opts *API7DeployOptions) *corev1.Service { if opts.ServiceName == "" { opts.ServiceName = "api7ee3-apisix-gateway-mtls" diff --git a/test/e2e/framework/manifests/dp.yaml b/test/e2e/framework/manifests/dp.yaml index 7608c9de6..63a541141 100644 --- a/test/e2e/framework/manifests/dp.yaml +++ b/test/e2e/framework/manifests/dp.yaml @@ -181,7 +181,7 @@ spec: spec: #serviceAccountName: ginkgo containers: - - image: ghcr.io/api7/api7-ee-3-gateway:dev + - image: {{ .Image }} imagePullPolicy: IfNotPresent env: {{- if not .TLSEnabled }} From c3f1f8c5eaacc8fc015458101f73a85a11c80aa0 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Thu, 20 Aug 2026 17:08:27 +0800 Subject: [PATCH 4/4] test(conformance): fix kind-lb aborting under .SHELLFLAGS -e Signed-off-by: AlinsRan --- .github/workflows/apisix-conformance-test.yml | 4 +++- .github/workflows/conformance-test.yml | 4 +++- Makefile | 7 +++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/apisix-conformance-test.yml b/.github/workflows/apisix-conformance-test.yml index fcd7f7bf9..1d934b42a 100644 --- a/.github/workflows/apisix-conformance-test.yml +++ b/.github/workflows/apisix-conformance-test.yml @@ -168,7 +168,9 @@ jobs: RUN_OUTCOME: ${{ steps.conformance.outcome }} run: | problem="" - if [ "${RUN_OUTCOME}" != "success" ]; then + if [ "${RUN_OUTCOME}" = "skipped" ]; then + problem="the suite never ran, an earlier step failed" + elif [ "${RUN_OUTCOME}" != "success" ]; then problem="the suite exited non-zero, see the failures above" elif ! ls ./*-report.yaml >/dev/null 2>&1; then problem="no report was produced" diff --git a/.github/workflows/conformance-test.yml b/.github/workflows/conformance-test.yml index 59f3f49c3..b429405c6 100644 --- a/.github/workflows/conformance-test.yml +++ b/.github/workflows/conformance-test.yml @@ -121,7 +121,9 @@ jobs: RUN_OUTCOME: ${{ steps.conformance.outcome }} run: | problem="" - if [ "${RUN_OUTCOME}" != "success" ]; then + if [ "${RUN_OUTCOME}" = "skipped" ]; then + problem="the suite never ran, an earlier step failed" + elif [ "${RUN_OUTCOME}" != "success" ]; then problem="the suite exited non-zero, see the failures above" elif ! ls ./*-report.yaml >/dev/null 2>&1; then problem="no report was produced" diff --git a/Makefile b/Makefile index 7880e7775..14243a329 100644 --- a/Makefile +++ b/Makefile @@ -243,12 +243,13 @@ conformance-test-api7ee: --mode="$(CONFORMANCE_MODE)" \ --report-output=$(CONFORMANCE_TEST_REPORT_OUTPUT) +# The last line is the API7 EE data plane, which the api7ee mode deploys instead +# of the APISIX one. .PHONY: conformance-images conformance-images: ## Print the images the conformance run deploys. @echo $(CONFORMANCE_INGRESS_IMAGE) @echo $(CONFORMANCE_ADC_IMAGE) @echo $(CONFORMANCE_DATAPLANE_IMAGE) - # the api7ee mode deploys this one instead of the APISIX data plane @echo $(CONFORMANCE_API7EE_DATAPLANE_IMAGE) .PHONY: lint @@ -267,7 +268,9 @@ kind-up: .PHONY: kind-lb kind-lb: ## Run cloud-provider-kind so LoadBalancer Services in kind get an address. - @pid=$$(cat $(CLOUD_PROVIDER_KIND_PID) 2>/dev/null); \ + @# `|| true`: .SHELLFLAGS carries -e, and cat exits non-zero when the pid + @# file does not exist yet, which would abort the recipe before the check. + @pid=$$(cat $(CLOUD_PROVIDER_KIND_PID) 2>/dev/null || true); \ if [ -n "$$pid" ] && ps -p $$pid -o args= 2>/dev/null | grep -q cloud-provider-kind; then \ echo "cloud-provider-kind already running"; \ else \