From 8dfeaa6a92819ec981347c9a24f9364d05f4bf74 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Mon, 31 Aug 2026 21:34:48 -0400 Subject: [PATCH 1/4] ci: make the PR gate fast and deterministic, move ct install off PRs Lint and Test Charts failed 15 of its last 17 runs. The cause was not flakiness. It was two deterministic faults. First, ct.yaml was never read. chart-testing resolves its config from CT_CONFIG_DIR inside the tool cache, not from the repository root, and the workflow never passed --config. The helm-extra-args timeout of 600s was silently dropped, helm fell back to its 300s default, and every run died at ~4m58s with "client rate limiter Wait returned an error: context deadline exceeded". ct-arm-64.yaml was dead for the same reason. Second, ct install could not converge regardless of the timeout. model-catalog failed both its readiness and liveness probes with HTTP 503 continuously, and hasura-auth-webhook refused connections on port 3000. The full stack does not come up on a single-node kind cluster on a 2-vCPU runner without real backing services. The PR gate is now ct lint plus a helm template and kubeconform matrix over four value permutations. It runs in about a minute and every failure is a real chart fault. kubeconform is the part helm lint cannot do: it validates the rendered manifests against the Kubernetes schemas and catches duplicate keys and bad apiVersions. ct install moves to chart-install.yaml on a nightly schedule and on demand, where it can report without blocking a merge. Also passes --config ct.yaml so ct.yaml applies, and bumps checkout to v4, setup-helm to v4.3.0, chart-testing-action to v2.7.0 and helm to v3.16.4, which clears the Node 20 deprecation warnings. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Gb5UJ1nbafcMMFbEoZfB41 --- .github/workflows/chart-install.yaml | 48 ++++++++++++++++++ .github/workflows/linter.yaml | 76 +++++++++++++++++++++++----- 2 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/chart-install.yaml diff --git a/.github/workflows/chart-install.yaml b/.github/workflows/chart-install.yaml new file mode 100644 index 0000000..04842fd --- /dev/null +++ b/.github/workflows/chart-install.yaml @@ -0,0 +1,48 @@ +name: Install Charts (kind) + +# Deliberately not on pull_request. Installing the full MINT stack on a +# single-node kind cluster on a 2-vCPU runner takes ~7 minutes and depends on +# every upstream image and backing service being reachable and healthy, so it +# cannot act as a merge gate. It runs nightly and on demand instead. +on: + workflow_dispatch: + schedule: + - cron: "0 7 * * *" + +concurrency: + group: chart-install + cancel-in-progress: false + +jobs: + install: + name: ct install + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v4.3.0 + with: + version: v3.16.4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + check-latest: true + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.7.0 + + - name: Create kind cluster + uses: helm/kind-action@v1.12.0 + + # --config is mandatory, see linter.yaml. Without it the 600s + # helm-extra-args timeout in ct.yaml is dropped and helm falls back to + # 300s, which is not enough for this stack. + # --all because a scheduled run has no PR diff for list-changed. + - name: Run chart-testing (install) + run: ct install --config ct.yaml --all diff --git a/.github/workflows/linter.yaml b/.github/workflows/linter.yaml index 957288c..1ae83f1 100644 --- a/.github/workflows/linter.yaml +++ b/.github/workflows/linter.yaml @@ -1,22 +1,27 @@ name: Lint and Test Charts -# on pull request or any branch on: pull_request: +concurrency: + group: lint-charts-${{ github.ref }} + cancel-in-progress: true + jobs: - lint-test: + lint: + name: Lint runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Helm - uses: azure/setup-helm@v4.2.0 + uses: azure/setup-helm@v4.3.0 with: - version: v3.14.4 + version: v3.16.4 - uses: actions/setup-python@v5 with: @@ -24,24 +29,67 @@ jobs: check-latest: true - name: Set up chart-testing - uses: helm/chart-testing-action@v2.6.1 + uses: helm/chart-testing-action@v2.7.0 + # --config is mandatory. chart-testing resolves its config from + # CT_CONFIG_DIR inside the tool cache, never from the repository root, so + # without this flag ct.yaml is silently ignored. - name: Run chart-testing (list-changed) id: list-changed run: | - changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }}) + changed=$(ct list-changed --config ct.yaml) if [[ -n "$changed" ]]; then echo "changed=true" >> "$GITHUB_OUTPUT" fi - name: Run chart-testing (lint) if: steps.list-changed.outputs.changed == 'true' - run: ct lint --target-branch ${{ github.event.repository.default_branch }} + run: ct lint --config ct.yaml - - name: Create kind cluster - if: steps.list-changed.outputs.changed == 'true' - uses: helm/kind-action@v1.10.0 + render: + name: Render (${{ matrix.name }}) + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + include: + - name: default + args: "" + - name: ui-react + args: "-f charts/mint/ci/ui-react-values.yaml" + - name: arm64 + args: "--set arm_support=true --set components.data_catalog.enabled=false" + # data_catalog is off by default, so this is the only permutation + # that renders datacatalog.yaml at all, and the only one that reaches + # its two arm_support branches. + - name: all-components-arm64 + args: "--set arm_support=true --set components.data_catalog.enabled=true" + steps: + - name: Checkout + uses: actions/checkout@v4 - - name: Run chart-testing (install) - if: steps.list-changed.outputs.changed == 'true' - run: ct install --target-branch ${{ github.event.repository.default_branch }} + - name: Set up Helm + uses: azure/setup-helm@v4.3.0 + with: + version: v3.16.4 + + - name: Install kubeconform + run: | + curl -sSL -o /tmp/kubeconform.tar.gz \ + https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz + tar -xzf /tmp/kubeconform.tar.gz -C /tmp kubeconform + sudo install /tmp/kubeconform /usr/local/bin/kubeconform + + # helm template proves the chart renders; kubeconform proves what it + # renders is valid Kubernetes. helm lint catches neither duplicate keys + # nor bad apiVersions. + - name: Render and validate + run: | + helm template mint charts/mint ${{ matrix.args }} > /tmp/manifests.yaml + kubeconform \ + -strict \ + -summary \ + -kubernetes-version 1.29.0 \ + -ignore-missing-schemas \ + /tmp/manifests.yaml From 9b6be688cf2fc232609c7a1db9c6e377b9647e46 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Mon, 31 Aug 2026 21:34:48 -0400 Subject: [PATCH 2/4] fix(chart): drop the duplicate resource-policy annotation on the Hasura PVC pvc-hasura.yaml hardcoded helm.sh/resource-policy: keep and values.yaml supplies the same key through persistence.annotations, so the rendered PVC carried the annotation twice. Helm resolves the duplicate last-wins, so one annotation reached the cluster and the behaviour was correct, but the manifest was not valid YAML. helm lint and helm template both pass it in silence. The new kubeconform step in CI does not. The four sibling PVC templates never hardcoded the annotation. Hasura was the only outlier. All five persistence blocks in values.yaml carry the keep annotation, so removing the hardcoded line makes Hasura consistent with the rest and leaves the rendered output unchanged. Verified: the full rendered output before and after differs by exactly the one duplicate line, in every value permutation. The keep annotation still survives both override paths, because Helm coalesces user values onto chart defaults rather than replacing the map. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Gb5UJ1nbafcMMFbEoZfB41 --- charts/mint/templates/pvc-hasura.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/mint/templates/pvc-hasura.yaml b/charts/mint/templates/pvc-hasura.yaml index 4e53ba0..e4e298c 100644 --- a/charts/mint/templates/pvc-hasura.yaml +++ b/charts/mint/templates/pvc-hasura.yaml @@ -7,7 +7,6 @@ metadata: namespace: {{ .Release.Namespace | quote }} {{- if or .Values.components.hasura_db.persistence.annotations .Values.commonAnnotations }} annotations: - helm.sh/resource-policy: keep {{- if .Values.components.hasura_db.persistence.annotations }} {{- include "common.tplvalues.render" ( dict "value" .Values.components.hasura_db.persistence.annotations "context" $ ) | nindent 4 }} {{- end }} From e002c22ffd0adff67d44713f4b7b3d111e32082e Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Mon, 31 Aug 2026 21:37:10 -0400 Subject: [PATCH 3/4] chore(chart): release 9.0.0-beta.12 The PVC annotation fix touches a template, so ct lint requires a version bump. Verified against the dynamo deployment, which pins 9.0.0-beta.11: the full rendered release differs by exactly the one duplicate line, and both PVCs it creates keep helm.sh/resource-policy: keep. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Gb5UJ1nbafcMMFbEoZfB41 --- charts/mint/Chart.yaml | 2 +- charts/mint/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mint/Chart.yaml b/charts/mint/Chart.yaml index 9fb55da..d27a67d 100644 --- a/charts/mint/Chart.yaml +++ b/charts/mint/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 9.0.0-beta.11 +version: 9.0.0-beta.12 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. diff --git a/charts/mint/README.md b/charts/mint/README.md index b6ea59e..7c34a70 100644 --- a/charts/mint/README.md +++ b/charts/mint/README.md @@ -1,6 +1,6 @@ # MINT -![Version: 9.0.0-beta.11](https://img.shields.io/badge/Version-9.0.0--beta.11-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.3](https://img.shields.io/badge/AppVersion-1.16.3-informational?style=flat-square) +![Version: 9.0.0-beta.12](https://img.shields.io/badge/Version-9.0.0--beta.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.3](https://img.shields.io/badge/AppVersion-1.16.3-informational?style=flat-square) A Helm chart for MINT From 3f27bff8348f132724b03460253f970d3e56a18b Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Mon, 31 Aug 2026 21:38:28 -0400 Subject: [PATCH 4/4] ci: pin Python to 3.11 so chart-testing's yamale keeps working ct lint crashed with "AttributeError: module 'ast' has no attribute 'Num'". chart-testing bundles yamale, yamale references ast.Num, and ast.Num was removed in Python 3.12. setup-python was configured with python-version "3.x" and check-latest true, so it floated to Python 3.14 and the workflow broke without any code change. That is the real source of the flakiness: the interpreter version was not pinned. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Gb5UJ1nbafcMMFbEoZfB41 --- .github/workflows/chart-install.yaml | 7 +++++-- .github/workflows/linter.yaml | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/chart-install.yaml b/.github/workflows/chart-install.yaml index 04842fd..a8c2fb9 100644 --- a/.github/workflows/chart-install.yaml +++ b/.github/workflows/chart-install.yaml @@ -29,10 +29,13 @@ jobs: with: version: v3.16.4 + # Pinned deliberately. "3.x" with check-latest floats to the newest + # Python, which is how this workflow broke without a code change: + # chart-testing bundles yamale, yamale references ast.Num, and ast.Num + # was removed in Python 3.12. - uses: actions/setup-python@v5 with: - python-version: "3.x" - check-latest: true + python-version: "3.11" - name: Set up chart-testing uses: helm/chart-testing-action@v2.7.0 diff --git a/.github/workflows/linter.yaml b/.github/workflows/linter.yaml index 1ae83f1..2c7241e 100644 --- a/.github/workflows/linter.yaml +++ b/.github/workflows/linter.yaml @@ -23,10 +23,13 @@ jobs: with: version: v3.16.4 + # Pinned deliberately. "3.x" with check-latest floats to the newest + # Python, which is how this workflow broke without a code change: + # chart-testing bundles yamale, yamale references ast.Num, and ast.Num + # was removed in Python 3.12. - uses: actions/setup-python@v5 with: - python-version: "3.x" - check-latest: true + python-version: "3.11" - name: Set up chart-testing uses: helm/chart-testing-action@v2.7.0