From a2f4cc7ad58ccd915400230cfb1837397a711486 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Mon, 17 Aug 2026 16:22:35 +0200 Subject: [PATCH 1/2] fix(test-insights): make the closing CTA one partial, not eleven copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every test framework recipe ends by sending the reader to the Test Insights dashboard, and each one carried its own copy of that sentence. Ten of the copies had drifted onto `https://dashboard.mergify.com/test-insights/jobs`, which is not a page — Test Insights has Prevention, Detection and Mitigation — and an eleventh named the dashboard with no link at all. #12490 has since corrected all eleven in place, so the links work today; what it could not fix is why one stale URL became ten broken links, which is that the sentence exists eleven times. So the paragraph becomes `_review-in-test-insights.mdx`, included by the recipes. The rendered text is byte-identical to what #12490 shipped; the next time that link moves it is one edit rather than eleven, and a recipe cannot quietly drift out of step with its siblings again. Shared partials are already how these files handle the Buildkite quarantine setup and the upload steps. Separately, two inline GitHub Actions examples were missing the `continue-on-error: true` that the quarantine partial they include requires: the Gradle example in the JUnit recipe and the `rake test` example in the Minitest one. Without it a failing test step ends the job before the upload step runs, so the run that most needs a report produces none — and in both files the sibling example directly above already had the line, so this was drift rather than a deliberate difference. Part of MRGFY-8720 Change-Id: I482ec81d7710f37d83dcffdf65abc029b73437fc --- .../test-frameworks/_review-in-test-insights.mdx | 3 +++ src/content/docs/test-insights/test-frameworks/cypress.mdx | 5 ++--- src/content/docs/test-insights/test-frameworks/golang.mdx | 5 ++--- src/content/docs/test-insights/test-frameworks/jest.mdx | 5 ++--- src/content/docs/test-insights/test-frameworks/junit.mdx | 6 +++--- src/content/docs/test-insights/test-frameworks/minitest.mdx | 6 +++--- src/content/docs/test-insights/test-frameworks/mstest.mdx | 5 ++--- src/content/docs/test-insights/test-frameworks/nunit.mdx | 5 ++--- src/content/docs/test-insights/test-frameworks/pest.mdx | 5 ++--- src/content/docs/test-insights/test-frameworks/phpunit.mdx | 5 ++--- src/content/docs/test-insights/test-frameworks/rust.mdx | 5 ++--- src/content/docs/test-insights/test-frameworks/testng.mdx | 5 ++--- 12 files changed, 27 insertions(+), 33 deletions(-) create mode 100644 src/content/docs/test-insights/test-frameworks/_review-in-test-insights.mdx diff --git a/src/content/docs/test-insights/test-frameworks/_review-in-test-insights.mdx b/src/content/docs/test-insights/test-frameworks/_review-in-test-insights.mdx new file mode 100644 index 0000000000..b8e4dd3a88 --- /dev/null +++ b/src/content/docs/test-insights/test-frameworks/_review-in-test-insights.mdx @@ -0,0 +1,3 @@ +You can then review your test results, including any failures or flaky tests, +directly in the [Test Insights +dashboard](https://dashboard.mergify.com/test-insights/detection). diff --git a/src/content/docs/test-insights/test-frameworks/cypress.mdx b/src/content/docs/test-insights/test-frameworks/cypress.mdx index c428781a08..08824d1649 100644 --- a/src/content/docs/test-insights/test-frameworks/cypress.mdx +++ b/src/content/docs/test-insights/test-frameworks/cypress.mdx @@ -14,6 +14,7 @@ import BuildkiteCIUploadStep from "../../../../components/BuildkiteCIUploadStep. import BuildkiteCIUploadStepMatrix from "../../../../components/BuildkiteCIUploadStepMatrix.astro" import BuildkiteCIQuarantineSetup from "./_buildkite_mergify_ci_quarantine_setup.mdx" import MergifyCliUploadStep from "../../../../components/MergifyCliUploadStep.astro" +import ReviewInTestInsights from "./_review-in-test-insights.mdx" @@ -90,9 +91,7 @@ After pushing these changes: 2. Cypress generates JUnit files. 3. The Mergify CI action uploads that file to Test Insights. -You can then review your test results, including any failures or flaky tests, -directly in the [Test Insights -dashboard](https://dashboard.mergify.com/test-insights/detection). + ## Troubleshooting Tips diff --git a/src/content/docs/test-insights/test-frameworks/golang.mdx b/src/content/docs/test-insights/test-frameworks/golang.mdx index 13e9758827..f42674081d 100644 --- a/src/content/docs/test-insights/test-frameworks/golang.mdx +++ b/src/content/docs/test-insights/test-frameworks/golang.mdx @@ -14,6 +14,7 @@ import BuildkiteCIUploadStep from "../../../../components/BuildkiteCIUploadStep. import BuildkiteCIUploadStepMatrix from "../../../../components/BuildkiteCIUploadStepMatrix.astro" import BuildkiteCIQuarantineSetup from "./_buildkite_mergify_ci_quarantine_setup.mdx" import MergifyCliUploadStep from "../../../../components/MergifyCliUploadStep.astro" +import ReviewInTestInsights from "./_review-in-test-insights.mdx" @@ -81,9 +82,7 @@ After pushing these changes: 2. A JUnit report (junit.xml) is generated. 3. The Mergify CI action uploads the report to Test Insights. -You can then review your test results, including any failures or flaky tests, -directly in the [Test Insights -dashboard](https://dashboard.mergify.com/test-insights/detection). + ## Troubleshooting Tips diff --git a/src/content/docs/test-insights/test-frameworks/jest.mdx b/src/content/docs/test-insights/test-frameworks/jest.mdx index ebefe6fb05..8a097bda60 100644 --- a/src/content/docs/test-insights/test-frameworks/jest.mdx +++ b/src/content/docs/test-insights/test-frameworks/jest.mdx @@ -14,6 +14,7 @@ import BuildkiteCIUploadStep from "../../../../components/BuildkiteCIUploadStep. import BuildkiteCIUploadStepMatrix from "../../../../components/BuildkiteCIUploadStepMatrix.astro" import BuildkiteCIQuarantineSetup from "./_buildkite_mergify_ci_quarantine_setup.mdx" import MergifyCliUploadStep from "../../../../components/MergifyCliUploadStep.astro" +import ReviewInTestInsights from "./_review-in-test-insights.mdx" @@ -105,9 +106,7 @@ After pushing these changes: 2. A JUnit report (`junit.xml`) is generated. 3. The Mergify CI action uploads the report to Test Insights. -You can then review your test results, including any failures or flaky tests, -directly in the [Test Insights -dashboard](https://dashboard.mergify.com/test-insights/detection). + ## Troubleshooting Tips diff --git a/src/content/docs/test-insights/test-frameworks/junit.mdx b/src/content/docs/test-insights/test-frameworks/junit.mdx index 5e14c87aec..8d21bb977f 100644 --- a/src/content/docs/test-insights/test-frameworks/junit.mdx +++ b/src/content/docs/test-insights/test-frameworks/junit.mdx @@ -12,6 +12,7 @@ import BuildkiteCIUploadStep from "../../../../components/BuildkiteCIUploadStep. import BuildkiteCIUploadStepMatrix from "../../../../components/BuildkiteCIUploadStepMatrix.astro" import BuildkiteCIQuarantineSetup from "./_buildkite_mergify_ci_quarantine_setup.mdx" import MergifyCliUploadStep from "../../../../components/MergifyCliUploadStep.astro" +import ReviewInTestInsights from "./_review-in-test-insights.mdx" This guide shows how to generate JUnit reports from your JUnit tests and upload them to **Test Insights** using your CI workflow. @@ -91,6 +92,7 @@ For Gradle projects: ```yaml - name: Run JUnit Tests and Generate Report id: tests + continue-on-error: true run: ./gradlew test ``` @@ -131,9 +133,7 @@ After pushing these changes: 2. JUnit XML reports are generated by Maven/Gradle. 3. The Mergify CI action uploads the reports to Test Insights. -You can then review your test results, including any failures or flaky tests, -directly in the [Test Insights -dashboard](https://dashboard.mergify.com/test-insights/detection). + ## Troubleshooting Tips diff --git a/src/content/docs/test-insights/test-frameworks/minitest.mdx b/src/content/docs/test-insights/test-frameworks/minitest.mdx index 309b9e92bf..4bb7f993b1 100644 --- a/src/content/docs/test-insights/test-frameworks/minitest.mdx +++ b/src/content/docs/test-insights/test-frameworks/minitest.mdx @@ -12,6 +12,7 @@ import BuildkiteCIUploadStep from "../../../../components/BuildkiteCIUploadStep. import BuildkiteCIUploadStepMatrix from "../../../../components/BuildkiteCIUploadStepMatrix.astro" import BuildkiteCIQuarantineSetup from "./_buildkite_mergify_ci_quarantine_setup.mdx" import MergifyCliUploadStep from "../../../../components/MergifyCliUploadStep.astro" +import ReviewInTestInsights from "./_review-in-test-insights.mdx" This guide shows how to generate JUnit reports from your Minitest tests and upload them to **Test Insights** using your CI workflow. @@ -142,6 +143,7 @@ Using `minitest-reporters` approach: ```yaml - name: Run Minitest Tests and Generate JUnit Report id: tests + continue-on-error: true run: bundle exec rake test ``` @@ -169,9 +171,7 @@ After pushing these changes: 2. A JUnit report (junit.xml) is generated. 3. The Mergify CI action uploads the report to Test Insights. -You can then review your test results, including any failures or flaky tests, -directly in the [Test Insights -dashboard](https://dashboard.mergify.com/test-insights/detection). + ## Troubleshooting Tips diff --git a/src/content/docs/test-insights/test-frameworks/mstest.mdx b/src/content/docs/test-insights/test-frameworks/mstest.mdx index 2a27db3fa7..b5035888e2 100644 --- a/src/content/docs/test-insights/test-frameworks/mstest.mdx +++ b/src/content/docs/test-insights/test-frameworks/mstest.mdx @@ -12,6 +12,7 @@ import BuildkiteCIUploadStep from "../../../../components/BuildkiteCIUploadStep. import BuildkiteCIUploadStepMatrix from "../../../../components/BuildkiteCIUploadStepMatrix.astro" import BuildkiteCIQuarantineSetup from "./_buildkite_mergify_ci_quarantine_setup.mdx" import MergifyCliUploadStep from "../../../../components/MergifyCliUploadStep.astro" +import ReviewInTestInsights from "./_review-in-test-insights.mdx" This guide shows how to generate JUnit reports from your MSTest tests and upload them to **Test Insights** using your CI workflow. @@ -115,9 +116,7 @@ After pushing these changes: 2. A JUnit report (junit.xml) is generated. 3. The Mergify CI action uploads the report to Test Insights. -You can then review your test results, including any failures or flaky tests, -directly in the [Test Insights -dashboard](https://dashboard.mergify.com/test-insights/detection). + ## Troubleshooting Tips diff --git a/src/content/docs/test-insights/test-frameworks/nunit.mdx b/src/content/docs/test-insights/test-frameworks/nunit.mdx index 1dcef6aed4..806a14b81c 100644 --- a/src/content/docs/test-insights/test-frameworks/nunit.mdx +++ b/src/content/docs/test-insights/test-frameworks/nunit.mdx @@ -12,6 +12,7 @@ import BuildkiteCIUploadStep from "../../../../components/BuildkiteCIUploadStep. import BuildkiteCIUploadStepMatrix from "../../../../components/BuildkiteCIUploadStepMatrix.astro" import BuildkiteCIQuarantineSetup from "./_buildkite_mergify_ci_quarantine_setup.mdx" import MergifyCliUploadStep from "../../../../components/MergifyCliUploadStep.astro" +import ReviewInTestInsights from "./_review-in-test-insights.mdx" This guide shows how to generate JUnit reports from your NUnit tests and upload them to **Test Insights** using your CI workflow. @@ -134,9 +135,7 @@ After pushing these changes: 2. A JUnit report (junit.xml) is generated. 3. The Mergify CI action uploads the report to Test Insights. -You can then review your test results, including any failures or flaky tests, -directly in the [Test Insights -dashboard](https://dashboard.mergify.com/test-insights/detection). + ## Troubleshooting Tips diff --git a/src/content/docs/test-insights/test-frameworks/pest.mdx b/src/content/docs/test-insights/test-frameworks/pest.mdx index a560cec2b9..cd17549b3a 100644 --- a/src/content/docs/test-insights/test-frameworks/pest.mdx +++ b/src/content/docs/test-insights/test-frameworks/pest.mdx @@ -12,6 +12,7 @@ import BuildkiteCIUploadStep from "../../../../components/BuildkiteCIUploadStep. import BuildkiteCIUploadStepMatrix from "../../../../components/BuildkiteCIUploadStepMatrix.astro" import BuildkiteCIQuarantineSetup from "./_buildkite_mergify_ci_quarantine_setup.mdx" import MergifyCliUploadStep from "../../../../components/MergifyCliUploadStep.astro" +import ReviewInTestInsights from "./_review-in-test-insights.mdx" This guide shows how to generate JUnit reports from your Pest tests and upload them to **Test Insights** using your CI workflow. @@ -126,9 +127,7 @@ After pushing these changes: 2. A JUnit report (junit.xml) is generated. 3. The Mergify CI action uploads the report to Test Insights. -You can then review your test results, including any failures or flaky tests, -directly in the [Test Insights -dashboard](https://dashboard.mergify.com/test-insights/detection). + ## Troubleshooting Tips diff --git a/src/content/docs/test-insights/test-frameworks/phpunit.mdx b/src/content/docs/test-insights/test-frameworks/phpunit.mdx index 9170f564f7..ced507f6b2 100644 --- a/src/content/docs/test-insights/test-frameworks/phpunit.mdx +++ b/src/content/docs/test-insights/test-frameworks/phpunit.mdx @@ -12,6 +12,7 @@ import BuildkiteCIUploadStep from "../../../../components/BuildkiteCIUploadStep. import BuildkiteCIUploadStepMatrix from "../../../../components/BuildkiteCIUploadStepMatrix.astro" import BuildkiteCIQuarantineSetup from "./_buildkite_mergify_ci_quarantine_setup.mdx" import MergifyCliUploadStep from "../../../../components/MergifyCliUploadStep.astro" +import ReviewInTestInsights from "./_review-in-test-insights.mdx" This guide shows how to generate JUnit reports from your PHPUnit tests and upload them to **Test Insights** using your CI workflow. @@ -113,9 +114,7 @@ After pushing these changes: 2. A JUnit report (junit.xml) is generated. 3. The Mergify CI action uploads the report to Test Insights. -You can then review your test results, including any failures or flaky tests, -directly in the [Test Insights -dashboard](https://dashboard.mergify.com/test-insights/detection). + ## Troubleshooting Tips diff --git a/src/content/docs/test-insights/test-frameworks/rust.mdx b/src/content/docs/test-insights/test-frameworks/rust.mdx index 8559caec7c..9b250ac2a0 100644 --- a/src/content/docs/test-insights/test-frameworks/rust.mdx +++ b/src/content/docs/test-insights/test-frameworks/rust.mdx @@ -14,6 +14,7 @@ import BuildkiteCIUploadStep from "../../../../components/BuildkiteCIUploadStep. import BuildkiteCIUploadStepMatrix from "../../../../components/BuildkiteCIUploadStepMatrix.astro" import BuildkiteCIQuarantineSetup from "./_buildkite_mergify_ci_quarantine_setup.mdx" import MergifyCliUploadStep from "../../../../components/MergifyCliUploadStep.astro" +import ReviewInTestInsights from "./_review-in-test-insights.mdx" @@ -132,9 +133,7 @@ After pushing these changes: 2. A JUnit report (junit.xml) is generated. 3. The Mergify CI action uploads the report to Test Insights. -You can then review your test results, including any failures or flaky tests, -directly in the [Test Insights -dashboard](https://dashboard.mergify.com/test-insights/detection). + ## Troubleshooting Tips diff --git a/src/content/docs/test-insights/test-frameworks/testng.mdx b/src/content/docs/test-insights/test-frameworks/testng.mdx index fce010db23..c43a36d7ce 100644 --- a/src/content/docs/test-insights/test-frameworks/testng.mdx +++ b/src/content/docs/test-insights/test-frameworks/testng.mdx @@ -12,6 +12,7 @@ import BuildkiteCIUploadStep from "../../../../components/BuildkiteCIUploadStep. import BuildkiteCIUploadStepMatrix from "../../../../components/BuildkiteCIUploadStepMatrix.astro" import BuildkiteCIQuarantineSetup from "./_buildkite_mergify_ci_quarantine_setup.mdx" import MergifyCliUploadStep from "../../../../components/MergifyCliUploadStep.astro" +import ReviewInTestInsights from "./_review-in-test-insights.mdx" This guide shows how to generate JUnit reports from your TestNG tests and upload them to **Test Insights** using your CI workflow. @@ -146,9 +147,7 @@ After pushing these changes: 2. JUnit-compatible XML reports are generated. 3. The Mergify CI action uploads the reports to Test Insights. -You can then review your test results, including any failures or flaky tests, -directly in the [Test Insights -dashboard](https://dashboard.mergify.com/test-insights/detection). + ## Troubleshooting Tips From f03d9dabd93e76ae52e7faf4a3ea6ca1a30e3562 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Tue, 18 Aug 2026 09:34:57 +0200 Subject: [PATCH 2/2] fix(test-insights): stop pinning the Buildkite plugin to its first release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two Buildkite upload snippets hardcode `mergifyio/mergify-ci#v1` in their `.astro` source. `v1` is the oldest of six releases — `v1` through `v6` are distinct tags, not a floating major — so every reader who copies one of these snippets pins the first version ever cut and never receives anything shipped since. Nothing else in the docs does this. Every Buildkite page writes `@@BUILDKITE_PLUGIN_VERSION@@`, which `plugins/remark-buildkite-version.ts` substitutes from `src/data/buildkite-plugin-version.json` at build time. That plugin only visits markdown nodes, so the sentinel does nothing inside an `.astro` component and these two were left pinning by hand. The fix is the one the GitHub Actions counterpart already uses: import the data file and interpolate it, exactly as `MergifyCIUploadStepMatrix.astro` does with `gha-mergify-ci-version.json`. Bumping `v1` to `v6` would have recreated the same problem one release later. This matters now because the commit at the bottom of this stack recommends the plugin's `job_name` property again. A reader who copies the matrix snippet and pins `v1` gets a plugin where that property predates the code reading it — the docs would recommend a knob that is dead in the version the snippet installs. The non-matrix component has the same pin and is the more widely used of the two: eleven test-framework recipes render it. Both are fixed here, since it is one bug with one cause. Verified on the built output: all 78 rendered occurrences of the plugin reference now read `v6`, none read `v1`, and no template literal leaked into the HTML. `pnpm check` 0 errors, `pnpm build` 384 pages, `pnpm test` 144 passed, `pnpm check:internal-leaks` clean. Refs MRGFY-8720 Change-Id: Ic5504d661f0c14e668fbb7b8cf55cf86b9dd5788 --- src/components/BuildkiteCIUploadStep.astro | 4 +++- src/components/BuildkiteCIUploadStepMatrix.astro | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/components/BuildkiteCIUploadStep.astro b/src/components/BuildkiteCIUploadStep.astro index 34f261b267..6efbb15237 100644 --- a/src/components/BuildkiteCIUploadStep.astro +++ b/src/components/BuildkiteCIUploadStep.astro @@ -2,6 +2,8 @@ import { Code as InlineCode } from 'astro:components'; import { Code } from 'astro-expressive-code/components'; +import buildkitePluginVersion from '~/data/buildkite-plugin-version.json'; + const inlineThemes = { light: 'github-light', dark: 'github-dark' } as const; export interface Props { @@ -15,7 +17,7 @@ const { reportPath } = Astro.props; - label: "Run tests" command: plugins: - - mergifyio/mergify-ci#v1: + - mergifyio/mergify-ci#${buildkitePluginVersion.version}: action: junit-process report_path: ${reportPath} token: "\${MERGIFY_TOKEN}"`} diff --git a/src/components/BuildkiteCIUploadStepMatrix.astro b/src/components/BuildkiteCIUploadStepMatrix.astro index 92a12ed391..69d55546b5 100644 --- a/src/components/BuildkiteCIUploadStepMatrix.astro +++ b/src/components/BuildkiteCIUploadStepMatrix.astro @@ -2,6 +2,7 @@ import { Code } from 'astro-expressive-code/components'; import Aside from '~/components/Aside.astro'; +import buildkitePluginVersion from '~/data/buildkite-plugin-version.json'; export interface Props { reportPath: string; @@ -23,7 +24,7 @@ const { reportPath } = Astro.props; - "3.12" command: plugins: - - mergifyio/mergify-ci#v1: + - mergifyio/mergify-ci#${buildkitePluginVersion.version}: action: junit-process job_name: "Tests ({{matrix}})" report_path: ${reportPath}