From 0a3c5229e4c63585991b2fdae64bd046260549af Mon Sep 17 00:00:00 2001 From: Julien Jomier Date: Tue, 26 May 2026 20:10:02 -0400 Subject: [PATCH 1/4] Handle skipped tests in "Not run" column This commit introduces a new `notRunSkippedDetailsRegex` property to the Project model, allowing users to define regex patterns for not-run test details. A corresponding validation rule is added to ensure the regex is valid. The Build model now includes a method to count not-run tests that do not match the specified patterns, and the GraphQL schema is updated to expose this functionality. Additionally, UI components are modified to display the warning count and adjust test status colors based on the new regex patterns. Tests are also added to verify the new functionality. --- .../UpdateProjectInputValidator.php | 6 + app/Models/Build.php | 19 +++ app/Models/Project.php | 4 + app/Rules/NotRunSkippedDetailsRegexRule.php | 26 ++++ app/Utils/TestDisplay.php | 142 ++++++++++++++++++ app/cdash/app/Controller/Api/Index.php | 14 ++ app/cdash/app/Controller/Api/QueryTests.php | 8 +- app/cdash/tests/test_testhistory.php | 3 + ...otrun_skipped_details_regex_to_project.php | 15 ++ graphql/schema.graphql | 19 +++ .../js/angular/views/partials/build.html | 2 +- resources/js/vue/components/BuildSummary.vue | 9 +- .../js/vue/components/BuildTestsPage.vue | 33 ++-- .../components/ProjectSettings/GeneralTab.vue | 13 +- .../js/vue/components/ProjectSettingsPage.vue | 17 +++ .../js/vue/components/TestDetailsPage.vue | 16 ++ .../components/shared/BuildSummaryCard.vue | 28 +++- .../js/vue/components/shared/FormSection.vue | 11 +- .../js/vue/components/shared/TestDisplay.js | 99 ++++++++++++ tests/Browser/Pages/BuildTestsPageTest.php | 46 ++++++ tests/Feature/GraphQL/BuildTypeTest.php | 54 +++++++ .../GraphQL/Mutations/UpdateProjectTest.php | 26 ++++ tests/Feature/GraphQL/ProjectTypeTest.php | 2 + tests/Unit/Utils/TestDisplayTest.php | 82 ++++++++++ 24 files changed, 665 insertions(+), 29 deletions(-) create mode 100644 app/Rules/NotRunSkippedDetailsRegexRule.php create mode 100644 app/Utils/TestDisplay.php create mode 100644 database/migrations/2026_05_26_120000_add_notrun_skipped_details_regex_to_project.php create mode 100644 resources/js/vue/components/shared/TestDisplay.js create mode 100644 tests/Unit/Utils/TestDisplayTest.php diff --git a/app/GraphQL/Validators/UpdateProjectInputValidator.php b/app/GraphQL/Validators/UpdateProjectInputValidator.php index 0dc314ed3a..9d176a6436 100644 --- a/app/GraphQL/Validators/UpdateProjectInputValidator.php +++ b/app/GraphQL/Validators/UpdateProjectInputValidator.php @@ -3,6 +3,7 @@ namespace App\GraphQL\Validators; use App\Models\Project; +use App\Rules\NotRunSkippedDetailsRegexRule; use App\Rules\ProjectAuthenticateSubmissionsRule; use App\Rules\ProjectNameRule; use App\Rules\ProjectVisibilityRule; @@ -25,6 +26,11 @@ public function rules(): array 'authenticateSubmissions' => [ new ProjectAuthenticateSubmissionsRule(), ], + 'notRunSkippedDetailsRegex' => [ + 'nullable', + 'string', + new NotRunSkippedDetailsRegexRule(), + ], ]; } diff --git a/app/Models/Build.php b/app/Models/Build.php index a0d8fe0185..98402a0684 100644 --- a/app/Models/Build.php +++ b/app/Models/Build.php @@ -2,6 +2,7 @@ namespace App\Models; +use App\Utils\TestDisplay; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; @@ -400,4 +401,22 @@ public function percentCoverageForPath(string $path): ?float return ($loctested / $total_lines) * 100; } + + /** + * The number of not-run tests whose details do not match the project's skipped pattern. + */ + public function notRunTestsWarningCount(): int + { + $patterns = $this->project?->notrun_skipped_details_regex + ?? TestDisplay::DEFAULT_NOTRUN_SKIPPED_DETAILS_REGEX; + + $count = 0; + foreach ($this->tests()->where('status', Test::NOTRUN)->pluck('details') as $details) { + if (!TestDisplay::isAcceptableNotRun($details, $patterns)) { + $count++; + } + } + + return $count; + } } diff --git a/app/Models/Project.php b/app/Models/Project.php index 11b7987ed4..c102b9200e 100644 --- a/app/Models/Project.php +++ b/app/Models/Project.php @@ -3,6 +3,7 @@ namespace App\Models; use App\Enums\ProjectRole; +use App\Utils\TestDisplay; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Model; @@ -49,6 +50,7 @@ * @property ?string $banner * @property ?string $logoUrl * @property ?string $cmakeprojectroot + * @property string $notrun_skipped_details_regex * * @method Builder forUser() * @method Builder administeredByUser() @@ -95,6 +97,7 @@ class Project extends Model 'ldapfilter', 'banner', 'cmakeprojectroot', + 'notrun_skipped_details_regex', ]; protected $casts = [ @@ -137,6 +140,7 @@ class Project extends Model 'uploadquota' => 10, 'showcoveragecode' => true, 'authenticatesubmissions' => false, + 'notrun_skipped_details_regex' => TestDisplay::DEFAULT_NOTRUN_SKIPPED_DETAILS_REGEX, ]; public const ACCESS_PRIVATE = 0; diff --git a/app/Rules/NotRunSkippedDetailsRegexRule.php b/app/Rules/NotRunSkippedDetailsRegexRule.php new file mode 100644 index 0000000000..d6d9c402dd --- /dev/null +++ b/app/Rules/NotRunSkippedDetailsRegexRule.php @@ -0,0 +1,26 @@ + + */ + public static function parsePatterns(?string $patternsText): array + { + if ($patternsText === null || trim($patternsText) === '') { + return []; + } + + $patterns = []; + foreach (preg_split("/\r\n|\n|\r/", $patternsText) as $line) { + $line = trim($line); + if ($line !== '') { + $patterns[] = $line; + } + } + + return $patterns; + } + + public static function patternToPcre(string $pattern): string + { + $pattern = trim($pattern); + if ($pattern === '') { + return ''; + } + + if (str_starts_with($pattern, '/')) { + return $pattern; + } + + $parts = preg_split('/(\*|\?)/', $pattern, -1, PREG_SPLIT_DELIM_CAPTURE); + if ($parts === false) { + return ''; + } + + $regex = ''; + foreach ($parts as $part) { + if ($part === '*') { + $regex .= '.*'; + } elseif ($part === '?') { + $regex .= '.'; + } else { + $regex .= preg_quote($part, '/'); + } + } + + return '/(?i)' . $regex . '/'; + } + + public static function detailsMatchesSkippedPattern(?string $details, ?string $patternsText): bool + { + if ($details === null || $details === '') { + return false; + } + + foreach (self::parsePatterns($patternsText) as $pattern) { + $pcre = self::patternToPcre($pattern); + if ($pcre === '') { + continue; + } + + $result = @preg_match($pcre, $details); + if ($result === 1) { + return true; + } + } + + return false; + } + + public static function isAcceptableNotRun(?string $details, ?string $patternsText): bool + { + return self::detailsMatchesSkippedPattern($details, $patternsText); + } + + public static function statusColorClass(string $status, ?string $details, ?string $patternsText): string + { + if ($status === Test::NOTRUN && self::isAcceptableNotRun($details, $patternsText)) { + return 'normal'; + } + + return match ($status) { + Test::PASSED => 'normal', + Test::FAILED => 'error', + Test::NOTRUN => 'warning', + default => '', + }; + } + + public static function statusTextColorClass(string $status, ?string $details, ?string $patternsText): string + { + return match (self::statusColorClass($status, $details, $patternsText)) { + 'normal' => 'normal-text', + 'warning' => 'warning-text', + 'error' => 'error-text', + default => '', + }; + } + + public static function graphqlStatusColorClass(string $status, ?string $details, ?string $patternsText): string + { + $dbStatus = match ($status) { + 'NOT_RUN' => Test::NOTRUN, + 'PASSED' => Test::PASSED, + 'FAILED' => Test::FAILED, + default => strtolower($status), + }; + + return self::statusColorClass($dbStatus, $details, $patternsText); + } + + public static function isValidPatternsText(?string $patternsText): bool + { + foreach (self::parsePatterns($patternsText) as $pattern) { + $pcre = self::patternToPcre($pattern); + if ($pcre === '') { + return false; + } + + set_error_handler(static fn (): bool => true); + $result = preg_match($pcre, ''); + restore_error_handler(); + + if ($result === false) { + return false; + } + } + + return true; + } +} diff --git a/app/cdash/app/Controller/Api/Index.php b/app/cdash/app/Controller/Api/Index.php index b70f94fd90..7542f771bf 100644 --- a/app/cdash/app/Controller/Api/Index.php +++ b/app/cdash/app/Controller/Api/Index.php @@ -17,6 +17,7 @@ namespace CDash\Controller\Api; +use App\Models\Build as EloquentBuild; use App\Utils\TestingDay; use CDash\Database; use CDash\Model\BuildGroup; @@ -1098,6 +1099,7 @@ public function generateBuildResponseFromRow(array $build_array): array|false } $test_response['notrun'] = $nnotrun; + $test_response['notrunwarning'] = $this->computeNotRunTestsWarningCount((int) $buildid, $nnotrun); $test_response['fail'] = $nfail; $test_response['pass'] = $npass; @@ -1555,4 +1557,16 @@ public function recordGenerationTime(array &$response): void { $this->pageTimer->end($response); } + + /** + * Count not-run tests that should display as warnings on the index page. + */ + private function computeNotRunTestsWarningCount(int $buildid, int $notRunCount): int + { + if ($notRunCount <= 0) { + return 0; + } + + return EloquentBuild::find($buildid)?->notRunTestsWarningCount() ?? 0; + } } diff --git a/app/cdash/app/Controller/Api/QueryTests.php b/app/cdash/app/Controller/Api/QueryTests.php index 7f00f75295..0899abe869 100644 --- a/app/cdash/app/Controller/Api/QueryTests.php +++ b/app/cdash/app/Controller/Api/QueryTests.php @@ -20,6 +20,7 @@ use App\Models\PinnedTestMeasurement; use App\Models\Project as EloquentProject; use App\Models\TestMeasurement; +use App\Utils\TestDisplay; use CDash\Database; use CDash\Model\Build; use CDash\Model\Project; @@ -375,6 +376,7 @@ public function getResponse(): array ", $query_params); // Rows of test data to be displayed to the user. + $notRunSkippedDetailsRegex = EloquentProject::findOrFail($this->project->Id)->notrun_skipped_details_regex; $tests = []; foreach ($rows as $row) { $test = []; @@ -423,7 +425,11 @@ public function getResponse(): array case 'notrun': $test['status'] = 'Not Run'; - $test['statusclass'] = 'warning'; + $test['statusclass'] = TestDisplay::statusColorClass( + 'notrun', + $row->details, + $notRunSkippedDetailsRegex, + ); break; } diff --git a/app/cdash/tests/test_testhistory.php b/app/cdash/tests/test_testhistory.php index 7f06fb55fc..e9bd2f3a24 100644 --- a/app/cdash/tests/test_testhistory.php +++ b/app/cdash/tests/test_testhistory.php @@ -134,6 +134,9 @@ private function generateXML($mm, $timestamp, $include_sporadic, $flaky_passed) ./notrun + + Skipped + diff --git a/database/migrations/2026_05_26_120000_add_notrun_skipped_details_regex_to_project.php b/database/migrations/2026_05_26_120000_add_notrun_skipped_details_regex_to_project.php new file mode 100644 index 0000000000..e2a61e4547 --- /dev/null +++ b/database/migrations/2026_05_26_120000_add_notrun_skipped_details_regex_to_project.php @@ -0,0 +1,15 @@ + - +
diff --git a/resources/js/vue/components/BuildSummary.vue b/resources/js/vue/components/BuildSummary.vue index 373471eaec..05ff0daa11 100644 --- a/resources/js/vue/components/BuildSummary.vue +++ b/resources/js/vue/components/BuildSummary.vue @@ -516,6 +516,7 @@ export default { buildWarningsCount failedTestsCount notRunTestsCount + notRunTestsWarningCount site { id name @@ -539,6 +540,7 @@ export default { buildWarningsCount failedTestsCount notRunTestsCount + notRunTestsWarningCount } nextBuild: build(id: $nextId) @include(if: $hasNext) { id @@ -548,6 +550,7 @@ export default { buildWarningsCount failedTestsCount notRunTestsCount + notRunTestsWarningCount } } `, @@ -576,7 +579,7 @@ export default { nerrors: Math.max(0, prev.buildErrorsCount), nwarnings: Math.max(0, prev.buildWarningsCount), ntestfailed: Math.max(0, prev.failedTestsCount), - ntestnotrun: Math.max(0, prev.notRunTestsCount), + ntestnotrun: Math.max(0, prev.notRunTestsWarningCount), }; } else { this.cdash.previousbuild = null; @@ -591,7 +594,7 @@ export default { nerrors: Math.max(0, next.buildErrorsCount), nwarnings: Math.max(0, next.buildWarningsCount), ntestfailed: Math.max(0, next.failedTestsCount), - ntestnotrun: Math.max(0, next.notRunTestsCount), + ntestnotrun: Math.max(0, next.notRunTestsWarningCount), }; } else { this.cdash.nextbuild = null; @@ -616,7 +619,7 @@ export default { this.cdash.test = { nfailed: Math.max(0, build.failedTestsCount), - nnotrun: Math.max(0, build.notRunTestsCount), + nnotrun: Math.max(0, build.notRunTestsWarningCount), }; this.cdash.projectname_encoded = encodeURIComponent(build.project.name); diff --git a/resources/js/vue/components/BuildTestsPage.vue b/resources/js/vue/components/BuildTestsPage.vue index 788c83b846..7422904b7b 100644 --- a/resources/js/vue/components/BuildTestsPage.vue +++ b/resources/js/vue/components/BuildTestsPage.vue @@ -73,6 +73,7 @@ import LoadingIndicator from './shared/LoadingIndicator.vue'; import BuildSummaryCard from './shared/BuildSummaryCard.vue'; import BuildSidebar from './shared/BuildSidebar.vue'; import { DateTime } from 'luxon'; +import { testStatusToColorClass } from './shared/TestDisplay'; const TEST_QUERY = gql` query( @@ -82,6 +83,9 @@ const TEST_QUERY = gql` ) { build(id: $buildid) { id + project { + notRunSkippedDetailsRegex + } tests(filters: $filters, first: 1000000) { edges { node { @@ -205,6 +209,11 @@ export default { }); return tests; }, + result({ data }) { + if (data?.build?.project?.notRunSkippedDetailsRegex !== undefined) { + this.notRunSkippedDetailsRegex = data.build.project.notRunSkippedDetailsRegex; + } + }, variables() { return { buildid: this.buildId, @@ -237,6 +246,11 @@ export default { }); return tests; }, + result({ data }) { + if (data?.build?.project?.notRunSkippedDetailsRegex !== undefined) { + this.notRunSkippedDetailsRegex = data.build.project.notRunSkippedDetailsRegex; + } + }, variables() { return { buildid: this.previousBuildId, @@ -259,6 +273,7 @@ export default { data() { return { changedFilters: JSON.parse(JSON.stringify(this.initialFilters)), + notRunSkippedDetailsRegex: '*skip*', }; }, @@ -308,6 +323,7 @@ export default { }, formattedTestRows() { + const notRunSkippedDetailsRegex = this.notRunSkippedDetailsRegex; return this.filteredTests?.map((edge) => { return { name: { @@ -325,14 +341,14 @@ export default { value: edge.node.status, text: this.humanReadableTestStatus(edge.node.status), href: `${this.$baseURL}/tests/${edge.node.id}`, - classes: [this.testStatusToColorClass(edge.node.status)], + classes: [testStatusToColorClass(edge.node.status, edge.node.details, notRunSkippedDetailsRegex)], }, subProject: edge.subProject ?? '', timeStatus: { value: edge.node.timeStatusCategory, text: this.humanReadableTestStatus(edge.node.timeStatusCategory), href: `${this.$baseURL}/tests/${edge.node.id}?graph=time`, - classes: [this.testStatusToColorClass(edge.node.timeStatusCategory)], + classes: [testStatusToColorClass(edge.node.timeStatusCategory, edge.node.details, notRunSkippedDetailsRegex)], }, history: { value: '', @@ -349,19 +365,6 @@ export default { }, methods: { - testStatusToColorClass(status) { - switch (status) { - case 'PASSED': - return 'normal'; - case 'FAILED': - return 'error'; - case 'NOT_RUN': - return 'warning'; - default: - return ''; - } - }, - humanReadableTestStatus(status) { switch (status) { case 'PASSED': diff --git a/resources/js/vue/components/ProjectSettings/GeneralTab.vue b/resources/js/vue/components/ProjectSettings/GeneralTab.vue index 2e68a274c6..96d9468323 100644 --- a/resources/js/vue/components/ProjectSettings/GeneralTab.vue +++ b/resources/js/vue/components/ProjectSettings/GeneralTab.vue @@ -362,7 +362,8 @@ + + @@ -541,6 +550,7 @@ export default { fileUploadLimit: 50, showCoverageCode: true, banner: '', + notRunSkippedDetailsRegex: '*skip*', }, validationErrors: {}, fatalError: null, @@ -584,6 +594,7 @@ export default { fileUploadLimit showCoverageCode banner + notRunSkippedDetailsRegex } } `, diff --git a/resources/js/vue/components/ProjectSettingsPage.vue b/resources/js/vue/components/ProjectSettingsPage.vue index 66a6d89bc7..eeb4012426 100644 --- a/resources/js/vue/components/ProjectSettingsPage.vue +++ b/resources/js/vue/components/ProjectSettingsPage.vue @@ -97,5 +97,22 @@ export default { currentSection: 'general', }; }, + + mounted() { + this.navigateToHashSection(); + }, + + methods: { + navigateToHashSection() { + if (window.location.hash !== '#Testing') { + return; + } + + this.currentSection = 'general'; + this.$nextTick(() => { + document.getElementById('Testing')?.scrollIntoView(); + }); + }, + }, }; diff --git a/resources/js/vue/components/TestDetailsPage.vue b/resources/js/vue/components/TestDetailsPage.vue index 65cfeb8f93..f6a4309d5e 100644 --- a/resources/js/vue/components/TestDetailsPage.vue +++ b/resources/js/vue/components/TestDetailsPage.vue @@ -334,6 +334,7 @@ import { faChartLine, faLink, } from '@fortawesome/free-solid-svg-icons'; +import { detailsMatchesSkippedPattern } from './shared/TestDisplay'; export default { name: 'TestDetails', @@ -382,6 +383,7 @@ export default { enableTestTiming testTimeStdMultiplier testTimeStdThreshold + notRunSkippedDetailsRegex } } } @@ -517,6 +519,20 @@ export default { }, testStatusPillClass() { + if (!this.test) { + return 'tw-bg-neutral tw-text-neutral-content'; + } + + if ( + this.test.status === 'NOT_RUN' + && detailsMatchesSkippedPattern( + this.test.details, + this.build?.project?.notRunSkippedDetailsRegex ?? '*skip*', + ) + ) { + return 'tw-bg-success tw-text-success-content'; + } + switch (this.test.status) { case 'PASSED': return 'tw-bg-success tw-text-success-content'; diff --git a/resources/js/vue/components/shared/BuildSummaryCard.vue b/resources/js/vue/components/shared/BuildSummaryCard.vue index ee46268fe1..5dfe2b74b7 100644 --- a/resources/js/vue/components/shared/BuildSummaryCard.vue +++ b/resources/js/vue/components/shared/BuildSummaryCard.vue @@ -348,6 +348,7 @@ export default { passedTestsCount failedTestsCount notRunTestsCount + notRunTestsWarningCount site { id name @@ -455,9 +456,15 @@ export default { return 'No Submission'; } + const notRunWarningCount = Math.max(0, this.build.notRunTestsWarningCount); + const notRunSkippedCount = Math.max(0, this.build.notRunTestsCount - notRunWarningCount); + let retval = ''; - if (this.build.notRunTestsCount > 0) { - retval += `${this.build.notRunTestsCount} Not Run${this.commaSeparator(this.build.failedTestsCount > 0 || this.build.passedTestsCount)}`; + if (notRunWarningCount > 0) { + retval += `${notRunWarningCount} Not Run${this.commaSeparator(this.build.failedTestsCount > 0 || this.build.passedTestsCount > 0 || notRunSkippedCount > 0)}`; + } + if (notRunSkippedCount > 0) { + retval += `${notRunSkippedCount} Skipped${this.commaSeparator(this.build.failedTestsCount > 0 || this.build.passedTestsCount > 0)}`; } if (this.build.failedTestsCount > 0) { retval += `${this.build.failedTestsCount} Failed${this.commaSeparator(this.build.passedTestsCount)}`; @@ -565,19 +572,26 @@ export default { testColor() { if (this.build.failedTestsCount > 0) { return 'tw-bg-red-400'; - } else { - return 'tw-bg-green-400'; } + if (this.build.notRunTestsWarningCount > 0) { + return 'tw-bg-orange-400'; + } + + return 'tw-bg-green-400'; }, testHighlightColor() { if (!this.hasTest) { return 'tw-border-x-gray-400'; - } else if (this.build.failedTestsCount > 0) { + } + if (this.build.failedTestsCount > 0) { return 'tw-border-x-red-400'; - } else { - return 'tw-border-x-green-400'; } + if (this.build.notRunTestsWarningCount > 0) { + return 'tw-border-x-orange-400'; + } + + return 'tw-border-x-green-400'; }, /** diff --git a/resources/js/vue/components/shared/FormSection.vue b/resources/js/vue/components/shared/FormSection.vue index c95a864cb6..3061ef9487 100644 --- a/resources/js/vue/components/shared/FormSection.vue +++ b/resources/js/vue/components/shared/FormSection.vue @@ -1,5 +1,8 @@