From 5f4925970f28932e9c181be637d92f50d53ab02c Mon Sep 17 00:00:00 2001 From: David Bond Date: Tue, 20 Oct 2026 09:48:43 +0100 Subject: [PATCH 1/2] fix: plain Verification Evidence heading, evidence sub-headings and struck-through criteria --- CHANGELOG.md | 4 + .../Rubrics/TicketContractAssessmentTests.cs | 128 ++++++++++++++++++ .../Rules/CriteriaClassificationMeasure.cs | 12 +- .../Text/IssueText.Patterns.cs | 3 +- .../Text/IssueText.Sections.cs | 16 +++ .../Text/IssueText.cs | 2 +- 6 files changed, 162 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f15aeeb..34612fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,10 @@ the new checks apply. a Reachability block, a Not covered block and a Suggested QA route of A to D. - New condition type `fieldsDiffer` and new measure `criteriaClassification`, for rubrics that need them. +### Fixed + +- TicketContract: a Verification Evidence table headed Criterion | Evidence (not only Criterion | Test) is evidence, not restated criteria, so it no longer triggers the duplication cap; a bold sub-heading inside the evidence section (such as *MUST criteria*) is not counted as a criteria heading; and an acceptance-criteria item struck through in full (-text-, optionally followed by a note or moved to KEY) is no longer counted as a criterion of the ticket. + ### Changed - Documentation and Certifications accept a justified `No documentation change: ` or diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs index f82260d..c0adfca 100644 --- a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs @@ -189,6 +189,134 @@ public async Task EvidenceTableDoesNotTriggerDuplicationCap() report.Findings.Should().NotContain(finding => finding.StartsWith("✗ Duplication", StringComparison.Ordinal)); } + + private const string PanelledPlainEvidence = """ + h2. Verification Evidence + {panel:title=Verification Evidence|titleBGColor=#E3FCEF|bgColor=#F3FFF9} + *Automated coverage of the MUST criteria* + || Criterion || Test || Location || + | MUST 1 (accumulated per tenant per month) | MerlinAllowanceStoreTests.TheMonthsFigure_IsReadable | EfCore.IntegrationTest/Merlin/MerlinAllowanceStoreTests.cs | + + *Reachability - how to make the covered code actually run* + * Refusal: the chat as a user of a tenant on the Free plan. + + *Not covered by automated tests, and why* + * The Admin markup has no component test. + + *Suggested QA route:* C (token-billing boundary, never Route A). + {panel} + """; + + private const string EvidenceWithEvidenceColumn = """ + h2. Verification Evidence + + *MUST criteria* + || Criterion || Evidence || Where || + | MUST 1 | Every image ends with a non-root user | root Dockerfile | + + *Reachability:* every pod on the next build. + + *Not covered by automated tests, and why:* container start-up has no harness here. + + *Suggested QA route:* D (infrastructure). + """; + + private const string EvidenceWithBulletsUnderBoldHeading = """ + h2. Verification Evidence + + *Coverage* + || Criterion || Test || Location || + | MUST 1 | ParserTests.ReadsTheSection | Parser/ParserTests.cs | + + *MUST criteria* + * MUST 1 is covered by the parser tests. + + *Reachability:* the import command. + + *Not covered by automated tests, and why:* nothing is uncovered. + + *Suggested QA route:* A + """; + + [Fact] + public async Task PlainEvidenceHeading_InsideAPanel_Passes() + { + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + PanelledPlainEvidence, status: "Ready for Test"); + + ContractItem(report).Finding.Should().Be("✓ Ticket Contract: Criteria classified; evidence present"); + report.Caps.Select(cap => cap.Name).Should().NotContain("Criteria duplication"); + } + + [Theory] + [InlineData("h2. Verification Evidence")] + [InlineData("h2. Verification Evidence (Retrospective)")] + public async Task EvidenceTableWithEvidenceColumn_IsNotRestatedCriteria(string heading) + { + string evidence = EvidenceWithEvidenceColumn.Replace("h2. Verification Evidence", heading, StringComparison.Ordinal); + + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + evidence, status: "Ready for Test"); + + ContractItem(report).RawScore.Should().Be(100, ContractItem(report).Finding); + report.Caps.Select(cap => cap.Name).Should().NotContain("Criteria duplication"); + } + + [Fact] + public async Task BoldSubHeadingInsideEvidence_IsNotACriteriaHeading() + { + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + EvidenceWithBulletsUnderBoldHeading, status: "Ready for Test"); + + report.Caps.Select(cap => cap.Name).Should().NotContain("Criteria duplication"); + } + + [Fact] + public async Task CriteriaStatedAgainOutsideEvidence_StillTriggersDuplicationCap() + { + const string again = "h2. Definition of Done\r\n# MUST: a schedule keeps its local run time.\r\n"; + + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + again); + + report.Caps.Select(cap => cap.Name).Should().Contain("Criteria duplication"); + } + + [Theory] + [InlineData("# -MUST: export the report as PDF- (moved to MS-27650 by Jane Smith)")] + [InlineData("# -Export the report as PDF- moved to MS-27650")] + [InlineData("# -MUST: export the report as PDF-")] + [InlineData("# -MUST: export the report as PDF- _Moved to MS-27650 by Jane Smith, 2026-10-19: waits on Legal._")] + [InlineData("* -SHOULD: export the report as PDF- (moved to MS-27650)")] + public async Task StruckThroughCriterion_IsNotACriterionOfThisTicket(string struck) + { + string criteria = $"h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n{struck}\r\n# SHOULD: the list shows the offset.\r\n"; + + AssessmentReport report = await AssessAsync(Summary + criteria); + + ContractItem(report).RawScore.Should().Be(100, ContractItem(report).Finding); + } + + [Fact] + public async Task StruckThroughMust_DoesNotCountAsTheOnlyMust() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# -MUST: export the report as PDF- (moved to MS-27650)\r\n# SHOULD: the list shows the offset.\r\n")) + .Finding.Should().Be("✗ Ticket Contract: No MUST criterion"); + + [Fact] + public async Task StruckThroughOnly_LeavesNoCriteria() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# -MUST: export the report as PDF- (moved to MS-27650)\r\n")) + .Finding.Should().Be("✗ Ticket Contract: Acceptance criteria section has no criteria"); + + [Theory] + [InlineData("# MUST: export the report as -PDF- only")] + [InlineData("# -export- the report as PDF")] + [InlineData("# -MUST: export- the report as PDF, and keep it -ready-")] + [InlineData("# -the old wording- replaced by a live statement of the rule")] + public async Task PartialStrike_IsStillACriterion_AndStaysUnclassifiedWhenNoKeyword(string item) + { + string criteria = $"h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n{item}\r\n"; + + AssessmentReport report = await AssessAsync(Summary + criteria); + + bool hasKeyword = item.Contains("MUST", StringComparison.Ordinal); + ContractItem(report).RawScore.Should().Be(hasKeyword ? 100 : 0, ContractItem(report).Finding); + } [Fact] public async Task Epic_NotApplicable() { diff --git a/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs index 82f0f9c..67e4b4d 100644 --- a/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs +++ b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs @@ -20,7 +20,7 @@ public enum CriteriaClassificationKind /// /// Counts the criteria under a section of the text, by their RFC 2119 classification. A criterion is a top-level list /// item (#, *, - or 1.) or a table row (|, not a || header). Keywords are -/// MUST, SHOULD and MAY as whole words in capitals. An absent section counts zero. +/// MUST, SHOULD and MAY as whole words in capitals. An absent section counts zero. An item struck through in full (-text-, optionally followed by a note) belongs to\n/// another ticket now and is not counted. /// /// /// A criterion (the item, or the row's first cell) is classified, in order, by: a keyword it starts with, optionally @@ -109,6 +109,12 @@ private static (int Items, int Classified, int Must) Tally(string body) } private static (string Text, bool IsListItem)? Criterion(string line) + { + (string Text, bool IsListItem)? found = Candidate(line); + return found is { } candidate && StruckThrough().IsMatch(candidate.Text) ? null : found; + } + + private static (string Text, bool IsListItem)? Candidate(string line) { Match item = ListItem().Match(line); if (item.Success) @@ -120,6 +126,10 @@ private static (string Text, bool IsListItem)? Criterion(string line) return row.Success && !MarkdownTableRule().IsMatch(row.Groups[1].Value) ? (row.Groups[1].Value, false) : null; } + // All of the text is struck through (-text-), then at most a "(note)", an "_note_" or "moved to KEY ..." remark. + [GeneratedRegex(@"^[ \t]*-(?=\S)(?:(?!-(?:\s|$)).)+-(?:[ \t]*|[ \t]+(?:\(.*\)|_.*_|moved[ \t]+to[ \t]+[A-Z][A-Z0-9]*-\d+.*)[ \t]*)$", RegexOptions.CultureInvariant, 1000)] + private static partial Regex StruckThrough(); + [GeneratedRegex(@"\r?\n", RegexOptions.CultureInvariant, 1000)] private static partial Regex LineBreak(); diff --git a/PanoramicData.IssueAssessment/Text/IssueText.Patterns.cs b/PanoramicData.IssueAssessment/Text/IssueText.Patterns.cs index fe309ab..05a65ca 100644 --- a/PanoramicData.IssueAssessment/Text/IssueText.Patterns.cs +++ b/PanoramicData.IssueAssessment/Text/IssueText.Patterns.cs @@ -29,7 +29,8 @@ public static partial class IssueText private static readonly Regex CriteriaHeadingLine = new(@"^[ \t]*(?:h([1-6])\.|(#{1,6})[ \t]|\*{1,2}(?=[^*\r\n]+\*{1,2}[ \t]*:?[ \t]*$))(.*)$", Options, Timeout); private static readonly Regex CriteriaHeadingText = new(@"(?i)criteria|definition\s+of\s+done|verification\s+evidence", Options, Timeout); - private static readonly Regex CriteriaMappingTable = new(@"(?im)^[ \t]*\|.*criteri.*\|.*\btests?\b", Options, Timeout); + private static readonly Regex VerificationEvidenceText = new(@"(?i)verification\s+evidence", Options, Timeout); + private static readonly Regex CriteriaMappingTable = new(@"(?im)^[ \t]*\|.*criteri.*\|.*\b(?:tests?|evidence)\b", Options, Timeout); private static readonly Regex PanelSectionHeading = new(@"^[ \t]*h([12])\.(.*)$", Options, Timeout); private static readonly Regex PanelOpening = new(@"^[ \t]*\{panel(?::([^}]*))?\}", Options | RegexOptions.IgnoreCase, Timeout); diff --git a/PanoramicData.IssueAssessment/Text/IssueText.Sections.cs b/PanoramicData.IssueAssessment/Text/IssueText.Sections.cs index 74fc209..2a950ed 100644 --- a/PanoramicData.IssueAssessment/Text/IssueText.Sections.cs +++ b/PanoramicData.IssueAssessment/Text/IssueText.Sections.cs @@ -106,4 +106,20 @@ private static bool IsMappingTableSection(string[] lines, List<(int Index, int L return end > index + 1 && CriteriaMappingTable.IsMatch(string.Join('\n', lines[(index + 1)..end])); } + + // A weaker heading under a Verification Evidence heading, such as a bold "MUST criteria" line, is a sub-heading of the evidence. + private static bool IsInsideEvidenceSection(List<(int Index, int Level, string Text)> headings, int current) + { + for (int previous = current - 1; previous >= 0; previous--) + { + if (headings[previous].Level >= headings[current].Level) + { + continue; + } + + return VerificationEvidenceText.IsMatch(headings[previous].Text); + } + + return false; + } } diff --git a/PanoramicData.IssueAssessment/Text/IssueText.cs b/PanoramicData.IssueAssessment/Text/IssueText.cs index cf18abf..810d2ce 100644 --- a/PanoramicData.IssueAssessment/Text/IssueText.cs +++ b/PanoramicData.IssueAssessment/Text/IssueText.cs @@ -92,7 +92,7 @@ public static IReadOnlyList CriteriaHeadings(string? text) for (int current = 0; current < headings.Count; current++) { string headingText = headings[current].Text; - if (CriteriaHeadingText.IsMatch(headingText) && !IsMappingTableSection(lines, headings, current)) + if (CriteriaHeadingText.IsMatch(headingText) && !IsMappingTableSection(lines, headings, current) && !IsInsideEvidenceSection(headings, current)) { found.Add(headingText); } From 9b24a9d468dddbd2249147d88515631a9e5fe120 Mon Sep 17 00:00:00 2001 From: David Bond Date: Tue, 20 Oct 2026 09:51:21 +0100 Subject: [PATCH 2/2] fix: strike rule keeps leading-minus criteria; doc comment --- .../Rubrics/TicketContractAssessmentTests.cs | 12 ++++++++++++ .../Rules/CriteriaClassificationMeasure.cs | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs index c0adfca..493f2bf 100644 --- a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs @@ -303,6 +303,18 @@ public async Task StruckThroughOnly_LeavesNoCriteria() => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# -MUST: export the report as PDF- (moved to MS-27650)\r\n")) .Finding.Should().Be("✗ Ticket Contract: Acceptance criteria section has no criteria"); + [Fact] + public async Task LeadingMinusNumericItem_IsStillACriterion() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n# -1 offset must be rejected\r\n")) + .Finding.Should().Be("✗ Ticket Contract: 1 criteria not classified MUST/SHOULD/MAY"); + + [Theory] + [InlineData("# MUST: show -5 when negative")] + [InlineData("# -1 offset MUST be rejected - (see MS-123)")] + public async Task LeadingMinusClassifiedItem_IsStillACriterion(string item) + => ContractItem(await AssessAsync(Summary + $"h2. Acceptance Criteria\r\n# SHOULD: the list shows the offset.\r\n{item}\r\n")) + .RawScore.Should().Be(100); + [Theory] [InlineData("# MUST: export the report as -PDF- only")] [InlineData("# -export- the report as PDF")] diff --git a/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs index 67e4b4d..e1b8dde 100644 --- a/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs +++ b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs @@ -20,7 +20,7 @@ public enum CriteriaClassificationKind /// /// Counts the criteria under a section of the text, by their RFC 2119 classification. A criterion is a top-level list /// item (#, *, - or 1.) or a table row (|, not a || header). Keywords are -/// MUST, SHOULD and MAY as whole words in capitals. An absent section counts zero. An item struck through in full (-text-, optionally followed by a note) belongs to\n/// another ticket now and is not counted. +/// MUST, SHOULD and MAY as whole words in capitals. An absent section counts zero. An item struck through in full is not counted. /// /// /// A criterion (the item, or the row's first cell) is classified, in order, by: a keyword it starts with, optionally @@ -127,7 +127,7 @@ private static (string Text, bool IsListItem)? Candidate(string line) } // All of the text is struck through (-text-), then at most a "(note)", an "_note_" or "moved to KEY ..." remark. - [GeneratedRegex(@"^[ \t]*-(?=\S)(?:(?!-(?:\s|$)).)+-(?:[ \t]*|[ \t]+(?:\(.*\)|_.*_|moved[ \t]+to[ \t]+[A-Z][A-Z0-9]*-\d+.*)[ \t]*)$", RegexOptions.CultureInvariant, 1000)] + [GeneratedRegex(@"^[ \t]*-(?=\S)(?:(?!-(?:\s|$)).)+(?<=\S)-(?:[ \t]*|[ \t]+(?:\(.*?\)|_.*?_|moved[ \t]+to[ \t]+[A-Z][A-Z0-9]*-\d+.*?)[ \t]*)$", RegexOptions.CultureInvariant, 1000)] private static partial Regex StruckThrough(); [GeneratedRegex(@"\r?\n", RegexOptions.CultureInvariant, 1000)]