Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ the new checks apply.
a Reachability block, a Not covered block and a Suggested QA route of A to D.
- New condition type `fieldsDiffer` and new measure `criteriaClassification`, for rubrics that need them.

### Fixed

- TicketContract: a Verification Evidence table headed Criterion | Evidence (not only Criterion | Test) is evidence, not restated criteria, so it no longer triggers the duplication cap; a bold sub-heading inside the evidence section (such as *MUST criteria*) is not counted as a criteria heading; and an acceptance-criteria item struck through in full (-text-, optionally followed by a note or moved to KEY) is no longer counted as a criterion of the ticket.

### Changed

- Documentation and Certifications accept a justified `No documentation change: <reason>` or
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,146 @@ public async Task EvidenceTableDoesNotTriggerDuplicationCap()
report.Findings.Should().NotContain(finding => finding.StartsWith("✗ Duplication", StringComparison.Ordinal));
}


private const string PanelledPlainEvidence = """
h2. Verification Evidence
{panel:title=Verification Evidence|titleBGColor=#E3FCEF|bgColor=#F3FFF9}
*Automated coverage of the MUST criteria*
|| Criterion || Test || Location ||
| MUST 1 (accumulated per tenant per month) | MerlinAllowanceStoreTests.TheMonthsFigure_IsReadable | EfCore.IntegrationTest/Merlin/MerlinAllowanceStoreTests.cs |

*Reachability - how to make the covered code actually run*
* Refusal: the chat as a user of a tenant on the Free plan.

*Not covered by automated tests, and why*
* The Admin markup has no component test.

*Suggested QA route:* C (token-billing boundary, never Route A).
{panel}
""";

private const string EvidenceWithEvidenceColumn = """
h2. Verification Evidence

*MUST criteria*
|| Criterion || Evidence || Where ||
| MUST 1 | Every image ends with a non-root user | root Dockerfile |

*Reachability:* every pod on the next build.

*Not covered by automated tests, and why:* container start-up has no harness here.

*Suggested QA route:* D (infrastructure).
""";

private const string EvidenceWithBulletsUnderBoldHeading = """
h2. Verification Evidence

*Coverage*
|| Criterion || Test || Location ||
| MUST 1 | ParserTests.ReadsTheSection | Parser/ParserTests.cs |

*MUST criteria*
* MUST 1 is covered by the parser tests.

*Reachability:* the import command.

*Not covered by automated tests, and why:* nothing is uncovered.

*Suggested QA route:* A
""";

[Fact]
public async Task PlainEvidenceHeading_InsideAPanel_Passes()
{
AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + PanelledPlainEvidence, status: "Ready for Test");

ContractItem(report).Finding.Should().Be("✓ Ticket Contract: Criteria classified; evidence present");
report.Caps.Select(cap => cap.Name).Should().NotContain("Criteria duplication");
}

[Theory]
[InlineData("h2. Verification Evidence")]
[InlineData("h2. Verification Evidence (Retrospective)")]
public async Task EvidenceTableWithEvidenceColumn_IsNotRestatedCriteria(string heading)
{
string evidence = EvidenceWithEvidenceColumn.Replace("h2. Verification Evidence", heading, StringComparison.Ordinal);

AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + evidence, status: "Ready for Test");

ContractItem(report).RawScore.Should().Be(100, ContractItem(report).Finding);
report.Caps.Select(cap => cap.Name).Should().NotContain("Criteria duplication");
}

[Fact]
public async Task BoldSubHeadingInsideEvidence_IsNotACriteriaHeading()
{
AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + EvidenceWithBulletsUnderBoldHeading, status: "Ready for Test");

report.Caps.Select(cap => cap.Name).Should().NotContain("Criteria duplication");
}

[Fact]
public async Task CriteriaStatedAgainOutsideEvidence_StillTriggersDuplicationCap()
{
const string again = "h2. Definition of Done\r\n# MUST: a schedule keeps its local run time.\r\n";

AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + again);

report.Caps.Select(cap => cap.Name).Should().Contain("Criteria duplication");
}

[Theory]
[InlineData("# -MUST: export the report as PDF- (moved to MS-27650 by Jane Smith)")]
[InlineData("# -Export the report as PDF- moved to MS-27650")]
[InlineData("# -MUST: export the report as PDF-")]
[InlineData("# -MUST: export the report as PDF- _Moved to MS-27650 by Jane Smith, 2026-10-19: waits on Legal._")]
[InlineData("* -SHOULD: export the report as PDF- (moved to MS-27650)")]
public async Task StruckThroughCriterion_IsNotACriterionOfThisTicket(string struck)
{
string criteria = $"h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n{struck}\r\n# SHOULD: the list shows the offset.\r\n";

AssessmentReport report = await AssessAsync(Summary + criteria);

ContractItem(report).RawScore.Should().Be(100, ContractItem(report).Finding);
}

[Fact]
public async Task StruckThroughMust_DoesNotCountAsTheOnlyMust()
=> ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# -MUST: export the report as PDF- (moved to MS-27650)\r\n# SHOULD: the list shows the offset.\r\n"))
.Finding.Should().Be("✗ Ticket Contract: No MUST criterion");

[Fact]
public async Task StruckThroughOnly_LeavesNoCriteria()
=> ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# -MUST: export the report as PDF- (moved to MS-27650)\r\n"))
.Finding.Should().Be("✗ Ticket Contract: Acceptance criteria section has no criteria");

[Fact]
public async Task LeadingMinusNumericItem_IsStillACriterion()
=> ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n# -1 offset must be rejected\r\n"))
.Finding.Should().Be("✗ Ticket Contract: 1 criteria not classified MUST/SHOULD/MAY");

[Theory]
[InlineData("# MUST: show -5 when negative")]
[InlineData("# -1 offset MUST be rejected - (see MS-123)")]
public async Task LeadingMinusClassifiedItem_IsStillACriterion(string item)
=> ContractItem(await AssessAsync(Summary + $"h2. Acceptance Criteria\r\n# SHOULD: the list shows the offset.\r\n{item}\r\n"))
.RawScore.Should().Be(100);

[Theory]
[InlineData("# MUST: export the report as -PDF- only")]
[InlineData("# -export- the report as PDF")]
[InlineData("# -MUST: export- the report as PDF, and keep it -ready-")]
[InlineData("# -the old wording- replaced by a live statement of the rule")]
public async Task PartialStrike_IsStillACriterion_AndStaysUnclassifiedWhenNoKeyword(string item)
{
string criteria = $"h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n{item}\r\n";

AssessmentReport report = await AssessAsync(Summary + criteria);

bool hasKeyword = item.Contains("MUST", StringComparison.Ordinal);
ContractItem(report).RawScore.Should().Be(hasKeyword ? 100 : 0, ContractItem(report).Finding);
}
[Fact]
public async Task Epic_NotApplicable()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public enum CriteriaClassificationKind
/// <summary>
/// Counts the criteria under a section of the text, by their RFC 2119 classification. A criterion is a top-level list
/// item (<c>#</c>, <c>*</c>, <c>-</c> or <c>1.</c>) or a table row (<c>|</c>, not a <c>||</c> header). Keywords are
/// MUST, SHOULD and MAY as whole words in capitals. An absent section counts zero.
/// MUST, SHOULD and MAY as whole words in capitals. An absent section counts zero. An item struck through in full is not counted.
/// </summary>
/// <remarks>
/// A criterion (the item, or the row's first cell) is classified, in order, by: a keyword it starts with, optionally
Expand Down Expand Up @@ -109,6 +109,12 @@ private static (int Items, int Classified, int Must) Tally(string body)
}

private static (string Text, bool IsListItem)? Criterion(string line)
{
(string Text, bool IsListItem)? found = Candidate(line);
return found is { } candidate && StruckThrough().IsMatch(candidate.Text) ? null : found;
}

private static (string Text, bool IsListItem)? Candidate(string line)
{
Match item = ListItem().Match(line);
if (item.Success)
Expand All @@ -120,6 +126,10 @@ private static (string Text, bool IsListItem)? Criterion(string line)
return row.Success && !MarkdownTableRule().IsMatch(row.Groups[1].Value) ? (row.Groups[1].Value, false) : null;
}

// All of the text is struck through (-text-), then at most a "(note)", an "_note_" or "moved to KEY ..." remark.
[GeneratedRegex(@"^[ \t]*-(?=\S)(?:(?!-(?:\s|$)).)+(?<=\S)-(?:[ \t]*|[ \t]+(?:\(.*?\)|_.*?_|moved[ \t]+to[ \t]+[A-Z][A-Z0-9]*-\d+.*?)[ \t]*)$", RegexOptions.CultureInvariant, 1000)]
private static partial Regex StruckThrough();

[GeneratedRegex(@"\r?\n", RegexOptions.CultureInvariant, 1000)]
private static partial Regex LineBreak();

Expand Down
3 changes: 2 additions & 1 deletion PanoramicData.IssueAssessment/Text/IssueText.Patterns.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ public static partial class IssueText

private static readonly Regex CriteriaHeadingLine = new(@"^[ \t]*(?:h([1-6])\.|(#{1,6})[ \t]|\*{1,2}(?=[^*\r\n]+\*{1,2}[ \t]*:?[ \t]*$))(.*)$", Options, Timeout);
private static readonly Regex CriteriaHeadingText = new(@"(?i)criteria|definition\s+of\s+done|verification\s+evidence", Options, Timeout);
private static readonly Regex CriteriaMappingTable = new(@"(?im)^[ \t]*\|.*criteri.*\|.*\btests?\b", Options, Timeout);
private static readonly Regex VerificationEvidenceText = new(@"(?i)verification\s+evidence", Options, Timeout);
private static readonly Regex CriteriaMappingTable = new(@"(?im)^[ \t]*\|.*criteri.*\|.*\b(?:tests?|evidence)\b", Options, Timeout);

private static readonly Regex PanelSectionHeading = new(@"^[ \t]*h([12])\.(.*)$", Options, Timeout);
private static readonly Regex PanelOpening = new(@"^[ \t]*\{panel(?::([^}]*))?\}", Options | RegexOptions.IgnoreCase, Timeout);
Expand Down
16 changes: 16 additions & 0 deletions PanoramicData.IssueAssessment/Text/IssueText.Sections.cs
Original file line number Diff line number Diff line change
Expand Up @@ -106,4 +106,20 @@ private static bool IsMappingTableSection(string[] lines, List<(int Index, int L

return end > index + 1 && CriteriaMappingTable.IsMatch(string.Join('\n', lines[(index + 1)..end]));
}

// A weaker heading under a Verification Evidence heading, such as a bold "MUST criteria" line, is a sub-heading of the evidence.
private static bool IsInsideEvidenceSection(List<(int Index, int Level, string Text)> headings, int current)
{
for (int previous = current - 1; previous >= 0; previous--)
{
if (headings[previous].Level >= headings[current].Level)
{
continue;
}

return VerificationEvidenceText.IsMatch(headings[previous].Text);
}

return false;
}
}
2 changes: 1 addition & 1 deletion PanoramicData.IssueAssessment/Text/IssueText.cs
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public static IReadOnlyList<string> CriteriaHeadings(string? text)
for (int current = 0; current < headings.Count; current++)
{
string headingText = headings[current].Text;
if (CriteriaHeadingText.IsMatch(headingText) && !IsMappingTableSection(lines, headings, current))
if (CriteriaHeadingText.IsMatch(headingText) && !IsMappingTableSection(lines, headings, current) && !IsInsideEvidenceSection(headings, current))
{
found.Add(headingText);
}
Expand Down
Loading