Skip to content

Release Notes check and cap apply from Ready for Test onwards #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
davidnmbond merged 1 commit into from
Oct 29, 2026
Merged

Release Notes check and cap apply from Ready for Test onwards #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ the new checks apply.

### Fixed

- ReleaseNotes and EpicReleaseNotes apply from Ready for Test onwards, the status at which Release Notes are set. Before
then an empty field scores 100 with "○ Release Notes: Not required yet (status: ...)", as Fix Version does, and the
"Release notes" mandatory cap does not apply, so a ticket in Ready for Progress can reach EXCELLENT. Notes already
written before Ready for Test are still checked and reported, but do not cap the score.
- TicketContract: a Verification Evidence table headed Criterion | Evidence (not only Criterion | Test) is evidence, not restated criteria, so it no longer triggers the duplication cap; a bold sub-heading inside the evidence section (such as *MUST criteria*) is not counted as a criteria heading; and an acceptance-criteria item struck through in full (-text-, optionally followed by a note or moved to KEY) is no longer counted as a criterion of the ticket.

### Changed
Expand Down
43 changes: 38 additions & 5 deletions PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
namespace PanoramicData.IssueAssessment.Tests.Rubrics;

/// <summary>
/// The MS-only ReleaseNotes assessment: customer-facing release notes are mandatory, so a failing tier caps the score.
/// The MS-only ReleaseNotes assessment: customer-facing release notes are mandatory from Ready for Test, so a failing tier caps the score from then on.
/// </summary>
public class ReleaseNotesAssessmentTests : TestBase
{
Expand Down Expand Up @@ -212,18 +212,51 @@ public async Task ResolutionFixed_IsAssessed(string resolution)
report.Caps.Select(cap => cap.Name).Should().Contain(CapName);
}

[Theory]
[InlineData("Improvement", "Ready for Progress")]
[InlineData("Improvement", "In Progress")]
[InlineData("Epic", "In Progress")]
public async Task EmptyBeforeReadyForTest_NotRequiredYet_NoCap(string issueType, string status)
{
AssessmentReport report = await AssessAsync(notes: null, issueType: issueType, status: status);

NotesItem(report).RawScore.Should().Be(100);
NotesItem(report).Finding.Should().Be($"○ Release Notes: Not required yet (status: {status})");
report.Caps.Select(cap => cap.Name).Should().NotContain(CapName);
}

[Fact]
public async Task FlawedBeforeReadyForTest_StillReported_NoCap()
{
AssessmentReport report = await AssessAsync("TBD", status: "Ready for Progress");

NotesItem(report).Finding.Should().Be("✗ Release Notes: Placeholder");
report.Caps.Select(cap => cap.Name).Should().NotContain(CapName);
}

[Theory]
[InlineData("In Test")]
[InlineData("Closed")]
public async Task EmptyFromReadyForTestOn_Caps(string status)
{
AssessmentReport report = await AssessAsync(notes: null, status: status);

NotesItem(report).Finding.Should().Be("✗ Release Notes: Empty");
report.Caps.Select(cap => cap.Name).Should().Contain(CapName);
}

private static AssessmentReportItem NotesItem(AssessmentReport report)
=> report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name.EndsWith("ReleaseNotes", StringComparison.Ordinal) && item.Outcome != AssessmentOutcome.NotApplicable);

private static NormalizedIssue Issue(string? notes, string title = "Keep schedule run times across timezone changes", string issueType = "Improvement", string? resolution = null, ImmutableList<string>? labels = null)
private static NormalizedIssue Issue(string? notes, string title = "Keep schedule run times across timezone changes", string issueType = "Improvement", string? resolution = null, ImmutableList<string>? labels = null, string status = "Ready for Test")
=> new()
{
Key = "MS-1",
Project = "MS",
Title = title,
Description = Description,
IssueType = issueType,
Status = "In Progress",
Status = status,
Platform = "Jira",
ReleaseNotes = notes,
Resolution = resolution,
Expand All @@ -236,6 +269,6 @@ private static async Task<AssessmentReport> ReportAsync(IssueAssessor assessor,
return AssessmentReport.From(await assessor.AssessAsync(document, CancellationToken));
}

private static Task<AssessmentReport> AssessAsync(string? notes, string title = "Keep schedule run times across timezone changes", string issueType = "Improvement", string? resolution = null, ImmutableList<string>? labels = null)
=> ReportAsync(Assessor, Issue(notes, title, issueType, resolution, labels));
private static Task<AssessmentReport> AssessAsync(string? notes, string title = "Keep schedule run times across timezone changes", string issueType = "Improvement", string? resolution = null, ImmutableList<string>? labels = null, string status = "Ready for Test")
=> ReportAsync(Assessor, Issue(notes, title, issueType, resolution, labels, status));
}
10 changes: 8 additions & 2 deletions PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"conditions": [ { "type": "fieldPresent", "path": "$.resolution" }, { "type": "not", "condition": { "type": "fieldEquals", "path": "$.resolution", "values": [ "Fixed", "Done" ] } } ]
},
"contractEvidenceDue": { "type": "fieldEquals", "path": "$.status", "values": [ "Ready for Test", "In Test", "Test Blocked", "Ready for Customer", "With Customer", "Ready for Release", "Closed", "Done", "Resolved" ] },
"releaseNotesDue": { "type": "ref", "name": "contractEvidenceDue" },
"isInvestigation": {
"type": "all",
"conditions": [
Expand Down Expand Up @@ -382,6 +383,7 @@
"definition": {
"type": "tiered",
"tiers": [
{ "when": { "type": "all", "conditions": [ { "type": "not", "condition": { "type": "ref", "name": "releaseNotesDue" } }, { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } } ] }, "score": 100, "finding": "○ Release Notes: Not required yet (status: {$.status})" },
{ "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Fill in the Release Notes field with one or two customer-facing sentences of at least 8 words saying what changed for the customer" },
{ "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Replace the placeholder in the Release Notes field with one or two sentences saying what changed for the customer" },
{ "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Say what changed and for whom: a generic note such as 'Fixed bug' tells the customer nothing" },
Expand Down Expand Up @@ -608,6 +610,7 @@
"definition": {
"type": "tiered",
"tiers": [
{ "when": { "type": "all", "conditions": [ { "type": "not", "condition": { "type": "ref", "name": "releaseNotesDue" } }, { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } } ] }, "score": 100, "finding": "○ Release Notes: Not required yet (status: {$.status})" },
{ "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Fill in the Release Notes field with one or two customer-facing sentences of at least 8 words saying what changed for the customer" },
{ "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Replace the placeholder in the Release Notes field with one or two sentences saying what changed for the customer" },
{ "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Say what changed and for whom: a generic note such as 'Fixed bug' tells the customer nothing" },
Expand Down Expand Up @@ -663,8 +666,11 @@
"group": "mandatory",
"cap": 0.79,
"when": {
"type": "any",
"conditions": [ { "type": "assessmentScore", "assessment": "ReleaseNotes", "lessThan": 100 }, { "type": "assessmentScore", "assessment": "EpicReleaseNotes", "lessThan": 100 } ]
"type": "all",
"conditions": [
{ "type": "ref", "name": "releaseNotesDue" },
{ "type": "any", "conditions": [ { "type": "assessmentScore", "assessment": "ReleaseNotes", "lessThan": 100 }, { "type": "assessmentScore", "assessment": "EpicReleaseNotes", "lessThan": 100 } ] }
]
}
},
{
Expand Down
Loading