From 469a8017bbe2e899218c315c1aa28d4944550603 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 11:48:52 +0100 Subject: [PATCH 01/15] feat: normalised issue carries release notes and resolution --- .../Mapping/MapperTests.cs | 34 +++++++++++++++++++ .../Mapping/JiraIssueMapper.cs | 12 ++++--- .../Mapping/NormalizedIssue.cs | 8 +++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs b/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs index 3242b61..3d4c7fd 100644 --- a/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs @@ -148,6 +148,40 @@ public void Jira_V3Description_ListItemParagraphsAndAttributelessNodes_Flatten() issue.Description.Should().Be("* first\nsecond\nend"); } + [Fact] + public void FromRestJson_ReadsReleaseNotesFromCustomField10600() + { + JiraIssueMapper.FromRestJson(Json("""{ "fields": { "customfield_10600": "Fixed the export." } }""")) + .ReleaseNotes.Should().Be("Fixed the export."); + + JiraIssueMapper.FromRestJson(Json(""" + { "fields": { "customfield_10600": { "type": "doc", "version": 1, "content": [ + { "type": "paragraph", "content": [ { "type": "text", "text": "Fixed the export." } ] } ] } } } + """)).ReleaseNotes.Should().Be("Fixed the export."); + + JiraIssueMapper.FromRestJson(Json("""{ "fields": {} }""")).ReleaseNotes.Should().BeNull(); + } + + [Fact] + public void FromRestJson_ReadsResolutionName() + { + JiraIssueMapper.FromRestJson(Json("""{ "fields": { "resolution": { "name": "Won't Do" } } }""")) + .Resolution.Should().Be("Won't Do"); + JiraIssueMapper.FromRestJson(Json("""{ "fields": { "resolution": null } }""")).Resolution.Should().BeNull(); + } + + [Fact] + public void ToJsonObject_OmitsNullReleaseNotesAndResolution() + { + var empty = new NormalizedIssue().ToJsonObject(); + empty.ContainsKey("releaseNotes").Should().BeFalse(); + empty.ContainsKey("resolution").Should().BeFalse(); + + var set = new NormalizedIssue { ReleaseNotes = "n", Resolution = "Fixed" }.ToJsonObject(); + set["releaseNotes"]!.GetValue().Should().Be("n"); + set["resolution"]!.GetValue().Should().Be("Fixed"); + } + [Fact] public void Jira_NoDescription_IsNull() { diff --git a/PanoramicData.IssueAssessment/Mapping/JiraIssueMapper.cs b/PanoramicData.IssueAssessment/Mapping/JiraIssueMapper.cs index 9ba5fb3..b57ac9a 100644 --- a/PanoramicData.IssueAssessment/Mapping/JiraIssueMapper.cs +++ b/PanoramicData.IssueAssessment/Mapping/JiraIssueMapper.cs @@ -48,6 +48,8 @@ public static NormalizedIssue FromRestJson(JsonElement issue, Uri? baseUrl) FixVersions = Names(fields, "fixVersions"), AttachmentsCount = ArrayLength(fields, "attachment"), DueDate = StringAt(fields, "duedate"), + ReleaseNotes = TextField(fields, "customfield_10600"), + Resolution = StringAt(fields, "resolution", "name"), Url = BrowseUrl(baseUrl, key), Platform = "Jira", }; @@ -81,11 +83,13 @@ public static string FlattenAdf(JsonElement document) return AdfFlattener.Flatten(document); } - private static string? Description(JsonElement fields) - => fields.TryGetProperty("description", out JsonElement description) ? description.ValueKind switch + private static string? Description(JsonElement fields) => TextField(fields, "description"); + + private static string? TextField(JsonElement fields, string property) + => fields.TryGetProperty(property, out JsonElement value) ? value.ValueKind switch { - JsonValueKind.String => description.GetString(), - JsonValueKind.Object => FlattenAdf(description), + JsonValueKind.String => value.GetString(), + JsonValueKind.Object => FlattenAdf(value), _ => null, } : null; diff --git a/PanoramicData.IssueAssessment/Mapping/NormalizedIssue.cs b/PanoramicData.IssueAssessment/Mapping/NormalizedIssue.cs index 073df6f..d711938 100644 --- a/PanoramicData.IssueAssessment/Mapping/NormalizedIssue.cs +++ b/PanoramicData.IssueAssessment/Mapping/NormalizedIssue.cs @@ -68,6 +68,12 @@ public sealed partial record NormalizedIssue /// Gets reproduction steps held in a field of their own, if the system has one. public string? ReproductionSteps { get; init; } + /// Gets the release notes held in a field of their own, if the system has one. + public string? ReleaseNotes { get; init; } + + /// Gets the resolution name, such as Fixed or Won't Do; null while unresolved. + public string? Resolution { get; init; } + /// Gets a link to the issue. public string? Url { get; init; } @@ -118,6 +124,8 @@ public JsonObject ToJsonObject() Add(issue, "dueDate", DueDate); Add(issue, "acceptanceCriteria", AcceptanceCriteria); Add(issue, "reproductionSteps", ReproductionSteps); + Add(issue, "releaseNotes", ReleaseNotes); + Add(issue, "resolution", Resolution); Add(issue, "url", Url); Add(issue, "platform", Platform); return issue; From da5b631695eaffb5e093bac66b51b47b017d5727 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 11:56:26 +0100 Subject: [PATCH 02/15] feat: MS release-notes assessment with mandatory cap Adds the fieldsDiffer condition, the isMs and isUndelivered named conditions, a ReleaseNotes assessment (EpicReleaseNotes for epics, since assessment names must be unique) weighted 6, and a mandatory 'Release notes' cap at 0.79. Not applicable outside MS and when resolution is set and not Fixed or Done. BuiltInRubricParityTests: no expectation changed. The Yoda fixtures with MS keys would now gain the new finding and cap, so the parity rubric drops the ReleaseNotes assessments and cap, as it already drops Presentation; ReleaseNotesAssessmentTests covers them. A 1.1.6 rubric copy is a test fixture for the OPS unchanged-score test. --- ...PanoramicData.IssueAssessment.Tests.csproj | 1 + .../Rubrics/BuiltInRubricParityTests.cs | 10 +- .../Rubrics/ReleaseNotesAssessmentTests.cs | 191 ++++++ .../Rubrics/ticket-quality-1.1.6.json | 609 ++++++++++++++++++ .../Rubrics/panoramicdata-ticket-quality.json | 56 ++ .../Rules/Conditions.cs | 1 + .../Rules/FieldsDifferCondition.cs | 36 ++ .../Validation/RuleValidator.cs | 1 + 8 files changed, 903 insertions(+), 2 deletions(-) create mode 100644 PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs create mode 100644 PanoramicData.IssueAssessment.Tests/Rubrics/ticket-quality-1.1.6.json create mode 100644 PanoramicData.IssueAssessment/Rules/FieldsDifferCondition.cs diff --git a/PanoramicData.IssueAssessment.Tests/PanoramicData.IssueAssessment.Tests.csproj b/PanoramicData.IssueAssessment.Tests/PanoramicData.IssueAssessment.Tests.csproj index b2fefb8..6a19d42 100644 --- a/PanoramicData.IssueAssessment.Tests/PanoramicData.IssueAssessment.Tests.csproj +++ b/PanoramicData.IssueAssessment.Tests/PanoramicData.IssueAssessment.Tests.csproj @@ -15,6 +15,7 @@ + diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs index 6959b9e..6edd676 100644 --- a/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs @@ -17,7 +17,7 @@ namespace PanoramicData.IssueAssessment.Tests.Rubrics; /// public class BuiltInRubricParityTests : TestBase { - // Yoda has no Presentation factor, so parity is proved over the rubric without it; PresentationTests covers that one. + // Yoda has no Presentation or Release Notes factor, so parity is proved over the rubric without them; PresentationTests and ReleaseNotesAssessmentTests cover those. private static readonly IssueAssessor Assessor = new(Serialization.AssessmentRubric.Load(WithoutPresentation(BuiltInRubrics.PanoramicDataTicketQualityJson))); public static TheoryData Cases => [.. Enumerable.Range(0, File.ReadAllLines(FixturePath).Length)]; @@ -69,12 +69,18 @@ private static string WithoutPresentation(string rubricJson) foreach (JsonNode? category in rubric["categories"]!.AsArray()) { JsonArray assessments = category!["assessments"]!.AsArray(); - foreach (JsonNode? assessment in assessments.Where(item => item!["name"]!.GetValue().EndsWith("Presentation", StringComparison.Ordinal)).ToList()) + foreach (JsonNode? assessment in assessments.Where(item => item!["name"]!.GetValue().EndsWith("Presentation", StringComparison.Ordinal) || item["name"]!.GetValue().EndsWith("ReleaseNotes", StringComparison.Ordinal)).ToList()) { assessments.Remove(assessment); } } + JsonArray caps = rubric["scoreCaps"]!.AsArray(); + foreach (JsonNode? cap in caps.Where(item => item!["name"]!.GetValue() == "Release notes").ToList()) + { + caps.Remove(cap); + } + return rubric.ToJsonString(); } diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs new file mode 100644 index 0000000..42ec9d3 --- /dev/null +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs @@ -0,0 +1,191 @@ +using PanoramicData.IssueAssessment.Mapping; +using PanoramicData.IssueAssessment.Rubrics; + +namespace PanoramicData.IssueAssessment.Tests.Rubrics; + +/// +/// The MS-only ReleaseNotes assessment: customer-facing release notes are mandatory, so a failing tier caps the score. +/// +public class ReleaseNotesAssessmentTests : TestBase +{ + private const string CapName = "Release notes"; + private const string GoodNotes = "Reports now keep their scheduled run time after the tenant timezone changes"; + private static readonly IssueAssessor Assessor = new(BuiltInRubrics.PanoramicDataTicketQuality); + + private const string Description = "h2. Summary\r\nRun times drift when the tenant timezone changes.\r\n\r\nh2. Acceptance Criteria\r\n# MUST: a schedule keeps its local run time after a timezone change.\r\n# SHOULD: the schedule list shows the new offset.\r\n\r\nh2. Business Case\r\nCustomers rely on reports arriving at the agreed time, which has a direct business value."; + + [Fact] + public async Task Empty_FailsAndCapsAt79() + { + AssessmentReport report = await AssessAsync(notes: null); + + NotesItem(report).RawScore.Should().Be(0); + NotesItem(report).Finding.Should().Be("✗ Release Notes: Empty"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + report.NormalizedScore.Should().BeLessThanOrEqualTo(0.79); + } + + [Theory] + [InlineData("TBD")] + [InlineData("None")] + [InlineData("See the ticket")] + [InlineData("---")] + public async Task Placeholder_Fails(string notes) + => NotesItem(await AssessAsync(notes)).Finding.Should().Be("✗ Release Notes: Placeholder"); + + [Fact] + public async Task Placeholder_TBD_Fails() + { + AssessmentReport report = await AssessAsync("TBD"); + + NotesItem(report).Finding.Should().Be("✗ Release Notes: Placeholder"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task SevenWords_Fails() + => NotesItem(await AssessAsync("Schedules now keep their run time changes")).Finding.Should().Be("✗ Release Notes: Under 8 words"); + + [Fact] + public async Task SameAsSummaryIgnoringPrefix_Fails() + => NotesItem(await AssessAsync("Add X to Y report for every tenant", title: "ReportMagic: Add X to Y report for every tenant")) + .Finding.Should().Be("✗ Release Notes: Same as the summary"); + + [Theory] + [InlineData("Fixes the issue raised in MS-12345 for all users today")] + [InlineData("Updated ReportJob.cs so that every scheduled report runs on time")] + [InlineData("Changed the file under C:\\inetpub\\reports so that every report loads")] + [InlineData("Updated the files in the repo/src/Reports folder so reports load faster")] + public async Task TicketKey_Fails(string notes) + => NotesItem(await AssessAsync(notes)).Finding.Should().StartWith("✗ Release Notes: Not customer-facing"); + + [Fact] + public async Task FilePath_Fails() + => NotesItem(await AssessAsync("The settings now load from appsettings.json when the service starts up")).Finding.Should().StartWith("✗ Release Notes: Not customer-facing"); + + [Fact] + public async Task ServiceIdentifier_Fails() + => NotesItem(await AssessAsync("ReportJobService now retries failed jobs three times before giving up")).Finding.Should().StartWith("✗ Release Notes: Not customer-facing"); + + [Fact] + public async Task ProductNames_Pass() + { + AssessmentReport report = await AssessAsync("ReportMagic and ConnectMagic now show the DataMagic sync time on every dashboard page"); + + NotesItem(report).RawScore.Should().Be(100); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Fact] + public async Task SentenceInitialCapital_Passes() + => NotesItem(await AssessAsync("Schedules now keep their local run time after the timezone changes")).RawScore.Should().Be(100); + + [Fact] + public async Task EmDash_Fails() + => NotesItem(await AssessAsync("Schedules now keep their run time \u2014 even when the timezone changes")).Finding.Should().Be("✗ Release Notes: Contains an em dash"); + + [Fact] + public async Task UsSpelling_Fails() + => NotesItem(await AssessAsync("Schedules now show the correct behavior when the timezone changes for tenants")).Finding.Should().Be("✗ Release Notes: US spelling"); + + [Theory] + [InlineData("Fixed bug")] + [InlineData("Various fixes")] + public async Task GenericFixedBug_Fails(string notes) + { + AssessmentReport report = await AssessAsync(notes); + + NotesItem(report).RawScore.Should().Be(0); + } + + [Fact] + public async Task Good_Scores100_NoCap() + { + AssessmentReport report = await AssessAsync(GoodNotes); + + NotesItem(report).Finding.Should().Be("✓ Release Notes: Present and customer-facing"); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Fact] + public async Task InternalLabel_StillRequiresNotes() + { + AssessmentReport report = await AssessAsync(notes: null, labels: ["internal"]); + + NotesItem(report).RawScore.Should().Be(0); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task Epic_IsAssessed() + { + AssessmentReport report = await AssessAsync(notes: null, issueType: "Epic"); + + report.Categories.Single(category => category.Name == "Epic Quality").Assessments + .Single(item => item.Name.EndsWith("ReleaseNotes", StringComparison.Ordinal)).RawScore.Should().Be(0); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task OpsProject_NotApplicable_ScoreUnchanged() + { + NormalizedIssue ops = Issue(notes: null) with { Key = "OPS-1", Project = "OPS", Description = Description }; + IssueAssessor previous = new(Serialization.AssessmentRubric.Load(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Rubrics", "ticket-quality-1.1.6.json")))); + + AssessmentReport now = await ReportAsync(Assessor, ops); + AssessmentReport before = await ReportAsync(previous, ops); + + now.Score.Should().Be(before.Score); + now.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + now.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == "ReleaseNotes").Outcome.Should().Be(AssessmentOutcome.NotApplicable); + } + + [Theory] + [InlineData("Won't Do")] + [InlineData("Duplicate")] + public async Task ResolutionWontDo_NotApplicable_NoCap(string resolution) + { + AssessmentReport report = await AssessAsync(notes: null, resolution: resolution); + + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == "ReleaseNotes").Outcome.Should().Be(AssessmentOutcome.NotApplicable); + } + + [Theory] + [InlineData("Fixed")] + [InlineData("Done")] + public async Task ResolutionFixed_IsAssessed(string resolution) + { + AssessmentReport report = await AssessAsync(notes: null, resolution: resolution); + + NotesItem(report).RawScore.Should().Be(0); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + private static AssessmentReportItem NotesItem(AssessmentReport report) + => report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name.EndsWith("ReleaseNotes", StringComparison.Ordinal) && item.Outcome != AssessmentOutcome.NotApplicable); + + private static NormalizedIssue Issue(string? notes, string title = "Keep schedule run times across timezone changes", string issueType = "Improvement", string? resolution = null, ImmutableList? labels = null) + => new() + { + Key = "MS-1", + Project = "MS", + Title = title, + Description = Description, + IssueType = issueType, + Status = "In Progress", + Platform = "Jira", + ReleaseNotes = notes, + Resolution = resolution, + Labels = labels ?? [], + }; + + private static async Task ReportAsync(IssueAssessor assessor, NormalizedIssue issue) + { + using JsonDocument document = issue.ToJsonDocument(); + return AssessmentReport.From(await assessor.AssessAsync(document, CancellationToken)); + } + + private static Task AssessAsync(string? notes, string title = "Keep schedule run times across timezone changes", string issueType = "Improvement", string? resolution = null, ImmutableList? labels = null) + => ReportAsync(Assessor, Issue(notes, title, issueType, resolution, labels)); +} diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/ticket-quality-1.1.6.json b/PanoramicData.IssueAssessment.Tests/Rubrics/ticket-quality-1.1.6.json new file mode 100644 index 0000000..47830d0 --- /dev/null +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/ticket-quality-1.1.6.json @@ -0,0 +1,609 @@ +{ + // Panoramic Data ticket quality rubric. + // Ported from PanoramicData.Skills .github/skills/_common/IssueQualityCore.ps1 (Yoda), which defined it first: + // factors, weights, patterns, findings and recommendations are transcribed from there, so both score a ticket the + // same. Epics use their own rubric; Change Requests add a rollback plan; Bugs need steps to reproduce. + "executionStrategy": "series", + "conditions": { + "isBug": { "type": "fieldMatches", "path": "$.issueType", "patterns": [ "Bug|Defect" ] }, + "isEpic": { "type": "fieldEquals", "path": "$.issueType", "values": [ "Epic" ] }, + "isChangeRequest": { "type": "fieldEquals", "path": "$.issueType", "values": [ "Change Request" ] }, + "isInvestigation": { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "ref", "name": "isEpic" } }, + { + "type": "any", + "conditions": [ + { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "ref", "name": "isBug" } }, + { + "type": "textMatches", + "source": { "paths": [ "$.title", "$.description" ] }, + "patterns": [ "\\binvestigate\\b", "\\bresearch\\b", "\\bspike\\b", "\\banalysis\\b", "\\bexplore\\b", "\\bfeasibility\\b", "\\bproof\\s+of\\s+concept\\b", "\\bpoc\\b", "\\bstudy\\b", "\\bevaluate\\b", "\\bassess\\b" ] + } + ] + }, + { "type": "fieldEquals", "path": "$.labels", "values": [ "investigation", "research", "spike" ] } + ] + } + ] + }, + "supportsComponents": { + "type": "any", + "conditions": [ + { "type": "fieldEquals", "path": "$.supportsComponents", "values": [ "true" ] }, + { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "fieldPresent", "path": "$.supportsComponents" } }, + { "type": "not", "condition": { "type": "fieldEquals", "path": "$.project", "values": [ "PS" ] } } + ] + } + ] + }, + "supportsFixVersion": { "type": "not", "condition": { "type": "fieldEquals", "path": "$.supportsFixVersion", "values": [ "false" ] } }, + "shouldHaveFixVersion": { "type": "fieldEquals", "path": "$.status", "values": [ "Ready for Test", "In Test", "Ready for Release", "Closed", "Done", "Resolved" ] }, + "hasFixVersion": { "type": "fieldPresent", "path": "$.fixVersions" }, + "hasEnvironmentMention": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "test2|alpha|beta|staging|production|environment" ] }, + "nonUiComponentsOnly": { + "type": "fieldMatches", + "path": "$.components", + "requireAll": true, + "patterns": [ "API", "API\\ Client", "Backend", "Database", "Data", "Connections", "Workers", "Worker", "Scheduler", "Schedules", "Infrastructure", "Security", "Authentication", "Migration", "Unit\\ tests", "Performance", "Configuration", "Logging" ] + }, + "uiConsistencyApplies": { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "ref", "name": "isBug" } }, + { "type": "not", "condition": { "type": "ref", "name": "isInvestigation" } }, + { "type": "fieldMatches", "path": "$.issueType", "patterns": [ "Feature|Improvement|Story|Enhancement" ] } + ] + }, + "isUiTicket": { + "type": "all", + "conditions": [ + { "type": "ref", "name": "uiConsistencyApplies" }, + { + "type": "any", + "conditions": [ + { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "ref", "name": "nonUiComponentsOnly" } }, + { "type": "fieldMatches", "path": "$.components", "patterns": [ "\\bUI\\b", "\\bFrontend\\b", "\\bFront-End\\b", "\\bWeb\\b", "\\bPortal\\b", "\\bDashboard\\b" ] } + ] + }, + { "type": "fieldMatches", "path": "$.labels", "patterns": [ "^ui$", "^frontend$", "^front-end$", "^web-ui$", "^ux$", "^visual$", "^design$" ] }, + { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "ref", "name": "nonUiComponentsOnly" } }, + { + "type": "measure", + "atLeast": 2, + "of": { + "type": "patternMatchCount", + "source": { "paths": [ "$.title", "$.description" ] }, + "patterns": [ "\\bUI\\b", "\\buser\\s+interface\\b", "\\bscreen\\b", "\\bdialog\\b", "\\bmodal\\b", "\\bform\\b", "\\bbutton\\b", "\\bpage\\b", "\\bpanel\\b", "\\bdashboard\\b", "\\bwidget\\b", "\\bview\\b", "\\blayout\\b", "\\btoolbar\\b", "\\bmenu\\b", "\\bdropdown\\b", "\\btab\\b", "\\bnavigation\\b", "\\bfront[- ]?end\\b", "\\bstyle\\b", "\\bCSS\\b", "\\btheme\\b", "\\bicon\\b", "\\btooltip\\b", "\\bpopup\\b", "\\bpopover\\b", "\\bgrid\\b", "\\btable\\b", "\\bcolumn\\b", "\\bheader\\b", "\\bfooter\\b", "\\bsidebar\\b" ] + } + } + ] + } + ] + } + ] + }, + "hasPanelableSections": { + "type": "all", + "conditions": [ + { "type": "fieldEquals", "path": "$.platform", "values": [ "Jira" ] }, + { "type": "measure", "atLeast": 2, "of": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "panelable" } } + ] + }, + "isCliRelated": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "magicsuite\\s+cli", "ms\\s+cli", "command[\\s-]?line", "msmagic", "cli\\s+command", "--\\w+", "invoke[\\s-]command" ] }, + "hasCliLabel": { "type": "fieldMatches", "path": "$.labels", "patterns": [ "CLI|Command" ] }, + "hasCliComponent": { "type": "fieldMatches", "path": "$.components", "patterns": [ "CLI" ] }, + "hasUiIndicators": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "button", "click", "dialog", "modal", "dropdown", "menu", "screen", "page", "form", "input\\s+field", "checkbox", "radio", "tab", "panel", "dashboard", "report\\s*studio", "ui\\s+", "user\\s+interface", "browser", "blazor" ] }, + "hasUiLabel": { "type": "fieldMatches", "path": "$.labels", "patterns": [ "^(UI|Blazor|ReportStudio)$" ] } + }, + "categories": [ + { + "name": "Ticket Quality", + "description": "Specification completeness of a bug, story, task or change request.", + "assessmentExecutionStrategy": "series", + "appliesWhen": { "type": "not", "condition": { "type": "ref", "name": "isEpic" } }, + "assessments": [ + { + "name": "Component", + "description": "The component the ticket concerns is identified.", + "weight": 14, + "appliesWhen": { "type": "ref", "name": "supportsComponents" }, + "notApplicableExplanation": "○ Component: Not applicable for the {$.project} project", + "definition": { + "type": "tiered", + "measures": { "components": { "type": "itemCount", "path": "$.components" } }, + "tiers": [ + { "when": { "type": "measure", "measure": "components", "atLeast": 1 }, "score": 100, "finding": "✓ Component: {$.components[0]}" }, + { "when": { "type": "fieldMatches", "path": "$.labels", "patterns": [ "DataMagic|ReportMagic|AlertMagic|Admin|Connect|CLI|Docs" ] }, "score": 70, "finding": "⚠ Component: Via label ({$.labels~DataMagic|ReportMagic|AlertMagic|Admin|Connect|CLI|Docs})", "recommendation": "Set component field" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.title" ] }, "patterns": [ "DataMagic|ReportMagic|AlertMagic|Admin|Connect|CLI|Docs|Report\\s*Studio" ] }, "score": 50, "finding": "⚠ Component: Mentioned in summary only", "recommendation": "Set component field explicitly" }, + { "score": 0, "finding": "✗ Component: Not identified", "recommendation": "Add component to issue" } + ] + } + }, + { + "name": "Description", + "description": "The description says enough.", + "weight": 14, + "definition": { + "type": "tiered", + "measures": { "words": { "type": "wordCount", "source": { "paths": [ "$.description" ] } } }, + "tiers": [ + { "when": { "type": "measure", "measure": "words", "atLeast": 100 }, "score": 100, "finding": "✓ Description: {words} words (comprehensive)" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 50 }, "score": 80, "finding": "✓ Description: {words} words (adequate)" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 20 }, "score": 50, "finding": "⚠ Description: {words} words (brief)", "recommendation": "Add more context to description" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 1 }, "score": 25, "finding": "⚠ Description: {words} words (minimal)", "recommendation": "Expand description with details" }, + { "score": 0, "finding": "✗ Description: Empty", "recommendation": "Add description" } + ] + } + }, + { + "name": "Testability", + "description": "Pass/fail criteria, or for an investigation its scope and deliverables, are stated.", + "weight": 16, + "definition": { + "type": "tiered", + "measures": { + "scope": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "scope:", "h[1-6]\\.[^\\r\\n]*scope", "\\*\\*[^\\r\\n]*scope", "questions?\\s+to\\s+(answer|investigate)", "investigate\\s+(whether|if|how|why)", "determine\\s+(whether|if|how|why)", "find\\s+out", "understand\\s+(how|why|what)" ] }, + "deliverables": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "deliverable", "findings", "recommendation", "report", "document", "conclusion", "outcome", "result\\s+of\\s+(investigation|research|analysis)" ] }, + "criteria": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "should\\s+(be|display|show|return|have|not)", "must\\s+(be|display|show|return|have|not)", "verify\\s+that", "acceptance\\s+criteria", "h[23]\\.\\s*acceptance\\s+criteria", "expected\\s+(result|behavior|behaviour|outcome)", "pass\\s+criteria", "success\\s+criteria" ] }, + "criteriaHeadings": { "type": "criteriaHeadingCount", "source": { "paths": [ "$.description" ] } } + }, + "tiers": [ + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "scope", "atLeast": 2 }, { "type": "measure", "measure": "deliverables", "atLeast": 1 } ] }, "score": 100, "finding": "✓ Investigation Clarity: Clear scope and deliverables" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "scope", "atLeast": 1 }, { "type": "measure", "measure": "deliverables", "atLeast": 1 } ] }, "score": 80, "finding": "✓ Investigation Clarity: Scope and deliverables defined" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "scope", "atLeast": 1 } ] }, "score": 50, "finding": "⚠ Investigation Clarity: Missing deliverables", "recommendation": "Define investigation deliverables (findings document, recommendation, report)" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "deliverables", "atLeast": 1 } ] }, "score": 50, "finding": "⚠ Investigation Clarity: Missing scope", "recommendation": "Define investigation scope and questions to answer" }, + { "when": { "type": "ref", "name": "isInvestigation" }, "score": 20, "finding": "✗ Investigation Clarity: No scope or deliverables", "recommendation": "Add investigation scope, questions to answer, and expected deliverables" }, + { "when": { "type": "measure", "measure": "criteria", "atLeast": 3 }, "score": 100, "finding": "✓ Testability: Clear criteria found" }, + { "when": { "type": "all", "conditions": [ { "type": "measure", "measure": "criteria", "atLeast": 1 }, { "type": "measure", "measure": "criteriaHeadings", "lessThan": 1 } ] }, "score": 60, "finding": "⚠ Testability: Some criteria found", "recommendation": "Add explicit pass/fail criteria" }, + { "when": { "type": "measure", "measure": "criteria", "atLeast": 1 }, "score": 60, "finding": "⚠ Testability: Some criteria found", "recommendation": "Make the existing acceptance criteria explicit pass/fail checks, rather than adding another section" }, + { "score": 20, "finding": "✗ Testability: No clear criteria", "recommendation": "Define acceptance criteria" } + ] + } + }, + { + "name": "StepsToReproduce", + "description": "A bug has numbered steps to reproduce under their own heading.", + "weight": 12, + "definition": { + "type": "tiered", + "measures": { + "steps": { "type": "occurrenceCount", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" }, "pattern": "(?m)^[ \\t]*(?:#(?!#)|\\d+[.)])[ \\t]+\\S" }, + "listedElsewhere": { "type": "occurrenceCount", "source": { "paths": [ "$.description" ] }, "pattern": "(?m)^[ \\t]*(?:#(?!#)|\\d+[.)])[ \\t]+\\S" }, + "sectionText": { "type": "alphanumericCount", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" } } + }, + "tiers": [ + { "when": { "type": "ref", "name": "isInvestigation" }, "score": 100, "finding": "○ Steps: Not required for investigations" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "not", "condition": { "type": "sectionPresent", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" } } }, { "type": "measure", "measure": "listedElsewhere", "atLeast": 2 } ] }, "score": 0, "finding": "✗ Steps to Reproduce: No steps-to-reproduce section (a numbered list was found, but not under a Steps to Reproduce heading)", "recommendation": "Add an 'h2. Steps to Reproduce' section with numbered steps (Jira '# step' or '1. step')" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "not", "condition": { "type": "sectionPresent", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" } } } ] }, "score": 0, "finding": "✗ Steps to Reproduce: No steps-to-reproduce section", "recommendation": "Add an 'h2. Steps to Reproduce' section with numbered steps (Jira '# step' or '1. step')" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "measure", "measure": "steps", "atLeast": 2 } ] }, "score": 100, "finding": "✓ Steps to Reproduce: {steps} numbered steps" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" }, "patterns": [ "(?i)\\b(?:not|never)\\s+(?:been\\s+)?(?:reliably\\s+|consistently\\s+)?reproduc|\\b(?:cannot|can't|could\\s+not|couldn't|unable\\s+to)\\s+(?:be\\s+)?(?:reliably\\s+|consistently\\s+)?reproduc|\\bintermittent" ] }, { "type": "measure", "measure": "sectionText", "atLeast": 60 } ] }, "score": 60, "finding": "⚠ Steps to Reproduce: Stated as not reproducible on demand, with observed conditions", "recommendation": "Add numbered steps once a reliable reproduction is found; until then the ticket stays at GOOD" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" }, "patterns": [ "(?i)\\b(?:not|never)\\s+(?:been\\s+)?(?:reliably\\s+|consistently\\s+)?reproduc|\\b(?:cannot|can't|could\\s+not|couldn't|unable\\s+to)\\s+(?:be\\s+)?(?:reliably\\s+|consistently\\s+)?reproduc|\\bintermittent" ] } ] }, "score": 30, "finding": "⚠ Steps to Reproduce: Stated as not reproducible, with no observed conditions", "recommendation": "Describe when it was seen: environment, frequency, time, data and what was being done" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "measure", "measure": "steps", "atLeast": 1 } ] }, "score": 60, "finding": "⚠ Steps to Reproduce: Only one numbered step", "recommendation": "Break the reproduction into numbered steps from a known starting state" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" } }, { "type": "measure", "measure": "sectionText", "lessThan": 15 } ] } ] }, "score": 0, "finding": "✗ Steps to Reproduce: Heading present but the section is empty or a placeholder", "recommendation": "Fill in the steps to reproduce - a heading over 'TBD' does not say how to see the bug" }, + { "when": { "type": "ref", "name": "isBug" }, "score": 40, "finding": "⚠ Steps to Reproduce: Section present but the steps are not numbered", "recommendation": "Rewrite the steps to reproduce as a numbered list" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "steps?\\s*(to\\s+reproduce|:)", "(?m)^\\s*\\d+\\.\\s+", "(?m)^\\s*#\\s+", "(?m)^\\s*-\\s+", "h[23]\\.\\s*steps?\\s*(to\\s+reproduce)?", "how\\s+to\\s+reproduce", "reproduction\\s+steps", "repro\\s+steps" ] }, "score": 100, "finding": "✓ Steps: Workflow documented" }, + { "score": 100, "finding": "○ Steps: Not required for this type" } + ] + } + }, + { + "name": "CurrentVsExpected", + "description": "Current and expected behaviour are documented, or for an investigation its background.", + "weight": 12, + "definition": { + "type": "tiered", + "measures": { + "context": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "background:", "h[1-6]\\.[^\\r\\n]*background", "context:", "h[1-6]\\.[^\\r\\n]*context", "\\*\\*[^\\r\\n]*(background|context)", "motivation:", "why\\s+it\\s+matters", "why\\s+(we|this)", "reason\\s+for", "because" ] }, + "current": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "current(ly)?(\\s+behavior|\\s+behaviour)?:", "h[23]\\.\\s*current(\\s*/\\s*actual)?\\s*(behavior|behaviour)", "\\*\\*current\\s*(behavior|behaviour)\\*\\*", "\\*current(\\s*/\\s*actual)?\\s*(behavior|behaviour)?\\*", "\\*actual\\s*(behavior|behaviour)?\\*", "actual(\\s+behavior|\\s+behaviour)?:", "what\\s+happens:", "observed:", "bug:", "issue:", "problem:" ] }, + "expected": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "expected(\\s+behavior|\\s+behaviour)?:", "h[23]\\.\\s*expected\\s*(behavior|behaviour)", "\\*\\*expected\\s*(behavior|behaviour)\\*\\*", "\\*expected\\s*(behavior|behaviour)?\\*", "should\\s+be:", "desired:", "correct\\s+behavior:", "fix:", "solution:", "h2\\.\\s*acceptance\\s+criteria", "h3\\.\\s*acceptance\\s+criteria", "acceptance\\s+criteria:", "\\*\\*acceptance\\s+criteria\\*\\*" ] } + }, + "tiers": [ + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "context", "atLeast": 2 } ] }, "score": 100, "finding": "✓ Investigation Context: Well documented" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "context", "atLeast": 1 } ] }, "score": 70, "finding": "⚠ Investigation Context: Partial context", "recommendation": "Add more background/motivation for the investigation" }, + { "when": { "type": "ref", "name": "isInvestigation" }, "score": 30, "finding": "⚠ Investigation Context: Missing context", "recommendation": "Add background and motivation for why this investigation is needed" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "measure", "measure": "current", "atLeast": 1 }, { "type": "measure", "measure": "expected", "atLeast": 1 } ] }, "score": 100, "finding": "✓ Current vs Expected: Both documented" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "measure", "measure": "current", "atLeast": 1 } ] }, "score": 50, "finding": "⚠ Current vs Expected: Missing expected behavior", "recommendation": "Document both current and expected behavior" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "measure", "measure": "expected", "atLeast": 1 } ] }, "score": 50, "finding": "⚠ Current vs Expected: Missing current behavior", "recommendation": "Document both current and expected behavior" }, + { "when": { "type": "ref", "name": "isBug" }, "score": 0, "finding": "✗ Current vs Expected: Neither documented", "recommendation": "Add current behavior and expected behavior sections" }, + { "when": { "type": "measure", "measure": "expected", "atLeast": 1 }, "score": 100, "finding": "✓ Expected Behavior: Documented" }, + { "score": 40, "finding": "⚠ Expected Behavior: Not explicit", "recommendation": "Clarify expected outcome" } + ] + } + }, + { + "name": "InvestigationTimebox", + "description": "An investigation states how long it may take. Recommendation only; it carries no weight.", + "weight": 0, + "appliesWhen": { "type": "ref", "name": "isInvestigation" }, + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "timebox", "time-?box", "\\d+\\s*(hour|day|week)s?", "maximum\\s+time", "time\\s+limit" ] }, "score": 100, "finding": "" }, + { "score": 0, "finding": "", "recommendation": "Consider adding a timebox for the investigation" } + ] + } + }, + { + "name": "EnvironmentVersion", + "description": "The fix version and the environment are stated.", + "weight": 7, + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "not", "condition": { "type": "ref", "name": "supportsFixVersion" } }, "score": 100, "finding": "○ Fix Version: Not applicable on {$.platform}" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "hasFixVersion" }, { "type": "ref", "name": "hasEnvironmentMention" } ] }, "score": 100, "finding": "✓ Fix Version: {$.fixVersions[0]}" }, + { "when": { "type": "ref", "name": "hasFixVersion" }, "score": 70, "finding": "✓ Fix Version: {$.fixVersions[0]}" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "shouldHaveFixVersion" }, { "type": "ref", "name": "hasEnvironmentMention" } ] }, "score": 30, "finding": "⚠ Fix Version: Not specified (expected for '{$.status}' status)", "recommendation": "Add fix version" }, + { "when": { "type": "ref", "name": "shouldHaveFixVersion" }, "score": 0, "finding": "⚠ Fix Version: Not specified (expected for '{$.status}' status)", "recommendation": "Add fix version" }, + { "score": 100, "finding": "○ Fix Version: Not required yet (status: {$.status})" } + ] + } + }, + { + "name": "Attachments", + "description": "A bug carries a screenshot or log.", + "weight": 3, + "definition": { + "type": "tiered", + "measures": { "attachments": { "type": "number", "path": "$.attachmentsCount" } }, + "tiers": [ + { "when": { "type": "measure", "measure": "attachments", "atLeast": 1 }, "score": 100, "finding": "✓ Attachments: {attachments} item(s)" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.description" ] }, "patterns": [ "!\\[|screenshot|image|http.*\\.(png|jpg|gif)" ] }, "score": 70, "finding": "✓ Attachments: Inline images found" }, + { "when": { "type": "ref", "name": "isBug" }, "score": 0, "finding": "⚠ Attachments: None (recommended for bugs)", "recommendation": "Add screenshot or error log" }, + { "score": 100, "finding": "○ Attachments: None (optional for non-bugs)" } + ] + } + }, + { + "name": "Documentation", + "description": "Documentation or help updates are referenced.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "references": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "documentation|docs?", "help\\s+(file|topic|section)", "user\\s+guide", "knowledge\\s+base|kb\\s+article", "wiki|confluence", "readme|readme\\.md", "api\\s+(reference|documentation|docs)", "manual", "guide|tutorial" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "references", "atLeast": 2 }, "score": 100, "finding": "✓ Documentation: Multiple references found" }, + { "when": { "type": "measure", "measure": "references", "atLeast": 1 }, "score": 70, "finding": "✓ Documentation: Referenced", "recommendation": "Consider adding more documentation updates" }, + { "score": 0, "finding": "⚠ Documentation: Not referenced", "recommendation": "Update documentation/help text for this change" } + ] + } + }, + { + "name": "Certifications", + "description": "A new feature has a certification exam question.", + "weight": 4, + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "ref", "name": "isBug" }, "score": 100, "finding": "○ Certifications: Not applicable for bugs" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "certifica?tion|exam|certification\\s+question", "cert\\s+question", "assessment\\s+question", "test\\s+question" ] }, "score": 100, "finding": "✓ Certifications: Exam question(s) created" }, + { "score": 0, "finding": "○ Certifications: No exam question created", "recommendation": "Consider creating a certification exam question for this new feature" } + ] + } + }, + { + "name": "UIConsistency", + "description": "A UI change references the look and feel of similar features, or says there are none.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "references": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "consistent\\s+(with|look|feel|style|design|behaviour|behavior)", "look\\s+and\\s+feel", "match(es|ing)?\\s+(the|existing|current|other|similar)", "similar\\s+(to|feature|screen|page|dialog|panel|view|component)", "same\\s+(style|approach|pattern|design|way)\\s+(as|used)", "in\\s+line\\s+with", "align(ed)?\\s+with", "follows?\\s+(the|existing|current)\\s+(pattern|convention|style|design)", "precedent", "no\\s+similar\\s+(feature|screen|page|component|view)", "no\\s+existing\\s+(feature|screen|page|component|view)", "first\\s+of\\s+(its|this)\\s+kind", "new\\s+pattern", "no\\s+precedent", "h[23]\\.\\s*UI\\s+Consistency", "\\*\\*UI\\s+Consistency\\*\\*", "UI\\s+consistency:" ] } }, + "tiers": [ + { "when": { "type": "not", "condition": { "type": "ref", "name": "uiConsistencyApplies" } }, "score": 100, "finding": "○ UI Consistency: Not applicable for {$.issueType}" }, + { "when": { "type": "not", "condition": { "type": "ref", "name": "isUiTicket" } }, "score": 100, "finding": "○ UI Consistency: Not a UI ticket" }, + { "when": { "type": "measure", "measure": "references", "atLeast": 2 }, "score": 100, "finding": "✓ UI Consistency: Look & feel references with similar features" }, + { "when": { "type": "measure", "measure": "references", "atLeast": 1 }, "score": 70, "finding": "⚠ UI Consistency: Partial reference to similar features", "recommendation": "Expand UI consistency details - reference specific similar features or state there are none" }, + { "score": 0, "finding": "✗ UI Consistency: No reference to look & feel or similar features", "recommendation": "Add UI consistency details or explicitly state no similar features exist" } + ] + } + }, + { + "name": "BusinessCase", + "description": "The value, impact or cost of not doing the work is stated.", + "weight": 12, + "definition": { + "type": "tiered", + "measures": { "justification": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "business\\s+case", "business\\s+value", "business\\s+justification", "business\\s+impact", "business\\s+benefit", "cost\\s+of\\s+(not\\s+doing|inaction|delay)", "return\\s+on\\s+investment", "\\bROI\\b", "value\\s+proposition", "revenue\\s+impact", "customer\\s+impact", "user\\s+impact", "compliance\\s+requirement", "regulatory\\s+requirement", "risk\\s+(reduction|mitigation)", "operational\\s+(benefit|efficiency|saving)", "h[23]\\.\\s*(Business\\s+Case|Business\\s+Value|Business\\s+Justification|Motivation|Rationale)", "\\*\\*(Business\\s+Case|Business\\s+Value|Justification|Motivation|Rationale)\\*\\*", "why\\s+(this|we)\\s+(is|are|need|should|must)", "justif(y|ication|ied)", "rationale" ] } }, + "tiers": [ + { "when": { "type": "ref", "name": "isBug" }, "score": 100, "finding": "○ Business Case: Not required for bugs" }, + { "when": { "type": "ref", "name": "isInvestigation" }, "score": 100, "finding": "○ Business Case: Not required for investigations" }, + { "when": { "type": "measure", "measure": "justification", "atLeast": 3 }, "score": 100, "finding": "✓ Business Case: Clear justification provided" }, + { "when": { "type": "measure", "measure": "justification", "atLeast": 2 }, "score": 80, "finding": "✓ Business Case: Justification referenced" }, + { "when": { "type": "measure", "measure": "justification", "atLeast": 1 }, "score": 50, "finding": "⚠ Business Case: Brief justification", "recommendation": "Expand business case with impact, value, or cost of inaction" }, + { "score": 0, "finding": "✗ Business Case: Not provided", "recommendation": "Add a business case explaining the value, impact, or cost of not doing this work" } + ] + } + }, + { + "name": "RollbackPlan", + "description": "A change request documents how the change is undone.", + "weight": 34, + "appliesWhen": { "type": "ref", "name": "isChangeRequest" }, + "definition": { + "type": "tiered", + "measures": { "planText": { "type": "alphanumericCount", "source": { "paths": [ "$.description" ], "section": "(?:rollback|mitigation)" } } }, + "tiers": [ + { "when": { "type": "not", "condition": { "type": "sectionPresent", "source": { "paths": [ "$.description" ], "section": "(?:rollback|mitigation)" } } }, "score": 0, "finding": "✗ Rollback Plan: No rollback or mitigation section", "recommendation": "Add a 'Rollback plan' section: what is captured before the change, what reverses it, and what is not reversible" }, + { "when": { "type": "placeholder", "source": { "paths": [ "$.description" ], "section": "(?:rollback|mitigation)" } }, "score": 0, "finding": "✗ Rollback Plan: Heading present but the section is empty or a placeholder", "recommendation": "Fill in the rollback plan - a heading over 'TBD' does not describe how the change is undone" }, + { "when": { "type": "measure", "measure": "planText", "lessThan": 15 }, "score": 0, "finding": "✗ Rollback Plan: Section too short to describe anything", "recommendation": "Expand the rollback plan to state what reverses the change and what cannot be reversed" }, + { "when": { "type": "measure", "measure": "planText", "lessThan": 120 }, "score": 60, "finding": "⚠ Rollback Plan: Present but brief", "recommendation": "Expand the rollback plan: name what is captured beforehand, and say what is irreversible and what compensates for it" }, + { "score": 100, "finding": "✓ Rollback Plan: Documented" } + ] + } + }, + { + "name": "Presentation", + "description": "Each section sits in an untitled panel directly under its h2 heading, so the layout can be scanned and found again; sections holding a table or code block are exempt.", + "weight": 8, + "appliesWhen": { "type": "ref", "name": "hasPanelableSections" }, + "definition": { + "type": "tiered", + "measures": { + "panelable": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "panelable" }, + "panelled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "panelled" }, + "unpanelled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "unpanelled" }, + "titled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "titled" } + }, + "tiers": [ + { "when": { "type": "all", "conditions": [ { "type": "measure", "measure": "unpanelled", "lessThan": 1 }, { "type": "measure", "measure": "titled", "lessThan": 1 } ] }, "score": 100, "finding": "✓ Presentation: {panelled} sections in untitled panels" }, + { "when": { "type": "measure", "measure": "unpanelled", "lessThan": 1 }, "score": 70, "finding": "⚠ Presentation: {titled} panel titles repeat their heading ({titled:text})", "recommendation": "Remove the title from each panel that sits directly under a heading, since the heading already names it: keep only the background colour, as in panel:bgColor=#F4F5F7" }, + { "when": { "type": "measure", "measure": "panelled", "atLeast": 1 }, "score": 50, "finding": "⚠ Presentation: {unpanelled} of {panelable} sections are not in a panel ({unpanelled:text})", "recommendation": "Put the content of each section in an untitled panel directly under its heading; a section holding a table or code block can stay as it is" }, + { "score": 20, "finding": "✗ Presentation: None of {panelable} sections is in a panel ({unpanelled:text})", "recommendation": "Put the content of each section in an untitled panel directly under its heading, for example one with only a background colour, panel:bgColor=#F4F5F7; a section holding a table or code block can stay as it is" } + ] + } + }, + { + "name": "CriteriaDuplication", + "description": "Acceptance criteria are stated under one heading only. It carries no weight; duplication caps the score instead.", + "weight": 0, + "definition": { + "type": "tiered", + "measures": { "headings": { "type": "criteriaHeadingCount", "source": { "paths": [ "$.description" ] } } }, + "tiers": [ + { "when": { "type": "measure", "measure": "headings", "atLeast": 2 }, "score": 0, "finding": "✗ Duplication: acceptance criteria stated under {headings} headings ({headings:text}), so EXCELLENT is unreachable (score capped at 79)", "recommendation": "State the acceptance criteria once, under one heading, and remove the others" }, + { "score": 100, "finding": "" } + ] + } + } + ] + }, + { + "name": "Epic Quality", + "description": "Specification completeness of an epic: design, breakdown, scope and success criteria.", + "assessmentExecutionStrategy": "series", + "appliesWhen": { "type": "ref", "name": "isEpic" }, + "assessments": [ + { + "name": "EpicComponent", + "description": "The component the epic concerns is set.", + "weight": 10, + "appliesWhen": { "type": "ref", "name": "supportsComponents" }, + "notApplicableExplanation": "○ Component: Not applicable for the {$.project} project", + "definition": { + "type": "tiered", + "measures": { "components": { "type": "itemCount", "path": "$.components" } }, + "tiers": [ + { "when": { "type": "measure", "measure": "components", "atLeast": 1 }, "score": 100, "finding": "✓ Component: {$.components[0]}" }, + { "score": 0, "finding": "✗ Component: Not set", "recommendation": "Set the component field" } + ] + } + }, + { + "name": "EpicDescription", + "description": "The epic's description states the problem and the solution.", + "weight": 18, + "definition": { + "type": "tiered", + "measures": { "words": { "type": "wordCount", "source": { "paths": [ "$.description" ] } } }, + "tiers": [ + { "when": { "type": "measure", "measure": "words", "atLeast": 200 }, "score": 100, "finding": "✓ Description: {words} words (comprehensive)" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 100 }, "score": 80, "finding": "✓ Description: {words} words (good)" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 50 }, "score": 50, "finding": "⚠ Description: {words} words (brief for an epic)", "recommendation": "Expand description with problem statement and solution overview" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 1 }, "score": 25, "finding": "⚠ Description: {words} words (minimal)", "recommendation": "Add comprehensive problem statement and solution overview" }, + { "score": 0, "finding": "✗ Description: Empty", "recommendation": "Add description with problem statement and solution overview" } + ] + } + }, + { + "name": "Architecture", + "description": "The technical design is described.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "design": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "architect(ure|ural)", "design\\s+(overview|decision|approach)", "technical\\s+(design|approach|solution|overview)", "flow\\s+diagram", "sequence\\s+diagram", "data\\s+flow", "system\\s+design", "component\\s+diagram", "integration\\s+point", "API\\s+(design|contract|endpoint)", "schema\\s+(design|change)", "database\\s+(design|change|migration)", "h[23]\\.\\s*(Technical|Architecture|Design)", "\\*\\*(Technical|Architecture|Design)\\s+(Design|Overview|Approach)\\*\\*" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "design", "atLeast": 3 }, "score": 100, "finding": "✓ Architecture: Comprehensive technical design" }, + { "when": { "type": "measure", "measure": "design", "atLeast": 2 }, "score": 80, "finding": "✓ Architecture: Technical approach documented" }, + { "when": { "type": "measure", "measure": "design", "atLeast": 1 }, "score": 50, "finding": "⚠ Architecture: Brief technical mention", "recommendation": "Add technical design details (approach, diagrams, integration points)" }, + { "score": 0, "finding": "✗ Architecture: No technical design", "recommendation": "Add architecture/technical design section" } + ] + } + }, + { + "name": "Decomposition", + "description": "The work is broken down into child tickets or phases.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "breakdown": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "child\\s+(ticket|issue|task|story)", "sub-task", "breakdown", "decompos(e|ed|ition)", "phase\\s*\\d", "step\\s*\\d", "work\\s*(stream|package|item)", "MS-\\d{4,5}", "implementation\\s+order", "dependencies", "h[23]\\.\\s*(Breakdown|Decomposition|Child|Sub-tasks|Implementation\\s+Order|Work\\s+Items)", "\\*\\*(Breakdown|Decomposition|Child|Sub-tasks|Implementation\\s+Order)\\*\\*" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "breakdown", "atLeast": 3 }, "score": 100, "finding": "✓ Decomposition: Clear work breakdown" }, + { "when": { "type": "measure", "measure": "breakdown", "atLeast": 2 }, "score": 80, "finding": "✓ Decomposition: Work items referenced" }, + { "when": { "type": "measure", "measure": "breakdown", "atLeast": 1 }, "score": 50, "finding": "⚠ Decomposition: Partial breakdown", "recommendation": "Add clear work breakdown with child tickets or phases" }, + { "score": 0, "finding": "✗ Decomposition: No work breakdown", "recommendation": "Decompose into child tickets or define implementation phases" } + ] + } + }, + { + "name": "ScopeDefinition", + "description": "What is in and out of scope is defined.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "scope": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "in\\s+scope", "out\\s+of\\s+scope", "scope\\s+(of|definition|boundary|boundaries)", "not\\s+included", "excluded", "boundaries", "goal(s)?", "objective(s)?", "non-goal", "h[23]\\.\\s*(Scope|Goal|Objective|Boundaries|Non-Goal|Out of Scope)", "\\*\\*(Scope|Goal|Objective|In Scope|Out of Scope|Non-Goal)\\*\\*" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "scope", "atLeast": 3 }, "score": 100, "finding": "✓ Scope: Well-defined boundaries" }, + { "when": { "type": "measure", "measure": "scope", "atLeast": 2 }, "score": 80, "finding": "✓ Scope: Goals documented" }, + { "when": { "type": "measure", "measure": "scope", "atLeast": 1 }, "score": 50, "finding": "⚠ Scope: Partially defined", "recommendation": "Clarify what is in scope and out of scope" }, + { "score": 0, "finding": "✗ Scope: Not defined", "recommendation": "Add scope boundaries (in scope, out of scope, goals)" } + ] + } + }, + { + "name": "AcceptanceCriteria", + "description": "The epic's success criteria and definition of done are stated.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "criteria": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "acceptance\\s+criteria", "success\\s+criteria", "done\\s+when", "definition\\s+of\\s+done", "complete\\s+when", "considered\\s+complete", "epic\\s+(is\\s+)?complete", "milestone", "deliverable(s)?", "h[23]\\.\\s*(Acceptance|Success|Done|Definition of Done|Deliverables)", "\\*\\*(Acceptance|Success)\\s+Criteria\\*\\*" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "criteria", "atLeast": 2 }, "score": 100, "finding": "✓ Acceptance Criteria: High-level success criteria defined" }, + { "when": { "type": "measure", "measure": "criteria", "atLeast": 1 }, "score": 70, "finding": "⚠ Acceptance Criteria: Partially defined", "recommendation": "Add clearer definition of done for this epic" }, + { "score": 0, "finding": "✗ Acceptance Criteria: Not defined", "recommendation": "Add high-level acceptance/success criteria and definition of done" } + ] + } + }, + { + "name": "FixVersion", + "description": "The epic's fix version is set once it is expected.", + "weight": 5, + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "not", "condition": { "type": "ref", "name": "supportsFixVersion" } }, "score": 100, "finding": "○ Fix Version: Not applicable on {$.platform}" }, + { "when": { "type": "ref", "name": "hasFixVersion" }, "score": 100, "finding": "✓ Fix Version: {$.fixVersions[0]}" }, + { "when": { "type": "ref", "name": "shouldHaveFixVersion" }, "score": 0, "finding": "⚠ Fix Version: Not set (expected for '{$.status}' status)", "recommendation": "Add fix version" }, + { "score": 100, "finding": "○ Fix Version: Not required yet (status: {$.status})" } + ] + } + }, + { + "name": "EpicAttachments", + "description": "The epic has diagrams or mockups.", + "weight": 5, + "definition": { + "type": "tiered", + "measures": { "attachments": { "type": "number", "path": "$.attachmentsCount" } }, + "tiers": [ + { "when": { "type": "measure", "measure": "attachments", "atLeast": 1 }, "score": 100, "finding": "✓ Attachments: {attachments} item(s)" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.description" ] }, "patterns": [ "!\\[|screenshot|image|diagram|http.*\\.(png|jpg|gif)" ] }, "score": 70, "finding": "✓ Attachments: Inline images/diagrams found" }, + { "score": 30, "finding": "⚠ Attachments: None (diagrams/mockups recommended for epics)", "recommendation": "Consider adding architecture diagrams or mockups" } + ] + } + }, + { + "name": "EpicPresentation", + "description": "Each section sits in an untitled panel directly under its h2 heading, so the layout can be scanned and found again; sections holding a table or code block are exempt.", + "weight": 8, + "appliesWhen": { "type": "ref", "name": "hasPanelableSections" }, + "definition": { + "type": "tiered", + "measures": { + "panelable": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "panelable" }, + "panelled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "panelled" }, + "unpanelled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "unpanelled" }, + "titled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "titled" } + }, + "tiers": [ + { "when": { "type": "all", "conditions": [ { "type": "measure", "measure": "unpanelled", "lessThan": 1 }, { "type": "measure", "measure": "titled", "lessThan": 1 } ] }, "score": 100, "finding": "✓ Presentation: {panelled} sections in untitled panels" }, + { "when": { "type": "measure", "measure": "unpanelled", "lessThan": 1 }, "score": 70, "finding": "⚠ Presentation: {titled} panel titles repeat their heading ({titled:text})", "recommendation": "Remove the title from each panel that sits directly under a heading, since the heading already names it: keep only the background colour, as in panel:bgColor=#F4F5F7" }, + { "when": { "type": "measure", "measure": "panelled", "atLeast": 1 }, "score": 50, "finding": "⚠ Presentation: {unpanelled} of {panelable} sections are not in a panel ({unpanelled:text})", "recommendation": "Put the content of each section in an untitled panel directly under its heading; a section holding a table or code block can stay as it is" }, + { "score": 20, "finding": "✗ Presentation: None of {panelable} sections is in a panel ({unpanelled:text})", "recommendation": "Put the content of each section in an untitled panel directly under its heading, for example one with only a background colour, panel:bgColor=#F4F5F7; a section holding a table or code block can stay as it is" } + ] + } + }, + { + "name": "EpicBusinessCase", + "description": "Why the epic is valuable is stated.", + "weight": 10, + "definition": { + "type": "tiered", + "measures": { "justification": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "business\\s+case", "business\\s+value", "business\\s+justification", "business\\s+impact", "cost\\s+of\\s+(not\\s+doing|inaction|delay)", "return\\s+on\\s+investment", "\\bROI\\b", "value\\s+proposition", "strategic\\s+(value|benefit|alignment)", "revenue\\s+impact", "customer\\s+impact", "compliance\\s+requirement", "risk\\s+(reduction|mitigation)", "operational\\s+(benefit|efficiency|saving)", "h[23]\\.\\s*(Business\\s+Case|Business\\s+Value|Business\\s+Justification|Motivation|Rationale)", "\\*\\*(Business\\s+Case|Business\\s+Value|Justification|Motivation|Rationale)\\*\\*", "justif(y|ication|ied)", "rationale", "why\\s+(this|we)\\s+(is|are|need|should|must)" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "justification", "atLeast": 3 }, "score": 100, "finding": "✓ Business Case: Clear justification provided" }, + { "when": { "type": "measure", "measure": "justification", "atLeast": 2 }, "score": 80, "finding": "✓ Business Case: Justification referenced" }, + { "when": { "type": "measure", "measure": "justification", "atLeast": 1 }, "score": 50, "finding": "⚠ Business Case: Brief justification", "recommendation": "Expand business case with impact, value, or cost of inaction" }, + { "score": 0, "finding": "✗ Business Case: Not provided", "recommendation": "Add business case explaining why this epic is valuable" } + ] + } + } + ] + } + ], + "scoreCaps": [ + { + "name": "Criteria duplication", + "group": "duplication", + "cap": 0.79, + "when": { "type": "assessmentScore", "assessment": "CriteriaDuplication", "lessThan": 100 } + }, + { + "name": "Steps to reproduce", + "group": "mandatory", + "cap": 0.79, + "when": { + "type": "all", + "conditions": [ + { "type": "ref", "name": "isBug" }, + { "type": "not", "condition": { "type": "ref", "name": "isInvestigation" } }, + { "type": "assessmentScore", "assessment": "StepsToReproduce", "lessThan": 100 } + ] + } + }, + { + "name": "Rollback plan", + "group": "mandatory", + "cap": 0.79, + "when": { "type": "assessmentScore", "assessment": "RollbackPlan", "lessThan": 1 } + } + ], + "gradeBands": [ + { "name": "EXCELLENT", "minimumScore": 0.8 }, + { "name": "GOOD", "minimumScore": 0.6 }, + { "name": "ADEQUATE", "minimumScore": 0.5 }, + { "name": "NEEDS IMPROVEMENT", "minimumScore": 0.3 }, + { "name": "POOR", "minimumScore": 0 } + ], + "classifications": { + "TestAutomationType": [ + { "when": { "type": "ref", "name": "isEpic" }, "value": "None" }, + { + "when": { + "type": "all", + "conditions": [ + { "type": "any", "conditions": [ { "type": "ref", "name": "isCliRelated" }, { "type": "ref", "name": "hasCliLabel" }, { "type": "ref", "name": "hasCliComponent" } ] }, + { "type": "not", "condition": { "type": "ref", "name": "hasUiIndicators" } }, + { "type": "not", "condition": { "type": "ref", "name": "hasUiLabel" } } + ] + }, + "value": "Ai" + }, + { + "when": { + "type": "all", + "conditions": [ + { "type": "any", "conditions": [ { "type": "ref", "name": "hasUiIndicators" }, { "type": "ref", "name": "hasUiLabel" } ] }, + { "type": "any", "conditions": [ { "type": "ref", "name": "isCliRelated" }, { "type": "ref", "name": "hasCliLabel" } ] } + ] + }, + "value": "Hybrid" + }, + { "when": { "type": "any", "conditions": [ { "type": "ref", "name": "hasUiIndicators" }, { "type": "ref", "name": "hasUiLabel" } ] }, "value": "Human" }, + { "value": "None" } + ] + } +} diff --git a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json index 47830d0..5507f9f 100644 --- a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json +++ b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json @@ -8,6 +8,11 @@ "isBug": { "type": "fieldMatches", "path": "$.issueType", "patterns": [ "Bug|Defect" ] }, "isEpic": { "type": "fieldEquals", "path": "$.issueType", "values": [ "Epic" ] }, "isChangeRequest": { "type": "fieldEquals", "path": "$.issueType", "values": [ "Change Request" ] }, + "isMs": { "type": "fieldEquals", "path": "$.project", "values": [ "MS" ] }, + "isUndelivered": { + "type": "all", + "conditions": [ { "type": "fieldPresent", "path": "$.resolution" }, { "type": "not", "condition": { "type": "fieldEquals", "path": "$.resolution", "values": [ "Fixed", "Done" ] } } ] + }, "isInvestigation": { "type": "all", "conditions": [ @@ -366,6 +371,27 @@ ] } }, + { + "name": "ReleaseNotes", + "description": "MS tickets carry customer-facing release notes, written for the person who reads the release notes, not for the team.", + "weight": 6, + "appliesWhen": { "type": "all", "conditions": [ { "type": "ref", "name": "isMs" }, { "type": "not", "condition": { "type": "ref", "name": "isUndelivered" } } ] }, + "notApplicableExplanation": "○ Release Notes: Not applicable (not an MS ticket, or not delivered)", + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "measure", "of": { "type": "wordCount", "source": { "paths": [ "$.releaseNotes" ] } }, "lessThan": 8 }, "score": 0, "finding": "✗ Release Notes: Under 8 words", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "not", "condition": { "type": "fieldsDiffer", "left": "$.releaseNotes", "right": "$.title", "ignorePrefixPattern": "^[^:]{1,40}:\\s*" } }, "score": 0, "finding": "✗ Release Notes: Same as the summary", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "\\u2014" ] }, "score": 0, "finding": "✗ Release Notes: Contains an em dash", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)\\b(behavior|color|organization|initialize|customize|analyze|center|favorite|canceled)s?\\b" ] }, "score": 0, "finding": "✗ Release Notes: US spelling", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "score": 100, "finding": "✓ Release Notes: Present and customer-facing" } + ] + } + }, { "name": "CriteriaDuplication", "description": "Acceptance criteria are stated under one heading only. It carries no weight; duplication caps the score instead.", @@ -526,6 +552,27 @@ ] } }, + { + "name": "EpicReleaseNotes", + "description": "MS tickets carry customer-facing release notes, written for the person who reads the release notes, not for the team.", + "weight": 6, + "appliesWhen": { "type": "all", "conditions": [ { "type": "ref", "name": "isMs" }, { "type": "not", "condition": { "type": "ref", "name": "isUndelivered" } } ] }, + "notApplicableExplanation": "○ Release Notes: Not applicable (not an MS ticket, or not delivered)", + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "measure", "of": { "type": "wordCount", "source": { "paths": [ "$.releaseNotes" ] } }, "lessThan": 8 }, "score": 0, "finding": "✗ Release Notes: Under 8 words", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "not", "condition": { "type": "fieldsDiffer", "left": "$.releaseNotes", "right": "$.title", "ignorePrefixPattern": "^[^:]{1,40}:\\s*" } }, "score": 0, "finding": "✗ Release Notes: Same as the summary", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "\\u2014" ] }, "score": 0, "finding": "✗ Release Notes: Contains an em dash", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)\\b(behavior|color|organization|initialize|customize|analyze|center|favorite|canceled)s?\\b" ] }, "score": 0, "finding": "✗ Release Notes: US spelling", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "score": 100, "finding": "✓ Release Notes: Present and customer-facing" } + ] + } + }, { "name": "EpicBusinessCase", "description": "Why the epic is valuable is stated.", @@ -564,6 +611,15 @@ ] } }, + { + "name": "Release notes", + "group": "mandatory", + "cap": 0.79, + "when": { + "type": "any", + "conditions": [ { "type": "assessmentScore", "assessment": "ReleaseNotes", "lessThan": 100 }, { "type": "assessmentScore", "assessment": "EpicReleaseNotes", "lessThan": 100 } ] + } + }, { "name": "Rollback plan", "group": "mandatory", diff --git a/PanoramicData.IssueAssessment/Rules/Conditions.cs b/PanoramicData.IssueAssessment/Rules/Conditions.cs index 7a51b11..fc567a5 100644 --- a/PanoramicData.IssueAssessment/Rules/Conditions.cs +++ b/PanoramicData.IssueAssessment/Rules/Conditions.cs @@ -18,6 +18,7 @@ namespace PanoramicData.IssueAssessment.Rules; [JsonDerivedType(typeof(FieldMatchesCondition), "fieldMatches")] [JsonDerivedType(typeof(FieldEqualsCondition), "fieldEquals")] [JsonDerivedType(typeof(FieldPresentCondition), "fieldPresent")] +[JsonDerivedType(typeof(FieldsDifferCondition), "fieldsDiffer")] [JsonDerivedType(typeof(SectionPresentCondition), "sectionPresent")] [JsonDerivedType(typeof(PlaceholderCondition), "placeholder")] [JsonDerivedType(typeof(AssessmentScoreCondition), "assessmentScore")] diff --git a/PanoramicData.IssueAssessment/Rules/FieldsDifferCondition.cs b/PanoramicData.IssueAssessment/Rules/FieldsDifferCondition.cs new file mode 100644 index 0000000..f2ac790 --- /dev/null +++ b/PanoramicData.IssueAssessment/Rules/FieldsDifferCondition.cs @@ -0,0 +1,36 @@ +namespace PanoramicData.IssueAssessment.Rules; + +/// +/// True when the text at differs from the text at after trimming, ignoring case +/// and removing from the start of each. A missing value reads as empty. +/// +public sealed record FieldsDifferCondition : ICondition +{ + /// Gets the JSONPath of the first text. + public required string Left { get; init; } + + /// Gets the JSONPath of the second text. + public required string Right { get; init; } + + /// Gets an optional regular expression removed from the start of each text before comparing. + public string? IgnorePrefixPattern { get; init; } + + /// + public bool Evaluate(EvaluationContext context) + { + ArgumentNullException.ThrowIfNull(context); + return !string.Equals(Normalise(context, Left), Normalise(context, Right), StringComparison.Ordinal); + } + + private string Normalise(EvaluationContext context, string path) + { + IReadOnlyList values = context.Strings(path); + string text = (values.Count > 0 ? values[0] : string.Empty).Trim(); + if (IgnorePrefixPattern is { } prefix) + { + text = RuleRegex.Get(prefix, caseSensitive: true).Replace(text, string.Empty, 1).Trim(); + } + + return text.ToLowerInvariant(); + } +} diff --git a/PanoramicData.IssueAssessment/Validation/RuleValidator.cs b/PanoramicData.IssueAssessment/Validation/RuleValidator.cs index a54e8ee..1e8a903 100644 --- a/PanoramicData.IssueAssessment/Validation/RuleValidator.cs +++ b/PanoramicData.IssueAssessment/Validation/RuleValidator.cs @@ -38,6 +38,7 @@ internal static partial class RuleValidator FieldMatchesCondition value => PathError(value.Path) ?? PatternsError(value.Patterns), FieldEqualsCondition value => PathError(value.Path) ?? (value.Values is not { Count: > 0 } ? "has a 'fieldEquals' condition without values." : null), FieldPresentCondition value => PathError(value.Path), + FieldsDifferCondition value => PathError(value.Left) ?? PathError(value.Right), SectionPresentCondition value => SourceError(value.Source) ?? (value.Source.Section is null ? "has a 'sectionPresent' condition whose source has no section." : null), PlaceholderCondition value => SourceError(value.Source), _ => null, From 85caede65ce590de428b6e8fe1ee87a53f60cd3d Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 12:02:14 +0100 Subject: [PATCH 03/15] fix: release-notes generic tier order, standards ids, no N/A line --- .../Rubrics/ReleaseNotesAssessmentTests.cs | 13 +++++++++++-- .../Rubrics/panoramicdata-ticket-quality.json | 10 ++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs index 42ec9d3..fd39a47 100644 --- a/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs @@ -59,6 +59,14 @@ public async Task SameAsSummaryIgnoringPrefix_Fails() public async Task TicketKey_Fails(string notes) => NotesItem(await AssessAsync(notes)).Finding.Should().StartWith("✗ Release Notes: Not customer-facing"); + [Theory] + [InlineData("Schedules now read and write UTF-8 files correctly for every tenant")] + [InlineData("Passwords are now hashed with SHA-256 before they are stored for tenants")] + [InlineData("Reports now meet the ISO-27001 audit trail wording for every customer")] + [InlineData("The library update addresses CVE-2026-12345 for every hosted customer")] + public async Task StandardsIdentifiers_Pass(string notes) + => NotesItem(await AssessAsync(notes)).RawScore.Should().Be(100); + [Fact] public async Task FilePath_Fails() => NotesItem(await AssessAsync("The settings now load from appsettings.json when the service starts up")).Finding.Should().StartWith("✗ Release Notes: Not customer-facing"); @@ -89,13 +97,13 @@ public async Task UsSpelling_Fails() => NotesItem(await AssessAsync("Schedules now show the correct behavior when the timezone changes for tenants")).Finding.Should().Be("✗ Release Notes: US spelling"); [Theory] - [InlineData("Fixed bug")] + [InlineData("Fixed bug.")] [InlineData("Various fixes")] public async Task GenericFixedBug_Fails(string notes) { AssessmentReport report = await AssessAsync(notes); - NotesItem(report).RawScore.Should().Be(0); + NotesItem(report).Finding.Should().Be("✗ Release Notes: Generic"); } [Fact] @@ -136,6 +144,7 @@ public async Task OpsProject_NotApplicable_ScoreUnchanged() AssessmentReport before = await ReportAsync(previous, ops); now.Score.Should().Be(before.Score); + now.Findings.Should().NotContain(finding => finding.Contains("Release Notes", StringComparison.Ordinal)); now.Caps.Select(cap => cap.Name).Should().NotContain(CapName); now.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == "ReleaseNotes").Outcome.Should().Be(AssessmentOutcome.NotApplicable); } diff --git a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json index 5507f9f..ea62e70 100644 --- a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json +++ b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json @@ -376,18 +376,17 @@ "description": "MS tickets carry customer-facing release notes, written for the person who reads the release notes, not for the team.", "weight": 6, "appliesWhen": { "type": "all", "conditions": [ { "type": "ref", "name": "isMs" }, { "type": "not", "condition": { "type": "ref", "name": "isUndelivered" } } ] }, - "notApplicableExplanation": "○ Release Notes: Not applicable (not an MS ticket, or not delivered)", "definition": { "type": "tiered", "tiers": [ { "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "when": { "type": "measure", "of": { "type": "wordCount", "source": { "paths": [ "$.releaseNotes" ] } }, "lessThan": 8 }, "score": 0, "finding": "✗ Release Notes: Under 8 words", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "when": { "type": "not", "condition": { "type": "fieldsDiffer", "left": "$.releaseNotes", "right": "$.title", "ignorePrefixPattern": "^[^:]{1,40}:\\s*" } }, "score": 0, "finding": "✗ Release Notes: Same as the summary", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b(?!(?:UTF|SHA|ISO|IEC|RFC|TLS|SSL|AES|RSA|CVE|CWE|PCI|DSS|ECMA|IEEE|HTTP|X)-)[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "\\u2014" ] }, "score": 0, "finding": "✗ Release Notes: Contains an em dash", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)\\b(behavior|color|organization|initialize|customize|analyze|center|favorite|canceled)s?\\b" ] }, "score": 0, "finding": "✗ Release Notes: US spelling", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "score": 100, "finding": "✓ Release Notes: Present and customer-facing" } ] } @@ -557,18 +556,17 @@ "description": "MS tickets carry customer-facing release notes, written for the person who reads the release notes, not for the team.", "weight": 6, "appliesWhen": { "type": "all", "conditions": [ { "type": "ref", "name": "isMs" }, { "type": "not", "condition": { "type": "ref", "name": "isUndelivered" } } ] }, - "notApplicableExplanation": "○ Release Notes: Not applicable (not an MS ticket, or not delivered)", "definition": { "type": "tiered", "tiers": [ { "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "when": { "type": "measure", "of": { "type": "wordCount", "source": { "paths": [ "$.releaseNotes" ] } }, "lessThan": 8 }, "score": 0, "finding": "✗ Release Notes: Under 8 words", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "when": { "type": "not", "condition": { "type": "fieldsDiffer", "left": "$.releaseNotes", "right": "$.title", "ignorePrefixPattern": "^[^:]{1,40}:\\s*" } }, "score": 0, "finding": "✗ Release Notes: Same as the summary", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b(?!(?:UTF|SHA|ISO|IEC|RFC|TLS|SSL|AES|RSA|CVE|CWE|PCI|DSS|ECMA|IEEE|HTTP|X)-)[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "\\u2014" ] }, "score": 0, "finding": "✗ Release Notes: Contains an em dash", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)\\b(behavior|color|organization|initialize|customize|analyze|center|favorite|canceled)s?\\b" ] }, "score": 0, "finding": "✗ Release Notes: US spelling", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, { "score": 100, "finding": "✓ Release Notes: Present and customer-facing" } ] } From c966751a2d8075b0c78fa42213879df177d3c279 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 12:12:52 +0100 Subject: [PATCH 04/15] feat: MS Ticket Contract assessment with mandatory cap Adds the criteriaClassification measure (items, unclassified, must under a section), the contractEvidenceDue condition, a TicketContract assessment weighted 6 for MS non-epic delivered tickets, and a mandatory 'Ticket contract' cap at 0.79. From Ready for Test on, a Verification Evidence section with a Criterion table, Reachability, Not covered and a Suggested QA route is required. BuiltInRubricParityTests: no expectation changed; the parity rubric also drops TicketContract and its cap. --- .../Rubrics/BuiltInRubricParityTests.cs | 6 +- .../Rubrics/TicketContractAssessmentTests.cs | 218 ++++++++++++++++++ .../Rubrics/panoramicdata-ticket-quality.json | 53 +++++ .../Rules/CriteriaClassificationMeasure.cs | 101 ++++++++ .../Rules/Measures.cs | 1 + .../Validation/RuleValidator.Measures.cs | 1 + 6 files changed, 377 insertions(+), 3 deletions(-) create mode 100644 PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs create mode 100644 PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs index 6edd676..30bc14a 100644 --- a/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs @@ -17,7 +17,7 @@ namespace PanoramicData.IssueAssessment.Tests.Rubrics; /// public class BuiltInRubricParityTests : TestBase { - // Yoda has no Presentation or Release Notes factor, so parity is proved over the rubric without them; PresentationTests and ReleaseNotesAssessmentTests cover those. + // Yoda has no Presentation, Release Notes or Ticket Contract factor, so parity is proved over the rubric without them; their own test classes cover those. private static readonly IssueAssessor Assessor = new(Serialization.AssessmentRubric.Load(WithoutPresentation(BuiltInRubrics.PanoramicDataTicketQualityJson))); public static TheoryData Cases => [.. Enumerable.Range(0, File.ReadAllLines(FixturePath).Length)]; @@ -69,14 +69,14 @@ private static string WithoutPresentation(string rubricJson) foreach (JsonNode? category in rubric["categories"]!.AsArray()) { JsonArray assessments = category!["assessments"]!.AsArray(); - foreach (JsonNode? assessment in assessments.Where(item => item!["name"]!.GetValue().EndsWith("Presentation", StringComparison.Ordinal) || item["name"]!.GetValue().EndsWith("ReleaseNotes", StringComparison.Ordinal)).ToList()) + foreach (JsonNode? assessment in assessments.Where(item => item!["name"]!.GetValue().EndsWith("Presentation", StringComparison.Ordinal) || item["name"]!.GetValue().EndsWith("ReleaseNotes", StringComparison.Ordinal) || item["name"]!.GetValue() == "TicketContract").ToList()) { assessments.Remove(assessment); } } JsonArray caps = rubric["scoreCaps"]!.AsArray(); - foreach (JsonNode? cap in caps.Where(item => item!["name"]!.GetValue() == "Release notes").ToList()) + foreach (JsonNode? cap in caps.Where(item => item!["name"]!.GetValue() is "Release notes" or "Ticket contract").ToList()) { caps.Remove(cap); } diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs new file mode 100644 index 0000000..3fd717b --- /dev/null +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs @@ -0,0 +1,218 @@ +using PanoramicData.IssueAssessment.Mapping; +using PanoramicData.IssueAssessment.Rubrics; + +namespace PanoramicData.IssueAssessment.Tests.Rubrics; + +/// +/// The MS-only TicketContract assessment: criteria are classified MUST/SHOULD/MAY, and from Ready for Test on a +/// Verification Evidence section is present. A failing tier caps the score. +/// +public class TicketContractAssessmentTests : TestBase +{ + private const string CapName = "Ticket contract"; + private static readonly IssueAssessor Assessor = new(BuiltInRubrics.PanoramicDataTicketQuality); + + private const string Summary = "h2. Summary\r\nRun times drift when the tenant timezone changes.\r\n\r\n"; + private const string ClassifiedCriteria = "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its local run time after a timezone change.\r\n# SHOULD: the schedule list shows the new offset.\r\n# MAY: the audit log records the change.\r\n\r\n"; + + private const string FullEvidence = """ + h2. Verification Evidence + + *Automated coverage of the MUST criteria* + || Criterion || Test || Location || + | MUST 1 | GitFileSystemProvider_RecoversFromMissingIndex | MagicSuite.SystemAccess.Test/Git/GitFileSystemProviderTests.cs | + | MUST 2 | (none - not reachable from a test) | - | + + *Reachability - how to make the covered code actually run* + Files > any Git-backed connection > open any file. The recovery path runs on the + first read after the clone is corrupted. + + *Not covered by automated tests, and why* + MUST 2 needs a corrupted clone on a shared RWX volume; not reproducible in-process. + + *Suggested QA route:* B (regression suite) + """; + + [Fact] + public async Task NoCriteriaSection_Fails() + { + AssessmentReport report = await AssessAsync(Summary + "h2. Notes\r\nNothing else to say."); + + ContractItem(report).RawScore.Should().Be(0); + ContractItem(report).Finding.Should().Be("✗ Ticket Contract: No acceptance criteria section"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + report.NormalizedScore.Should().BeLessThanOrEqualTo(0.79); + } + + [Fact] + public async Task EmptyCriteriaSection_Fails() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\nTo be agreed with the team.\r\n")) + .Finding.Should().Be("✗ Ticket Contract: Acceptance criteria section has no criteria"); + + [Fact] + public async Task OneUnclassified_Fails() + { + AssessmentReport report = await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n# the schedule list shows the new offset.\r\n"); + + ContractItem(report).Finding.Should().Be("✗ Ticket Contract: 1 criteria not classified MUST/SHOULD/MAY"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task AllShouldNoMust_Fails() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# SHOULD: a schedule keeps its run time.\r\n# SHOULD: the list shows the offset.\r\n")) + .Finding.Should().Be("✗ Ticket Contract: No MUST criterion"); + + [Fact] + public async Task ClassifiedInProgress_NoEvidenceNeeded_Passes() + { + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria); + + ContractItem(report).RawScore.Should().Be(100); + ContractItem(report).Finding.Should().Be("✓ Ticket Contract: Criteria classified"); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Fact] + public async Task ReadyForTest_NoEvidence_Fails() + { + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria, status: "Ready for Test"); + + ContractItem(report).Finding.Should().Be("✗ Ticket Contract: No Verification Evidence section"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task ReadyForTest_EvidenceMissingRoute_Fails() + { + string evidence = FullEvidence.Replace("*Suggested QA route:* B (regression suite)", string.Empty, StringComparison.Ordinal); + + ContractItem(await AssessAsync(Summary + ClassifiedCriteria + evidence, status: "Ready for Test")) + .Finding.Should().Be("✗ Ticket Contract: Verification Evidence incomplete"); + } + + [Theory] + [InlineData("Ready for Test")] + [InlineData("In Test")] + [InlineData("Closed")] + public async Task ReadyForTest_FullEvidence_Passes(string status) + { + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + FullEvidence, status: status); + + ContractItem(report).RawScore.Should().Be(100); + ContractItem(report).Finding.Should().Be("✓ Ticket Contract: Criteria classified; evidence present"); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Fact] + public async Task RetrospectiveHeading_Passes() + { + string evidence = FullEvidence.Replace("h2. Verification Evidence", "h2. Verification Evidence (Retrospective)", StringComparison.Ordinal); + + ContractItem(await AssessAsync(Summary + ClassifiedCriteria + evidence, status: "Ready for Test")).RawScore.Should().Be(100); + } + + [Theory] + [InlineData("# MUST: x")] + [InlineData("# *MUST* x")] + [InlineData("* (SHOULD) x")] + [InlineData("| MAY 1 | x |")] + [InlineData("1. MUST - x")] + public async Task FormsAccepted(string criterion) + { + string criteria = $"h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n{criterion}\r\n"; + + AssessmentReport report = await AssessAsync(Summary + criteria); + + ContractItem(report).RawScore.Should().Be(100, ContractItem(report).Finding); + } + + [Fact] + public async Task LowercaseMay_IsNotAClassification() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n# may show the offset.\r\n")) + .Finding.Should().Be("✗ Ticket Contract: 1 criteria not classified MUST/SHOULD/MAY"); + + [Fact] + public async Task NestedSubPointsAndTableRules_AreNotCriteria() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n## including daylight saving changes\r\n - and leap days\r\n| MAY | x |\r\n|---|---|\r\n")) + .RawScore.Should().Be(100); + + [Fact] + public async Task EvidenceTableDoesNotTriggerDuplicationCap() + { + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + FullEvidence, status: "Ready for Test"); + + report.Caps.Select(cap => cap.Name).Should().NotContain("Criteria duplication"); + report.Findings.Should().NotContain(finding => finding.StartsWith("✗ Duplication", StringComparison.Ordinal)); + } + + [Fact] + public async Task Epic_NotApplicable() + { + AssessmentReport report = await AssessAsync(Summary + "h2. Notes\r\nNo criteria here.", issueType: "Epic"); + + report.Categories.SelectMany(category => category.Assessments) + .Where(item => item.Name == "TicketContract") + .Should().OnlyContain(item => item.Outcome == AssessmentOutcome.NotApplicable); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Fact] + public async Task Bug_RequiresCriteriaHeading() + { + const string bug = "h2. Steps to Reproduce\r\n# Open a schedule.\r\n# Change the tenant timezone.\r\n\r\nh2. Current Behaviour\r\nThe run time moves.\r\n\r\nh2. Expected Behaviour\r\nThe run time stays.\r\n"; + + AssessmentReport report = await AssessAsync(bug, issueType: "Bug"); + + ContractItem(report).Finding.Should().Be("✗ Ticket Contract: No acceptance criteria section"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task OpsProject_NotApplicable() + { + NormalizedIssue ops = Issue(Summary, "Improvement", "In Progress", null) with { Key = "OPS-1", Project = "OPS" }; + + AssessmentReport report = await ReportAsync(ops); + + report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == "TicketContract").Outcome.Should().Be(AssessmentOutcome.NotApplicable); + report.Findings.Should().NotContain(finding => finding.Contains("Ticket Contract", StringComparison.Ordinal)); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Fact] + public async Task Undelivered_NotApplicable() + { + AssessmentReport report = await AssessAsync(Summary, resolution: "Won't Do"); + + report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == "TicketContract").Outcome.Should().Be(AssessmentOutcome.NotApplicable); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + private static AssessmentReportItem ContractItem(AssessmentReport report) + => report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == "TicketContract" && item.Outcome != AssessmentOutcome.NotApplicable); + + private static NormalizedIssue Issue(string description, string issueType, string status, string? resolution) + => new() + { + Key = "MS-1", + Project = "MS", + Title = "Keep schedule run times across timezone changes", + Description = description, + IssueType = issueType, + Status = status, + Platform = "Jira", + ReleaseNotes = "Reports now keep their scheduled run time after the tenant timezone changes", + Resolution = resolution, + Labels = [], + }; + + private static async Task ReportAsync(NormalizedIssue issue) + { + using JsonDocument document = issue.ToJsonDocument(); + return AssessmentReport.From(await Assessor.AssessAsync(document, CancellationToken)); + } + + private static Task AssessAsync(string description, string issueType = "Improvement", string status = "In Progress", string? resolution = null) + => ReportAsync(Issue(description, issueType, status, resolution)); +} diff --git a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json index ea62e70..af9dfce 100644 --- a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json +++ b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json @@ -13,6 +13,7 @@ "type": "all", "conditions": [ { "type": "fieldPresent", "path": "$.resolution" }, { "type": "not", "condition": { "type": "fieldEquals", "path": "$.resolution", "values": [ "Fixed", "Done" ] } } ] }, + "contractEvidenceDue": { "type": "fieldEquals", "path": "$.status", "values": [ "Ready for Test", "In Test", "Test Blocked", "Ready for Release", "Closed", "Done", "Resolved" ] }, "isInvestigation": { "type": "all", "conditions": [ @@ -391,6 +392,52 @@ ] } }, + { + "name": "TicketContract", + "description": "MS acceptance criteria are classified MUST, SHOULD or MAY, with at least one MUST, and from Ready for Test on a Verification Evidence section says how each MUST was verified.", + "weight": 6, + "appliesWhen": { "type": "all", "conditions": [ { "type": "ref", "name": "isMs" }, { "type": "not", "condition": { "type": "ref", "name": "isEpic" } }, { "type": "not", "condition": { "type": "ref", "name": "isUndelivered" } } ] }, + "definition": { + "type": "tiered", + "measures": { + "items": { "type": "criteriaClassification", "path": "$.description", "section": "acceptance\\s+criteria", "count": "items" }, + "unclassified": { "type": "criteriaClassification", "path": "$.description", "section": "acceptance\\s+criteria", "count": "unclassified" }, + "must": { "type": "criteriaClassification", "path": "$.description", "section": "acceptance\\s+criteria", "count": "must" } + }, + "tiers": [ + { "when": { "type": "not", "condition": { "type": "sectionPresent", "source": { "paths": [ "$.description" ], "section": "acceptance\\s+criteria" } } }, "score": 0, "finding": "✗ Ticket Contract: No acceptance criteria section", "recommendation": "Add an 'h2. Acceptance Criteria' section listing each criterion as a numbered item classified MUST, SHOULD or MAY" }, + { "when": { "type": "measure", "measure": "items", "lessThan": 1 }, "score": 0, "finding": "✗ Ticket Contract: Acceptance criteria section has no criteria", "recommendation": "List each acceptance criterion as a numbered item or table row, classified MUST, SHOULD or MAY" }, + { "when": { "type": "measure", "measure": "unclassified", "atLeast": 1 }, "score": 0, "finding": "✗ Ticket Contract: {unclassified} criteria not classified MUST/SHOULD/MAY", "recommendation": "Start each acceptance criterion with MUST, SHOULD or MAY" }, + { "when": { "type": "measure", "measure": "must", "lessThan": 1 }, "score": 0, "finding": "✗ Ticket Contract: No MUST criterion", "recommendation": "Classify the criteria the ticket cannot pass without as MUST (aim for 2 to 5)" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "contractEvidenceDue" }, { "type": "not", "condition": { "type": "sectionPresent", "source": { "paths": [ "$.description" ], "section": "verification\\s+evidence" } } } ] }, "score": 0, "finding": "✗ Ticket Contract: No Verification Evidence section", "recommendation": "Add an 'h2. Verification Evidence' section: a Criterion/Test/Location table, Reachability, what is not covered and why, and the suggested QA route" }, + { + "when": { + "type": "all", + "conditions": [ + { "type": "ref", "name": "contractEvidenceDue" }, + { + "type": "not", + "condition": { + "type": "all", + "conditions": [ + { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "verification\\s+evidence" }, "patterns": [ "\\|\\|\\s*Criterion\\s*\\|\\|" ] }, + { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "verification\\s+evidence" }, "patterns": [ "(?i)reachability" ] }, + { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "verification\\s+evidence" }, "patterns": [ "(?i)not covered" ] }, + { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "verification\\s+evidence" }, "caseSensitive": true, "patterns": [ "(?i)suggested qa route:?\\*?\\s*(?-i:[ABCD])\\b" ] } + ] + } + } + ] + }, + "score": 0, + "finding": "✗ Ticket Contract: Verification Evidence incomplete", + "recommendation": "Complete the Verification Evidence section: a Criterion/Test/Location table, Reachability, what is not covered and why, and 'Suggested QA route:' followed by A, B, C or D" + }, + { "when": { "type": "ref", "name": "contractEvidenceDue" }, "score": 100, "finding": "✓ Ticket Contract: Criteria classified; evidence present" }, + { "score": 100, "finding": "✓ Ticket Contract: Criteria classified" } + ] + } + }, { "name": "CriteriaDuplication", "description": "Acceptance criteria are stated under one heading only. It carries no weight; duplication caps the score instead.", @@ -618,6 +665,12 @@ "conditions": [ { "type": "assessmentScore", "assessment": "ReleaseNotes", "lessThan": 100 }, { "type": "assessmentScore", "assessment": "EpicReleaseNotes", "lessThan": 100 } ] } }, + { + "name": "Ticket contract", + "group": "mandatory", + "cap": 0.79, + "when": { "type": "assessmentScore", "assessment": "TicketContract", "lessThan": 100 } + }, { "name": "Rollback plan", "group": "mandatory", diff --git a/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs new file mode 100644 index 0000000..dae9335 --- /dev/null +++ b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs @@ -0,0 +1,101 @@ +using System.Text.RegularExpressions; + +namespace PanoramicData.IssueAssessment.Rules; + +/// +/// Which criteria a counts. +/// +public enum CriteriaClassificationKind +{ + /// Every criterion. + Items, + + /// Criteria not classified MUST, SHOULD or MAY. + Unclassified, + + /// Criteria classified MUST. + Must, +} + +/// +/// Counts the criteria under a section of the text, by their RFC 2119 classification. A criterion is a top-level list +/// item (#, *, - or 1.) or a table row (|, not a || header). It is classified +/// when the item, or the row's first cell, starts with MUST, SHOULD or MAY in capitals, optionally after * or +/// (. An absent section counts zero. +/// +public sealed partial record CriteriaClassificationMeasure : IMeasure +{ + /// Gets the JSONPath of the text to read. + public required string Path { get; init; } + + /// Gets the regular expression fragment matched against the heading of the section holding the criteria. + public required string Section { get; init; } + + /// Gets which criteria to count. + public required CriteriaClassificationKind Count { get; init; } + + /// + public double Evaluate(EvaluationContext context) + { + ArgumentNullException.ThrowIfNull(context); + string? body = context.Text(new TextSource { Paths = [Path], Section = Section }); + if (body is null) + { + return 0; + } + + int items = 0; + int classified = 0; + int must = 0; + foreach (string line in LineBreak().Split(body)) + { + if (CriterionText(line) is not { } criterion) + { + continue; + } + + items++; + Match classification = Classification().Match(criterion); + if (classification.Success) + { + classified++; + must += classification.Groups[1].Value == "MUST" ? 1 : 0; + } + } + + return Count switch + { + CriteriaClassificationKind.Unclassified => items - classified, + CriteriaClassificationKind.Must => must, + _ => items, + }; + } + + private static string? CriterionText(string line) + { + Match item = ListItem().Match(line); + if (item.Success) + { + return item.Groups[1].Value; + } + + Match row = TableRow().Match(line); + return row.Success && !MarkdownTableRule().IsMatch(row.Groups[1].Value) ? row.Groups[1].Value : null; + } + + [GeneratedRegex(@"\r?\n", RegexOptions.CultureInvariant, 1000)] + private static partial Regex LineBreak(); + + // More indentation, or a doubled wiki marker, is a sub-point of the criterion above it. + [GeneratedRegex(@"^ ?(?:[#*-]|\d+\.)[ \t]+(\S.*)$", RegexOptions.CultureInvariant, 1000)] + private static partial Regex ListItem(); + + [GeneratedRegex(@"^[ \t]*\|(?!\|)([^|]*)", RegexOptions.CultureInvariant, 1000)] + private static partial Regex TableRow(); + + [GeneratedRegex(@"^[ \t]*:?-+:?[ \t]*$", RegexOptions.CultureInvariant, 1000)] + private static partial Regex MarkdownTableRule(); + + [GeneratedRegex(@"^[ \t]*[*(]?[ \t]*(MUST|SHOULD|MAY)\b", RegexOptions.CultureInvariant, 1000)] + private static partial Regex Classification(); +} diff --git a/PanoramicData.IssueAssessment/Rules/Measures.cs b/PanoramicData.IssueAssessment/Rules/Measures.cs index 4eefa20..f4f412d 100644 --- a/PanoramicData.IssueAssessment/Rules/Measures.cs +++ b/PanoramicData.IssueAssessment/Rules/Measures.cs @@ -17,6 +17,7 @@ namespace PanoramicData.IssueAssessment.Rules; [JsonDerivedType(typeof(NumberMeasure), "number")] [JsonDerivedType(typeof(CriteriaHeadingCountMeasure), "criteriaHeadingCount")] [JsonDerivedType(typeof(SectionPanelCountMeasure), "sectionPanelCount")] +[JsonDerivedType(typeof(CriteriaClassificationMeasure), "criteriaClassification")] public interface IMeasure { /// diff --git a/PanoramicData.IssueAssessment/Validation/RuleValidator.Measures.cs b/PanoramicData.IssueAssessment/Validation/RuleValidator.Measures.cs index e545142..2253cd8 100644 --- a/PanoramicData.IssueAssessment/Validation/RuleValidator.Measures.cs +++ b/PanoramicData.IssueAssessment/Validation/RuleValidator.Measures.cs @@ -17,6 +17,7 @@ internal static partial class RuleValidator SectionPanelCountMeasure value => SourceError(value.Source), ItemCountMeasure value => PathError(value.Path), NumberMeasure value => PathError(value.Path), + CriteriaClassificationMeasure value => PathError(value.Path) ?? (string.IsNullOrWhiteSpace(value.Section) ? "has a 'criteriaClassification' measure without a section." : PatternError(value.Section)), _ => null, }; From a9a8fd7c04c365438faf10ac8bf6540f5195095a Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 12:17:49 +0100 Subject: [PATCH 05/15] test: pin real-ticket criteria forms --- .../Rubrics/TicketContractAssessmentTests.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs index 3fd717b..55fe944 100644 --- a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs @@ -118,6 +118,7 @@ public async Task RetrospectiveHeading_Passes() [InlineData("* (SHOULD) x")] [InlineData("| MAY 1 | x |")] [InlineData("1. MUST - x")] + [InlineData("# *MUST* - PASS when a reference is struck through on its page")] public async Task FormsAccepted(string criterion) { string criteria = $"h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n{criterion}\r\n"; @@ -132,6 +133,17 @@ public async Task LowercaseMay_IsNotAClassification() => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n# may show the offset.\r\n")) .Finding.Should().Be("✗ Ticket Contract: 1 criteria not classified MUST/SHOULD/MAY"); + [Fact] + public async Task BoldGroupLabelLine_IsNotACriterion() + { + const string criteria = "h2. Acceptance Criteria\r\n*MUST* (the ticket cannot pass without these)\r\n# *MUST* A value encrypted under the old key is readable\r\n# *SHOULD* Something else\r\n"; + + AssessmentReport report = await AssessAsync(Summary + criteria); + + ContractItem(report).RawScore.Should().Be(100, ContractItem(report).Finding); + ContractItem(report).Finding.Should().Be("✓ Ticket Contract: Criteria classified"); + } + [Fact] public async Task NestedSubPointsAndTableRules_AreNotCriteria() => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n## including daylight saving changes\r\n - and leap days\r\n| MAY | x |\r\n|---|---|\r\n")) From 6e6cb020d084f63d2f08d8063f9e46beeb4c81ca Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 12:20:27 +0100 Subject: [PATCH 06/15] feat: justified not-needed tiers for documentation and certifications No Yoda parity fixture contains a matching line, so no parity score changes. --- .../Rubrics/NotNeededTierTests.cs | 70 +++++++++++++++++++ .../Rubrics/panoramicdata-ticket-quality.json | 2 + 2 files changed, 72 insertions(+) create mode 100644 PanoramicData.IssueAssessment.Tests/Rubrics/NotNeededTierTests.cs diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/NotNeededTierTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/NotNeededTierTests.cs new file mode 100644 index 0000000..e95252d --- /dev/null +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/NotNeededTierTests.cs @@ -0,0 +1,70 @@ +using PanoramicData.IssueAssessment.Mapping; +using PanoramicData.IssueAssessment.Rubrics; + +namespace PanoramicData.IssueAssessment.Tests.Rubrics; + +public class NotNeededTierTests : TestBase +{ + private static readonly IssueAssessor Assessor = new(BuiltInRubrics.PanoramicDataTicketQuality); + + [Fact] + public async Task NoDocumentationChangeWithReason_Scores100() + { + AssessmentReportItem item = await Item("Documentation", "No documentation change: internal refactor with no user-visible behaviour", "Improvement", "MS"); + item.RawScore.Should().Be(100); + item.Finding.Should().Be("✓ Documentation: Not needed (reason given)"); + } + + [Fact] + public async Task NoDocumentationChangeBare_Scores0() + => (await Item("Documentation", "No documentation change.", "Improvement", "MS")).Finding.Should().NotContain("Not needed"); + + [Fact] + public async Task NoDocumentationChangeNA_Scores0() + => (await Item("Documentation", "No documentation change: N/A", "Improvement", "MS")).Finding.Should().NotContain("Not needed"); + + [Fact] + public async Task NoCertificationQuestionWithReason_Scores100() + { + AssessmentReportItem item = await Item("Certifications", "No certification question: internal refactor with no user-visible behaviour", "Improvement", "MS"); + item.RawScore.Should().Be(100); + item.Finding.Should().Be("✓ Certifications: Not needed (reason given)"); + } + + [Fact] + public async Task NoCertificationQuestionBare_Scores0() + => (await Item("Certifications", "No certification question.", "Improvement", "MS")).Finding.Should().NotContain("Not needed"); + + [Fact] + public async Task NoCertificationQuestionNA_Scores0() + => (await Item("Certifications", "No certification question: N/A", "Improvement", "MS")).Finding.Should().NotContain("Not needed"); + + [Fact] + public async Task AppliesToOpsToo() + { + (await Item("Documentation", "No documentation change: internal refactor with no user-visible behaviour", "Improvement", "OPS")).RawScore.Should().Be(100); + (await Item("Certifications", "No certification question: internal refactor with no user-visible behaviour", "Improvement", "OPS")).RawScore.Should().Be(100); + } + + [Fact] + public async Task BugCertifications_StillNotApplicable() + => (await Item("Certifications", "No certification question: internal refactor with no user-visible behaviour", "Bug", "MS")).Finding.Should().Be("○ Certifications: Not applicable for bugs"); + + private static async Task Item(string name, string description, string issueType, string project) + { + NormalizedIssue issue = new() + { + Key = project + "-1", + Project = project, + Title = "Tidy the schedule code", + Description = description, + IssueType = issueType, + Status = "In Progress", + Platform = "Jira", + Labels = [], + }; + using JsonDocument document = issue.ToJsonDocument(); + AssessmentReport report = AssessmentReport.From(await Assessor.AssessAsync(document, CancellationToken)); + return report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == name); + } +} diff --git a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json index af9dfce..627c638 100644 --- a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json +++ b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json @@ -282,6 +282,7 @@ "type": "tiered", "measures": { "references": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "documentation|docs?", "help\\s+(file|topic|section)", "user\\s+guide", "knowledge\\s+base|kb\\s+article", "wiki|confluence", "readme|readme\\.md", "api\\s+(reference|documentation|docs)", "manual", "guide|tutorial" ] } }, "tiers": [ + { "when": { "type": "textMatches", "source": { "paths": [ "$.description" ] }, "patterns": [ "(?im)^\\W*no documentation change:\\s*(?!n/?a\\b)\\S.{14,}" ] }, "score": 100, "finding": "✓ Documentation: Not needed (reason given)" }, { "when": { "type": "measure", "measure": "references", "atLeast": 2 }, "score": 100, "finding": "✓ Documentation: Multiple references found" }, { "when": { "type": "measure", "measure": "references", "atLeast": 1 }, "score": 70, "finding": "✓ Documentation: Referenced", "recommendation": "Consider adding more documentation updates" }, { "score": 0, "finding": "⚠ Documentation: Not referenced", "recommendation": "Update documentation/help text for this change" } @@ -296,6 +297,7 @@ "type": "tiered", "tiers": [ { "when": { "type": "ref", "name": "isBug" }, "score": 100, "finding": "○ Certifications: Not applicable for bugs" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.description" ] }, "patterns": [ "(?im)^\\W*no certification question:\\s*(?!n/?a\\b)\\S.{14,}" ] }, "score": 100, "finding": "✓ Certifications: Not needed (reason given)" }, { "when": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "certifica?tion|exam|certification\\s+question", "cert\\s+question", "assessment\\s+question", "test\\s+question" ] }, "score": 100, "finding": "✓ Certifications: Exam question(s) created" }, { "score": 0, "finding": "○ Certifications: No exam question created", "recommendation": "Consider creating a certification exam question for this new feature" } ] From 81228ffe3e734e1d3f29a4ad9a550f72f9600a56 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 12:23:20 +0100 Subject: [PATCH 07/15] chore: 1.2.0 release notes and version Co-Authored-By: Claude Haiku 5.5 --- CHANGELOG.md | 25 +++++++++++++++++++++++++ README.md | 6 ++++++ version.json | 2 +- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fdee81..d4d62a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,31 @@ All notable changes to this project are documented here. The format follows [Semantic Versioning](https://semver.org/). Versions are `major.minor.height`, where the height comes from Nerdbank.GitVersioning. +## [1.2.0] - Unreleased + +Adds two Magic Suite ticket checks to the built-in rubric and a justified "not needed" option for documentation and +certifications. Everything is additive; 1.1 rubrics load and score as before, though a ticket's score can move where +the new checks apply. + +### Added + +- `NormalizedIssue.ReleaseNotes` (Jira `customfield_10600`, plain text or ADF) and `NormalizedIssue.Resolution` + (`fields.resolution.name`), read from the `releaseNotes` and `resolution` JSON keys. +- Built-in ticket-quality rubric: MS-only `ReleaseNotes` assessment (and `EpicReleaseNotes` in Epic Quality), weight 6, + mandatory cap 0.79. It checks that the notes are present, not a placeholder or generic, at least 8 words, different + from the summary, customer-facing (no ticket keys other than standards ids, file paths or code identifiers), free of + em dashes and in British spelling. It does not apply to other projects or to undelivered resolutions. +- Built-in ticket-quality rubric: MS-only `TicketContract` assessment (not for Epics), weight 6, mandatory cap 0.79. + Acceptance criteria must each be classified MUST, SHOULD or MAY, with at least one MUST. From Ready for Test onwards + the ticket needs an `h2. Verification Evidence` section (optionally marked "(Retrospective)") with a criterion table, + a Reachability block, a Not covered block and a Suggested QA route of A to D. +- New condition type `fieldsDiffer` and new measure `criteriaClassification`, for rubrics that need them. + +### Changed + +- Documentation and Certifications accept a justified `No documentation change: ` or + `No certification question: ` line, for all projects. + ## [1.1] - Unreleased Adds rubrics: whole configurations written as JSON, with a built-in Panoramic Data ticket-quality rubric. Everything diff --git a/README.md b/README.md index 06a14f2..54ff1c6 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,12 @@ report.Recommendations.ForEach(Console.WriteLine); // what to add to raise the Console.WriteLine(report.ToJson()); ``` +For Magic Suite (`MS`) tickets the rubric also scores release notes (`ReleaseNotes`, and `EpicReleaseNotes` on epics) +and, for tickets that are not epics, the MUST/SHOULD/MAY classification of acceptance criteria and the Verification +Evidence section (`TicketContract`). Both have weight 6 and a mandatory cap of 0.79, and neither applies to a ticket +whose resolution is anything other than Fixed or Done. Documentation and Certifications accept a justified +`No documentation change: ` or `No certification question: ` line in place of the item. + `BuiltInRubrics.PanoramicDataTicketQualityJson` returns its JSON, a good starting point for your own. ### Writing a rubric diff --git a/version.json b/version.json index 28d4cab..7f28ebf 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/AArnott/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", - "version": "1.1", + "version": "1.2", "publicReleaseRefSpec": [ "^refs/heads/main$", "^refs/tags/\\d+\\.\\d+\\.\\d+$" From 51a758bb9bedea92c554416e3ea15e38538afd50 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 12:51:05 +0100 Subject: [PATCH 08/15] fix: validate fieldsDiffer ignorePrefixPattern at load and pin its contract Co-Authored-By: Claude Opus 5.5 --- .../Rules/RuleEngineTests.cs | 20 +++++++++++++++++++ .../Validation/RubricValidationTests.cs | 3 +++ .../Validation/RuleValidator.cs | 2 +- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/PanoramicData.IssueAssessment.Tests/Rules/RuleEngineTests.cs b/PanoramicData.IssueAssessment.Tests/Rules/RuleEngineTests.cs index 5b80916..9ebd5e6 100644 --- a/PanoramicData.IssueAssessment.Tests/Rules/RuleEngineTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rules/RuleEngineTests.cs @@ -138,4 +138,24 @@ public void RuleRegex_CaseSensitive_DoesNotIgnoreCase() new TextMatchesCondition { Source = Source("$.d"), Patterns = ["Bug"], CaseSensitive = true }.Evaluate(Context("""{ "d": "bug" }""")) .Should().BeFalse(); } + + [Theory] + [InlineData("""{ "l": "Keep run times", "r": "Keep run times" }""", false)] + [InlineData("""{ "l": "Keep run times", "r": "Keep start times" }""", true)] + [InlineData("""{ "l": " KEEP Run Times ", "r": "keep run times" }""", false)] + [InlineData("""{ "l": "Keep run times", "r": "Keep run times" }""", true)] + [InlineData("""{ "r": "" }""", false)] + [InlineData("""{ "r": "Keep run times" }""", true)] + [InlineData("""{ }""", false)] + public void FieldsDiffer_ComparesTrimmedTextIgnoringCase(string json, bool expected) + => new FieldsDifferCondition { Left = "$.l", Right = "$.r" }.Evaluate(Context(json)).Should().Be(expected); + + [Theory] + [InlineData("""{ "l": "Schedules: Keep run times", "r": "Keep run times" }""", false)] + [InlineData("""{ "l": "Keep run times", "r": "Schedules: Keep run times" }""", false)] + [InlineData("""{ "l": "Fix: Keep run times", "r": "Bug: keep run times" }""", false)] + [InlineData("""{ "l": "Schedules: Keep run times", "r": "Keep start times" }""", true)] + [InlineData("""{ "l": "Keep run times: always", "r": "always" }""", false)] + public void FieldsDiffer_StripsThePrefixFromBothSides(string json, bool expected) + => new FieldsDifferCondition { Left = "$.l", Right = "$.r", IgnorePrefixPattern = @"^[^:]{1,40}:\s*" }.Evaluate(Context(json)).Should().Be(expected); } diff --git a/PanoramicData.IssueAssessment.Tests/Validation/RubricValidationTests.cs b/PanoramicData.IssueAssessment.Tests/Validation/RubricValidationTests.cs index 652cdb0..84418dc 100644 --- a/PanoramicData.IssueAssessment.Tests/Validation/RubricValidationTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Validation/RubricValidationTests.cs @@ -73,6 +73,9 @@ public void ValidRubric_IsAccepted() { """{ "type": "fieldEquals", "path": "$.a", "values": [] }""", "'fieldEquals' condition without values" }, { """{ "type": "fieldEquals", "path": "bad", "values": [ "x" ] }""", "invalid JSONPath 'bad'" }, { """{ "type": "fieldPresent", "path": "bad" }""", "invalid JSONPath 'bad'" }, + { """{ "type": "fieldsDiffer", "left": "bad", "right": "$.b" }""", "invalid JSONPath 'bad'" }, + { """{ "type": "fieldsDiffer", "left": "$.a", "right": "bad" }""", "invalid JSONPath 'bad'" }, + { """{ "type": "fieldsDiffer", "left": "$.a", "right": "$.b", "ignorePrefixPattern": "(" }""", "invalid regex pattern '('" }, { """{ "type": "sectionPresent", "source": { "paths": [ "$.a" ] } }""", "whose source has no section" }, { """{ "type": "sectionPresent", "source": { "paths": [ "bad" ], "section": "x" } }""", "invalid JSONPath 'bad'" }, { """{ "type": "placeholder", "source": { "paths": [ "bad" ] } }""", "invalid JSONPath 'bad'" }, diff --git a/PanoramicData.IssueAssessment/Validation/RuleValidator.cs b/PanoramicData.IssueAssessment/Validation/RuleValidator.cs index 1e8a903..6061128 100644 --- a/PanoramicData.IssueAssessment/Validation/RuleValidator.cs +++ b/PanoramicData.IssueAssessment/Validation/RuleValidator.cs @@ -38,7 +38,7 @@ internal static partial class RuleValidator FieldMatchesCondition value => PathError(value.Path) ?? PatternsError(value.Patterns), FieldEqualsCondition value => PathError(value.Path) ?? (value.Values is not { Count: > 0 } ? "has a 'fieldEquals' condition without values." : null), FieldPresentCondition value => PathError(value.Path), - FieldsDifferCondition value => PathError(value.Left) ?? PathError(value.Right), + FieldsDifferCondition value => PathError(value.Left) ?? PathError(value.Right) ?? (value.IgnorePrefixPattern is null ? null : PatternError(value.IgnorePrefixPattern)), SectionPresentCondition value => SourceError(value.Source) ?? (value.Source.Section is null ? "has a 'sectionPresent' condition whose source has no section." : null), PlaceholderCondition value => SourceError(value.Source), _ => null, From d40ba52322e7de7415198710446a6977a74a52d4 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 12:52:11 +0100 Subject: [PATCH 09/15] fix: Verification Evidence is due in Ready for Customer and With Customer Both statuses follow In Test in the MS workflow, so a ticket in either needed no evidence. Co-Authored-By: Claude Opus 5.5 --- .../Rubrics/TicketContractAssessmentTests.cs | 11 ++++++++--- .../Rubrics/panoramicdata-ticket-quality.json | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs index 55fe944..d1d2225 100644 --- a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs @@ -73,10 +73,13 @@ public async Task ClassifiedInProgress_NoEvidenceNeeded_Passes() report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); } - [Fact] - public async Task ReadyForTest_NoEvidence_Fails() + [Theory] + [InlineData("Ready for Test")] + [InlineData("Ready for Customer")] + [InlineData("With Customer")] + public async Task EvidenceDue_NoEvidence_Fails(string status) { - AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria, status: "Ready for Test"); + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria, status: status); ContractItem(report).Finding.Should().Be("✗ Ticket Contract: No Verification Evidence section"); report.Caps.Select(cap => cap.Name).Should().Contain(CapName); @@ -94,6 +97,8 @@ public async Task ReadyForTest_EvidenceMissingRoute_Fails() [Theory] [InlineData("Ready for Test")] [InlineData("In Test")] + [InlineData("Ready for Customer")] + [InlineData("With Customer")] [InlineData("Closed")] public async Task ReadyForTest_FullEvidence_Passes(string status) { diff --git a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json index 627c638..ef93b9e 100644 --- a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json +++ b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json @@ -13,7 +13,7 @@ "type": "all", "conditions": [ { "type": "fieldPresent", "path": "$.resolution" }, { "type": "not", "condition": { "type": "fieldEquals", "path": "$.resolution", "values": [ "Fixed", "Done" ] } } ] }, - "contractEvidenceDue": { "type": "fieldEquals", "path": "$.status", "values": [ "Ready for Test", "In Test", "Test Blocked", "Ready for Release", "Closed", "Done", "Resolved" ] }, + "contractEvidenceDue": { "type": "fieldEquals", "path": "$.status", "values": [ "Ready for Test", "In Test", "Test Blocked", "Ready for Customer", "With Customer", "Ready for Release", "Closed", "Done", "Resolved" ] }, "isInvestigation": { "type": "all", "conditions": [ From 006981d36cc1c9ceaf376dfe74550cb7ad9b26c1 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 12:53:33 +0100 Subject: [PATCH 10/15] fix: each failing release-notes tier recommends a fix for its own rule Co-Authored-By: Claude Opus 5.5 --- .../Rubrics/ReleaseNotesAssessmentTests.cs | 40 +++++++++++++++++++ .../Rubrics/panoramicdata-ticket-quality.json | 32 +++++++-------- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs index fd39a47..7cc04de 100644 --- a/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs @@ -106,6 +106,46 @@ public async Task GenericFixedBug_Fails(string notes) NotesItem(report).Finding.Should().Be("✗ Release Notes: Generic"); } + public static TheoryData FailingTiers => new() + { + { "Improvement", null, "✗ Release Notes: Empty", EmptyAdvice }, + { "Epic", null, "✗ Release Notes: Empty", EmptyAdvice }, + { "Improvement", "TBD", "✗ Release Notes: Placeholder", PlaceholderAdvice }, + { "Epic", "TBD", "✗ Release Notes: Placeholder", PlaceholderAdvice }, + { "Improvement", "Various fixes", "✗ Release Notes: Generic", GenericAdvice }, + { "Epic", "Various fixes", "✗ Release Notes: Generic", GenericAdvice }, + { "Improvement", "Schedules now keep their run time changes", "✗ Release Notes: Under 8 words", ShortAdvice }, + { "Epic", "Schedules now keep their run time changes", "✗ Release Notes: Under 8 words", ShortAdvice }, + { "Improvement", "Schedules: Keep schedule run times across timezone changes", "✗ Release Notes: Same as the summary", SameAsSummaryAdvice }, + { "Epic", "Schedules: Keep schedule run times across timezone changes", "✗ Release Notes: Same as the summary", SameAsSummaryAdvice }, + { "Improvement", "Fixes the issue raised in MS-12345 for all users today", "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", NotCustomerFacingAdvice }, + { "Epic", "Fixes the issue raised in MS-12345 for all users today", "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", NotCustomerFacingAdvice }, + { "Improvement", "Schedules now keep their run time — even when the timezone changes", "✗ Release Notes: Contains an em dash", EmDashAdvice }, + { "Epic", "Schedules now keep their run time — even when the timezone changes", "✗ Release Notes: Contains an em dash", EmDashAdvice }, + { "Improvement", "Schedules now show the correct behavior when the timezone changes for tenants", "✗ Release Notes: US spelling", UsSpellingAdvice }, + { "Epic", "Schedules now show the correct behavior when the timezone changes for tenants", "✗ Release Notes: US spelling", UsSpellingAdvice }, + }; + + private const string EmptyAdvice = "Fill in the Release Notes field with one or two customer-facing sentences of at least 8 words saying what changed for the customer"; + private const string PlaceholderAdvice = "Replace the placeholder in the Release Notes field with one or two sentences saying what changed for the customer"; + private const string GenericAdvice = "Say what changed and for whom: a generic note such as 'Fixed bug' tells the customer nothing"; + private const string ShortAdvice = "Expand the Release Notes to at least 8 words, saying what changed and who it affects"; + private const string SameAsSummaryAdvice = "Describe the outcome for the customer rather than repeating the ticket summary"; + private const string NotCustomerFacingAdvice = "Remove ticket keys, file paths and code identifiers, and describe the change as the customer sees it"; + private const string EmDashAdvice = "Replace the em dash with a comma, colon or full stop"; + private const string UsSpellingAdvice = "Use British spelling, for example behaviour, colour, organisation, centre, favourite and cancelled"; + + [Theory] + [MemberData(nameof(FailingTiers))] + public async Task FailingTier_RecommendsAFixForItsOwnRule(string issueType, string? notes, string finding, string recommendation) + { + AssessmentReportItem item = NotesItem(await AssessAsync(notes, issueType: issueType)); + + item.Finding.Should().Be(finding); + item.Recommendation.Should().Be(recommendation); + item.Recommendation.Should().NotContain("—"); + } + [Fact] public async Task Good_Scores100_NoCap() { diff --git a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json index ef93b9e..40c40b8 100644 --- a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json +++ b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json @@ -382,14 +382,14 @@ "definition": { "type": "tiered", "tiers": [ - { "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "measure", "of": { "type": "wordCount", "source": { "paths": [ "$.releaseNotes" ] } }, "lessThan": 8 }, "score": 0, "finding": "✗ Release Notes: Under 8 words", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "not", "condition": { "type": "fieldsDiffer", "left": "$.releaseNotes", "right": "$.title", "ignorePrefixPattern": "^[^:]{1,40}:\\s*" } }, "score": 0, "finding": "✗ Release Notes: Same as the summary", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b(?!(?:UTF|SHA|ISO|IEC|RFC|TLS|SSL|AES|RSA|CVE|CWE|PCI|DSS|ECMA|IEEE|HTTP|X)-)[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "\\u2014" ] }, "score": 0, "finding": "✗ Release Notes: Contains an em dash", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)\\b(behavior|color|organization|initialize|customize|analyze|center|favorite|canceled)s?\\b" ] }, "score": 0, "finding": "✗ Release Notes: US spelling", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Fill in the Release Notes field with one or two customer-facing sentences of at least 8 words saying what changed for the customer" }, + { "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Replace the placeholder in the Release Notes field with one or two sentences saying what changed for the customer" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Say what changed and for whom: a generic note such as 'Fixed bug' tells the customer nothing" }, + { "when": { "type": "measure", "of": { "type": "wordCount", "source": { "paths": [ "$.releaseNotes" ] } }, "lessThan": 8 }, "score": 0, "finding": "✗ Release Notes: Under 8 words", "recommendation": "Expand the Release Notes to at least 8 words, saying what changed and who it affects" }, + { "when": { "type": "not", "condition": { "type": "fieldsDiffer", "left": "$.releaseNotes", "right": "$.title", "ignorePrefixPattern": "^[^:]{1,40}:\\s*" } }, "score": 0, "finding": "✗ Release Notes: Same as the summary", "recommendation": "Describe the outcome for the customer rather than repeating the ticket summary" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b(?!(?:UTF|SHA|ISO|IEC|RFC|TLS|SSL|AES|RSA|CVE|CWE|PCI|DSS|ECMA|IEEE|HTTP|X)-)[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Remove ticket keys, file paths and code identifiers, and describe the change as the customer sees it" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "\\u2014" ] }, "score": 0, "finding": "✗ Release Notes: Contains an em dash", "recommendation": "Replace the em dash with a comma, colon or full stop" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)\\b(behavior|color|organization|initialize|customize|analyze|center|favorite|canceled)s?\\b" ] }, "score": 0, "finding": "✗ Release Notes: US spelling", "recommendation": "Use British spelling, for example behaviour, colour, organisation, centre, favourite and cancelled" }, { "score": 100, "finding": "✓ Release Notes: Present and customer-facing" } ] } @@ -608,14 +608,14 @@ "definition": { "type": "tiered", "tiers": [ - { "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "measure", "of": { "type": "wordCount", "source": { "paths": [ "$.releaseNotes" ] } }, "lessThan": 8 }, "score": 0, "finding": "✗ Release Notes: Under 8 words", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "not", "condition": { "type": "fieldsDiffer", "left": "$.releaseNotes", "right": "$.title", "ignorePrefixPattern": "^[^:]{1,40}:\\s*" } }, "score": 0, "finding": "✗ Release Notes: Same as the summary", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b(?!(?:UTF|SHA|ISO|IEC|RFC|TLS|SSL|AES|RSA|CVE|CWE|PCI|DSS|ECMA|IEEE|HTTP|X)-)[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "\\u2014" ] }, "score": 0, "finding": "✗ Release Notes: Contains an em dash", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, - { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)\\b(behavior|color|organization|initialize|customize|analyze|center|favorite|canceled)s?\\b" ] }, "score": 0, "finding": "✗ Release Notes: US spelling", "recommendation": "Write the Release Notes field as one or two customer-facing sentences of at least 8 words" }, + { "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Fill in the Release Notes field with one or two customer-facing sentences of at least 8 words saying what changed for the customer" }, + { "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Replace the placeholder in the Release Notes field with one or two sentences saying what changed for the customer" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Say what changed and for whom: a generic note such as 'Fixed bug' tells the customer nothing" }, + { "when": { "type": "measure", "of": { "type": "wordCount", "source": { "paths": [ "$.releaseNotes" ] } }, "lessThan": 8 }, "score": 0, "finding": "✗ Release Notes: Under 8 words", "recommendation": "Expand the Release Notes to at least 8 words, saying what changed and who it affects" }, + { "when": { "type": "not", "condition": { "type": "fieldsDiffer", "left": "$.releaseNotes", "right": "$.title", "ignorePrefixPattern": "^[^:]{1,40}:\\s*" } }, "score": 0, "finding": "✗ Release Notes: Same as the summary", "recommendation": "Describe the outcome for the customer rather than repeating the ticket summary" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b(?!(?:UTF|SHA|ISO|IEC|RFC|TLS|SSL|AES|RSA|CVE|CWE|PCI|DSS|ECMA|IEEE|HTTP|X)-)[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Remove ticket keys, file paths and code identifiers, and describe the change as the customer sees it" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "\\u2014" ] }, "score": 0, "finding": "✗ Release Notes: Contains an em dash", "recommendation": "Replace the em dash with a comma, colon or full stop" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)\\b(behavior|color|organization|initialize|customize|analyze|center|favorite|canceled)s?\\b" ] }, "score": 0, "finding": "✗ Release Notes: US spelling", "recommendation": "Use British spelling, for example behaviour, colour, organisation, centre, favourite and cancelled" }, { "score": 100, "finding": "✓ Release Notes: Present and customer-facing" } ] } From d8ab46cd05873ace4f6d17b23cebf2ab572b1ae3 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 12:59:08 +0100 Subject: [PATCH 11/15] fix: classify RFC 2119 sentence criteria and items under a label line A criterion is classified when it starts with MUST, SHOULD or MAY, contains exactly one distinct keyword, or is a list item under a label line such as *MUST* or MUST: (scope ends at the next label or wiki heading). Items mixing keywords without a leading one stay unclassified. Covers the MS-27557 and MS-27609 forms. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- .../Rubrics/TicketContractAssessmentTests.cs | 26 +++++ .../CriteriaClassificationMeasureTests.cs | 101 ++++++++++++++++++ .../Rules/CriteriaClassificationMeasure.cs | 51 +++++++-- 4 files changed, 171 insertions(+), 11 deletions(-) create mode 100644 PanoramicData.IssueAssessment.Tests/Rules/CriteriaClassificationMeasureTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index d4d62a0..92678ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,9 @@ the new checks apply. from the summary, customer-facing (no ticket keys other than standards ids, file paths or code identifiers), free of em dashes and in British spelling. It does not apply to other projects or to undelivered resolutions. - Built-in ticket-quality rubric: MS-only `TicketContract` assessment (not for Epics), weight 6, mandatory cap 0.79. - Acceptance criteria must each be classified MUST, SHOULD or MAY, with at least one MUST. From Ready for Test onwards + Acceptance criteria must each be classified MUST, SHOULD or MAY, with at least one MUST. A criterion is classified + when it starts with a keyword, contains exactly one distinct keyword (the RFC 2119 sentence form), or sits under a + label line such as `*MUST*` or `MUST:`. Only capitals count. From Ready for Test onwards the ticket needs an `h2. Verification Evidence` section (optionally marked "(Retrospective)") with a criterion table, a Reachability block, a Not covered block and a Suggested QA route of A to D. - New condition type `fieldsDiffer` and new measure `criteriaClassification`, for rubrics that need them. diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs index d1d2225..f82260d 100644 --- a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs @@ -133,6 +133,32 @@ public async Task FormsAccepted(string criterion) ContractItem(report).RawScore.Should().Be(100, ContractItem(report).Finding); } + [Fact] + public async Task InlineRfc2119Criteria_Ms27557Form_Pass() + { + const string criteria = "h2. Acceptance Criteria\r\n# SECTIONPAGES in a header or footer MUST show the page count of its section\r\n# NUMPAGES SHOULD still show the page count of the whole document\r\n"; + + ContractItem(await AssessAsync(Summary + criteria)).Finding.Should().Be("✓ Ticket Contract: Criteria classified"); + } + + [Fact] + public async Task LabelThenUnlabelledItems_Ms27609Form_Pass() + { + const string criteria = "h2. Acceptance Criteria\r\n*MUST*\r\n# A value encrypted under the old key is readable after rotation\r\n# The rotation is recorded in the audit log\r\n*SHOULD*\r\n# The admin is told when the rotation finishes\r\n"; + + ContractItem(await AssessAsync(Summary + criteria)).Finding.Should().Be("✓ Ticket Contract: Criteria classified"); + } + + [Fact] + public async Task MixedMustAndMayItem_IsNotClassified() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n# the list MUST refresh and MAY animate.\r\n")) + .Finding.Should().Be("✗ Ticket Contract: 1 criteria not classified MUST/SHOULD/MAY"); + + [Fact] + public async Task ShouldAndMayLabelsOnly_FailNoMust() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n*SHOULD*\r\n# a schedule keeps its run time.\r\n*MAY*\r\n# the list shows the offset.\r\n")) + .Finding.Should().Be("✗ Ticket Contract: No MUST criterion"); + [Fact] public async Task LowercaseMay_IsNotAClassification() => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n# may show the offset.\r\n")) diff --git a/PanoramicData.IssueAssessment.Tests/Rules/CriteriaClassificationMeasureTests.cs b/PanoramicData.IssueAssessment.Tests/Rules/CriteriaClassificationMeasureTests.cs new file mode 100644 index 0000000..cb7c078 --- /dev/null +++ b/PanoramicData.IssueAssessment.Tests/Rules/CriteriaClassificationMeasureTests.cs @@ -0,0 +1,101 @@ +namespace PanoramicData.IssueAssessment.Tests.Rules; + +/// +/// Unit tests for : which criteria count as classified MUST, SHOULD or MAY. +/// +public class CriteriaClassificationMeasureTests : TestBase +{ + private const string Heading = "h2. Acceptance Criteria\r\n"; + + [Theory] + [InlineData("# MUST: x")] + [InlineData("# *MUST* x")] + [InlineData("* (SHOULD) x")] + [InlineData("| MAY 1 | x |")] + [InlineData("1. MUST - x")] + [InlineData("# *MUST* - PASS when a reference is struck through on its page")] + [InlineData("- MAY x")] + public void LeadingKeywordForms_AreClassified(string criterion) + => Counts(criterion).Should().Be((1, 0, criterion.Contains("MUST", StringComparison.Ordinal) ? 1 : 0)); + + [Fact] + public void InlineKeyword_Ms27557ProseForm_IsClassified() + => Counts("# SECTIONPAGES in a header or footer MUST show the page count of its section\r\n# ChartMagic MUST draw X and Y axis titles in regular weight") + .Should().Be((2, 0, 2)); + + [Theory] + [InlineData("# The schedule list SHOULD show the new offset", 0)] + [InlineData("# The audit log MAY record the change.", 0)] + [InlineData("# The run time MUST stay the same, and it MUST not drift", 1)] + public void OneDistinctInlineKeyword_IsClassified(string criterion, int must) + => Counts(criterion).Should().Be((1, 0, must)); + + [Theory] + [InlineData("# The run time MUST stay the same and MAY be logged")] + [InlineData("# SECTIONPAGES SHOULD refresh; NUMPAGES MUST refresh")] + public void MixedInlineKeywords_AreUnclassified(string criterion) + => Counts(criterion).Should().Be((1, 1, 0)); + + [Fact] + public void MixedKeywords_StartingWithOne_TakeTheLeadingOne() + => Counts("# MUST: the run time stays the same and MAY be logged").Should().Be((1, 0, 1)); + + [Theory] + [InlineData("# the run time must stay the same")] + [InlineData("# the audit log may record the change and should be kept")] + [InlineData("# the MUSTARD theme keeps its MAYBE flag")] + public void LowercaseOrEmbeddedKeywords_AreUnclassified(string criterion) + => Counts(criterion).Should().Be((1, 1, 0)); + + [Fact] + public void LabelLine_Ms27609Form_ClassifiesTheUnlabelledItemsBelowIt() + => Counts("*MUST*\r\n# A value encrypted under the old key is readable\r\n# The rotation is logged") + .Should().Be((2, 0, 2)); + + [Theory] + [InlineData("*MUST*")] + [InlineData("*MUST* (the ticket cannot pass without these)")] + [InlineData("MUST:")] + [InlineData("(MUST)")] + [InlineData("[MUST]")] + public void LabelLineForms_ClassifyTheItemsBelow(string label) + => Counts($"{label}\r\n# A value encrypted under the old key is readable").Should().Be((1, 0, 1)); + + [Fact] + public void LabelScope_EndsAtTheNextLabel() + => Counts("*MUST*\r\n# The key rotates\r\n# The rotation is logged\r\n\r\n*SHOULD*\r\n# The admin is emailed\r\n\r\n*MAY*\r\n# The dashboard shows a banner") + .Should().Be((4, 0, 2)); + + [Fact] + public void LabelScope_EndsAtAHeading() + => Counts("*MUST*\r\n# The key rotates\r\nh3. Notes\r\n# The rotation is logged") + .Should().Be((2, 1, 1)); + + [Fact] + public void ItemsBeforeAnyLabel_AreUnclassified() + => Counts("# The key rotates\r\n*MUST*\r\n# The rotation is logged").Should().Be((2, 1, 1)); + + [Fact] + public void ItemWithItsOwnKeyword_KeepsItUnderALabel() + => Counts("*MUST*\r\n# The rotation SHOULD be logged\r\n# SHOULD: the admin is emailed\r\n# The key rotates") + .Should().Be((3, 0, 1)); + + [Fact] + public void NestedSubPoint_IsNotALabel() + => Counts("*SHOULD*\r\n# The admin is emailed\r\n** MUST include the tenant name\r\n# The dashboard shows a banner") + .Should().Be((2, 0, 0)); + + [Fact] + public void LowercaseLabelLine_IsNotALabel() + => Counts("must:\r\n# The key rotates").Should().Be((1, 1, 0)); + + private static (double Items, double Unclassified, double Must) Counts(string criteria) + { + using JsonDocument document = JsonDocument.Parse(JsonSerializer.Serialize(new { d = Heading + criteria + "\r\n" })); + EvaluationContext context = new(document); + double Measure(CriteriaClassificationKind kind) + => new CriteriaClassificationMeasure { Path = "$.d", Section = @"acceptance\s+criteria", Count = kind }.Evaluate(context); + + return (Measure(CriteriaClassificationKind.Items), Measure(CriteriaClassificationKind.Unclassified), Measure(CriteriaClassificationKind.Must)); + } +} diff --git a/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs index dae9335..2631a55 100644 --- a/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs +++ b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs @@ -19,10 +19,16 @@ public enum CriteriaClassificationKind /// /// Counts the criteria under a section of the text, by their RFC 2119 classification. A criterion is a top-level list -/// item (#, *, - or 1.) or a table row (|, not a || header). It is classified -/// when the item, or the row's first cell, starts with MUST, SHOULD or MAY in capitals, optionally after * or -/// (. An absent section counts zero. +/// item (#, *, - or 1.) or a table row (|, not a || header). Keywords are +/// MUST, SHOULD and MAY as whole words in capitals. An absent section counts zero. /// +/// +/// A criterion (the item, or the row's first cell) is classified, in order, by: a keyword it starts with, optionally +/// after * or (; otherwise the one keyword it contains, when it contains exactly one distinct keyword; +/// otherwise, for a list item, the label line above it. A label line is a line that is not a criterion and starts +/// with a keyword, optionally bolded or bracketed (*MUST*, MUST:). Its scope runs to the next label line +/// or wiki heading. +/// public sealed partial record CriteriaClassificationMeasure : IMeasure { /// Gets the JSONPath of the text to read. @@ -47,19 +53,22 @@ public double Evaluate(EvaluationContext context) int items = 0; int classified = 0; int must = 0; + string? label = null; foreach (string line in LineBreak().Split(body)) { - if (CriterionText(line) is not { } criterion) + if (Criterion(line) is not { } criterion) { + label = LabelLine().Match(line) is { Success: true } labelLine ? labelLine.Groups[1].Value + : WikiHeading().IsMatch(line) ? null + : label; continue; } items++; - Match classification = Classification().Match(criterion); - if (classification.Success) + if (Classify(criterion.Text, criterion.IsListItem ? label : null) is { } keyword) { classified++; - must += classification.Groups[1].Value == "MUST" ? 1 : 0; + must += keyword == "MUST" ? 1 : 0; } } @@ -71,16 +80,28 @@ public double Evaluate(EvaluationContext context) }; } - private static string? CriterionText(string line) + private static string? Classify(string criterion, string? label) + { + Match leading = Classification().Match(criterion); + if (leading.Success) + { + return leading.Groups[1].Value; + } + + List keywords = [.. Keyword().Matches(criterion).Select(match => match.Value).Distinct(StringComparer.Ordinal)]; + return keywords.Count == 1 ? keywords[0] : label; + } + + private static (string Text, bool IsListItem)? Criterion(string line) { Match item = ListItem().Match(line); if (item.Success) { - return item.Groups[1].Value; + return (item.Groups[1].Value, true); } Match row = TableRow().Match(line); - return row.Success && !MarkdownTableRule().IsMatch(row.Groups[1].Value) ? row.Groups[1].Value : null; + return row.Success && !MarkdownTableRule().IsMatch(row.Groups[1].Value) ? (row.Groups[1].Value, false) : null; } [GeneratedRegex(@"\r?\n", RegexOptions.CultureInvariant, 1000)] @@ -98,4 +119,14 @@ public double Evaluate(EvaluationContext context) [GeneratedRegex(@"^[ \t]*[*(]?[ \t]*(MUST|SHOULD|MAY)\b", RegexOptions.CultureInvariant, 1000)] private static partial Regex Classification(); + + [GeneratedRegex(@"\b(?:MUST|SHOULD|MAY)\b", RegexOptions.CultureInvariant, 1000)] + private static partial Regex Keyword(); + + // No space after the marker, so a "** MUST ..." nested sub-point is not a label. + [GeneratedRegex(@"^[ \t]*(?:\*{1,2}|_|\(|\[)?(MUST|SHOULD|MAY)\b", RegexOptions.CultureInvariant, 1000)] + private static partial Regex LabelLine(); + + [GeneratedRegex(@"^[ \t]*h[1-6]\.", RegexOptions.CultureInvariant, 1000)] + private static partial Regex WikiHeading(); } From a20025c77150f6a6cdb9de71a76fab1e466d1bc4 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 12:59:19 +0100 Subject: [PATCH 12/15] docs: CHANGELOG heading [1.2] to match the major.minor style NBGV appends the git height, so 1.2.0 is never published. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92678ee..f15aeeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project are documented here. The format follows [Semantic Versioning](https://semver.org/). Versions are `major.minor.height`, where the height comes from Nerdbank.GitVersioning. -## [1.2.0] - Unreleased +## [1.2] - Unreleased Adds two Magic Suite ticket checks to the built-in rubric and a justified "not needed" option for documentation and certifications. Everything is additive; 1.1 rubrics load and score as before, though a ticket's score can move where From 0a46f1030b68fbef62ca6ecdf3f95274a59930d8 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 12:59:47 +0100 Subject: [PATCH 13/15] docs: README lists the fieldsDiffer and criteriaClassification building blocks Co-Authored-By: Claude Opus 5.5 --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 54ff1c6..5cec667 100644 --- a/README.md +++ b/README.md @@ -186,11 +186,13 @@ The building blocks: - **Tiered assessments** (`"type": "tiered"`): tiers are tried in order and the first whose `when` holds gives the score (0–100 by default), the finding and an optional recommendation. The last tier has no condition. - **Measures** count things: `patternMatchCount`, `occurrenceCount`, `wordCount`, `alphanumericCount`, `length`, - `itemCount`, `number`, `criteriaHeadingCount` and `sectionPanelCount` (how many `h2.` sections of Jira wiki text - are panelable, panelled, unpanelled or carry a panel title that repeats the heading). + `itemCount`, `number`, `criteriaHeadingCount`, `sectionPanelCount` (how many `h2.` sections of Jira wiki text + are panelable, panelled, unpanelled or carry a panel title that repeats the heading) and `criteriaClassification` + (how many criteria in a section there are, how many are not classified MUST, SHOULD or MAY, and how many are MUST). - **Conditions**: `all`, `any`, `not`, `ref` (a named condition), `measure`, `textMatches`, `fieldMatches`, - `fieldEquals`, `fieldPresent`, `sectionPresent`, `placeholder` (a section that only says "TBD") and, for caps and - classifications, `assessmentScore`. + `fieldEquals`, `fieldPresent`, `fieldsDiffer` (two fields differ, ignoring case, surrounding space and an optional + prefix), `sectionPresent`, `placeholder` (a section that only says "TBD") and, for caps and classifications, + `assessmentScore`. - **Text sources** join one or more fields and can narrow to a section by heading, in Jira wiki or Markdown. - **Placeholders** in findings and recommendations: `{measure}`, `{measure:text}`, `{$.path}` and `{$.path~pattern}` (the first value matching the pattern). From fe9f9404ff33789cecb5928997a9cd617fbff2c8 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 13:05:14 +0100 Subject: [PATCH 14/15] test: release-notes tier data writes the em dash as a code point Co-Authored-By: Claude Opus 5.5 --- .../Rubrics/ReleaseNotesAssessmentTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs index 7cc04de..1ab9d20 100644 --- a/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs @@ -120,12 +120,13 @@ public async Task GenericFixedBug_Fails(string notes) { "Epic", "Schedules: Keep schedule run times across timezone changes", "✗ Release Notes: Same as the summary", SameAsSummaryAdvice }, { "Improvement", "Fixes the issue raised in MS-12345 for all users today", "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", NotCustomerFacingAdvice }, { "Epic", "Fixes the issue raised in MS-12345 for all users today", "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", NotCustomerFacingAdvice }, - { "Improvement", "Schedules now keep their run time — even when the timezone changes", "✗ Release Notes: Contains an em dash", EmDashAdvice }, - { "Epic", "Schedules now keep their run time — even when the timezone changes", "✗ Release Notes: Contains an em dash", EmDashAdvice }, + { "Improvement", $"Schedules now keep their run time {EmDash} even when the timezone changes", "✗ Release Notes: Contains an em dash", EmDashAdvice }, + { "Epic", $"Schedules now keep their run time {EmDash} even when the timezone changes", "✗ Release Notes: Contains an em dash", EmDashAdvice }, { "Improvement", "Schedules now show the correct behavior when the timezone changes for tenants", "✗ Release Notes: US spelling", UsSpellingAdvice }, { "Epic", "Schedules now show the correct behavior when the timezone changes for tenants", "✗ Release Notes: US spelling", UsSpellingAdvice }, }; + private static readonly string EmDash = ((char)0x2014).ToString(); private const string EmptyAdvice = "Fill in the Release Notes field with one or two customer-facing sentences of at least 8 words saying what changed for the customer"; private const string PlaceholderAdvice = "Replace the placeholder in the Release Notes field with one or two sentences saying what changed for the customer"; private const string GenericAdvice = "Say what changed and for whom: a generic note such as 'Fixed bug' tells the customer nothing"; @@ -143,7 +144,7 @@ public async Task FailingTier_RecommendsAFixForItsOwnRule(string issueType, stri item.Finding.Should().Be(finding); item.Recommendation.Should().Be(recommendation); - item.Recommendation.Should().NotContain("—"); + item.Recommendation.Should().NotContain(EmDash); } [Fact] From 7c8aaa3e45fdae4c8786fc6bd3e026d8b923612b Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 19 Oct 2026 16:29:11 +0100 Subject: [PATCH 15/15] refactor: keep criteria classification and mapper tests under Codacy limits Co-Authored-By: Claude Opus 5.5 --- .../Mapping/MapperTests.cs | 15 +++++---- .../Rules/CriteriaClassificationMeasure.cs | 32 ++++++++++++++----- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs b/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs index 3d4c7fd..150d649 100644 --- a/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs @@ -162,13 +162,16 @@ public void FromRestJson_ReadsReleaseNotesFromCustomField10600() JiraIssueMapper.FromRestJson(Json("""{ "fields": {} }""")).ReleaseNotes.Should().BeNull(); } + [Theory] + [InlineData("Won't Do")] + [InlineData("Fixed")] + public void FromRestJson_ReadsResolutionName(string resolution) + => JiraIssueMapper.FromRestJson(Json(JsonSerializer.Serialize(new { fields = new { resolution = new { name = resolution } } }))) + .Resolution.Should().Be(resolution); + [Fact] - public void FromRestJson_ReadsResolutionName() - { - JiraIssueMapper.FromRestJson(Json("""{ "fields": { "resolution": { "name": "Won't Do" } } }""")) - .Resolution.Should().Be("Won't Do"); - JiraIssueMapper.FromRestJson(Json("""{ "fields": { "resolution": null } }""")).Resolution.Should().BeNull(); - } + public void FromRestJson_NullResolution_IsNull() + => JiraIssueMapper.FromRestJson(Json("""{ "fields": { "resolution": null } }""")).Resolution.Should().BeNull(); [Fact] public void ToJsonObject_OmitsNullReleaseNotesAndResolution() diff --git a/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs index 2631a55..82f0f9c 100644 --- a/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs +++ b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs @@ -50,6 +50,17 @@ public double Evaluate(EvaluationContext context) return 0; } + (int items, int classified, int must) = Tally(body); + return Count switch + { + CriteriaClassificationKind.Unclassified => items - classified, + CriteriaClassificationKind.Must => must, + _ => items, + }; + } + + private static (int Items, int Classified, int Must) Tally(string body) + { int items = 0; int classified = 0; int must = 0; @@ -58,9 +69,7 @@ public double Evaluate(EvaluationContext context) { if (Criterion(line) is not { } criterion) { - label = LabelLine().Match(line) is { Success: true } labelLine ? labelLine.Groups[1].Value - : WikiHeading().IsMatch(line) ? null - : label; + label = LabelAfter(line, label); continue; } @@ -72,12 +81,19 @@ public double Evaluate(EvaluationContext context) } } - return Count switch + return (items, classified, must); + } + + /// The label in force after a line that is not a criterion: a label line sets it, a wiki heading clears it. + private static string? LabelAfter(string line, string? label) + { + Match labelLine = LabelLine().Match(line); + if (labelLine.Success) { - CriteriaClassificationKind.Unclassified => items - classified, - CriteriaClassificationKind.Must => must, - _ => items, - }; + return labelLine.Groups[1].Value; + } + + return WikiHeading().IsMatch(line) ? null : label; } private static string? Classify(string criterion, string? label)