diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fdee81..f15aeeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,33 @@ All notable changes to this project are documented here. The format follows [Semantic Versioning](https://semver.org/). Versions are `major.minor.height`, where the height comes from Nerdbank.GitVersioning. +## [1.2] - Unreleased + +Adds two Magic Suite ticket checks to the built-in rubric and a justified "not needed" option for documentation and +certifications. Everything is additive; 1.1 rubrics load and score as before, though a ticket's score can move where +the new checks apply. + +### Added + +- `NormalizedIssue.ReleaseNotes` (Jira `customfield_10600`, plain text or ADF) and `NormalizedIssue.Resolution` + (`fields.resolution.name`), read from the `releaseNotes` and `resolution` JSON keys. +- Built-in ticket-quality rubric: MS-only `ReleaseNotes` assessment (and `EpicReleaseNotes` in Epic Quality), weight 6, + mandatory cap 0.79. It checks that the notes are present, not a placeholder or generic, at least 8 words, different + from the summary, customer-facing (no ticket keys other than standards ids, file paths or code identifiers), free of + em dashes and in British spelling. It does not apply to other projects or to undelivered resolutions. +- Built-in ticket-quality rubric: MS-only `TicketContract` assessment (not for Epics), weight 6, mandatory cap 0.79. + Acceptance criteria must each be classified MUST, SHOULD or MAY, with at least one MUST. A criterion is classified + when it starts with a keyword, contains exactly one distinct keyword (the RFC 2119 sentence form), or sits under a + label line such as `*MUST*` or `MUST:`. Only capitals count. From Ready for Test onwards + the ticket needs an `h2. Verification Evidence` section (optionally marked "(Retrospective)") with a criterion table, + a Reachability block, a Not covered block and a Suggested QA route of A to D. +- New condition type `fieldsDiffer` and new measure `criteriaClassification`, for rubrics that need them. + +### Changed + +- Documentation and Certifications accept a justified `No documentation change: ` or + `No certification question: ` line, for all projects. + ## [1.1] - Unreleased Adds rubrics: whole configurations written as JSON, with a built-in Panoramic Data ticket-quality rubric. Everything diff --git a/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs b/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs index 3242b61..150d649 100644 --- a/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Mapping/MapperTests.cs @@ -148,6 +148,43 @@ public void Jira_V3Description_ListItemParagraphsAndAttributelessNodes_Flatten() issue.Description.Should().Be("* first\nsecond\nend"); } + [Fact] + public void FromRestJson_ReadsReleaseNotesFromCustomField10600() + { + JiraIssueMapper.FromRestJson(Json("""{ "fields": { "customfield_10600": "Fixed the export." } }""")) + .ReleaseNotes.Should().Be("Fixed the export."); + + JiraIssueMapper.FromRestJson(Json(""" + { "fields": { "customfield_10600": { "type": "doc", "version": 1, "content": [ + { "type": "paragraph", "content": [ { "type": "text", "text": "Fixed the export." } ] } ] } } } + """)).ReleaseNotes.Should().Be("Fixed the export."); + + JiraIssueMapper.FromRestJson(Json("""{ "fields": {} }""")).ReleaseNotes.Should().BeNull(); + } + + [Theory] + [InlineData("Won't Do")] + [InlineData("Fixed")] + public void FromRestJson_ReadsResolutionName(string resolution) + => JiraIssueMapper.FromRestJson(Json(JsonSerializer.Serialize(new { fields = new { resolution = new { name = resolution } } }))) + .Resolution.Should().Be(resolution); + + [Fact] + public void FromRestJson_NullResolution_IsNull() + => JiraIssueMapper.FromRestJson(Json("""{ "fields": { "resolution": null } }""")).Resolution.Should().BeNull(); + + [Fact] + public void ToJsonObject_OmitsNullReleaseNotesAndResolution() + { + var empty = new NormalizedIssue().ToJsonObject(); + empty.ContainsKey("releaseNotes").Should().BeFalse(); + empty.ContainsKey("resolution").Should().BeFalse(); + + var set = new NormalizedIssue { ReleaseNotes = "n", Resolution = "Fixed" }.ToJsonObject(); + set["releaseNotes"]!.GetValue().Should().Be("n"); + set["resolution"]!.GetValue().Should().Be("Fixed"); + } + [Fact] public void Jira_NoDescription_IsNull() { diff --git a/PanoramicData.IssueAssessment.Tests/PanoramicData.IssueAssessment.Tests.csproj b/PanoramicData.IssueAssessment.Tests/PanoramicData.IssueAssessment.Tests.csproj index b2fefb8..6a19d42 100644 --- a/PanoramicData.IssueAssessment.Tests/PanoramicData.IssueAssessment.Tests.csproj +++ b/PanoramicData.IssueAssessment.Tests/PanoramicData.IssueAssessment.Tests.csproj @@ -15,6 +15,7 @@ + diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs index 6959b9e..30bc14a 100644 --- a/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/BuiltInRubricParityTests.cs @@ -17,7 +17,7 @@ namespace PanoramicData.IssueAssessment.Tests.Rubrics; /// public class BuiltInRubricParityTests : TestBase { - // Yoda has no Presentation factor, so parity is proved over the rubric without it; PresentationTests covers that one. + // Yoda has no Presentation, Release Notes or Ticket Contract factor, so parity is proved over the rubric without them; their own test classes cover those. private static readonly IssueAssessor Assessor = new(Serialization.AssessmentRubric.Load(WithoutPresentation(BuiltInRubrics.PanoramicDataTicketQualityJson))); public static TheoryData Cases => [.. Enumerable.Range(0, File.ReadAllLines(FixturePath).Length)]; @@ -69,12 +69,18 @@ private static string WithoutPresentation(string rubricJson) foreach (JsonNode? category in rubric["categories"]!.AsArray()) { JsonArray assessments = category!["assessments"]!.AsArray(); - foreach (JsonNode? assessment in assessments.Where(item => item!["name"]!.GetValue().EndsWith("Presentation", StringComparison.Ordinal)).ToList()) + foreach (JsonNode? assessment in assessments.Where(item => item!["name"]!.GetValue().EndsWith("Presentation", StringComparison.Ordinal) || item["name"]!.GetValue().EndsWith("ReleaseNotes", StringComparison.Ordinal) || item["name"]!.GetValue() == "TicketContract").ToList()) { assessments.Remove(assessment); } } + JsonArray caps = rubric["scoreCaps"]!.AsArray(); + foreach (JsonNode? cap in caps.Where(item => item!["name"]!.GetValue() is "Release notes" or "Ticket contract").ToList()) + { + caps.Remove(cap); + } + return rubric.ToJsonString(); } diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/NotNeededTierTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/NotNeededTierTests.cs new file mode 100644 index 0000000..e95252d --- /dev/null +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/NotNeededTierTests.cs @@ -0,0 +1,70 @@ +using PanoramicData.IssueAssessment.Mapping; +using PanoramicData.IssueAssessment.Rubrics; + +namespace PanoramicData.IssueAssessment.Tests.Rubrics; + +public class NotNeededTierTests : TestBase +{ + private static readonly IssueAssessor Assessor = new(BuiltInRubrics.PanoramicDataTicketQuality); + + [Fact] + public async Task NoDocumentationChangeWithReason_Scores100() + { + AssessmentReportItem item = await Item("Documentation", "No documentation change: internal refactor with no user-visible behaviour", "Improvement", "MS"); + item.RawScore.Should().Be(100); + item.Finding.Should().Be("✓ Documentation: Not needed (reason given)"); + } + + [Fact] + public async Task NoDocumentationChangeBare_Scores0() + => (await Item("Documentation", "No documentation change.", "Improvement", "MS")).Finding.Should().NotContain("Not needed"); + + [Fact] + public async Task NoDocumentationChangeNA_Scores0() + => (await Item("Documentation", "No documentation change: N/A", "Improvement", "MS")).Finding.Should().NotContain("Not needed"); + + [Fact] + public async Task NoCertificationQuestionWithReason_Scores100() + { + AssessmentReportItem item = await Item("Certifications", "No certification question: internal refactor with no user-visible behaviour", "Improvement", "MS"); + item.RawScore.Should().Be(100); + item.Finding.Should().Be("✓ Certifications: Not needed (reason given)"); + } + + [Fact] + public async Task NoCertificationQuestionBare_Scores0() + => (await Item("Certifications", "No certification question.", "Improvement", "MS")).Finding.Should().NotContain("Not needed"); + + [Fact] + public async Task NoCertificationQuestionNA_Scores0() + => (await Item("Certifications", "No certification question: N/A", "Improvement", "MS")).Finding.Should().NotContain("Not needed"); + + [Fact] + public async Task AppliesToOpsToo() + { + (await Item("Documentation", "No documentation change: internal refactor with no user-visible behaviour", "Improvement", "OPS")).RawScore.Should().Be(100); + (await Item("Certifications", "No certification question: internal refactor with no user-visible behaviour", "Improvement", "OPS")).RawScore.Should().Be(100); + } + + [Fact] + public async Task BugCertifications_StillNotApplicable() + => (await Item("Certifications", "No certification question: internal refactor with no user-visible behaviour", "Bug", "MS")).Finding.Should().Be("○ Certifications: Not applicable for bugs"); + + private static async Task Item(string name, string description, string issueType, string project) + { + NormalizedIssue issue = new() + { + Key = project + "-1", + Project = project, + Title = "Tidy the schedule code", + Description = description, + IssueType = issueType, + Status = "In Progress", + Platform = "Jira", + Labels = [], + }; + using JsonDocument document = issue.ToJsonDocument(); + AssessmentReport report = AssessmentReport.From(await Assessor.AssessAsync(document, CancellationToken)); + return report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == name); + } +} diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs new file mode 100644 index 0000000..1ab9d20 --- /dev/null +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/ReleaseNotesAssessmentTests.cs @@ -0,0 +1,241 @@ +using PanoramicData.IssueAssessment.Mapping; +using PanoramicData.IssueAssessment.Rubrics; + +namespace PanoramicData.IssueAssessment.Tests.Rubrics; + +/// +/// The MS-only ReleaseNotes assessment: customer-facing release notes are mandatory, so a failing tier caps the score. +/// +public class ReleaseNotesAssessmentTests : TestBase +{ + private const string CapName = "Release notes"; + private const string GoodNotes = "Reports now keep their scheduled run time after the tenant timezone changes"; + private static readonly IssueAssessor Assessor = new(BuiltInRubrics.PanoramicDataTicketQuality); + + private const string Description = "h2. Summary\r\nRun times drift when the tenant timezone changes.\r\n\r\nh2. Acceptance Criteria\r\n# MUST: a schedule keeps its local run time after a timezone change.\r\n# SHOULD: the schedule list shows the new offset.\r\n\r\nh2. Business Case\r\nCustomers rely on reports arriving at the agreed time, which has a direct business value."; + + [Fact] + public async Task Empty_FailsAndCapsAt79() + { + AssessmentReport report = await AssessAsync(notes: null); + + NotesItem(report).RawScore.Should().Be(0); + NotesItem(report).Finding.Should().Be("✗ Release Notes: Empty"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + report.NormalizedScore.Should().BeLessThanOrEqualTo(0.79); + } + + [Theory] + [InlineData("TBD")] + [InlineData("None")] + [InlineData("See the ticket")] + [InlineData("---")] + public async Task Placeholder_Fails(string notes) + => NotesItem(await AssessAsync(notes)).Finding.Should().Be("✗ Release Notes: Placeholder"); + + [Fact] + public async Task Placeholder_TBD_Fails() + { + AssessmentReport report = await AssessAsync("TBD"); + + NotesItem(report).Finding.Should().Be("✗ Release Notes: Placeholder"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task SevenWords_Fails() + => NotesItem(await AssessAsync("Schedules now keep their run time changes")).Finding.Should().Be("✗ Release Notes: Under 8 words"); + + [Fact] + public async Task SameAsSummaryIgnoringPrefix_Fails() + => NotesItem(await AssessAsync("Add X to Y report for every tenant", title: "ReportMagic: Add X to Y report for every tenant")) + .Finding.Should().Be("✗ Release Notes: Same as the summary"); + + [Theory] + [InlineData("Fixes the issue raised in MS-12345 for all users today")] + [InlineData("Updated ReportJob.cs so that every scheduled report runs on time")] + [InlineData("Changed the file under C:\\inetpub\\reports so that every report loads")] + [InlineData("Updated the files in the repo/src/Reports folder so reports load faster")] + public async Task TicketKey_Fails(string notes) + => NotesItem(await AssessAsync(notes)).Finding.Should().StartWith("✗ Release Notes: Not customer-facing"); + + [Theory] + [InlineData("Schedules now read and write UTF-8 files correctly for every tenant")] + [InlineData("Passwords are now hashed with SHA-256 before they are stored for tenants")] + [InlineData("Reports now meet the ISO-27001 audit trail wording for every customer")] + [InlineData("The library update addresses CVE-2026-12345 for every hosted customer")] + public async Task StandardsIdentifiers_Pass(string notes) + => NotesItem(await AssessAsync(notes)).RawScore.Should().Be(100); + + [Fact] + public async Task FilePath_Fails() + => NotesItem(await AssessAsync("The settings now load from appsettings.json when the service starts up")).Finding.Should().StartWith("✗ Release Notes: Not customer-facing"); + + [Fact] + public async Task ServiceIdentifier_Fails() + => NotesItem(await AssessAsync("ReportJobService now retries failed jobs three times before giving up")).Finding.Should().StartWith("✗ Release Notes: Not customer-facing"); + + [Fact] + public async Task ProductNames_Pass() + { + AssessmentReport report = await AssessAsync("ReportMagic and ConnectMagic now show the DataMagic sync time on every dashboard page"); + + NotesItem(report).RawScore.Should().Be(100); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Fact] + public async Task SentenceInitialCapital_Passes() + => NotesItem(await AssessAsync("Schedules now keep their local run time after the timezone changes")).RawScore.Should().Be(100); + + [Fact] + public async Task EmDash_Fails() + => NotesItem(await AssessAsync("Schedules now keep their run time \u2014 even when the timezone changes")).Finding.Should().Be("✗ Release Notes: Contains an em dash"); + + [Fact] + public async Task UsSpelling_Fails() + => NotesItem(await AssessAsync("Schedules now show the correct behavior when the timezone changes for tenants")).Finding.Should().Be("✗ Release Notes: US spelling"); + + [Theory] + [InlineData("Fixed bug.")] + [InlineData("Various fixes")] + public async Task GenericFixedBug_Fails(string notes) + { + AssessmentReport report = await AssessAsync(notes); + + NotesItem(report).Finding.Should().Be("✗ Release Notes: Generic"); + } + + public static TheoryData FailingTiers => new() + { + { "Improvement", null, "✗ Release Notes: Empty", EmptyAdvice }, + { "Epic", null, "✗ Release Notes: Empty", EmptyAdvice }, + { "Improvement", "TBD", "✗ Release Notes: Placeholder", PlaceholderAdvice }, + { "Epic", "TBD", "✗ Release Notes: Placeholder", PlaceholderAdvice }, + { "Improvement", "Various fixes", "✗ Release Notes: Generic", GenericAdvice }, + { "Epic", "Various fixes", "✗ Release Notes: Generic", GenericAdvice }, + { "Improvement", "Schedules now keep their run time changes", "✗ Release Notes: Under 8 words", ShortAdvice }, + { "Epic", "Schedules now keep their run time changes", "✗ Release Notes: Under 8 words", ShortAdvice }, + { "Improvement", "Schedules: Keep schedule run times across timezone changes", "✗ Release Notes: Same as the summary", SameAsSummaryAdvice }, + { "Epic", "Schedules: Keep schedule run times across timezone changes", "✗ Release Notes: Same as the summary", SameAsSummaryAdvice }, + { "Improvement", "Fixes the issue raised in MS-12345 for all users today", "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", NotCustomerFacingAdvice }, + { "Epic", "Fixes the issue raised in MS-12345 for all users today", "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", NotCustomerFacingAdvice }, + { "Improvement", $"Schedules now keep their run time {EmDash} even when the timezone changes", "✗ Release Notes: Contains an em dash", EmDashAdvice }, + { "Epic", $"Schedules now keep their run time {EmDash} even when the timezone changes", "✗ Release Notes: Contains an em dash", EmDashAdvice }, + { "Improvement", "Schedules now show the correct behavior when the timezone changes for tenants", "✗ Release Notes: US spelling", UsSpellingAdvice }, + { "Epic", "Schedules now show the correct behavior when the timezone changes for tenants", "✗ Release Notes: US spelling", UsSpellingAdvice }, + }; + + private static readonly string EmDash = ((char)0x2014).ToString(); + private const string EmptyAdvice = "Fill in the Release Notes field with one or two customer-facing sentences of at least 8 words saying what changed for the customer"; + private const string PlaceholderAdvice = "Replace the placeholder in the Release Notes field with one or two sentences saying what changed for the customer"; + private const string GenericAdvice = "Say what changed and for whom: a generic note such as 'Fixed bug' tells the customer nothing"; + private const string ShortAdvice = "Expand the Release Notes to at least 8 words, saying what changed and who it affects"; + private const string SameAsSummaryAdvice = "Describe the outcome for the customer rather than repeating the ticket summary"; + private const string NotCustomerFacingAdvice = "Remove ticket keys, file paths and code identifiers, and describe the change as the customer sees it"; + private const string EmDashAdvice = "Replace the em dash with a comma, colon or full stop"; + private const string UsSpellingAdvice = "Use British spelling, for example behaviour, colour, organisation, centre, favourite and cancelled"; + + [Theory] + [MemberData(nameof(FailingTiers))] + public async Task FailingTier_RecommendsAFixForItsOwnRule(string issueType, string? notes, string finding, string recommendation) + { + AssessmentReportItem item = NotesItem(await AssessAsync(notes, issueType: issueType)); + + item.Finding.Should().Be(finding); + item.Recommendation.Should().Be(recommendation); + item.Recommendation.Should().NotContain(EmDash); + } + + [Fact] + public async Task Good_Scores100_NoCap() + { + AssessmentReport report = await AssessAsync(GoodNotes); + + NotesItem(report).Finding.Should().Be("✓ Release Notes: Present and customer-facing"); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Fact] + public async Task InternalLabel_StillRequiresNotes() + { + AssessmentReport report = await AssessAsync(notes: null, labels: ["internal"]); + + NotesItem(report).RawScore.Should().Be(0); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task Epic_IsAssessed() + { + AssessmentReport report = await AssessAsync(notes: null, issueType: "Epic"); + + report.Categories.Single(category => category.Name == "Epic Quality").Assessments + .Single(item => item.Name.EndsWith("ReleaseNotes", StringComparison.Ordinal)).RawScore.Should().Be(0); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task OpsProject_NotApplicable_ScoreUnchanged() + { + NormalizedIssue ops = Issue(notes: null) with { Key = "OPS-1", Project = "OPS", Description = Description }; + IssueAssessor previous = new(Serialization.AssessmentRubric.Load(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Rubrics", "ticket-quality-1.1.6.json")))); + + AssessmentReport now = await ReportAsync(Assessor, ops); + AssessmentReport before = await ReportAsync(previous, ops); + + now.Score.Should().Be(before.Score); + now.Findings.Should().NotContain(finding => finding.Contains("Release Notes", StringComparison.Ordinal)); + now.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + now.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == "ReleaseNotes").Outcome.Should().Be(AssessmentOutcome.NotApplicable); + } + + [Theory] + [InlineData("Won't Do")] + [InlineData("Duplicate")] + public async Task ResolutionWontDo_NotApplicable_NoCap(string resolution) + { + AssessmentReport report = await AssessAsync(notes: null, resolution: resolution); + + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == "ReleaseNotes").Outcome.Should().Be(AssessmentOutcome.NotApplicable); + } + + [Theory] + [InlineData("Fixed")] + [InlineData("Done")] + public async Task ResolutionFixed_IsAssessed(string resolution) + { + AssessmentReport report = await AssessAsync(notes: null, resolution: resolution); + + NotesItem(report).RawScore.Should().Be(0); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + private static AssessmentReportItem NotesItem(AssessmentReport report) + => report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name.EndsWith("ReleaseNotes", StringComparison.Ordinal) && item.Outcome != AssessmentOutcome.NotApplicable); + + private static NormalizedIssue Issue(string? notes, string title = "Keep schedule run times across timezone changes", string issueType = "Improvement", string? resolution = null, ImmutableList? labels = null) + => new() + { + Key = "MS-1", + Project = "MS", + Title = title, + Description = Description, + IssueType = issueType, + Status = "In Progress", + Platform = "Jira", + ReleaseNotes = notes, + Resolution = resolution, + Labels = labels ?? [], + }; + + private static async Task ReportAsync(IssueAssessor assessor, NormalizedIssue issue) + { + using JsonDocument document = issue.ToJsonDocument(); + return AssessmentReport.From(await assessor.AssessAsync(document, CancellationToken)); + } + + private static Task AssessAsync(string? notes, string title = "Keep schedule run times across timezone changes", string issueType = "Improvement", string? resolution = null, ImmutableList? labels = null) + => ReportAsync(Assessor, Issue(notes, title, issueType, resolution, labels)); +} diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs new file mode 100644 index 0000000..f82260d --- /dev/null +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/TicketContractAssessmentTests.cs @@ -0,0 +1,261 @@ +using PanoramicData.IssueAssessment.Mapping; +using PanoramicData.IssueAssessment.Rubrics; + +namespace PanoramicData.IssueAssessment.Tests.Rubrics; + +/// +/// The MS-only TicketContract assessment: criteria are classified MUST/SHOULD/MAY, and from Ready for Test on a +/// Verification Evidence section is present. A failing tier caps the score. +/// +public class TicketContractAssessmentTests : TestBase +{ + private const string CapName = "Ticket contract"; + private static readonly IssueAssessor Assessor = new(BuiltInRubrics.PanoramicDataTicketQuality); + + private const string Summary = "h2. Summary\r\nRun times drift when the tenant timezone changes.\r\n\r\n"; + private const string ClassifiedCriteria = "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its local run time after a timezone change.\r\n# SHOULD: the schedule list shows the new offset.\r\n# MAY: the audit log records the change.\r\n\r\n"; + + private const string FullEvidence = """ + h2. Verification Evidence + + *Automated coverage of the MUST criteria* + || Criterion || Test || Location || + | MUST 1 | GitFileSystemProvider_RecoversFromMissingIndex | MagicSuite.SystemAccess.Test/Git/GitFileSystemProviderTests.cs | + | MUST 2 | (none - not reachable from a test) | - | + + *Reachability - how to make the covered code actually run* + Files > any Git-backed connection > open any file. The recovery path runs on the + first read after the clone is corrupted. + + *Not covered by automated tests, and why* + MUST 2 needs a corrupted clone on a shared RWX volume; not reproducible in-process. + + *Suggested QA route:* B (regression suite) + """; + + [Fact] + public async Task NoCriteriaSection_Fails() + { + AssessmentReport report = await AssessAsync(Summary + "h2. Notes\r\nNothing else to say."); + + ContractItem(report).RawScore.Should().Be(0); + ContractItem(report).Finding.Should().Be("✗ Ticket Contract: No acceptance criteria section"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + report.NormalizedScore.Should().BeLessThanOrEqualTo(0.79); + } + + [Fact] + public async Task EmptyCriteriaSection_Fails() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\nTo be agreed with the team.\r\n")) + .Finding.Should().Be("✗ Ticket Contract: Acceptance criteria section has no criteria"); + + [Fact] + public async Task OneUnclassified_Fails() + { + AssessmentReport report = await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n# the schedule list shows the new offset.\r\n"); + + ContractItem(report).Finding.Should().Be("✗ Ticket Contract: 1 criteria not classified MUST/SHOULD/MAY"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task AllShouldNoMust_Fails() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# SHOULD: a schedule keeps its run time.\r\n# SHOULD: the list shows the offset.\r\n")) + .Finding.Should().Be("✗ Ticket Contract: No MUST criterion"); + + [Fact] + public async Task ClassifiedInProgress_NoEvidenceNeeded_Passes() + { + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria); + + ContractItem(report).RawScore.Should().Be(100); + ContractItem(report).Finding.Should().Be("✓ Ticket Contract: Criteria classified"); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Theory] + [InlineData("Ready for Test")] + [InlineData("Ready for Customer")] + [InlineData("With Customer")] + public async Task EvidenceDue_NoEvidence_Fails(string status) + { + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria, status: status); + + ContractItem(report).Finding.Should().Be("✗ Ticket Contract: No Verification Evidence section"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task ReadyForTest_EvidenceMissingRoute_Fails() + { + string evidence = FullEvidence.Replace("*Suggested QA route:* B (regression suite)", string.Empty, StringComparison.Ordinal); + + ContractItem(await AssessAsync(Summary + ClassifiedCriteria + evidence, status: "Ready for Test")) + .Finding.Should().Be("✗ Ticket Contract: Verification Evidence incomplete"); + } + + [Theory] + [InlineData("Ready for Test")] + [InlineData("In Test")] + [InlineData("Ready for Customer")] + [InlineData("With Customer")] + [InlineData("Closed")] + public async Task ReadyForTest_FullEvidence_Passes(string status) + { + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + FullEvidence, status: status); + + ContractItem(report).RawScore.Should().Be(100); + ContractItem(report).Finding.Should().Be("✓ Ticket Contract: Criteria classified; evidence present"); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Fact] + public async Task RetrospectiveHeading_Passes() + { + string evidence = FullEvidence.Replace("h2. Verification Evidence", "h2. Verification Evidence (Retrospective)", StringComparison.Ordinal); + + ContractItem(await AssessAsync(Summary + ClassifiedCriteria + evidence, status: "Ready for Test")).RawScore.Should().Be(100); + } + + [Theory] + [InlineData("# MUST: x")] + [InlineData("# *MUST* x")] + [InlineData("* (SHOULD) x")] + [InlineData("| MAY 1 | x |")] + [InlineData("1. MUST - x")] + [InlineData("# *MUST* - PASS when a reference is struck through on its page")] + public async Task FormsAccepted(string criterion) + { + string criteria = $"h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n{criterion}\r\n"; + + AssessmentReport report = await AssessAsync(Summary + criteria); + + ContractItem(report).RawScore.Should().Be(100, ContractItem(report).Finding); + } + + [Fact] + public async Task InlineRfc2119Criteria_Ms27557Form_Pass() + { + const string criteria = "h2. Acceptance Criteria\r\n# SECTIONPAGES in a header or footer MUST show the page count of its section\r\n# NUMPAGES SHOULD still show the page count of the whole document\r\n"; + + ContractItem(await AssessAsync(Summary + criteria)).Finding.Should().Be("✓ Ticket Contract: Criteria classified"); + } + + [Fact] + public async Task LabelThenUnlabelledItems_Ms27609Form_Pass() + { + const string criteria = "h2. Acceptance Criteria\r\n*MUST*\r\n# A value encrypted under the old key is readable after rotation\r\n# The rotation is recorded in the audit log\r\n*SHOULD*\r\n# The admin is told when the rotation finishes\r\n"; + + ContractItem(await AssessAsync(Summary + criteria)).Finding.Should().Be("✓ Ticket Contract: Criteria classified"); + } + + [Fact] + public async Task MixedMustAndMayItem_IsNotClassified() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n# the list MUST refresh and MAY animate.\r\n")) + .Finding.Should().Be("✗ Ticket Contract: 1 criteria not classified MUST/SHOULD/MAY"); + + [Fact] + public async Task ShouldAndMayLabelsOnly_FailNoMust() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n*SHOULD*\r\n# a schedule keeps its run time.\r\n*MAY*\r\n# the list shows the offset.\r\n")) + .Finding.Should().Be("✗ Ticket Contract: No MUST criterion"); + + [Fact] + public async Task LowercaseMay_IsNotAClassification() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n# may show the offset.\r\n")) + .Finding.Should().Be("✗ Ticket Contract: 1 criteria not classified MUST/SHOULD/MAY"); + + [Fact] + public async Task BoldGroupLabelLine_IsNotACriterion() + { + const string criteria = "h2. Acceptance Criteria\r\n*MUST* (the ticket cannot pass without these)\r\n# *MUST* A value encrypted under the old key is readable\r\n# *SHOULD* Something else\r\n"; + + AssessmentReport report = await AssessAsync(Summary + criteria); + + ContractItem(report).RawScore.Should().Be(100, ContractItem(report).Finding); + ContractItem(report).Finding.Should().Be("✓ Ticket Contract: Criteria classified"); + } + + [Fact] + public async Task NestedSubPointsAndTableRules_AreNotCriteria() + => ContractItem(await AssessAsync(Summary + "h2. Acceptance Criteria\r\n# MUST: a schedule keeps its run time.\r\n## including daylight saving changes\r\n - and leap days\r\n| MAY | x |\r\n|---|---|\r\n")) + .RawScore.Should().Be(100); + + [Fact] + public async Task EvidenceTableDoesNotTriggerDuplicationCap() + { + AssessmentReport report = await AssessAsync(Summary + ClassifiedCriteria + FullEvidence, status: "Ready for Test"); + + report.Caps.Select(cap => cap.Name).Should().NotContain("Criteria duplication"); + report.Findings.Should().NotContain(finding => finding.StartsWith("✗ Duplication", StringComparison.Ordinal)); + } + + [Fact] + public async Task Epic_NotApplicable() + { + AssessmentReport report = await AssessAsync(Summary + "h2. Notes\r\nNo criteria here.", issueType: "Epic"); + + report.Categories.SelectMany(category => category.Assessments) + .Where(item => item.Name == "TicketContract") + .Should().OnlyContain(item => item.Outcome == AssessmentOutcome.NotApplicable); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Fact] + public async Task Bug_RequiresCriteriaHeading() + { + const string bug = "h2. Steps to Reproduce\r\n# Open a schedule.\r\n# Change the tenant timezone.\r\n\r\nh2. Current Behaviour\r\nThe run time moves.\r\n\r\nh2. Expected Behaviour\r\nThe run time stays.\r\n"; + + AssessmentReport report = await AssessAsync(bug, issueType: "Bug"); + + ContractItem(report).Finding.Should().Be("✗ Ticket Contract: No acceptance criteria section"); + report.Caps.Select(cap => cap.Name).Should().Contain(CapName); + } + + [Fact] + public async Task OpsProject_NotApplicable() + { + NormalizedIssue ops = Issue(Summary, "Improvement", "In Progress", null) with { Key = "OPS-1", Project = "OPS" }; + + AssessmentReport report = await ReportAsync(ops); + + report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == "TicketContract").Outcome.Should().Be(AssessmentOutcome.NotApplicable); + report.Findings.Should().NotContain(finding => finding.Contains("Ticket Contract", StringComparison.Ordinal)); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + [Fact] + public async Task Undelivered_NotApplicable() + { + AssessmentReport report = await AssessAsync(Summary, resolution: "Won't Do"); + + report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == "TicketContract").Outcome.Should().Be(AssessmentOutcome.NotApplicable); + report.Caps.Select(cap => cap.Name).Should().NotContain(CapName); + } + + private static AssessmentReportItem ContractItem(AssessmentReport report) + => report.Categories.SelectMany(category => category.Assessments).Single(item => item.Name == "TicketContract" && item.Outcome != AssessmentOutcome.NotApplicable); + + private static NormalizedIssue Issue(string description, string issueType, string status, string? resolution) + => new() + { + Key = "MS-1", + Project = "MS", + Title = "Keep schedule run times across timezone changes", + Description = description, + IssueType = issueType, + Status = status, + Platform = "Jira", + ReleaseNotes = "Reports now keep their scheduled run time after the tenant timezone changes", + Resolution = resolution, + Labels = [], + }; + + private static async Task ReportAsync(NormalizedIssue issue) + { + using JsonDocument document = issue.ToJsonDocument(); + return AssessmentReport.From(await Assessor.AssessAsync(document, CancellationToken)); + } + + private static Task AssessAsync(string description, string issueType = "Improvement", string status = "In Progress", string? resolution = null) + => ReportAsync(Issue(description, issueType, status, resolution)); +} diff --git a/PanoramicData.IssueAssessment.Tests/Rubrics/ticket-quality-1.1.6.json b/PanoramicData.IssueAssessment.Tests/Rubrics/ticket-quality-1.1.6.json new file mode 100644 index 0000000..47830d0 --- /dev/null +++ b/PanoramicData.IssueAssessment.Tests/Rubrics/ticket-quality-1.1.6.json @@ -0,0 +1,609 @@ +{ + // Panoramic Data ticket quality rubric. + // Ported from PanoramicData.Skills .github/skills/_common/IssueQualityCore.ps1 (Yoda), which defined it first: + // factors, weights, patterns, findings and recommendations are transcribed from there, so both score a ticket the + // same. Epics use their own rubric; Change Requests add a rollback plan; Bugs need steps to reproduce. + "executionStrategy": "series", + "conditions": { + "isBug": { "type": "fieldMatches", "path": "$.issueType", "patterns": [ "Bug|Defect" ] }, + "isEpic": { "type": "fieldEquals", "path": "$.issueType", "values": [ "Epic" ] }, + "isChangeRequest": { "type": "fieldEquals", "path": "$.issueType", "values": [ "Change Request" ] }, + "isInvestigation": { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "ref", "name": "isEpic" } }, + { + "type": "any", + "conditions": [ + { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "ref", "name": "isBug" } }, + { + "type": "textMatches", + "source": { "paths": [ "$.title", "$.description" ] }, + "patterns": [ "\\binvestigate\\b", "\\bresearch\\b", "\\bspike\\b", "\\banalysis\\b", "\\bexplore\\b", "\\bfeasibility\\b", "\\bproof\\s+of\\s+concept\\b", "\\bpoc\\b", "\\bstudy\\b", "\\bevaluate\\b", "\\bassess\\b" ] + } + ] + }, + { "type": "fieldEquals", "path": "$.labels", "values": [ "investigation", "research", "spike" ] } + ] + } + ] + }, + "supportsComponents": { + "type": "any", + "conditions": [ + { "type": "fieldEquals", "path": "$.supportsComponents", "values": [ "true" ] }, + { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "fieldPresent", "path": "$.supportsComponents" } }, + { "type": "not", "condition": { "type": "fieldEquals", "path": "$.project", "values": [ "PS" ] } } + ] + } + ] + }, + "supportsFixVersion": { "type": "not", "condition": { "type": "fieldEquals", "path": "$.supportsFixVersion", "values": [ "false" ] } }, + "shouldHaveFixVersion": { "type": "fieldEquals", "path": "$.status", "values": [ "Ready for Test", "In Test", "Ready for Release", "Closed", "Done", "Resolved" ] }, + "hasFixVersion": { "type": "fieldPresent", "path": "$.fixVersions" }, + "hasEnvironmentMention": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "test2|alpha|beta|staging|production|environment" ] }, + "nonUiComponentsOnly": { + "type": "fieldMatches", + "path": "$.components", + "requireAll": true, + "patterns": [ "API", "API\\ Client", "Backend", "Database", "Data", "Connections", "Workers", "Worker", "Scheduler", "Schedules", "Infrastructure", "Security", "Authentication", "Migration", "Unit\\ tests", "Performance", "Configuration", "Logging" ] + }, + "uiConsistencyApplies": { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "ref", "name": "isBug" } }, + { "type": "not", "condition": { "type": "ref", "name": "isInvestigation" } }, + { "type": "fieldMatches", "path": "$.issueType", "patterns": [ "Feature|Improvement|Story|Enhancement" ] } + ] + }, + "isUiTicket": { + "type": "all", + "conditions": [ + { "type": "ref", "name": "uiConsistencyApplies" }, + { + "type": "any", + "conditions": [ + { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "ref", "name": "nonUiComponentsOnly" } }, + { "type": "fieldMatches", "path": "$.components", "patterns": [ "\\bUI\\b", "\\bFrontend\\b", "\\bFront-End\\b", "\\bWeb\\b", "\\bPortal\\b", "\\bDashboard\\b" ] } + ] + }, + { "type": "fieldMatches", "path": "$.labels", "patterns": [ "^ui$", "^frontend$", "^front-end$", "^web-ui$", "^ux$", "^visual$", "^design$" ] }, + { + "type": "all", + "conditions": [ + { "type": "not", "condition": { "type": "ref", "name": "nonUiComponentsOnly" } }, + { + "type": "measure", + "atLeast": 2, + "of": { + "type": "patternMatchCount", + "source": { "paths": [ "$.title", "$.description" ] }, + "patterns": [ "\\bUI\\b", "\\buser\\s+interface\\b", "\\bscreen\\b", "\\bdialog\\b", "\\bmodal\\b", "\\bform\\b", "\\bbutton\\b", "\\bpage\\b", "\\bpanel\\b", "\\bdashboard\\b", "\\bwidget\\b", "\\bview\\b", "\\blayout\\b", "\\btoolbar\\b", "\\bmenu\\b", "\\bdropdown\\b", "\\btab\\b", "\\bnavigation\\b", "\\bfront[- ]?end\\b", "\\bstyle\\b", "\\bCSS\\b", "\\btheme\\b", "\\bicon\\b", "\\btooltip\\b", "\\bpopup\\b", "\\bpopover\\b", "\\bgrid\\b", "\\btable\\b", "\\bcolumn\\b", "\\bheader\\b", "\\bfooter\\b", "\\bsidebar\\b" ] + } + } + ] + } + ] + } + ] + }, + "hasPanelableSections": { + "type": "all", + "conditions": [ + { "type": "fieldEquals", "path": "$.platform", "values": [ "Jira" ] }, + { "type": "measure", "atLeast": 2, "of": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "panelable" } } + ] + }, + "isCliRelated": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "magicsuite\\s+cli", "ms\\s+cli", "command[\\s-]?line", "msmagic", "cli\\s+command", "--\\w+", "invoke[\\s-]command" ] }, + "hasCliLabel": { "type": "fieldMatches", "path": "$.labels", "patterns": [ "CLI|Command" ] }, + "hasCliComponent": { "type": "fieldMatches", "path": "$.components", "patterns": [ "CLI" ] }, + "hasUiIndicators": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "button", "click", "dialog", "modal", "dropdown", "menu", "screen", "page", "form", "input\\s+field", "checkbox", "radio", "tab", "panel", "dashboard", "report\\s*studio", "ui\\s+", "user\\s+interface", "browser", "blazor" ] }, + "hasUiLabel": { "type": "fieldMatches", "path": "$.labels", "patterns": [ "^(UI|Blazor|ReportStudio)$" ] } + }, + "categories": [ + { + "name": "Ticket Quality", + "description": "Specification completeness of a bug, story, task or change request.", + "assessmentExecutionStrategy": "series", + "appliesWhen": { "type": "not", "condition": { "type": "ref", "name": "isEpic" } }, + "assessments": [ + { + "name": "Component", + "description": "The component the ticket concerns is identified.", + "weight": 14, + "appliesWhen": { "type": "ref", "name": "supportsComponents" }, + "notApplicableExplanation": "○ Component: Not applicable for the {$.project} project", + "definition": { + "type": "tiered", + "measures": { "components": { "type": "itemCount", "path": "$.components" } }, + "tiers": [ + { "when": { "type": "measure", "measure": "components", "atLeast": 1 }, "score": 100, "finding": "✓ Component: {$.components[0]}" }, + { "when": { "type": "fieldMatches", "path": "$.labels", "patterns": [ "DataMagic|ReportMagic|AlertMagic|Admin|Connect|CLI|Docs" ] }, "score": 70, "finding": "⚠ Component: Via label ({$.labels~DataMagic|ReportMagic|AlertMagic|Admin|Connect|CLI|Docs})", "recommendation": "Set component field" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.title" ] }, "patterns": [ "DataMagic|ReportMagic|AlertMagic|Admin|Connect|CLI|Docs|Report\\s*Studio" ] }, "score": 50, "finding": "⚠ Component: Mentioned in summary only", "recommendation": "Set component field explicitly" }, + { "score": 0, "finding": "✗ Component: Not identified", "recommendation": "Add component to issue" } + ] + } + }, + { + "name": "Description", + "description": "The description says enough.", + "weight": 14, + "definition": { + "type": "tiered", + "measures": { "words": { "type": "wordCount", "source": { "paths": [ "$.description" ] } } }, + "tiers": [ + { "when": { "type": "measure", "measure": "words", "atLeast": 100 }, "score": 100, "finding": "✓ Description: {words} words (comprehensive)" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 50 }, "score": 80, "finding": "✓ Description: {words} words (adequate)" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 20 }, "score": 50, "finding": "⚠ Description: {words} words (brief)", "recommendation": "Add more context to description" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 1 }, "score": 25, "finding": "⚠ Description: {words} words (minimal)", "recommendation": "Expand description with details" }, + { "score": 0, "finding": "✗ Description: Empty", "recommendation": "Add description" } + ] + } + }, + { + "name": "Testability", + "description": "Pass/fail criteria, or for an investigation its scope and deliverables, are stated.", + "weight": 16, + "definition": { + "type": "tiered", + "measures": { + "scope": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "scope:", "h[1-6]\\.[^\\r\\n]*scope", "\\*\\*[^\\r\\n]*scope", "questions?\\s+to\\s+(answer|investigate)", "investigate\\s+(whether|if|how|why)", "determine\\s+(whether|if|how|why)", "find\\s+out", "understand\\s+(how|why|what)" ] }, + "deliverables": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "deliverable", "findings", "recommendation", "report", "document", "conclusion", "outcome", "result\\s+of\\s+(investigation|research|analysis)" ] }, + "criteria": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "should\\s+(be|display|show|return|have|not)", "must\\s+(be|display|show|return|have|not)", "verify\\s+that", "acceptance\\s+criteria", "h[23]\\.\\s*acceptance\\s+criteria", "expected\\s+(result|behavior|behaviour|outcome)", "pass\\s+criteria", "success\\s+criteria" ] }, + "criteriaHeadings": { "type": "criteriaHeadingCount", "source": { "paths": [ "$.description" ] } } + }, + "tiers": [ + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "scope", "atLeast": 2 }, { "type": "measure", "measure": "deliverables", "atLeast": 1 } ] }, "score": 100, "finding": "✓ Investigation Clarity: Clear scope and deliverables" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "scope", "atLeast": 1 }, { "type": "measure", "measure": "deliverables", "atLeast": 1 } ] }, "score": 80, "finding": "✓ Investigation Clarity: Scope and deliverables defined" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "scope", "atLeast": 1 } ] }, "score": 50, "finding": "⚠ Investigation Clarity: Missing deliverables", "recommendation": "Define investigation deliverables (findings document, recommendation, report)" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "deliverables", "atLeast": 1 } ] }, "score": 50, "finding": "⚠ Investigation Clarity: Missing scope", "recommendation": "Define investigation scope and questions to answer" }, + { "when": { "type": "ref", "name": "isInvestigation" }, "score": 20, "finding": "✗ Investigation Clarity: No scope or deliverables", "recommendation": "Add investigation scope, questions to answer, and expected deliverables" }, + { "when": { "type": "measure", "measure": "criteria", "atLeast": 3 }, "score": 100, "finding": "✓ Testability: Clear criteria found" }, + { "when": { "type": "all", "conditions": [ { "type": "measure", "measure": "criteria", "atLeast": 1 }, { "type": "measure", "measure": "criteriaHeadings", "lessThan": 1 } ] }, "score": 60, "finding": "⚠ Testability: Some criteria found", "recommendation": "Add explicit pass/fail criteria" }, + { "when": { "type": "measure", "measure": "criteria", "atLeast": 1 }, "score": 60, "finding": "⚠ Testability: Some criteria found", "recommendation": "Make the existing acceptance criteria explicit pass/fail checks, rather than adding another section" }, + { "score": 20, "finding": "✗ Testability: No clear criteria", "recommendation": "Define acceptance criteria" } + ] + } + }, + { + "name": "StepsToReproduce", + "description": "A bug has numbered steps to reproduce under their own heading.", + "weight": 12, + "definition": { + "type": "tiered", + "measures": { + "steps": { "type": "occurrenceCount", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" }, "pattern": "(?m)^[ \\t]*(?:#(?!#)|\\d+[.)])[ \\t]+\\S" }, + "listedElsewhere": { "type": "occurrenceCount", "source": { "paths": [ "$.description" ] }, "pattern": "(?m)^[ \\t]*(?:#(?!#)|\\d+[.)])[ \\t]+\\S" }, + "sectionText": { "type": "alphanumericCount", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" } } + }, + "tiers": [ + { "when": { "type": "ref", "name": "isInvestigation" }, "score": 100, "finding": "○ Steps: Not required for investigations" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "not", "condition": { "type": "sectionPresent", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" } } }, { "type": "measure", "measure": "listedElsewhere", "atLeast": 2 } ] }, "score": 0, "finding": "✗ Steps to Reproduce: No steps-to-reproduce section (a numbered list was found, but not under a Steps to Reproduce heading)", "recommendation": "Add an 'h2. Steps to Reproduce' section with numbered steps (Jira '# step' or '1. step')" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "not", "condition": { "type": "sectionPresent", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" } } } ] }, "score": 0, "finding": "✗ Steps to Reproduce: No steps-to-reproduce section", "recommendation": "Add an 'h2. Steps to Reproduce' section with numbered steps (Jira '# step' or '1. step')" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "measure", "measure": "steps", "atLeast": 2 } ] }, "score": 100, "finding": "✓ Steps to Reproduce: {steps} numbered steps" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" }, "patterns": [ "(?i)\\b(?:not|never)\\s+(?:been\\s+)?(?:reliably\\s+|consistently\\s+)?reproduc|\\b(?:cannot|can't|could\\s+not|couldn't|unable\\s+to)\\s+(?:be\\s+)?(?:reliably\\s+|consistently\\s+)?reproduc|\\bintermittent" ] }, { "type": "measure", "measure": "sectionText", "atLeast": 60 } ] }, "score": 60, "finding": "⚠ Steps to Reproduce: Stated as not reproducible on demand, with observed conditions", "recommendation": "Add numbered steps once a reliable reproduction is found; until then the ticket stays at GOOD" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" }, "patterns": [ "(?i)\\b(?:not|never)\\s+(?:been\\s+)?(?:reliably\\s+|consistently\\s+)?reproduc|\\b(?:cannot|can't|could\\s+not|couldn't|unable\\s+to)\\s+(?:be\\s+)?(?:reliably\\s+|consistently\\s+)?reproduc|\\bintermittent" ] } ] }, "score": 30, "finding": "⚠ Steps to Reproduce: Stated as not reproducible, with no observed conditions", "recommendation": "Describe when it was seen: environment, frequency, time, data and what was being done" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "measure", "measure": "steps", "atLeast": 1 } ] }, "score": 60, "finding": "⚠ Steps to Reproduce: Only one numbered step", "recommendation": "Break the reproduction into numbered steps from a known starting state" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.description" ], "section": "(?:reproduc|\\brepro\\b)" } }, { "type": "measure", "measure": "sectionText", "lessThan": 15 } ] } ] }, "score": 0, "finding": "✗ Steps to Reproduce: Heading present but the section is empty or a placeholder", "recommendation": "Fill in the steps to reproduce - a heading over 'TBD' does not say how to see the bug" }, + { "when": { "type": "ref", "name": "isBug" }, "score": 40, "finding": "⚠ Steps to Reproduce: Section present but the steps are not numbered", "recommendation": "Rewrite the steps to reproduce as a numbered list" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "steps?\\s*(to\\s+reproduce|:)", "(?m)^\\s*\\d+\\.\\s+", "(?m)^\\s*#\\s+", "(?m)^\\s*-\\s+", "h[23]\\.\\s*steps?\\s*(to\\s+reproduce)?", "how\\s+to\\s+reproduce", "reproduction\\s+steps", "repro\\s+steps" ] }, "score": 100, "finding": "✓ Steps: Workflow documented" }, + { "score": 100, "finding": "○ Steps: Not required for this type" } + ] + } + }, + { + "name": "CurrentVsExpected", + "description": "Current and expected behaviour are documented, or for an investigation its background.", + "weight": 12, + "definition": { + "type": "tiered", + "measures": { + "context": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "background:", "h[1-6]\\.[^\\r\\n]*background", "context:", "h[1-6]\\.[^\\r\\n]*context", "\\*\\*[^\\r\\n]*(background|context)", "motivation:", "why\\s+it\\s+matters", "why\\s+(we|this)", "reason\\s+for", "because" ] }, + "current": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "current(ly)?(\\s+behavior|\\s+behaviour)?:", "h[23]\\.\\s*current(\\s*/\\s*actual)?\\s*(behavior|behaviour)", "\\*\\*current\\s*(behavior|behaviour)\\*\\*", "\\*current(\\s*/\\s*actual)?\\s*(behavior|behaviour)?\\*", "\\*actual\\s*(behavior|behaviour)?\\*", "actual(\\s+behavior|\\s+behaviour)?:", "what\\s+happens:", "observed:", "bug:", "issue:", "problem:" ] }, + "expected": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "expected(\\s+behavior|\\s+behaviour)?:", "h[23]\\.\\s*expected\\s*(behavior|behaviour)", "\\*\\*expected\\s*(behavior|behaviour)\\*\\*", "\\*expected\\s*(behavior|behaviour)?\\*", "should\\s+be:", "desired:", "correct\\s+behavior:", "fix:", "solution:", "h2\\.\\s*acceptance\\s+criteria", "h3\\.\\s*acceptance\\s+criteria", "acceptance\\s+criteria:", "\\*\\*acceptance\\s+criteria\\*\\*" ] } + }, + "tiers": [ + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "context", "atLeast": 2 } ] }, "score": 100, "finding": "✓ Investigation Context: Well documented" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isInvestigation" }, { "type": "measure", "measure": "context", "atLeast": 1 } ] }, "score": 70, "finding": "⚠ Investigation Context: Partial context", "recommendation": "Add more background/motivation for the investigation" }, + { "when": { "type": "ref", "name": "isInvestigation" }, "score": 30, "finding": "⚠ Investigation Context: Missing context", "recommendation": "Add background and motivation for why this investigation is needed" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "measure", "measure": "current", "atLeast": 1 }, { "type": "measure", "measure": "expected", "atLeast": 1 } ] }, "score": 100, "finding": "✓ Current vs Expected: Both documented" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "measure", "measure": "current", "atLeast": 1 } ] }, "score": 50, "finding": "⚠ Current vs Expected: Missing expected behavior", "recommendation": "Document both current and expected behavior" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "isBug" }, { "type": "measure", "measure": "expected", "atLeast": 1 } ] }, "score": 50, "finding": "⚠ Current vs Expected: Missing current behavior", "recommendation": "Document both current and expected behavior" }, + { "when": { "type": "ref", "name": "isBug" }, "score": 0, "finding": "✗ Current vs Expected: Neither documented", "recommendation": "Add current behavior and expected behavior sections" }, + { "when": { "type": "measure", "measure": "expected", "atLeast": 1 }, "score": 100, "finding": "✓ Expected Behavior: Documented" }, + { "score": 40, "finding": "⚠ Expected Behavior: Not explicit", "recommendation": "Clarify expected outcome" } + ] + } + }, + { + "name": "InvestigationTimebox", + "description": "An investigation states how long it may take. Recommendation only; it carries no weight.", + "weight": 0, + "appliesWhen": { "type": "ref", "name": "isInvestigation" }, + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "timebox", "time-?box", "\\d+\\s*(hour|day|week)s?", "maximum\\s+time", "time\\s+limit" ] }, "score": 100, "finding": "" }, + { "score": 0, "finding": "", "recommendation": "Consider adding a timebox for the investigation" } + ] + } + }, + { + "name": "EnvironmentVersion", + "description": "The fix version and the environment are stated.", + "weight": 7, + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "not", "condition": { "type": "ref", "name": "supportsFixVersion" } }, "score": 100, "finding": "○ Fix Version: Not applicable on {$.platform}" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "hasFixVersion" }, { "type": "ref", "name": "hasEnvironmentMention" } ] }, "score": 100, "finding": "✓ Fix Version: {$.fixVersions[0]}" }, + { "when": { "type": "ref", "name": "hasFixVersion" }, "score": 70, "finding": "✓ Fix Version: {$.fixVersions[0]}" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "shouldHaveFixVersion" }, { "type": "ref", "name": "hasEnvironmentMention" } ] }, "score": 30, "finding": "⚠ Fix Version: Not specified (expected for '{$.status}' status)", "recommendation": "Add fix version" }, + { "when": { "type": "ref", "name": "shouldHaveFixVersion" }, "score": 0, "finding": "⚠ Fix Version: Not specified (expected for '{$.status}' status)", "recommendation": "Add fix version" }, + { "score": 100, "finding": "○ Fix Version: Not required yet (status: {$.status})" } + ] + } + }, + { + "name": "Attachments", + "description": "A bug carries a screenshot or log.", + "weight": 3, + "definition": { + "type": "tiered", + "measures": { "attachments": { "type": "number", "path": "$.attachmentsCount" } }, + "tiers": [ + { "when": { "type": "measure", "measure": "attachments", "atLeast": 1 }, "score": 100, "finding": "✓ Attachments: {attachments} item(s)" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.description" ] }, "patterns": [ "!\\[|screenshot|image|http.*\\.(png|jpg|gif)" ] }, "score": 70, "finding": "✓ Attachments: Inline images found" }, + { "when": { "type": "ref", "name": "isBug" }, "score": 0, "finding": "⚠ Attachments: None (recommended for bugs)", "recommendation": "Add screenshot or error log" }, + { "score": 100, "finding": "○ Attachments: None (optional for non-bugs)" } + ] + } + }, + { + "name": "Documentation", + "description": "Documentation or help updates are referenced.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "references": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "documentation|docs?", "help\\s+(file|topic|section)", "user\\s+guide", "knowledge\\s+base|kb\\s+article", "wiki|confluence", "readme|readme\\.md", "api\\s+(reference|documentation|docs)", "manual", "guide|tutorial" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "references", "atLeast": 2 }, "score": 100, "finding": "✓ Documentation: Multiple references found" }, + { "when": { "type": "measure", "measure": "references", "atLeast": 1 }, "score": 70, "finding": "✓ Documentation: Referenced", "recommendation": "Consider adding more documentation updates" }, + { "score": 0, "finding": "⚠ Documentation: Not referenced", "recommendation": "Update documentation/help text for this change" } + ] + } + }, + { + "name": "Certifications", + "description": "A new feature has a certification exam question.", + "weight": 4, + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "ref", "name": "isBug" }, "score": 100, "finding": "○ Certifications: Not applicable for bugs" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "certifica?tion|exam|certification\\s+question", "cert\\s+question", "assessment\\s+question", "test\\s+question" ] }, "score": 100, "finding": "✓ Certifications: Exam question(s) created" }, + { "score": 0, "finding": "○ Certifications: No exam question created", "recommendation": "Consider creating a certification exam question for this new feature" } + ] + } + }, + { + "name": "UIConsistency", + "description": "A UI change references the look and feel of similar features, or says there are none.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "references": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "consistent\\s+(with|look|feel|style|design|behaviour|behavior)", "look\\s+and\\s+feel", "match(es|ing)?\\s+(the|existing|current|other|similar)", "similar\\s+(to|feature|screen|page|dialog|panel|view|component)", "same\\s+(style|approach|pattern|design|way)\\s+(as|used)", "in\\s+line\\s+with", "align(ed)?\\s+with", "follows?\\s+(the|existing|current)\\s+(pattern|convention|style|design)", "precedent", "no\\s+similar\\s+(feature|screen|page|component|view)", "no\\s+existing\\s+(feature|screen|page|component|view)", "first\\s+of\\s+(its|this)\\s+kind", "new\\s+pattern", "no\\s+precedent", "h[23]\\.\\s*UI\\s+Consistency", "\\*\\*UI\\s+Consistency\\*\\*", "UI\\s+consistency:" ] } }, + "tiers": [ + { "when": { "type": "not", "condition": { "type": "ref", "name": "uiConsistencyApplies" } }, "score": 100, "finding": "○ UI Consistency: Not applicable for {$.issueType}" }, + { "when": { "type": "not", "condition": { "type": "ref", "name": "isUiTicket" } }, "score": 100, "finding": "○ UI Consistency: Not a UI ticket" }, + { "when": { "type": "measure", "measure": "references", "atLeast": 2 }, "score": 100, "finding": "✓ UI Consistency: Look & feel references with similar features" }, + { "when": { "type": "measure", "measure": "references", "atLeast": 1 }, "score": 70, "finding": "⚠ UI Consistency: Partial reference to similar features", "recommendation": "Expand UI consistency details - reference specific similar features or state there are none" }, + { "score": 0, "finding": "✗ UI Consistency: No reference to look & feel or similar features", "recommendation": "Add UI consistency details or explicitly state no similar features exist" } + ] + } + }, + { + "name": "BusinessCase", + "description": "The value, impact or cost of not doing the work is stated.", + "weight": 12, + "definition": { + "type": "tiered", + "measures": { "justification": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "business\\s+case", "business\\s+value", "business\\s+justification", "business\\s+impact", "business\\s+benefit", "cost\\s+of\\s+(not\\s+doing|inaction|delay)", "return\\s+on\\s+investment", "\\bROI\\b", "value\\s+proposition", "revenue\\s+impact", "customer\\s+impact", "user\\s+impact", "compliance\\s+requirement", "regulatory\\s+requirement", "risk\\s+(reduction|mitigation)", "operational\\s+(benefit|efficiency|saving)", "h[23]\\.\\s*(Business\\s+Case|Business\\s+Value|Business\\s+Justification|Motivation|Rationale)", "\\*\\*(Business\\s+Case|Business\\s+Value|Justification|Motivation|Rationale)\\*\\*", "why\\s+(this|we)\\s+(is|are|need|should|must)", "justif(y|ication|ied)", "rationale" ] } }, + "tiers": [ + { "when": { "type": "ref", "name": "isBug" }, "score": 100, "finding": "○ Business Case: Not required for bugs" }, + { "when": { "type": "ref", "name": "isInvestigation" }, "score": 100, "finding": "○ Business Case: Not required for investigations" }, + { "when": { "type": "measure", "measure": "justification", "atLeast": 3 }, "score": 100, "finding": "✓ Business Case: Clear justification provided" }, + { "when": { "type": "measure", "measure": "justification", "atLeast": 2 }, "score": 80, "finding": "✓ Business Case: Justification referenced" }, + { "when": { "type": "measure", "measure": "justification", "atLeast": 1 }, "score": 50, "finding": "⚠ Business Case: Brief justification", "recommendation": "Expand business case with impact, value, or cost of inaction" }, + { "score": 0, "finding": "✗ Business Case: Not provided", "recommendation": "Add a business case explaining the value, impact, or cost of not doing this work" } + ] + } + }, + { + "name": "RollbackPlan", + "description": "A change request documents how the change is undone.", + "weight": 34, + "appliesWhen": { "type": "ref", "name": "isChangeRequest" }, + "definition": { + "type": "tiered", + "measures": { "planText": { "type": "alphanumericCount", "source": { "paths": [ "$.description" ], "section": "(?:rollback|mitigation)" } } }, + "tiers": [ + { "when": { "type": "not", "condition": { "type": "sectionPresent", "source": { "paths": [ "$.description" ], "section": "(?:rollback|mitigation)" } } }, "score": 0, "finding": "✗ Rollback Plan: No rollback or mitigation section", "recommendation": "Add a 'Rollback plan' section: what is captured before the change, what reverses it, and what is not reversible" }, + { "when": { "type": "placeholder", "source": { "paths": [ "$.description" ], "section": "(?:rollback|mitigation)" } }, "score": 0, "finding": "✗ Rollback Plan: Heading present but the section is empty or a placeholder", "recommendation": "Fill in the rollback plan - a heading over 'TBD' does not describe how the change is undone" }, + { "when": { "type": "measure", "measure": "planText", "lessThan": 15 }, "score": 0, "finding": "✗ Rollback Plan: Section too short to describe anything", "recommendation": "Expand the rollback plan to state what reverses the change and what cannot be reversed" }, + { "when": { "type": "measure", "measure": "planText", "lessThan": 120 }, "score": 60, "finding": "⚠ Rollback Plan: Present but brief", "recommendation": "Expand the rollback plan: name what is captured beforehand, and say what is irreversible and what compensates for it" }, + { "score": 100, "finding": "✓ Rollback Plan: Documented" } + ] + } + }, + { + "name": "Presentation", + "description": "Each section sits in an untitled panel directly under its h2 heading, so the layout can be scanned and found again; sections holding a table or code block are exempt.", + "weight": 8, + "appliesWhen": { "type": "ref", "name": "hasPanelableSections" }, + "definition": { + "type": "tiered", + "measures": { + "panelable": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "panelable" }, + "panelled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "panelled" }, + "unpanelled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "unpanelled" }, + "titled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "titled" } + }, + "tiers": [ + { "when": { "type": "all", "conditions": [ { "type": "measure", "measure": "unpanelled", "lessThan": 1 }, { "type": "measure", "measure": "titled", "lessThan": 1 } ] }, "score": 100, "finding": "✓ Presentation: {panelled} sections in untitled panels" }, + { "when": { "type": "measure", "measure": "unpanelled", "lessThan": 1 }, "score": 70, "finding": "⚠ Presentation: {titled} panel titles repeat their heading ({titled:text})", "recommendation": "Remove the title from each panel that sits directly under a heading, since the heading already names it: keep only the background colour, as in panel:bgColor=#F4F5F7" }, + { "when": { "type": "measure", "measure": "panelled", "atLeast": 1 }, "score": 50, "finding": "⚠ Presentation: {unpanelled} of {panelable} sections are not in a panel ({unpanelled:text})", "recommendation": "Put the content of each section in an untitled panel directly under its heading; a section holding a table or code block can stay as it is" }, + { "score": 20, "finding": "✗ Presentation: None of {panelable} sections is in a panel ({unpanelled:text})", "recommendation": "Put the content of each section in an untitled panel directly under its heading, for example one with only a background colour, panel:bgColor=#F4F5F7; a section holding a table or code block can stay as it is" } + ] + } + }, + { + "name": "CriteriaDuplication", + "description": "Acceptance criteria are stated under one heading only. It carries no weight; duplication caps the score instead.", + "weight": 0, + "definition": { + "type": "tiered", + "measures": { "headings": { "type": "criteriaHeadingCount", "source": { "paths": [ "$.description" ] } } }, + "tiers": [ + { "when": { "type": "measure", "measure": "headings", "atLeast": 2 }, "score": 0, "finding": "✗ Duplication: acceptance criteria stated under {headings} headings ({headings:text}), so EXCELLENT is unreachable (score capped at 79)", "recommendation": "State the acceptance criteria once, under one heading, and remove the others" }, + { "score": 100, "finding": "" } + ] + } + } + ] + }, + { + "name": "Epic Quality", + "description": "Specification completeness of an epic: design, breakdown, scope and success criteria.", + "assessmentExecutionStrategy": "series", + "appliesWhen": { "type": "ref", "name": "isEpic" }, + "assessments": [ + { + "name": "EpicComponent", + "description": "The component the epic concerns is set.", + "weight": 10, + "appliesWhen": { "type": "ref", "name": "supportsComponents" }, + "notApplicableExplanation": "○ Component: Not applicable for the {$.project} project", + "definition": { + "type": "tiered", + "measures": { "components": { "type": "itemCount", "path": "$.components" } }, + "tiers": [ + { "when": { "type": "measure", "measure": "components", "atLeast": 1 }, "score": 100, "finding": "✓ Component: {$.components[0]}" }, + { "score": 0, "finding": "✗ Component: Not set", "recommendation": "Set the component field" } + ] + } + }, + { + "name": "EpicDescription", + "description": "The epic's description states the problem and the solution.", + "weight": 18, + "definition": { + "type": "tiered", + "measures": { "words": { "type": "wordCount", "source": { "paths": [ "$.description" ] } } }, + "tiers": [ + { "when": { "type": "measure", "measure": "words", "atLeast": 200 }, "score": 100, "finding": "✓ Description: {words} words (comprehensive)" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 100 }, "score": 80, "finding": "✓ Description: {words} words (good)" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 50 }, "score": 50, "finding": "⚠ Description: {words} words (brief for an epic)", "recommendation": "Expand description with problem statement and solution overview" }, + { "when": { "type": "measure", "measure": "words", "atLeast": 1 }, "score": 25, "finding": "⚠ Description: {words} words (minimal)", "recommendation": "Add comprehensive problem statement and solution overview" }, + { "score": 0, "finding": "✗ Description: Empty", "recommendation": "Add description with problem statement and solution overview" } + ] + } + }, + { + "name": "Architecture", + "description": "The technical design is described.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "design": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "architect(ure|ural)", "design\\s+(overview|decision|approach)", "technical\\s+(design|approach|solution|overview)", "flow\\s+diagram", "sequence\\s+diagram", "data\\s+flow", "system\\s+design", "component\\s+diagram", "integration\\s+point", "API\\s+(design|contract|endpoint)", "schema\\s+(design|change)", "database\\s+(design|change|migration)", "h[23]\\.\\s*(Technical|Architecture|Design)", "\\*\\*(Technical|Architecture|Design)\\s+(Design|Overview|Approach)\\*\\*" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "design", "atLeast": 3 }, "score": 100, "finding": "✓ Architecture: Comprehensive technical design" }, + { "when": { "type": "measure", "measure": "design", "atLeast": 2 }, "score": 80, "finding": "✓ Architecture: Technical approach documented" }, + { "when": { "type": "measure", "measure": "design", "atLeast": 1 }, "score": 50, "finding": "⚠ Architecture: Brief technical mention", "recommendation": "Add technical design details (approach, diagrams, integration points)" }, + { "score": 0, "finding": "✗ Architecture: No technical design", "recommendation": "Add architecture/technical design section" } + ] + } + }, + { + "name": "Decomposition", + "description": "The work is broken down into child tickets or phases.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "breakdown": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "child\\s+(ticket|issue|task|story)", "sub-task", "breakdown", "decompos(e|ed|ition)", "phase\\s*\\d", "step\\s*\\d", "work\\s*(stream|package|item)", "MS-\\d{4,5}", "implementation\\s+order", "dependencies", "h[23]\\.\\s*(Breakdown|Decomposition|Child|Sub-tasks|Implementation\\s+Order|Work\\s+Items)", "\\*\\*(Breakdown|Decomposition|Child|Sub-tasks|Implementation\\s+Order)\\*\\*" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "breakdown", "atLeast": 3 }, "score": 100, "finding": "✓ Decomposition: Clear work breakdown" }, + { "when": { "type": "measure", "measure": "breakdown", "atLeast": 2 }, "score": 80, "finding": "✓ Decomposition: Work items referenced" }, + { "when": { "type": "measure", "measure": "breakdown", "atLeast": 1 }, "score": 50, "finding": "⚠ Decomposition: Partial breakdown", "recommendation": "Add clear work breakdown with child tickets or phases" }, + { "score": 0, "finding": "✗ Decomposition: No work breakdown", "recommendation": "Decompose into child tickets or define implementation phases" } + ] + } + }, + { + "name": "ScopeDefinition", + "description": "What is in and out of scope is defined.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "scope": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "in\\s+scope", "out\\s+of\\s+scope", "scope\\s+(of|definition|boundary|boundaries)", "not\\s+included", "excluded", "boundaries", "goal(s)?", "objective(s)?", "non-goal", "h[23]\\.\\s*(Scope|Goal|Objective|Boundaries|Non-Goal|Out of Scope)", "\\*\\*(Scope|Goal|Objective|In Scope|Out of Scope|Non-Goal)\\*\\*" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "scope", "atLeast": 3 }, "score": 100, "finding": "✓ Scope: Well-defined boundaries" }, + { "when": { "type": "measure", "measure": "scope", "atLeast": 2 }, "score": 80, "finding": "✓ Scope: Goals documented" }, + { "when": { "type": "measure", "measure": "scope", "atLeast": 1 }, "score": 50, "finding": "⚠ Scope: Partially defined", "recommendation": "Clarify what is in scope and out of scope" }, + { "score": 0, "finding": "✗ Scope: Not defined", "recommendation": "Add scope boundaries (in scope, out of scope, goals)" } + ] + } + }, + { + "name": "AcceptanceCriteria", + "description": "The epic's success criteria and definition of done are stated.", + "weight": 13, + "definition": { + "type": "tiered", + "measures": { "criteria": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "acceptance\\s+criteria", "success\\s+criteria", "done\\s+when", "definition\\s+of\\s+done", "complete\\s+when", "considered\\s+complete", "epic\\s+(is\\s+)?complete", "milestone", "deliverable(s)?", "h[23]\\.\\s*(Acceptance|Success|Done|Definition of Done|Deliverables)", "\\*\\*(Acceptance|Success)\\s+Criteria\\*\\*" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "criteria", "atLeast": 2 }, "score": 100, "finding": "✓ Acceptance Criteria: High-level success criteria defined" }, + { "when": { "type": "measure", "measure": "criteria", "atLeast": 1 }, "score": 70, "finding": "⚠ Acceptance Criteria: Partially defined", "recommendation": "Add clearer definition of done for this epic" }, + { "score": 0, "finding": "✗ Acceptance Criteria: Not defined", "recommendation": "Add high-level acceptance/success criteria and definition of done" } + ] + } + }, + { + "name": "FixVersion", + "description": "The epic's fix version is set once it is expected.", + "weight": 5, + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "not", "condition": { "type": "ref", "name": "supportsFixVersion" } }, "score": 100, "finding": "○ Fix Version: Not applicable on {$.platform}" }, + { "when": { "type": "ref", "name": "hasFixVersion" }, "score": 100, "finding": "✓ Fix Version: {$.fixVersions[0]}" }, + { "when": { "type": "ref", "name": "shouldHaveFixVersion" }, "score": 0, "finding": "⚠ Fix Version: Not set (expected for '{$.status}' status)", "recommendation": "Add fix version" }, + { "score": 100, "finding": "○ Fix Version: Not required yet (status: {$.status})" } + ] + } + }, + { + "name": "EpicAttachments", + "description": "The epic has diagrams or mockups.", + "weight": 5, + "definition": { + "type": "tiered", + "measures": { "attachments": { "type": "number", "path": "$.attachmentsCount" } }, + "tiers": [ + { "when": { "type": "measure", "measure": "attachments", "atLeast": 1 }, "score": 100, "finding": "✓ Attachments: {attachments} item(s)" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.description" ] }, "patterns": [ "!\\[|screenshot|image|diagram|http.*\\.(png|jpg|gif)" ] }, "score": 70, "finding": "✓ Attachments: Inline images/diagrams found" }, + { "score": 30, "finding": "⚠ Attachments: None (diagrams/mockups recommended for epics)", "recommendation": "Consider adding architecture diagrams or mockups" } + ] + } + }, + { + "name": "EpicPresentation", + "description": "Each section sits in an untitled panel directly under its h2 heading, so the layout can be scanned and found again; sections holding a table or code block are exempt.", + "weight": 8, + "appliesWhen": { "type": "ref", "name": "hasPanelableSections" }, + "definition": { + "type": "tiered", + "measures": { + "panelable": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "panelable" }, + "panelled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "panelled" }, + "unpanelled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "unpanelled" }, + "titled": { "type": "sectionPanelCount", "source": { "paths": [ "$.description" ] }, "count": "titled" } + }, + "tiers": [ + { "when": { "type": "all", "conditions": [ { "type": "measure", "measure": "unpanelled", "lessThan": 1 }, { "type": "measure", "measure": "titled", "lessThan": 1 } ] }, "score": 100, "finding": "✓ Presentation: {panelled} sections in untitled panels" }, + { "when": { "type": "measure", "measure": "unpanelled", "lessThan": 1 }, "score": 70, "finding": "⚠ Presentation: {titled} panel titles repeat their heading ({titled:text})", "recommendation": "Remove the title from each panel that sits directly under a heading, since the heading already names it: keep only the background colour, as in panel:bgColor=#F4F5F7" }, + { "when": { "type": "measure", "measure": "panelled", "atLeast": 1 }, "score": 50, "finding": "⚠ Presentation: {unpanelled} of {panelable} sections are not in a panel ({unpanelled:text})", "recommendation": "Put the content of each section in an untitled panel directly under its heading; a section holding a table or code block can stay as it is" }, + { "score": 20, "finding": "✗ Presentation: None of {panelable} sections is in a panel ({unpanelled:text})", "recommendation": "Put the content of each section in an untitled panel directly under its heading, for example one with only a background colour, panel:bgColor=#F4F5F7; a section holding a table or code block can stay as it is" } + ] + } + }, + { + "name": "EpicBusinessCase", + "description": "Why the epic is valuable is stated.", + "weight": 10, + "definition": { + "type": "tiered", + "measures": { "justification": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "business\\s+case", "business\\s+value", "business\\s+justification", "business\\s+impact", "cost\\s+of\\s+(not\\s+doing|inaction|delay)", "return\\s+on\\s+investment", "\\bROI\\b", "value\\s+proposition", "strategic\\s+(value|benefit|alignment)", "revenue\\s+impact", "customer\\s+impact", "compliance\\s+requirement", "risk\\s+(reduction|mitigation)", "operational\\s+(benefit|efficiency|saving)", "h[23]\\.\\s*(Business\\s+Case|Business\\s+Value|Business\\s+Justification|Motivation|Rationale)", "\\*\\*(Business\\s+Case|Business\\s+Value|Justification|Motivation|Rationale)\\*\\*", "justif(y|ication|ied)", "rationale", "why\\s+(this|we)\\s+(is|are|need|should|must)" ] } }, + "tiers": [ + { "when": { "type": "measure", "measure": "justification", "atLeast": 3 }, "score": 100, "finding": "✓ Business Case: Clear justification provided" }, + { "when": { "type": "measure", "measure": "justification", "atLeast": 2 }, "score": 80, "finding": "✓ Business Case: Justification referenced" }, + { "when": { "type": "measure", "measure": "justification", "atLeast": 1 }, "score": 50, "finding": "⚠ Business Case: Brief justification", "recommendation": "Expand business case with impact, value, or cost of inaction" }, + { "score": 0, "finding": "✗ Business Case: Not provided", "recommendation": "Add business case explaining why this epic is valuable" } + ] + } + } + ] + } + ], + "scoreCaps": [ + { + "name": "Criteria duplication", + "group": "duplication", + "cap": 0.79, + "when": { "type": "assessmentScore", "assessment": "CriteriaDuplication", "lessThan": 100 } + }, + { + "name": "Steps to reproduce", + "group": "mandatory", + "cap": 0.79, + "when": { + "type": "all", + "conditions": [ + { "type": "ref", "name": "isBug" }, + { "type": "not", "condition": { "type": "ref", "name": "isInvestigation" } }, + { "type": "assessmentScore", "assessment": "StepsToReproduce", "lessThan": 100 } + ] + } + }, + { + "name": "Rollback plan", + "group": "mandatory", + "cap": 0.79, + "when": { "type": "assessmentScore", "assessment": "RollbackPlan", "lessThan": 1 } + } + ], + "gradeBands": [ + { "name": "EXCELLENT", "minimumScore": 0.8 }, + { "name": "GOOD", "minimumScore": 0.6 }, + { "name": "ADEQUATE", "minimumScore": 0.5 }, + { "name": "NEEDS IMPROVEMENT", "minimumScore": 0.3 }, + { "name": "POOR", "minimumScore": 0 } + ], + "classifications": { + "TestAutomationType": [ + { "when": { "type": "ref", "name": "isEpic" }, "value": "None" }, + { + "when": { + "type": "all", + "conditions": [ + { "type": "any", "conditions": [ { "type": "ref", "name": "isCliRelated" }, { "type": "ref", "name": "hasCliLabel" }, { "type": "ref", "name": "hasCliComponent" } ] }, + { "type": "not", "condition": { "type": "ref", "name": "hasUiIndicators" } }, + { "type": "not", "condition": { "type": "ref", "name": "hasUiLabel" } } + ] + }, + "value": "Ai" + }, + { + "when": { + "type": "all", + "conditions": [ + { "type": "any", "conditions": [ { "type": "ref", "name": "hasUiIndicators" }, { "type": "ref", "name": "hasUiLabel" } ] }, + { "type": "any", "conditions": [ { "type": "ref", "name": "isCliRelated" }, { "type": "ref", "name": "hasCliLabel" } ] } + ] + }, + "value": "Hybrid" + }, + { "when": { "type": "any", "conditions": [ { "type": "ref", "name": "hasUiIndicators" }, { "type": "ref", "name": "hasUiLabel" } ] }, "value": "Human" }, + { "value": "None" } + ] + } +} diff --git a/PanoramicData.IssueAssessment.Tests/Rules/CriteriaClassificationMeasureTests.cs b/PanoramicData.IssueAssessment.Tests/Rules/CriteriaClassificationMeasureTests.cs new file mode 100644 index 0000000..cb7c078 --- /dev/null +++ b/PanoramicData.IssueAssessment.Tests/Rules/CriteriaClassificationMeasureTests.cs @@ -0,0 +1,101 @@ +namespace PanoramicData.IssueAssessment.Tests.Rules; + +/// +/// Unit tests for : which criteria count as classified MUST, SHOULD or MAY. +/// +public class CriteriaClassificationMeasureTests : TestBase +{ + private const string Heading = "h2. Acceptance Criteria\r\n"; + + [Theory] + [InlineData("# MUST: x")] + [InlineData("# *MUST* x")] + [InlineData("* (SHOULD) x")] + [InlineData("| MAY 1 | x |")] + [InlineData("1. MUST - x")] + [InlineData("# *MUST* - PASS when a reference is struck through on its page")] + [InlineData("- MAY x")] + public void LeadingKeywordForms_AreClassified(string criterion) + => Counts(criterion).Should().Be((1, 0, criterion.Contains("MUST", StringComparison.Ordinal) ? 1 : 0)); + + [Fact] + public void InlineKeyword_Ms27557ProseForm_IsClassified() + => Counts("# SECTIONPAGES in a header or footer MUST show the page count of its section\r\n# ChartMagic MUST draw X and Y axis titles in regular weight") + .Should().Be((2, 0, 2)); + + [Theory] + [InlineData("# The schedule list SHOULD show the new offset", 0)] + [InlineData("# The audit log MAY record the change.", 0)] + [InlineData("# The run time MUST stay the same, and it MUST not drift", 1)] + public void OneDistinctInlineKeyword_IsClassified(string criterion, int must) + => Counts(criterion).Should().Be((1, 0, must)); + + [Theory] + [InlineData("# The run time MUST stay the same and MAY be logged")] + [InlineData("# SECTIONPAGES SHOULD refresh; NUMPAGES MUST refresh")] + public void MixedInlineKeywords_AreUnclassified(string criterion) + => Counts(criterion).Should().Be((1, 1, 0)); + + [Fact] + public void MixedKeywords_StartingWithOne_TakeTheLeadingOne() + => Counts("# MUST: the run time stays the same and MAY be logged").Should().Be((1, 0, 1)); + + [Theory] + [InlineData("# the run time must stay the same")] + [InlineData("# the audit log may record the change and should be kept")] + [InlineData("# the MUSTARD theme keeps its MAYBE flag")] + public void LowercaseOrEmbeddedKeywords_AreUnclassified(string criterion) + => Counts(criterion).Should().Be((1, 1, 0)); + + [Fact] + public void LabelLine_Ms27609Form_ClassifiesTheUnlabelledItemsBelowIt() + => Counts("*MUST*\r\n# A value encrypted under the old key is readable\r\n# The rotation is logged") + .Should().Be((2, 0, 2)); + + [Theory] + [InlineData("*MUST*")] + [InlineData("*MUST* (the ticket cannot pass without these)")] + [InlineData("MUST:")] + [InlineData("(MUST)")] + [InlineData("[MUST]")] + public void LabelLineForms_ClassifyTheItemsBelow(string label) + => Counts($"{label}\r\n# A value encrypted under the old key is readable").Should().Be((1, 0, 1)); + + [Fact] + public void LabelScope_EndsAtTheNextLabel() + => Counts("*MUST*\r\n# The key rotates\r\n# The rotation is logged\r\n\r\n*SHOULD*\r\n# The admin is emailed\r\n\r\n*MAY*\r\n# The dashboard shows a banner") + .Should().Be((4, 0, 2)); + + [Fact] + public void LabelScope_EndsAtAHeading() + => Counts("*MUST*\r\n# The key rotates\r\nh3. Notes\r\n# The rotation is logged") + .Should().Be((2, 1, 1)); + + [Fact] + public void ItemsBeforeAnyLabel_AreUnclassified() + => Counts("# The key rotates\r\n*MUST*\r\n# The rotation is logged").Should().Be((2, 1, 1)); + + [Fact] + public void ItemWithItsOwnKeyword_KeepsItUnderALabel() + => Counts("*MUST*\r\n# The rotation SHOULD be logged\r\n# SHOULD: the admin is emailed\r\n# The key rotates") + .Should().Be((3, 0, 1)); + + [Fact] + public void NestedSubPoint_IsNotALabel() + => Counts("*SHOULD*\r\n# The admin is emailed\r\n** MUST include the tenant name\r\n# The dashboard shows a banner") + .Should().Be((2, 0, 0)); + + [Fact] + public void LowercaseLabelLine_IsNotALabel() + => Counts("must:\r\n# The key rotates").Should().Be((1, 1, 0)); + + private static (double Items, double Unclassified, double Must) Counts(string criteria) + { + using JsonDocument document = JsonDocument.Parse(JsonSerializer.Serialize(new { d = Heading + criteria + "\r\n" })); + EvaluationContext context = new(document); + double Measure(CriteriaClassificationKind kind) + => new CriteriaClassificationMeasure { Path = "$.d", Section = @"acceptance\s+criteria", Count = kind }.Evaluate(context); + + return (Measure(CriteriaClassificationKind.Items), Measure(CriteriaClassificationKind.Unclassified), Measure(CriteriaClassificationKind.Must)); + } +} diff --git a/PanoramicData.IssueAssessment.Tests/Rules/RuleEngineTests.cs b/PanoramicData.IssueAssessment.Tests/Rules/RuleEngineTests.cs index 5b80916..9ebd5e6 100644 --- a/PanoramicData.IssueAssessment.Tests/Rules/RuleEngineTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Rules/RuleEngineTests.cs @@ -138,4 +138,24 @@ public void RuleRegex_CaseSensitive_DoesNotIgnoreCase() new TextMatchesCondition { Source = Source("$.d"), Patterns = ["Bug"], CaseSensitive = true }.Evaluate(Context("""{ "d": "bug" }""")) .Should().BeFalse(); } + + [Theory] + [InlineData("""{ "l": "Keep run times", "r": "Keep run times" }""", false)] + [InlineData("""{ "l": "Keep run times", "r": "Keep start times" }""", true)] + [InlineData("""{ "l": " KEEP Run Times ", "r": "keep run times" }""", false)] + [InlineData("""{ "l": "Keep run times", "r": "Keep run times" }""", true)] + [InlineData("""{ "r": "" }""", false)] + [InlineData("""{ "r": "Keep run times" }""", true)] + [InlineData("""{ }""", false)] + public void FieldsDiffer_ComparesTrimmedTextIgnoringCase(string json, bool expected) + => new FieldsDifferCondition { Left = "$.l", Right = "$.r" }.Evaluate(Context(json)).Should().Be(expected); + + [Theory] + [InlineData("""{ "l": "Schedules: Keep run times", "r": "Keep run times" }""", false)] + [InlineData("""{ "l": "Keep run times", "r": "Schedules: Keep run times" }""", false)] + [InlineData("""{ "l": "Fix: Keep run times", "r": "Bug: keep run times" }""", false)] + [InlineData("""{ "l": "Schedules: Keep run times", "r": "Keep start times" }""", true)] + [InlineData("""{ "l": "Keep run times: always", "r": "always" }""", false)] + public void FieldsDiffer_StripsThePrefixFromBothSides(string json, bool expected) + => new FieldsDifferCondition { Left = "$.l", Right = "$.r", IgnorePrefixPattern = @"^[^:]{1,40}:\s*" }.Evaluate(Context(json)).Should().Be(expected); } diff --git a/PanoramicData.IssueAssessment.Tests/Validation/RubricValidationTests.cs b/PanoramicData.IssueAssessment.Tests/Validation/RubricValidationTests.cs index 652cdb0..84418dc 100644 --- a/PanoramicData.IssueAssessment.Tests/Validation/RubricValidationTests.cs +++ b/PanoramicData.IssueAssessment.Tests/Validation/RubricValidationTests.cs @@ -73,6 +73,9 @@ public void ValidRubric_IsAccepted() { """{ "type": "fieldEquals", "path": "$.a", "values": [] }""", "'fieldEquals' condition without values" }, { """{ "type": "fieldEquals", "path": "bad", "values": [ "x" ] }""", "invalid JSONPath 'bad'" }, { """{ "type": "fieldPresent", "path": "bad" }""", "invalid JSONPath 'bad'" }, + { """{ "type": "fieldsDiffer", "left": "bad", "right": "$.b" }""", "invalid JSONPath 'bad'" }, + { """{ "type": "fieldsDiffer", "left": "$.a", "right": "bad" }""", "invalid JSONPath 'bad'" }, + { """{ "type": "fieldsDiffer", "left": "$.a", "right": "$.b", "ignorePrefixPattern": "(" }""", "invalid regex pattern '('" }, { """{ "type": "sectionPresent", "source": { "paths": [ "$.a" ] } }""", "whose source has no section" }, { """{ "type": "sectionPresent", "source": { "paths": [ "bad" ], "section": "x" } }""", "invalid JSONPath 'bad'" }, { """{ "type": "placeholder", "source": { "paths": [ "bad" ] } }""", "invalid JSONPath 'bad'" }, diff --git a/PanoramicData.IssueAssessment/Mapping/JiraIssueMapper.cs b/PanoramicData.IssueAssessment/Mapping/JiraIssueMapper.cs index 9ba5fb3..b57ac9a 100644 --- a/PanoramicData.IssueAssessment/Mapping/JiraIssueMapper.cs +++ b/PanoramicData.IssueAssessment/Mapping/JiraIssueMapper.cs @@ -48,6 +48,8 @@ public static NormalizedIssue FromRestJson(JsonElement issue, Uri? baseUrl) FixVersions = Names(fields, "fixVersions"), AttachmentsCount = ArrayLength(fields, "attachment"), DueDate = StringAt(fields, "duedate"), + ReleaseNotes = TextField(fields, "customfield_10600"), + Resolution = StringAt(fields, "resolution", "name"), Url = BrowseUrl(baseUrl, key), Platform = "Jira", }; @@ -81,11 +83,13 @@ public static string FlattenAdf(JsonElement document) return AdfFlattener.Flatten(document); } - private static string? Description(JsonElement fields) - => fields.TryGetProperty("description", out JsonElement description) ? description.ValueKind switch + private static string? Description(JsonElement fields) => TextField(fields, "description"); + + private static string? TextField(JsonElement fields, string property) + => fields.TryGetProperty(property, out JsonElement value) ? value.ValueKind switch { - JsonValueKind.String => description.GetString(), - JsonValueKind.Object => FlattenAdf(description), + JsonValueKind.String => value.GetString(), + JsonValueKind.Object => FlattenAdf(value), _ => null, } : null; diff --git a/PanoramicData.IssueAssessment/Mapping/NormalizedIssue.cs b/PanoramicData.IssueAssessment/Mapping/NormalizedIssue.cs index 073df6f..d711938 100644 --- a/PanoramicData.IssueAssessment/Mapping/NormalizedIssue.cs +++ b/PanoramicData.IssueAssessment/Mapping/NormalizedIssue.cs @@ -68,6 +68,12 @@ public sealed partial record NormalizedIssue /// Gets reproduction steps held in a field of their own, if the system has one. public string? ReproductionSteps { get; init; } + /// Gets the release notes held in a field of their own, if the system has one. + public string? ReleaseNotes { get; init; } + + /// Gets the resolution name, such as Fixed or Won't Do; null while unresolved. + public string? Resolution { get; init; } + /// Gets a link to the issue. public string? Url { get; init; } @@ -118,6 +124,8 @@ public JsonObject ToJsonObject() Add(issue, "dueDate", DueDate); Add(issue, "acceptanceCriteria", AcceptanceCriteria); Add(issue, "reproductionSteps", ReproductionSteps); + Add(issue, "releaseNotes", ReleaseNotes); + Add(issue, "resolution", Resolution); Add(issue, "url", Url); Add(issue, "platform", Platform); return issue; diff --git a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json index 47830d0..40c40b8 100644 --- a/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json +++ b/PanoramicData.IssueAssessment/Rubrics/panoramicdata-ticket-quality.json @@ -8,6 +8,12 @@ "isBug": { "type": "fieldMatches", "path": "$.issueType", "patterns": [ "Bug|Defect" ] }, "isEpic": { "type": "fieldEquals", "path": "$.issueType", "values": [ "Epic" ] }, "isChangeRequest": { "type": "fieldEquals", "path": "$.issueType", "values": [ "Change Request" ] }, + "isMs": { "type": "fieldEquals", "path": "$.project", "values": [ "MS" ] }, + "isUndelivered": { + "type": "all", + "conditions": [ { "type": "fieldPresent", "path": "$.resolution" }, { "type": "not", "condition": { "type": "fieldEquals", "path": "$.resolution", "values": [ "Fixed", "Done" ] } } ] + }, + "contractEvidenceDue": { "type": "fieldEquals", "path": "$.status", "values": [ "Ready for Test", "In Test", "Test Blocked", "Ready for Customer", "With Customer", "Ready for Release", "Closed", "Done", "Resolved" ] }, "isInvestigation": { "type": "all", "conditions": [ @@ -276,6 +282,7 @@ "type": "tiered", "measures": { "references": { "type": "patternMatchCount", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "documentation|docs?", "help\\s+(file|topic|section)", "user\\s+guide", "knowledge\\s+base|kb\\s+article", "wiki|confluence", "readme|readme\\.md", "api\\s+(reference|documentation|docs)", "manual", "guide|tutorial" ] } }, "tiers": [ + { "when": { "type": "textMatches", "source": { "paths": [ "$.description" ] }, "patterns": [ "(?im)^\\W*no documentation change:\\s*(?!n/?a\\b)\\S.{14,}" ] }, "score": 100, "finding": "✓ Documentation: Not needed (reason given)" }, { "when": { "type": "measure", "measure": "references", "atLeast": 2 }, "score": 100, "finding": "✓ Documentation: Multiple references found" }, { "when": { "type": "measure", "measure": "references", "atLeast": 1 }, "score": 70, "finding": "✓ Documentation: Referenced", "recommendation": "Consider adding more documentation updates" }, { "score": 0, "finding": "⚠ Documentation: Not referenced", "recommendation": "Update documentation/help text for this change" } @@ -290,6 +297,7 @@ "type": "tiered", "tiers": [ { "when": { "type": "ref", "name": "isBug" }, "score": 100, "finding": "○ Certifications: Not applicable for bugs" }, + { "when": { "type": "textMatches", "source": { "paths": [ "$.description" ] }, "patterns": [ "(?im)^\\W*no certification question:\\s*(?!n/?a\\b)\\S.{14,}" ] }, "score": 100, "finding": "✓ Certifications: Not needed (reason given)" }, { "when": { "type": "textMatches", "source": { "paths": [ "$.title", "$.description" ] }, "patterns": [ "certifica?tion|exam|certification\\s+question", "cert\\s+question", "assessment\\s+question", "test\\s+question" ] }, "score": 100, "finding": "✓ Certifications: Exam question(s) created" }, { "score": 0, "finding": "○ Certifications: No exam question created", "recommendation": "Consider creating a certification exam question for this new feature" } ] @@ -366,6 +374,72 @@ ] } }, + { + "name": "ReleaseNotes", + "description": "MS tickets carry customer-facing release notes, written for the person who reads the release notes, not for the team.", + "weight": 6, + "appliesWhen": { "type": "all", "conditions": [ { "type": "ref", "name": "isMs" }, { "type": "not", "condition": { "type": "ref", "name": "isUndelivered" } } ] }, + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Fill in the Release Notes field with one or two customer-facing sentences of at least 8 words saying what changed for the customer" }, + { "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Replace the placeholder in the Release Notes field with one or two sentences saying what changed for the customer" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Say what changed and for whom: a generic note such as 'Fixed bug' tells the customer nothing" }, + { "when": { "type": "measure", "of": { "type": "wordCount", "source": { "paths": [ "$.releaseNotes" ] } }, "lessThan": 8 }, "score": 0, "finding": "✗ Release Notes: Under 8 words", "recommendation": "Expand the Release Notes to at least 8 words, saying what changed and who it affects" }, + { "when": { "type": "not", "condition": { "type": "fieldsDiffer", "left": "$.releaseNotes", "right": "$.title", "ignorePrefixPattern": "^[^:]{1,40}:\\s*" } }, "score": 0, "finding": "✗ Release Notes: Same as the summary", "recommendation": "Describe the outcome for the customer rather than repeating the ticket summary" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b(?!(?:UTF|SHA|ISO|IEC|RFC|TLS|SSL|AES|RSA|CVE|CWE|PCI|DSS|ECMA|IEEE|HTTP|X)-)[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Remove ticket keys, file paths and code identifiers, and describe the change as the customer sees it" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "\\u2014" ] }, "score": 0, "finding": "✗ Release Notes: Contains an em dash", "recommendation": "Replace the em dash with a comma, colon or full stop" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)\\b(behavior|color|organization|initialize|customize|analyze|center|favorite|canceled)s?\\b" ] }, "score": 0, "finding": "✗ Release Notes: US spelling", "recommendation": "Use British spelling, for example behaviour, colour, organisation, centre, favourite and cancelled" }, + { "score": 100, "finding": "✓ Release Notes: Present and customer-facing" } + ] + } + }, + { + "name": "TicketContract", + "description": "MS acceptance criteria are classified MUST, SHOULD or MAY, with at least one MUST, and from Ready for Test on a Verification Evidence section says how each MUST was verified.", + "weight": 6, + "appliesWhen": { "type": "all", "conditions": [ { "type": "ref", "name": "isMs" }, { "type": "not", "condition": { "type": "ref", "name": "isEpic" } }, { "type": "not", "condition": { "type": "ref", "name": "isUndelivered" } } ] }, + "definition": { + "type": "tiered", + "measures": { + "items": { "type": "criteriaClassification", "path": "$.description", "section": "acceptance\\s+criteria", "count": "items" }, + "unclassified": { "type": "criteriaClassification", "path": "$.description", "section": "acceptance\\s+criteria", "count": "unclassified" }, + "must": { "type": "criteriaClassification", "path": "$.description", "section": "acceptance\\s+criteria", "count": "must" } + }, + "tiers": [ + { "when": { "type": "not", "condition": { "type": "sectionPresent", "source": { "paths": [ "$.description" ], "section": "acceptance\\s+criteria" } } }, "score": 0, "finding": "✗ Ticket Contract: No acceptance criteria section", "recommendation": "Add an 'h2. Acceptance Criteria' section listing each criterion as a numbered item classified MUST, SHOULD or MAY" }, + { "when": { "type": "measure", "measure": "items", "lessThan": 1 }, "score": 0, "finding": "✗ Ticket Contract: Acceptance criteria section has no criteria", "recommendation": "List each acceptance criterion as a numbered item or table row, classified MUST, SHOULD or MAY" }, + { "when": { "type": "measure", "measure": "unclassified", "atLeast": 1 }, "score": 0, "finding": "✗ Ticket Contract: {unclassified} criteria not classified MUST/SHOULD/MAY", "recommendation": "Start each acceptance criterion with MUST, SHOULD or MAY" }, + { "when": { "type": "measure", "measure": "must", "lessThan": 1 }, "score": 0, "finding": "✗ Ticket Contract: No MUST criterion", "recommendation": "Classify the criteria the ticket cannot pass without as MUST (aim for 2 to 5)" }, + { "when": { "type": "all", "conditions": [ { "type": "ref", "name": "contractEvidenceDue" }, { "type": "not", "condition": { "type": "sectionPresent", "source": { "paths": [ "$.description" ], "section": "verification\\s+evidence" } } } ] }, "score": 0, "finding": "✗ Ticket Contract: No Verification Evidence section", "recommendation": "Add an 'h2. Verification Evidence' section: a Criterion/Test/Location table, Reachability, what is not covered and why, and the suggested QA route" }, + { + "when": { + "type": "all", + "conditions": [ + { "type": "ref", "name": "contractEvidenceDue" }, + { + "type": "not", + "condition": { + "type": "all", + "conditions": [ + { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "verification\\s+evidence" }, "patterns": [ "\\|\\|\\s*Criterion\\s*\\|\\|" ] }, + { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "verification\\s+evidence" }, "patterns": [ "(?i)reachability" ] }, + { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "verification\\s+evidence" }, "patterns": [ "(?i)not covered" ] }, + { "type": "textMatches", "source": { "paths": [ "$.description" ], "section": "verification\\s+evidence" }, "caseSensitive": true, "patterns": [ "(?i)suggested qa route:?\\*?\\s*(?-i:[ABCD])\\b" ] } + ] + } + } + ] + }, + "score": 0, + "finding": "✗ Ticket Contract: Verification Evidence incomplete", + "recommendation": "Complete the Verification Evidence section: a Criterion/Test/Location table, Reachability, what is not covered and why, and 'Suggested QA route:' followed by A, B, C or D" + }, + { "when": { "type": "ref", "name": "contractEvidenceDue" }, "score": 100, "finding": "✓ Ticket Contract: Criteria classified; evidence present" }, + { "score": 100, "finding": "✓ Ticket Contract: Criteria classified" } + ] + } + }, { "name": "CriteriaDuplication", "description": "Acceptance criteria are stated under one heading only. It carries no weight; duplication caps the score instead.", @@ -526,6 +600,26 @@ ] } }, + { + "name": "EpicReleaseNotes", + "description": "MS tickets carry customer-facing release notes, written for the person who reads the release notes, not for the team.", + "weight": 6, + "appliesWhen": { "type": "all", "conditions": [ { "type": "ref", "name": "isMs" }, { "type": "not", "condition": { "type": "ref", "name": "isUndelivered" } } ] }, + "definition": { + "type": "tiered", + "tiers": [ + { "when": { "type": "not", "condition": { "type": "fieldPresent", "path": "$.releaseNotes" } }, "score": 0, "finding": "✗ Release Notes: Empty", "recommendation": "Fill in the Release Notes field with one or two customer-facing sentences of at least 8 words saying what changed for the customer" }, + { "when": { "type": "any", "conditions": [ { "type": "placeholder", "source": { "paths": [ "$.releaseNotes" ] } }, { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(see (the )?(ticket|description)|none|-+)\\s*\\.?\\s*$" ] } ] }, "score": 0, "finding": "✗ Release Notes: Placeholder", "recommendation": "Replace the placeholder in the Release Notes field with one or two sentences saying what changed for the customer" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)^\\s*(fixed (a )?bugs?|bug ?fix(es)?|various (fixes|improvements)|minor (fixes|improvements))\\s*\\.?\\s*$" ] }, "score": 0, "finding": "✗ Release Notes: Generic", "recommendation": "Say what changed and for whom: a generic note such as 'Fixed bug' tells the customer nothing" }, + { "when": { "type": "measure", "of": { "type": "wordCount", "source": { "paths": [ "$.releaseNotes" ] } }, "lessThan": 8 }, "score": 0, "finding": "✗ Release Notes: Under 8 words", "recommendation": "Expand the Release Notes to at least 8 words, saying what changed and who it affects" }, + { "when": { "type": "not", "condition": { "type": "fieldsDiffer", "left": "$.releaseNotes", "right": "$.title", "ignorePrefixPattern": "^[^:]{1,40}:\\s*" } }, "score": 0, "finding": "✗ Release Notes: Same as the summary", "recommendation": "Describe the outcome for the customer rather than repeating the ticket summary" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "caseSensitive": true, "patterns": [ "\\b(?!(?:UTF|SHA|ISO|IEC|RFC|TLS|SSL|AES|RSA|CVE|CWE|PCI|DSS|ECMA|IEEE|HTTP|X)-)[A-Z][A-Z0-9]+-\\d+\\b|\\.(cs|razor|ps1|json|csproj)\\b|[\\\\/]src[\\\\/]|[A-Za-z]:\\\\|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)+\\.[A-Z][A-Za-z]+\\(|\\b[A-Z][a-z]+(?:[A-Z][a-z]+)*(Service|Controller|Repository|Provider|Manager|Handler)\\b" ] }, "score": 0, "finding": "✗ Release Notes: Not customer-facing (ticket key, file path or code identifier)", "recommendation": "Remove ticket keys, file paths and code identifiers, and describe the change as the customer sees it" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "\\u2014" ] }, "score": 0, "finding": "✗ Release Notes: Contains an em dash", "recommendation": "Replace the em dash with a comma, colon or full stop" }, + { "when": { "type": "fieldMatches", "path": "$.releaseNotes", "patterns": [ "(?i)\\b(behavior|color|organization|initialize|customize|analyze|center|favorite|canceled)s?\\b" ] }, "score": 0, "finding": "✗ Release Notes: US spelling", "recommendation": "Use British spelling, for example behaviour, colour, organisation, centre, favourite and cancelled" }, + { "score": 100, "finding": "✓ Release Notes: Present and customer-facing" } + ] + } + }, { "name": "EpicBusinessCase", "description": "Why the epic is valuable is stated.", @@ -564,6 +658,21 @@ ] } }, + { + "name": "Release notes", + "group": "mandatory", + "cap": 0.79, + "when": { + "type": "any", + "conditions": [ { "type": "assessmentScore", "assessment": "ReleaseNotes", "lessThan": 100 }, { "type": "assessmentScore", "assessment": "EpicReleaseNotes", "lessThan": 100 } ] + } + }, + { + "name": "Ticket contract", + "group": "mandatory", + "cap": 0.79, + "when": { "type": "assessmentScore", "assessment": "TicketContract", "lessThan": 100 } + }, { "name": "Rollback plan", "group": "mandatory", diff --git a/PanoramicData.IssueAssessment/Rules/Conditions.cs b/PanoramicData.IssueAssessment/Rules/Conditions.cs index 7a51b11..fc567a5 100644 --- a/PanoramicData.IssueAssessment/Rules/Conditions.cs +++ b/PanoramicData.IssueAssessment/Rules/Conditions.cs @@ -18,6 +18,7 @@ namespace PanoramicData.IssueAssessment.Rules; [JsonDerivedType(typeof(FieldMatchesCondition), "fieldMatches")] [JsonDerivedType(typeof(FieldEqualsCondition), "fieldEquals")] [JsonDerivedType(typeof(FieldPresentCondition), "fieldPresent")] +[JsonDerivedType(typeof(FieldsDifferCondition), "fieldsDiffer")] [JsonDerivedType(typeof(SectionPresentCondition), "sectionPresent")] [JsonDerivedType(typeof(PlaceholderCondition), "placeholder")] [JsonDerivedType(typeof(AssessmentScoreCondition), "assessmentScore")] diff --git a/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs new file mode 100644 index 0000000..82f0f9c --- /dev/null +++ b/PanoramicData.IssueAssessment/Rules/CriteriaClassificationMeasure.cs @@ -0,0 +1,148 @@ +using System.Text.RegularExpressions; + +namespace PanoramicData.IssueAssessment.Rules; + +/// +/// Which criteria a counts. +/// +public enum CriteriaClassificationKind +{ + /// Every criterion. + Items, + + /// Criteria not classified MUST, SHOULD or MAY. + Unclassified, + + /// Criteria classified MUST. + Must, +} + +/// +/// Counts the criteria under a section of the text, by their RFC 2119 classification. A criterion is a top-level list +/// item (#, *, - or 1.) or a table row (|, not a || header). Keywords are +/// MUST, SHOULD and MAY as whole words in capitals. An absent section counts zero. +/// +/// +/// A criterion (the item, or the row's first cell) is classified, in order, by: a keyword it starts with, optionally +/// after * or (; otherwise the one keyword it contains, when it contains exactly one distinct keyword; +/// otherwise, for a list item, the label line above it. A label line is a line that is not a criterion and starts +/// with a keyword, optionally bolded or bracketed (*MUST*, MUST:). Its scope runs to the next label line +/// or wiki heading. +/// +public sealed partial record CriteriaClassificationMeasure : IMeasure +{ + /// Gets the JSONPath of the text to read. + public required string Path { get; init; } + + /// Gets the regular expression fragment matched against the heading of the section holding the criteria. + public required string Section { get; init; } + + /// Gets which criteria to count. + public required CriteriaClassificationKind Count { get; init; } + + /// + public double Evaluate(EvaluationContext context) + { + ArgumentNullException.ThrowIfNull(context); + string? body = context.Text(new TextSource { Paths = [Path], Section = Section }); + if (body is null) + { + return 0; + } + + (int items, int classified, int must) = Tally(body); + return Count switch + { + CriteriaClassificationKind.Unclassified => items - classified, + CriteriaClassificationKind.Must => must, + _ => items, + }; + } + + private static (int Items, int Classified, int Must) Tally(string body) + { + int items = 0; + int classified = 0; + int must = 0; + string? label = null; + foreach (string line in LineBreak().Split(body)) + { + if (Criterion(line) is not { } criterion) + { + label = LabelAfter(line, label); + continue; + } + + items++; + if (Classify(criterion.Text, criterion.IsListItem ? label : null) is { } keyword) + { + classified++; + must += keyword == "MUST" ? 1 : 0; + } + } + + return (items, classified, must); + } + + /// The label in force after a line that is not a criterion: a label line sets it, a wiki heading clears it. + private static string? LabelAfter(string line, string? label) + { + Match labelLine = LabelLine().Match(line); + if (labelLine.Success) + { + return labelLine.Groups[1].Value; + } + + return WikiHeading().IsMatch(line) ? null : label; + } + + private static string? Classify(string criterion, string? label) + { + Match leading = Classification().Match(criterion); + if (leading.Success) + { + return leading.Groups[1].Value; + } + + List keywords = [.. Keyword().Matches(criterion).Select(match => match.Value).Distinct(StringComparer.Ordinal)]; + return keywords.Count == 1 ? keywords[0] : label; + } + + private static (string Text, bool IsListItem)? Criterion(string line) + { + Match item = ListItem().Match(line); + if (item.Success) + { + return (item.Groups[1].Value, true); + } + + Match row = TableRow().Match(line); + return row.Success && !MarkdownTableRule().IsMatch(row.Groups[1].Value) ? (row.Groups[1].Value, false) : null; + } + + [GeneratedRegex(@"\r?\n", RegexOptions.CultureInvariant, 1000)] + private static partial Regex LineBreak(); + + // More indentation, or a doubled wiki marker, is a sub-point of the criterion above it. + [GeneratedRegex(@"^ ?(?:[#*-]|\d+\.)[ \t]+(\S.*)$", RegexOptions.CultureInvariant, 1000)] + private static partial Regex ListItem(); + + [GeneratedRegex(@"^[ \t]*\|(?!\|)([^|]*)", RegexOptions.CultureInvariant, 1000)] + private static partial Regex TableRow(); + + [GeneratedRegex(@"^[ \t]*:?-+:?[ \t]*$", RegexOptions.CultureInvariant, 1000)] + private static partial Regex MarkdownTableRule(); + + [GeneratedRegex(@"^[ \t]*[*(]?[ \t]*(MUST|SHOULD|MAY)\b", RegexOptions.CultureInvariant, 1000)] + private static partial Regex Classification(); + + [GeneratedRegex(@"\b(?:MUST|SHOULD|MAY)\b", RegexOptions.CultureInvariant, 1000)] + private static partial Regex Keyword(); + + // No space after the marker, so a "** MUST ..." nested sub-point is not a label. + [GeneratedRegex(@"^[ \t]*(?:\*{1,2}|_|\(|\[)?(MUST|SHOULD|MAY)\b", RegexOptions.CultureInvariant, 1000)] + private static partial Regex LabelLine(); + + [GeneratedRegex(@"^[ \t]*h[1-6]\.", RegexOptions.CultureInvariant, 1000)] + private static partial Regex WikiHeading(); +} diff --git a/PanoramicData.IssueAssessment/Rules/FieldsDifferCondition.cs b/PanoramicData.IssueAssessment/Rules/FieldsDifferCondition.cs new file mode 100644 index 0000000..f2ac790 --- /dev/null +++ b/PanoramicData.IssueAssessment/Rules/FieldsDifferCondition.cs @@ -0,0 +1,36 @@ +namespace PanoramicData.IssueAssessment.Rules; + +/// +/// True when the text at differs from the text at after trimming, ignoring case +/// and removing from the start of each. A missing value reads as empty. +/// +public sealed record FieldsDifferCondition : ICondition +{ + /// Gets the JSONPath of the first text. + public required string Left { get; init; } + + /// Gets the JSONPath of the second text. + public required string Right { get; init; } + + /// Gets an optional regular expression removed from the start of each text before comparing. + public string? IgnorePrefixPattern { get; init; } + + /// + public bool Evaluate(EvaluationContext context) + { + ArgumentNullException.ThrowIfNull(context); + return !string.Equals(Normalise(context, Left), Normalise(context, Right), StringComparison.Ordinal); + } + + private string Normalise(EvaluationContext context, string path) + { + IReadOnlyList values = context.Strings(path); + string text = (values.Count > 0 ? values[0] : string.Empty).Trim(); + if (IgnorePrefixPattern is { } prefix) + { + text = RuleRegex.Get(prefix, caseSensitive: true).Replace(text, string.Empty, 1).Trim(); + } + + return text.ToLowerInvariant(); + } +} diff --git a/PanoramicData.IssueAssessment/Rules/Measures.cs b/PanoramicData.IssueAssessment/Rules/Measures.cs index 4eefa20..f4f412d 100644 --- a/PanoramicData.IssueAssessment/Rules/Measures.cs +++ b/PanoramicData.IssueAssessment/Rules/Measures.cs @@ -17,6 +17,7 @@ namespace PanoramicData.IssueAssessment.Rules; [JsonDerivedType(typeof(NumberMeasure), "number")] [JsonDerivedType(typeof(CriteriaHeadingCountMeasure), "criteriaHeadingCount")] [JsonDerivedType(typeof(SectionPanelCountMeasure), "sectionPanelCount")] +[JsonDerivedType(typeof(CriteriaClassificationMeasure), "criteriaClassification")] public interface IMeasure { /// diff --git a/PanoramicData.IssueAssessment/Validation/RuleValidator.Measures.cs b/PanoramicData.IssueAssessment/Validation/RuleValidator.Measures.cs index e545142..2253cd8 100644 --- a/PanoramicData.IssueAssessment/Validation/RuleValidator.Measures.cs +++ b/PanoramicData.IssueAssessment/Validation/RuleValidator.Measures.cs @@ -17,6 +17,7 @@ internal static partial class RuleValidator SectionPanelCountMeasure value => SourceError(value.Source), ItemCountMeasure value => PathError(value.Path), NumberMeasure value => PathError(value.Path), + CriteriaClassificationMeasure value => PathError(value.Path) ?? (string.IsNullOrWhiteSpace(value.Section) ? "has a 'criteriaClassification' measure without a section." : PatternError(value.Section)), _ => null, }; diff --git a/PanoramicData.IssueAssessment/Validation/RuleValidator.cs b/PanoramicData.IssueAssessment/Validation/RuleValidator.cs index a54e8ee..6061128 100644 --- a/PanoramicData.IssueAssessment/Validation/RuleValidator.cs +++ b/PanoramicData.IssueAssessment/Validation/RuleValidator.cs @@ -38,6 +38,7 @@ internal static partial class RuleValidator FieldMatchesCondition value => PathError(value.Path) ?? PatternsError(value.Patterns), FieldEqualsCondition value => PathError(value.Path) ?? (value.Values is not { Count: > 0 } ? "has a 'fieldEquals' condition without values." : null), FieldPresentCondition value => PathError(value.Path), + FieldsDifferCondition value => PathError(value.Left) ?? PathError(value.Right) ?? (value.IgnorePrefixPattern is null ? null : PatternError(value.IgnorePrefixPattern)), SectionPresentCondition value => SourceError(value.Source) ?? (value.Source.Section is null ? "has a 'sectionPresent' condition whose source has no section." : null), PlaceholderCondition value => SourceError(value.Source), _ => null, diff --git a/README.md b/README.md index 06a14f2..5cec667 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,12 @@ report.Recommendations.ForEach(Console.WriteLine); // what to add to raise the Console.WriteLine(report.ToJson()); ``` +For Magic Suite (`MS`) tickets the rubric also scores release notes (`ReleaseNotes`, and `EpicReleaseNotes` on epics) +and, for tickets that are not epics, the MUST/SHOULD/MAY classification of acceptance criteria and the Verification +Evidence section (`TicketContract`). Both have weight 6 and a mandatory cap of 0.79, and neither applies to a ticket +whose resolution is anything other than Fixed or Done. Documentation and Certifications accept a justified +`No documentation change: ` or `No certification question: ` line in place of the item. + `BuiltInRubrics.PanoramicDataTicketQualityJson` returns its JSON, a good starting point for your own. ### Writing a rubric @@ -180,11 +186,13 @@ The building blocks: - **Tiered assessments** (`"type": "tiered"`): tiers are tried in order and the first whose `when` holds gives the score (0–100 by default), the finding and an optional recommendation. The last tier has no condition. - **Measures** count things: `patternMatchCount`, `occurrenceCount`, `wordCount`, `alphanumericCount`, `length`, - `itemCount`, `number`, `criteriaHeadingCount` and `sectionPanelCount` (how many `h2.` sections of Jira wiki text - are panelable, panelled, unpanelled or carry a panel title that repeats the heading). + `itemCount`, `number`, `criteriaHeadingCount`, `sectionPanelCount` (how many `h2.` sections of Jira wiki text + are panelable, panelled, unpanelled or carry a panel title that repeats the heading) and `criteriaClassification` + (how many criteria in a section there are, how many are not classified MUST, SHOULD or MAY, and how many are MUST). - **Conditions**: `all`, `any`, `not`, `ref` (a named condition), `measure`, `textMatches`, `fieldMatches`, - `fieldEquals`, `fieldPresent`, `sectionPresent`, `placeholder` (a section that only says "TBD") and, for caps and - classifications, `assessmentScore`. + `fieldEquals`, `fieldPresent`, `fieldsDiffer` (two fields differ, ignoring case, surrounding space and an optional + prefix), `sectionPresent`, `placeholder` (a section that only says "TBD") and, for caps and classifications, + `assessmentScore`. - **Text sources** join one or more fields and can narrow to a section by heading, in Jira wiki or Markdown. - **Placeholders** in findings and recommendations: `{measure}`, `{measure:text}`, `{$.path}` and `{$.path~pattern}` (the first value matching the pattern). diff --git a/version.json b/version.json index 28d4cab..7f28ebf 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/AArnott/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", - "version": "1.1", + "version": "1.2", "publicReleaseRefSpec": [ "^refs/heads/main$", "^refs/tags/\\d+\\.\\d+\\.\\d+$"