From 298f14f0254761a32ef7f9f0713b69f0b95639eb Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 01:29:35 +0000 Subject: [PATCH 01/10] docs(examples): record analysis runs in the model and report them in a document Co-Authored-By: jason.han --- .../unreleased/analysis-results-demo.added.md | 1 + cmd/sysml/examples_analysis_results_test.go | 64 +++ docs/manual/query-kinds.md | 10 + examples/README.md | 1 + examples/analysis-results-demo/README.md | 214 ++++++++ .../lander-results.sysml | 485 ++++++++++++++++++ examples/analysis-results-demo/report.md | 77 +++ .../testdata/api_json_roundtrip_expected.txt | 3 +- .../testdata/corpus_roundtrip_expected.txt | 3 +- 9 files changed, 856 insertions(+), 2 deletions(-) create mode 100644 changes/unreleased/analysis-results-demo.added.md create mode 100644 cmd/sysml/examples_analysis_results_test.go create mode 100644 examples/analysis-results-demo/README.md create mode 100644 examples/analysis-results-demo/lander-results.sysml create mode 100644 examples/analysis-results-demo/report.md diff --git a/changes/unreleased/analysis-results-demo.added.md b/changes/unreleased/analysis-results-demo.added.md new file mode 100644 index 0000000000..4ab1967d07 --- /dev/null +++ b/changes/unreleased/analysis-results-demo.added.md @@ -0,0 +1 @@ +- **Added `examples/analysis-results-demo/`, a worked example of saving analysis runs into the model and reporting them in a document.** Analysis and sweep output is printed and discarded, so the demo records each run as a `part` usage typed by a result-record definition — inputs, outputs and objective as attributes, `@RecordedRun` provenance metadata, and a `ref part` to the subject — then renders a report that groups, filters and lists the records, flags the record a later model edit made stale through a derived `drift`/`stale` pair, and contrasts them with `Verdicts` recomputed live at render time. diff --git a/cmd/sysml/examples_analysis_results_test.go b/cmd/sysml/examples_analysis_results_test.go new file mode 100644 index 0000000000..e2dba1bbf2 --- /dev/null +++ b/cmd/sysml/examples_analysis_results_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// TestAnalysisResultsExample runs the analysis-results demo end to end: the +// rendered report matches its committed output, the stale-records query +// returns exactly the record the model moved away from, and the analysis +// still prints the fuelLeft the scout record saved. +func TestAnalysisResultsExample(t *testing.T) { + binary := buildCLI(t) + examples := filepath.Join("..", "..", "examples", "analysis-results-demo") + source := filepath.Join(examples, "lander-results.sysml") + + committed, err := os.ReadFile(filepath.Join(examples, "report.md")) + if err != nil { + t.Fatal(err) + } + out := filepath.Join(t.TempDir(), "report.md") + render := exec.Command(binary, source, "-render-document", "Reporting::AnalysisReport", "-o", out) + if output, err := render.CombinedOutput(); err != nil { + t.Fatalf("render: %v\n%s", err, output) + } + written, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if string(written) != string(committed) { + t.Errorf("rendered example differs from examples/analysis-results-demo/report.md:\n%s", written) + } + + stale := exec.Command(binary, source, "-run-query", "Reporting::StaleRuns root=results") + output, err := stale.CombinedOutput() + if err != nil { + t.Fatalf("stale query: %v\n%s", err, output) + } + for _, want := range []string{ + "returned 1 row", + "Row 1: Results::results::relayRun", + "drift = 30.0", + "liveFuelLeft = 110.0", + } { + if !strings.Contains(string(output), want) { + t.Errorf("stale query output is missing %q:\n%s", want, output) + } + } + + // The recorded baseline must match what the case still prints; if the + // model moves so the printed value differs from scoutRun's fuelLeft, this + // catches the record silently drifting. + run := exec.Command(binary, source, "-analysis", "Descent::scoutBudget") + output, err = run.CombinedOutput() + if err != nil { + t.Fatalf("analysis: %v\n%s", err, output) + } + if !strings.Contains(string(output), "fuelLeft = 130.0") { + t.Errorf("scoutBudget no longer prints the recorded fuelLeft = 130.0:\n%s", output) + } +} diff --git a/docs/manual/query-kinds.md b/docs/manual/query-kinds.md index 78787b201d..92210625b7 100644 --- a/docs/manual/query-kinds.md +++ b/docs/manual/query-kinds.md @@ -75,6 +75,16 @@ Over gRPC, `RunDocumentQuery` answers an object row in the `object` arm of `DocumentValue` and a verdict row in the `verdict` arm; see [Native document queries and rendering over gRPC](../reference/api.md#native-document-queries-and-rendering-over-grpc). +One kind of result neither sees: what an analysis run printed. `-analysis`, +`-sweep` and trade studies report their outputs and verdicts on the terminal +and discard them — no element and no held object records that a run +happened, so no `Project` or `Verdicts` table can tabulate the runs a model +has had. The workaround is to write the runs back into the model as +result-record usages, which then filter, sort and project like anything else; +[the analysis-results demo](../../examples/analysis-results-demo/README.md) +works the pattern end to end, including records that flag themselves stale +when the model moves. + ## Runtime state and event queries Object rows tell you *what an object holds*; three more operations tell you diff --git a/examples/README.md b/examples/README.md index ef7e5fb107..5b58c05548 100644 --- a/examples/README.md +++ b/examples/README.md @@ -32,6 +32,7 @@ Each of these is a model and a walkthrough of the commands that exercise it. | [relay-probe-demo/mission.sysml](relay-probe-demo/mission.sysml) | [relay-probe-demo/README.md](relay-probe-demo/README.md) | one individual probe across its mission phases: event occurrences ordered in time, snapshots and a timeslice of one individual, occurrences with multiplicity, a calculation reading across two snapshots, a requirement whose subject is a snapshot, and a beacon inside a timeslice sending telemetry through its probe's own port | | [analysis-demo/lander.sysml](analysis-demo/lander.sysml) | [analysis-demo/README.md](analysis-demo/README.md) | analysis cases, asked every way the tool answers them: an analysis whose action steps feed each other and whose objective is a requirement, run bound, with arguments and on an object; a verification case whose body decides its verdict beside its objective; a parameter sweep and a seeded sample; two trade studies choosing among three landers; an action and a state machine due at the same instant of one clock, under each scheduling policy and explored; `-trace`, `-json`, the REPL forms and [the same questions from Python](analysis-demo/lander_demo.py) | | [verdicts-demo/rover.sysml](verdicts-demo/rover.sysml) | [verdicts-demo/README.md](verdicts-demo/README.md) | `Verdicts(...)` in a document query: one row per assertion about a rover and every object it holds — its constraints, a requirement it carries, a `satisfy` of its battery with the verification case verifying it, six wheels from a multiplicity and a heater no value decides — over the declared object, then over the object a session holds after a drive; what the table sees that evaluating one expression does not | +| [analysis-results-demo/lander-results.sysml](analysis-results-demo/lander-results.sysml) | [analysis-results-demo/README.md](analysis-results-demo/README.md) | saving analysis runs into the model: `-analysis` and `-sweep` print and discard, so each run is written back as a `part` usage typed by a result-record definition with `@RecordedRun` provenance metadata and a `ref part` to its subject — then a document groups the records by subject, filters the sweep rows and the trade study, lists every annotated record's command, and flags the record a later model edit made stale, beside live `Verdicts` recomputed at render time | | [solver-demo.sysml](solver-demo.sysml) | [SOLVER-DEMO.md](SOLVER-DEMO.md) | `%check`, `%explain`, `%solve`, `%configure` and `%optimize` — what conditions *can* hold, which conflict, what satisfies them, which variants are permitted, what is best (needs z3 or cvc5) | | [oosem-demo/oosem-demo.sysml](oosem-demo/oosem-demo.sysml) | [oosem-demo/README.md](oosem-demo/README.md) | the `OOSEM` library on a small Earth-observation mission: as-is and to-be enterprise, causal analysis, stakeholder needs derived down to component requirements with `#moe`/`#mop`, the black-box system context and its use case, the logical scenario and components, and the physical architecture distributed over nodes | | [mosa-demo/mosa-demo.sysml](mosa-demo/mosa-demo.sysml) | [mosa-demo/README.md](mosa-demo/README.md) | the `MOSA` library on a modular ground vehicle: the major system platform, its major system components and a modular autonomy system, the modular system interfaces between them (one written as `#keyInterface`), the consensus standards they conform to, data rights and proprietary elements, interface control, MOSA requirements with their traces, a conformance assessment, the MOSA views and a generated interface control document; `-validate` reports the openness gaps the model leaves on purpose | diff --git a/examples/analysis-results-demo/README.md b/examples/analysis-results-demo/README.md new file mode 100644 index 0000000000..087d555076 --- /dev/null +++ b/examples/analysis-results-demo/README.md @@ -0,0 +1,214 @@ +# Analysis results demo: saving runs into the model and reporting them + +[`lander-results.sysml`](lander-results.sysml) asks one question: **can the +results of analysis runs be saved into the model itself, so a generated +document can tabulate them later?** + +The honest answer is *not by itself, but yes by pattern*. `-analysis` and +`-sweep` print each run's inputs, outputs and verdicts and then discard them: +a run leaves no element in the model and no held object a document query can +read, and `-render-document` cannot be combined with `-analysis`, so the +document never sees a run happen. What a document *can* see is anything the +model declares — so the recording pattern is to write each run back yourself: +a `part` usage typed by a result-record definition, holding the run's inputs, +outputs and objective as attribute values, annotated with provenance metadata +and pointing at the part the run was about. [`report.md`](report.md) is what +the document then renders. + +## The recording pattern + +The `Records` package declares the vocabulary: + +```sysml +metadata def RecordedRun { + attribute runAt : String; + attribute tool : String; + attribute revision : String; + attribute command : String; +} + +part def AnalysisRun { + attribute caseName : String; + attribute kind : String; // "run" | "sweep" | "trade" + attribute 'objective' : String; // "satisfied" | "not satisfied" | "undecided" + attribute command : String; +} + +part def FuelBudgetRun :> AnalysisRun { + ref part lander : Lander; + attribute subjectName : String; + attribute burnTime : Real; + attribute fuelUsed : Real; + attribute wetMass : Real; + attribute fuelLeft : Real; + attribute liveFuelLeft : Real = lander.fuel - burnTime * lander.burnRate; + attribute drift : Real = liveFuelLeft - fuelLeft; + attribute stale : Boolean = drift != 0.0; +} +``` + +Each run is then one usage in the `Results` package, filled in from the +printed output: + +```sysml +part scoutRun : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:14:00Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; + } + ref part :>> lander = scout; + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "satisfied"; + attribute :>> subjectName = "scout"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 120.0; + attribute :>> wetMass = 730.0; + attribute :>> fuelLeft = 130.0; +} +``` + +Two limitations shape the record. The annotation's attribute values are not +projectable — `Project(properties = ("runAt"))` reports `unknown property` — +so `command` is also carried as a plain attribute for the provenance table to +show; the `@RecordedRun` metadata still answers `WhereMetadata` filters and +keeps the provenance machine-readable. And `objective` is a reserved word, +written `'objective'` wherever a name is needed. + +## The runs that were recorded + +One baseline run per candidate, at `FuelBudget`'s default 40 s burn: + +```bash +./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget +``` + +``` +✓ Descent::scoutBudget + fuelUsed = 120.0 + wetMass = 730.0 + fuelLeft = 130.0 + objective reserveHeld: satisfied +``` + +```bash +./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget +./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget +``` + +``` + fuelUsed = 320.0 fuelUsed = 100.0 + wetMass = 1980.0 wetMass = 530.0 + fuelLeft = 580.0 fuelLeft = 80.0 +``` + +A sweep of `scoutBudget`, one record per row (this README's tables omit the +`time` column, which is wall time): + +```bash +./bin/sysml examples/analysis-results-demo/lander-results.sysml \ + -analysis Descent::scoutBudget -sweep "burnTime=40.0..80.0:20.0" +``` + +``` +burnTime | fuelUsed | wetMass | fuelLeft | verdict +---------+----------+---------+----------+---------------------------- +40.0 | 120.0 | 730.0 | 130.0 | reserveHeld: satisfied +60.0 | 180.0 | 670.0 | 70.0 | reserveHeld: satisfied +80.0 | 240.0 | 610.0 | 10.0 | reserveHeld: not satisfied +``` + +And the trade study: + +```bash +./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest +``` + +``` +✓ Selection::lightest + selectedAlternative = Landers::relay (object #3) + objective tradeStudyObjective: satisfied + evaluationFunction(Landers::scout (object #1)) = 850.0 + evaluationFunction(Landers::hauler (object #2)) = 2300.0 + evaluationFunction(Landers::relay (object #3)) = 630.0 [selected] +``` + +## Records go stale — and can say so + +The `relay` part was edited *after* its run was recorded: `fuel` went from +`180.0` to `210.0` (the attribute carries a comment saying so). The record +still holds the printed `fuelLeft = 80.0`, but `liveFuelLeft` recomputes the +same formula from the model as it now stands — `210.0 - 40.0 * 2.5 = 110.0` — +so `drift = 30.0` and `stale = true`. That is the one thing a printed report +can never give you: a record that notices the model moved. + +```bash +./bin/sysml examples/analysis-results-demo/lander-results.sysml \ + -run-query "Reporting::StaleRuns root=results" +``` + +``` +✓ Query Reporting::StaleRuns returned 1 row + Columns: name, subjectName, burnTime, fuelLeft, liveFuelLeft, drift + Row 1: Results::results::relayRun + fuelLeft = 80.0 + liveFuelLeft = 110.0 + drift = 30.0 +``` + +The comparison is a derived Boolean on the record definition rather than a +`Column` expression, because computed columns do not support `!=`. Any model +edit can produce drift this way — only records typed by a definition that +recomputes an output can detect it, and only for the values it rederives. + +## Rendering the records + +```bash +./bin/sysml examples/analysis-results-demo/lander-results.sysml \ + -render-document Reporting::AnalysisReport -o examples/analysis-results-demo/report.md +``` + +[`report.md`](report.md) is committed so the test suite can compare the +render byte-for-byte. It shows a grouped table of every fuel-budget record by +subject, the sweep rows alone, the stale-records table (exactly `relayRun`), +a provenance table over `WhereMetadata(... 'metadata' = "Records::RecordedRun")`, +the trade-study record, and — the contrast — a `Verdicts` table of the +assertions about `scout` **evaluated live at render time**: the records say +what a run printed; the verdicts say what holds now. + +HTML and PDF render the same document tree: + +```bash +./bin/sysml examples/analysis-results-demo/lander-results.sysml \ + -render-document Reporting::AnalysisReport -doc-form html -o report.html +./bin/sysml examples/analysis-results-demo/lander-results.sysml \ + -render-document Reporting::AnalysisReport -doc-form pdf -o report.pdf +``` + +## Why isn't this automatic? + +Because nothing bridges two surfaces the tool already has — an implementation +gap, not an architectural limitation. The runtime already holds each +analysis run's results as typed values, `%save` already serializes the +session model, and document queries already read declared elements and held +objects; but analysis output is printed and discarded, no step writes it +back, and `-render-document` cannot run alongside `-analysis`. The pattern +this demo records by hand — a `part` usage typed by a result-record +definition, provenance metadata, a `ref part` to the subject — is the shape +an automated record step would emit. + +## Where to read more + +- [analysis-demo](../analysis-demo/README.md): the same lander asked with + `-analysis`, `-sweep`, trade studies and scheduling policies — everything + this demo records. +- [verdicts-demo](../verdicts-demo/README.md): `Verdicts(...)` rows in depth. +- The [query cookbook](../../docs/manual/query-cookbook.md): + [metadata filters](../../docs/manual/query-cookbook.md#metadata-filters), + [property filters](../../docs/manual/query-cookbook.md#property-filters), + [sorting](../../docs/manual/query-cookbook.md#sorting) and + [derived values](../../docs/manual/query-cookbook.md#derived-values), and + the [authoring manual](../../docs/manual/authoring.md) for + `Table`, `groupBy` and `Definitions`. diff --git a/examples/analysis-results-demo/lander-results.sysml b/examples/analysis-results-demo/lander-results.sysml new file mode 100644 index 0000000000..d2a19c8c3f --- /dev/null +++ b/examples/analysis-results-demo/lander-results.sysml @@ -0,0 +1,485 @@ +// Analysis runs are printed, not saved: `-analysis` and `-sweep` report each +// run's inputs, outputs and verdicts and then discard them, so nothing a +// document query can read records that a run happened. This model shows the +// workaround — write each run back into the model yourself as a `part` usage +// typed by a result-record definition, annotated with provenance metadata — +// and the one thing the pattern cannot fake: a derived attribute on the +// record recomputes the run's output from the model *as it stands now*, so a +// record whose numbers the model has moved away from reports its own drift. +// See README.md for the commands that produced each recorded row. + +package Landers { + private import ScalarValues::*; + + part def Lander { + attribute dryMass : Real; // kg, without fuel + attribute fuel : Real; // kg loaded at the start of descent + attribute burnRate : Real; // kg/s the engine consumes at full thrust + attribute payload : Real; // kg of instruments carried + attribute touchdownSpeed : Real; // m/s, from the descent profile + } + + part scout : Lander { + attribute :>> dryMass = 600.0; + attribute :>> fuel = 250.0; + attribute :>> burnRate = 3.0; + attribute :>> payload = 40.0; + attribute :>> touchdownSpeed = 1.2; + } + part hauler : Lander { + attribute :>> dryMass = 1400.0; + attribute :>> fuel = 900.0; + attribute :>> burnRate = 8.0; + attribute :>> payload = 300.0; + attribute :>> touchdownSpeed = 1.9; + } + part relay : Lander { + attribute :>> dryMass = 450.0; + // The recorded relay run predates this edit: it was captured when fuel + // was 180.0, so its record now reads stale. + attribute :>> fuel = 210.0; + attribute :>> burnRate = 2.5; + attribute :>> payload = 25.0; + attribute :>> touchdownSpeed = 1.4; + } +} + +package Descent { + private import ScalarValues::*; + private import Landers::*; + + // The requirement the analysis objective is typed by; the case binds its subject. + requirement def FuelReserve { + subject lander : Lander; + in attribute fuelLeft : Real; + attribute reserve : Real = 20.0; + require constraint { fuelLeft >= reserve } + } + + // Fuel a descent burn leaves: `then`-chained action steps feed each other + // through parameter bindings, and the case's outputs are read afterwards. + analysis def FuelBudget { + subject lander : Lander; + in attribute burnTime : Real = 40.0; // s of engine burn + + action burn { + in rate : Real = lander.burnRate; + in seconds : Real = burnTime; + out used : Real; + assign used := rate * seconds; + } + then action remaining { + in loaded : Real = lander.fuel; + in used : Real = burn.used; + out left : Real; + assign left := loaded - used; + } + + out fuelUsed : Real = burn.used; + out wetMass : Real = lander.dryMass + remaining.left; + + objective reserveHeld : FuelReserve { + subject = lander; + in fuelLeft = remaining.left; + } + + return fuelLeft : Real = remaining.left; + } + + analysis scoutBudget : FuelBudget { + subject lander = scout; + } + analysis haulerBudget : FuelBudget { + subject lander = hauler; + } + analysis relayBudget : FuelBudget { + subject lander = relay; + } + + // The touchdown requirement, stated of each candidate and satisfied by it. + requirement def SoftLanding { + subject lander : Lander; + attribute limit : Real = 1.5; + require constraint { lander.touchdownSpeed <= limit } + } + requirement scoutLandsSoftly : SoftLanding; + satisfy scoutLandsSoftly by scout; + requirement haulerLandsSoftly : SoftLanding; + satisfy haulerLandsSoftly by hauler; + requirement relayLandsSoftly : SoftLanding; + satisfy relayLandsSoftly by relay; + + // Verifies the requirement from its objective; `PassIf` decides the verdict. + verification def CheckScout { + subject lander : Lander; + objective { + verify scoutLandsSoftly; + require constraint { lander.touchdownSpeed <= 1.5 } + } + VerificationCases::PassIf(lander.touchdownSpeed <= 1.5) + } + verification def CheckHauler { + subject lander : Lander; + objective { + verify haulerLandsSoftly; + require constraint { lander.touchdownSpeed <= 1.5 } + } + VerificationCases::PassIf(lander.touchdownSpeed <= 1.5) + } + verification def CheckRelay { + subject lander : Lander; + objective { + verify relayLandsSoftly; + require constraint { lander.touchdownSpeed <= 1.5 } + } + VerificationCases::PassIf(lander.touchdownSpeed <= 1.5) + } + verification checkScout : CheckScout { + subject lander = scout; + } + verification checkHauler : CheckHauler { + subject lander = hauler; + } + verification checkRelay : CheckRelay { + subject lander = relay; + } +} + +package Selection { + private import ScalarValues::*; + private import TradeStudies::*; + private import Landers::*; + + // Lightest on the pad: every alternative is scored, and the objective picks the least. + analysis lightest : TradeStudy { + subject : Lander[1..*] = (scout, hauler, relay); + objective : MinimizeObjective; + calc :>> evaluationFunction { + in part l :>> alternative : Lander; + return :>> result : Real = l.dryMass + l.fuel; + } + return part :>> selectedAlternative : Lander; + } +} + +package Records { + private import ScalarValues::*; + private import Landers::*; + + // Provenance of a saved run: when, by what, at which revision, which command. + metadata def RecordedRun { + attribute runAt : String; + attribute tool : String; + attribute revision : String; + attribute command : String; + } + + // The vocabulary of a saved analysis run. `kind` is "run", "sweep" or + // "trade"; `objective` records the verdict the run printed. + // `command` is a plain attribute, not metadata, because a document + // query's `Project` cannot read an annotation's attribute values — it is + // what the provenance table shows for each record. + part def AnalysisRun { + attribute caseName : String; + attribute kind : String; + attribute 'objective' : String; + attribute command : String; + } + + // A FuelBudget run written back: subject, input and outputs as recorded, + // plus the output recomputed from the model as it stands now. `drift` + // measures how far the model has moved since the record was written, and + // `stale` names a record that no longer matches. + part def FuelBudgetRun :> AnalysisRun { + ref part lander : Lander; + attribute subjectName : String; + attribute burnTime : Real; + attribute fuelUsed : Real; + attribute wetMass : Real; + attribute fuelLeft : Real; + attribute liveFuelLeft : Real = lander.fuel - burnTime * lander.burnRate; + attribute drift : Real = liveFuelLeft - fuelLeft; + attribute stale : Boolean = drift != 0.0; + } + + // A TradeStudy run written back: the selection and every score it printed. + part def TradeStudyRun :> AnalysisRun { + attribute selected : String; + attribute scoutScore : Real; + attribute haulerScore : Real; + attribute relayScore : Real; + } +} + +package Results { + private import ScalarValues::*; + private import Landers::*; + private import Records::*; + + // The saved runs, held as subparts of one part so a query can walk them. + part results { + + // One baseline run per candidate, at FuelBudget's default 40 s burn. + part scoutRun : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:14:00Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; + } + ref part :>> lander = scout; + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; + attribute :>> subjectName = "scout"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 120.0; + attribute :>> wetMass = 730.0; + attribute :>> fuelLeft = 130.0; + } + part haulerRun : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:14:30Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; + } + ref part :>> lander = hauler; + attribute :>> caseName = "Descent::haulerBudget"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; + attribute :>> subjectName = "hauler"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 320.0; + attribute :>> wetMass = 1980.0; + attribute :>> fuelLeft = 580.0; + } + part relayRun : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:15:00Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; + } + ref part :>> lander = relay; + attribute :>> caseName = "Descent::relayBudget"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; + attribute :>> subjectName = "relay"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 100.0; + attribute :>> wetMass = 530.0; + attribute :>> fuelLeft = 80.0; + } + + // A sweep of scoutBudget over burnTime, one record per row the sweep printed. + part scoutSweep40 : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:20:10Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + } + ref part :>> lander = scout; + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + attribute :>> subjectName = "scout"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 120.0; + attribute :>> wetMass = 730.0; + attribute :>> fuelLeft = 130.0; + } + part scoutSweep60 : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:20:10Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + } + ref part :>> lander = scout; + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + attribute :>> subjectName = "scout"; + attribute :>> burnTime = 60.0; + attribute :>> fuelUsed = 180.0; + attribute :>> wetMass = 670.0; + attribute :>> fuelLeft = 70.0; + } + part scoutSweep80 : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:20:10Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + } + ref part :>> lander = scout; + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "not satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + attribute :>> subjectName = "scout"; + attribute :>> burnTime = 80.0; + attribute :>> fuelUsed = 240.0; + attribute :>> wetMass = 610.0; + attribute :>> fuelLeft = 10.0; + } + + // The trade study run: the selection and each alternative's score. + part lightestRun : TradeStudyRun { + @RecordedRun { + runAt = "2025-11-02T09:25:00Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; + } + attribute :>> caseName = "Selection::lightest"; + attribute :>> kind = "trade"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; + attribute :>> selected = "relay"; + attribute :>> scoutScore = 850.0; + attribute :>> haulerScore = 2300.0; + attribute :>> relayScore = 630.0; + } + } +} + +package Reporting { + private import ScalarValues::*; + private import DocumentQueries::*; + private import KerML::Root::Element; + private import Results::*; + + // Every saved fuel-budget run, sorted by burn time; the document groups it by subject. + calc def FuelBudgetRuns :> Query { + in root : Element; + Project( + source = OrderBy( + source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), + property = "burnTime", direction = "ascending", missing = "last", multiple = "error"), + properties = ("name", "subjectName", "kind", "burnTime", "fuelUsed", "wetMass", "fuelLeft", "objective")) + } + + // The sweep rows alone. + calc def SweepRuns :> Query { + in root : Element; + Project( + source = OrderBy( + source = WhereFeature( + source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), + 'feature' = "kind", operator = "=", value = "sweep"), + property = "burnTime", direction = "ascending", missing = "last", multiple = "error"), + properties = ("name", "caseName", "burnTime", "fuelUsed", "fuelLeft", "objective")) + } + + // Records whose saved numbers no longer match the model as it stands. + calc def StaleRuns :> Query { + in root : Element; + Project( + source = WhereFeature( + source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), + 'feature' = "stale", operator = "=", value = "true"), + properties = ("name", "subjectName", "burnTime", "fuelLeft", "liveFuelLeft", "drift")) + } + + // The recorded runs with their provenance annotation's command line. + calc def Provenance :> Query { + in root : Element; + Project( + source = WhereMetadata(source = Descendants(source = root, maxDepth = 1), 'metadata' = "Records::RecordedRun"), + properties = ("name", "kind", "caseName", "command")) + } + + // The trade-study record. + calc def TradeRuns :> Query { + in root : Element; + Project( + source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "TradeStudyRun"), + properties = ("name", "caseName", "selected", "scoutScore", "haulerScore", "relayScore", "objective")) + } + + // Assertions about the scout, checked live at render time. + calc def ScoutVerdicts :> Query { + in root : Element; + Project( + source = Verdicts(source = root), + properties = ("kind", "name", "verdict", "reason")) + } + + // The document itself. + part def AnalysisReport :> Document { + attribute redefines title = "Recorded analysis runs"; + + part intro : Paragraph { + attribute redefines text = "Analysis runs are printed and discarded; nothing writes them back. Every row below is a declared record — a part typed by a run definition whose attributes hold the inputs, outputs and objective a run printed, annotated with the command that produced it — except the last table, whose verdicts are recomputed live."; + } + + part budgets : Section { + attribute redefines title = "Recorded fuel budgets"; + part runs : Table { + attribute redefines caption = "One record per run, grouped by subject"; + attribute redefines groupBy = "subjectName"; + calc rows : FuelBudgetRuns { + in root = results; + } + } + } + + part sweep : Section { + attribute redefines title = "Sweep of scoutBudget"; + part runs : Table { + attribute redefines caption = "One record per sweep row"; + calc rows : SweepRuns { + in root = results; + } + } + } + + part stale : Section { + attribute redefines title = "Runs that no longer match the model"; + part runs : Table { + attribute redefines caption = "Records whose saved fuelLeft differs from the value the model now derives"; + calc rows : StaleRuns { + in root = results; + } + } + } + + part provenance : Section { + attribute redefines title = "Provenance"; + part runs : Table { + attribute redefines caption = "Every element annotated @RecordedRun"; + calc rows : Provenance { + in root = results; + } + } + } + + part trade : Section { + attribute redefines title = "Trade study"; + part runs : Table { + attribute redefines caption = "The recorded selection and scores"; + calc rows : TradeRuns { + in root = results; + } + } + } + + part live : Section { + attribute redefines title = "Live verdicts"; + part runs : Table { + attribute redefines caption = "Assertions about the scout, evaluated at render time"; + calc rows : ScoutVerdicts { + in root = Landers::scout; + } + } + } + } +} diff --git a/examples/analysis-results-demo/report.md b/examples/analysis-results-demo/report.md new file mode 100644 index 0000000000..b837b3990a --- /dev/null +++ b/examples/analysis-results-demo/report.md @@ -0,0 +1,77 @@ +# Recorded analysis runs + +Analysis runs are printed and discarded; nothing writes them back. Every row below is a declared record — a part typed by a run definition whose attributes hold the inputs, outputs and objective a run printed, annotated with the command that produced it — except the last table, whose verdicts are recomputed live. + +## Recorded fuel budgets + +*One record per run, grouped by subject* + +**subjectName: scout** + +| name | subjectName | kind | burnTime | fuelUsed | wetMass | fuelLeft | objective | +| --- | --- | --- | --- | --- | --- | --- | --- | +| scoutRun | scout | run | 40 | 120 | 730 | 130 | satisfied | +| scoutSweep40 | scout | sweep | 40 | 120 | 730 | 130 | satisfied | +| scoutSweep60 | scout | sweep | 60 | 180 | 670 | 70 | satisfied | +| scoutSweep80 | scout | sweep | 80 | 240 | 610 | 10 | not satisfied | + +**subjectName: hauler** + +| name | subjectName | kind | burnTime | fuelUsed | wetMass | fuelLeft | objective | +| --- | --- | --- | --- | --- | --- | --- | --- | +| haulerRun | hauler | run | 40 | 320 | 1980 | 580 | satisfied | + +**subjectName: relay** + +| name | subjectName | kind | burnTime | fuelUsed | wetMass | fuelLeft | objective | +| --- | --- | --- | --- | --- | --- | --- | --- | +| relayRun | relay | run | 40 | 100 | 530 | 80 | satisfied | + +## Sweep of scoutBudget + +*One record per sweep row* + +| name | caseName | burnTime | fuelUsed | fuelLeft | objective | +| --- | --- | --- | --- | --- | --- | +| scoutSweep40 | Descent::scoutBudget | 40 | 120 | 130 | satisfied | +| scoutSweep60 | Descent::scoutBudget | 60 | 180 | 70 | satisfied | +| scoutSweep80 | Descent::scoutBudget | 80 | 240 | 10 | not satisfied | + +## Runs that no longer match the model + +*Records whose saved fuelLeft differs from the value the model now derives* + +| name | subjectName | burnTime | fuelLeft | liveFuelLeft | drift | +| --- | --- | --- | --- | --- | --- | +| relayRun | relay | 40 | 80 | 110 | 30 | + +## Provenance + +*Every element annotated @RecordedRun* + +| name | kind | caseName | command | +| --- | --- | --- | --- | +| scoutRun | run | Descent::scoutBudget | ./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget | +| haulerRun | run | Descent::haulerBudget | ./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget | +| relayRun | run | Descent::relayBudget | ./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget | +| scoutSweep40 | sweep | Descent::scoutBudget | ./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0 | +| scoutSweep60 | sweep | Descent::scoutBudget | ./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0 | +| scoutSweep80 | sweep | Descent::scoutBudget | ./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0 | +| lightestRun | trade | Selection::lightest | ./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest | + +## Trade study + +*The recorded selection and scores* + +| name | caseName | selected | scoutScore | haulerScore | relayScore | objective | +| --- | --- | --- | --- | --- | --- | --- | +| lightestRun | Selection::lightest | relay | 850 | 2300 | 630 | satisfied | + +## Live verdicts + +*Assertions about the scout, evaluated at render time* + +| kind | name | verdict | reason | +| --- | --- | --- | --- | +| satisfaction | | holds | | +| verification | checkScout | holds | | diff --git a/tests/corpus/testdata/api_json_roundtrip_expected.txt b/tests/corpus/testdata/api_json_roundtrip_expected.txt index 5d2ee4f305..ba2a430968 100644 --- a/tests/corpus/testdata/api_json_roundtrip_expected.txt +++ b/tests/corpus/testdata/api_json_roundtrip_expected.txt @@ -9,13 +9,14 @@ # This is a per-file ratchet, not a claim that any verdict is right; # see docs/project/rdf-corpus-roundtrip.md. Regenerate with: # go test ./tests/corpus -run TestCorpusAPIJSONRoundTrip -update-api-json-roundtrip -# files: committed 43 +# files: committed 44 # files: sysml-v2-training 100 # files: pilot-corpora/kerml-examples 58 # files: pilot-corpora/sysml-examples 99 # files: pilot-corpora/sysml-validation 56 stable action-executor-demo.sysml stable analysis-demo/lander.sysml +stable analysis-results-demo/lander-results.sysml stable combined-behavioral-demo.sysml stable disposal-robot-demo/robot.sysml stable disposal-team-demo/team.sysml diff --git a/tests/corpus/testdata/corpus_roundtrip_expected.txt b/tests/corpus/testdata/corpus_roundtrip_expected.txt index e173edf202..12a9b13540 100644 --- a/tests/corpus/testdata/corpus_roundtrip_expected.txt +++ b/tests/corpus/testdata/corpus_roundtrip_expected.txt @@ -8,13 +8,14 @@ # is a per-file ratchet, not a claim that any verdict is right; see # docs/project/rdf-corpus-roundtrip.md. Regenerate with: # go test ./tests/corpus -run TestCorpusRoundTrip -update-corpus-roundtrip -# files: committed 43 +# files: committed 44 # files: sysml-v2-training 100 # files: pilot-corpora/kerml-examples 58 # files: pilot-corpora/sysml-examples 99 # files: pilot-corpora/sysml-validation 56 stable action-executor-demo.sysml stable analysis-demo/lander.sysml +stable analysis-results-demo/lander-results.sysml stable combined-behavioral-demo.sysml stable disposal-robot-demo/robot.sysml stable disposal-team-demo/team.sysml From 4830a9163935db449c00e062b3a81623d0176372 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 01:32:12 +0000 Subject: [PATCH 02/10] docs(examples): state which recorded runs predate the relay edit Co-Authored-By: jason.han --- examples/analysis-results-demo/README.md | 45 ++- .../lander-results.sysml | 274 +++++++++--------- 2 files changed, 176 insertions(+), 143 deletions(-) diff --git a/examples/analysis-results-demo/README.md b/examples/analysis-results-demo/README.md index 087d555076..2540f3a374 100644 --- a/examples/analysis-results-demo/README.md +++ b/examples/analysis-results-demo/README.md @@ -79,7 +79,11 @@ written `'objective'` wherever a name is needed. ## The runs that were recorded -One baseline run per candidate, at `FuelBudget`'s default 40 s burn: +One baseline run per candidate, at `FuelBudget`'s default 40 s burn. **Two of +these transcripts are records, not what the commands print today**: the relay +run and the trade study were captured before `relay.fuel` was edited from +`180.0` to `210.0`, which is what makes the staleness section below possible. +Each is labelled; the scout and hauler runs print the same numbers now. ```bash ./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget @@ -95,13 +99,33 @@ One baseline run per candidate, at `FuelBudget`'s default 40 s burn: ```bash ./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget +``` + +``` + fuelUsed = 320.0 + wetMass = 1980.0 + fuelLeft = 580.0 +``` + +The relay run, **as recorded** (when `relay.fuel` was `180.0`): + +```bash ./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget ``` ``` - fuelUsed = 320.0 fuelUsed = 100.0 - wetMass = 1980.0 wetMass = 530.0 - fuelLeft = 580.0 fuelLeft = 80.0 + fuelUsed = 100.0 + wetMass = 530.0 + fuelLeft = 80.0 +``` + +The same command **now prints** — the difference the `relayRun` record +detects: + +``` + fuelUsed = 100.0 + wetMass = 560.0 + fuelLeft = 110.0 ``` A sweep of `scoutBudget`, one record per row (this README's tables omit the @@ -120,7 +144,7 @@ burnTime | fuelUsed | wetMass | fuelLeft | verdict 80.0 | 240.0 | 610.0 | 10.0 | reserveHeld: not satisfied ``` -And the trade study: +And the trade study, **as recorded** before the relay edit: ```bash ./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest @@ -135,6 +159,9 @@ And the trade study: evaluationFunction(Landers::relay (object #3)) = 630.0 [selected] ``` +The same command **now prints** `660.0` for relay (`450.0 + 210.0`) — the +selection is still `relay`, but the score the record saved no longer is. + ## Records go stale — and can say so The `relay` part was edited *after* its run was recorded: `fuel` went from @@ -158,6 +185,14 @@ can never give you: a record that notices the model moved. drift = 30.0 ``` +The relay edit made a second record stale too: `lightestRun` still reports +`relayScore = 630.0` while `-analysis Selection::lightest` now scores relay +`660.0`. Nothing flags it — `TradeStudyRun` rederives nothing, so it has no +`liveFuelLeft` to compare against. That is the limitation the paragraph above +describes, made concrete: drift detection only exists where the record +definition recomputes the value itself, which is exactly what an automated +record step would have to emit for every output it saves. + The comparison is a derived Boolean on the record definition rather than a `Column` expression, because computed columns do not support `!=`. Any model edit can produce drift this way — only records typed by a definition that diff --git a/examples/analysis-results-demo/lander-results.sysml b/examples/analysis-results-demo/lander-results.sysml index d2a19c8c3f..527c7ef875 100644 --- a/examples/analysis-results-demo/lander-results.sysml +++ b/examples/analysis-results-demo/lander-results.sysml @@ -7,16 +7,15 @@ // record recomputes the run's output from the model *as it stands now*, so a // record whose numbers the model has moved away from reports its own drift. // See README.md for the commands that produced each recorded row. - package Landers { private import ScalarValues::*; part def Lander { - attribute dryMass : Real; // kg, without fuel - attribute fuel : Real; // kg loaded at the start of descent - attribute burnRate : Real; // kg/s the engine consumes at full thrust - attribute payload : Real; // kg of instruments carried - attribute touchdownSpeed : Real; // m/s, from the descent profile + attribute dryMass : Real; // kg, without fuel + attribute fuel : Real; // kg loaded at the start of descent + attribute burnRate : Real; // kg/s the engine consumes at full thrust + attribute payload : Real; // kg of instruments carried + attribute touchdownSpeed : Real; // m/s, from the descent profile } part scout : Lander { @@ -35,8 +34,8 @@ package Landers { } part relay : Lander { attribute :>> dryMass = 450.0; - // The recorded relay run predates this edit: it was captured when fuel - // was 180.0, so its record now reads stale. + // The recorded relay run and trade study predate this edit: fuel was + // 180.0 then, so relayRun reads stale and lightestRun silently disagrees. attribute :>> fuel = 210.0; attribute :>> burnRate = 2.5; attribute :>> payload = 25.0; @@ -60,8 +59,7 @@ package Descent { // through parameter bindings, and the case's outputs are read afterwards. analysis def FuelBudget { subject lander : Lander; - in attribute burnTime : Real = 40.0; // s of engine burn - + in attribute burnTime : Real = 40.0; // s of engine burn action burn { in rate : Real = lander.burnRate; in seconds : Real = burnTime; @@ -219,135 +217,135 @@ package Results { // The saved runs, held as subparts of one part so a query can walk them. part results { - // One baseline run per candidate, at FuelBudget's default 40 s burn. - part scoutRun : FuelBudgetRun { - @RecordedRun { - runAt = "2025-11-02T09:14:00Z"; - tool = "sysml"; - revision = "v0.8"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; + // One baseline run per candidate, at FuelBudget's default 40 s burn. + part scoutRun : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:14:00Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; + } + ref part :>> lander = scout; + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; + attribute :>> subjectName = "scout"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 120.0; + attribute :>> wetMass = 730.0; + attribute :>> fuelLeft = 130.0; } - ref part :>> lander = scout; - attribute :>> caseName = "Descent::scoutBudget"; - attribute :>> kind = "run"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; - attribute :>> subjectName = "scout"; - attribute :>> burnTime = 40.0; - attribute :>> fuelUsed = 120.0; - attribute :>> wetMass = 730.0; - attribute :>> fuelLeft = 130.0; - } - part haulerRun : FuelBudgetRun { - @RecordedRun { - runAt = "2025-11-02T09:14:30Z"; - tool = "sysml"; - revision = "v0.8"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; + part haulerRun : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:14:30Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; + } + ref part :>> lander = hauler; + attribute :>> caseName = "Descent::haulerBudget"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; + attribute :>> subjectName = "hauler"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 320.0; + attribute :>> wetMass = 1980.0; + attribute :>> fuelLeft = 580.0; } - ref part :>> lander = hauler; - attribute :>> caseName = "Descent::haulerBudget"; - attribute :>> kind = "run"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; - attribute :>> subjectName = "hauler"; - attribute :>> burnTime = 40.0; - attribute :>> fuelUsed = 320.0; - attribute :>> wetMass = 1980.0; - attribute :>> fuelLeft = 580.0; - } - part relayRun : FuelBudgetRun { - @RecordedRun { - runAt = "2025-11-02T09:15:00Z"; - tool = "sysml"; - revision = "v0.8"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; + part relayRun : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:15:00Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; + } + ref part :>> lander = relay; + attribute :>> caseName = "Descent::relayBudget"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; + attribute :>> subjectName = "relay"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 100.0; + attribute :>> wetMass = 530.0; + attribute :>> fuelLeft = 80.0; } - ref part :>> lander = relay; - attribute :>> caseName = "Descent::relayBudget"; - attribute :>> kind = "run"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; - attribute :>> subjectName = "relay"; - attribute :>> burnTime = 40.0; - attribute :>> fuelUsed = 100.0; - attribute :>> wetMass = 530.0; - attribute :>> fuelLeft = 80.0; - } - // A sweep of scoutBudget over burnTime, one record per row the sweep printed. - part scoutSweep40 : FuelBudgetRun { - @RecordedRun { - runAt = "2025-11-02T09:20:10Z"; - tool = "sysml"; - revision = "v0.8"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + // A sweep of scoutBudget over burnTime, one record per row the sweep printed. + part scoutSweep40 : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:20:10Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + } + ref part :>> lander = scout; + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + attribute :>> subjectName = "scout"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 120.0; + attribute :>> wetMass = 730.0; + attribute :>> fuelLeft = 130.0; } - ref part :>> lander = scout; - attribute :>> caseName = "Descent::scoutBudget"; - attribute :>> kind = "sweep"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - attribute :>> subjectName = "scout"; - attribute :>> burnTime = 40.0; - attribute :>> fuelUsed = 120.0; - attribute :>> wetMass = 730.0; - attribute :>> fuelLeft = 130.0; - } - part scoutSweep60 : FuelBudgetRun { - @RecordedRun { - runAt = "2025-11-02T09:20:10Z"; - tool = "sysml"; - revision = "v0.8"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + part scoutSweep60 : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:20:10Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + } + ref part :>> lander = scout; + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + attribute :>> subjectName = "scout"; + attribute :>> burnTime = 60.0; + attribute :>> fuelUsed = 180.0; + attribute :>> wetMass = 670.0; + attribute :>> fuelLeft = 70.0; } - ref part :>> lander = scout; - attribute :>> caseName = "Descent::scoutBudget"; - attribute :>> kind = "sweep"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - attribute :>> subjectName = "scout"; - attribute :>> burnTime = 60.0; - attribute :>> fuelUsed = 180.0; - attribute :>> wetMass = 670.0; - attribute :>> fuelLeft = 70.0; - } - part scoutSweep80 : FuelBudgetRun { - @RecordedRun { - runAt = "2025-11-02T09:20:10Z"; - tool = "sysml"; - revision = "v0.8"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + part scoutSweep80 : FuelBudgetRun { + @RecordedRun { + runAt = "2025-11-02T09:20:10Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + } + ref part :>> lander = scout; + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "not satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + attribute :>> subjectName = "scout"; + attribute :>> burnTime = 80.0; + attribute :>> fuelUsed = 240.0; + attribute :>> wetMass = 610.0; + attribute :>> fuelLeft = 10.0; } - ref part :>> lander = scout; - attribute :>> caseName = "Descent::scoutBudget"; - attribute :>> kind = "sweep"; - attribute :>> 'objective' = "not satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - attribute :>> subjectName = "scout"; - attribute :>> burnTime = 80.0; - attribute :>> fuelUsed = 240.0; - attribute :>> wetMass = 610.0; - attribute :>> fuelLeft = 10.0; - } - // The trade study run: the selection and each alternative's score. - part lightestRun : TradeStudyRun { - @RecordedRun { - runAt = "2025-11-02T09:25:00Z"; - tool = "sysml"; - revision = "v0.8"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; + // The trade study run: the selection and each alternative's score. + part lightestRun : TradeStudyRun { + @RecordedRun { + runAt = "2025-11-02T09:25:00Z"; + tool = "sysml"; + revision = "v0.8"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; + } + attribute :>> caseName = "Selection::lightest"; + attribute :>> kind = "trade"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; + attribute :>> selected = "relay"; + attribute :>> scoutScore = 850.0; + attribute :>> haulerScore = 2300.0; + attribute :>> relayScore = 630.0; } - attribute :>> caseName = "Selection::lightest"; - attribute :>> kind = "trade"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; - attribute :>> selected = "relay"; - attribute :>> scoutScore = 850.0; - attribute :>> haulerScore = 2300.0; - attribute :>> relayScore = 630.0; - } } } @@ -362,8 +360,8 @@ package Reporting { in root : Element; Project( source = OrderBy( - source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), - property = "burnTime", direction = "ascending", missing = "last", multiple = "error"), + source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), + property = "burnTime", direction = "ascending", missing = "last", multiple = "error"), properties = ("name", "subjectName", "kind", "burnTime", "fuelUsed", "wetMass", "fuelLeft", "objective")) } @@ -372,10 +370,10 @@ package Reporting { in root : Element; Project( source = OrderBy( - source = WhereFeature( - source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), - 'feature' = "kind", operator = "=", value = "sweep"), - property = "burnTime", direction = "ascending", missing = "last", multiple = "error"), + source = WhereFeature( + source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), + 'feature' = "kind", operator = "=", value = "sweep"), + property = "burnTime", direction = "ascending", missing = "last", multiple = "error"), properties = ("name", "caseName", "burnTime", "fuelUsed", "fuelLeft", "objective")) } @@ -384,8 +382,8 @@ package Reporting { in root : Element; Project( source = WhereFeature( - source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), - 'feature' = "stale", operator = "=", value = "true"), + source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), + 'feature' = "stale", operator = "=", value = "true"), properties = ("name", "subjectName", "burnTime", "fuelLeft", "liveFuelLeft", "drift")) } From 0234f4c2f874cc953b9924614bfc33d6cd7d2261 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 01:37:33 +0000 Subject: [PATCH 03/10] docs(examples): note the sweep's exit status in the analysis-results demo Co-Authored-By: jason.han --- examples/analysis-results-demo/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/analysis-results-demo/README.md b/examples/analysis-results-demo/README.md index 2540f3a374..a01a4c3b54 100644 --- a/examples/analysis-results-demo/README.md +++ b/examples/analysis-results-demo/README.md @@ -144,6 +144,10 @@ burnTime | fuelUsed | wetMass | fuelLeft | verdict 80.0 | 240.0 | 610.0 | 10.0 | reserveHeld: not satisfied ``` +The 80 s row leaves the objective unsatisfied, so the sweep exits `1`; the +transcripts here and above show the result lines only, not the package +loading and `standing` lines every run also prints. + And the trade study, **as recorded** before the relay edit: ```bash From 29d9d10cff7848bd91af3007ab58783cf687f36a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 02:26:26 +0000 Subject: [PATCH 04/10] docs(examples): re-record the pilot-differential baseline for the new example Co-Authored-By: jason.han --- docs/project/pilot-differential-baseline.json | 614 +++++++++++++++++- 1 file changed, 602 insertions(+), 12 deletions(-) diff --git a/docs/project/pilot-differential-baseline.json b/docs/project/pilot-differential-baseline.json index aa3a97c662..5786ac55dd 100644 --- a/docs/project/pilot-differential-baseline.json +++ b/docs/project/pilot-differential-baseline.json @@ -60,8 +60,8 @@ "name": "examples", "dir": "examples", "origin": "ours", - "files": 43, - "digest": "sha256:aa5a980eb63371fbea1ac5c52904d8639913da1b0486e544b1c3d3a8af00680a" + "files": 44, + "digest": "sha256:1fe87517a434896a7c3e87d57a28d790fc9074839cccb2f8adb3dd26c56d132f" }, { "name": "probes", @@ -71,17 +71,17 @@ "digest": "sha256:b0153c55bbfcdacabab911725dc44e0e7f4d3a501a281b1f3f2a13cda19737c6" } ], - "recorded": "2026-09-22" + "recorded": "2026-09-24" }, "totals": { - "files": 378, + "files": 379, "filesFullyAgreeing": 347, "agreement": 38, "severityMismatch": 3, "openSysMLOnly": 38, - "pilotOnly": 1185, + "pilotOnly": 1333, "openSysMLDiagnostics": 79, - "pilotDiagnostics": 1226 + "pilotDiagnostics": 1374 }, "roots": [ { @@ -746,16 +746,606 @@ "name": "examples", "dir": "examples", "totals": { - "files": 43, + "files": 44, "filesFullyAgreeing": 30, "agreement": 4, "severityMismatch": 2, "openSysMLOnly": 7, - "pilotOnly": 1165, + "pilotOnly": 1313, "openSysMLDiagnostics": 13, - "pilotDiagnostics": 1171 + "pilotDiagnostics": 1319 }, "files": [ + { + "path": "analysis-results-demo/lander-results.sysml", + "agreement": [], + "severityMismatch": [], + "openSysMLOnly": [], + "pilotOnly": [ + { + "line": 354, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 359, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 361, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 361, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 362, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 362, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 363, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 363, + "severity": "error", + "category": "unresolved-reference", + "count": 7 + }, + { + "line": 364, + "severity": "error", + "category": "unresolved-reference", + "count": 4 + }, + { + "line": 365, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 369, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 371, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 371, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 372, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 372, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 373, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 373, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 374, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 374, + "severity": "error", + "category": "unresolved-reference", + "count": 7 + }, + { + "line": 375, + "severity": "error", + "category": "unresolved-reference", + "count": 3 + }, + { + "line": 376, + "severity": "error", + "category": "unresolved-reference", + "count": 4 + }, + { + "line": 377, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 381, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 383, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 383, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 384, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 384, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 385, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 385, + "severity": "error", + "category": "unresolved-reference", + "count": 7 + }, + { + "line": 386, + "severity": "error", + "category": "unresolved-reference", + "count": 3 + }, + { + "line": 387, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 391, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 393, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 393, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 394, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 394, + "severity": "error", + "category": "unresolved-reference", + "count": 7 + }, + { + "line": 395, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 399, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 401, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 401, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 402, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 402, + "severity": "error", + "category": "unresolved-reference", + "count": 7 + }, + { + "line": 403, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 407, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 409, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 409, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 410, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 410, + "severity": "error", + "category": "unresolved-reference", + "count": 3 + }, + { + "line": 411, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 415, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 416, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 418, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 418, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 419, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 422, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 422, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 423, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 424, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 424, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 425, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 426, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 428, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 433, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 433, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 434, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 435, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 435, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 436, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 438, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 443, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 443, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 444, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 445, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 445, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 446, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 448, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 453, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 453, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 454, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 455, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 455, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 456, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 458, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 463, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 463, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 464, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 465, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 465, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 466, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 468, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 473, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 473, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 474, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 475, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 475, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 476, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 478, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + } + ] + }, { "path": "disposal-robot-demo/robot.sysml", "agreement": [], @@ -6436,14 +7026,14 @@ } ], "totals": { - "files": 378, + "files": 379, "filesFullyAgreeing": 348, "agreement": 38, "severityMismatch": 3, "openSysMLOnly": 37, - "pilotOnly": 1185, + "pilotOnly": 1333, "openSysMLDiagnostics": 78, - "pilotDiagnostics": 1226 + "pilotDiagnostics": 1374 }, "findings": [ { From d6a670b6f9f6c11693fcfbea533db139d116a1c6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 02:55:18 +0000 Subject: [PATCH 05/10] docs(project): update the differential figures for the new example Co-Authored-By: jason.han --- .../testing-pilot-corpora-gate/SKILL.md | 4 +-- .../testing-pilot-differential/SKILL.md | 8 ++--- .../testing-pilot-execution-referee/SKILL.md | 4 +-- .agents/skills/testing-pilot-xpect/SKILL.md | 4 +-- README.md | 6 ++-- docs/internals/architecture.md | 4 +-- docs/project/pilot-differential.md | 31 +++++++++++++------ 7 files changed, 37 insertions(+), 24 deletions(-) diff --git a/.agents/skills/testing-pilot-corpora-gate/SKILL.md b/.agents/skills/testing-pilot-corpora-gate/SKILL.md index 133c67d117..aa41f422c2 100644 --- a/.agents/skills/testing-pilot-corpora-gate/SKILL.md +++ b/.agents/skills/testing-pilot-corpora-gate/SKILL.md @@ -183,8 +183,8 @@ gate's own helpers are package-private but reusable (`pilotCorporaGate.files(t)` `actionlint`, `shellcheck`, `python3 scripts/check-doc-links.py`, `gofmt`, `go vet`, `go run -C tools ./cmd/pilot-diff` (validators pre-downloaded; ~4min, prints e.g. -the headline the committed baseline holds — `378 file(s), 347 fully agreeing; 38 agreed -diagnostic(s), 38 only ours, 1185 only the pilot's` after the Legend of the Red Dragon example left for its own repository at the `2026-08` pin, so read it from +the headline the committed baseline holds — `379 file(s), 347 fully agreeing; 38 agreed +diagnostic(s), 38 only ours, 1333 only the pilot's` at the `2026-08` pin, so read it from `docs/project/pilot-differential-baseline.json` rather than from this line) and `make lint` (staticcheck+gosec, ~2min) all work. There is **no** `yamllint` and **no** `circleci` CLI, so `.circleci/config.yml` can only be parsed as YAML, not schema-validated — say so diff --git a/.agents/skills/testing-pilot-differential/SKILL.md b/.agents/skills/testing-pilot-differential/SKILL.md index ab2579db15..4971b493b8 100644 --- a/.agents/skills/testing-pilot-differential/SKILL.md +++ b/.agents/skills/testing-pilot-differential/SKILL.md @@ -21,9 +21,9 @@ GNU-format diagnostics **relative to `--root`**. Consequences for testing: - The pin `tools/referee/diff` reports comes from `build/pilot-sysml-validator/pilot-pin.txt` (written by the new script), not from the DeciSym `pom.xml`. - `-validator /nonexistent` now says `run ./scripts/download-pilot-sysml-validator.sh`. -- Measured after the Legend of the Red Dragon example left for its own repository at the `2026-08` pin, with a fresh library cache: `378 file(s), 347 fully agreeing; 38 agreed, - 38 only ours, 1185 only the pilot's`, JSON totals `openSysMLDiagnostics 79 / pilotDiagnostics - 1226 / severityMismatch 3`; ~2 min wall, byte-identical across runs *and* after a from-scratch +- Measured at the `2026-08` pin, with a fresh library cache: `379 file(s), 347 fully agreeing; 38 agreed, + 38 only ours, 1333 only the pilot's`, JSON totals `openSysMLDiagnostics 79 / pilotDiagnostics + 1374 / severityMismatch 3`; ~2 min wall, byte-identical across runs *and* after a from-scratch rebuild of `build/pilot-validator`. The six `kerml-examples` pilot-only rows the `2026-07` run carried (`The opposite features 'owningType' … do not refer to each other`) are gone: the pilot fixed its `ownedDisjoining` delegate, and nothing on our side moved. `kerml-examples` carries no `syntax` diagnostic on either @@ -137,7 +137,7 @@ The harness compares OpenSysML diagnostics against the OMG SysML v2 Pilot Implem `build/pilot-diff/pilot-diff.{txt,json}`. `docs/project/pilot-differential-baseline.json` is the committed result of the *last refreshed* run, so **the harness is testable by reproduction** — but only while the baseline is current. Check that first. As of the rebaseline that came when the Legend of the Red Dragon example left for its own repository it **is** -current: a live run gives `378 file(s), 347 fully agreeing; 38 agreed, 38 only ours, 1185 only the +current: a live run gives `379 file(s), 347 fully agreeing; 38 agreed, 38 only ours, 1333 only the pilot's`, byte-identical to the committed baseline, and `docs/project/pilot-differential.md`'s "Results" table matches. The rebaseline before it, at the architecture self-model's landing, covered two rounds, because the succession-shorthand removal before it landed without refreshing the baseline; a control run of its merge commit gives diff --git a/.agents/skills/testing-pilot-execution-referee/SKILL.md b/.agents/skills/testing-pilot-execution-referee/SKILL.md index 0696b77dd5..e212010d36 100644 --- a/.agents/skills/testing-pilot-execution-referee/SKILL.md +++ b/.agents/skills/testing-pilot-execution-referee/SKILL.md @@ -148,8 +148,8 @@ pilot answers the representation's own. See `pilot-exec-diff: :: model no/such/model.sysml: stat : no such file or directory`. - **Additivity.** `go run -C tools ./cmd/pilot-diff` must still print the headline the - committed baseline holds (`378 file(s), 347 fully agreeing; 38 agreed - diagnostic(s), 38 only ours, 1185 only the pilot's` after the Legend of the Red Dragon example left for its own repository at the `2026-08` pin — read it from the baseline JSON, not from this line, since each + committed baseline holds (`379 file(s), 347 fully agreeing; 38 agreed + diagnostic(s), 38 only ours, 1333 only the pilot's` at the `2026-08` pin — read it from the baseline JSON, not from this line, since each fix round moves it) and `jq -S` diff clean against `docs/project/pilot-differential-baseline.json`; `git status --porcelain` empty at the end. diff --git a/.agents/skills/testing-pilot-xpect/SKILL.md b/.agents/skills/testing-pilot-xpect/SKILL.md index 0f31b8c8b9..8e051dfaef 100644 --- a/.agents/skills/testing-pilot-xpect/SKILL.md +++ b/.agents/skills/testing-pilot-xpect/SKILL.md @@ -416,8 +416,8 @@ census in `w5c_census_test.go` is live two ways: perturb one pinned triple (e.g. ## Regression neighbour `go run -C tools ./cmd/pilot-diff` (~1m12s) must still print the headline the *committed* baseline holds — -after the Legend of the Red Dragon example left for its own repository at the `2026-08` pin that is `378 file(s), 347 fully agreeing; 38 agreed diagnostic(s), 38 -only ours, 1185 only the pilot's`. Read the number out of +at the `2026-08` pin that is `379 file(s), 347 fully agreeing; 38 agreed diagnostic(s), 38 +only ours, 1333 only the pilot's`. Read the number out of `docs/project/pilot-differential-baseline.json` rather than trusting this line, since a landing fix round moves it. When the baseline is itself stale (it was at `19a3ce03`, holding 273 / 281 / 317), a failing `cmp` against it is *not* evidence of an Xpect regression — compare the summary line, and see diff --git a/README.md b/README.md index 11006da9f9..7e6f8bd77c 100644 --- a/README.md +++ b/README.md @@ -313,11 +313,11 @@ The project is under active development, with the core infrastructure operationa **Measured against the pinned reference** (`PILOT_TAG=2026-08`, artifact `0.62.0`). Every number below is generated by `make docs-counts` from the committed baselines and gated; none of them is typed in by hand. -- **Corpus agreement:** 347 of 378 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1185 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](docs/project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). +- **Corpus agreement:** 347 of 379 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1333 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](docs/project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). - **Declared-diagnostic silence:** of the 512 declared `errors` rows in the reference's own Xpect suites, we report nothing for 0. 245 we report word-for-word; 248 wording-only and 7 location-only differences are agreement in substance and are not counted as gaps; 0 more we report as a warning and 2 elsewhere in the file ([Xpect oracle](docs/project/pilot-xpect.md), `go run -C tools ./cmd/pilot-xpect`). - **Scope agreement:** 230 of 230 declared scope assertions match exactly (same source). - **Permissiveness gaps:** of 306 invalid models we wrote ourselves, the reference rejects 4 that we accept by default, and 293 both reject; 4 further cases agree only when we are asked strictly. We authored every one of these cases ourselves, so the denominator measures the reach of our own corpus and not our conformance; agreement reached only under an opt-in strict mode is weaker evidence than agreement by default ([rejection oracle](docs/project/pilot-rejection.md), `go run -C tools ./cmd/pilot-reject`). -- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](docs/project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 378 files agreeing, 37 diagnostics ours alone and 1185 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. +- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](docs/project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 379 files agreeing, 37 diagnostics ours alone and 1333 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. - **Self-assessed surface:** the action, state-machine and classifier-behavior rows have no external referee at all — the four refereed figures above cannot see them, because the pinned artifact evaluates expressions but executes neither actions nor state machines. [Spec compliance](docs/project/spec-compliance.md) counts them. What these numbers cannot show: the OMG corpora are demonstrations rather than an official conformance suite; the differential is one-directional, comparing the diagnostics the two implementations report on the same files; the Xpect suites are the pilot authors' test intent rather than a certification oracle; and none of these is a percentage of the specification — no global compliance figure is claimed anywhere. @@ -329,7 +329,7 @@ What these numbers cannot show: the OMG corpora are demonstrations rather than a **Test coverage:** top-level `Test` functions (counted from the `_test.go` files, as `go test ./...` runs them) covering parsers, semantics, runtime (actions, states, instances, operators, validation), behind golden ASTs, negatives, execution conformance cases, golden traces, runtime robustness cases and gRPC conformance and robustness cases. The figures are counted from the tree when the documentation site is built into the test inventory of [spec compliance](docs/project/spec-compliance.md), never committed, so a branch adding a test does not rewrite this page. A test skips only for want of something the run did not provide, and says what: the held-image round trip declines a conformance case that creates no instance, a few gate on a PDF or Mermaid toolchain, a pinned pilot artifact, the PSSM suite, a locale, a case-insensitive filesystem or a live Flexo stack, and the OMG corpus gates skip until the corpora are downloaded unless asked to fail. **Parser coverage:** 104/104 bundled library files parse cleanly — the 94 official SysML v2 standard library files and the non-normative `OpenSysML Libraries/OpenSysMLMathFunctions.kerml`, `OpenSysML Libraries/DocumentQueries.sysml`, `OpenSysML Libraries/IdentityMetadata.sysml`, `OpenSysML Libraries/DiagramLayout.sysml`, `OpenSysML Libraries/OOSEM.sysml`, `OpenSysML Libraries/MOSA.sysml`, `OpenSysML Libraries/StateSpaceIntegration.sysml`, `OpenSysML Libraries/Stochastic.sysml`, `OpenSysML Libraries/RandomFunctions.kerml` and `OpenSysML Libraries/Simulation.sysml` extensions. Conformance verified by [stdlib_conformance_test.go](internal/workspace/libs/stdlib_conformance_test.go). Grammar reference: [OMG Xtext grammar](https://github.com/Systems-Modeling/SysML-v2-Pilot-Implementation/tree/master/org.omg.kerml.xtext/src/org/omg/kerml/xtext). **Behavioral execution:** Calc/constraint/requirement/satisfy functional. Action/state executors handle nested invocation, control flow keywords, loop and conditional statements and the send statement (every conformance case passing). Coverage is self-assessed against the specification text and the normative library: the pinned OMG pilot implementation evaluates expressions but does not execute actions or state machines headlessly, so no external implementation currently adjudicates these rows. See [spec compliance](docs/project/spec-compliance.md). -**Reference differential:** 378 files compared diagnostic-by-diagnostic against the pinned OMG pilot implementation (`2026-08`), 347 in full agreement; every divergence is enumerated and adjudicated in [the differential](docs/project/pilot-differential.md), reproducible with `go run -C tools ./cmd/pilot-diff`. +**Reference differential:** 379 files compared diagnostic-by-diagnostic against the pinned OMG pilot implementation (`2026-08`), 347 in full agreement; every divergence is enumerated and adjudicated in [the differential](docs/project/pilot-differential.md), reproducible with `go run -C tools ./cmd/pilot-diff`. **Rejection oracle:** the reverse direction — do we reject what the reference rejects? 306 hand-written invalid models validated by both implementations, 297 rejected by both, 0 the pinned pilot rejects and we accept; the remainder only we reject — the control-node succession rules the pinned pilot leaves unimplemented and a non-Boolean succession guard it accepts once the standard library types it — and every permissiveness gap is enumerated with a reproducer and likely root cause in [the rejection oracle](docs/project/pilot-rejection.md), reproducible with `go run -C tools ./cmd/pilot-reject`. We wrote every case, so the count measures our coverage of the rejection surface, not our conformance — a sample, not a proof. **Training examples:** 100/100 files clean, gated by `tests/corpus/testdata/training_examples_expected.txt`. Download with `./scripts/download-training-examples.sh` (from the [OMG training directory](https://github.com/Systems-Modeling/SysML-v2-Pilot-Implementation/tree/master/sysml/src/training)). See [training examples](docs/project/training-examples.md) for analysis. **Semantic layer:** a complete implementation of runtime operators, feature chains and validation rules. See [examples/semantic-layer/](examples/semantic-layer/) for a full demonstration. diff --git a/docs/internals/architecture.md b/docs/internals/architecture.md index de7d3849c0..9a26444924 100644 --- a/docs/internals/architecture.md +++ b/docs/internals/architecture.md @@ -840,11 +840,11 @@ Every behavioral feature must have: **Measured against the pinned reference** (`PILOT_TAG=2026-08`, artifact `0.62.0`). Every number below is generated by `make docs-counts` from the committed baselines and gated; none of them is typed in by hand. -- **Corpus agreement:** 347 of 378 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1185 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](../project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). +- **Corpus agreement:** 347 of 379 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1333 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](../project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). - **Declared-diagnostic silence:** of the 512 declared `errors` rows in the reference's own Xpect suites, we report nothing for 0. 245 we report word-for-word; 248 wording-only and 7 location-only differences are agreement in substance and are not counted as gaps; 0 more we report as a warning and 2 elsewhere in the file ([Xpect oracle](../project/pilot-xpect.md), `go run -C tools ./cmd/pilot-xpect`). - **Scope agreement:** 230 of 230 declared scope assertions match exactly (same source). - **Permissiveness gaps:** of 306 invalid models we wrote ourselves, the reference rejects 4 that we accept by default, and 293 both reject; 4 further cases agree only when we are asked strictly. We authored every one of these cases ourselves, so the denominator measures the reach of our own corpus and not our conformance; agreement reached only under an opt-in strict mode is weaker evidence than agreement by default ([rejection oracle](../project/pilot-rejection.md), `go run -C tools ./cmd/pilot-reject`). -- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](../project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 378 files agreeing, 37 diagnostics ours alone and 1185 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. +- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](../project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 379 files agreeing, 37 diagnostics ours alone and 1333 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. - **Self-assessed surface:** the action, state-machine and classifier-behavior rows have no external referee at all — the four refereed figures above cannot see them, because the pinned artifact evaluates expressions but executes neither actions nor state machines. [Spec compliance](../project/spec-compliance.md) counts them. What these numbers cannot show: the OMG corpora are demonstrations rather than an official conformance suite; the differential is one-directional, comparing the diagnostics the two implementations report on the same files; the Xpect suites are the pilot authors' test intent rather than a certification oracle; and none of these is a percentage of the specification — no global compliance figure is claimed anywhere. diff --git a/docs/project/pilot-differential.md b/docs/project/pilot-differential.md index eeea64846d..36da542e46 100644 --- a/docs/project/pilot-differential.md +++ b/docs/project/pilot-differential.md @@ -209,7 +209,7 @@ nor double-counted as two independent disagreements. --- -## Results (pilot `2026-08`, 378 files) +## Results (pilot `2026-08`, 379 files) | Root | Files | Fully agreeing | Ours | Pilot | Agreed | Severity-only | Only ours | Only pilot | |---|---:|---:|---:|---:|---:|---:|---:|---:| @@ -218,9 +218,9 @@ nor double-counted as two independent disagreements. | `examples/pilot-corpora/sysml-validation` | 56 | 56 | 0 | 0 | 0 | 0 | 0 | 0 | | `examples/pilot-corpora/kerml-examples` | 58 | 55 | 10 | 0 | 0 | 0 | 10 | 0 | | `tests/testdata` | 18 | 10 | 43 | 55 | 34 | 1 | 8 | 20 | -| `examples` | 43 | 30 | 13 | 1171 | 4 | 2 | 7 | 1165 | +| `examples` | 44 | 30 | 13 | 1319 | 4 | 2 | 7 | 1313 | | `tools/referee/diff/testdata` (probes) | 4 | 1 | 6 | 0 | 0 | 0 | 6 | 0 | -| **Total** | **378** | **347** | **79** | **1226** | **38** | **3** | **38** | **1185** | +| **Total** | **379** | **347** | **79** | **1374** | **38** | **3** | **38** | **1333** | **Read the `only ours` total by root, never as one number.** Step 2 removes nine resolver false positives from the reference's **own** corpora: `pilot-examples` 16 → **7** and @@ -791,8 +791,8 @@ cascades through the rest of the file. The movement is entirely one file, | Count | Before the initializer rewrite | Now | |---|---:|---:| -| only pilot | 82 | **1185** | -| pilot diagnostics | 123 | **1226** | +| only pilot | 82 | **1333** | +| pilot diagnostics | 123 | **1374** | | severity-only | 9 | **3** | The rewrite itself took only-pilot to 61 and pilot diagnostics to 101; the `Now` column states @@ -925,7 +925,7 @@ Per category, the only-ours totals are: `pilot-examples` 4 `unmapped`, 2 advisory of the [runtime showcase round](#runtime-showcase-round)); `testdata` 7 `unmapped`, 1 `multiplicity`; `probes` 6 `unmapped`. Only-pilot: `testdata` 12 `kind-mismatch`, 3 `unmapped`, 3 syntax, 2 `unresolved-reference`; -`examples` 10 syntax, 19 `unmapped`, 547 `kind-mismatch`, 589 `unresolved-reference` — of which +`examples` 10 syntax, 19 `unmapped`, 587 `kind-mismatch`, 697 `unresolved-reference` — of which `relay-probe-demo/mission.sysml` carries none: it carried a `kind-mismatch` on its send of a `Telemetry` invocation until the send-argument round above, and the demo now writes the constructor, `send new Telemetry(…) via antenna`, which both implementations accept, so the row @@ -1017,13 +1017,13 @@ page's history. | Count | Now | |---|---:| | overall: fully agreeing / only ours / our diagnostics | **347 / 38 / 79** | -| only pilot | **1185** | -| pilot diagnostics | **1226** | +| only pilot | **1333** | +| pilot diagnostics | **1374** | | severity-only | **3** | | unmapped, our side | **34** | | kerml-examples: only ours | **10** | | pilot-examples: only ours | **7** | -| examples: only pilot | **1165** | +| examples: only pilot | **1313** | The KerML root is now the *cleanest* of the three OMG roots in proportion: **10** only-ours against 6 only-pilot, with 49 of 58 files fully @@ -1168,6 +1168,19 @@ of them the `kind-mismatch` rows adjudicated below where a `calc def` is passed Nothing else moves: the file draws no diagnostic from this implementation, so `fully agreeing`, `only ours` and `agreed` stay where the analysis walkthrough left them. +### Analysis results recording round + +`examples/analysis-results-demo/lander-results.sysml` is one file added to the `examples` root: +files 43 → **44** on the root, 378 → **379** overall, and pilot diagnostics 1226 → **1374** / +only pilot 1185 → **1333** — 148 diagnostics in 97 reported rows, 62 `unresolved-reference` +(counted 103) and 35 `kind-mismatch` (counted 45), all inside the file's `Results` and +`Reporting` packages. Every row is a construct the pinned artifact has no support for: the +document-query calls (`Project`, `OrderBy`, `WhereType`, `WhereFeature`, `WhereMetadata`, +`Verdicts`, `Descendants`), the `@RecordedRun` metadata annotations, and the run-record part +usages' `'objective'` quoted name and `ref part :>>` redefinitions. The file draws no diagnostic +from this implementation — it validates clean — so `fully agreeing`, `only ours`, `agreed` and +`severity-only` all stay where the expressions walkthrough left them. + ## Adjudications ### Only ours — candidate false positives (3, SysML side) From 209819c6ae06a0c1193c603469c3c801836d201b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 06:17:29 +0000 Subject: [PATCH 06/10] docs(examples): build the analysis-results demo on the AnalysisRecords library Co-Authored-By: jason.han --- .../unreleased/analysis-results-demo.added.md | 2 +- examples/analysis-results-demo/README.md | 161 +++++++++----- .../lander-results.sysml | 203 ++++++++++++------ examples/analysis-results-demo/report.md | 40 ++-- 4 files changed, 270 insertions(+), 136 deletions(-) diff --git a/changes/unreleased/analysis-results-demo.added.md b/changes/unreleased/analysis-results-demo.added.md index 4ab1967d07..9c14ad6347 100644 --- a/changes/unreleased/analysis-results-demo.added.md +++ b/changes/unreleased/analysis-results-demo.added.md @@ -1 +1 @@ -- **Added `examples/analysis-results-demo/`, a worked example of saving analysis runs into the model and reporting them in a document.** Analysis and sweep output is printed and discarded, so the demo records each run as a `part` usage typed by a result-record definition — inputs, outputs and objective as attributes, `@RecordedRun` provenance metadata, and a `ref part` to the subject — then renders a report that groups, filters and lists the records, flags the record a later model edit made stale through a derived `drift`/`stale` pair, and contrasts them with `Verdicts` recomputed live at render time. +- **Added `examples/analysis-results-demo/`, a worked example of saving analysis runs into the model and reporting them in a document.** The records are `part` usages on the bundled `AnalysisRecords` vocabulary — the same shape `-record-run`/`%record` emits, written by hand to keep pre-edit values — and a generated report groups, filters and lists them, flags the record a later model edit made stale through a derived `drift`/`stale` pair, and contrasts them with `Verdicts` recomputed live at render time. diff --git a/examples/analysis-results-demo/README.md b/examples/analysis-results-demo/README.md index a01a4c3b54..f35c0b7427 100644 --- a/examples/analysis-results-demo/README.md +++ b/examples/analysis-results-demo/README.md @@ -4,39 +4,32 @@ results of analysis runs be saved into the model itself, so a generated document can tabulate them later?** -The honest answer is *not by itself, but yes by pattern*. `-analysis` and -`-sweep` print each run's inputs, outputs and verdicts and then discard them: -a run leaves no element in the model and no held object a document query can -read, and `-render-document` cannot be combined with `-analysis`, so the -document never sees a run happen. What a document *can* see is anything the -model declares — so the recording pattern is to write each run back yourself: -a `part` usage typed by a result-record definition, holding the run's inputs, -outputs and objective as attribute values, annotated with provenance metadata -and pointing at the part the run was about. [`report.md`](report.md) is what -the document then renders. +The honest answer is *yes — by recording them, which the tool can now do +for you*. `-analysis` and `-sweep` print each run's inputs, outputs and +verdicts, but a run leaves nothing a document query can read until it is +**recorded**: `%record` and `-record-run` write the run back into the model as +a `part` usage typed by a result-record definition in the bundled +`AnalysisRecords` library — inputs, outputs, objective status, verdicts and +evaluations, annotated with provenance metadata and a `ref` to the part the +run was about. This demo's records are the same shape, written by hand (why, +below). [`report.md`](report.md) is what the document then renders. ## The recording pattern -The `Records` package declares the vocabulary: +The `Records` package builds the demo's record definitions on the bundled +`AnalysisRecords` library, which supplies `RecordedRun` (provenance +metadata: `runAt`, `tool`, `command`, `kind`), `AnalysisRun` (`caseName`, +`kind`, `'objective'`, `iteration`, `'subject'`, `subjectName`, `verdict`, +plus `verdicts` and `evaluations` collections), `VerdictRecord` and +`EvaluationRecord`: ```sysml -metadata def RecordedRun { - attribute runAt : String; - attribute tool : String; - attribute revision : String; +part def DemoRun :> AnalysisRecords::AnalysisRun { attribute command : String; } -part def AnalysisRun { - attribute caseName : String; - attribute kind : String; // "run" | "sweep" | "trade" - attribute 'objective' : String; // "satisfied" | "not satisfied" | "undecided" - attribute command : String; -} - -part def FuelBudgetRun :> AnalysisRun { +part def FuelBudgetRun :> DemoRun { ref part lander : Lander; - attribute subjectName : String; attribute burnTime : Real; attribute fuelUsed : Real; attribute wetMass : Real; @@ -52,30 +45,41 @@ printed output: ```sysml part scoutRun : FuelBudgetRun { - @RecordedRun { + @AnalysisRecords::RecordedRun { runAt = "2025-11-02T09:14:00Z"; tool = "sysml"; - revision = "v0.8"; command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; + kind = "run"; } - ref part :>> lander = scout; attribute :>> caseName = "Descent::scoutBudget"; attribute :>> kind = "run"; attribute :>> 'objective' = "satisfied"; - attribute :>> subjectName = "scout"; + ref :>> 'subject' = Landers::scout; + ref part :>> lander = scout; + attribute :>> subjectName = "Landers::scout"; attribute :>> burnTime = 40.0; attribute :>> fuelUsed = 120.0; attribute :>> wetMass = 730.0; attribute :>> fuelLeft = 130.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; + } } ``` -Two limitations shape the record. The annotation's attribute values are not -projectable — `Project(properties = ("runAt"))` reports `unknown property` — -so `command` is also carried as a plain attribute for the provenance table to -show; the `@RecordedRun` metadata still answers `WhereMetadata` filters and -keeps the provenance machine-readable. And `objective` is a reserved word, -written `'objective'` wherever a name is needed. +Two limitations shape the record. The annotation's attribute values are +not projectable — `Project(properties = ("runAt"))` reports +`unknown property` — so `command` is also carried as a plain attribute on +`DemoRun` for the provenance table to show; the `@AnalysisRecords::RecordedRun` +metadata still answers `WhereMetadata` filters and keeps the provenance +machine-readable. And the library's untyped `'subject'` ref cannot drive the +`liveFuelLeft` formula, so `FuelBudgetRun` keeps a typed `lander` ref and each +record binds both to the same part. Sweep records set the library's +`iteration` attribute (1, 2, 3); the trade-study record carries its scores as +`EvaluationRecord` usages under `evaluations`, the same shape `-record-run` +emits. ## The runs that were recorded @@ -189,13 +193,14 @@ can never give you: a record that notices the model moved. drift = 30.0 ``` -The relay edit made a second record stale too: `lightestRun` still reports -`relayScore = 630.0` while `-analysis Selection::lightest` now scores relay -`660.0`. Nothing flags it — `TradeStudyRun` rederives nothing, so it has no -`liveFuelLeft` to compare against. That is the limitation the paragraph above -describes, made concrete: drift detection only exists where the record -definition recomputes the value itself, which is exactly what an automated -record step would have to emit for every output it saves. +The relay edit made a second record stale too: `lightestRun`'s third +`EvaluationRecord` still says `score = 630.0` while `-analysis +Selection::lightest` now scores relay `660.0`. Nothing flags it — +`EvaluationRecord` rederives nothing, so it has no `liveFuelLeft` to compare +against. That is the limitation the paragraph above describes, made concrete: +drift detection only exists where the record definition recomputes the value +itself, which neither the library's records nor `-record-run`'s output does — +a recompute-def like `FuelBudgetRun` is something a modeler writes on purpose. The comparison is a derived Boolean on the record definition rather than a `Column` expression, because computed columns do not support `!=`. Any model @@ -212,7 +217,7 @@ recomputes an output can detect it, and only for the values it rederives. [`report.md`](report.md) is committed so the test suite can compare the render byte-for-byte. It shows a grouped table of every fuel-budget record by subject, the sweep rows alone, the stale-records table (exactly `relayRun`), -a provenance table over `WhereMetadata(... 'metadata' = "Records::RecordedRun")`, +a provenance table over `WhereMetadata(... 'metadata' = "AnalysisRecords::RecordedRun")`, the trade-study record, and — the contrast — a `Verdicts` table of the assertions about `scout` **evaluated live at render time**: the records say what a run printed; the verdicts say what holds now. @@ -226,17 +231,69 @@ HTML and PDF render the same document tree: -render-document Reporting::AnalysisReport -doc-form pdf -o report.pdf ``` -## Why isn't this automatic? +## Recording runs automatically + +Everything above was written by hand; the same vocabulary is what +`-record-run` emits. Running a case with `-record-run` records it into a +`Records` package beside the case and, with `-convert sysml`, writes the +model — records included — back out: + +```bash +./bin/sysml examples/analysis-results-demo/lander-results.sysml \ + -record-run "Descent::scoutBudget" -convert sysml -o recorded.sysml +``` + +``` +✓ Descent::scoutBudget + fuelUsed = 120.0 + wetMass = 730.0 + fuelLeft = 130.0 + objective reserveHeld: satisfied + standing: value (observed: 1 run under reverse) + recorded Records::scoutBudget_run1 (Records::ScoutBudgetRun) +``` + +The generated record is the same shape this demo writes by hand — a def per +case specializing `AnalysisRecords::AnalysisRun`, the library annotation, the +subject ref, and a `VerdictRecord` per check: + +```sysml +part def ScoutBudgetRun :> AnalysisRecords::AnalysisRun { + attribute :>> caseName default = "Descent::scoutBudget"; + attribute burnTime : ScalarValues::Real; + ... +} +part scoutBudget_run1 : ScoutBudgetRun { + @AnalysisRecords::RecordedRun { + runAt = "2026-09-24T06:08:21Z"; + tool = "sysml v0.8.1-2129-ge8b389eea"; + command = "-record-run \"Descent::scoutBudget\""; + kind = "run"; + } + ... + ref :>> 'subject' = Landers::scout; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; + } +} +``` -Because nothing bridges two surfaces the tool already has — an implementation -gap, not an architectural limitation. The runtime already holds each -analysis run's results as typed values, `%save` already serializes the -session model, and document queries already read declared elements and held -objects; but analysis output is printed and discarded, no step writes it -back, and `-render-document` cannot run alongside `-analysis`. The pattern -this demo records by hand — a `part` usage typed by a result-record -definition, provenance metadata, a `ref part` to the subject — is the shape -an automated record step would emit. +Because both speak `AnalysisRecords`, a document written against +`WhereMetadata(... 'metadata' = "AnalysisRecords::RecordedRun")` and +`WhereType(... "FuelBudgetRun")`-style filters reads hand-written and +`-record-run` records alike. The REPL form is `%record`; sweeps record one +record per row (`recorded 3 runs as Records::scoutBudget_run1 …`), though +`-convert` refuses a sweep — write the model out after a single +`-record-run`, or record each row explicitly. The full flag reference is +[the manual](../../docs/manual/recording-analysis-runs.md). + +These records stay hand-written for two reasons: the stale-relay story needs +values from *before* `relay.fuel` changed — a `-record-run` today would +record `fuelLeft = 110.0` and no drift — and the derived `liveFuelLeft` / +`drift` / `stale` columns live on the shared `DemoRun`/`FuelBudgetRun` defs, +while `-record-run` writes one def per case with no recompute. ## Where to read more diff --git a/examples/analysis-results-demo/lander-results.sysml b/examples/analysis-results-demo/lander-results.sysml index 527c7ef875..7d874f2f75 100644 --- a/examples/analysis-results-demo/lander-results.sysml +++ b/examples/analysis-results-demo/lander-results.sysml @@ -1,9 +1,7 @@ -// Analysis runs are printed, not saved: `-analysis` and `-sweep` report each -// run's inputs, outputs and verdicts and then discard them, so nothing a -// document query can read records that a run happened. This model shows the -// workaround — write each run back into the model yourself as a `part` usage -// typed by a result-record definition, annotated with provenance metadata — -// and the one thing the pattern cannot fake: a derived attribute on the +// An analysis run's printed results are discarded unless recorded — `%record` +// and `-record-run` write them back on the `AnalysisRecords` library +// vocabulary, or, as here, by hand. These records are hand-written because the +// story needs values from before a model edit: a derived attribute on the // record recomputes the run's output from the model *as it stands now*, so a // record whose numbers the model has moved away from reports its own drift. // See README.md for the commands that produced each recorded row. @@ -163,34 +161,24 @@ package Selection { package Records { private import ScalarValues::*; private import Landers::*; + private import AnalysisRecords::*; - // Provenance of a saved run: when, by what, at which revision, which command. - metadata def RecordedRun { - attribute runAt : String; - attribute tool : String; - attribute revision : String; - attribute command : String; - } - - // The vocabulary of a saved analysis run. `kind` is "run", "sweep" or - // "trade"; `objective` records the verdict the run printed. - // `command` is a plain attribute, not metadata, because a document - // query's `Project` cannot read an annotation's attribute values — it is - // what the provenance table shows for each record. - part def AnalysisRun { - attribute caseName : String; - attribute kind : String; - attribute 'objective' : String; + // Shared definition for the demo's hand-written records: the library's + // AnalysisRun plus `command` as a plain attribute — a document query's + // `Project` cannot read an annotation's attribute values, so the provenance + // table needs it declared. + part def DemoRun :> AnalysisRecords::AnalysisRun { attribute command : String; } // A FuelBudget run written back: subject, input and outputs as recorded, // plus the output recomputed from the model as it stands now. `drift` // measures how far the model has moved since the record was written, and - // `stale` names a record that no longer matches. - part def FuelBudgetRun :> AnalysisRun { + // `stale` names a record that no longer matches. `lander` is the typed + // subject the formula reads; the library's untyped `'subject'` ref is + // bound to the same part on each record. + part def FuelBudgetRun :> DemoRun { ref part lander : Lander; - attribute subjectName : String; attribute burnTime : Real; attribute fuelUsed : Real; attribute wetMass : Real; @@ -200,151 +188,218 @@ package Records { attribute stale : Boolean = drift != 0.0; } - // A TradeStudy run written back: the selection and every score it printed. - part def TradeStudyRun :> AnalysisRun { + // A TradeStudy run written back: the selection, plus the run's checks and + // evaluations carried under the library's `verdicts` and `evaluations`. + part def TradeStudyRun :> DemoRun { attribute selected : String; - attribute scoutScore : Real; - attribute haulerScore : Real; - attribute relayScore : Real; } } package Results { private import ScalarValues::*; private import Landers::*; + private import AnalysisRecords::*; private import Records::*; + // The saved runs, held as subparts of one part so a query can walk them. // The saved runs, held as subparts of one part so a query can walk them. part results { // One baseline run per candidate, at FuelBudget's default 40 s burn. part scoutRun : FuelBudgetRun { - @RecordedRun { + @AnalysisRecords::RecordedRun { runAt = "2025-11-02T09:14:00Z"; tool = "sysml"; - revision = "v0.8"; command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; + kind = "run"; } - ref part :>> lander = scout; attribute :>> caseName = "Descent::scoutBudget"; attribute :>> kind = "run"; attribute :>> 'objective' = "satisfied"; attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; - attribute :>> subjectName = "scout"; + ref :>> 'subject' = Landers::scout; + ref part :>> lander = scout; + attribute :>> subjectName = "Landers::scout"; attribute :>> burnTime = 40.0; attribute :>> fuelUsed = 120.0; attribute :>> wetMass = 730.0; attribute :>> fuelLeft = 130.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; + } } part haulerRun : FuelBudgetRun { - @RecordedRun { + @AnalysisRecords::RecordedRun { runAt = "2025-11-02T09:14:30Z"; tool = "sysml"; - revision = "v0.8"; command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; + kind = "run"; } - ref part :>> lander = hauler; attribute :>> caseName = "Descent::haulerBudget"; attribute :>> kind = "run"; attribute :>> 'objective' = "satisfied"; attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; - attribute :>> subjectName = "hauler"; + ref :>> 'subject' = Landers::hauler; + ref part :>> lander = hauler; + attribute :>> subjectName = "Landers::hauler"; attribute :>> burnTime = 40.0; attribute :>> fuelUsed = 320.0; attribute :>> wetMass = 1980.0; attribute :>> fuelLeft = 580.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; + } } part relayRun : FuelBudgetRun { - @RecordedRun { + @AnalysisRecords::RecordedRun { runAt = "2025-11-02T09:15:00Z"; tool = "sysml"; - revision = "v0.8"; command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; + kind = "run"; } - ref part :>> lander = relay; attribute :>> caseName = "Descent::relayBudget"; attribute :>> kind = "run"; attribute :>> 'objective' = "satisfied"; attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; - attribute :>> subjectName = "relay"; + ref :>> 'subject' = Landers::relay; + ref part :>> lander = relay; + attribute :>> subjectName = "Landers::relay"; attribute :>> burnTime = 40.0; attribute :>> fuelUsed = 100.0; attribute :>> wetMass = 530.0; attribute :>> fuelLeft = 80.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; + } } // A sweep of scoutBudget over burnTime, one record per row the sweep printed. part scoutSweep40 : FuelBudgetRun { - @RecordedRun { + @AnalysisRecords::RecordedRun { runAt = "2025-11-02T09:20:10Z"; tool = "sysml"; - revision = "v0.8"; command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + kind = "sweep"; } - ref part :>> lander = scout; attribute :>> caseName = "Descent::scoutBudget"; attribute :>> kind = "sweep"; attribute :>> 'objective' = "satisfied"; attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - attribute :>> subjectName = "scout"; + ref :>> 'subject' = Landers::scout; + ref part :>> lander = scout; + attribute :>> subjectName = "Landers::scout"; + attribute :>> iteration = 1; attribute :>> burnTime = 40.0; attribute :>> fuelUsed = 120.0; attribute :>> wetMass = 730.0; attribute :>> fuelLeft = 130.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; + } } part scoutSweep60 : FuelBudgetRun { - @RecordedRun { + @AnalysisRecords::RecordedRun { runAt = "2025-11-02T09:20:10Z"; tool = "sysml"; - revision = "v0.8"; command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + kind = "sweep"; } - ref part :>> lander = scout; attribute :>> caseName = "Descent::scoutBudget"; attribute :>> kind = "sweep"; attribute :>> 'objective' = "satisfied"; attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - attribute :>> subjectName = "scout"; + ref :>> 'subject' = Landers::scout; + ref part :>> lander = scout; + attribute :>> subjectName = "Landers::scout"; + attribute :>> iteration = 2; attribute :>> burnTime = 60.0; attribute :>> fuelUsed = 180.0; attribute :>> wetMass = 670.0; attribute :>> fuelLeft = 70.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; + } } part scoutSweep80 : FuelBudgetRun { - @RecordedRun { + @AnalysisRecords::RecordedRun { runAt = "2025-11-02T09:20:10Z"; tool = "sysml"; - revision = "v0.8"; command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + kind = "sweep"; } - ref part :>> lander = scout; attribute :>> caseName = "Descent::scoutBudget"; attribute :>> kind = "sweep"; attribute :>> 'objective' = "not satisfied"; attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - attribute :>> subjectName = "scout"; + ref :>> 'subject' = Landers::scout; + ref part :>> lander = scout; + attribute :>> subjectName = "Landers::scout"; + attribute :>> iteration = 3; attribute :>> burnTime = 80.0; attribute :>> fuelUsed = 240.0; attribute :>> wetMass = 610.0; attribute :>> fuelLeft = 10.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "not satisfied"; + } } - // The trade study run: the selection and each alternative's score. + // The trade study run: the selection, its objective verdict, and each + // alternative's evaluation — recorded before relay.fuel changed, so the + // relay score here is the old 630.0. part lightestRun : TradeStudyRun { - @RecordedRun { + @AnalysisRecords::RecordedRun { runAt = "2025-11-02T09:25:00Z"; tool = "sysml"; - revision = "v0.8"; command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; + kind = "trade"; } attribute :>> caseName = "Selection::lightest"; attribute :>> kind = "trade"; attribute :>> 'objective' = "satisfied"; attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; - attribute :>> selected = "relay"; - attribute :>> scoutScore = 850.0; - attribute :>> haulerScore = 2300.0; - attribute :>> relayScore = 630.0; + attribute :>> selected = "Landers::relay"; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "tradeStudyObjective"; + attribute :>> status = "satisfied"; + } + part evaluation1 : AnalysisRecords::EvaluationRecord :> evaluations { + attribute :>> 'function' = "Selection::lightest::evaluationFunction"; + attribute :>> alternative = "Landers::scout (object #1)"; + attribute :>> score = 850.0; + attribute :>> result = "850.0"; + attribute :>> selected = false; + attribute :>> tied = false; + } + part evaluation2 : AnalysisRecords::EvaluationRecord :> evaluations { + attribute :>> 'function' = "Selection::lightest::evaluationFunction"; + attribute :>> alternative = "Landers::hauler (object #2)"; + attribute :>> score = 2300.0; + attribute :>> result = "2300.0"; + attribute :>> selected = false; + attribute :>> tied = false; + } + part evaluation3 : AnalysisRecords::EvaluationRecord :> evaluations { + attribute :>> 'function' = "Selection::lightest::evaluationFunction"; + attribute :>> alternative = "Landers::relay (object #3)"; + attribute :>> score = 630.0; + attribute :>> result = "630.0"; + attribute :>> selected = true; + attribute :>> tied = false; + } } } } @@ -391,7 +446,7 @@ package Reporting { calc def Provenance :> Query { in root : Element; Project( - source = WhereMetadata(source = Descendants(source = root, maxDepth = 1), 'metadata' = "Records::RecordedRun"), + source = WhereMetadata(source = Descendants(source = root, maxDepth = 1), 'metadata' = "AnalysisRecords::RecordedRun"), properties = ("name", "kind", "caseName", "command")) } @@ -400,7 +455,15 @@ package Reporting { in root : Element; Project( source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "TradeStudyRun"), - properties = ("name", "caseName", "selected", "scoutScore", "haulerScore", "relayScore", "objective")) + properties = ("name", "caseName", "selected", "objective")) + } + + // The evaluations nested under the trade-study record. + calc def TradeEvaluations :> Query { + in root : Element; + Project( + source = WhereType(source = Descendants(source = root, maxDepth = 2), type = "EvaluationRecord"), + properties = ("alternative", "score", "selected", "tied")) } // Assertions about the scout, checked live at render time. @@ -416,7 +479,7 @@ package Reporting { attribute redefines title = "Recorded analysis runs"; part intro : Paragraph { - attribute redefines text = "Analysis runs are printed and discarded; nothing writes them back. Every row below is a declared record — a part typed by a run definition whose attributes hold the inputs, outputs and objective a run printed, annotated with the command that produced it — except the last table, whose verdicts are recomputed live."; + attribute redefines text = "An analysis run's printed results are discarded unless they are recorded — by -record-run or by hand, in the vocabulary the AnalysisRecords library defines. Every row below is a declared record typed by a run definition on that vocabulary, annotated with the command that produced it — except the last table, whose verdicts are recomputed live."; } part budgets : Section { @@ -453,7 +516,7 @@ package Reporting { part provenance : Section { attribute redefines title = "Provenance"; part runs : Table { - attribute redefines caption = "Every element annotated @RecordedRun"; + attribute redefines caption = "Every element annotated @AnalysisRecords::RecordedRun"; calc rows : Provenance { in root = results; } @@ -463,11 +526,17 @@ package Reporting { part trade : Section { attribute redefines title = "Trade study"; part runs : Table { - attribute redefines caption = "The recorded selection and scores"; + attribute redefines caption = "The recorded selection and verdict"; calc rows : TradeRuns { in root = results; } } + part evals : Table { + attribute redefines caption = "Each alternative's evaluation, as recorded"; + calc rows : TradeEvaluations { + in root = results; + } + } } part live : Section { diff --git a/examples/analysis-results-demo/report.md b/examples/analysis-results-demo/report.md index b837b3990a..5eaa424351 100644 --- a/examples/analysis-results-demo/report.md +++ b/examples/analysis-results-demo/report.md @@ -1,31 +1,31 @@ # Recorded analysis runs -Analysis runs are printed and discarded; nothing writes them back. Every row below is a declared record — a part typed by a run definition whose attributes hold the inputs, outputs and objective a run printed, annotated with the command that produced it — except the last table, whose verdicts are recomputed live. +An analysis run's printed results are discarded unless they are recorded — by -record-run or by hand, in the vocabulary the AnalysisRecords library defines. Every row below is a declared record typed by a run definition on that vocabulary, annotated with the command that produced it — except the last table, whose verdicts are recomputed live. ## Recorded fuel budgets *One record per run, grouped by subject* -**subjectName: scout** +**subjectName: Landers::scout** | name | subjectName | kind | burnTime | fuelUsed | wetMass | fuelLeft | objective | | --- | --- | --- | --- | --- | --- | --- | --- | -| scoutRun | scout | run | 40 | 120 | 730 | 130 | satisfied | -| scoutSweep40 | scout | sweep | 40 | 120 | 730 | 130 | satisfied | -| scoutSweep60 | scout | sweep | 60 | 180 | 670 | 70 | satisfied | -| scoutSweep80 | scout | sweep | 80 | 240 | 610 | 10 | not satisfied | +| scoutRun | Landers::scout | run | 40 | 120 | 730 | 130 | satisfied | +| scoutSweep40 | Landers::scout | sweep | 40 | 120 | 730 | 130 | satisfied | +| scoutSweep60 | Landers::scout | sweep | 60 | 180 | 670 | 70 | satisfied | +| scoutSweep80 | Landers::scout | sweep | 80 | 240 | 610 | 10 | not satisfied | -**subjectName: hauler** +**subjectName: Landers::hauler** | name | subjectName | kind | burnTime | fuelUsed | wetMass | fuelLeft | objective | | --- | --- | --- | --- | --- | --- | --- | --- | -| haulerRun | hauler | run | 40 | 320 | 1980 | 580 | satisfied | +| haulerRun | Landers::hauler | run | 40 | 320 | 1980 | 580 | satisfied | -**subjectName: relay** +**subjectName: Landers::relay** | name | subjectName | kind | burnTime | fuelUsed | wetMass | fuelLeft | objective | | --- | --- | --- | --- | --- | --- | --- | --- | -| relayRun | relay | run | 40 | 100 | 530 | 80 | satisfied | +| relayRun | Landers::relay | run | 40 | 100 | 530 | 80 | satisfied | ## Sweep of scoutBudget @@ -43,11 +43,11 @@ Analysis runs are printed and discarded; nothing writes them back. Every row bel | name | subjectName | burnTime | fuelLeft | liveFuelLeft | drift | | --- | --- | --- | --- | --- | --- | -| relayRun | relay | 40 | 80 | 110 | 30 | +| relayRun | Landers::relay | 40 | 80 | 110 | 30 | ## Provenance -*Every element annotated @RecordedRun* +*Every element annotated @AnalysisRecords::RecordedRun* | name | kind | caseName | command | | --- | --- | --- | --- | @@ -61,11 +61,19 @@ Analysis runs are printed and discarded; nothing writes them back. Every row bel ## Trade study -*The recorded selection and scores* +*The recorded selection and verdict* -| name | caseName | selected | scoutScore | haulerScore | relayScore | objective | -| --- | --- | --- | --- | --- | --- | --- | -| lightestRun | Selection::lightest | relay | 850 | 2300 | 630 | satisfied | +| name | caseName | selected | objective | +| --- | --- | --- | --- | +| lightestRun | Selection::lightest | Landers::relay | satisfied | + +*Each alternative's evaluation, as recorded* + +| alternative | score | selected | tied | +| --- | --- | --- | --- | +| Landers::scout (object \#1) | 850 | false | false | +| Landers::hauler (object \#2) | 2300 | false | false | +| Landers::relay (object \#3) | 630 | true | false | ## Live verdicts From e2f23843997d8c97137706e88ef51cc153389b40 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 06:17:29 +0000 Subject: [PATCH 07/10] docs(project): re-record the differential baseline for the library-based demo Co-Authored-By: jason.han --- .../testing-pilot-corpora-gate/SKILL.md | 2 +- .../testing-pilot-differential/SKILL.md | 6 +- .../testing-pilot-execution-referee/SKILL.md | 2 +- .agents/skills/testing-pilot-xpect/SKILL.md | 2 +- README.md | 4 +- docs/internals/architecture.md | 4 +- docs/project/pilot-differential-baseline.json | 1703 ++++++++++++++++- docs/project/pilot-differential.md | 35 +- 8 files changed, 1720 insertions(+), 38 deletions(-) diff --git a/.agents/skills/testing-pilot-corpora-gate/SKILL.md b/.agents/skills/testing-pilot-corpora-gate/SKILL.md index aa41f422c2..6ee7843de6 100644 --- a/.agents/skills/testing-pilot-corpora-gate/SKILL.md +++ b/.agents/skills/testing-pilot-corpora-gate/SKILL.md @@ -184,7 +184,7 @@ gate's own helpers are package-private but reusable (`pilotCorporaGate.files(t)` `actionlint`, `shellcheck`, `python3 scripts/check-doc-links.py`, `gofmt`, `go vet`, `go run -C tools ./cmd/pilot-diff` (validators pre-downloaded; ~4min, prints e.g. the headline the committed baseline holds — `379 file(s), 347 fully agreeing; 38 agreed -diagnostic(s), 38 only ours, 1333 only the pilot's` at the `2026-08` pin, so read it from +diagnostic(s), 38 only ours, 1545 only the pilot's` at the `2026-08` pin, so read it from `docs/project/pilot-differential-baseline.json` rather than from this line) and `make lint` (staticcheck+gosec, ~2min) all work. There is **no** `yamllint` and **no** `circleci` CLI, so `.circleci/config.yml` can only be parsed as YAML, not schema-validated — say so diff --git a/.agents/skills/testing-pilot-differential/SKILL.md b/.agents/skills/testing-pilot-differential/SKILL.md index 4971b493b8..e0414ca16e 100644 --- a/.agents/skills/testing-pilot-differential/SKILL.md +++ b/.agents/skills/testing-pilot-differential/SKILL.md @@ -22,8 +22,8 @@ GNU-format diagnostics **relative to `--root`**. Consequences for testing: (written by the new script), not from the DeciSym `pom.xml`. - `-validator /nonexistent` now says `run ./scripts/download-pilot-sysml-validator.sh`. - Measured at the `2026-08` pin, with a fresh library cache: `379 file(s), 347 fully agreeing; 38 agreed, - 38 only ours, 1333 only the pilot's`, JSON totals `openSysMLDiagnostics 79 / pilotDiagnostics - 1374 / severityMismatch 3`; ~2 min wall, byte-identical across runs *and* after a from-scratch + 38 only ours, 1545 only the pilot's`, JSON totals `openSysMLDiagnostics 79 / pilotDiagnostics + 1586 / severityMismatch 3`; ~2 min wall, byte-identical across runs *and* after a from-scratch rebuild of `build/pilot-validator`. The six `kerml-examples` pilot-only rows the `2026-07` run carried (`The opposite features 'owningType' … do not refer to each other`) are gone: the pilot fixed its `ownedDisjoining` delegate, and nothing on our side moved. `kerml-examples` carries no `syntax` diagnostic on either @@ -137,7 +137,7 @@ The harness compares OpenSysML diagnostics against the OMG SysML v2 Pilot Implem `build/pilot-diff/pilot-diff.{txt,json}`. `docs/project/pilot-differential-baseline.json` is the committed result of the *last refreshed* run, so **the harness is testable by reproduction** — but only while the baseline is current. Check that first. As of the rebaseline that came when the Legend of the Red Dragon example left for its own repository it **is** -current: a live run gives `379 file(s), 347 fully agreeing; 38 agreed, 38 only ours, 1333 only the +current: a live run gives `379 file(s), 347 fully agreeing; 38 agreed, 38 only ours, 1545 only the pilot's`, byte-identical to the committed baseline, and `docs/project/pilot-differential.md`'s "Results" table matches. The rebaseline before it, at the architecture self-model's landing, covered two rounds, because the succession-shorthand removal before it landed without refreshing the baseline; a control run of its merge commit gives diff --git a/.agents/skills/testing-pilot-execution-referee/SKILL.md b/.agents/skills/testing-pilot-execution-referee/SKILL.md index e212010d36..3d4730b86c 100644 --- a/.agents/skills/testing-pilot-execution-referee/SKILL.md +++ b/.agents/skills/testing-pilot-execution-referee/SKILL.md @@ -149,7 +149,7 @@ pilot answers the representation's own. See such file or directory`. - **Additivity.** `go run -C tools ./cmd/pilot-diff` must still print the headline the committed baseline holds (`379 file(s), 347 fully agreeing; 38 agreed - diagnostic(s), 38 only ours, 1333 only the pilot's` at the `2026-08` pin — read it from the baseline JSON, not from this line, since each + diagnostic(s), 38 only ours, 1545 only the pilot's` at the `2026-08` pin — read it from the baseline JSON, not from this line, since each fix round moves it) and `jq -S` diff clean against `docs/project/pilot-differential-baseline.json`; `git status --porcelain` empty at the end. diff --git a/.agents/skills/testing-pilot-xpect/SKILL.md b/.agents/skills/testing-pilot-xpect/SKILL.md index 8e051dfaef..30f0b71e98 100644 --- a/.agents/skills/testing-pilot-xpect/SKILL.md +++ b/.agents/skills/testing-pilot-xpect/SKILL.md @@ -417,7 +417,7 @@ census in `w5c_census_test.go` is live two ways: perturb one pinned triple (e.g. `go run -C tools ./cmd/pilot-diff` (~1m12s) must still print the headline the *committed* baseline holds — at the `2026-08` pin that is `379 file(s), 347 fully agreeing; 38 agreed diagnostic(s), 38 -only ours, 1333 only the pilot's`. Read the number out of +only ours, 1545 only the pilot's`. Read the number out of `docs/project/pilot-differential-baseline.json` rather than trusting this line, since a landing fix round moves it. When the baseline is itself stale (it was at `19a3ce03`, holding 273 / 281 / 317), a failing `cmp` against it is *not* evidence of an Xpect regression — compare the summary line, and see diff --git a/README.md b/README.md index 7e6f8bd77c..47980f55df 100644 --- a/README.md +++ b/README.md @@ -313,11 +313,11 @@ The project is under active development, with the core infrastructure operationa **Measured against the pinned reference** (`PILOT_TAG=2026-08`, artifact `0.62.0`). Every number below is generated by `make docs-counts` from the committed baselines and gated; none of them is typed in by hand. -- **Corpus agreement:** 347 of 379 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1333 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](docs/project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). +- **Corpus agreement:** 347 of 379 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1545 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](docs/project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). - **Declared-diagnostic silence:** of the 512 declared `errors` rows in the reference's own Xpect suites, we report nothing for 0. 245 we report word-for-word; 248 wording-only and 7 location-only differences are agreement in substance and are not counted as gaps; 0 more we report as a warning and 2 elsewhere in the file ([Xpect oracle](docs/project/pilot-xpect.md), `go run -C tools ./cmd/pilot-xpect`). - **Scope agreement:** 230 of 230 declared scope assertions match exactly (same source). - **Permissiveness gaps:** of 306 invalid models we wrote ourselves, the reference rejects 4 that we accept by default, and 293 both reject; 4 further cases agree only when we are asked strictly. We authored every one of these cases ourselves, so the denominator measures the reach of our own corpus and not our conformance; agreement reached only under an opt-in strict mode is weaker evidence than agreement by default ([rejection oracle](docs/project/pilot-rejection.md), `go run -C tools ./cmd/pilot-reject`). -- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](docs/project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 379 files agreeing, 37 diagnostics ours alone and 1333 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. +- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](docs/project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 379 files agreeing, 37 diagnostics ours alone and 1545 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. - **Self-assessed surface:** the action, state-machine and classifier-behavior rows have no external referee at all — the four refereed figures above cannot see them, because the pinned artifact evaluates expressions but executes neither actions nor state machines. [Spec compliance](docs/project/spec-compliance.md) counts them. What these numbers cannot show: the OMG corpora are demonstrations rather than an official conformance suite; the differential is one-directional, comparing the diagnostics the two implementations report on the same files; the Xpect suites are the pilot authors' test intent rather than a certification oracle; and none of these is a percentage of the specification — no global compliance figure is claimed anywhere. diff --git a/docs/internals/architecture.md b/docs/internals/architecture.md index 025c1629b7..062e216c08 100644 --- a/docs/internals/architecture.md +++ b/docs/internals/architecture.md @@ -840,11 +840,11 @@ Every behavioral feature must have: **Measured against the pinned reference** (`PILOT_TAG=2026-08`, artifact `0.62.0`). Every number below is generated by `make docs-counts` from the committed baselines and gated; none of them is typed in by hand. -- **Corpus agreement:** 347 of 379 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1333 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](../project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). +- **Corpus agreement:** 347 of 379 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1545 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](../project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). - **Declared-diagnostic silence:** of the 512 declared `errors` rows in the reference's own Xpect suites, we report nothing for 0. 245 we report word-for-word; 248 wording-only and 7 location-only differences are agreement in substance and are not counted as gaps; 0 more we report as a warning and 2 elsewhere in the file ([Xpect oracle](../project/pilot-xpect.md), `go run -C tools ./cmd/pilot-xpect`). - **Scope agreement:** 230 of 230 declared scope assertions match exactly (same source). - **Permissiveness gaps:** of 306 invalid models we wrote ourselves, the reference rejects 4 that we accept by default, and 293 both reject; 4 further cases agree only when we are asked strictly. We authored every one of these cases ourselves, so the denominator measures the reach of our own corpus and not our conformance; agreement reached only under an opt-in strict mode is weaker evidence than agreement by default ([rejection oracle](../project/pilot-rejection.md), `go run -C tools ./cmd/pilot-reject`). -- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](../project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 379 files agreeing, 37 diagnostics ours alone and 1333 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. +- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](../project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 379 files agreeing, 37 diagnostics ours alone and 1545 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. - **Self-assessed surface:** the action, state-machine and classifier-behavior rows have no external referee at all — the four refereed figures above cannot see them, because the pinned artifact evaluates expressions but executes neither actions nor state machines. [Spec compliance](../project/spec-compliance.md) counts them. What these numbers cannot show: the OMG corpora are demonstrations rather than an official conformance suite; the differential is one-directional, comparing the diagnostics the two implementations report on the same files; the Xpect suites are the pilot authors' test intent rather than a certification oracle; and none of these is a percentage of the specification — no global compliance figure is claimed anywhere. diff --git a/docs/project/pilot-differential-baseline.json b/docs/project/pilot-differential-baseline.json index 9a0b2695b9..60cc6ecfdb 100644 --- a/docs/project/pilot-differential-baseline.json +++ b/docs/project/pilot-differential-baseline.json @@ -60,8 +60,8 @@ "name": "examples", "dir": "examples", "origin": "ours", - "files": 43, - "digest": "sha256:18b2fbd67ec48deea2777902bd8a7f6bbb238f44107f28c52a7cda2ef0ff9e83" + "files": 44, + "digest": "sha256:f8583984a81fa527b7afea26d5fcead936ad129d2d89133b7c1d57c3bcb0e2de" }, { "name": "probes", @@ -74,14 +74,14 @@ "recorded": "2026-09-24" }, "totals": { - "files": 378, + "files": 379, "filesFullyAgreeing": 347, "agreement": 38, "severityMismatch": 3, "openSysMLOnly": 38, - "pilotOnly": 1185, + "pilotOnly": 1545, "openSysMLDiagnostics": 79, - "pilotDiagnostics": 1226 + "pilotDiagnostics": 1586 }, "roots": [ { @@ -746,16 +746,1692 @@ "name": "examples", "dir": "examples", "totals": { - "files": 43, + "files": 44, "filesFullyAgreeing": 30, "agreement": 4, "severityMismatch": 2, "openSysMLOnly": 7, - "pilotOnly": 1165, + "pilotOnly": 1525, "openSysMLDiagnostics": 13, - "pilotDiagnostics": 1171 + "pilotDiagnostics": 1531 }, "files": [ + { + "path": "analysis-results-demo/lander-results.sysml", + "agreement": [], + "severityMismatch": [], + "openSysMLOnly": [], + "pilotOnly": [ + { + "line": 164, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 170, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 201, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 210, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 210, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 211, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 211, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 212, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 212, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 213, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 213, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 214, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 214, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 216, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 217, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 218, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 220, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 222, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 227, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 227, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 228, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 229, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 230, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 234, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 234, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 235, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 235, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 236, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 236, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 237, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 237, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 238, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 238, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 240, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 241, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 242, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 244, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 246, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 251, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 251, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 252, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 253, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 254, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 258, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 258, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 259, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 259, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 260, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 260, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 261, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 261, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 262, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 262, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 264, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 265, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 266, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 268, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 270, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 275, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 275, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 276, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 277, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 278, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 284, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 284, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 285, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 285, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 286, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 286, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 287, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 287, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 288, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 288, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 290, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 291, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 292, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 294, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 296, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 297, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 302, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 302, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 303, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 304, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 305, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 309, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 309, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 310, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 310, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 311, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 311, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 312, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 312, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 313, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 313, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 315, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 316, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 317, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 319, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 321, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 322, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 327, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 327, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 328, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 329, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 330, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 334, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 334, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 335, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 335, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 336, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 336, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 337, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 337, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 338, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 338, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 340, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 341, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 342, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 344, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 346, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 347, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 352, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 352, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 353, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 354, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 355, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 363, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 363, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 364, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 364, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 365, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 365, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 366, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 366, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 367, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 367, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 369, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 370, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 371, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 374, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 374, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 375, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 376, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 377, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 379, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 379, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 380, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 381, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 382, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 383, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 384, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 384, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 385, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 387, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 387, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 388, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 389, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 390, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 391, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 392, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 392, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 393, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 395, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 395, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 396, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 397, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 398, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 399, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 400, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 400, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 401, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 409, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 414, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 416, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 416, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 417, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 417, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 418, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 418, + "severity": "error", + "category": "unresolved-reference", + "count": 7 + }, + { + "line": 419, + "severity": "error", + "category": "unresolved-reference", + "count": 4 + }, + { + "line": 420, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 424, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 426, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 426, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 427, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 427, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 428, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 428, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 429, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 429, + "severity": "error", + "category": "unresolved-reference", + "count": 7 + }, + { + "line": 430, + "severity": "error", + "category": "unresolved-reference", + "count": 3 + }, + { + "line": 431, + "severity": "error", + "category": "unresolved-reference", + "count": 4 + }, + { + "line": 432, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 436, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 438, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 438, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 439, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 439, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 440, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 440, + "severity": "error", + "category": "unresolved-reference", + "count": 7 + }, + { + "line": 441, + "severity": "error", + "category": "unresolved-reference", + "count": 3 + }, + { + "line": 442, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 446, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 448, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 448, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 449, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 449, + "severity": "error", + "category": "unresolved-reference", + "count": 7 + }, + { + "line": 450, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 454, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 456, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 456, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 457, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 457, + "severity": "error", + "category": "unresolved-reference", + "count": 7 + }, + { + "line": 458, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 462, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 464, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 464, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 465, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 465, + "severity": "error", + "category": "unresolved-reference", + "count": 7 + }, + { + "line": 466, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 470, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 472, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 472, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 473, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 473, + "severity": "error", + "category": "unresolved-reference", + "count": 3 + }, + { + "line": 474, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 478, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 479, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 481, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 481, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 482, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 485, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 485, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 486, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 487, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 487, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 488, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 489, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 491, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 496, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 496, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 497, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 498, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 498, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 499, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 501, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 506, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 506, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 507, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 508, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 508, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 509, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 511, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 516, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 516, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 517, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 518, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 518, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 519, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 521, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 526, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 526, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 527, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 528, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 528, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 529, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 531, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 534, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 534, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 535, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 537, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 542, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 542, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 543, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 544, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 544, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 545, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 547, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + } + ] + }, { "path": "disposal-robot-demo/robot.sysml", "agreement": [], @@ -6086,6 +7762,11 @@ } ], "unmapped": [ + { + "side": "pilot", + "message": "Cannot override a binding feature value", + "count": 10 + }, { "side": "pilot", "message": "Duplicate of other owned member name", @@ -6436,14 +8117,14 @@ } ], "totals": { - "files": 378, + "files": 379, "filesFullyAgreeing": 348, "agreement": 38, "severityMismatch": 3, "openSysMLOnly": 37, - "pilotOnly": 1185, + "pilotOnly": 1545, "openSysMLDiagnostics": 78, - "pilotDiagnostics": 1226 + "pilotDiagnostics": 1586 }, "findings": [ { diff --git a/docs/project/pilot-differential.md b/docs/project/pilot-differential.md index 36da542e46..deb4c1e2bc 100644 --- a/docs/project/pilot-differential.md +++ b/docs/project/pilot-differential.md @@ -218,9 +218,9 @@ nor double-counted as two independent disagreements. | `examples/pilot-corpora/sysml-validation` | 56 | 56 | 0 | 0 | 0 | 0 | 0 | 0 | | `examples/pilot-corpora/kerml-examples` | 58 | 55 | 10 | 0 | 0 | 0 | 10 | 0 | | `tests/testdata` | 18 | 10 | 43 | 55 | 34 | 1 | 8 | 20 | -| `examples` | 44 | 30 | 13 | 1319 | 4 | 2 | 7 | 1313 | +| `examples` | 44 | 30 | 13 | 1531 | 4 | 2 | 7 | 1525 | | `tools/referee/diff/testdata` (probes) | 4 | 1 | 6 | 0 | 0 | 0 | 6 | 0 | -| **Total** | **379** | **347** | **79** | **1374** | **38** | **3** | **38** | **1333** | +| **Total** | **379** | **347** | **79** | **1586** | **38** | **3** | **38** | **1545** | **Read the `only ours` total by root, never as one number.** Step 2 removes nine resolver false positives from the reference's **own** corpora: `pilot-examples` 16 → **7** and @@ -791,8 +791,8 @@ cascades through the rest of the file. The movement is entirely one file, | Count | Before the initializer rewrite | Now | |---|---:|---:| -| only pilot | 82 | **1333** | -| pilot diagnostics | 123 | **1374** | +| only pilot | 82 | **1545** | +| pilot diagnostics | 123 | **1586** | | severity-only | 9 | **3** | The rewrite itself took only-pilot to 61 and pilot diagnostics to 101; the `Now` column states @@ -925,7 +925,7 @@ Per category, the only-ours totals are: `pilot-examples` 4 `unmapped`, 2 advisory of the [runtime showcase round](#runtime-showcase-round)); `testdata` 7 `unmapped`, 1 `multiplicity`; `probes` 6 `unmapped`. Only-pilot: `testdata` 12 `kind-mismatch`, 3 `unmapped`, 3 syntax, 2 `unresolved-reference`; -`examples` 10 syntax, 19 `unmapped`, 587 `kind-mismatch`, 697 `unresolved-reference` — of which +`examples` 10 syntax, 29 `unmapped`, 654 `kind-mismatch`, 832 `unresolved-reference` — of which `relay-probe-demo/mission.sysml` carries none: it carried a `kind-mismatch` on its send of a `Telemetry` invocation until the send-argument round above, and the demo now writes the constructor, `send new Telemetry(…) via antenna`, which both implementations accept, so the row @@ -1017,13 +1017,13 @@ page's history. | Count | Now | |---|---:| | overall: fully agreeing / only ours / our diagnostics | **347 / 38 / 79** | -| only pilot | **1333** | -| pilot diagnostics | **1374** | +| only pilot | **1545** | +| pilot diagnostics | **1586** | | severity-only | **3** | | unmapped, our side | **34** | | kerml-examples: only ours | **10** | | pilot-examples: only ours | **7** | -| examples: only pilot | **1313** | +| examples: only pilot | **1525** | The KerML root is now the *cleanest* of the three OMG roots in proportion: **10** only-ours against 6 only-pilot, with 49 of 58 files fully @@ -1171,15 +1171,16 @@ Nothing else moves: the file draws no diagnostic from this implementation, so `f ### Analysis results recording round `examples/analysis-results-demo/lander-results.sysml` is one file added to the `examples` root: -files 43 → **44** on the root, 378 → **379** overall, and pilot diagnostics 1226 → **1374** / -only pilot 1185 → **1333** — 148 diagnostics in 97 reported rows, 62 `unresolved-reference` -(counted 103) and 35 `kind-mismatch` (counted 45), all inside the file's `Results` and -`Reporting` packages. Every row is a construct the pinned artifact has no support for: the -document-query calls (`Project`, `OrderBy`, `WhereType`, `WhereFeature`, `WhereMetadata`, -`Verdicts`, `Descendants`), the `@RecordedRun` metadata annotations, and the run-record part -usages' `'objective'` quoted name and `ref part :>>` redefinitions. The file draws no diagnostic -from this implementation — it validates clean — so `fully agreeing`, `only ours`, `agreed` and -`severity-only` all stay where the expressions walkthrough left them. +files 43 → **44** on the root, 378 → **379** overall, and pilot diagnostics 1226 → **1586** / +only pilot 1185 → **1545** — 360 diagnostics in 278 reported rows: 181 `unresolved-reference` +(counted 243), 87 `kind-mismatch` (counted 107) and 10 `unmapped`, all inside the file's +`Results` and `Reporting` packages. Every row is a construct the pinned artifact has no support +for: the document-query calls (`Project`, `OrderBy`, `WhereType`, `WhereFeature`, +`WhereMetadata`, `Verdicts`, `Descendants`), the `@AnalysisRecords::RecordedRun` metadata +annotations and specializations of the `AnalysisRecords` library defs, and the run-record part +usages' quoted `'objective'`/`'subject'` names and `ref part :>>`/`part :>>` redefinitions. The +file draws no diagnostic from this implementation — it validates clean — so `fully agreeing`, +`only ours`, `agreed` and `severity-only` all stay where the expressions walkthrough left them. ## Adjudications From 8531a54e361180eda2b8ebbc707832e4aa10afa8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 06:33:58 +0000 Subject: [PATCH 08/10] docs(examples): root the analysis-results report at the Records package so recorded runs join it Co-Authored-By: jason.han --- cmd/sysml/examples_analysis_results_test.go | 31 +- examples/analysis-results-demo/README.md | 70 ++- .../lander-results.sysml | 513 +++++++++--------- examples/analysis-results-demo/report.md | 14 + 4 files changed, 350 insertions(+), 278 deletions(-) diff --git a/cmd/sysml/examples_analysis_results_test.go b/cmd/sysml/examples_analysis_results_test.go index e2dba1bbf2..78e2c0e653 100644 --- a/cmd/sysml/examples_analysis_results_test.go +++ b/cmd/sysml/examples_analysis_results_test.go @@ -34,14 +34,14 @@ func TestAnalysisResultsExample(t *testing.T) { t.Errorf("rendered example differs from examples/analysis-results-demo/report.md:\n%s", written) } - stale := exec.Command(binary, source, "-run-query", "Reporting::StaleRuns root=results") + stale := exec.Command(binary, source, "-run-query", "Reporting::StaleRuns") output, err := stale.CombinedOutput() if err != nil { t.Fatalf("stale query: %v\n%s", err, output) } for _, want := range []string{ "returned 1 row", - "Row 1: Results::results::relayRun", + "Row 1: Records::relayRun", "drift = 30.0", "liveFuelLeft = 110.0", } { @@ -61,4 +61,31 @@ func TestAnalysisResultsExample(t *testing.T) { if !strings.Contains(string(output), "fuelLeft = 130.0") { t.Errorf("scoutBudget no longer prints the recorded fuelLeft = 130.0:\n%s", output) } + + // A run written back with -record-run lands in the same Records package + // the report's queries walk, so the document must show it alongside the + // hand-written records — and only it, in the run-table filters that name + // the hand-written defs. + t.Run("RecordedRunJoinsTheReport", func(t *testing.T) { + recorded := filepath.Join(t.TempDir(), "recorded-report.md") + render = exec.Command(binary, source, "-record-run", "Descent::scoutBudget", + "-render-document", "Reporting::AnalysisReport", "-o", recorded) + if output, err := render.CombinedOutput(); err != nil { + t.Fatalf("render with -record-run: %v\n%s", err, output) + } + written, err := os.ReadFile(recorded) + if err != nil { + t.Fatal(err) + } + text := string(written) + if !strings.Contains(text, "| scoutBudget\\_run1 | Descent::scoutBudget | run | Landers::scout | satisfied |") { + t.Errorf("recorded run missing from the every-recorded-run table:\n%s", text) + } + if !strings.Contains(text, "| scoutBudget\\_run1 | run | Descent::scoutBudget |") { + t.Errorf("recorded run missing from the provenance table:\n%s", text) + } + if strings.Count(text, "| relayRun | Landers::relay | 40 | 80 | 110 | 30 |") != 1 { + t.Errorf("stale table does not contain exactly the relay record:\n%s", text) + } + }) } diff --git a/examples/analysis-results-demo/README.md b/examples/analysis-results-demo/README.md index f35c0b7427..9c29b43b2f 100644 --- a/examples/analysis-results-demo/README.md +++ b/examples/analysis-results-demo/README.md @@ -40,8 +40,12 @@ part def FuelBudgetRun :> DemoRun { } ``` -Each run is then one usage in the `Results` package, filled in from the -printed output: +Each run is then one usage at the top level of the `Records` package — the +same package `-record-run` writes its records into — filled in from the +printed output. The definitions themselves sit in `Records::Vocab`, one +nesting level down: the report's queries walk the package's direct members, +and a `part def` matching `WhereType` would surface its own unbound features +as a bogus row. ```sysml part scoutRun : FuelBudgetRun { @@ -181,13 +185,13 @@ can never give you: a record that notices the model moved. ```bash ./bin/sysml examples/analysis-results-demo/lander-results.sysml \ - -run-query "Reporting::StaleRuns root=results" + -run-query "Reporting::StaleRuns" ``` ``` ✓ Query Reporting::StaleRuns returned 1 row Columns: name, subjectName, burnTime, fuelLeft, liveFuelLeft, drift - Row 1: Results::results::relayRun + Row 1: Records::relayRun fuelLeft = 80.0 liveFuelLeft = 110.0 drift = 30.0 @@ -215,12 +219,15 @@ recomputes an output can detect it, and only for the values it rederives. ``` [`report.md`](report.md) is committed so the test suite can compare the -render byte-for-byte. It shows a grouped table of every fuel-budget record by -subject, the sweep rows alone, the stale-records table (exactly `relayRun`), -a provenance table over `WhereMetadata(... 'metadata' = "AnalysisRecords::RecordedRun")`, -the trade-study record, and — the contrast — a `Verdicts` table of the -assertions about `scout` **evaluated live at render time**: the records say -what a run printed; the verdicts say what holds now. +render byte-for-byte. It opens with a table of **every recorded run** — +`WhereMetadata(... 'metadata' = "AnalysisRecords::RecordedRun")` filtered to +`WhereType(... type = "AnalysisRecords::AnalysisRun")` over the `Records` +package's direct members — then a grouped table of every fuel-budget record +by subject, the sweep rows alone, the stale-records table (exactly +`relayRun`), a provenance table over the same `WhereMetadata` filter, the +trade-study record and its `EvaluationRecord`s, and — the contrast — a +`Verdicts` table of the assertions about `scout` **evaluated live at render +time**: the records say what a run printed; the verdicts say what holds now. HTML and PDF render the same document tree: @@ -280,13 +287,42 @@ part scoutBudget_run1 : ScoutBudgetRun { } ``` -Because both speak `AnalysisRecords`, a document written against -`WhereMetadata(... 'metadata' = "AnalysisRecords::RecordedRun")` and -`WhereType(... "FuelBudgetRun")`-style filters reads hand-written and -`-record-run` records alike. The REPL form is `%record`; sweeps record one -record per row (`recorded 3 runs as Records::scoutBudget_run1 …`), though -`-convert` refuses a sweep — write the model out after a single -`-record-run`, or record each row explicitly. The full flag reference is +`-record-run` targets the `Records` package because it is the plain +top-level package beside `Descent` — the same package this demo's +hand-written records live in, so the generated record lands where the +report's queries already walk. Render the document in the same invocation +and the new record joins it: + +```bash +./bin/sysml examples/analysis-results-demo/lander-results.sysml \ + -record-run "Descent::scoutBudget" \ + -render-document Reporting::AnalysisReport -o recorded-report.md +``` + +The *Every recorded run* table — annotated `AnalysisRun`s, whatever their +definition — picks it up as an eighth row: + +``` +| name | caseName | kind | subjectName | objective | +| --- | --- | --- | --- | --- | +| scoutRun | Descent::scoutBudget | run | Landers::scout | satisfied | +| haulerRun | Descent::haulerBudget | run | Landers::hauler | satisfied | +| relayRun | Descent::relayBudget | run | Landers::relay | satisfied | +| scoutSweep40 | Descent::scoutBudget | sweep | Landers::scout | satisfied | +| scoutSweep60 | Descent::scoutBudget | sweep | Landers::scout | satisfied | +| scoutSweep80 | Descent::scoutBudget | sweep | Landers::scout | not satisfied | +| lightestRun | Selection::lightest | trade | | satisfied | +| scoutBudget\_run1 | Descent::scoutBudget | run | Landers::scout | satisfied | +``` + +The fuel-budget, sweep and stale tables do *not* pick it up — they filter +`WhereType(... type = "FuelBudgetRun")`, and the generated def specializes +`AnalysisRun`, not `FuelBudgetRun` — and the provenance table lists it with +an empty `command` cell, since only the hand-written `DemoRun` declares that +projectable attribute. The REPL form is `%record`; sweeps record one record +per row (`recorded 3 runs as Records::scoutBudget_run1 …`), though `-convert` +refuses a sweep — write the model out after a single `-record-run`, or +record each row explicitly. The full flag reference is [the manual](../../docs/manual/recording-analysis-runs.md). These records stay hand-written for two reasons: the stale-relay story needs diff --git a/examples/analysis-results-demo/lander-results.sysml b/examples/analysis-results-demo/lander-results.sysml index 7d874f2f75..74c4c01054 100644 --- a/examples/analysis-results-demo/lander-results.sysml +++ b/examples/analysis-results-demo/lander-results.sysml @@ -163,243 +163,237 @@ package Records { private import Landers::*; private import AnalysisRecords::*; - // Shared definition for the demo's hand-written records: the library's - // AnalysisRun plus `command` as a plain attribute — a document query's - // `Project` cannot read an annotation's attribute values, so the provenance - // table needs it declared. - part def DemoRun :> AnalysisRecords::AnalysisRun { - attribute command : String; - } - - // A FuelBudget run written back: subject, input and outputs as recorded, - // plus the output recomputed from the model as it stands now. `drift` - // measures how far the model has moved since the record was written, and - // `stale` names a record that no longer matches. `lander` is the typed - // subject the formula reads; the library's untyped `'subject'` ref is - // bound to the same part on each record. - part def FuelBudgetRun :> DemoRun { - ref part lander : Lander; - attribute burnTime : Real; - attribute fuelUsed : Real; - attribute wetMass : Real; - attribute fuelLeft : Real; - attribute liveFuelLeft : Real = lander.fuel - burnTime * lander.burnRate; - attribute drift : Real = liveFuelLeft - fuelLeft; - attribute stale : Boolean = drift != 0.0; - } - - // A TradeStudy run written back: the selection, plus the run's checks and - // evaluations carried under the library's `verdicts` and `evaluations`. - part def TradeStudyRun :> DemoRun { - attribute selected : String; - } -} + // The record definitions live one nesting level down: queries walk the + // package's direct members, and a `part def` matching `WhereType` would + // surface its own unbound features. + private package Vocab { + // Shared definition for the demo's hand-written records: the library's + // AnalysisRun plus `command` as a plain attribute — a document query's + // `Project` cannot read an annotation's attribute values, so the + // provenance table needs it declared. + part def DemoRun :> AnalysisRecords::AnalysisRun { + attribute command : String; + } -package Results { - private import ScalarValues::*; - private import Landers::*; - private import AnalysisRecords::*; - private import Records::*; - - // The saved runs, held as subparts of one part so a query can walk them. - // The saved runs, held as subparts of one part so a query can walk them. - part results { - - // One baseline run per candidate, at FuelBudget's default 40 s burn. - part scoutRun : FuelBudgetRun { - @AnalysisRecords::RecordedRun { - runAt = "2025-11-02T09:14:00Z"; - tool = "sysml"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; - kind = "run"; - } - attribute :>> caseName = "Descent::scoutBudget"; - attribute :>> kind = "run"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; - ref :>> 'subject' = Landers::scout; - ref part :>> lander = scout; - attribute :>> subjectName = "Landers::scout"; - attribute :>> burnTime = 40.0; - attribute :>> fuelUsed = 120.0; - attribute :>> wetMass = 730.0; - attribute :>> fuelLeft = 130.0; - part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { - attribute :>> kind = "objective"; - attribute :>> name = "reserveHeld"; - attribute :>> status = "satisfied"; - } + // A FuelBudget run written back: subject, input and outputs as + // recorded, plus the output recomputed from the model as it stands + // now. `drift` measures how far the model has moved since the record + // was written, and `stale` names a record that no longer matches. + // `lander` is the typed subject the formula reads; the library's + // untyped `'subject'` ref is bound to the same part on each record. + part def FuelBudgetRun :> DemoRun { + ref part lander : Lander; + attribute burnTime : Real; + attribute fuelUsed : Real; + attribute wetMass : Real; + attribute fuelLeft : Real; + attribute liveFuelLeft : Real = lander.fuel - burnTime * lander.burnRate; + attribute drift : Real = liveFuelLeft - fuelLeft; + attribute stale : Boolean = drift != 0.0; } - part haulerRun : FuelBudgetRun { - @AnalysisRecords::RecordedRun { - runAt = "2025-11-02T09:14:30Z"; - tool = "sysml"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; - kind = "run"; - } - attribute :>> caseName = "Descent::haulerBudget"; - attribute :>> kind = "run"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; - ref :>> 'subject' = Landers::hauler; - ref part :>> lander = hauler; - attribute :>> subjectName = "Landers::hauler"; - attribute :>> burnTime = 40.0; - attribute :>> fuelUsed = 320.0; - attribute :>> wetMass = 1980.0; - attribute :>> fuelLeft = 580.0; - part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { - attribute :>> kind = "objective"; - attribute :>> name = "reserveHeld"; - attribute :>> status = "satisfied"; - } + + // A TradeStudy run written back: the selection, plus the run's checks + // and evaluations carried under the library's `verdicts` and + // `evaluations`. + part def TradeStudyRun :> DemoRun { + attribute selected : String; } - part relayRun : FuelBudgetRun { - @AnalysisRecords::RecordedRun { - runAt = "2025-11-02T09:15:00Z"; - tool = "sysml"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; - kind = "run"; - } - attribute :>> caseName = "Descent::relayBudget"; - attribute :>> kind = "run"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; - ref :>> 'subject' = Landers::relay; - ref part :>> lander = relay; - attribute :>> subjectName = "Landers::relay"; - attribute :>> burnTime = 40.0; - attribute :>> fuelUsed = 100.0; - attribute :>> wetMass = 530.0; - attribute :>> fuelLeft = 80.0; - part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { - attribute :>> kind = "objective"; - attribute :>> name = "reserveHeld"; - attribute :>> status = "satisfied"; - } + } + private import Vocab::*; + // One baseline run per candidate, at FuelBudget's default 40 s burn. + part scoutRun : FuelBudgetRun { + @AnalysisRecords::RecordedRun { + runAt = "2025-11-02T09:14:00Z"; + tool = "sysml"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; + kind = "run"; } + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget"; + ref :>> 'subject' = Landers::scout; + ref part :>> lander = scout; + attribute :>> subjectName = "Landers::scout"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 120.0; + attribute :>> wetMass = 730.0; + attribute :>> fuelLeft = 130.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; + } + } + part haulerRun : FuelBudgetRun { + @AnalysisRecords::RecordedRun { + runAt = "2025-11-02T09:14:30Z"; + tool = "sysml"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; + kind = "run"; + } + attribute :>> caseName = "Descent::haulerBudget"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::haulerBudget"; + ref :>> 'subject' = Landers::hauler; + ref part :>> lander = hauler; + attribute :>> subjectName = "Landers::hauler"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 320.0; + attribute :>> wetMass = 1980.0; + attribute :>> fuelLeft = 580.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; + } + } + part relayRun : FuelBudgetRun { + @AnalysisRecords::RecordedRun { + runAt = "2025-11-02T09:15:00Z"; + tool = "sysml"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; + kind = "run"; + } + attribute :>> caseName = "Descent::relayBudget"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::relayBudget"; + ref :>> 'subject' = Landers::relay; + ref part :>> lander = relay; + attribute :>> subjectName = "Landers::relay"; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 100.0; + attribute :>> wetMass = 530.0; + attribute :>> fuelLeft = 80.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; + } + } - // A sweep of scoutBudget over burnTime, one record per row the sweep printed. - part scoutSweep40 : FuelBudgetRun { - @AnalysisRecords::RecordedRun { - runAt = "2025-11-02T09:20:10Z"; - tool = "sysml"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - kind = "sweep"; - } - attribute :>> caseName = "Descent::scoutBudget"; - attribute :>> kind = "sweep"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - ref :>> 'subject' = Landers::scout; - ref part :>> lander = scout; - attribute :>> subjectName = "Landers::scout"; - attribute :>> iteration = 1; - attribute :>> burnTime = 40.0; - attribute :>> fuelUsed = 120.0; - attribute :>> wetMass = 730.0; - attribute :>> fuelLeft = 130.0; - part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { - attribute :>> kind = "objective"; - attribute :>> name = "reserveHeld"; - attribute :>> status = "satisfied"; - } + // A sweep of scoutBudget over burnTime, one record per row the sweep printed. + part scoutSweep40 : FuelBudgetRun { + @AnalysisRecords::RecordedRun { + runAt = "2025-11-02T09:20:10Z"; + tool = "sysml"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + kind = "sweep"; } - part scoutSweep60 : FuelBudgetRun { - @AnalysisRecords::RecordedRun { - runAt = "2025-11-02T09:20:10Z"; - tool = "sysml"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - kind = "sweep"; - } - attribute :>> caseName = "Descent::scoutBudget"; - attribute :>> kind = "sweep"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - ref :>> 'subject' = Landers::scout; - ref part :>> lander = scout; - attribute :>> subjectName = "Landers::scout"; - attribute :>> iteration = 2; - attribute :>> burnTime = 60.0; - attribute :>> fuelUsed = 180.0; - attribute :>> wetMass = 670.0; - attribute :>> fuelLeft = 70.0; - part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { - attribute :>> kind = "objective"; - attribute :>> name = "reserveHeld"; - attribute :>> status = "satisfied"; - } + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + ref :>> 'subject' = Landers::scout; + ref part :>> lander = scout; + attribute :>> subjectName = "Landers::scout"; + attribute :>> iteration = 1; + attribute :>> burnTime = 40.0; + attribute :>> fuelUsed = 120.0; + attribute :>> wetMass = 730.0; + attribute :>> fuelLeft = 130.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; } - part scoutSweep80 : FuelBudgetRun { - @AnalysisRecords::RecordedRun { - runAt = "2025-11-02T09:20:10Z"; - tool = "sysml"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - kind = "sweep"; - } - attribute :>> caseName = "Descent::scoutBudget"; - attribute :>> kind = "sweep"; - attribute :>> 'objective' = "not satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; - ref :>> 'subject' = Landers::scout; - ref part :>> lander = scout; - attribute :>> subjectName = "Landers::scout"; - attribute :>> iteration = 3; - attribute :>> burnTime = 80.0; - attribute :>> fuelUsed = 240.0; - attribute :>> wetMass = 610.0; - attribute :>> fuelLeft = 10.0; - part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { - attribute :>> kind = "objective"; - attribute :>> name = "reserveHeld"; - attribute :>> status = "not satisfied"; - } + } + part scoutSweep60 : FuelBudgetRun { + @AnalysisRecords::RecordedRun { + runAt = "2025-11-02T09:20:10Z"; + tool = "sysml"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + kind = "sweep"; + } + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + ref :>> 'subject' = Landers::scout; + ref part :>> lander = scout; + attribute :>> subjectName = "Landers::scout"; + attribute :>> iteration = 2; + attribute :>> burnTime = 60.0; + attribute :>> fuelUsed = 180.0; + attribute :>> wetMass = 670.0; + attribute :>> fuelLeft = 70.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "satisfied"; + } + } + part scoutSweep80 : FuelBudgetRun { + @AnalysisRecords::RecordedRun { + runAt = "2025-11-02T09:20:10Z"; + tool = "sysml"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + kind = "sweep"; + } + attribute :>> caseName = "Descent::scoutBudget"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "not satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Descent::scoutBudget -sweep burnTime=40.0..80.0:20.0"; + ref :>> 'subject' = Landers::scout; + ref part :>> lander = scout; + attribute :>> subjectName = "Landers::scout"; + attribute :>> iteration = 3; + attribute :>> burnTime = 80.0; + attribute :>> fuelUsed = 240.0; + attribute :>> wetMass = 610.0; + attribute :>> fuelLeft = 10.0; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "reserveHeld"; + attribute :>> status = "not satisfied"; } + } - // The trade study run: the selection, its objective verdict, and each - // alternative's evaluation — recorded before relay.fuel changed, so the - // relay score here is the old 630.0. - part lightestRun : TradeStudyRun { - @AnalysisRecords::RecordedRun { - runAt = "2025-11-02T09:25:00Z"; - tool = "sysml"; - command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; - kind = "trade"; - } - attribute :>> caseName = "Selection::lightest"; - attribute :>> kind = "trade"; - attribute :>> 'objective' = "satisfied"; - attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; - attribute :>> selected = "Landers::relay"; - part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { - attribute :>> kind = "objective"; - attribute :>> name = "tradeStudyObjective"; - attribute :>> status = "satisfied"; - } - part evaluation1 : AnalysisRecords::EvaluationRecord :> evaluations { - attribute :>> 'function' = "Selection::lightest::evaluationFunction"; - attribute :>> alternative = "Landers::scout (object #1)"; - attribute :>> score = 850.0; - attribute :>> result = "850.0"; - attribute :>> selected = false; - attribute :>> tied = false; - } - part evaluation2 : AnalysisRecords::EvaluationRecord :> evaluations { - attribute :>> 'function' = "Selection::lightest::evaluationFunction"; - attribute :>> alternative = "Landers::hauler (object #2)"; - attribute :>> score = 2300.0; - attribute :>> result = "2300.0"; - attribute :>> selected = false; - attribute :>> tied = false; - } - part evaluation3 : AnalysisRecords::EvaluationRecord :> evaluations { - attribute :>> 'function' = "Selection::lightest::evaluationFunction"; - attribute :>> alternative = "Landers::relay (object #3)"; - attribute :>> score = 630.0; - attribute :>> result = "630.0"; - attribute :>> selected = true; - attribute :>> tied = false; - } + // The trade study run: the selection, its objective verdict, and each + // alternative's evaluation — recorded before relay.fuel changed, so the + // relay score here is the old 630.0. + part lightestRun : TradeStudyRun { + @AnalysisRecords::RecordedRun { + runAt = "2025-11-02T09:25:00Z"; + tool = "sysml"; + command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; + kind = "trade"; + } + attribute :>> caseName = "Selection::lightest"; + attribute :>> kind = "trade"; + attribute :>> 'objective' = "satisfied"; + attribute :>> command = "./bin/sysml examples/analysis-results-demo/lander-results.sysml -analysis Selection::lightest"; + attribute :>> selected = "Landers::relay"; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "tradeStudyObjective"; + attribute :>> status = "satisfied"; + } + part evaluation1 : AnalysisRecords::EvaluationRecord :> evaluations { + attribute :>> 'function' = "Selection::lightest::evaluationFunction"; + attribute :>> alternative = "Landers::scout (object #1)"; + attribute :>> score = 850.0; + attribute :>> result = "850.0"; + attribute :>> selected = false; + attribute :>> tied = false; + } + part evaluation2 : AnalysisRecords::EvaluationRecord :> evaluations { + attribute :>> 'function' = "Selection::lightest::evaluationFunction"; + attribute :>> alternative = "Landers::hauler (object #2)"; + attribute :>> score = 2300.0; + attribute :>> result = "2300.0"; + attribute :>> selected = false; + attribute :>> tied = false; + } + part evaluation3 : AnalysisRecords::EvaluationRecord :> evaluations { + attribute :>> 'function' = "Selection::lightest::evaluationFunction"; + attribute :>> alternative = "Landers::relay (object #3)"; + attribute :>> score = 630.0; + attribute :>> result = "630.0"; + attribute :>> selected = true; + attribute :>> tied = false; } } } @@ -408,25 +402,34 @@ package Reporting { private import ScalarValues::*; private import DocumentQueries::*; private import KerML::Root::Element; - private import Results::*; + + // The records package's members walked by name, so a run `-record-run` + // writes into the same `Records` package joins the query results. + calc def AllRuns :> Query { + Project( + source = WhereType( + source = WhereMetadata( + source = Descendants(source = Named(qualifiedName = "Records"), maxDepth = 1), + 'metadata' = "AnalysisRecords::RecordedRun"), + type = "AnalysisRecords::AnalysisRun"), + properties = ("name", "caseName", "kind", "subjectName", "objective")) + } // Every saved fuel-budget run, sorted by burn time; the document groups it by subject. calc def FuelBudgetRuns :> Query { - in root : Element; Project( source = OrderBy( - source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), + source = WhereType(source = Descendants(source = Named(qualifiedName = "Records"), maxDepth = 1), type = "FuelBudgetRun"), property = "burnTime", direction = "ascending", missing = "last", multiple = "error"), properties = ("name", "subjectName", "kind", "burnTime", "fuelUsed", "wetMass", "fuelLeft", "objective")) } // The sweep rows alone. calc def SweepRuns :> Query { - in root : Element; Project( source = OrderBy( source = WhereFeature( - source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), + source = WhereType(source = Descendants(source = Named(qualifiedName = "Records"), maxDepth = 1), type = "FuelBudgetRun"), 'feature' = "kind", operator = "=", value = "sweep"), property = "burnTime", direction = "ascending", missing = "last", multiple = "error"), properties = ("name", "caseName", "burnTime", "fuelUsed", "fuelLeft", "objective")) @@ -434,35 +437,31 @@ package Reporting { // Records whose saved numbers no longer match the model as it stands. calc def StaleRuns :> Query { - in root : Element; Project( source = WhereFeature( - source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "FuelBudgetRun"), + source = WhereType(source = Descendants(source = Named(qualifiedName = "Records"), maxDepth = 1), type = "FuelBudgetRun"), 'feature' = "stale", operator = "=", value = "true"), properties = ("name", "subjectName", "burnTime", "fuelLeft", "liveFuelLeft", "drift")) } // The recorded runs with their provenance annotation's command line. calc def Provenance :> Query { - in root : Element; Project( - source = WhereMetadata(source = Descendants(source = root, maxDepth = 1), 'metadata' = "AnalysisRecords::RecordedRun"), + source = WhereMetadata(source = Descendants(source = Named(qualifiedName = "Records"), maxDepth = 1), 'metadata' = "AnalysisRecords::RecordedRun"), properties = ("name", "kind", "caseName", "command")) } // The trade-study record. calc def TradeRuns :> Query { - in root : Element; Project( - source = WhereType(source = Descendants(source = root, maxDepth = 1), type = "TradeStudyRun"), + source = WhereType(source = Descendants(source = Named(qualifiedName = "Records"), maxDepth = 1), type = "TradeStudyRun"), properties = ("name", "caseName", "selected", "objective")) } // The evaluations nested under the trade-study record. calc def TradeEvaluations :> Query { - in root : Element; Project( - source = WhereType(source = Descendants(source = root, maxDepth = 2), type = "EvaluationRecord"), + source = WhereType(source = Descendants(source = Named(qualifiedName = "Records"), maxDepth = 2), type = "EvaluationRecord"), properties = ("alternative", "score", "selected", "tied")) } @@ -482,14 +481,20 @@ package Reporting { attribute redefines text = "An analysis run's printed results are discarded unless they are recorded — by -record-run or by hand, in the vocabulary the AnalysisRecords library defines. Every row below is a declared record typed by a run definition on that vocabulary, annotated with the command that produced it — except the last table, whose verdicts are recomputed live."; } + part every : Section { + attribute redefines title = "Every recorded run"; + part runs : Table { + attribute redefines caption = "Every @AnalysisRecords::RecordedRun-annotated AnalysisRun in the Records package"; + calc rows : AllRuns; + } + } + part budgets : Section { attribute redefines title = "Recorded fuel budgets"; part runs : Table { attribute redefines caption = "One record per run, grouped by subject"; attribute redefines groupBy = "subjectName"; - calc rows : FuelBudgetRuns { - in root = results; - } + calc rows : FuelBudgetRuns; } } @@ -497,9 +502,7 @@ package Reporting { attribute redefines title = "Sweep of scoutBudget"; part runs : Table { attribute redefines caption = "One record per sweep row"; - calc rows : SweepRuns { - in root = results; - } + calc rows : SweepRuns; } } @@ -507,9 +510,7 @@ package Reporting { attribute redefines title = "Runs that no longer match the model"; part runs : Table { attribute redefines caption = "Records whose saved fuelLeft differs from the value the model now derives"; - calc rows : StaleRuns { - in root = results; - } + calc rows : StaleRuns; } } @@ -517,9 +518,7 @@ package Reporting { attribute redefines title = "Provenance"; part runs : Table { attribute redefines caption = "Every element annotated @AnalysisRecords::RecordedRun"; - calc rows : Provenance { - in root = results; - } + calc rows : Provenance; } } @@ -527,15 +526,11 @@ package Reporting { attribute redefines title = "Trade study"; part runs : Table { attribute redefines caption = "The recorded selection and verdict"; - calc rows : TradeRuns { - in root = results; - } + calc rows : TradeRuns; } part evals : Table { attribute redefines caption = "Each alternative's evaluation, as recorded"; - calc rows : TradeEvaluations { - in root = results; - } + calc rows : TradeEvaluations; } } diff --git a/examples/analysis-results-demo/report.md b/examples/analysis-results-demo/report.md index 5eaa424351..f8b8209f46 100644 --- a/examples/analysis-results-demo/report.md +++ b/examples/analysis-results-demo/report.md @@ -2,6 +2,20 @@ An analysis run's printed results are discarded unless they are recorded — by -record-run or by hand, in the vocabulary the AnalysisRecords library defines. Every row below is a declared record typed by a run definition on that vocabulary, annotated with the command that produced it — except the last table, whose verdicts are recomputed live. +## Every recorded run + +*Every @AnalysisRecords::RecordedRun-annotated AnalysisRun in the Records package* + +| name | caseName | kind | subjectName | objective | +| --- | --- | --- | --- | --- | +| scoutRun | Descent::scoutBudget | run | Landers::scout | satisfied | +| haulerRun | Descent::haulerBudget | run | Landers::hauler | satisfied | +| relayRun | Descent::relayBudget | run | Landers::relay | satisfied | +| scoutSweep40 | Descent::scoutBudget | sweep | Landers::scout | satisfied | +| scoutSweep60 | Descent::scoutBudget | sweep | Landers::scout | satisfied | +| scoutSweep80 | Descent::scoutBudget | sweep | Landers::scout | not satisfied | +| lightestRun | Selection::lightest | trade | | satisfied | + ## Recorded fuel budgets *One record per run, grouped by subject* From 4b30041113bc7d9da96b51bb194d7c9b69121b11 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 06:33:58 +0000 Subject: [PATCH 09/10] docs(project): re-record the differential baseline for the Records-rooted demo Co-Authored-By: jason.han --- .../testing-pilot-corpora-gate/SKILL.md | 2 +- .../testing-pilot-differential/SKILL.md | 6 +- .../testing-pilot-execution-referee/SKILL.md | 2 +- .agents/skills/testing-pilot-xpect/SKILL.md | 2 +- README.md | 4 +- docs/internals/architecture.md | 4 +- docs/project/pilot-differential-baseline.json | 682 ++++++++++-------- docs/project/pilot-differential.md | 24 +- 8 files changed, 396 insertions(+), 330 deletions(-) diff --git a/.agents/skills/testing-pilot-corpora-gate/SKILL.md b/.agents/skills/testing-pilot-corpora-gate/SKILL.md index 6ee7843de6..eeeeacca87 100644 --- a/.agents/skills/testing-pilot-corpora-gate/SKILL.md +++ b/.agents/skills/testing-pilot-corpora-gate/SKILL.md @@ -184,7 +184,7 @@ gate's own helpers are package-private but reusable (`pilotCorporaGate.files(t)` `actionlint`, `shellcheck`, `python3 scripts/check-doc-links.py`, `gofmt`, `go vet`, `go run -C tools ./cmd/pilot-diff` (validators pre-downloaded; ~4min, prints e.g. the headline the committed baseline holds — `379 file(s), 347 fully agreeing; 38 agreed -diagnostic(s), 38 only ours, 1545 only the pilot's` at the `2026-08` pin, so read it from +diagnostic(s), 38 only ours, 1582 only the pilot's` at the `2026-08` pin, so read it from `docs/project/pilot-differential-baseline.json` rather than from this line) and `make lint` (staticcheck+gosec, ~2min) all work. There is **no** `yamllint` and **no** `circleci` CLI, so `.circleci/config.yml` can only be parsed as YAML, not schema-validated — say so diff --git a/.agents/skills/testing-pilot-differential/SKILL.md b/.agents/skills/testing-pilot-differential/SKILL.md index e0414ca16e..bea4275b73 100644 --- a/.agents/skills/testing-pilot-differential/SKILL.md +++ b/.agents/skills/testing-pilot-differential/SKILL.md @@ -22,8 +22,8 @@ GNU-format diagnostics **relative to `--root`**. Consequences for testing: (written by the new script), not from the DeciSym `pom.xml`. - `-validator /nonexistent` now says `run ./scripts/download-pilot-sysml-validator.sh`. - Measured at the `2026-08` pin, with a fresh library cache: `379 file(s), 347 fully agreeing; 38 agreed, - 38 only ours, 1545 only the pilot's`, JSON totals `openSysMLDiagnostics 79 / pilotDiagnostics - 1586 / severityMismatch 3`; ~2 min wall, byte-identical across runs *and* after a from-scratch + 38 only ours, 1582 only the pilot's`, JSON totals `openSysMLDiagnostics 79 / pilotDiagnostics + 1623 / severityMismatch 3`; ~2 min wall, byte-identical across runs *and* after a from-scratch rebuild of `build/pilot-validator`. The six `kerml-examples` pilot-only rows the `2026-07` run carried (`The opposite features 'owningType' … do not refer to each other`) are gone: the pilot fixed its `ownedDisjoining` delegate, and nothing on our side moved. `kerml-examples` carries no `syntax` diagnostic on either @@ -137,7 +137,7 @@ The harness compares OpenSysML diagnostics against the OMG SysML v2 Pilot Implem `build/pilot-diff/pilot-diff.{txt,json}`. `docs/project/pilot-differential-baseline.json` is the committed result of the *last refreshed* run, so **the harness is testable by reproduction** — but only while the baseline is current. Check that first. As of the rebaseline that came when the Legend of the Red Dragon example left for its own repository it **is** -current: a live run gives `379 file(s), 347 fully agreeing; 38 agreed, 38 only ours, 1545 only the +current: a live run gives `379 file(s), 347 fully agreeing; 38 agreed, 38 only ours, 1582 only the pilot's`, byte-identical to the committed baseline, and `docs/project/pilot-differential.md`'s "Results" table matches. The rebaseline before it, at the architecture self-model's landing, covered two rounds, because the succession-shorthand removal before it landed without refreshing the baseline; a control run of its merge commit gives diff --git a/.agents/skills/testing-pilot-execution-referee/SKILL.md b/.agents/skills/testing-pilot-execution-referee/SKILL.md index 3d4730b86c..01e3443194 100644 --- a/.agents/skills/testing-pilot-execution-referee/SKILL.md +++ b/.agents/skills/testing-pilot-execution-referee/SKILL.md @@ -149,7 +149,7 @@ pilot answers the representation's own. See such file or directory`. - **Additivity.** `go run -C tools ./cmd/pilot-diff` must still print the headline the committed baseline holds (`379 file(s), 347 fully agreeing; 38 agreed - diagnostic(s), 38 only ours, 1545 only the pilot's` at the `2026-08` pin — read it from the baseline JSON, not from this line, since each + diagnostic(s), 38 only ours, 1582 only the pilot's` at the `2026-08` pin — read it from the baseline JSON, not from this line, since each fix round moves it) and `jq -S` diff clean against `docs/project/pilot-differential-baseline.json`; `git status --porcelain` empty at the end. diff --git a/.agents/skills/testing-pilot-xpect/SKILL.md b/.agents/skills/testing-pilot-xpect/SKILL.md index 30f0b71e98..e36f10d4e1 100644 --- a/.agents/skills/testing-pilot-xpect/SKILL.md +++ b/.agents/skills/testing-pilot-xpect/SKILL.md @@ -417,7 +417,7 @@ census in `w5c_census_test.go` is live two ways: perturb one pinned triple (e.g. `go run -C tools ./cmd/pilot-diff` (~1m12s) must still print the headline the *committed* baseline holds — at the `2026-08` pin that is `379 file(s), 347 fully agreeing; 38 agreed diagnostic(s), 38 -only ours, 1545 only the pilot's`. Read the number out of +only ours, 1582 only the pilot's`. Read the number out of `docs/project/pilot-differential-baseline.json` rather than trusting this line, since a landing fix round moves it. When the baseline is itself stale (it was at `19a3ce03`, holding 273 / 281 / 317), a failing `cmp` against it is *not* evidence of an Xpect regression — compare the summary line, and see diff --git a/README.md b/README.md index 47980f55df..1209d4480b 100644 --- a/README.md +++ b/README.md @@ -313,11 +313,11 @@ The project is under active development, with the core infrastructure operationa **Measured against the pinned reference** (`PILOT_TAG=2026-08`, artifact `0.62.0`). Every number below is generated by `make docs-counts` from the committed baselines and gated; none of them is typed in by hand. -- **Corpus agreement:** 347 of 379 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1545 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](docs/project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). +- **Corpus agreement:** 347 of 379 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1582 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](docs/project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). - **Declared-diagnostic silence:** of the 512 declared `errors` rows in the reference's own Xpect suites, we report nothing for 0. 245 we report word-for-word; 248 wording-only and 7 location-only differences are agreement in substance and are not counted as gaps; 0 more we report as a warning and 2 elsewhere in the file ([Xpect oracle](docs/project/pilot-xpect.md), `go run -C tools ./cmd/pilot-xpect`). - **Scope agreement:** 230 of 230 declared scope assertions match exactly (same source). - **Permissiveness gaps:** of 306 invalid models we wrote ourselves, the reference rejects 4 that we accept by default, and 293 both reject; 4 further cases agree only when we are asked strictly. We authored every one of these cases ourselves, so the denominator measures the reach of our own corpus and not our conformance; agreement reached only under an opt-in strict mode is weaker evidence than agreement by default ([rejection oracle](docs/project/pilot-rejection.md), `go run -C tools ./cmd/pilot-reject`). -- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](docs/project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 379 files agreeing, 37 diagnostics ours alone and 1545 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. +- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](docs/project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 379 files agreeing, 37 diagnostics ours alone and 1582 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. - **Self-assessed surface:** the action, state-machine and classifier-behavior rows have no external referee at all — the four refereed figures above cannot see them, because the pinned artifact evaluates expressions but executes neither actions nor state machines. [Spec compliance](docs/project/spec-compliance.md) counts them. What these numbers cannot show: the OMG corpora are demonstrations rather than an official conformance suite; the differential is one-directional, comparing the diagnostics the two implementations report on the same files; the Xpect suites are the pilot authors' test intent rather than a certification oracle; and none of these is a percentage of the specification — no global compliance figure is claimed anywhere. diff --git a/docs/internals/architecture.md b/docs/internals/architecture.md index 062e216c08..aafce55fe4 100644 --- a/docs/internals/architecture.md +++ b/docs/internals/architecture.md @@ -840,11 +840,11 @@ Every behavioral feature must have: **Measured against the pinned reference** (`PILOT_TAG=2026-08`, artifact `0.62.0`). Every number below is generated by `make docs-counts` from the committed baselines and gated; none of them is typed in by hand. -- **Corpus agreement:** 347 of 379 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1545 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](../project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). +- **Corpus agreement:** 347 of 379 files agree diagnostic-by-diagnostic; 38 diagnostics are ours alone and 1582 the reference's alone, and the first number must be read by root: our diagnostics against the reference's own corpora fell while our non-standard-notation warnings on our own example models rose ([differential](../project/pilot-differential.md), `go run -C tools ./cmd/pilot-diff`). - **Declared-diagnostic silence:** of the 512 declared `errors` rows in the reference's own Xpect suites, we report nothing for 0. 245 we report word-for-word; 248 wording-only and 7 location-only differences are agreement in substance and are not counted as gaps; 0 more we report as a warning and 2 elsewhere in the file ([Xpect oracle](../project/pilot-xpect.md), `go run -C tools ./cmd/pilot-xpect`). - **Scope agreement:** 230 of 230 declared scope assertions match exactly (same source). - **Permissiveness gaps:** of 306 invalid models we wrote ourselves, the reference rejects 4 that we accept by default, and 293 both reject; 4 further cases agree only when we are asked strictly. We authored every one of these cases ourselves, so the denominator measures the reach of our own corpus and not our conformance; agreement reached only under an opt-in strict mode is weaker evidence than agreement by default ([rejection oracle](../project/pilot-rejection.md), `go run -C tools ./cmd/pilot-reject`). -- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](../project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 379 files agreeing, 37 diagnostics ours alone and 1545 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. +- **Declared errata:** the registry declares 12 defect(s) in the published reference material — 4 with a specification-derived correction, 8 documented without one, since no intended reading can be inferred ([OMG issues](../project/omg-issues.md), `tools/oracle/errata`). Every figure above is as published and stays the conformance statement; running the same oracles over the corrected text instead reports 348 of 379 files agreeing, 37 diagnostics ours alone and 1582 the reference's alone, 0 declared rows we are silent on, and 0 of 306 authored cases the reference alone rejects. The corrected figures are diagnostic only: an erratum never reclassifies a divergence category, and the published corpus is never edited. - **Self-assessed surface:** the action, state-machine and classifier-behavior rows have no external referee at all — the four refereed figures above cannot see them, because the pinned artifact evaluates expressions but executes neither actions nor state machines. [Spec compliance](../project/spec-compliance.md) counts them. What these numbers cannot show: the OMG corpora are demonstrations rather than an official conformance suite; the differential is one-directional, comparing the diagnostics the two implementations report on the same files; the Xpect suites are the pilot authors' test intent rather than a certification oracle; and none of these is a percentage of the specification — no global compliance figure is claimed anywhere. diff --git a/docs/project/pilot-differential-baseline.json b/docs/project/pilot-differential-baseline.json index 60cc6ecfdb..2c4586178e 100644 --- a/docs/project/pilot-differential-baseline.json +++ b/docs/project/pilot-differential-baseline.json @@ -61,7 +61,7 @@ "dir": "examples", "origin": "ours", "files": 44, - "digest": "sha256:f8583984a81fa527b7afea26d5fcead936ad129d2d89133b7c1d57c3bcb0e2de" + "digest": "sha256:f2a378084ae0ea965ebe6e4289b1ef1fd7fe7df0e614593bc96f2fcd11a37721" }, { "name": "probes", @@ -79,9 +79,9 @@ "agreement": 38, "severityMismatch": 3, "openSysMLOnly": 38, - "pilotOnly": 1545, + "pilotOnly": 1582, "openSysMLDiagnostics": 79, - "pilotDiagnostics": 1586 + "pilotDiagnostics": 1623 }, "roots": [ { @@ -751,9 +751,9 @@ "agreement": 4, "severityMismatch": 2, "openSysMLOnly": 7, - "pilotOnly": 1525, + "pilotOnly": 1562, "openSysMLDiagnostics": 13, - "pilotDiagnostics": 1531 + "pilotDiagnostics": 1568 }, "files": [ { @@ -769,1249 +769,1315 @@ "count": 1 }, { - "line": 170, - "severity": "error", - "category": "unresolved-reference", - "count": 1 - }, - { - "line": 201, + "line": 174, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 210, + "line": 205, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 210, + "line": 205, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 211, + "line": 206, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 211, + "line": 206, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 212, + "line": 207, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 212, + "line": 207, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 213, + "line": 208, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 213, + "line": 208, "severity": "error", "category": "unmapped", "count": 1 }, { - "line": 214, + "line": 209, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 214, + "line": 209, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 216, + "line": 211, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 217, + "line": 212, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 218, + "line": 213, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 220, + "line": 215, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 222, + "line": 217, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 227, + "line": 222, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 227, + "line": 222, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 228, + "line": 223, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 229, + "line": 224, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 230, + "line": 225, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 234, + "line": 229, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 234, + "line": 229, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 235, + "line": 230, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 235, + "line": 230, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 236, + "line": 231, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 236, + "line": 231, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 237, + "line": 232, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 237, + "line": 232, "severity": "error", "category": "unmapped", "count": 1 }, { - "line": 238, + "line": 233, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 238, + "line": 233, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 240, + "line": 235, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 241, + "line": 236, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 242, + "line": 237, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 244, + "line": 239, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 246, + "line": 241, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 251, + "line": 246, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 251, + "line": 246, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 252, + "line": 247, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 253, + "line": 248, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 254, + "line": 249, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 258, + "line": 253, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 258, + "line": 253, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 259, + "line": 254, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 259, + "line": 254, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 260, + "line": 255, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 260, + "line": 255, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 261, + "line": 256, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 261, + "line": 256, "severity": "error", "category": "unmapped", "count": 1 }, { - "line": 262, + "line": 257, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 262, + "line": 257, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 264, + "line": 259, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 265, + "line": 260, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 266, + "line": 261, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 268, + "line": 263, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 270, + "line": 265, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 275, + "line": 270, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 275, + "line": 270, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 276, + "line": 271, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 277, + "line": 272, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 278, + "line": 273, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 284, + "line": 279, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 284, + "line": 279, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 285, + "line": 280, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 285, + "line": 280, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 286, + "line": 281, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 286, + "line": 281, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 287, + "line": 282, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 287, + "line": 282, "severity": "error", "category": "unmapped", "count": 1 }, { - "line": 288, + "line": 283, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 288, + "line": 283, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 290, + "line": 285, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 291, + "line": 286, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 292, + "line": 287, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 294, + "line": 289, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 296, + "line": 291, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 297, + "line": 292, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 302, + "line": 297, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 302, + "line": 297, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 303, + "line": 298, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 304, + "line": 299, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 305, + "line": 300, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 309, + "line": 304, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 309, + "line": 304, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 310, + "line": 305, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 310, + "line": 305, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 311, + "line": 306, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 311, + "line": 306, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 312, + "line": 307, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 312, + "line": 307, "severity": "error", "category": "unmapped", "count": 1 }, { - "line": 313, + "line": 308, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 313, + "line": 308, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 315, + "line": 310, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 316, + "line": 311, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 317, + "line": 312, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 319, + "line": 314, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 321, + "line": 316, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 322, + "line": 317, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 327, + "line": 322, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 327, + "line": 322, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 328, + "line": 323, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 329, + "line": 324, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 330, + "line": 325, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 334, + "line": 329, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 334, + "line": 329, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 335, + "line": 330, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 335, + "line": 330, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 336, + "line": 331, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 336, + "line": 331, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 337, + "line": 332, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 337, + "line": 332, "severity": "error", "category": "unmapped", "count": 1 }, { - "line": 338, + "line": 333, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 338, + "line": 333, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 340, + "line": 335, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 341, + "line": 336, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 342, + "line": 337, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 344, + "line": 339, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 346, + "line": 341, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 347, + "line": 342, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 352, + "line": 347, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 352, + "line": 347, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 353, + "line": 348, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 354, + "line": 349, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 355, + "line": 350, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 363, + "line": 358, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 363, + "line": 358, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 364, + "line": 359, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 364, + "line": 359, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 365, + "line": 360, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 365, + "line": 360, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 366, + "line": 361, "severity": "error", "category": "kind-mismatch", "count": 2 }, { - "line": 366, + "line": 361, "severity": "error", "category": "unmapped", "count": 1 }, { - "line": 367, + "line": 362, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 367, + "line": 362, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 369, + "line": 364, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 370, + "line": 365, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 371, + "line": 366, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 374, + "line": 369, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 374, + "line": 369, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 375, + "line": 370, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 376, + "line": 371, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 377, + "line": 372, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 379, + "line": 374, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 379, + "line": 374, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 380, + "line": 375, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 381, + "line": 376, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 382, + "line": 377, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 383, + "line": 378, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 384, + "line": 379, "severity": "error", "category": "unmapped", "count": 1 }, { - "line": 384, + "line": 379, "severity": "warning", "category": "kind-mismatch", "count": 1 }, { - "line": 385, + "line": 380, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 387, + "line": 382, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 387, + "line": 382, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 388, + "line": 383, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 389, + "line": 384, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 390, + "line": 385, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 391, + "line": 386, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 392, + "line": 387, "severity": "error", "category": "unmapped", "count": 1 }, { - "line": 392, + "line": 387, "severity": "warning", "category": "kind-mismatch", "count": 1 }, { - "line": 393, + "line": 388, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 395, + "line": 390, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 395, + "line": 390, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 396, + "line": 391, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 397, + "line": 392, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 398, + "line": 393, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 399, + "line": 394, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 400, + "line": 395, "severity": "error", "category": "unmapped", "count": 1 }, { - "line": 400, + "line": 395, "severity": "warning", "category": "kind-mismatch", "count": 1 }, { - "line": 401, + "line": 396, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 409, + "line": 403, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 414, + "line": 408, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 416, + "line": 409, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 416, + "line": 409, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 417, + "line": 410, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 417, + "line": 410, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 418, + "line": 411, "severity": "error", "category": "kind-mismatch", + "count": 1 + }, + { + "line": 411, + "severity": "error", + "category": "unresolved-reference", "count": 2 }, { - "line": 418, + "line": 412, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 412, "severity": "error", "category": "unresolved-reference", - "count": 7 + "count": 6 }, { - "line": 419, + "line": 413, "severity": "error", "category": "unresolved-reference", - "count": 4 + "count": 1 }, { - "line": 420, + "line": 414, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 424, + "line": 415, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 419, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 426, + "line": 420, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 426, + "line": 420, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 427, + "line": 421, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 427, + "line": 421, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 428, + "line": 422, "severity": "error", "category": "kind-mismatch", + "count": 3 + }, + { + "line": 422, + "severity": "error", + "category": "unresolved-reference", + "count": 9 + }, + { + "line": 423, + "severity": "error", + "category": "unresolved-reference", + "count": 4 + }, + { + "line": 424, + "severity": "error", + "category": "unresolved-reference", "count": 1 }, { "line": 428, "severity": "error", "category": "unresolved-reference", - "count": 2 + "count": 1 }, { "line": 429, "severity": "error", "category": "kind-mismatch", - "count": 2 + "count": 1 }, { "line": 429, "severity": "error", "category": "unresolved-reference", - "count": 7 + "count": 1 + }, + { + "line": 430, + "severity": "error", + "category": "kind-mismatch", + "count": 1 }, { "line": 430, "severity": "error", "category": "unresolved-reference", - "count": 3 + "count": 2 + }, + { + "line": 431, + "severity": "error", + "category": "kind-mismatch", + "count": 1 }, { "line": 431, "severity": "error", "category": "unresolved-reference", - "count": 4 + "count": 2 }, { "line": 432, "severity": "error", + "category": "kind-mismatch", + "count": 3 + }, + { + "line": 432, + "severity": "error", + "category": "unresolved-reference", + "count": 9 + }, + { + "line": 433, + "severity": "error", + "category": "unresolved-reference", + "count": 3 + }, + { + "line": 434, + "severity": "error", + "category": "unresolved-reference", + "count": 4 + }, + { + "line": 435, + "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 436, + "line": 439, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 438, + "line": 440, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 438, + "line": 440, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 439, + "line": 441, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 439, + "line": 441, "severity": "error", "category": "unresolved-reference", "count": 2 }, { - "line": 440, + "line": 442, "severity": "error", "category": "kind-mismatch", - "count": 2 + "count": 3 }, { - "line": 440, + "line": 442, "severity": "error", "category": "unresolved-reference", - "count": 7 + "count": 9 }, { - "line": 441, + "line": 443, "severity": "error", "category": "unresolved-reference", "count": 3 }, { - "line": 442, + "line": 444, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 446, + "line": 448, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 448, + "line": 449, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 448, + "line": 449, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 449, + "line": 450, "severity": "error", "category": "kind-mismatch", - "count": 2 + "count": 3 }, { - "line": 449, + "line": 450, "severity": "error", "category": "unresolved-reference", - "count": 7 + "count": 9 }, { - "line": 450, + "line": 451, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 454, + "line": 455, "severity": "error", "category": "unresolved-reference", "count": 1 @@ -2032,13 +2098,13 @@ "line": 457, "severity": "error", "category": "kind-mismatch", - "count": 2 + "count": 3 }, { "line": 457, "severity": "error", "category": "unresolved-reference", - "count": 7 + "count": 9 }, { "line": 458, @@ -2053,241 +2119,247 @@ "count": 1 }, { - "line": 464, + "line": 463, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 464, + "line": 463, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 465, + "line": 464, "severity": "error", "category": "kind-mismatch", - "count": 2 + "count": 3 }, { - "line": 465, + "line": 464, "severity": "error", "category": "unresolved-reference", - "count": 7 + "count": 9 }, { - "line": 466, + "line": 465, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 470, + "line": 469, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 472, + "line": 471, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 472, + "line": 471, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 473, + "line": 472, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 473, + "line": 472, "severity": "error", "category": "unresolved-reference", "count": 3 }, { - "line": 474, + "line": 473, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 478, + "line": 477, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 479, + "line": 478, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 481, + "line": 480, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 481, + "line": 480, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 482, + "line": 481, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 485, + "line": 484, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 485, + "line": 484, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 486, + "line": 485, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 487, + "line": 486, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 487, + "line": 486, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 488, + "line": 487, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 489, + "line": 492, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 492, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 491, - "severity": "warning", - "category": "kind-mismatch", + "line": 493, + "severity": "error", + "category": "unresolved-reference", "count": 1 }, { - "line": 496, + "line": 494, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 496, + "line": 494, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 497, + "line": 495, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 498, + "line": 496, "severity": "error", - "category": "kind-mismatch", + "category": "unresolved-reference", "count": 1 }, { - "line": 498, + "line": 501, "severity": "error", - "category": "unresolved-reference", + "category": "kind-mismatch", "count": 1 }, { - "line": 499, + "line": 501, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 501, - "severity": "warning", - "category": "kind-mismatch", + "line": 502, + "severity": "error", + "category": "unresolved-reference", "count": 1 }, { - "line": 506, + "line": 503, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 506, + "line": 503, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 507, + "line": 504, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 508, + "line": 509, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 508, + "line": 509, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 509, + "line": 510, "severity": "error", "category": "unresolved-reference", "count": 1 }, { "line": 511, - "severity": "warning", + "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 516, + "line": 511, "severity": "error", - "category": "kind-mismatch", + "category": "unresolved-reference", "count": 1 }, { - "line": 516, + "line": 512, "severity": "error", "category": "unresolved-reference", "count": 1 @@ -2295,13 +2367,13 @@ { "line": 517, "severity": "error", - "category": "unresolved-reference", + "category": "kind-mismatch", "count": 1 }, { - "line": 518, + "line": 517, "severity": "error", - "category": "kind-mismatch", + "category": "unresolved-reference", "count": 1 }, { @@ -2310,6 +2382,12 @@ "category": "unresolved-reference", "count": 1 }, + { + "line": 519, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, { "line": 519, "severity": "error", @@ -2317,115 +2395,103 @@ "count": 1 }, { - "line": 521, - "severity": "warning", - "category": "kind-mismatch", + "line": 520, + "severity": "error", + "category": "unresolved-reference", "count": 1 }, { - "line": 526, + "line": 525, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 526, + "line": 525, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 527, + "line": 526, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 528, + "line": 527, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 528, + "line": 527, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 529, + "line": 528, "severity": "error", "category": "unresolved-reference", "count": 1 }, { "line": 531, - "severity": "warning", - "category": "kind-mismatch", - "count": 1 - }, - { - "line": 534, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 534, + "line": 531, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 535, + "line": 532, "severity": "error", "category": "unresolved-reference", "count": 1 }, { "line": 537, - "severity": "warning", - "category": "kind-mismatch", - "count": 1 - }, - { - "line": 542, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 542, + "line": 537, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 543, + "line": 538, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 544, + "line": 539, "severity": "error", "category": "kind-mismatch", "count": 1 }, { - "line": 544, + "line": 539, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 545, + "line": 540, "severity": "error", "category": "unresolved-reference", "count": 1 }, { - "line": 547, + "line": 542, "severity": "warning", "category": "kind-mismatch", "count": 1 @@ -8122,9 +8188,9 @@ "agreement": 38, "severityMismatch": 3, "openSysMLOnly": 37, - "pilotOnly": 1545, + "pilotOnly": 1582, "openSysMLDiagnostics": 78, - "pilotDiagnostics": 1586 + "pilotDiagnostics": 1623 }, "findings": [ { diff --git a/docs/project/pilot-differential.md b/docs/project/pilot-differential.md index deb4c1e2bc..a7c7d5fdcf 100644 --- a/docs/project/pilot-differential.md +++ b/docs/project/pilot-differential.md @@ -218,9 +218,9 @@ nor double-counted as two independent disagreements. | `examples/pilot-corpora/sysml-validation` | 56 | 56 | 0 | 0 | 0 | 0 | 0 | 0 | | `examples/pilot-corpora/kerml-examples` | 58 | 55 | 10 | 0 | 0 | 0 | 10 | 0 | | `tests/testdata` | 18 | 10 | 43 | 55 | 34 | 1 | 8 | 20 | -| `examples` | 44 | 30 | 13 | 1531 | 4 | 2 | 7 | 1525 | +| `examples` | 44 | 30 | 13 | 1568 | 4 | 2 | 7 | 1562 | | `tools/referee/diff/testdata` (probes) | 4 | 1 | 6 | 0 | 0 | 0 | 6 | 0 | -| **Total** | **379** | **347** | **79** | **1586** | **38** | **3** | **38** | **1545** | +| **Total** | **379** | **347** | **79** | **1623** | **38** | **3** | **38** | **1582** | **Read the `only ours` total by root, never as one number.** Step 2 removes nine resolver false positives from the reference's **own** corpora: `pilot-examples` 16 → **7** and @@ -791,8 +791,8 @@ cascades through the rest of the file. The movement is entirely one file, | Count | Before the initializer rewrite | Now | |---|---:|---:| -| only pilot | 82 | **1545** | -| pilot diagnostics | 123 | **1586** | +| only pilot | 82 | **1582** | +| pilot diagnostics | 123 | **1623** | | severity-only | 9 | **3** | The rewrite itself took only-pilot to 61 and pilot diagnostics to 101; the `Now` column states @@ -925,7 +925,7 @@ Per category, the only-ours totals are: `pilot-examples` 4 `unmapped`, 2 advisory of the [runtime showcase round](#runtime-showcase-round)); `testdata` 7 `unmapped`, 1 `multiplicity`; `probes` 6 `unmapped`. Only-pilot: `testdata` 12 `kind-mismatch`, 3 `unmapped`, 3 syntax, 2 `unresolved-reference`; -`examples` 10 syntax, 29 `unmapped`, 654 `kind-mismatch`, 832 `unresolved-reference` — of which +`examples` 10 syntax, 29 `unmapped`, 661 `kind-mismatch`, 862 `unresolved-reference` — of which `relay-probe-demo/mission.sysml` carries none: it carried a `kind-mismatch` on its send of a `Telemetry` invocation until the send-argument round above, and the demo now writes the constructor, `send new Telemetry(…) via antenna`, which both implementations accept, so the row @@ -1017,13 +1017,13 @@ page's history. | Count | Now | |---|---:| | overall: fully agreeing / only ours / our diagnostics | **347 / 38 / 79** | -| only pilot | **1545** | -| pilot diagnostics | **1586** | +| only pilot | **1582** | +| pilot diagnostics | **1623** | | severity-only | **3** | | unmapped, our side | **34** | | kerml-examples: only ours | **10** | | pilot-examples: only ours | **7** | -| examples: only pilot | **1525** | +| examples: only pilot | **1562** | The KerML root is now the *cleanest* of the three OMG roots in proportion: **10** only-ours against 6 only-pilot, with 49 of 58 files fully @@ -1171,10 +1171,10 @@ Nothing else moves: the file draws no diagnostic from this implementation, so `f ### Analysis results recording round `examples/analysis-results-demo/lander-results.sysml` is one file added to the `examples` root: -files 43 → **44** on the root, 378 → **379** overall, and pilot diagnostics 1226 → **1586** / -only pilot 1185 → **1545** — 360 diagnostics in 278 reported rows: 181 `unresolved-reference` -(counted 243), 87 `kind-mismatch` (counted 107) and 10 `unmapped`, all inside the file's -`Results` and `Reporting` packages. Every row is a construct the pinned artifact has no support +files 43 → **44** on the root, 378 → **379** overall, and pilot diagnostics 1226 → **1623** / +only pilot 1185 → **1582** — 397 diagnostics in 289 reported rows: 192 `unresolved-reference` +(counted 273), 87 `kind-mismatch` (counted 114) and 10 `unmapped`, all inside the file's +`Records` and `Reporting` packages. Every row is a construct the pinned artifact has no support for: the document-query calls (`Project`, `OrderBy`, `WhereType`, `WhereFeature`, `WhereMetadata`, `Verdicts`, `Descendants`), the `@AnalysisRecords::RecordedRun` metadata annotations and specializations of the `AnalysisRecords` library defs, and the run-record part From d6933421c661c96e1d86e82bf1c3e830252ce4d3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 13:36:57 +0000 Subject: [PATCH 10/10] docs(project): re-record the differential baseline after merging develop Co-Authored-By: jason.han --- docs/project/pilot-differential-baseline.json | 1769 ++++++++++++++++- 1 file changed, 1758 insertions(+), 11 deletions(-) diff --git a/docs/project/pilot-differential-baseline.json b/docs/project/pilot-differential-baseline.json index 7b76b7f2ae..d005f55e47 100644 --- a/docs/project/pilot-differential-baseline.json +++ b/docs/project/pilot-differential-baseline.json @@ -60,8 +60,8 @@ "name": "examples", "dir": "examples", "origin": "ours", - "files": 43, - "digest": "sha256:3e4522077d7c4903aa3c6cad01dc84abc79588f9f13c62e7c5aee8c9bee1a02f" + "files": 44, + "digest": "sha256:e1102754c82d55de49dd9a70010c43e515784f8237de200fc7e29d79ff8a7bcd" }, { "name": "probes", @@ -74,14 +74,14 @@ "recorded": "2026-09-24" }, "totals": { - "files": 378, + "files": 379, "filesFullyAgreeing": 347, "agreement": 38, "severityMismatch": 3, "openSysMLOnly": 38, - "pilotOnly": 1185, + "pilotOnly": 1582, "openSysMLDiagnostics": 79, - "pilotDiagnostics": 1226 + "pilotDiagnostics": 1623 }, "roots": [ { @@ -746,16 +746,1758 @@ "name": "examples", "dir": "examples", "totals": { - "files": 43, + "files": 44, "filesFullyAgreeing": 30, "agreement": 4, "severityMismatch": 2, "openSysMLOnly": 7, - "pilotOnly": 1165, + "pilotOnly": 1562, "openSysMLDiagnostics": 13, - "pilotDiagnostics": 1171 + "pilotDiagnostics": 1568 }, "files": [ + { + "path": "analysis-results-demo/lander-results.sysml", + "agreement": [], + "severityMismatch": [], + "openSysMLOnly": [], + "pilotOnly": [ + { + "line": 164, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 174, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 205, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 205, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 206, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 206, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 207, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 207, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 208, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 208, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 209, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 209, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 211, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 212, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 213, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 215, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 217, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 222, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 222, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 223, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 224, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 225, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 229, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 229, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 230, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 230, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 231, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 231, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 232, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 232, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 233, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 233, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 235, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 236, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 237, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 239, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 241, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 246, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 246, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 247, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 248, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 249, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 253, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 253, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 254, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 254, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 255, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 255, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 256, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 256, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 257, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 257, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 259, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 260, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 261, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 263, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 265, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 270, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 270, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 271, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 272, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 273, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 279, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 279, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 280, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 280, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 281, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 281, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 282, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 282, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 283, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 283, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 285, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 286, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 287, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 289, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 291, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 292, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 297, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 297, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 298, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 299, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 300, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 304, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 304, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 305, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 305, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 306, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 306, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 307, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 307, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 308, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 308, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 310, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 311, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 312, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 314, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 316, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 317, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 322, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 322, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 323, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 324, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 325, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 329, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 329, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 330, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 330, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 331, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 331, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 332, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 332, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 333, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 333, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 335, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 336, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 337, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 339, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 341, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 342, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 347, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 347, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 348, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 349, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 350, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 358, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 358, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 359, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 359, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 360, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 360, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 361, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 361, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 362, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 362, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 364, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 365, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 366, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 369, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 369, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 370, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 371, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 372, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 374, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 374, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 375, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 376, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 377, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 378, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 379, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 379, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 380, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 382, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 382, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 383, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 384, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 385, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 386, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 387, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 387, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 388, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 390, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 390, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 391, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 392, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 393, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 394, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 395, + "severity": "error", + "category": "unmapped", + "count": 1 + }, + { + "line": 395, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 396, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 403, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 408, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 409, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 409, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 410, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 410, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 411, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 411, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 412, + "severity": "error", + "category": "kind-mismatch", + "count": 2 + }, + { + "line": 412, + "severity": "error", + "category": "unresolved-reference", + "count": 6 + }, + { + "line": 413, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 414, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 415, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 419, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 420, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 420, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 421, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 421, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 422, + "severity": "error", + "category": "kind-mismatch", + "count": 3 + }, + { + "line": 422, + "severity": "error", + "category": "unresolved-reference", + "count": 9 + }, + { + "line": 423, + "severity": "error", + "category": "unresolved-reference", + "count": 4 + }, + { + "line": 424, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 428, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 429, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 429, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 430, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 430, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 431, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 431, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 432, + "severity": "error", + "category": "kind-mismatch", + "count": 3 + }, + { + "line": 432, + "severity": "error", + "category": "unresolved-reference", + "count": 9 + }, + { + "line": 433, + "severity": "error", + "category": "unresolved-reference", + "count": 3 + }, + { + "line": 434, + "severity": "error", + "category": "unresolved-reference", + "count": 4 + }, + { + "line": 435, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 439, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 440, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 440, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 441, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 441, + "severity": "error", + "category": "unresolved-reference", + "count": 2 + }, + { + "line": 442, + "severity": "error", + "category": "kind-mismatch", + "count": 3 + }, + { + "line": 442, + "severity": "error", + "category": "unresolved-reference", + "count": 9 + }, + { + "line": 443, + "severity": "error", + "category": "unresolved-reference", + "count": 3 + }, + { + "line": 444, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 448, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 449, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 449, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 450, + "severity": "error", + "category": "kind-mismatch", + "count": 3 + }, + { + "line": 450, + "severity": "error", + "category": "unresolved-reference", + "count": 9 + }, + { + "line": 451, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 455, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 456, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 456, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 457, + "severity": "error", + "category": "kind-mismatch", + "count": 3 + }, + { + "line": 457, + "severity": "error", + "category": "unresolved-reference", + "count": 9 + }, + { + "line": 458, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 462, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 463, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 463, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 464, + "severity": "error", + "category": "kind-mismatch", + "count": 3 + }, + { + "line": 464, + "severity": "error", + "category": "unresolved-reference", + "count": 9 + }, + { + "line": 465, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 469, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 471, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 471, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 472, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 472, + "severity": "error", + "category": "unresolved-reference", + "count": 3 + }, + { + "line": 473, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 477, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 478, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 480, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 480, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 481, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 484, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 484, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 485, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 486, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 486, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 487, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 492, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 492, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 493, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 494, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 494, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 495, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 496, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 501, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 501, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 502, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 503, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 503, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 504, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 509, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 509, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 510, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 511, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 511, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 512, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 517, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 517, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 518, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 519, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 519, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 520, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 525, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 525, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 526, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 527, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 527, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 528, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 531, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 531, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 532, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 537, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 537, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 538, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 539, + "severity": "error", + "category": "kind-mismatch", + "count": 1 + }, + { + "line": 539, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 540, + "severity": "error", + "category": "unresolved-reference", + "count": 1 + }, + { + "line": 542, + "severity": "warning", + "category": "kind-mismatch", + "count": 1 + } + ] + }, { "path": "disposal-robot-demo/robot.sysml", "agreement": [], @@ -6086,6 +7828,11 @@ } ], "unmapped": [ + { + "side": "pilot", + "message": "Cannot override a binding feature value", + "count": 10 + }, { "side": "pilot", "message": "Duplicate of other owned member name", @@ -6436,14 +8183,14 @@ } ], "totals": { - "files": 378, + "files": 379, "filesFullyAgreeing": 348, "agreement": 38, "severityMismatch": 3, "openSysMLOnly": 37, - "pilotOnly": 1185, + "pilotOnly": 1582, "openSysMLDiagnostics": 78, - "pilotDiagnostics": 1226 + "pilotDiagnostics": 1623 }, "findings": [ {