diff --git a/changes/unreleased/record-analysis-runs.added.md b/changes/unreleased/record-analysis-runs.added.md new file mode 100644 index 000000000..903571c96 --- /dev/null +++ b/changes/unreleased/record-analysis-runs.added.md @@ -0,0 +1 @@ +- **Record analysis runs into the model.** `%record [into ]` at the REPL and `-record-run ` on the command line run an analysis case as `%analysis`/`-analysis` does and write the run into the model as `AnalysisRecords` elements — a record definition per case, one part per run carrying the inputs bound and outputs produced, and `@AnalysisRecords::RecordedRun` provenance metadata. Sweeps (`-sweep`) and Monte Carlo samples (`-runs`/`-seed`) record one part per run, plus one for the sample's conclusion under kind `sample`; records compose with `-convert sysml -o` and `-render-document`, are found by document queries, and an `inout` records the value the run left and a `In` companion for the value it was bound with, and values supplied as Integer and Real alike settle a member to Real and scalar-valued enum literals keep their literal, and a failed run records nothing. A verification case's record carries the verdict its body decided — the `verdict` attribute — and one `VerdictRecord` row apiece for it and each subcase's. diff --git a/cmd/sysml/check.go b/cmd/sysml/check.go index 696d31a44..c63c8099f 100644 --- a/cmd/sysml/check.go +++ b/cmd/sysml/check.go @@ -27,6 +27,8 @@ type checks struct { satisfy optionalNames calcs stringSlice analyses stringSlice + records stringSlice + recordInto string sweeps stringSlice samples sweepCount seed sweepSeed @@ -231,6 +233,7 @@ func (a *advanceTime) Set(value string) error { func (c *checks) requested() bool { return c.validate.given || c.jsonOut || c.advance.given || c.satisfy.given || len(c.instantiate) > 0 || len(c.constraints) > 0 || len(c.requirements) > 0 || len(c.calcs) > 0 || len(c.analyses) > 0 || + len(c.records) > 0 || len(c.queries) > 0 || len(c.actions) > 0 || len(c.states) > 0 || c.sweeping() || c.running() || c.compare != "" || c.checker.given() } @@ -328,11 +331,11 @@ func (c *checks) runsMisuse() string { switch { case !c.runs.given: return "-observe names what -runs reports; ask for the runs, as -runs " - case len(c.actions) == 0 && len(c.analyses) == 0: + case len(c.actions) == 0 && len(c.analyses) == 0 && len(c.records) == 0: return "-runs runs an action or a Simulation::MonteCarlo analysis case; name one, as -action or -analysis " - case len(c.actions)+len(c.analyses) > 1: + case len(c.actions)+len(c.analyses)+len(c.records) > 1: return "-runs runs one action or analysis case; name a single -action or -analysis" - case len(c.analyses) > 0 && len(c.observe) > 0: + case len(c.analyses)+len(c.records) > 0 && len(c.observe) > 0: return "-runs of an analysis case observes what the case declares as observed; -observe names the features of an -action" case len(c.states) > 0: return "-runs runs an action; a state machine is run once, as -state without -runs" @@ -355,7 +358,7 @@ func (c *checks) compareMisuse() string { switch { case len(c.states) > 0 || c.sweeping() || c.advance.given || c.checker.given() || c.validate.given || c.satisfy.given || len(c.instantiate) > 0 || len(c.constraints) > 0 || - len(c.requirements) > 0 || len(c.calcs) > 0 || len(c.analyses) > 0 || len(c.queries) > 0: + len(c.requirements) > 0 || len(c.calcs) > 0 || len(c.analyses) > 0 || len(c.records) > 0 || len(c.queries) > 0: return "-compare-results runs the migrated configurations the results index and compares the runs with the tool's; the other checks are made in a run of their own" } for _, pair := range c.observe { @@ -407,7 +410,7 @@ func (c *checks) sweepMisuse() string { if !c.sweeping() { return "" } - targets := len(c.calcs) + len(c.analyses) + targets := len(c.calcs) + len(c.analyses) + len(c.records) switch { case targets == 0: return "-sweep runs an analysis case or a calc; name one, as -analysis or -calc " @@ -426,14 +429,53 @@ func (c *checks) sweepMisuse() string { func (c *checks) instantiatesOnly() bool { return len(c.instantiate) > 0 && !c.validate.given && !c.jsonOut && !c.advance.given && !c.satisfy.given && len(c.constraints) == 0 && len(c.requirements) == 0 && len(c.calcs) == 0 && len(c.analyses) == 0 && + len(c.records) == 0 && len(c.queries) == 0 && len(c.actions) == 0 && len(c.states) == 0 && !c.sweeping() && !c.running() && c.compare == "" && !c.checker.given() } +// recordsOnly reports whether the run makes records and decides nothing else, +// so a document can be rendered over, or a file written from, what was recorded. +// The bounds the records run under — a sweep's ranges, a Monte Carlo's runs, +// seed and draws — and the objects -instantiate materializes for them, serve them. +func (c *checks) recordsOnly() bool { + return len(c.records) > 0 && !c.validate.given && !c.jsonOut && !c.advance.given && !c.satisfy.given && + len(c.constraints) == 0 && len(c.requirements) == 0 && len(c.calcs) == 0 && + len(c.analyses) == 0 && len(c.observe) == 0 && + len(c.queries) == 0 && len(c.actions) == 0 && len(c.states) == 0 && c.compare == "" && !c.checker.given() +} + +// recordMisuse reports why the flags the records were asked for with record +// none, and "" when they record one. +func (c *checks) recordMisuse() string { + if len(c.records) == 0 { + return "" + } + switch { + case c.samples.given: + return "-samples draws values for a sweep it does not run; -record-run records a run, a -sweep's rows or a -runs sample" + case len(c.records) > 1 && c.runs.given: + return "-runs runs one analysis case; name a single -record-run" + } + return "" +} + +// boundsMisuse reports why the bounds a records run was asked for make no run: +// the same refusal runChecks gives for them. +func (c *checks) boundsMisuse() string { + for _, message := range []string{c.sweepMisuse(), c.runsMisuse(), c.recordMisuse()} { + if message != "" { + return message + } + } + return "" +} + // checksOnly reports whether anything was asked about the model itself, as // against how to report the answer. func (c *checks) checksOnly() bool { return len(c.validate.targets) > 0 || len(c.instantiate) > 0 || len(c.constraints) > 0 || len(c.requirements) > 0 || len(c.satisfy.targets) > 0 || len(c.calcs) > 0 || len(c.analyses) > 0 || + len(c.records) > 0 || len(c.queries) > 0 || len(c.actions) > 0 || len(c.states) > 0 || c.compare != "" } @@ -533,6 +575,10 @@ func runChecks(files []string, exprs []string, c checks) int { rep.failed(message) return rep.finish() } + if message := c.recordMisuse(); message != "" { + rep.failed(message) + return rep.finish() + } if message := c.checkerMisuse(engine.text); message != "" { rep.failed(message) return rep.finish() @@ -708,6 +754,9 @@ func runChecks(files []string, exprs []string, c checks) int { rep.verdict(sess.RunAnalysis(invocation)) } } + for _, invocation := range c.records { + rep.verdict(c.record(sess, invocation)) + } // With -advance every behavior named is started first and the clock they share // is moved once, so an action's signal reaches a machine that accepts it later; // under the check engine it bounds the search of the invocation's schedules. @@ -753,6 +802,56 @@ func behaviors(values []string) []repl.Behavior { return out } +// record runs one invocation as its -analysis twin does — swept or sampled over +// -runs when the flags say — then writes the run into the model as records. +func (c *checks) record(sess *repl.Session, invocation string) repl.Verdict { + command := c.recordCommand(invocation) + switch { + case c.sweeping(): + return sess.RecordSweep(invocation, c.sweeps, c.recordInto, command) + case c.runs.given: + return sess.RecordMonteCarlo(invocation, c.runs.value, c.seed.seed(), c.recordInto, command) + default: + return sess.RecordAnalysis(invocation, c.recordInto, command) + } +} + +// recordCommand is the invocation text a record's provenance carries: the flags +// the run was made with, as written. +func (c *checks) recordCommand(invocation string) string { + parts := []string{fmt.Sprintf("-record-run %q", invocation)} + for _, r := range c.sweeps { + parts = append(parts, fmt.Sprintf("-sweep %q", r)) + } + if c.runs.given { + parts = append(parts, "-runs "+c.runs.text) + } + if c.seed.given { + parts = append(parts, "-seed "+c.seed.text) + } + if c.draws.text != "" { + parts = append(parts, "-draws "+c.draws.text) + } + // The flags the session runs under decide what the run computed and which + // objects it ran on, so the command records them as written too. + if schedule.text != "" { + parts = append(parts, "-schedule "+schedule.text) + } + if c.clockStep.given { + parts = append(parts, "-clock-step "+c.clockStep.text) + } + if engine.text != "" { + parts = append(parts, "-engine "+engine.text) + } + for _, name := range c.instantiate { + parts = append(parts, fmt.Sprintf("-instantiate %q", name)) + } + if c.recordInto != "" { + parts = append(parts, "-record-into "+c.recordInto) + } + return strings.Join(parts, " ") +} + // sweep runs one invocation once per row of the ranges given: over every value // of each range, or over values drawn from them when -samples was asked for. func (c *checks) sweep(sess *repl.Session, invocation string) repl.Verdict { diff --git a/cmd/sysml/compare_test.go b/cmd/sysml/compare_test.go index 3f72cb9c7..52da12b47 100644 --- a/cmd/sysml/compare_test.go +++ b/cmd/sysml/compare_test.go @@ -166,6 +166,7 @@ func TestMigrationResultsThroughCLI(t *testing.T) { "missing sidecar": {[]string{model, "-compare-results", filepath.Join(dir, "none.json")}, "-compare-results: open"}, "sidecar not JSON": {[]string{model, "-compare-results", model}, "the results are not the JSON -migration-results writes"}, "with convert": {[]string{model, "-compare-results", sidecar, "-convert", "ttl"}, "cannot be combined with -convert"}, + "with record-run": {[]string{model, "-compare-results", sidecar, "-record-run", "Group 0"}, "the other checks are made in a run of their own"}, "results without xmi": {[]string{model, "-convert", "ttl", "-migration-results", sidecar}, "-migration-results indexes the result snapshots of a SysML v1 migration"}, "results without convert": {[]string{model, "-migration-results", sidecar}, "-migration-results accompanies -convert"}, "results over the model": {[]string{simconfigXMI, "-convert", "sysml", "-o", sidecar, "-migration-results", sidecar}, "-migration-results and -o both name"}, diff --git a/cmd/sysml/convert.go b/cmd/sysml/convert.go index 48af38aab..943a184fb 100644 --- a/cmd/sysml/convert.go +++ b/cmd/sysml/convert.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" + "github.com/Open-MBEE/OpenSysML/internal/frontend/repl" "github.com/Open-MBEE/OpenSysML/internal/translate/convert" "github.com/Open-MBEE/OpenSysML/internal/translate/export" "github.com/Open-MBEE/OpenSysML/internal/translate/interop/flexo" @@ -61,6 +62,15 @@ func runConvert(files []string) (int, error) { } input := files[0] + // A run asked to record puts the records in the session's buffer rather than + // in the file, so what is converted is that buffer's text, as %save writes it. + if len(modelChecks.records) > 0 { + if err := recordedConvertMisuse(input); err != nil { + return 0, err + } + return convertRecorded(input, to) + } + inputRef, inputIsURL, err := flexo.ParseBranchURL(input) if err != nil { return 0, err @@ -134,25 +144,18 @@ func runConvert(files []string) (int, error) { // convertInput runs the conversion the input format asks for: a SysML v1 model // is migrated and its report written, anything else converted. func convertInput(name string, data []byte, from, to convert.Format) ([]byte, error) { - if idForm != "" && (from != convert.FormatSysML || (to != convert.FormatTurtle && to != convert.FormatAPIJSON)) { - return nil, fmt.Errorf("-id applies to -convert ttl or api-json from SysML notation") + opts, err := convertOptions(from, to) + if err != nil { + return nil, err } if from != convert.FormatXMI { - opts := convert.Options{} - if idForm != "" { - form, ok := export.ParseIDForm(idForm) - if !ok { - return nil, fmt.Errorf("-id wants qualified or uuid, not %q", idForm) - } - opts.ID = form - } return convert.ConvertWith(name, data, from, to, opts) } - opts, err := migrationOptions() + migOpts, err := migrationOptions() if err != nil { return nil, err } - migrated, err := convert.Migrate(name, data, to, opts) + migrated, err := convert.Migrate(name, data, to, migOpts) if err != nil { return nil, err } @@ -165,6 +168,115 @@ func convertInput(name string, data []byte, from, to convert.Format) ([]byte, er return migrated.Output, nil } +// recordedConvertMisuse is why a flag cannot share the run -record-run +// converts: what is converted is the session the records join, not a file +// migrated or a branch read or pushed. +func recordedConvertMisuse(input string) error { + inRef, inputIsURL, err := flexo.ParseBranchURL(input) + if err != nil { + return err + } + if inputIsURL { + return fmt.Errorf("-record-run converts the recorded session model; a repository branch is not an input it reads (%s)", inRef) + } + if outputPath != "" { + outRef, outputIsURL, err := flexo.ParseBranchURL(outputPath) + if err != nil { + return err + } + if outputIsURL { + return fmt.Errorf("-record-run converts the recorded session model; -o cannot push it to a repository branch (%s)", outRef) + } + } + switch { + case syncState != "": + return errors.New("-record-run converts the recorded session model; -sync-state does not apply") + case migrationReport != "": + return errors.New("-record-run converts the recorded session model; -migration-report does not apply") + case migrationResults != "": + return errors.New("-record-run converts the recorded session model; -migration-results does not apply") + case layoutPath != "": + return errors.New("-record-run converts the recorded session model; -layout does not apply") + } + return nil +} + +// convertRecorded loads the file, makes the runs -record-run names so the +// records join the session's buffer, and converts that text; a load that did +// not analyse or a run that failed converts nothing. +func convertRecorded(input string, to convert.Format) (int, error) { + if fromFormat != "" && fromFormat != "sysml" { + return 0, fmt.Errorf("-record-run records into SysML notation; -from %s does not apply", fromFormat) + } + sess := newSession() + report, err := sess.LoadPathsReport([]string{input}) + if err != nil { + return 0, err + } + writeLines(os.Stderr, report.Loaded) + writeLines(os.Stderr, report.Found) + writeLines(os.Stderr, report.Declared) + if report.Errors { + return 0, fmt.Errorf("%s did not analyse cleanly; nothing was converted", input) + } + // The objects -instantiate names are materialized first, so a run named on + // one has it to record. + for _, name := range modelChecks.instantiate { + created, err := sess.InstantiateReport(name) + if err != nil { + return 0, err + } + writeLines(os.Stderr, created.Lines) + if len(created.FeatureValueErrors) > 0 { + writeLines(os.Stderr, created.FeatureValueErrors) + return 0, fmt.Errorf("%s did not materialize cleanly; nothing was converted", name) + } + } + for _, invocation := range modelChecks.records { + verdict := modelChecks.record(sess, invocation) + writeLines(os.Stderr, verdict.Lines) + if verdict.Status != repl.VerdictHolds { + return 0, fmt.Errorf("%s: the run was not recorded; nothing was converted", invocation) + } + } + opts, err := convertOptions(convert.FormatSysML, to) + if err != nil { + return 0, err + } + out, tolerated, err := convert.ConvertTolerantWith(repl.SessionOrigin, []byte(sess.Text()), convert.FormatSysML, to, opts) + if err != nil { + return 0, err + } + if tolerated != nil { + for _, line := range strings.Split("warning: "+tolerated.Error(), "\n") { + fmt.Fprintln(os.Stderr, line) + } + } + if outputPath != "" { + return exitHolds, writeConversion(outputPath, out, to) + } + _, err = os.Stdout.Write(out) + return exitHolds, err +} + +// convertOptions are the conversion settings -id asks for, refusing it for a +// direction it does not apply to. +func convertOptions(from, to convert.Format) (convert.Options, error) { + opts := convert.Options{} + if idForm == "" { + return opts, nil + } + if from != convert.FormatSysML || (to != convert.FormatTurtle && to != convert.FormatAPIJSON) { + return opts, fmt.Errorf("-id applies to -convert ttl or api-json from SysML notation") + } + form, ok := export.ParseIDForm(idForm) + if !ok { + return opts, fmt.Errorf("-id wants qualified or uuid, not %q", idForm) + } + opts.ID = form + return opts, nil +} + // writeConversion writes converted output to a file and reports it. func writeConversion(path string, out []byte, to convert.Format) error { replaced, err := export.WriteFile(path, out) diff --git a/cmd/sysml/main.go b/cmd/sysml/main.go index a488d5d40..c62067464 100644 --- a/cmd/sysml/main.go +++ b/cmd/sysml/main.go @@ -455,6 +455,14 @@ func runCLI() int { fmt.Fprintln(os.Stderr, "sysml: -layout accompanies -convert of a SysML v1 model; write `sysml model.xmi -convert sysml -layout model_mtip.xml`") return 2 } + if flagGiven("record-into") && len(modelChecks.records) == 0 { + fmt.Fprintln(os.Stderr, "sysml: -record-into accompanies -record-run; write `sysml model.sysml -record-run \"Pkg::Case\" -record-into Pkg::Log`") + return 2 + } + if flagGiven("record-into") && modelChecks.recordInto == "" { + fmt.Fprintln(os.Stderr, "sysml: -record-into needs a package name; write `sysml model.sysml -record-run \"Pkg::Case\" -record-into Pkg::Log`") + return 2 + } if flagGiven("layout") && layoutPath == "" { fmt.Fprintln(os.Stderr, "sysml: -layout is empty; name the MTIP export to lay the migrated views out from") return 2 @@ -582,7 +590,7 @@ func runCLI() int { fmt.Fprintln(os.Stderr, "sysml: -convert and -query are mutually exclusive") return 2 } - if modelChecks.requested() { + if modelChecks.requested() && !modelChecks.recordsOnly() { return refuse(modelChecks, "-convert writes the model out and decides nothing about it; check it in its own run") } @@ -590,6 +598,15 @@ func runCLI() int { fmt.Fprintln(os.Stderr, "sysml: -convert, -render and -render-document each write a document out; ask for one per run") return 2 } + if modelChecks.recordsOnly() { + if message := modelChecks.boundsMisuse(); message != "" { + fmt.Fprintf(os.Stderr, "sysml: %s\n", message) + return 2 + } + if status := resolveRunBounds(); status != 0 { + return status + } + } return runConvertExit(args) } @@ -628,12 +645,15 @@ func runCLI() int { case modelChecks.jsonOut && !modelChecks.checksOnly(): fmt.Fprintln(os.Stderr, "sysml: -render-document writes a document, not JSON; -json reports checks") return 2 - case modelChecks.requested() && !modelChecks.instantiatesOnly(): + case modelChecks.requested() && !modelChecks.instantiatesOnly() && !modelChecks.recordsOnly(): return refuse(modelChecks, "-render-document writes a document out and decides nothing about the model; check it in its own run") case len(evalExprs) > 0 || fromFormat != "": fmt.Fprintln(os.Stderr, "sysml: -render-document cannot be combined with -eval or -from") return 2 + case modelChecks.recordsOnly() && modelChecks.boundsMisuse() != "": + fmt.Fprintf(os.Stderr, "sysml: %s\n", modelChecks.boundsMisuse()) + return 2 } if status := resolveRunBounds(); status != 0 { return status @@ -681,6 +701,7 @@ func resolveRunBounds() int { // the run bounds resolved at startup. func newSession() *repl.Session { sess := repl.NewSession() + sess.SetToolVersion("sysml " + Version) if err := sess.SetBudgets(budgets); err != nil { // Unreachable: budgets are validated in main before any session exists. fmt.Fprintln(os.Stderr, errPrefix, err) diff --git a/cmd/sysml/record_test.go b/cmd/sysml/record_test.go new file mode 100644 index 000000000..9fa142dbf --- /dev/null +++ b/cmd/sysml/record_test.go @@ -0,0 +1,358 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/Open-MBEE/OpenSysML/internal/exec/runtime" +) + +// recordModel declares the cases the -record-run tests record: one case whose +// subject the model binds, and one without a subject. +const recordModel = `package Demo { + private import ScalarValues::*; + part def Probe { + attribute t : Real = 3.0; + } + part probe : Probe; + analysis def Check { + subject s : Probe; + in gain : Real; + out x : Real = s.t + gain; + } + analysis timed : Check { subject s = probe; in gain = 2.0; } +}` + +func writeRecordModel(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "model.sysml") + if err := os.WriteFile(path, []byte(recordModel), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +// TestRecordRunReportsWhatItRecorded runs a case and records it, and the run +// reports both the case's verdict and the element the record became. +func TestRecordRunReportsWhatItRecorded(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + cmd := exec.Command(binary, source, "-record-run", "Demo::timed") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("-record-run: %v\n%s", err, out) + } + for _, want := range []string{"x = 5.0", "recorded Records::timed_run1"} { + if !strings.Contains(string(out), want) { + t.Errorf("-record-run output is missing %q:\n%s", want, out) + } + } +} + +// TestRecordRunConvertWritesTheRecords converts the session text a -record-run +// produced: the file it writes carries the Records package, validates clean as +// a model of its own, and a second run records _run2 into it. +func TestRecordRunConvertWritesTheRecords(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + dir := t.TempDir() + first := filepath.Join(dir, "first.sysml") + cmd := exec.Command(binary, source, "-record-run", "Demo::timed", "-convert", "sysml", "-o", first) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("record + convert: %v\n%s", err, out) + } + written, err := os.ReadFile(first) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"part timed_run1 : TimedRun", `caseName = "Demo::timed"`, "attribute :>> x = 5.0"} { + if !strings.Contains(string(written), want) { + t.Errorf("converted model is missing %q:\n%s", want, written) + } + } + if out, err := exec.Command(binary, first, "-validate").CombinedOutput(); err != nil { + t.Fatalf("converted model does not validate: %v\n%s", err, out) + } + second := filepath.Join(dir, "second.sysml") + cmd = exec.Command(binary, first, "-record-run", "Demo::timed", "-convert", "sysml", "-o", second) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("re-record + convert: %v\n%s", err, out) + } + written, err = os.ReadFile(second) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(written), "part timed_run2 : TimedRun") { + t.Errorf("converted model is missing the renumbered run:\n%s", written) + } +} + +// TestRecordRunSweepRecordsEveryRow sweeps one input of a case and records one +// run per value the range steps through. +func TestRecordRunSweepRecordsEveryRow(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + out := filepath.Join(t.TempDir(), "swept.sysml") + cmd := exec.Command(binary, source, "-record-run", "Demo::timed", "-sweep", "gain=1..3", "-convert", "sysml", "-o", out) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("record + sweep + convert: %v\n%s", err, output) + } + written, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"part timed_run1 :", "part timed_run2 :", "part timed_run3 :", `attribute :>> kind = "sweep"`, "attribute :>> iteration = 3"} { + if !strings.Contains(string(written), want) { + t.Errorf("converted sweep is missing %q:\n%s", want, written) + } + } +} + +// TestRecordRunMonteCarloRecordsEveryRun samples a MonteCarlo case under -runs +// and records each run it made. +func TestRecordRunMonteCarloRecordsEveryRun(t *testing.T) { + binary := buildCLI(t) + model := `package MC { + private import ScalarValues::*; + private import RandomFunctions::*; + part def Probe { + attribute t : Real; + action settle { first start; then assign t := uniform(1.0, 5.0); then done; } + } + individual def probe :> Probe; + analysis def Mc :> Simulation::MonteCarlo { + subject analysed : Probe; + perform action run ::> analysed.settle; + attribute :>> observed : Real = analysed.t; + return Mean : Real = mean; + } +} +` + source := filepath.Join(t.TempDir(), "model.sysml") + if err := os.WriteFile(source, []byte(model), 0o644); err != nil { + t.Fatal(err) + } + out := filepath.Join(t.TempDir(), "runs.sysml") + cmd := exec.Command(binary, source, "-instantiate", "MC::probe", "-record-run", "MC::Mc MC::probe", "-runs", "2", "-seed", "7", "-convert", "sysml", "-o", out) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("record + runs + convert: %v\n%s", err, output) + } + written, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"part Mc_run1 :", "part Mc_run2 :", `attribute :>> kind = "runs"`, "attribute :>> iteration = 2"} { + if !strings.Contains(string(written), want) { + t.Errorf("converted MonteCarlo records are missing %q:\n%s", want, written) + } + } +} + +// TestRecordRunRenderDocumentSeesTheRecords renders a document over the model +// after a run was recorded, so the document's table lists the record it made. +func TestRecordRunRenderDocumentSeesTheRecords(t *testing.T) { + binary := buildCLI(t) + model := `package Demo { + private import ScalarValues::*; + private import DocumentQueries::*; + part def Probe { + attribute t : Real = 3.0; + } + part probe : Probe; + analysis def Check { + subject s : Probe; + in gain : Real; + out x : Real = s.t + gain; + } + analysis timed : Check { subject s = probe; in gain = 2.0; } + calc def RecordedRuns :> DocumentQueries::Query { + in names : String[1..*]; + Project( + source = Named(qualifiedName = names), + properties = ("name", "kind") + ) + } + part def Log :> DocumentQueries::Document { + attribute redefines title = "Run Log"; + part runs : Table { + calc rows : RecordedRuns { in names = "Records::timed_run1"; } + } + } +} +` + source := filepath.Join(t.TempDir(), "model.sysml") + if err := os.WriteFile(source, []byte(model), 0o644); err != nil { + t.Fatal(err) + } + out := filepath.Join(t.TempDir(), "report.md") + cmd := exec.Command(binary, source, "-record-run", "Demo::timed", "-render-document", "Demo::Log", "-o", out) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("record + render-document: %v\n%s", err, output) + } + written, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(written), "timed\\_run1") && !strings.Contains(string(written), "timed_run1") { + t.Errorf("rendered document does not list the record:\n%s", written) + } +} + +// TestRecordRunChecksStayExclusive keeps the guard that a decision and a render +// do not share a run: -render-document with -analysis is still refused. +func TestRecordRunChecksStayExclusive(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + cmd := exec.Command(binary, source, "-analysis", "Demo::timed", "-render-document", "Demo::Doc", "-o", filepath.Join(t.TempDir(), "x.md")) + if out, err := cmd.CombinedOutput(); err == nil { + t.Fatalf("-analysis + -render-document succeeded:\n%s", out) + } else if !strings.Contains(string(out), "-render-document writes a document") { + t.Errorf("unexpected refusal:\n%s", out) + } +} + +// TestRecordIntoWithoutRecordRunRefused rejects -record-into on its own. +func TestRecordIntoWithoutRecordRunRefused(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + cmd := exec.Command(binary, source, "-record-into", "Demo::Log", "-convert", "sysml") + if out, err := cmd.CombinedOutput(); err == nil { + t.Fatalf("-record-into alone succeeded:\n%s", out) + } else if !strings.Contains(string(out), "-record-into accompanies -record-run") { + t.Errorf("unexpected refusal:\n%s", out) + } +} + +// TestRecordRunCommandCarriesTheRunFlags records the flags the session ran +// under in the record's provenance command. +func TestRecordRunCommandCarriesTheRunFlags(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + out := filepath.Join(t.TempDir(), "saved.sysml") + cmd := exec.Command(binary, source, "-record-run", "Demo::timed", + "-clock-step", "0.5", "-instantiate", "Demo::probe", "-convert", "sysml", "-o", out) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("record with run flags: %v\n%s", err, output) + } + written, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`-clock-step 0.5`, `-instantiate \"Demo::probe\"`} { + if !strings.Contains(string(written), want) { + t.Errorf("recorded command is missing %q:\n%s", want, written) + } + } +} + +// TestRecordIntoEmptyRefused rejects -record-into given without a package. +func TestRecordIntoEmptyRefused(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + cmd := exec.Command(binary, source, "-record-run", "Demo::timed", "-record-into=", "-convert", "sysml") + if out, err := cmd.CombinedOutput(); err == nil { + t.Fatalf("-record-into= succeeded:\n%s", out) + } else if !strings.Contains(string(out), "-record-into needs a package name") { + t.Errorf("unexpected refusal:\n%s", out) + } +} + +// TestRecordRunConvertHonoursID converts the session a -record-run produced +// with -id applied to it, as -convert honours it on a file. +func TestRecordRunConvertHonoursID(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + out := run(t, binary, source, "-record-run", "Demo::timed", "-convert", "ttl", "-id", "uuid") + uuid := regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`) + if !uuid.MatchString(out) { + t.Errorf("no UUID ids in the converted records:\n%s", out) + } +} + +// TestRecordRunConvertRefusesFrom refuses -from on a recorded conversion: the +// input is still SysML notation. +func TestRecordRunConvertRefusesFrom(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + cmd := exec.Command(binary, source, "-record-run", "Demo::timed", "-from", "kerml", "-convert", "sysml") + if out, err := cmd.CombinedOutput(); err == nil { + t.Fatalf("-record-run + -from kerml succeeded:\n%s", out) + } else if !strings.Contains(string(out), "-from") { + t.Errorf("unexpected refusal:\n%s", out) + } +} + +// TestRecordRunBoundsApplyToConvert refuses the run bounds runChecks refuses +// when -convert shares the run, exiting as the same misuse does. +func TestRecordRunBoundsApplyToConvert(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + out, code := exitCode(t, exec.Command(binary, source, + "-record-run", "Demo::timed", "-sweep", "gain=1..3", "-samples", "2", "-seed", "7", "-convert", "sysml")) + if code != 2 || !strings.Contains(out, "-samples draws values for a sweep it does not run") { + t.Errorf("-samples with -record-run -convert: code %d:\n%s", code, out) + } +} + +// TestRecordRunBoundsApplyToRenderDocument refuses a -runs/-sweep conflict on +// a -render-document run the same as runChecks does. +func TestRecordRunBoundsApplyToRenderDocument(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + out, code := exitCode(t, exec.Command(binary, source, + "-record-run", "Demo::timed", "-runs", "3", "-seed", "7", "-sweep", "gain=1..3", "-render-document", "Demo::Doc")) + if code != 2 || !strings.Contains(out, "-runs runs an action; -sweep and -samples run an analysis case or calc") { + t.Errorf("-runs + -sweep with -record-run -render-document: code %d:\n%s", code, out) + } +} + +// TestRecordRunConvertHonoursRunBounds applies the run bounds the env vars +// ask for on a recorded conversion, as runChecks does on a plain -sweep. +func TestRecordRunConvertHonoursRunBounds(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + t.Setenv(runtime.MaxSweepRunsEnvVar, "notanumber") + out, code := exitCode(t, exec.Command(binary, source, "-record-run", "Demo::timed", "-convert", "sysml")) + if code != 2 || !strings.Contains(out, runtime.MaxSweepRunsEnvVar) { + t.Errorf("an unusable %s went unreported: code %d:\n%s", runtime.MaxSweepRunsEnvVar, code, out) + } +} + +// TestRecordRunConvertHonoursTheSweepBudget caps a recorded sweep the same +// way a plain -sweep is capped. +func TestRecordRunConvertHonoursTheSweepBudget(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + t.Setenv(runtime.MaxSweepRunsEnvVar, "2") + out, code := exitCode(t, exec.Command(binary, source, + "-record-run", "Demo::timed", "-sweep", "gain=1..5", "-convert", "sysml")) + if code == 0 || !strings.Contains(out, runtime.MaxSweepRunsEnvVar) { + t.Errorf("a sweep over the budget converted: code %d:\n%s", code, out) + } +} + +// TestRecordRunConvertRefusesIrrelevantFlags refuses the flags a recorded +// conversion does not honour, before the model is even loaded. +func TestRecordRunConvertRefusesIrrelevantFlags(t *testing.T) { + binary := buildCLI(t) + source := writeRecordModel(t) + for name, args := range map[string][]string{ + "branch input": {"flexo://proj-1/main", "-record-run", "Demo::timed", "-convert", "sysml"}, + "branch output": {source, "-record-run", "Demo::timed", "-convert", "ttl", "-o", "flexo://proj-1/main"}, + "sync-state": {source, "-record-run", "Demo::timed", "-convert", "ttl", "-sync-state", filepath.Join(t.TempDir(), "s.ttl")}, + "migration report": {source, "-record-run", "Demo::timed", "-convert", "sysml", "-migration-report", filepath.Join(t.TempDir(), "r.json")}, + "migration results": {source, "-record-run", "Demo::timed", "-convert", "sysml", "-migration-results", filepath.Join(t.TempDir(), "r.txt")}, + "layout": {source, "-record-run", "Demo::timed", "-convert", "sysml", "-layout", filepath.Join(t.TempDir(), "l.json")}, + } { + t.Run(name, func(t *testing.T) { + out, code := exitCode(t, exec.Command(binary, args...)) + if code != 2 || !strings.Contains(out, "-record-run converts the recorded session model") { + t.Errorf("%v: code %d:\n%s", args, code, out) + } + }) + } +} diff --git a/cmd/sysml/render.go b/cmd/sysml/render.go index cb8b1fff9..5a225edf5 100644 --- a/cmd/sysml/render.go +++ b/cmd/sysml/render.go @@ -232,6 +232,15 @@ func loadRenderingModel(files []string) (*repl.Session, error) { fmt.Fprintf(os.Stderr, "%s: materialization is bounded; not every feature value was materialized\n", name) } } + // The runs -record-run names are made and written into the model before a + // document is rendered, so its queries see the records. + for _, invocation := range modelChecks.records { + verdict := modelChecks.record(sess, invocation) + writeLines(os.Stderr, verdict.Lines) + if verdict.Status != repl.VerdictHolds { + return nil, fmt.Errorf("%s: the run was not recorded; nothing was rendered", invocation) + } + } return sess, nil } diff --git a/cmd/sysml/usage.go b/cmd/sysml/usage.go index 4369a6bbf..2bbf857e7 100644 --- a/cmd/sysml/usage.go +++ b/cmd/sysml/usage.go @@ -534,6 +534,8 @@ func registerFlags(fs *flag.FlagSet) { fs.Var(&modelChecks.satisfy, "satisfy", "Evaluate every satisfaction assertion, or with -satisfy= those the named element states, and exit (repeatable)") fs.Var(&modelChecks.calcs, "calc", "Invoke this calculation and report its result, as -calc \"Fall(3, 4)\" (repeatable)") fs.Var(&modelChecks.analyses, "analysis", "Run this analysis or verification case and report its outputs and verdict, as -analysis \"Pkg::Case(3.0) Pkg::part\" (repeatable)") + fs.Var(&modelChecks.records, "record-run", "Run this analysis case as -analysis does and record the run into the model as AnalysisRecords elements, one record per -sweep value or -runs run (repeatable)") + fs.StringVar(&modelChecks.recordInto, "record-into", "", "Record -record-run runs into this package instead of a Records package beside the case's") fs.Var(&modelChecks.queries, "run-query", "Execute this document query and report its rows, as -run-query \"Heavy root=telescope\" (repeatable)") fs.Var(&modelChecks.instantiate, "instantiate", "Create an object of this definition or usage before the checks, so a verdict is about it (repeatable)") fs.BoolVar(&modelChecks.jsonOut, "json", false, "Report checks as one JSON document rather than as lines") @@ -649,6 +651,8 @@ func optionGroups() []usage.OptionGroup { usage.Opt("satisfy", "[=]"), usage.Opt("calc", ""), usage.Opt("analysis", ""), + usage.Opt("record-run", ""), + usage.Opt("record-into", ""), usage.Opt("run-query", ""), usage.Opt("instantiate", nameArg), usage.Opt("json", ""), diff --git a/docs/internals/architecture.md b/docs/internals/architecture.md index de7d3849c..137507962 100644 --- a/docs/internals/architecture.md +++ b/docs/internals/architecture.md @@ -648,7 +648,7 @@ See [the guide](../guide/) for VS Code configuration. | Component | Status | |-----------|--------| -| Lexer/Parser (structural + behavioral) | ✅ Operational (98/98 stdlib clean - see [conformance gate](../../internal/workspace/libs/stdlib_conformance_test.go)) | +| Lexer/Parser (structural + behavioral) | ✅ Operational (105/105 stdlib clean - see [conformance gate](../../internal/workspace/libs/stdlib_conformance_test.go)) | | Symbol resolution & type system | ✅ Complete | | Validation passes (syntax → constraints) | ✅ Complete | | Expression evaluator & instance model (Tiers 1-3) | ✅ Complete | @@ -662,7 +662,7 @@ See [the guide](../guide/) for VS Code configuration. | Standard library bundling | ✅ Complete | | LSP server implementation | ✅ Complete | -**Parser coverage:** 98/98 bundled library files parse cleanly — the 94 official SysML v2 standard library files and four non-normative OpenSysML extensions: `OpenSysML Libraries/OpenSysMLMathFunctions.kerml`, `OpenSysML Libraries/DocumentQueries.sysml`, `OpenSysML Libraries/IdentityMetadata.sysml` and `OpenSysML Libraries/OOSEM.sysml`. Conformance verified by [stdlib_conformance_test.go](../../internal/workspace/libs/stdlib_conformance_test.go). Grammar reference available at [OMG Xtext grammar](https://github.com/Systems-Modeling/SysML-v2-Pilot-Implementation/tree/master/org.omg.kerml.xtext/src/org/omg/kerml/xtext). +**Parser coverage:** 105/105 bundled library files parse cleanly — the 94 official SysML v2 standard library files and the 11 non-normative OpenSysML extensions: `OpenSysML Libraries/OpenSysMLMathFunctions.kerml`, `DocumentQueries.sysml`, `IdentityMetadata.sysml`, `OOSEM.sysml`, `DiagramLayout.sysml`, `MOSA.sysml`, `RandomFunctions.kerml`, `Simulation.sysml`, `StateSpaceIntegration.sysml`, `Stochastic.sysml` and `AnalysisRecords.sysml`. Conformance verified by [stdlib_conformance_test.go](../../internal/workspace/libs/stdlib_conformance_test.go). Grammar reference available at [OMG Xtext grammar](https://github.com/Systems-Modeling/SysML-v2-Pilot-Implementation/tree/master/org.omg.kerml.xtext/src/org/omg/kerml/xtext). --- @@ -675,8 +675,8 @@ New grammar features require a **four-layer test contract** to ensure correctnes #### 1. Conformance Gate - **Purpose:** Ensure stdlib continues to parse cleanly - **Location:** `internal/workspace/libs/stdlib_conformance_test.go` -- **Test:** `TestStdlibConformance` loads all 96 bundled library files -- **Acceptance:** 98/98 files parse without errors +- **Test:** `TestStdlibConformance` loads all 105 bundled library files +- **Acceptance:** 105/105 files parse without errors - **Allowlist:** `testdata/stdlib_known_failures.txt` (currently empty) - **Failure mode:** Regression breaks previously-working stdlib files diff --git a/docs/manual/README.md b/docs/manual/README.md index 7461b1cd7..a95b24fc2 100644 --- a/docs/manual/README.md +++ b/docs/manual/README.md @@ -36,11 +36,16 @@ was produced by it. graded reports, from three flat requirements and their satisfiers to a multi-team program with derivation chains, verdicts, coverage gaps and a grouped matrix, each with its source and rendered output -10. [Limitations and troubleshooting](troubleshooting.md) — the typed error +10. [Recording analysis runs](recording-analysis-runs.md) — writing the runs a + case makes into the model as `AnalysisRecords` elements, and reading them + back with document queries +11. [Limitations and troubleshooting](troubleshooting.md) — the typed error catalog and the current limitations The document-query vocabulary is a **non-normative OpenSysML extension** — it is not part of the OMG SysML v2 or KerML standard. Models that use it remain standard SysML v2: the vocabulary is ordinary `calc def`s and `part def`s from the bundled `DocumentQueries` library package, and using it does not alter the -language's semantics. +language's semantics. The bundled `AnalysisRecords` library is the same kind +of extension: the declarations `%record`/`-record-run` writes a run into the +model with (see [Recording analysis runs](recording-analysis-runs.md)). diff --git a/docs/manual/query-kinds.md b/docs/manual/query-kinds.md index 78787b201..b8ea0920f 100644 --- a/docs/manual/query-kinds.md +++ b/docs/manual/query-kinds.md @@ -75,6 +75,13 @@ Over gRPC, `RunDocumentQuery` answers an object row in the `object` arm of `DocumentValue` and a verdict row in the `verdict` arm; see [Native document queries and rendering over gRPC](../reference/api.md#native-document-queries-and-rendering-over-grpc). +A run `%record`/`-record-run` makes is **model** rows, not object rows: the +record is written into the model as elements annotated +`@AnalysisRecords::RecordedRun`, so `WhereMetadata` finds each one and +`WhereFeature`/`Project`/`OrderBy` read the values it bound — `caseName`, +`kind`, `iteration`, and a property per input and output. See +[Recording analysis runs](recording-analysis-runs.md). + ## Runtime state and event queries Object rows tell you *what an object holds*; three more operations tell you diff --git a/docs/manual/recording-analysis-runs.md b/docs/manual/recording-analysis-runs.md new file mode 100644 index 000000000..0e0e9c6e7 --- /dev/null +++ b/docs/manual/recording-analysis-runs.md @@ -0,0 +1,198 @@ +# Recording analysis runs + +`%record` at the prompt — `-record-run` on the command line — runs an analysis +case exactly as `%analysis`/`-analysis` does, reports the same verdict, and then +writes the run **into the model** as elements of the bundled `AnalysisRecords` +library: a record definition, one part per run carrying every input bound and +output produced, and provenance metadata stating when the run was made, by what +tool, with what command, and of which kind (`run`, `trade`, `sweep`, `runs` +or `sample`). + +The records are ordinary model elements, so a document query finds them and a +document renders them — a run log lives in the model beside the cases it +records. + +## Recording a run + +```sysml +package Demo { + private import ScalarValues::*; + part def Probe { attribute t : Real = 3.0; } + part probe : Probe; + + analysis def Check { + subject s : Probe; + in gain : Real; + out x : Real = s.t + gain; + } + analysis timed : Check { subject s = probe; in gain = 2.0; } +} +``` + +```text +%record Demo::timed +✓ Demo::timed + x = 5.0 + standing: value (observed: 1 run under reverse) + recorded Records::timed_run1 (Records::TimedRun) +``` + +The run goes into a `Records` package beside the package enclosing the case's — +a top-level `Records` when the case is `Demo::timed`, `A::Records` for a case in +a package `A::Descent` declares — and into a record definition named for the case +(`TimedRun`), specializing `AnalysisRecords::AnalysisRun`. `%record ... into +` names the package instead. The record part holds a redefinition for +each input and output of the case, plus the `caseName`, `kind`, `'objective'` +and `subjectName`/`subject` features `AnalysisRun` declares — `iteration` +only on a sweep or sample's records — and a `ref` to its subject; verdicts and +evaluations a trade study or verification made become +`VerdictRecord`/`EvaluationRecord` parts under `verdicts`/`evaluations`. A +parameter declared `inout` is one member — the record carries the value the +run left in it — plus a `In` companion carrying the value it was bound +with, just as a quantity carries a `Unit` companion naming the unit. A +verification case's record also carries what its body decided — the `verdict` +attribute (`"pass"`, `"fail"`, `"inconclusive"` or `"error"`) — and one +`VerdictRecord` row apiece for the body's verdict (`kind` `"verification"`) +and each subcase's (`kind` `"subcase"`): + +```sysml +package Records { + part def TimedRun :> AnalysisRecords::AnalysisRun { + attribute :>> caseName default = "Demo::timed"; + attribute gain : ScalarValues::Real; + attribute x : ScalarValues::Real; + } + part timed_run1 : TimedRun { + @AnalysisRecords::RecordedRun { + runAt = "2026-09-24T02:21:14Z"; + tool = "sysml dev"; + command = "%record Demo::timed"; + kind = "run"; + } + attribute :>> caseName = "Demo::timed"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "undecided"; + ref :>> 'subject' = Demo::probe; + attribute :>> subjectName = "Demo::probe"; + attribute :>> gain = 2.0; + attribute :>> x = 5.0; + } +} +``` + +The definition's `caseName` marks the case it records. A sibling case of the +same name (`Demo::B::check` beside `Demo::A::check`) gets a definition of its +own, named from its owner (`B_checkRun`), rather than taking the first case's +over. + +Recording a second run of the same case reuses the definition and numbers the +part on (`timed_run2`) — including after the model was saved and reloaded. + +## Sweeps and Monte Carlo + +On the command line the run a `-record-run` makes takes the same bounds the +matching check takes: with `-sweep` the case runs once per row as `-sweep` +makes it, and one record per row is written (`kind = "sweep"`, `iteration` +the row); with `-runs ` and `-seed` a `Simulation::MonteCarlo` case is +sampled as `-runs` does and each seeded run is recorded (`kind = "runs"`), +with the sample's conclusion — the statistics, the result and the checks that +are the sample's — recorded once more beside them (`kind = "sample"`): + +```bash +$ sysml model.sysml -record-run "Demo::timed" -sweep "gain=1..3" -convert sysml -o saved.sysml +sweep Demo::timed — 3 run(s) + recorded 3 runs as Records::timed_run1 … timed_run3 +wrote saved.sysml (sysml, 2278 bytes) +``` + +`-convert sysml` writes the session text the records joined — the model plus +the `Records` package — formatted; loading `saved.sysml` and recording again +produces `timed_run4`. `-record-into ` names the records' package; +`-render-document` composes the same way, recording first so the document's +queries see the records. A failed sweep row is skipped and counted; a sampled +run whose declared output could not be read is not recorded — `run N not +recorded: ` names it. An output bound to a statistic of the sample +(`mean`, `deviation`, …) is the sample's and appears only on its record. An +output whose binding draws a random value is evaluated afresh on every read, +as the runtime's checks and conclusion do: its recorded value is one such +evaluation, made without moving the draws the sample's runs and conclusion see. +A run that fails records nothing and leaves the +model untouched — the record submission is atomic: the diagnostics it produced +are reported and the model is as it was. + +## Reading the records + +The `@AnalysisRecords::RecordedRun` metadata makes every record findable by +`WhereMetadata`, and its features are ordinary values `WhereFeature` and +`Project` read — see [Which query is which](query-kinds.md#object-rows-and-verdict-rows). +This run goes into `Demo::Log` instead, so the query's `root=Demo` subtree +contains it: + +```text +%record Demo::timed into Demo::Log +✓ Demo::timed + x = 5.0 + standing: value (observed: 1 run under reverse) + recorded Demo::Log::timed_run1 (Demo::Log::TimedRun) +``` + +```sysml +calc def TimedRuns :> DocumentQueries::Query { + in root : Element; + Project(source = WhereFeature( + source = WhereMetadata( + source = Descendants(source = root, maxDepth = 10), + 'metadata' = "AnalysisRecords::RecordedRun"), + 'feature' = "caseName", + operator = "=", + value = "Demo::timed"), + properties = ("name", "gain", "x")) +} +``` + +```text +%run-query TimedRuns root=Demo +✓ Query Demo::TimedRuns returned 1 row + Columns: name, gain, x + Row 1: Demo::Log::timed_run1 + name = "timed_run1" + gain = 2.0 + x = 5.0 +``` + +## The AnalysisRecords library + +`internal/workspace/libs/stdlib/OpenSysML Libraries/AnalysisRecords.sysml`, a +non-normative OpenSysML extension bundled like `DocumentQueries`, declares the +vocabulary the records are written in: + +- `RecordedRun` — the metadata annotation a record carries: `runAt` (the UTC + timestamp), `tool`, `command`, `kind` (`"run"`, `"trade"`, `"sweep"`, + `"runs"` or `"sample"`). +- `AnalysisRun` — the record definition's supertype: `caseName`, `kind`, + `'objective'` (the run's objective verdict, `"undecided"` when the case + declares none), `iteration` (its position in a sweep or sample), `'subject'` + and `subjectName` (the object it ran on), `verdict` (what a verification + case's body decided — `"pass"`, `"fail"`, `"inconclusive"` or `"error"`; + unset for a case that is not a verification), and `verdicts`/`evaluations`. +- `VerdictRecord` — one check a run made: `kind` (`"objective"`, `"assertion"`, + `"verification"` or `"subcase"`), `name`, `status`, `detail` (the violated + condition, or why an undecided check could not be evaluated). +- `EvaluationRecord` — one trade-study evaluation: `function`, `alternative`, + `score`, `result`, `selected`, `tied`, `error`. + +## Limitations + +- A value that is not a scalar, enum literal, quantity or resolvable reference + is recorded as its printed String. +- A quantity is recorded as its Real magnitude plus a `Unit` String + companion naming the unit; an `inout` parameter as the value the run left + plus a `In` companion carrying the value it was bound with. +- An input or output left unset is declared on the record but not redefined. +- A member's type is settled from the values the runs supply; Integer and + Real are one numeric family for it — either way the member is Real (an + Integer literal is valid under it). A recorded value that is a scalar-valued + enum literal keeps the literal (`= Grade::high`), not the scalar it equals. +- Records join only a package whose header is a plain `package Name`. +- Recording is exposed at the REPL and CLI; the gRPC surface does not expose + it yet. diff --git a/docs/project/pilot-differential-baseline.json b/docs/project/pilot-differential-baseline.json index aa3a97c66..9a0b2695b 100644 --- a/docs/project/pilot-differential-baseline.json +++ b/docs/project/pilot-differential-baseline.json @@ -61,7 +61,7 @@ "dir": "examples", "origin": "ours", "files": 43, - "digest": "sha256:aa5a980eb63371fbea1ac5c52904d8639913da1b0486e544b1c3d3a8af00680a" + "digest": "sha256:18b2fbd67ec48deea2777902bd8a7f6bbb238f44107f28c52a7cda2ef0ff9e83" }, { "name": "probes", @@ -71,7 +71,7 @@ "digest": "sha256:b0153c55bbfcdacabab911725dc44e0e7f4d3a501a281b1f3f2a13cda19737c6" } ], - "recorded": "2026-09-22" + "recorded": "2026-09-24" }, "totals": { "files": 378, diff --git a/docs/reference/cli.md b/docs/reference/cli.md index a4be5b836..a37285684 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -258,6 +258,8 @@ written in, so the verdicts are about that object: | `-instantiate ` | Creates an object first, so the verdicts are about it; with `-run-query` or `-render-document`, so the query reads it ([Objects the session holds](../manual/query-cookbook.md#objects-the-session-holds)). Under `-schedule explore`, `-engine check`, `smt` or `all` each run creates an object of the declaration of its own before its behaviors start, one per `-instantiate` as the session holds one per `-instantiate`, which a `-state` or `-action` named alone attaches to and a path such as `Mission::mission.vehicle` walks into ([Objects an exploration runs on](#objects-an-exploration-runs-on)) | | `-calc "()"` | Invokes a calculation and reports what it computed | | `-analysis "[()] [object]"` | Runs an analysis or [verification](#verification-case-verdicts) case — a [trade study](#trade-studies) included — and reports its `out` and `return` values with their units, then the verdict of its `objective` — `satisfied`, `not satisfied` with the violated condition, or `undecided` with the reason — as `%analysis` does. An objective typed by a requirement def binds the def's subject as a requirement usage does (`subject = ship;`, `subject s = ship;` or `subject :>> s = ship;`); one binding none checks the case's result, the library's default for it, and is `undecided` naming the type when that result is not of the subject's type. Arguments bind the case's `in` parameters, positionally (`Pkg::Case(3.0)`) or by name (`Pkg::Case(limit = 3.0)`); the object, one `-instantiate` created and named as `-state` names its performer, is the case's `subject`. A usage that binds its subject (`subject s = ship;`) needs no object; a definition, or a usage that binds none, is refused by name without one. A verification case runs the same way and reports beside those verdicts the `VerdictKind` its body produced. Repeatable | +| `-record-run "[()] [object]"` | Runs an analysis case as `-analysis` does and records the run into the model as `AnalysisRecords` elements: a record definition named for the case in a `Records` package beside the case's, and one part under it per run carrying the inputs bound and the outputs produced, annotated `@AnalysisRecords::RecordedRun` with when the run was made, the tool and command, and its kind. With `-sweep` the case sweeps as `-sweep` makes it and one record per row is written (`kind = "sweep"`); with `-runs ` and `-seed` a `Simulation::MonteCarlo` case is sampled as `-runs` makes it and each run recorded (`kind = "runs"`). Composes with `-convert sysml -o`, which writes the session text the records joined, and with `-render-document`, whose queries then see the records; a run that fails records nothing and leaves the model untouched. Repeatable. See [Recording analysis runs](#recording-analysis-runs) | +| `-record-into ` | Records the `-record-run` runs into the package named instead of a `Records` package beside the case's; refused without `-record-run` | | `-run-query " [

=...]"` | Executes a document query and reports its rows, as `%run-query` does — including any computed `Column(name = "", expression = )` and relationship-derived `RelatedColumn(...)` projections evaluated per row. Each binding is written as `=`; a name binds the object `-instantiate` created under it while the run holds one (`#2` and `car.wheels[2]` bind an object by id and by path), and the element otherwise. A query over `Verdicts` reports each row as ` on : ` ([Which constraints and requirements hold](../manual/query-cookbook.md#which-constraints-and-requirements-hold)). The queries run after `-state`, `-action` and `-advance` have run, so `States`, `InState` and `Events` read where the run left the objects and, with `-trace`, what it recorded — a state row as `. in `, an event row as `t= .: ` ([Where the objects stand and what they did](../manual/query-cookbook.md#where-the-objects-stand-and-what-they-did)) | | `-action " [object]"` | Runs an action to completion and reports its outputs, on the object named as `-state` names its performer when one is; under `-schedule explore` each run performs it on an object of its own ([Objects an exploration runs on](#objects-an-exploration-runs-on)) | | `-state " [object]"` | Runs a state machine and reports where it settled. The object is one `-instantiate` created, named as `%state` names it: a usage's name, a feature path to a part it holds (`Fleet::driver.r`), or the id the report prints (`#2`). Naming the machine the object exhibits attaches to its running machine rather than performing it again (a definition exhibited as several usages is refused with the usages to name instead); naming a usage whose definition alone was instantiated says which usage to `-instantiate`. Under `-schedule explore` the object is one each run creates of its own: a definition or usage to instantiate, a path from one into a part it holds (`Mission::mission.vehicle`, `Fleet::fleet.rovers[2]`) or, named alone, the run's one `-instantiate` object exhibiting the machine ([Objects an exploration runs on](#objects-an-exploration-runs-on)) | @@ -1138,6 +1140,40 @@ draws the model's values from `n` too, on a stream of its own. Every draw is rec witness the checker writes, as `draw = ` lines, and `-schedule replay:` consumes them instead of drawing again. +## Recording analysis runs + +`-record-run` runs an analysis case as `-analysis` does — the same verdict lines +and the same bindings — and then writes the run into the model as elements of the +bundled `AnalysisRecords` library: a record definition named for the case +(`TimedRun` for `Demo::timed`) specializing `AnalysisRecords::AnalysisRun`, in a +`Records` package beside the case's enclosing package or the one `-record-into` +names, and one part per run carrying a redefinition of each input bound and +output produced, `caseName`, `kind` and `objective` — `iteration` on a sweep or +sample's records — a `ref` to the subject and +`@AnalysisRecords::RecordedRun` provenance metadata (`runAt`, `tool`, `command`, +`kind`). Verdicts a trade study or verification made become +`VerdictRecord`/`EvaluationRecord` parts under `verdicts`/`evaluations`. + +```bash +$ sysml model.sysml -record-run "Demo::timed" +✓ Demo::timed + x = 5.0 + standing: value (observed: 1 run under reverse) + recorded Records::timed_run1 (Records::TimedRun) +``` + +With `-sweep` the case runs once per row and one record per row is written +(`kind = "sweep"`); with `-runs ` and `-seed` a `Simulation::MonteCarlo` case +is sampled and each run recorded (`kind = "runs"`). Recording again of the same +case reuses the definition and numbers the parts on (`timed_run2`), including +after the model was saved and reloaded — the probe for the next number reads the +model. `-convert sysml` writes the session text the records joined, so +`-record-run ... -convert sysml -o saved.sysml` is how a recorded model is +saved; `-render-document` composes the same way, the records made before the +document's queries run. A run that fails, or a record submission that produces +diagnostics, records nothing and leaves the model untouched. See +[Recording analysis runs](../manual/recording-analysis-runs.md). + ## Comparing a migrated configuration with the tool's results A SysML v1 model migrated from a simulation tool (see diff --git a/docs/reference/repl-commands.md b/docs/reference/repl-commands.md index 784762e3a..1492e3a04 100644 --- a/docs/reference/repl-commands.md +++ b/docs/reference/repl-commands.md @@ -67,6 +67,7 @@ into the parts it holds (`car.fl.hub`, `#3.fl`, `car.wheels[2]`). | **Behavioral Execution** | | | `%calc [args...]` | Invoke calculation with arguments | | `%analysis [()] []` | Run an analysis or verification case — a calculation performed as an action — and print its `out` and `return` values with their units, then the verdict of its `objective` and of each `assert constraint` in its body: `satisfied`, `not satisfied` with the violated condition, or `undecided` with the reason the condition could not be evaluated. An objective typed by a requirement def binds the def's subject as a requirement usage does (`subject = ship;`, `subject s = ship;` or `subject :>> s = ship;`), reading the case's subject, parameters and steps' outputs; one binding none checks the case's result, the library's default for it (`Cases::Case::obj`), and is `undecided` naming the type when that result is not of the subject's type. Arguments in parentheses bind the case's `in` parameters, positionally (`%analysis An::Case(3.0)`) or by name (`%analysis An::Case(limit = 3.0)`), evaluated at the prompt like `%calc`'s; an [object reference](#object-references) after them is the case's `subject` (`%analysis An::CostAnalysis An::barge`). A usage that binds its subject (`subject s = ship;`) needs no object; a definition, or a usage that binds none, is refused by name without one, as `%requirement` refuses an unbound subject. A usage nested in a part runs as a feature of the object the session holds for that part (`%instantiate An::holder`, then `%analysis An::Holder::inner`). The body's `action` steps, sequenced by `then` or by declaration order, run as an action does, later steps reading earlier steps' outputs; a step that fails, a body that deadlocks or exhausts the step budget, a case that runs itself, and an `in` parameter with no argument and no default are errors naming the case; a case recursing without bound through a nested `analysis` step reports the depth limit on one line, its repeated frames collapsed to a count as `%calc`'s are. `%calc` refuses an analysis case and says to run it this way. A `verification def` or `verification` usage runs the same way and reports in addition the `VerdictKind` its body produced: `pass`/`fail` from the library's own `VerificationCases::PassIf` calculation, a `VerdictKind` literal the body bound, `inconclusive` for a body producing no verdict value, or `error` with the reason a body's run could not be carried out; each nested verification step is reported on its own line, marked `(subcase)`, the library stating no roll-up. A `TradeStudies::TradeStudy` runs the same way — the library's own expressions apply the case's `evaluationFunction` to each alternative the subject lists, in subject order, and `selectOne` returns the first scoring the objective's `best` — and the report adds each evaluation the run made of the case's own calc as a value, with `[selected]` on the alternative returned and `[tied]` on a later one scoring the same; an evaluation that fails is listed with its error and leaves the objective `undecided`, a subject listing no alternative or redeclared `[1]` and bound to several is a `multiplicity violation` ([Trade studies](../guide/06-behavior.md#trade-studies)) | +| `%record [()] [] [into ]` | Run an analysis case as `%analysis` does — the same verdict lines and the same bindings — and record the run into the model as `AnalysisRecords` elements: a record definition named for the case (`TimedRun` for `Demo::timed`) specializing `AnalysisRecords::AnalysisRun`, in a `Records` package beside the case's enclosing package or the one `into` names, and one part per run carrying a redefinition of each input bound and output produced, `caseName`, `kind`, `iteration`, a `ref` to the subject, `VerdictRecord`/`EvaluationRecord` parts for the verdicts and evaluations it made, and `@AnalysisRecords::RecordedRun` provenance metadata (`runAt`, `tool`, `command`, `kind`). The last line names what was recorded (`recorded Records::timed_run1 (Records::TimedRun)`); recording again of the same case numbers the part on (`timed_run2`), including after a `%save` and reload. The record submission is atomic: a run that fails, a record definition that does not specialize `AnalysisRecords::AnalysisRun`, a declared feature that collides with a reserved one, or a merge that would drop or produce diagnostics leaves the session's model untouched and reports the error. The records are ordinary elements — `%save` writes them with the model, and a `%run-query`/`%render-document` reads them by `WhereMetadata('metadata' = "AnalysisRecords::RecordedRun")`, `WhereFeature` and `Project`. Same recording as the CLI's [`-record-run`](cli.md#recording-analysis-runs); sweeps and Monte Carlo samples are recorded through `-record-run` with `-sweep`/`-runs`. See [Recording analysis runs](../manual/recording-analysis-runs.md) | | `%run-query [

=...]` | Execute a document query (a `calc def` specializing `DocumentQueries::Query`) and print its rows and projected cells. A projection lists declared property names and may add computed columns: `Column(name = "", expression = )` entries evaluated once per row over the row element's features, with arithmetic (`+`, `-`, `*`, `/`), string concatenation and `??` defaults for absent values. A column expression that fails (including a reference that resolves to no value and has no `??` default) fails the query with a typed error rather than producing an empty cell. A `RelatedColumn(name, relationshipKind, direction, maxDepth, aggregate = "list")` entry derives a column from the elements `RelatedElements` reaches from each row — the elements themselves as a multi-valued cell (`[a, b]`, `(none)` when there are none), their `"count"` or `"any"` of them — and `WhereFeature`/`OrderBy` read it by name ([Traceability matrix](../manual/query-cookbook.md#traceability-matrix)). Each binding is written as `=`; a name binds the object the session holds under it while it holds one (`car` after `%instantiate car`; `#2` and `car.wheels[2]` bind an object by id and by path), else the element it refers to, and anything else is evaluated as an expression. Over an object the operations read what the session holds: its parts as `OwnedElements`/`Descendants`, the values it holds now as its properties, and `Objects(type = "")` enumerates every object held that is of the type ([Objects the session holds](../manual/query-cookbook.md#objects-the-session-holds)). `Verdicts(source = )` checks the object behind each row as `%validate` does — the held object, or the element's declared object — and answers one verdict row per assertion about it and the objects it holds, printed as ` on : `; `Project`, `WhereFeature` and `OrderBy` read its `path`, `kind`, `verdict`, `condition`, `reason` and `verification` beside the assertion's own properties ([Which constraints and requirements hold](../manual/query-cookbook.md#which-constraints-and-requirements-hold)). `States(source = )` answers the state each object's machine is in now, one row per active leaf (`. in `, with `machine`, `name`, `statePath`, `region` and `enclosing`), `InState(name = "")` the held objects in that state, and `Events(source, kind, since, before)` the trace the session records — `%trace on` first, or the query is refused — as rows (`t= .: `, with `kind`, `time`, `state`, `from`, `to`, `target`, `event`, `payload`, `alternatives`, `taken`), `[since, before)` inclusive at the start and exclusive at the end ([Where the objects stand and what they did](../manual/query-cookbook.md#where-the-objects-stand-and-what-they-did)). A parameter left unbound takes its declared default (`in root : Element = telescope;`, `in pattern : String default "m";`) under the same rule: a name binds the element it refers to, anything else is evaluated once in the query that declared it, and a redefining parameter's default replaces the inherited one. Named query invocation and relationship traversal (`RelatedElements` over specialization, subsetting, redefinition, typing, connection, allocation, satisfaction and verification edges, outgoing or incoming) are supported. See the [query cookbook](../manual/query-cookbook.md) | | `%render-document [mermaid\|dot\|plantuml]` | Compile a document definition (a `part def` specializing `DocumentQueries::Document`), run its queries against the model — and against the objects the session holds, a parameter bound to a usage's name binding the object held under it while one is — and print the rendered Markdown. A document binds its queries' parameters in the model, so the name is the whole invocation apart from the optional diagram form: `mermaid` (the default), `dot` or `plantuml`, the form every graph-shaped diagram block of the document is written in. The output is deterministic CommonMark: the title and sections as ATX headings; paragraphs from text runs (`Span` runs with a `plain`/`emphasis`/`strong`/`code` style, `Link` runs to a URL, `Ref` runs linking to another content block's anchor, and query-produced values styled through nested `SpanColumn`/`LinkColumn` column runs); GitHub-flavored pipe tables with the projected column names (one subtable per group value when the table has a `groupBy` column); bullet and numbered lists; diagram blocks as fenced ` ```mermaid ` blocks rendered through the view engine (` ```dot ` blocks of Graphviz DOT with the `dot` form, ` ```plantuml ` blocks with `plantuml`; a table-kind view as a pipe table whichever form), with an optional caption and `TB`/`LR`/`RL`/`BT` flow direction. Markdown metacharacters in content are escaped. Markdown is the only form the REPL writes; the CLI's `-doc-form html` renders the same document tree as semantic HTML ([Rendering a document as HTML](cli.md#rendering-a-document-as-html)) and `-doc-form pdf` converts the Markdown to PDF ([Rendering a document as PDF](cli.md#rendering-a-document-as-pdf)). See the [document generation manual](../manual/README.md) | | `%sweep [()] [] =..[:] ...` | Run an analysis case or a calc once per value of a range and print the runs as a table: one row per run, carrying the values bound for it, the run's `out`/`return` values, the verdict of its `objective` where it has one, and the wall time of that run. The invocation is written as `%analysis`/`%calc` writes it — arguments in parentheses, an [object reference](#object-references) after them as the case's `subject` — and each range that follows names a parameter the case declares and the arguments do not bind. Endpoints and step are expressions evaluated at the prompt, units included (`speed=0.0 [SI::'m/s']..10.0 [SI::'m/s']:2.0 [SI::'m/s']`), converted to the unit `` carries; `` is included where the step lands on it. The values are produced in the parameter's declared type, not the literals' — a `Real` or `Rational` parameter swept over `1..4:1` is bound to `1.0`, `2.0`, `3.0`, `4.0` and shown so; an `Integer`, `Natural` or `Positive` parameter swept over `1.0..3.0:1.0` to `1`, `2`, `3`, and an endpoint or step of it that is no Integer, or below what a `Natural` or `Positive` holds, is refused naming the parameter and its type before any run; an `attribute def` specializing a scalar takes that scalar's values; a quantity-typed parameter is typed through its `num` (`Number` in the library, so its magnitudes are read as written; `Integer` where a model redefines it so, a `Natural` or `Positive` one refusing a magnitude below what it holds and one holding no number refusing the range) and takes the unit `` carries; a `Number`-typed parameter and one declaring no type take the range as written — Integers between Integer literals, reals otherwise — and an untyped one is noted under the table; a `Boolean`, `String`, enumeration or non-scalar parameter refuses a range naming its type. A range between whole numbers with no `:` steps by one, up or down as its endpoints direct; a range with a fractional endpoint and no step is refused. A range read as reals takes an Integer endpoint or step only where a Real holds it without rounding, and steps only where the reals tell its rows apart, so a range whose rows would repeat one value is refused rather than run. Several ranges run their cartesian product, the first written varying slowest, and rows come out in that order. A run that fails is a row numbering its typed error, printed in full under the table, rather than an abort of the table. A parameter whose name needs the quotes of an unrestricted name is swept under that name, quotes included (`'launch mass'=1..3`). A step of zero, a step whose sign never reaches ``, an endpoint that is no number or is not finite, incompatible units, an undeclared parameter, the case's subject, one the arguments already bind — by name or by holding the position it is bound from — and a plan asking for more runs than `OPENSYSML_MAX_SWEEP_RUNS` allows are errors naming what was asked for. A trade study's rows carry an `evaluations` column, each run's evaluations of its alternatives with the one selected marked, a failed row keeping the ones it made. Each row is a run in a context of its own — the subject and the `self` of a nested case instantiated there, the arguments evaluated once at the prompt (a held feature a run wrote reads as written) and their values carried in, an argument naming a held object bound to the object the row makes for it — so no row sees another's writes; rows run `%jobs` at a time, and the table is in range order whatever order they finish in, the `time` column alone varying with the count. A sweep on an object the session holds runs each row on an object of its own that is the held object as the sweep found it, and leaves the held object as it was: a fresh object of the held object's declaration (one reached through a feature of another, `fleet.flagship`, on the like of `fleet` walked to its `flagship`) while the object is as the declaration made it, the behaviors its type exhibits or performs still as their start left them — as one fresh from `%instantiate` is; otherwise — named by `#`, a feature of it written by a run, a signal sent to it, its state machine moved or its performed action past a wait — a copy from one image of the held object and what it holds, taken as the sweep begins and made in each row's context under the same identities, executors and posted signals included. One destroyed, or in a state no copy carries (a body paused mid-statement, a debugger inside a step), is refused naming the reason, never swept on shared state; an argument naming a held object binds the row's copy of it, and is refused naming the argument when no copy can be made or its value is bound to the run that made it. Same tables as the CLI's [`-sweep`](cli.md#sweeping-a-parameter) | diff --git a/examples/self-model/pipeline.sysml b/examples/self-model/pipeline.sysml index 811f8b7d6..3b1d41a8f 100644 --- a/examples/self-model/pipeline.sysml +++ b/examples/self-model/pipeline.sysml @@ -334,7 +334,7 @@ package OpenSysMLPipeline { // snapshot when that matches the files, and parsed from the files otherwise. #SemanticEngine part def StandardLibrary :> Stage { attribute :>> goPackage = "internal/workspace/libs"; - attribute bundledFileCount : Integer = 104; + attribute bundledFileCount : Integer = 105; attribute cached : Boolean = true; attribute snapshotFile : String = "internal/workspace/libs/stdlib.snapshot"; attribute snapshotEmbedded : Boolean = true; diff --git a/examples/self-model/quality.sysml b/examples/self-model/quality.sysml index ea21147c6..01dfa4ab1 100644 --- a/examples/self-model/quality.sysml +++ b/examples/self-model/quality.sysml @@ -71,7 +71,7 @@ package OpenSysMLInvariants { doc /* Every bundled standard library file parses clean. */ subject stdlib : StandardLibrary; require constraint { - stdlib.bundledFileCount == 104 + stdlib.bundledFileCount == 105 } } diff --git a/internal/exec/analysis/record/record.go b/internal/exec/analysis/record/record.go new file mode 100644 index 000000000..e5ac813f1 --- /dev/null +++ b/internal/exec/analysis/record/record.go @@ -0,0 +1,831 @@ +// Package record generates the SysML declarations that record an analysis +// run, a sweep or a Monte-Carlo sample into the model it ran on, as usages of +// the bundled AnalysisRecords library. +package record + +import ( + "fmt" + "strings" + "time" + + "github.com/Open-MBEE/OpenSysML/internal/exec/runtime" + "github.com/Open-MBEE/OpenSysML/internal/semantic/semantics" + "github.com/Open-MBEE/OpenSysML/internal/semantic/symbols" + "github.com/Open-MBEE/OpenSysML/internal/syntax/source" +) + +// Kind is the run shape a record's `kind` feature reports. +type Kind string + +const ( + // KindRun records one run of an analysis case. + KindRun Kind = "run" + // KindTrade records one run of an analysis case that evaluated alternatives. + KindTrade Kind = "trade" + // KindSweep records the runs of a sweep. + KindSweep Kind = "sweep" + // KindRuns records the runs of a Monte-Carlo sample. + KindRuns Kind = "runs" + // KindSample records the conclusion a Monte-Carlo sample made, beside the + // runs it was made of. + KindSample Kind = "sample" +) + +// Provenance is what a recorded run reports about how it was made. +type Provenance struct { + // RunAt is when the run was made; recorded in UTC. + RunAt time.Time + + // Tool names the program that ran it. + Tool string + + // Command is the invocation text that ran it. + Command string + + // Kind is the run shape the records report. + Kind Kind +} + +// Subject is how a run's object is recorded: its usage in the model, and its +// text for subjectName. +type Subject struct { + // Usage is the subject object's qualified name, or "" when it has none. + Usage string + + // Text is the subject as subjectName reports it. + Text string +} + +// Run is everything one run of a case contributes to its record. +type Run struct { + // Iteration is the run's position in a sweep or sample; 0 for a single run. + Iteration int + + // Subject is the object the run was made on. + Subject Subject + + // Inputs are the values the run bound the case's input parameters to. + Inputs []runtime.InputBinding + + // Outputs are the values the run's declared outputs came to. + Outputs []runtime.CalcOutputValue + + // Verdicts are what the run's checks decided. + Verdicts []runtime.AnalysisVerdict + + // Evaluations are the calc applications the run made. + Evaluations []runtime.AnalysisEvaluation + + // Kind is the run shape this record makes; empty, the request's kind. + Kind Kind + + // Verifications are what a verification case's body and its subcases + // decided, for a case that is one. + Verifications []runtime.VerificationVerdict + + // Spell renders the run's values for the text it cannot supply itself, in + // the context it was made in — a sweep's rows each carry their own. + Spell Spelling +} + +// Spelling renders a run's values for the text it cannot supply itself. +type Spelling struct { + // ObjectUsage is the qualified name of the usage an object value is an + // occurrence of, or "" when the value names no single usage. + ObjectUsage func(v runtime.Value) string + + // Text is a value's text for the surfaces that record it as a string. + Text func(v runtime.Value) string + + // Unset reports a value records nothing: null, or a feature holding none. + Unset func(v runtime.Value) bool +} + +// Feature is one member an existing record definition declares. +type Feature struct { + // Ref marks a `ref part`, an object-valued feature; unset is an attribute. + Ref bool + + // TypeFQN is the qualified name of the feature's declared type. + TypeFQN string +} + +// Existing is what Generate must fit the records it makes into. +type Existing struct { + // Package marks the target package already declared. + Package bool + + // Definition marks the per-case record definition already declared in it. + Definition bool + + // Attributes are the features the existing definition declares, by name. + Attributes map[string]Feature + + // Taken are the N for which _runN is already declared in the package. + Taken map[int]bool + + // Stem is the name the records and their definition are built on — the + // case's short name, or an owner-prefixed fallback a sibling case's + // definition forced. + Stem string +} + +// Request is one call to Generate. +type Request struct { + // Package is the qualified name of the package the records go into, e.g. + // "Records" or "Mission::Records". + Package string + + // Case is the qualified name of the analysis case the runs were made of. + Case string + + // CaseName is the case's declared name as written, quoting needs included; + // the records and their definition are named from it. Empty falls back to + // Case's last qualified-name segment. + CaseName string + + // Provenance is what every record reports about how it was made. + Provenance Provenance + + // Runs are the runs to record, in order. + Runs []Run + + // Existing is what the target package already holds. + Existing Existing +} + +// Result is what Generate made. +type Result struct { + // Source is the declaration text: the target package's full nesting down to + // it, holding the record definition when new and every record usage. + Source string + + // Definition is the qualified name of the per-case record definition. + Definition string + + // Records are the qualified names of the record usages, in order. + Records []string +} + +// reservedFeatures are AnalysisRun's own features a parameter name may not take. +var reservedFeatures = map[string]bool{ + "caseName": true, "kind": true, "objective": true, "iteration": true, + "subject": true, "subjectName": true, "verdict": true, "verdicts": true, + "evaluations": true, +} + +// feature is one member the record definition declares for a run value. +type feature struct { + name string + ref bool // object-valued + typ string // declared type as written, "" for a ref + unitOf string // nonempty: this feature is the unit companion of the named one +} + +// valueKind classifies how a value is spelled: its declared type and, for a +// quantity, the companion unit feature it needs. +type valueKind int + +const ( + kindUnset valueKind = iota + kindRef + kindInteger + kindReal + kindBoolean + kindString + kindEnum + kindQuantity +) + +// shape is how a value is recorded: its feature kind, the literal spelling, +// and — for a quantity — the unit text its companion feature records. +type shape struct { + kind valueKind + literal string + typ string + unit string +} + +// classify decides the feature shape a value asks for. +func classify(v runtime.Value, r *Run) shape { + if r.Spell.Unset != nil && r.Spell.Unset(v) { + return shape{kind: kindUnset} + } + // An enumeration literal keeps its identity through a scalar payload too. + if lit := v.EnumerationLiteral(); lit != nil { + enum := semantics.EnumerationOwning(lit) + fqn := qualifiedName(enum) + return shape{kind: kindEnum, typ: fqn, literal: source.QualifiedNameText(fqn + "::" + lit.Name)} + } + switch v.Kind { + case runtime.ValNull: + return shape{kind: kindUnset} + case runtime.ValConst: + if kind, typ, ok := constScalar(v.Const.Kind); ok { + return shape{kind: kind, typ: typ, literal: semantics.FormatConst(v.Const)} + } + // A constant without a literal spelling, Infinity included, is + // recorded as a string of its text. + return shape{kind: kindString, typ: "ScalarValues::String", literal: source.StringText(semantics.FormatConst(v.Const))} + case runtime.ValString: + return shape{kind: kindString, typ: "ScalarValues::String", literal: source.StringText(v.Str())} + case runtime.ValQuantity: + q := v.Quantity() + return shape{kind: kindQuantity, typ: "ScalarValues::Real", literal: semantics.FormatConst(q.Num), unit: q.Unit.String()} + case runtime.ValInstance, runtime.ValVariant: + if r.Spell.ObjectUsage != nil { + if usage := r.Spell.ObjectUsage(v); usage != "" { + return shape{kind: kindRef, literal: source.QualifiedNameText(usage)} + } + } + } + // Everything else — a structured value, or an object naming no usage — is + // recorded by its text. + return shape{kind: kindString, typ: "ScalarValues::String", literal: source.StringText(spellText(v, r))} +} + +// constScalar is the feature kind and ScalarValues type a scalar literal's +// kind is recorded under; a constant without a literal spelling, Infinity +// included, has none and is recorded as a string. +func constScalar(k semantics.ValueKind) (valueKind, string, bool) { + switch k { + case semantics.ValInt: + return kindInteger, "ScalarValues::Integer", true + case semantics.ValReal: + return kindReal, "ScalarValues::Real", true + case semantics.ValBool: + return kindBoolean, "ScalarValues::Boolean", true + } + return 0, "", false +} + +// spellText is a value's text for the string features, Text when supplied and +// the runtime's own formatting otherwise. +func spellText(v runtime.Value, r *Run) string { + if r.Spell.Text != nil { + return r.Spell.Text(v) + } + return runtime.FormatValue(v) +} + +// qualifiedName is a symbol's qualified name by its owning chain. +func qualifiedName(sym *symbols.Symbol) string { + if sym == nil { + return "" + } + var names []string + for cur := sym; cur != nil; cur = cur.Owner() { + if cur.Name == "" { + break + } + names = append([]string{cur.Name}, names...) + } + return strings.Join(names, "::") +} + +// shortName is the last qualified-name segment of a case's name. +func shortName(fqn string) string { + segs, ok := source.QualifiedNameSegments(fqn) + if !ok || len(segs) == 0 { + return fqn + } + return segs[len(segs)-1] +} + +// upperFirst capitalizes a name's leading letter. +func upperFirst(name string) string { + if name == "" { + return name + } + return strings.ToUpper(name[:1]) + name[1:] +} + +// member is one input or output value a run declares. inOf marks the In +// companion of an inout: one member in the run's value, one in its binding. +type member struct { + name string + value runtime.Value + inOf string +} + +// members are the input and output values a run declares, in order. A name +// on both sides is one inout parameter: the value it ran to, then an +// In companion holding the value it was bound with. +func members(r Run) []member { + var out []member + outs := map[string]bool{} + for _, o := range r.Outputs { + outs[o.Name] = true + } + inouts := map[string]runtime.Value{} + for _, in := range r.Inputs { + if outs[in.Name] { + inouts[in.Name] = in.Value + continue + } + out = append(out, member{name: in.Name, value: in.Value}) + } + for _, o := range r.Outputs { + out = append(out, member{name: o.Name, value: o.Value}) + if v, ok := inouts[o.Name]; ok { + out = append(out, member{name: o.Name + "In", value: v, inOf: o.Name}) + } + } + return out +} + +// buildFeatures decides the members the record definition needs: every +// distinct member name in first-encounter order, its shape settled from the +// runs that supply it a value, and a unit companion after each quantity. +func buildFeatures(req *Request) ([]feature, error) { + // An inout's In companion is the run's own member, but a member a run + // declares of the same name collides with it, as a Unit companion's does. + companionOf := map[string]string{} + for i := range req.Runs { + for _, m := range members(req.Runs[i]) { + if m.inOf != "" { + companionOf[m.name] = m.inOf + } + } + } + // First pass: settle each member's shape over every run that supplies it. + var names []string + shapes := map[string]shape{} + for i := range req.Runs { + for _, m := range members(req.Runs[i]) { + if m.inOf == "" { + if owner, ok := companionOf[m.name]; ok { + return nil, fmt.Errorf("case %s: member %q collides with the in companion of inout %q", req.Case, m.name, owner) + } + } + if reservedFeatures[m.name] { + return nil, fmt.Errorf("case %s: parameter %q shares a name with a feature of AnalysisRecords::AnalysisRun", req.Case, m.name) + } + sh := classify(m.value, &req.Runs[i]) + cur, seen := shapes[m.name] + if !seen { + names = append(names, m.name) + shapes[m.name] = sh + continue + } + if sh.kind == kindUnset { + continue + } + if cur.kind == kindUnset { + shapes[m.name] = sh + continue + } + // A quantity member takes a plain-number row either order: the + // row keeps its literal and takes no unit. + if sh.kind == kindQuantity && (cur.kind == kindInteger || cur.kind == kindReal) { + shapes[m.name] = sh + continue + } + if cur.kind == kindQuantity && (sh.kind == kindInteger || sh.kind == kindReal) { + continue + } + // Integer and Real are one numeric family for the record + // definition: either way the member settles to Real, an Integer + // literal remaining valid under it. + if numericPair(cur.typ, sh.typ) { + cur = shape{kind: kindReal, typ: "ScalarValues::Real"} + shapes[m.name] = cur + continue + } + f := feature{name: m.name} + applyShape(&f, cur) + if err := compatible(&f, sh); err != nil { + return nil, fmt.Errorf("case %s: member %q: %w", req.Case, m.name, err) + } + } + } + // The two sides of an inout settle to one shape: a quantity side wins over + // a plain number, Integer and Real settle to Real, anything else must match. + for companion, owner := range companionOf { + o, c := shapes[owner], shapes[companion] + switch { + case o.kind == kindUnset || c.kind == kindUnset: + case o.kind == kindQuantity && (c.kind == kindInteger || c.kind == kindReal): + shapes[companion] = o + case c.kind == kindQuantity && (o.kind == kindInteger || o.kind == kindReal): + shapes[owner] = c + case numericPair(o.typ, c.typ): + shapes[owner] = shape{kind: kindReal, typ: "ScalarValues::Real"} + shapes[companion] = shape{kind: kindReal, typ: "ScalarValues::Real"} + default: + f := feature{name: owner} + applyShape(&f, o) + if err := compatible(&f, c); err != nil { + return nil, fmt.Errorf("case %s: inout %q: %w", req.Case, owner, err) + } + } + } + // Emit the features, each quantity's unit companion after it; a member + // named for one is a collision whatever order they met in. + units := map[string]string{} + for _, name := range names { + if shapes[name].kind == kindQuantity { + units[name+"Unit"] = name + } + } + var feats []feature + for _, name := range names { + if q, ok := units[name]; ok { + return nil, fmt.Errorf("case %s: member %q collides with the unit companion of quantity %q", req.Case, name, q) + } + f := feature{name: name} + applyShape(&f, shapes[name]) + feats = append(feats, f) + if shapes[name].kind == kindQuantity { + feats = append(feats, feature{name: name + "Unit", typ: "ScalarValues::String", unitOf: name}) + } + } + return feats, nil +} + +// numericPair reports whether the types are Integer and Real in either order: +// one numeric family for the record definition, settling to Real. +func numericPair(a, b string) bool { + return (a == "ScalarValues::Integer" && b == "ScalarValues::Real") || + (a == "ScalarValues::Real" && b == "ScalarValues::Integer") +} + +// applyShape gives a feature the declared shape a value's first supply asks for. +func applyShape(f *feature, sh shape) { + f.ref = sh.kind == kindRef + switch sh.kind { + case kindUnset: + f.typ = "ScalarValues::ScalarValue" + case kindRef: + f.typ = "" + default: + f.typ = source.QualifiedNameText(sh.typ) + } +} + +// compatible checks a later run's value against the shape a feature took. +func compatible(f *feature, sh shape) error { + switch { + case f.ref && sh.kind != kindRef: + return fmt.Errorf("an object value cannot be recorded in the value member") + case !f.ref && f.typ != "" && sh.kind == kindRef: + return fmt.Errorf("a non-object value cannot be recorded in the reference member") + } + if f.ref || sh.kind == kindRef { + return nil + } + if f.typ == "ScalarValues::ScalarValue" { + // The first supply was unset; a settled value gives the member its type. + f.typ = source.QualifiedNameText(sh.typ) + return nil + } + if f.typ != source.QualifiedNameText(sh.typ) { + return fmt.Errorf("value recorded as %s cannot follow %s", sh.typ, f.typ) + } + return nil +} + +// Generate renders the declarations recording req's runs. +func Generate(req Request) (Result, error) { + if len(req.Runs) == 0 { + return Result{}, fmt.Errorf("case %s: nothing to record", req.Case) + } + for _, r := range req.Runs { + if len(r.Outputs) == 0 && len(r.Verdicts) == 0 && len(r.Evaluations) == 0 && len(r.Verifications) == 0 { + return Result{}, fmt.Errorf("case %s produced no outputs to record", req.Case) + } + } + for _, r := range req.Runs { + seen := map[string]bool{} + for _, in := range r.Inputs { + if seen[in.Name] { + return Result{}, fmt.Errorf("case %s: member %q is listed twice", req.Case, in.Name) + } + seen[in.Name] = true + } + seen = map[string]bool{} + for _, o := range r.Outputs { + if seen[o.Name] { + return Result{}, fmt.Errorf("case %s: member %q is listed twice", req.Case, o.Name) + } + seen[o.Name] = true + } + } + + stem := req.Existing.Stem + if stem == "" { + stem = req.CaseName + } + if stem == "" { + stem = shortName(req.Case) + } + defName := upperFirst(stem) + "Run" + feats, err := buildFeatures(&req) + if err != nil { + return Result{}, err + } + + if req.Existing.Definition { + if err := checkExisting(&req, feats, defName); err != nil { + return Result{}, err + } + } + + var src strings.Builder + segs, _ := source.QualifiedNameSegments(req.Package) + depth := 0 + for _, seg := range segs { + writeIndent(&src, depth) + src.WriteString("package ") + src.WriteString(source.QualifiedNameText(seg)) + src.WriteString(" {\n") + depth++ + } + + recordNames := make([]string, len(req.Runs)) + if !req.Existing.Definition { + writeDefinition(&src, depth, defName, feats, req.Case) + } + // Number each record the smallest free N, the package's taken numbers and + // this batch's both skipped. + taken := map[int]bool{} + for n := range req.Existing.Taken { + taken[n] = true + } + next := 1 + for i := range req.Runs { + for taken[next] { + next++ + } + name := stem + "_run" + fmt.Sprint(next) + taken[next] = true + recordNames[i] = name + writeRecord(&src, depth, name, defName, feats, &req.Runs[i], &req) + } + + for d := depth - 1; d >= 0; d-- { + writeIndent(&src, d) + src.WriteString("}\n") + } + + records := make([]string, len(recordNames)) + for i, n := range recordNames { + records[i] = req.Package + "::" + n + } + return Result{Source: src.String(), Definition: req.Package + "::" + defName, Records: records}, nil +} + +// checkExisting verifies every feature the records need is declared +// compatibly by the existing definition. +func checkExisting(req *Request, feats []feature, defName string) error { + def := req.Package + "::" + defName + for _, f := range feats { + decl, ok := req.Existing.Attributes[f.name] + if !ok { + return fmt.Errorf("record definition %s declares no member %q; record into another package with `into`", def, f.name) + } + if decl.Ref != f.ref { + kind := "an attribute" + if decl.Ref { + kind = "a reference" + } + want := "a reference" + if !f.ref { + want = "an attribute" + } + return fmt.Errorf("record definition %s declares %s as %s but the run values need %s; record into another package with `into`", def, f.name, kind, want) + } + if !f.ref && f.typ != "" && decl.TypeFQN != "" && decl.TypeFQN != f.typ && + f.typ != "ScalarValues::ScalarValue" && decl.TypeFQN != "ScalarValues::ScalarValue" { + // An Integer literal is valid under a declared Real; the + // reverse would widen a definition the model owns, so it stays + // refused. + if decl.TypeFQN == "ScalarValues::Real" && f.typ == "ScalarValues::Integer" { + continue + } + return fmt.Errorf("record definition %s declares %s : %s but the run values need %s : %s; record into another package with `into`", def, f.name, decl.TypeFQN, f.name, f.typ) + } + } + return nil +} + +// writeIndent writes depth levels of indentation. +func writeIndent(src *strings.Builder, depth int) { + src.WriteString(strings.Repeat(" ", depth)) +} + +// writeDefinition writes the per-case record definition, its caseName default +// marking the case it records. +func writeDefinition(src *strings.Builder, depth int, name string, feats []feature, caseFQN string) { + writeIndent(src, depth) + src.WriteString("part def ") + src.WriteString(source.NameText(name)) + src.WriteString(" :> AnalysisRecords::AnalysisRun {\n") + writeIndent(src, depth+1) + src.WriteString("attribute :>> caseName default = ") + src.WriteString(source.StringText(caseFQN)) + src.WriteString(";\n") + for _, f := range feats { + writeIndent(src, depth+1) + if f.ref { + src.WriteString("ref part ") + src.WriteString(source.NameText(f.name)) + src.WriteString(";\n") + } else { + src.WriteString("attribute ") + src.WriteString(source.NameText(f.name)) + src.WriteString(" : ") + src.WriteString(f.typ) + src.WriteString(";\n") + } + } + writeIndent(src, depth) + src.WriteString("}\n") +} + +// writeRecord writes one run's record usage. +func writeRecord(src *strings.Builder, depth int, name, defName string, feats []feature, r *Run, req *Request) { + writeIndent(src, depth) + src.WriteString("part ") + src.WriteString(source.NameText(name)) + src.WriteString(" : ") + src.WriteString(source.NameText(defName)) + src.WriteString(" {\n") + + writeIndent(src, depth+1) + src.WriteString("@AnalysisRecords::RecordedRun {\n") + for _, m := range []struct{ name, value string }{ + {"runAt", source.StringText(req.Provenance.RunAt.UTC().Format(time.RFC3339))}, + {"tool", source.StringText(req.Provenance.Tool)}, + {"command", source.StringText(req.Provenance.Command)}, + {"kind", source.StringText(string(kindOf(r, req)))}, + } { + writeIndent(src, depth+2) + src.WriteString(m.name) + src.WriteString(" = ") + src.WriteString(m.value) + src.WriteString(";\n") + } + writeIndent(src, depth+1) + src.WriteString("}\n") + + writeFeature(src, depth+1, "caseName", source.StringText(req.Case)) + writeFeature(src, depth+1, "kind", source.StringText(string(kindOf(r, req)))) + writeFeature(src, depth+1, "'objective'", source.StringText(objectiveOf(r))) + for _, v := range r.Verifications { + if !v.Subcase { + writeFeature(src, depth+1, "verdict", source.StringText(string(v.Kind))) + } + } + if r.Iteration > 0 { + writeIndent(src, depth+1) + src.WriteString("attribute :>> iteration = ") + src.WriteString(fmt.Sprint(r.Iteration)) + src.WriteString(";\n") + } + if r.Subject.Usage != "" || r.Subject.Text != "" { + if r.Subject.Usage != "" { + writeIndent(src, depth+1) + src.WriteString("ref :>> 'subject' = ") + src.WriteString(source.QualifiedNameText(r.Subject.Usage)) + src.WriteString(";\n") + } + writeFeature(src, depth+1, "subjectName", source.StringText(r.Subject.Text)) + } + + shapeByName := map[string]feature{} + for _, f := range feats { + shapeByName[f.name] = f + } + for _, m := range members(*r) { + sh := classify(m.value, r) + if sh.kind == kindUnset { + continue + } + writeIndent(src, depth+1) + if sh.kind == kindRef { + src.WriteString("ref :>> ") + } else { + src.WriteString("attribute :>> ") + } + src.WriteString(source.NameText(m.name)) + src.WriteString(" = ") + src.WriteString(sh.literal) + src.WriteString(";\n") + if sh.kind == kindQuantity { + writeFeature(src, depth+1, m.name+"Unit", source.StringText(sh.unit)) + } + } + + for i, v := range r.Verdicts { + writeVerdict(src, depth+1, i+1, v) + } + for i, v := range r.Verifications { + writeVerification(src, depth+1, i+1, v) + } + for i, e := range r.Evaluations { + writeEvaluation(src, depth+1, i+1, e, r) + } + + writeIndent(src, depth) + src.WriteString("}\n") +} + +// objectiveOf is the status the run's objective verdict reports. +func objectiveOf(r *Run) string { + for _, v := range r.Verdicts { + if v.Kind == "objective" { + return v.Status.String() + } + } + return "undecided" +} + +// kindOf is the kind a run's record reports: its own where set, the +// request's otherwise. +func kindOf(r *Run, req *Request) Kind { + if r.Kind != "" { + return r.Kind + } + return req.Provenance.Kind +} + +// writeFeature writes `attribute :>> name = literal;`. +func writeFeature(src *strings.Builder, depth int, name, literal string) { + writeIndent(src, depth) + src.WriteString("attribute :>> ") + if strings.HasPrefix(name, "'") { + src.WriteString(name) + } else { + src.WriteString(source.NameText(name)) + } + src.WriteString(" = ") + src.WriteString(literal) + src.WriteString(";\n") +} + +// writeVerdict writes one verdict record part. +func writeVerdict(src *strings.Builder, depth, n int, v runtime.AnalysisVerdict) { + writeIndent(src, depth) + src.WriteString(fmt.Sprintf("part verdict%d : AnalysisRecords::VerdictRecord :> verdicts {\n", n)) + writeFeature(src, depth+1, "kind", source.StringText(v.Kind)) + writeFeature(src, depth+1, "name", source.StringText(v.Name)) + writeFeature(src, depth+1, "status", source.StringText(v.Status.String())) + if v.Detail != "" { + writeFeature(src, depth+1, "detail", source.StringText(v.Detail)) + } + writeIndent(src, depth) + src.WriteString("}\n") +} + +// writeVerification writes a verification case's body or subcase verdict as +// one more VerdictRecord: its kind says which, its name the case that ran. +func writeVerification(src *strings.Builder, depth, n int, v runtime.VerificationVerdict) { + kind := "verification" + if v.Subcase { + kind = "subcase" + } + writeIndent(src, depth) + src.WriteString(fmt.Sprintf("part verification%d : AnalysisRecords::VerdictRecord :> verdicts {\n", n)) + writeFeature(src, depth+1, "kind", source.StringText(kind)) + writeFeature(src, depth+1, "name", source.StringText(v.Case)) + writeFeature(src, depth+1, "status", source.StringText(string(v.Kind))) + if v.Detail != "" { + writeFeature(src, depth+1, "detail", source.StringText(v.Detail)) + } + writeIndent(src, depth) + src.WriteString("}\n") +} + +// writeEvaluation writes one evaluation record part. +func writeEvaluation(src *strings.Builder, depth, n int, e runtime.AnalysisEvaluation, r *Run) { + writeIndent(src, depth) + src.WriteString(fmt.Sprintf("part evaluation%d : AnalysisRecords::EvaluationRecord :> evaluations {\n", n)) + writeFeature(src, depth+1, "function", source.StringText(e.Function)) + var args []string + for _, a := range e.Arguments { + args = append(args, spellText(a, r)) + } + writeFeature(src, depth+1, "alternative", source.StringText(strings.Join(args, ", "))) + if sh := classify(e.Result, r); sh.kind == kindInteger || sh.kind == kindReal || sh.kind == kindQuantity { + writeFeature(src, depth+1, "score", sh.literal) + writeFeature(src, depth+1, "result", source.StringText(spellText(e.Result, r))) + } else if e.Error == nil { + writeFeature(src, depth+1, "result", source.StringText(spellText(e.Result, r))) + } + writeFeature(src, depth+1, "selected", boolText(e.Selected)) + writeFeature(src, depth+1, "tied", boolText(e.Tied)) + if e.Error != nil { + writeFeature(src, depth+1, "error", source.StringText(e.Error.Error())) + } + writeIndent(src, depth) + src.WriteString("}\n") +} + +// boolText spells a Boolean literal. +func boolText(b bool) string { + if b { + return "true" + } + return "false" +} diff --git a/internal/exec/analysis/record/record_test.go b/internal/exec/analysis/record/record_test.go new file mode 100644 index 000000000..683792cf0 --- /dev/null +++ b/internal/exec/analysis/record/record_test.go @@ -0,0 +1,764 @@ +package record + +import ( + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Open-MBEE/OpenSysML/internal/exec/runtime" + "github.com/Open-MBEE/OpenSysML/internal/semantic/semantics" + "github.com/Open-MBEE/OpenSysML/internal/semantic/symbols" + "github.com/Open-MBEE/OpenSysML/internal/syntax/format" + "github.com/Open-MBEE/OpenSysML/internal/syntax/parser" + "github.com/Open-MBEE/OpenSysML/internal/syntax/source" +) + +var update = flag.Bool("update", false, "rewrite the golden files from the current generator") + +func real(f float64) runtime.Value { + return runtime.Value{Kind: runtime.ValConst, Const: semantics.Value{Kind: semantics.ValReal, Real: f}} +} + +func integer(n int64) runtime.Value { + return runtime.Value{Kind: runtime.ValConst, Const: semantics.Value{Kind: semantics.ValInt, Int: n}} +} + +func boolean(b bool) runtime.Value { + return runtime.Value{Kind: runtime.ValConst, Const: semantics.Value{Kind: semantics.ValBool, Bool: b}} +} + +func enumLiteral(t *testing.T) runtime.Value { + file := parser.New(source.New("", []byte( + `package P { enum def Fuel { enum leaded; enum unleaded; } }`))).ParseFile() + scope := symbols.Build(file) + p, ok := scope.LookupLocal("P") + if !ok { + t.Fatal("package not built") + } + fuel, ok := p.Scope.LookupLocal("Fuel") + if !ok { + t.Fatal("enum def not built") + } + lit, ok := fuel.Scope.LookupLocal("leaded") + if !ok { + t.Fatal("enum literal not built") + } + return runtime.NewEnumLiteral(lit) +} + +// spell renders values the way a session would: no object resolves to a usage. +func spell() Spelling { + return Spelling{ + ObjectUsage: func(runtime.Value) string { return "" }, + Text: runtime.FormatValue, + Unset: func(v runtime.Value) bool { return v.Kind == runtime.ValNull }, + } +} + +func provenance(kind Kind) Provenance { + return Provenance{ + RunAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + Tool: "sysml test", + Command: "%record P::check", + Kind: kind, + } +} + +// golden runs Generate and compares Source against the golden at name. +func golden(t *testing.T, name string, req Request) Result { + t.Helper() + res, err := Generate(req) + if err != nil { + t.Fatalf("Generate: %v", err) + } + path := filepath.Join("testdata", name) + if *update { + if err := os.WriteFile(path, []byte(res.Source), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%s missing; run with -update", path) + } + if res.Source != string(want) { + t.Errorf("%s differs from the generated text:\n%s", name, res.Source) + } + return res +} + +// A single run records every value spelling, its verdicts and its provenance. +func TestGenerateSingleRun(t *testing.T) { + obj := runtime.Value{Kind: runtime.ValInstance, Instance: 7} + sp := spell() + sp.ObjectUsage = func(v runtime.Value) string { + if v == obj { + return "P::scout" + } + return "" + } + res := golden(t, "single_run.sysml.golden", Request{ + Package: "Records", Case: "P::scoutBudget", Provenance: provenance(KindRun), + Runs: []Run{{ + Spell: sp, + Subject: Subject{Usage: "P::scout", Text: "P::scout"}, + Inputs: []runtime.InputBinding{ + {Name: "burnTime", Value: real(3.0)}, + {Name: "trials", Value: integer(8)}, + {Name: "crewOk", Value: boolean(true)}, + {Name: "label", Value: runtime.NewStringValue(`a "b"`)}, + {Name: "grade", Value: enumLiteral(t)}, + {Name: "tank", Value: runtime.NewQuantityValue(&runtime.Quantity{ + Num: semantics.Value{Kind: semantics.ValReal, Real: 12.5}, + Unit: semantics.Unit{Text: "kg"}, + })}, + {Name: "target", Value: obj}, + }, + Outputs: []runtime.CalcOutputValue{ + {Name: "fuelUsed", Value: real(12.5)}, + {Name: "memo", Value: runtime.Value{Kind: runtime.ValNull}}, + {Name: "plan", Value: runtime.NewSequenceValue(nil)}, + }, + Verdicts: []runtime.AnalysisVerdict{ + {Kind: "objective", Name: "fuelFits", Status: runtime.VerdictSatisfied, Detail: "holds"}, + {Kind: "assertion", Name: "crewReady", Status: runtime.VerdictNotSatisfied}, + }, + }}, + }) + if res.Definition != "Records::ScoutBudgetRun" { + t.Errorf("definition %q", res.Definition) + } + if len(res.Records) != 1 || res.Records[0] != "Records::scoutBudget_run1" { + t.Errorf("records %v", res.Records) + } +} + +// A trade run records its evaluations: selected, tied and failed alternatives. +func TestGenerateTradeRun(t *testing.T) { + failed := errors.New("boom") + res := golden(t, "trade_run.sysml.golden", Request{ + Package: "P::Records", Case: "P::choose", Provenance: provenance(KindTrade), + Runs: []Run{{ + Spell: spell(), + Subject: Subject{Text: "P::fleet"}, + Outputs: []runtime.CalcOutputValue{{Name: "best", Value: real(1.0)}}, + Evaluations: []runtime.AnalysisEvaluation{ + {Function: "P::score", Arguments: []runtime.Value{real(1.0)}, Result: real(0.75), Selected: true}, + {Function: "P::score", Arguments: []runtime.Value{real(2.0)}, Result: real(0.75), Tied: true}, + {Function: "P::score", Arguments: []runtime.Value{real(3.0)}, Result: runtime.Value{Kind: runtime.ValNull}, Error: failed}, + }, + }}, + }) + if res.Definition != "P::Records::ChooseRun" { + t.Errorf("definition %q", res.Definition) + } +} + +// A sweep reuses the existing definition and numbers its records on. +func TestGenerateSweepRuns(t *testing.T) { + res := golden(t, "sweep_runs.sysml.golden", Request{ + Package: "Records", Case: "P::check", Provenance: provenance(KindSweep), + Existing: Existing{ + Package: true, Definition: true, Taken: map[int]bool{1: true, 2: true, 3: true}, + Attributes: map[string]Feature{ + "load": {TypeFQN: "ScalarValues::Real"}, + "done": {TypeFQN: "ScalarValues::Boolean"}, + }, + }, + Runs: []Run{ + {Iteration: 1, Spell: spell(), Inputs: []runtime.InputBinding{{Name: "load", Value: real(1.0)}}, Outputs: []runtime.CalcOutputValue{{Name: "done", Value: boolean(true)}}}, + {Iteration: 2, Spell: spell(), Inputs: []runtime.InputBinding{{Name: "load", Value: real(3.0)}}, Outputs: []runtime.CalcOutputValue{{Name: "done", Value: boolean(false)}}}, + {Iteration: 3, Spell: spell(), Inputs: []runtime.InputBinding{{Name: "load", Value: real(5.0)}}, Outputs: []runtime.CalcOutputValue{{Name: "done", Value: boolean(true)}}}, + }, + }) + want := []string{"Records::check_run4", "Records::check_run5", "Records::check_run6"} + if fmt.Sprint(res.Records) != fmt.Sprint(want) { + t.Errorf("records %v, want %v", res.Records, want) + } +} + +// Generate refuses the shapes it cannot record. +func TestGenerateErrors(t *testing.T) { + base := func() Request { + return Request{Package: "Records", Case: "P::check", Provenance: provenance(KindRun)} + } + empty := Request{Package: "Records", Case: "P::check"} + if _, err := Generate(empty); err == nil { + t.Error("no runs: want an error") + } + req := base() + req.Runs = []Run{{Spell: spell(), Inputs: []runtime.InputBinding{{Name: "x", Value: real(1)}}}} + if _, err := Generate(req); err == nil { + t.Error("a run with no outputs, verdicts or evaluations: want an error") + } + req = base() + req.Runs = []Run{{Spell: spell(), Outputs: []runtime.CalcOutputValue{{Name: "kind", Value: real(1)}}}} + if _, err := Generate(req); err == nil { + t.Error("member colliding with an AnalysisRun feature: want an error") + } + req = base() + req.Runs = []Run{{ + Spell: spell(), + Outputs: []runtime.CalcOutputValue{{Name: "x", Value: real(2)}, {Name: "x", Value: real(3)}}, + }} + if _, err := Generate(req); err == nil { + t.Error("one side of the run listing a name twice: want an error") + } + req = base() + req.Runs = []Run{{ + Spell: spell(), + Inputs: []runtime.InputBinding{{Name: "x", Value: real(1)}, {Name: "xIn", Value: real(0)}}, + Outputs: []runtime.CalcOutputValue{{Name: "x", Value: real(2)}}, + }} + if _, err := Generate(req); err == nil { + t.Error("member colliding with an inout's in companion: want an error") + } + req = base() + req.Existing = Existing{Package: true, Definition: true, Attributes: map[string]Feature{ + "load": {TypeFQN: "ScalarValues::String"}, + }} + req.Runs = []Run{{ + Spell: spell(), + Inputs: []runtime.InputBinding{{Name: "load", Value: real(1)}}, + Outputs: []runtime.CalcOutputValue{{Name: "done", Value: boolean(true)}}, + }} + if _, err := Generate(req); err == nil { + t.Error("existing def with a type-mismatched member: want an error") + } + req = base() + req.Existing = Existing{Package: true, Definition: true, Attributes: map[string]Feature{ + "load": {Ref: true}, + "done": {TypeFQN: "ScalarValues::Boolean"}, + }} + req.Runs = []Run{{ + Spell: spell(), + Inputs: []runtime.InputBinding{{Name: "load", Value: real(1)}}, + Outputs: []runtime.CalcOutputValue{{Name: "done", Value: boolean(true)}}, + }} + if _, err := Generate(req); err == nil { + t.Error("existing ref member fed an attribute value: want an error") + } + dose := runtime.NewQuantityValue(&runtime.Quantity{ + Num: semantics.Value{Kind: semantics.ValReal, Real: 1.5}, + Unit: semantics.Unit{Text: "kg"}, + }) + req = base() + req.Runs = []Run{{Spell: spell(), Outputs: []runtime.CalcOutputValue{ + {Name: "dose", Value: dose}, + {Name: "doseUnit", Value: real(1)}, + }}} + if _, err := Generate(req); err == nil { + t.Error("a member named as a quantity's unit companion: want an error") + } + req = base() + req.Runs = []Run{{Spell: spell(), Outputs: []runtime.CalcOutputValue{ + {Name: "doseUnit", Value: real(1)}, + {Name: "dose", Value: dose}, + }}} + if _, err := Generate(req); err == nil { + t.Error("a quantity whose unit companion names a member: want an error") + } +} + +// A member unset in one run takes the type the settled run gives it, wherever +// it sits among the members the definition declares. +func TestGenerateSettlesAnUnsetMember(t *testing.T) { + res, err := Generate(Request{ + Package: "Records", Case: "P::check", Provenance: provenance(KindSweep), + Runs: []Run{ + {Iteration: 1, Spell: spell(), Outputs: []runtime.CalcOutputValue{ + {Name: "x", Value: runtime.Value{Kind: runtime.ValNull}}, + {Name: "a", Value: real(1)}, + {Name: "b", Value: real(2)}, + {Name: "c", Value: real(3)}, + }}, + {Iteration: 2, Spell: spell(), Outputs: []runtime.CalcOutputValue{ + {Name: "x", Value: real(3.0)}, + {Name: "a", Value: real(1)}, + {Name: "b", Value: real(2)}, + {Name: "c", Value: real(3)}, + }}, + }, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + for _, want := range []string{"attribute x : ScalarValues::Real;", "attribute :>> x = 3.0;"} { + if !strings.Contains(res.Source, want) { + t.Errorf("generated source is missing %q:\n%s", want, res.Source) + } + } +} + +// Infinity has no literal of a typed attribute: it records as a string. +func TestGenerateInfinityValue(t *testing.T) { + res, err := Generate(Request{ + Package: "Records", Case: "P::check", Provenance: provenance(KindRun), + Runs: []Run{{Spell: spell(), Outputs: []runtime.CalcOutputValue{ + {Name: "value", Value: runtime.Value{Kind: runtime.ValConst, Const: semantics.Value{Kind: semantics.ValInfinity}}}, + }}}, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + for _, want := range []string{"attribute value : ScalarValues::String;", `attribute :>> value = "*";`} { + if !strings.Contains(res.Source, want) { + t.Errorf("generated source is missing %q:\n%s", want, res.Source) + } + } + if _, err := format.Source("", []byte(res.Source), format.DefaultOptions); err != nil { + t.Errorf("generated source does not parse: %v", err) + } +} + +// The generated text is formatter-stable: formatting it changes nothing. +func TestGeneratedSourceIsFormatterStable(t *testing.T) { + for _, name := range []string{"single_run.sysml.golden", "trade_run.sysml.golden", "sweep_runs.sysml.golden"} { + src, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatalf("%s missing; run with -update", name) + } + out, err := format.Source(name, src, format.DefaultOptions) + if err != nil { + t.Fatalf("format %s: %v", name, err) + } + if string(out) != string(src) { + t.Errorf("%s is not formatter-stable:\n%s", name, out) + } + } +} + +// A member unset in an early row and a quantity in a later one still gains +// its unit companion, settled to Real. +func TestGenerateSettlesAnUnsetMemberToQuantity(t *testing.T) { + res, err := Generate(Request{ + Package: "Records", Case: "P::check", Provenance: provenance(KindSweep), + Runs: []Run{ + {Iteration: 1, Spell: spell(), Outputs: []runtime.CalcOutputValue{ + {Name: "x", Value: runtime.Value{Kind: runtime.ValNull}}, + {Name: "a", Value: real(1)}, + {Name: "b", Value: real(2)}, + }}, + {Iteration: 2, Spell: spell(), Outputs: []runtime.CalcOutputValue{ + {Name: "x", Value: runtime.NewQuantityValue(&runtime.Quantity{ + Num: semantics.Value{Kind: semantics.ValReal, Real: 2.0}, + Unit: semantics.Unit{Text: "kg"}, + })}, + {Name: "a", Value: real(1)}, + {Name: "b", Value: real(2)}, + }}, + }, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + for _, want := range []string{ + "attribute x : ScalarValues::Real;", + "attribute xUnit : ScalarValues::String;", + "attribute :>> x = 2.0;", + `attribute :>> xUnit = "kg";`, + } { + if !strings.Contains(res.Source, want) { + t.Errorf("generated source is missing %q:\n%s", want, res.Source) + } + } + if _, err := format.Source("", []byte(res.Source), format.DefaultOptions); err != nil { + t.Errorf("generated source does not parse: %v", err) + } +} + +// A member Real in one row and a quantity in a later one declares the unit +// companion, which only the rows with a unit redefine. +func TestGenerateSettlesARealMemberToQuantity(t *testing.T) { + res, err := Generate(Request{ + Package: "Records", Case: "P::check", Provenance: provenance(KindSweep), + Runs: []Run{ + {Iteration: 1, Spell: spell(), Outputs: []runtime.CalcOutputValue{{Name: "x", Value: real(1)}}}, + {Iteration: 2, Spell: spell(), Outputs: []runtime.CalcOutputValue{ + {Name: "x", Value: runtime.NewQuantityValue(&runtime.Quantity{ + Num: semantics.Value{Kind: semantics.ValReal, Real: 2.0}, + Unit: semantics.Unit{Text: "kg"}, + })}, + }}, + }, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + for _, want := range []string{ + "attribute x : ScalarValues::Real;", + "attribute xUnit : ScalarValues::String;", + "attribute :>> x = 1.0;", + `attribute :>> xUnit = "kg";`, + } { + if !strings.Contains(res.Source, want) { + t.Errorf("generated source is missing %q:\n%s", want, res.Source) + } + } + if strings.Count(res.Source, "xUnit = ") != 1 { + t.Errorf("only the row with a unit should redefine xUnit:\n%s", res.Source) + } + if _, err := format.Source("", []byte(res.Source), format.DefaultOptions); err != nil { + t.Errorf("generated source does not parse: %v", err) + } +} + +// Record numbers fill the gaps a package's earlier records leave. +func TestGenerateNumbersIntoTheGaps(t *testing.T) { + res, err := Generate(Request{ + Package: "Records", Case: "P::check", Provenance: provenance(KindSweep), + Existing: Existing{ + Package: true, Definition: true, Taken: map[int]bool{2: true}, + Attributes: map[string]Feature{"load": {TypeFQN: "ScalarValues::Real"}}, + }, + Runs: []Run{ + {Iteration: 1, Spell: spell(), Outputs: []runtime.CalcOutputValue{{Name: "load", Value: real(1)}}}, + {Iteration: 2, Spell: spell(), Outputs: []runtime.CalcOutputValue{{Name: "load", Value: real(2)}}}, + }, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + want := []string{"Records::check_run1", "Records::check_run3"} + if fmt.Sprint(res.Records) != fmt.Sprint(want) { + t.Errorf("records %v, want %v", res.Records, want) + } +} + +// A case name that needs quoting is carried through quoted record names, and +// the def marks the case it records. +func TestGenerateQuotedName(t *testing.T) { + res, err := Generate(Request{ + Package: "Records", Case: "Demo::fuel budget", CaseName: "fuel budget", + Provenance: provenance(KindRun), + Runs: []Run{{Spell: spell(), Outputs: []runtime.CalcOutputValue{{Name: "y", Value: real(3)}}}}, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + for _, want := range []string{ + "part def 'Fuel budgetRun' :> AnalysisRecords::AnalysisRun", + `attribute :>> caseName default = "Demo::fuel budget";`, + "part 'fuel budget_run1' : 'Fuel budgetRun'", + } { + if !strings.Contains(res.Source, want) { + t.Errorf("generated source is missing %q:\n%s", want, res.Source) + } + } + if _, err := format.Source("", []byte(res.Source), format.DefaultOptions); err != nil { + t.Errorf("generated source does not parse: %v", err) + } +} + +// An owner-prefixed stem names the definition and the records of a case whose +// short name a sibling's definition already took. +func TestGenerateOwnerStem(t *testing.T) { + res, err := Generate(Request{ + Package: "Records", Case: "Demo::B::check", + Provenance: provenance(KindRun), + Existing: Existing{Stem: "B_check"}, + Runs: []Run{{Spell: spell(), Outputs: []runtime.CalcOutputValue{{Name: "y", Value: real(2)}}}}, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + for _, want := range []string{ + "part def B_checkRun :> AnalysisRecords::AnalysisRun", + "part B_check_run1 : B_checkRun", + } { + if !strings.Contains(res.Source, want) { + t.Errorf("generated source is missing %q:\n%s", want, res.Source) + } + } +} + +// A member declared ScalarValue by an earlier, unset run accepts a concrete +// type the next run settles it to. +func TestGenerateExistingScalarValueAcceptsASettledType(t *testing.T) { + res, err := Generate(Request{ + Package: "Records", Case: "P::check", Provenance: provenance(KindRun), + Existing: Existing{ + Package: true, Definition: true, + Attributes: map[string]Feature{"x": {TypeFQN: "ScalarValues::ScalarValue"}}, + Stem: "check", + }, + Runs: []Run{{Spell: spell(), Outputs: []runtime.CalcOutputValue{{Name: "x", Value: real(2)}}}}, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if !strings.Contains(res.Source, "attribute :>> x = 2.0;") { + t.Errorf("generated source is missing the redefinition:\n%s", res.Source) + } +} + +// A verification run records what its body and subcases decided: the body's +// verdict beside the case's features, each verdict a VerdictRecord row. +func TestGenerateVerificationRun(t *testing.T) { + res, err := Generate(Request{ + Package: "P::Records", Case: "P::fire", Provenance: provenance(KindRun), + Runs: []Run{{ + Outputs: []runtime.CalcOutputValue{{Name: "margin", Value: real(-200.0)}}, + Verdicts: []runtime.AnalysisVerdict{{Kind: "objective", Name: "thrust", Status: runtime.VerdictNotSatisfied}}, + Verifications: []runtime.VerificationVerdict{ + {Case: "P::fire", Kind: runtime.VerdictFail}, + {Case: "P::cold", Kind: runtime.VerdictPass, Subcase: true}, + {Case: "P::hot", Kind: runtime.VerdictInconclusive, Subcase: true, Detail: "no data"}, + }, + Spell: spell(), + }}, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + for _, want := range []string{ + `attribute :>> verdict = "fail";`, + `part verification1 : AnalysisRecords::VerdictRecord :> verdicts {`, + `attribute :>> kind = "verification";`, + `attribute :>> name = "P::fire";`, + `attribute :>> status = "fail";`, + `part verification2`, + `attribute :>> kind = "subcase";`, + `attribute :>> name = "P::cold";`, + `attribute :>> status = "pass";`, + `part verification3`, + `attribute :>> status = "inconclusive";`, + `attribute :>> detail = "no data";`, + } { + if !strings.Contains(res.Source, want) { + t.Errorf("source is missing %q:\n%s", want, res.Source) + } + } + reparsed := parser.New(source.New("", []byte(res.Source))) + reparsed.ParseFile() + if len(reparsed.Diagnostics) > 0 { + t.Fatalf("generated source does not parse: %v\n%s", reparsed.Diagnostics[0], res.Source) + } +} + +// A run that decided only verification verdicts is still a record worth +// making: verdicts count toward "nothing to record". +func TestGenerateVerificationOnlyRun(t *testing.T) { + res, err := Generate(Request{ + Package: "P::Records", Case: "P::fire", Provenance: provenance(KindRun), + Runs: []Run{{ + Verifications: []runtime.VerificationVerdict{{Case: "P::fire", Kind: runtime.VerdictPass}}, + Spell: spell(), + }}, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if !strings.Contains(res.Source, `attribute :>> verdict = "pass";`) { + t.Errorf("source is missing the verdict:\n%s", res.Source) + } +} + +// An inout records as one member carrying the value the run left in it, +// plus an In companion carrying the value it was bound with. +func TestGenerateInoutRun(t *testing.T) { + res, err := Generate(Request{ + Package: "P::Records", Case: "P::count", Provenance: provenance(KindRun), + Runs: []Run{{ + Inputs: []runtime.InputBinding{{Name: "counter", Value: integer(3)}}, + Outputs: []runtime.CalcOutputValue{{Name: "counter", Value: integer(6)}, {Name: "doubled", Value: integer(6)}}, + Spell: spell(), + }}, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + for _, want := range []string{ + "attribute counter : ScalarValues::Integer;", + "attribute counterIn : ScalarValues::Integer;", + "attribute doubled : ScalarValues::Integer;", + "attribute :>> counter = 6;", + "attribute :>> counterIn = 3;", + "attribute :>> doubled = 6;", + } { + if !strings.Contains(res.Source, want) { + t.Errorf("source is missing %q:\n%s", want, res.Source) + } + } + // The In companion follows its value member. + if strings.Index(res.Source, "attribute counterIn") < strings.Index(res.Source, "attribute counter :") { + t.Errorf("the In companion precedes its member:\n%s", res.Source) + } + reparsed := parser.New(source.New("", []byte(res.Source))) + reparsed.ParseFile() + if len(reparsed.Diagnostics) > 0 { + t.Fatalf("generated source does not parse: %v\n%s", reparsed.Diagnostics[0], res.Source) + } +} + +// A scalar-valued enumeration literal records as the literal it is, not the +// scalar it equals. +func TestGenerateScalarValuedEnumLiteral(t *testing.T) { + file := parser.New(source.New("", []byte( + `package P { enum def Grade { enum high = 3; enum low = 1; } }`))).ParseFile() + scope := symbols.Build(file) + p, _ := scope.LookupLocal("P") + grade, _ := p.Scope.LookupLocal("Grade") + high, ok := grade.Scope.LookupLocal("high") + if !ok { + t.Fatal("enum literal not built") + } + res, err := Generate(Request{ + Package: "P::Records", Case: "P::mix", Provenance: provenance(KindRun), + Runs: []Run{{ + Outputs: []runtime.CalcOutputValue{ + {Name: "g", Value: runtime.EnumeratedValue(high, integer(3))}, + {Name: "half", Value: real(1.5)}, + }, + Spell: spell(), + }}, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + for _, want := range []string{ + "attribute g : P::Grade;", + "attribute :>> g = P::Grade::high;", + } { + if !strings.Contains(res.Source, want) { + t.Errorf("source is missing %q:\n%s", want, res.Source) + } + } +} + +// Integer and Real are one numeric family for the record definition: rows of +// either settle the member to Real, each literal staying its own. +func TestGenerateNumericFamilySettlesToReal(t *testing.T) { + res, err := Generate(Request{ + Package: "P::Records", Case: "P::mix", Provenance: provenance(KindSweep), + Runs: []Run{ + {Outputs: []runtime.CalcOutputValue{{Name: "half", Value: integer(3)}}, Spell: spell()}, + {Outputs: []runtime.CalcOutputValue{{Name: "half", Value: real(3.5)}}, Spell: spell()}, + }, + }) + if err != nil { + t.Fatalf("Generate: %v", err) + } + for _, want := range []string{ + "attribute half : ScalarValues::Real;", + "attribute :>> half = 3;", + "attribute :>> half = 3.5;", + } { + if !strings.Contains(res.Source, want) { + t.Errorf("source is missing %q:\n%s", want, res.Source) + } + } +} + +// A declared Real member accepts Integer values — Integer specializes it — +// while a declared Integer member cannot take a Real back. +func TestGenerateExistingRealAcceptsAnInteger(t *testing.T) { + req := Request{ + Package: "P::Records", Case: "P::mix", Provenance: provenance(KindRun), + Existing: Existing{ + Package: true, Definition: true, + Attributes: map[string]Feature{"half": {TypeFQN: "ScalarValues::Real"}}, + }, + Runs: []Run{{ + Outputs: []runtime.CalcOutputValue{{Name: "half", Value: integer(3)}}, + Spell: spell(), + }}, + } + if _, err := Generate(req); err != nil { + t.Fatalf("an Integer under a declared Real: %v", err) + } + req.Existing.Attributes["half"] = Feature{TypeFQN: "ScalarValues::Integer"} + req.Runs[0].Outputs[0].Value = real(3.5) + if _, err := Generate(req); err == nil { + t.Error("a Real under a declared Integer: want an error") + } +} + +// A quantity member takes a plain-number row in either order: the row keeps +// its literal and takes no unit. +func TestGenerateQuantityAndPlainNumbers(t *testing.T) { + kg := runtime.NewQuantityValue(&runtime.Quantity{ + Num: semantics.Value{Kind: semantics.ValReal, Real: 3.0}, + Unit: semantics.Unit{Text: "kg"}, + }) + orders := map[string][]runtime.Value{ + "integer then quantity": {integer(2), kg}, + "quantity then integer": {kg, integer(2)}, + } + for name, values := range orders { + var runs []Run + for _, v := range values { + runs = append(runs, Run{ + Outputs: []runtime.CalcOutputValue{{Name: "x", Value: v}}, + Spell: spell(), + }) + } + res, err := Generate(Request{ + Package: "P::Records", Case: "P::mix", Provenance: provenance(KindSweep), Runs: runs, + }) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + for _, want := range []string{ + "attribute x : ScalarValues::Real;", + "attribute xUnit : ScalarValues::String;", + "attribute :>> x = 2;", + "attribute :>> x = 3.0;", + `attribute :>> xUnit = "kg";`, + } { + if !strings.Contains(res.Source, want) { + t.Errorf("%s: source is missing %q:\n%s", name, want, res.Source) + } + } + if strings.Count(res.Source, "xUnit = ") != 1 { + t.Errorf("%s: a plain-number row wrote a unit:\n%s", name, res.Source) + } + } +} + +// An inout whose two sides mix a quantity with a plain number settles both +// members to the quantity shape, each with its unit companion declared. +func TestGenerateInoutQuantityAndPlainNumber(t *testing.T) { + kg := runtime.NewQuantityValue(&runtime.Quantity{ + Num: semantics.Value{Kind: semantics.ValReal, Real: 3.0}, + Unit: semantics.Unit{Text: "kg"}, + }) + sides := map[string][2]runtime.Value{ + "integer out, quantity in": {integer(2), kg}, + "real out, quantity in": {real(2.5), kg}, + "quantity out, integer in": {kg, integer(2)}, + "quantity out, real in": {kg, real(2.5)}, + } + for name, v := range sides { + res, err := Generate(Request{ + Package: "P::Records", Case: "P::mix", Provenance: provenance(KindRun), + Runs: []Run{{ + Inputs: []runtime.InputBinding{{Name: "x", Value: v[1]}}, + Outputs: []runtime.CalcOutputValue{{Name: "x", Value: v[0]}}, + Spell: spell(), + }}, + }) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + for _, want := range []string{ + "attribute x : ScalarValues::Real;", + "attribute xUnit : ScalarValues::String;", + "attribute xIn : ScalarValues::Real;", + "attribute xInUnit : ScalarValues::String;", + `Unit = "kg";`, + } { + if !strings.Contains(res.Source, want) { + t.Errorf("%s: source is missing %q:\n%s", name, want, res.Source) + } + } + if strings.Count(res.Source, `Unit = "kg";`) != 1 { + t.Errorf("%s: want exactly one unit redefinition:\n%s", name, res.Source) + } + } +} diff --git a/internal/exec/analysis/record/testdata/single_run.sysml.golden b/internal/exec/analysis/record/testdata/single_run.sysml.golden new file mode 100644 index 000000000..157351d91 --- /dev/null +++ b/internal/exec/analysis/record/testdata/single_run.sysml.golden @@ -0,0 +1,50 @@ +package Records { + part def ScoutBudgetRun :> AnalysisRecords::AnalysisRun { + attribute :>> caseName default = "P::scoutBudget"; + attribute burnTime : ScalarValues::Real; + attribute trials : ScalarValues::Integer; + attribute crewOk : ScalarValues::Boolean; + attribute label : ScalarValues::String; + attribute grade : P::Fuel; + attribute tank : ScalarValues::Real; + attribute tankUnit : ScalarValues::String; + ref part target; + attribute fuelUsed : ScalarValues::Real; + attribute memo : ScalarValues::ScalarValue; + attribute plan : ScalarValues::String; + } + part scoutBudget_run1 : ScoutBudgetRun { + @AnalysisRecords::RecordedRun { + runAt = "2026-01-01T00:00:00Z"; + tool = "sysml test"; + command = "%record P::check"; + kind = "run"; + } + attribute :>> caseName = "P::scoutBudget"; + attribute :>> kind = "run"; + attribute :>> 'objective' = "satisfied"; + ref :>> 'subject' = P::scout; + attribute :>> subjectName = "P::scout"; + attribute :>> burnTime = 3.0; + attribute :>> trials = 8; + attribute :>> crewOk = true; + attribute :>> label = "a \"b\""; + attribute :>> grade = P::Fuel::leaded; + attribute :>> tank = 12.5; + attribute :>> tankUnit = "kg"; + ref :>> target = P::scout; + attribute :>> fuelUsed = 12.5; + attribute :>> plan = "[]"; + part verdict1 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "objective"; + attribute :>> name = "fuelFits"; + attribute :>> status = "satisfied"; + attribute :>> detail = "holds"; + } + part verdict2 : AnalysisRecords::VerdictRecord :> verdicts { + attribute :>> kind = "assertion"; + attribute :>> name = "crewReady"; + attribute :>> status = "not satisfied"; + } + } +} diff --git a/internal/exec/analysis/record/testdata/sweep_runs.sysml.golden b/internal/exec/analysis/record/testdata/sweep_runs.sysml.golden new file mode 100644 index 000000000..294e8da3e --- /dev/null +++ b/internal/exec/analysis/record/testdata/sweep_runs.sysml.golden @@ -0,0 +1,44 @@ +package Records { + part check_run4 : CheckRun { + @AnalysisRecords::RecordedRun { + runAt = "2026-01-01T00:00:00Z"; + tool = "sysml test"; + command = "%record P::check"; + kind = "sweep"; + } + attribute :>> caseName = "P::check"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "undecided"; + attribute :>> iteration = 1; + attribute :>> load = 1.0; + attribute :>> done = true; + } + part check_run5 : CheckRun { + @AnalysisRecords::RecordedRun { + runAt = "2026-01-01T00:00:00Z"; + tool = "sysml test"; + command = "%record P::check"; + kind = "sweep"; + } + attribute :>> caseName = "P::check"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "undecided"; + attribute :>> iteration = 2; + attribute :>> load = 3.0; + attribute :>> done = false; + } + part check_run6 : CheckRun { + @AnalysisRecords::RecordedRun { + runAt = "2026-01-01T00:00:00Z"; + tool = "sysml test"; + command = "%record P::check"; + kind = "sweep"; + } + attribute :>> caseName = "P::check"; + attribute :>> kind = "sweep"; + attribute :>> 'objective' = "undecided"; + attribute :>> iteration = 3; + attribute :>> load = 5.0; + attribute :>> done = true; + } +} diff --git a/internal/exec/analysis/record/testdata/trade_run.sysml.golden b/internal/exec/analysis/record/testdata/trade_run.sysml.golden new file mode 100644 index 000000000..c3c3ca34f --- /dev/null +++ b/internal/exec/analysis/record/testdata/trade_run.sysml.golden @@ -0,0 +1,44 @@ +package P { + package Records { + part def ChooseRun :> AnalysisRecords::AnalysisRun { + attribute :>> caseName default = "P::choose"; + attribute best : ScalarValues::Real; + } + part choose_run1 : ChooseRun { + @AnalysisRecords::RecordedRun { + runAt = "2026-01-01T00:00:00Z"; + tool = "sysml test"; + command = "%record P::check"; + kind = "trade"; + } + attribute :>> caseName = "P::choose"; + attribute :>> kind = "trade"; + attribute :>> 'objective' = "undecided"; + attribute :>> subjectName = "P::fleet"; + attribute :>> best = 1.0; + part evaluation1 : AnalysisRecords::EvaluationRecord :> evaluations { + attribute :>> 'function' = "P::score"; + attribute :>> alternative = "1.0"; + attribute :>> score = 0.75; + attribute :>> result = "0.75"; + attribute :>> selected = true; + attribute :>> tied = false; + } + part evaluation2 : AnalysisRecords::EvaluationRecord :> evaluations { + attribute :>> 'function' = "P::score"; + attribute :>> alternative = "2.0"; + attribute :>> score = 0.75; + attribute :>> result = "0.75"; + attribute :>> selected = false; + attribute :>> tied = true; + } + part evaluation3 : AnalysisRecords::EvaluationRecord :> evaluations { + attribute :>> 'function' = "P::score"; + attribute :>> alternative = "3.0"; + attribute :>> selected = false; + attribute :>> tied = false; + attribute :>> error = "boom"; + } + } + } +} diff --git a/internal/exec/runtime/analysis_inputs_test.go b/internal/exec/runtime/analysis_inputs_test.go new file mode 100644 index 000000000..6759b0983 --- /dev/null +++ b/internal/exec/runtime/analysis_inputs_test.go @@ -0,0 +1,295 @@ +package runtime + +import ( + "reflect" + "strings" + "testing" +) + +// The inputs a run reports are the values its parameters were bound to, in +// declaration order: arguments by position or by name, a default evaluated +// where none was given, and the subject parameter never among them. +func TestAnalysisResultInputsAreTheBoundParameters(t *testing.T) { + ctx, scope := analysisFixture(t, ` + package test { + private import ScalarValues::*; + part def Probe; + individual def probe :> Probe; + analysis def Check { + subject s : Probe; + in burnTime : Real; + in margin : Real = 1.5; + in flag : Boolean = true; + out fuelUsed : Real = burnTime + margin; + } + }`) + sym := requirementNamed(t, scope, "Check") + probe, err := ctx.Instantiate(requirementNamed(t, scope, "probe")) + if err != nil { + t.Fatal(err) + } + + textOf := func(inputs []InputBinding) []string { + out := make([]string, len(inputs)) + for i, in := range inputs { + out[i] = in.Name + "=" + FormatValue(in.Value) + } + return out + } + + result, err := ctx.RunAnalysis(sym, AnalysisArgs{ + Subject: probe, + Positional: []Value{constValue(drawnReal(3.0))}, + }, scope, nil) + if err != nil { + t.Fatal(err) + } + want := []string{"burnTime=3.0", "margin=1.5", "flag=true"} + if got := textOf(result.Inputs); !reflect.DeepEqual(got, want) { + t.Errorf("inputs %v, want %v", got, want) + } + + result, err = ctx.RunAnalysis(sym, AnalysisArgs{ + Subject: probe, + Named: map[string]Value{"margin": constValue(drawnReal(0.25)), "burnTime": constValue(drawnReal(4.0))}, + }, scope, nil) + if err != nil { + t.Fatal(err) + } + want = []string{"burnTime=4.0", "margin=0.25", "flag=true"} + if got := textOf(result.Inputs); !reflect.DeepEqual(got, want) { + t.Errorf("inputs %v, want %v", got, want) + } +} + +// A sweep row carries the inputs its run bound, the row's own overlay included. +func TestSweepRowCarriesTheRunsInputs(t *testing.T) { + ctx, _ := analysisFixture(t, `package test { calc def Idle { return k : Integer = 0; } }`) + bound := []InputBinding{{Name: "n", Value: constValue(drawnReal(7.0))}} + row := runSweepRow(ctx, []SweepBinding{{Param: "n", Value: constValue(drawnReal(7.0))}}, + func(*Context, []SweepBinding) (SweepRunResult, error) { + return SweepRunResult{Inputs: bound}, nil + }) + if !reflect.DeepEqual(row.Inputs, bound) { + t.Errorf("row inputs %v, want %v", row.Inputs, bound) + } +} + +// A Monte Carlo run reports the inputs its iteration bound and every declared +// output of that iteration, `observed` appended when it declares no output. +func TestMonteCarloRunInputsAndIterationOutputs(t *testing.T) { + ctx, scope := analysisFixture(t, ` + package test { + private import ScalarValues::*; + private import RandomFunctions::*; + part def Probe { + attribute t : Real; + action settle { first start; then assign t := uniform(1.0, 5.0); then done; } + } + individual def probe :> Probe; + analysis def Mc :> Simulation::MonteCarlo { + subject analysed : Probe; + in gain : Real = 2.0; + perform action run ::> analysed.settle; + attribute :>> observed : Real = analysed.t; + return Mean : Real = mean; + } + }`) + sym := requirementNamed(t, scope, "Mc") + probe, err := ctx.Instantiate(requirementNamed(t, scope, "probe")) + if err != nil { + t.Fatal(err) + } + ctx.SetModelSeed(RunSeed(1, 1)) + run, err := ctx.ObserveMonteCarlo(sym, AnalysisArgs{Subject: probe}, scope, nil) + if err != nil { + t.Fatal(err) + } + if len(run.Inputs) != 1 || run.Inputs[0].Name != "gain" { + t.Fatalf("inputs %+v, want the one binding of gain", run.Inputs) + } + var names []string + for _, out := range run.Outputs { + names = append(names, out.Name) + } + // The statistics are bound over the sample, so the one output this + // iteration establishes is the observed value; the return reading a + // statistic is unread until the conclusion supplies it. + want := []string{"observed"} + if !reflect.DeepEqual(names, want) { + t.Errorf("outputs %v, want %v", names, want) + } + if len(run.Unread) != 0 { + t.Errorf("the stat-bound return is the sample's, not unread: %v", run.Unread) + } +} + +// An output erroring for its own reason — not the statistics the sample has +// not supplied — is reported on the run, which still completes and observes. +func TestMonteCarloRunReportsAnOutputError(t *testing.T) { + ctx, scope := analysisFixture(t, ` + package test { + private import ScalarValues::*; + private import RandomFunctions::*; + part def Probe { + attribute t : Real; + action settle { first start; then assign t := uniform(1.0, 5.0); then done; } + } + individual def probe :> Probe; + analysis def Mc :> Simulation::MonteCarlo { + subject analysed : Probe; + perform action run ::> analysed.settle; + attribute :>> observed : Real = analysed.t; + return Mean : Real = mean; + out Bad : Real = 1.0 / 0.0; + } + } + `) + sym := requirementNamed(t, scope, "Mc") + probe, err := ctx.Instantiate(requirementNamed(t, scope, "probe")) + if err != nil { + t.Fatal(err) + } + ctx.SetModelSeed(RunSeed(1, 1)) + run, err := ctx.ObserveMonteCarlo(sym, AnalysisArgs{Subject: probe}, scope, nil) + if err != nil { + t.Fatal(err) + } + if run.Unread["Bad"] == nil { + t.Error("an output erroring for its own reason vanished") + } + if run.Observed.Kind != ValConst { + t.Errorf("the run's observed is invalid: %v", run.Observed) + } +} + +// The outputs an observation captures are read in a probe: capturing them +// draws nothing, so the conclusion over the sample — and the draws taken to +// make it — are the ones a capture-free observation makes, whether or not an +// output is a random draw of its own. +func TestMonteCarloIterationOutputsMemoizeNothing(t *testing.T) { + sample := func(extraOutput string) (AnalysisResult, []DrawTaken, error) { + ctx, scope := analysisFixture(t, ` + package test { + private import ScalarValues::*; + private import RandomFunctions::*; + part def Probe { + attribute t : Real; + action settle { first start; then assign t := uniform(1.0, 5.0); then done; } + } + individual def probe :> Probe; + analysis def Mc :> Simulation::MonteCarlo { + subject analysed : Probe; + perform action run ::> analysed.settle; + attribute :>> observed : Real = analysed.t; + return Mean : Real = mean;`+extraOutput+` + } + }`) + sym := requirementNamed(t, scope, "Mc") + probe, err := ctx.Instantiate(requirementNamed(t, scope, "probe")) + if err != nil { + t.Fatal(err) + } + ctx.SetModelSeed(RunSeed(1, 1)) + var runs []*MonteCarloRun + for i := int64(1); i <= 3; i++ { + run, err := ctx.ObserveMonteCarlo(sym, AnalysisArgs{Subject: probe}, scope, nil) + if err != nil { + t.Fatal(err) + } + if len(run.run.outputs) != 0 { + t.Fatalf("capturing the outputs memoized into the run: %v", run.run.outputs) + } + runs = append(runs, run) + } + stats, err := MonteCarloSample(runs) + if err != nil { + t.Fatal(err) + } + res, err := ConcludeMonteCarlo(runs, stats) + var observed []DrawTaken + for _, d := range ctx.DrawsTaken() { + if strings.HasPrefix(d.What, "uniform(1.0") { + observed = append(observed, d) + } + } + return res, observed, err + } + + plain, drawsPlain, err := sample("") + if err != nil { + t.Fatal(err) + } + drawn, drawsDrawn, err := sample(` + out Again : Real = uniform(0.0, 1.0);`) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(drawsDrawn, drawsPlain) { + t.Errorf("capturing a draw-made output moved the sample's draws: %v vs %v", drawsDrawn, drawsPlain) + } + for i, out := range plain.Outputs { + if i < len(drawn.Outputs) && drawn.Outputs[i].Name == out.Name && FormatValue(drawn.Outputs[i].Value) != FormatValue(out.Value) { + t.Errorf("output %s changed: %v vs %v", out.Name, drawn.Outputs[i].Value, out.Value) + } + } +} + +// An output that reads a statistic is the sample's, as surely as the +// statistic is: the rows carry neither it nor an error for it. An output that +// fails for a reason of its own is in Unread. +func TestMonteCarloStatBoundOutputsAreTheSamples(t *testing.T) { + ctx, scope := analysisFixture(t, ` + package test { + private import ScalarValues::*; + private import RandomFunctions::*; + part def Probe { + attribute t : Real; + action settle { first start; then assign t := uniform(1.0, 5.0); then done; } + } + individual def probe :> Probe; + analysis def Mc :> Simulation::MonteCarlo { + subject analysed : Probe; + perform action run ::> analysed.settle; + attribute :>> observed : Real = analysed.t; + return Mean : Real = mean; + out Dev : Real = deviation + 1.0; + out Half : Real = Mean / 2.0; + out Bad : Real; + out Ratio : Real = 1.0 / (analysed.t - analysed.t); + } + }`) + sym := requirementNamed(t, scope, "Mc") + probe, err := ctx.Instantiate(requirementNamed(t, scope, "probe")) + if err != nil { + t.Fatal(err) + } + ctx.SetModelSeed(RunSeed(1, 1)) + var runs []*MonteCarloRun + for i := 0; i < 2; i++ { + run, err := ctx.ObserveMonteCarlo(sym, AnalysisArgs{Subject: probe}, scope, nil) + if err != nil { + t.Fatal(err) + } + runs = append(runs, run) + } + for _, run := range runs { + for _, out := range run.Outputs { + for _, statBound := range []string{"Mean", "Dev", "Half"} { + if out.Name == statBound { + t.Errorf("run %d carries the sample's %s as its own", run.Number, statBound) + } + } + } + for _, statBound := range []string{"Mean", "Dev", "Half"} { + if run.Unread[statBound] != nil { + t.Errorf("run %d's %s is the sample's, not unread: %v", run.Number, statBound, run.Unread[statBound]) + } + } + for _, failed := range []string{"Bad", "Ratio"} { + if run.Unread[failed] == nil { + t.Errorf("run %d's %s failed for its own reason but is not in Unread", run.Number, failed) + } + } + } +} diff --git a/internal/exec/runtime/analysis_run.go b/internal/exec/runtime/analysis_run.go index d8b37023d..98eabdd0d 100644 --- a/internal/exec/runtime/analysis_run.go +++ b/internal/exec/runtime/analysis_run.go @@ -221,6 +221,13 @@ type AnalysisEvaluation struct { Tied bool } +// InputBinding is the value one input parameter of a case was bound to for a +// run: its argument, or the default its declaration evaluated to. +type InputBinding struct { + Name string + Value Value +} + // AnalysisResult is what one run of an analysis case produced: its output // values in declaration order, and the verdict of each objective and assertion. type AnalysisResult struct { @@ -231,6 +238,10 @@ type AnalysisResult struct { // taken from the enclosing case; nil for a case declaring no subject. Subject *Instance + // Inputs are the values the run bound the case's input parameters to, in + // declaration order, the subject parameter excluded: what the body ran with. + Inputs []InputBinding + // Outputs are the case's out and return parameters, in declaration order; // a value the body returned into an unnamed result is named "result". Outputs []CalcOutputValue @@ -300,7 +311,7 @@ func (ctx *Context) runCase(sym *symbols.Symbol, args AnalysisArgs, scope *symbo // The outputs computed before one failed stay reported; the verdicts and the // pick do not, since the case established neither. - result := AnalysisResult{Case: shape.Name, Subject: run.boundSubject(ctx)} + result := AnalysisResult{Case: shape.Name, Subject: run.boundSubject(ctx), Inputs: run.inputs()} outputs, err := run.outputValues(ctx) result.Outputs = outputs if err != nil { diff --git a/internal/exec/runtime/calc_usage.go b/internal/exec/runtime/calc_usage.go index 74e88cd77..676f139a1 100644 --- a/internal/exec/runtime/calc_usage.go +++ b/internal/exec/runtime/calc_usage.go @@ -453,6 +453,29 @@ type calcRun struct { // perf is the case's performance, whose steps an output binding reads by name // (`step.pin`); nil for a calc, which performs none. perf *actionFrame + // boundInputs are the values the run's input parameters were bound to. + boundInputs []InputBinding +} + +// boundInputs are the values each non-subject input parameter of shape was +// bound to in env, in declaration order. +func boundInputs(shape *calcShape, env frame) []InputBinding { + var inputs []InputBinding + for i := range shape.Params { + param := &shape.Params[i] + if param.IsSubject { + continue + } + if value, ok := env.lookup(param.Name); ok { + inputs = append(inputs, InputBinding{Name: param.Name, Value: value}) + } + } + return inputs +} + +// inputs are the values the run's input parameters were bound to. +func (run *calcRun) inputs() []InputBinding { + return run.boundInputs } // newCalcRun holds the environment one evaluation of a calc computed. @@ -574,6 +597,9 @@ type calcUsageStart struct { // deferResults leaves the results ending the steps unrun, for a Monte Carlo to // evaluate over its sample once the run's observation is in. deferResults bool + // inputs are the values each input parameter was bound to, captured before the + // body runs so a body's assignments cannot rewrite what it was given. + inputs []InputBinding } // beginCalcUsage binds the usage's inputs and makes ready to run its body; the @@ -630,6 +656,7 @@ func (ctx *Context) startCalcUsage(shape *calcShape, key calcUsageKey, reader *E if err != nil { return nil, err } + start.inputs = boundInputs(shape, start.env) start.host = &calcStmtHost{ctx: ctx, shape: shape, self: reader.self} // A usage nested in a behavior body computes over that body's bindings, as // an invocation of it does. @@ -885,7 +912,7 @@ func (ctx *Context) runCalcUsage(start *calcUsageStart) (*calcRun, error) { run := newCalcRun(shape, reader.scope, reader.self, env) run.outer, run.result, run.returned = nested, result, returned - run.activation, run.perf = engine.activation, host.performance() + run.activation, run.perf, run.boundInputs = engine.activation, host.performance(), start.inputs // The returned value is the result parameter's, read under its name or as // `result`; every other output states its own value, never the returned one. if returned { @@ -938,9 +965,7 @@ func (run *calcRun) value(ctx *Context, out calcOutput) (Value, error) { } } if out.Value == nil { - return Value{}, fmt.Errorf( - "%w: output %s of %s", ErrOutputNotAssigned, run.outputDescription(out), run.shape.Label, - ) + return Value{}, &UnassignedOutputError{Output: run.outputDescription(out), Calc: run.shape.Label} } if run.computing[out.Name] { return Value{}, fmt.Errorf( diff --git a/internal/exec/runtime/errors.go b/internal/exec/runtime/errors.go index 1af0f171a..56174c32b 100644 --- a/internal/exec/runtime/errors.go +++ b/internal/exec/runtime/errors.go @@ -505,6 +505,21 @@ type budgetExceededError struct { errs []error } +// UnassignedOutputError names the declared output a run could not read +// because the activation never assigned it, and the calc that declares it. +type UnassignedOutputError struct { + Output string + Calc string +} + +// Error keeps the diagnostic text an ErrOutputNotAssigned carried. +func (e *UnassignedOutputError) Error() string { + return fmt.Sprintf("%s: output %s of %s", ErrOutputNotAssigned, e.Output, e.Calc) +} + +// Unwrap reports the failure as an ErrOutputNotAssigned. +func (e *UnassignedOutputError) Unwrap() error { return ErrOutputNotAssigned } + func (e *budgetExceededError) Error() string { return e.message } func (e *budgetExceededError) Unwrap() []error { return e.errs } diff --git a/internal/exec/runtime/montecarlo_case.go b/internal/exec/runtime/montecarlo_case.go index 0147380a9..e3a4bc30d 100644 --- a/internal/exec/runtime/montecarlo_case.go +++ b/internal/exec/runtime/montecarlo_case.go @@ -75,6 +75,15 @@ type MonteCarloRun struct { // Observed is the value the run's `observed` came to, null when the run left it unbound. Observed Value + // Inputs are the values the run bound the case's input parameters to, in + // declaration order. Outputs are every declared output of this iteration + // as the iteration established it — the sample's statistics excluded — with + // the observed feature appended when it declares no output of its own. + // Unread holds the outputs that could not be read, by name. + Inputs []InputBinding + Outputs []CalcOutputValue + Unread map[string]error + // Verdicts are the case's checks over this run: on its own until ConcludeMonteCarlo // settles them over the sample, which keeps the checks that are the sample's alone. Verdicts []AnalysisVerdict @@ -130,8 +139,10 @@ func (ctx *Context) ObserveMonteCarlo(sym *symbols.Symbol, args AnalysisArgs, sc Case: shape.Name, Subject: run.boundSubject(ctx), Observed: observed, + Inputs: run.inputs(), } r.Verdicts = r.checks() + r.Outputs, r.Unread = r.iterationOutputs() r.left = make([]bool, len(r.Verdicts)) for i, v := range r.Verdicts { r.left[i] = v.Status == VerdictUndecided @@ -139,6 +150,61 @@ func (ctx *Context) ObserveMonteCarlo(sym *symbols.Symbol, args AnalysisArgs, sc return r, nil } +// iterationOutputs are the values this run's declared outputs came to, the +// observed feature appended when it is not among them. The outputs that are +// the sample's — runs, mean, deviation and outOfSpec — are left out whatever +// their binding; any other output that cannot be read is in the error map +// returned beside them. The whole read runs in a probe: nothing it evaluates +// is kept, drawn or written in the run or its context. +func (r *MonteCarloRun) iterationOutputs() ([]CalcOutputValue, map[string]error) { + ctx := r.ctx + defer ctx.beginRun()() + defer ctx.beginProbe()() + kept := r.run.outputs + r.run.outputs = maps.Clone(kept) + defer func() { r.run.outputs = kept }() + stats := map[string]bool{} + for _, feature := range []string{ + MonteCarloRunsOutput, MonteCarloMeanOutput, + MonteCarloDeviationOutput, MonteCarloOutOfSpecOutput, + } { + if name, ok := ctx.monteCarloMember(r.run.shape, feature); ok { + stats[name] = true + } + } + var outputs []CalcOutputValue + unread := map[string]error{} + for _, out := range r.run.shape.Outputs { + if out.Name == "" || stats[out.Name] { + continue + } + value, err := r.run.output(ctx, out.Name) + if err != nil { + var u *UnassignedOutputError + // Reading an unbound statistic through the binding leaves the + // output to the conclusion as much as the statistic itself. + if !(errors.As(err, &u) && stats[u.Output]) { + unread[out.Name] = err + } + continue + } + outputs = append(outputs, CalcOutputValue{Name: out.Name, Value: value}) + } + if name, ok := ctx.monteCarloMember(r.run.shape, monteCarloObserved); ok { + seen := false + for _, out := range outputs { + if out.Name == name { + seen = true + break + } + } + if !seen { + outputs = append(outputs, CalcOutputValue{Name: name, Value: r.Observed}) + } + } + return outputs, unread +} + // checks decides the case's checks over the run as it stands. The outputs they read are // evaluated for them alone, not kept: the conclusion evaluates each once, over the sample. func (r *MonteCarloRun) checks() []AnalysisVerdict { diff --git a/internal/exec/runtime/sweep.go b/internal/exec/runtime/sweep.go index 92da3b912..5c380e1b0 100644 --- a/internal/exec/runtime/sweep.go +++ b/internal/exec/runtime/sweep.go @@ -121,6 +121,9 @@ func (p SweepPlan) Drawn() bool { return p.Sampled || (p.IsMonteCarlo() && !p.Se type SweepRunResult struct { Outputs []CalcOutputValue Verdicts []AnalysisVerdict + // Inputs are the values the run bound the case's input parameters to, in + // declaration order, the row's own binding included. + Inputs []InputBinding // The object the run was about, where a case ran on one. Subject *Instance // Evaluations are the applications the run made of a calc held as a value, @@ -142,6 +145,7 @@ type SweepRow struct { // The object this run's verdicts are about, where a case ran on one. Subject *Instance Evaluations []AnalysisEvaluation + Inputs []InputBinding Elapsed time.Duration Err error // Context is the context the run was made in, which its outputs, subject and @@ -211,6 +215,7 @@ func runSweepRow(ctx *Context, bindings []SweepBinding, run SweepRun) SweepRow { row.Err = err row.Outputs, row.Verdicts = result.Outputs, result.Verdicts row.Subject, row.Evaluations = result.Subject, result.Evaluations + row.Inputs = result.Inputs return row } diff --git a/internal/frontend/repl/analysis.go b/internal/frontend/repl/analysis.go index 22e5b4e2f..39e8483fb 100644 --- a/internal/frontend/repl/analysis.go +++ b/internal/frontend/repl/analysis.go @@ -99,11 +99,17 @@ func (s *Session) doAnalysis(tail string) ([]string, bool, error) { // after evaluating some of what it declares reports those evaluations and the // verdicts left undecided beneath the error. func (s *Session) analysisVerdict(inv analysisInvocation) Verdict { + run, err := s.runAnalysis(inv) + return s.caseVerdict(inv, run, err) +} + +// caseVerdict reports a run of the case inv names, run already made; err is the +// error the run ended with, nil when it completed. +func (s *Session) caseVerdict(inv analysisInvocation, run caseRun, err error) Verdict { label := inv.name if inv.argText != "" { label += "(" + strings.TrimSpace(inv.argText) + ")" } - run, err := s.runAnalysis(inv) if err != nil { verdict := unresolvedVerdict(label, err.Error()) s.reportCaseRun(&verdict, run.result) diff --git a/internal/frontend/repl/merge.go b/internal/frontend/repl/merge.go index b56cff4a6..be66dc868 100644 --- a/internal/frontend/repl/merge.go +++ b/internal/frontend/repl/merge.go @@ -39,25 +39,46 @@ type edit struct { // // The returned spans locate the submitted text inside the merged result, so a // report still covers what was typed rather than the whole absorbed snippet. -func (s *Session) mergeSubmission(src string, root *ast.RootNamespace, comments string) (string, []source.Span, dropReport, bool) { +// The last result reports the merge rewrote a snippet where it stands: the +// submission's text is wholly inside it, so the caller appends nothing. +func (s *Session) mergeSubmission(src string, root *ast.RootNamespace, comments string) (string, []source.Span, dropReport, bool, bool) { newDecl, ok := soleNamespace(src, root) if !ok || len(newDecl.members) == 0 { - return "", nil, dropReport{}, false - } + return "", nil, dropReport{}, false, false + } + // A record nests its target package inside the same-named top namespace + // several files may open: the file already holding the deepest prefix of + // that nesting is where it belongs. + chain := namespaceChain(src, newDecl) + target, depth := -1, -1 + var oldDecl nsDecl for i, sn := range s.snippets { // Only what the prompt typed in an earlier submission merges: a loaded // file keeps its identity, so re-typing its package supersedes it, and // two snippets of one submission are both part of that submission. A // masked submission is not merged into either: its text is not analyzed, // so folding it in would put what the parser could not read back into the - // buffer. - if sn.origin != "" || sn.gen == s.version || sn.open { + // buffer. A recorded run is the one exception: it merges into a loaded + // file's package, which keeps its file's identity on the result. + if sn.gen == s.version || sn.open || (sn.origin != "" && !s.recordMerge) { continue } - oldDecl, ok := namedNamespace(sn.src, newDecl.name) + cand, ok := namedNamespace(sn.src, newDecl.name) // A different header is a different declaration, whatever it names: it // replaces the old one rather than adding to a body it did not write. - if !ok || oldDecl.header != newDecl.header { + if !ok || cand.header != newDecl.header { + continue + } + if !s.recordMerge { + target, oldDecl = i, cand + break + } + if d := namespaceDepth(sn.src, cand, chain); d > depth { + target, depth, oldDecl = i, d, cand + } + } + for i, sn := range s.snippets { + if i != target { continue } edits, replaced, gone := mergeEdits(sn.src, oldDecl, src, newDecl, newDecl.name) @@ -70,10 +91,18 @@ func (s *Session) mergeSubmission(src string, root *ast.RootNamespace, comments // even when the body it re-typed added nothing new. edits = append(edits, edit{start: oldDecl.start, end: oldDecl.start, own: true}) merged, own := applyEdits(sn.src, edits) + if sn.origin != "" { + // A file's text is updated where it is, its origin and key kept, so + // a later reload of the file still supersedes it; the submission's + // own text is wholly inside it and is not appended again. + names := declaredNames(parser.New(source.New(parseDocName(sn.origin), []byte(merged))).ParseFile()) + s.snippets[i] = snippet{src: merged, names: names, origin: sn.origin, key: sn.key, gen: s.version, own: own} + return "", nil, dropReport{merged: true, decl: newDecl.desc, lost: replaced, gone: gone}, true, true + } s.snippets = append(s.snippets[:i:i], s.snippets[i+1:]...) - return merged, own, dropReport{merged: true, decl: newDecl.desc, lost: replaced, gone: gone}, true + return merged, own, dropReport{merged: true, decl: newDecl.desc, lost: replaced, gone: gone}, true, false } - return "", nil, dropReport{}, false + return "", nil, dropReport{}, false, false } // reopenedNamespaces reports the namespaces a loaded file opens that another @@ -143,6 +172,48 @@ func soleNamespace(src string, root *ast.RootNamespace) (nsDecl, bool) { return namespaceDeclOf(src, root.Members[0]) } +// namespaceChain names the nested namespace declarations inside decl, top to +// innermost — the package nesting a generated record writes. +func namespaceChain(src string, decl nsDecl) []string { + var segs []string + for { + var next nsDecl + for _, m := range decl.members { + if sub, ok := namespaceDeclOf(src, m); ok { + next = sub + break + } + } + if next.name == "" { + return segs + } + segs = append(segs, next.name) + decl = next + } +} + +// namespaceDepth reports how many of the chain's segments decl's body already +// holds as nested namespaces, counting from the first until one is missing. +func namespaceDepth(src string, decl nsDecl, segs []string) int { + for i, seg := range segs { + found := false + for _, m := range decl.members { + if memberName(m) != seg { + continue + } + if sub, ok := namespaceDeclOf(src, m); ok { + decl = sub + found = true + break + } + } + if !found { + return i + } + } + return len(segs) +} + // namedNamespace finds the namespace declaration of the given name in an // accepted snippet. It reports false unless exactly one member declares that // name, so an ambiguous snippet keeps the replacement behavior. diff --git a/internal/frontend/repl/meta.go b/internal/frontend/repl/meta.go index d9af679eb..64e8761a5 100644 --- a/internal/frontend/repl/meta.go +++ b/internal/frontend/repl/meta.go @@ -123,6 +123,7 @@ func opensName(sofar string, rest []rune) bool { const ( cmdQuery = "%query" cmdAnalysis = "%analysis" + cmdRecord = "%record" cmdInvoke = "%invoke" cmdSweep = "%sweep" cmdSamples = "%samples" @@ -198,6 +199,7 @@ var metaCommandTable = []metaCommand{ {name: "%calc", group: groupBehavioral, args: " ", desc: "invoke a calculation with arguments"}, {name: cmdAnalysis, group: groupBehavioral, args: "[()] []", desc: "run an analysis case and report its outputs and the verdict of its objective; arguments bind its inputs and an object is its subject"}, + {name: cmdRecord, group: groupBehavioral, args: "[()] [] [into ]", desc: "run an analysis case as %analysis does and record the run into the model as AnalysisRecords elements, into the package named or a Records package beside the case's"}, {name: cmdSweep, group: groupBehavioral, args: "[()] []

=..[:]...", desc: "run an analysis case or calc once per value of each range, one run per row of the cartesian product, and print the table"}, {name: cmdSamples, group: groupBehavioral, args: " [()] []

=.....", desc: "run an analysis case or calc over values drawn uniformly from each range with the given seed, and print the table"}, {name: cmdRuns, group: groupBehavioral, args: " [] [...]", desc: "run an action times, each run's modeled randomness seeded from the given seed — left out under %draws min, max or average — and print the table of the observables with each one's distribution"}, @@ -497,6 +499,11 @@ func (s *Session) metaModelCommand(fields []string, line string) (metaResult, bo return metaOut([]string{analysisUsage}, false, nil), true } return metaOut(s.doAnalysis(strings.TrimPrefix(strings.TrimSpace(line), cmdAnalysis))), true + case cmdRecord: + if len(fields) < 2 { + return metaOut([]string{recordUsage}, false, nil), true + } + return metaOut(s.doRecord(strings.TrimPrefix(strings.TrimSpace(line), cmdRecord))), true case cmdSweep: if len(fields) < 2 { return metaOut([]string{sweepUsage}, false, nil), true diff --git a/internal/frontend/repl/montecarlo.go b/internal/frontend/repl/montecarlo.go index 02585507f..0e8232ac5 100644 --- a/internal/frontend/repl/montecarlo.go +++ b/internal/frontend/repl/montecarlo.go @@ -43,11 +43,17 @@ func (s *Session) RunMonteCarlo(invocation string, count int64, seed *uint64) Ve // monteCarloVerdict reports the runs as an action's table with the concluded case after it; // a failed run or unsatisfied concluding check fails it, a failed in-run check only counts. func (s *Session) monteCarloVerdict(inv analysisInvocation, count int64, seed *uint64) Verdict { + sample, answered, err := s.monteCarloSample(inv, count, seed) + return s.monteCarloReport(inv, sample, answered, err) +} + +// monteCarloReport is the verdict a Monte Carlo sample reports, sample already +// made; err is the error the sample ended with, nil when it completed. +func (s *Session) monteCarloReport(inv analysisInvocation, sample *monteCarloRuns, answered *analysis.Plan, err error) Verdict { label := "runs " + inv.name if inv.argText != "" { label += "(" + strings.TrimSpace(inv.argText) + ")" } - sample, answered, err := s.monteCarloSample(inv, count, seed) if err != nil { return standing(unresolvedVerdict(label, err.Error()), answered) } @@ -129,6 +135,11 @@ type monteCarloRuns struct { // unconcluded says why the case is not concluded over the table: no run completed, // or the sample of the completed ones was refused. unconcluded error + // concluded and concludeErr are the conclusion made once, for the report and + // the recorder to share. + concluded runtime.AnalysisResult + concludeErr error + concludedOnce bool } // last is the completed run the conclusion is read through. @@ -138,7 +149,12 @@ func (m *monteCarloRuns) last() *runtime.MonteCarloRun { // conclude settles every completed run's checks over the sample and concludes the case in // the last; each row then carries the run's settled checks, the sample's own left to the conclusion. +// It runs once: every caller reads the same conclusion. func (m *monteCarloRuns) conclude() (runtime.AnalysisResult, error) { + if m.concludedOnce { + return m.concluded, m.concludeErr + } + m.concludedOnce = true concluded, err := runtime.ConcludeMonteCarlo(m.completed, m.stats) byNumber := make(map[int64]*runtime.MonteCarloRun, len(m.completed)) for _, run := range m.completed { @@ -149,9 +165,18 @@ func (m *monteCarloRuns) conclude() (runtime.AnalysisResult, error) { m.table.Rows[k].Verdicts = byNumber[i].Verdicts } } + m.concluded, m.concludeErr = concluded, err return concluded, err } +// conclusion is the conclusion a completed sample made, or why it has none. +func (m *monteCarloRuns) conclusion() (runtime.AnalysisResult, error) { + if m.unconcluded != nil { + return runtime.AnalysisResult{}, m.unconcluded + } + return m.conclude() +} + // monteCarloSample makes count runs of the invocation, each in its own context on objects // made from their declarations; the plan is returned beside a refusal made after an engine ran. func (s *Session) monteCarloSample(inv analysisInvocation, count int64, seed *uint64) (*monteCarloRuns, *analysis.Plan, error) { diff --git a/internal/frontend/repl/record.go b/internal/frontend/repl/record.go new file mode 100644 index 000000000..67ef610c0 --- /dev/null +++ b/internal/frontend/repl/record.go @@ -0,0 +1,679 @@ +package repl + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/Open-MBEE/OpenSysML/internal/exec/analysis/record" + "github.com/Open-MBEE/OpenSysML/internal/exec/runtime" + "github.com/Open-MBEE/OpenSysML/internal/semantic/resolve" + "github.com/Open-MBEE/OpenSysML/internal/semantic/semantics" + "github.com/Open-MBEE/OpenSysML/internal/semantic/symbols" + "github.com/Open-MBEE/OpenSysML/internal/syntax/ast" + "github.com/Open-MBEE/OpenSysML/internal/syntax/diag" + "github.com/Open-MBEE/OpenSysML/internal/syntax/source" +) + +const recordUsage = "usage: %record [()] [] [into ]" + +// doRecord carries out %record at the prompt: the run %analysis would make, +// recorded into the model beside the case or into the package `into` names. +func (s *Session) doRecord(tail string) ([]string, bool, error) { + inv, into, err := splitRecordArgs(tail) + if err != nil { + return []string{errPrefix + err.Error(), recordUsage}, false, nil + } + if inv.name == "" { + return []string{recordUsage}, false, nil + } + return s.withTrace(s.recordAnalysisInv(inv, into, "%record "+strings.TrimSpace(tail))).Lines, false, nil +} + +// splitRecordArgs takes apart %record's tail: the invocation %analysis takes, +// then `into ` written last when it is. +func splitRecordArgs(tail string) (analysisInvocation, string, error) { + if i := lastTopLevelInto(tail); i >= 0 { + into := strings.TrimSpace(tail[i+len(" into "):]) + if _, ok := source.QualifiedNameSegments(into); !ok { + return analysisInvocation{}, "", fmt.Errorf("%q does not name a package", into) + } + inv, err := splitAnalysisArgs(tail[:i]) + return inv, into, err + } + inv, err := splitAnalysisArgs(tail) + return inv, "", err +} + +// lastTopLevelInto is the index of the last ` into ` token written at top +// level — outside quoted names, string literals and parentheses — or -1. +func lastTopLevelInto(tail string) int { + last, depth := -1, 0 + var quote byte + escaped := false + for i := 0; i < len(tail); i++ { + c := tail[i] + switch { + case escaped: + escaped = false + case c == '\\': + escaped = true + case quote != 0: + if c == quote { + quote = 0 + } + case c == '\'' || c == '"': + quote = c + case c == '(': + depth++ + case c == ')': + if depth > 0 { + depth-- + } + case depth == 0 && strings.HasPrefix(tail[i:], " into "): + last = i + } + } + return last +} + +// RecordAnalysis runs the invocation as %analysis does and records the run +// into the model as AnalysisRecords elements, into the package named or a +// Records package beside the case's. command is the invocation text the +// record's provenance carries. +func (s *Session) RecordAnalysis(invocation, into, command string) Verdict { + defer s.enter()() + inv, err := splitAnalysisArgs(invocation) + if err != nil { + return s.withTrace(unresolvedVerdict(invocation, err.Error())) + } + return s.withTrace(s.recordAnalysisInv(inv, into, command)) +} + +// recordAnalysisInv is RecordAnalysis on an invocation already parsed. +func (s *Session) recordAnalysisInv(inv analysisInvocation, into, command string) Verdict { + run, err := s.runAnalysis(inv) + verdict := s.caseVerdict(inv, run, err) + if err != nil { + return verdict + } + _, fqn, serr := s.analysisSymbol(inv) + if serr != nil { + return s.recordFailed(verdict, serr) + } + kind := record.KindRun + if len(run.result.Evaluations) > 0 { + kind = record.KindTrade + } + contexts := map[*runtime.Context]bool{} + if s.rtCtx != nil { + contexts[s.rtCtx] = true + } + inst := run.subject + if inst == nil { + inst = run.result.Subject + } + rec := record.Run{ + Subject: s.recordSubject(inst, run.label, contexts), + Inputs: run.result.Inputs, + Outputs: run.result.Outputs, + Verdicts: run.result.Verdicts, + Evaluations: run.result.Evaluations, + Verifications: run.verdicts, + Spell: s.recordSpelling(contexts), + } + res, rerr := s.recordRuns(fqn, kind, into, command, []record.Run{rec}) + return s.recorded(verdict, res, rerr, 0, nil) +} + +// RecordSweep runs the invocation once per row of the ranges as RunSweep does +// and records each completed row, numbered in table order. +func (s *Session) RecordSweep(invocation string, ranges []string, into, command string) Verdict { + defer s.enter()() + inv, trailing, err := splitSweepTail(invocation) + if err != nil { + return s.withTrace(unresolvedVerdict(invocation, err.Error())) + } + specs := make([]sweepSpec, 0, len(ranges)+len(trailing)) + specs = append(specs, trailing...) + for _, text := range ranges { + spec, err := parseSweepSpec(text) + if err != nil { + return s.withTrace(unresolvedVerdict(invocation, err.Error())) + } + specs = append(specs, spec) + } + return s.withTrace(s.recordSweepInv(inv, specs, into, command)) +} + +// recordSweepInv is RecordSweep on an invocation and its specs already parsed. +func (s *Session) recordSweepInv(inv analysisInvocation, specs []sweepSpec, into, command string) Verdict { + sym, fqn, err := s.lookupSymbolOfKinds(inv.name, + symbols.SymbolAnalysisCaseDef, symbols.SymbolAnalysisCaseUsage, + symbols.SymbolCalcDef, symbols.SymbolCalcUsage) + if err == nil && !runtime.IsRunnableCaseSymbol(sym) { + err = fmt.Errorf("%s is a calc, not a case", inv.name) + } + if err != nil { + return unresolvedVerdict(sweepLabel(inv, sweepDraws{}), err.Error()) + } + table, plan, err := s.runSweep(inv, specs, sweepDraws{}) + if err != nil { + return standing(unresolvedVerdict(sweepLabel(inv, sweepDraws{}), err.Error()), plan) + } + verdict := standing(s.sweepReport(inv, table, sweepDraws{}), plan) + var runs []record.Run + skipped := 0 + for i, row := range table.Rows { + if row.Err != nil { + skipped++ + continue + } + // Each row's values spell in its own context: instance ids restart per + // row, so a value means nothing read through another row's. + own := map[*runtime.Context]bool{row.Context: true} + runs = append(runs, record.Run{ + Iteration: i + 1, + Subject: s.recordSubject(row.Subject, inv.object, own), + Inputs: row.Inputs, + Outputs: row.Outputs, + Verdicts: row.Verdicts, + Evaluations: row.Evaluations, + Spell: s.recordSpelling(own), + }) + } + if len(runs) == 0 { + return s.recorded(verdict, record.Result{}, nil, skipped, []string{"nothing recorded: every row failed"}) + } + res, rerr := s.recordRuns(fqn, record.KindSweep, into, command, runs) + return s.recorded(verdict, res, rerr, skipped, nil) +} + +// RecordMonteCarlo runs the invocation count times as RunMonteCarlo does and +// records each completed run, numbered by its run number. +func (s *Session) RecordMonteCarlo(invocation string, count int64, seed *uint64, into, command string) Verdict { + defer s.enter()() + inv, err := splitAnalysisArgs(invocation) + if err != nil { + return s.withTrace(unresolvedVerdict(invocation, err.Error())) + } + return s.withTrace(s.recordMonteCarloInv(inv, count, seed, into, command)) +} + +// recordMonteCarloInv is RecordMonteCarlo on an invocation already parsed. +func (s *Session) recordMonteCarloInv(inv analysisInvocation, count int64, seed *uint64, into, command string) Verdict { + sample, answered, err := s.monteCarloSample(inv, count, seed) + verdict := s.monteCarloReport(inv, sample, answered, err) + if err != nil || sample == nil { + return verdict + } + fqn := sampleRunsCase(sample) + var runs []record.Run + var reasons []string + skipped := 0 + concluded, cerr := sample.conclusion() + for _, run := range sample.completed { + // Outputs that are the sample's were already left out of Unread; what + // is left failed the iteration. + if len(run.Unread) > 0 { + missing := make([]string, 0, len(run.Unread)) + for name := range run.Unread { + missing = append(missing, name) + } + sort.Strings(missing) + skipped++ + reasons = append(reasons, fmt.Sprintf("run %d not recorded: %s", run.Number, run.Unread[missing[0]])) + continue + } + own := map[*runtime.Context]bool{run.Context(): true} + runs = append(runs, record.Run{ + Iteration: int(run.Number), + Subject: s.recordSubject(run.Subject, inv.object, own), + Inputs: run.Inputs, + Outputs: run.Outputs, + Verdicts: run.Verdicts, + Spell: s.recordSpelling(own), + }) + } + // The sample's own record carries what the run rows cannot: the statistics, + // the result and the sample's checks of the case's conclusion. + switch { + case cerr != nil: + reasons = append(reasons, fmt.Sprintf("sample not recorded: %s", cerr)) + case len(sample.completed) > 0: + last := sample.last() + own := map[*runtime.Context]bool{last.Context(): true} + runs = append(runs, record.Run{ + Kind: record.KindSample, + Subject: s.recordSubject(last.Subject, inv.object, own), + Inputs: last.Inputs, + Outputs: concluded.Outputs, + Verdicts: concluded.Verdicts, + Evaluations: concluded.Evaluations, + Spell: s.recordSpelling(own), + }) + } + skipped += len(sample.table.Rows) - len(sample.completed) + if len(runs) == 0 { + return s.recorded(verdict, record.Result{}, nil, skipped, append(reasons, "nothing recorded: no run completed")) + } + res, rerr := s.recordRuns(fqn, record.KindRuns, into, command, runs) + return s.recorded(verdict, res, rerr, skipped, reasons) +} + +// sampleRunsCase is the qualified name of the case a sample's runs were made +// of, from any completed run's report. +func sampleRunsCase(sample *monteCarloRuns) string { + for _, run := range sample.completed { + return run.Case + } + return "" +} + +// recordSubject is how a run's object is recorded: its usage in the model when +// that resolves to exactly one element, and its text for subjectName. +func (s *Session) recordSubject(inst *runtime.Instance, label string, contexts map[*runtime.Context]bool) record.Subject { + if inst == nil { + return record.Subject{} + } + for ctx := range contexts { + if _, ok := ctx.Instance(inst.ID); ok { + if usage := ctx.OccurrenceUsage(inst); usage != "" && len(s.symbolIndex().LookupQualified(usage)) == 1 { + return record.Subject{Usage: usage, Text: usage} + } + return record.Subject{Text: objectText(ctx, runtime.Value{Kind: runtime.ValInstance, Instance: inst.ID})} + } + } + return record.Subject{Text: label} +} + +// recordRuns generates the record declarations for runs of the case fqn names, +// submits them, and returns what was generated. +func (s *Session) recordRuns(fqn string, kind record.Kind, into, command string, runs []record.Run) (record.Result, error) { + pkg, err := s.recordPackage(fqn, into) + if err != nil { + return record.Result{}, err + } + caseSym := s.recordCaseSymbol(fqn) + existing, err := s.recordExisting(caseSym, fqn, pkg) + if err != nil { + return record.Result{}, err + } + caseName := fqn + if caseSym != nil { + caseName = caseSym.Name + } + res, err := record.Generate(record.Request{ + Package: pkg, + Case: fqn, + CaseName: caseName, + Provenance: record.Provenance{ + RunAt: s.now(), + Tool: s.toolVersion, + Command: command, + Kind: kind, + }, + Runs: runs, + Existing: existing, + }) + if err != nil { + return record.Result{}, err + } + if err := s.submitRecord(res.Source); err != nil { + return record.Result{}, err + } + return res, nil +} + +// recordPackage is the package a case's records go into: the one into names, +// or Records beside the package enclosing the case's own package. +func (s *Session) recordPackage(fqn, into string) (string, error) { + if into != "" { + segs, ok := source.QualifiedNameSegments(into) + if !ok { + return "", fmt.Errorf("%q does not name a package", into) + } + for i := 1; i <= len(segs); i++ { + prefix := strings.Join(segs[:i], "::") + for _, sym := range s.symbolIndex().LookupQualified(prefix) { + if sym.Kind != symbols.SymbolPackage { + return "", fmt.Errorf("%s names a %s, not a package", prefix, sym.Kind) + } + } + } + return into, nil + } + idx := s.symbolIndex() + for _, sym := range idx.LookupQualified(fqn) { + pkg := enclosingPackage(sym) + if pkg == nil { + continue + } + parent := enclosingPackage(pkg.Owner()) + if parent == nil { + return "Records", nil + } + return idx.GetFQN(parent) + "::Records", nil + } + return "Records", nil +} + +// enclosingPackage walks a symbol's owners to the nearest package, nil when +// there is none. +func enclosingPackage(sym *symbols.Symbol) *symbols.Symbol { + for s := sym; s != nil; s = s.Owner() { + if s.Kind == symbols.SymbolPackage { + return s + } + } + return nil +} + +// recordCaseSymbol is the analysis case symbol fqn names, nil when the index +// names none or several. +func (s *Session) recordCaseSymbol(fqn string) *symbols.Symbol { + syms := s.symbolIndex().LookupQualified(fqn) + if len(syms) == 1 { + return syms[0] + } + return nil +} + +// recordExisting is what the target package already declares of the shape +// Generate must fit: the package itself, the case's record definition and the +// record numbers already taken. The stem the records are named from is the +// case's name, owner-prefixed when a definition of the same name belongs to a +// sibling case. +func (s *Session) recordExisting(caseSym *symbols.Symbol, fqn, pkg string) (record.Existing, error) { + idx := s.symbolIndex() + var existing record.Existing + if len(idx.LookupQualified(pkg)) > 0 { + existing.Package = true + } + short := shortName(fqn) + if caseSym != nil { + short = caseSym.Name + } + // Candidate stems: the case's name, then each owner up the chain prefixed. + stems := []string{short} + if caseSym != nil { + var owners []string + for cur := caseSym.Owner(); cur != nil && cur.Name != ""; cur = cur.Owner() { + owners = append([]string{cur.Name}, owners...) + stems = append(stems, strings.Join(append(append([]string{}, owners...), short), "_")) + } + } + var sem *semantics.Model + for _, stem := range stems { + def := pkg + "::" + upperFirst(stem) + "Run" + defSyms := idx.LookupQualified(def) + if len(defSyms) == 0 { + existing.Stem = stem + break + } + runSyms := idx.LookupQualified("AnalysisRecords::AnalysisRun") + if len(runSyms) == 0 { + return existing, fmt.Errorf("the AnalysisRecords library is not loaded") + } + if sem == nil { + resolver := resolve.New(idx) + sem = semantics.NewModel(resolver) + resolver.SetModel(sem) + } + if !specializesOne(sem, defSyms[0], runSyms[0]) { + return existing, fmt.Errorf("%s is not an analysis record definition", def) + } + owner := recordDefOwner(defSyms[0]) + switch { + case owner == "" || owner == fqn: + // Unowned or this case's own: reused. + existing.Definition = true + existing.Attributes = recordAttributes(idx, sem, defSyms[0]) + existing.Stem = stem + default: + if stem == stems[len(stems)-1] { + return existing, fmt.Errorf("record definition %s belongs to %s; record into another package with `into`", def, owner) + } + } + if existing.Stem != "" { + break + } + } + stem := existing.Stem + if stem == "" { + stem = short + } + existing.Taken = map[int]bool{} + for _, sym := range idx.LookupQualified(pkg) { + if sym.Scope == nil { + continue + } + prefix := stem + "_run" + for _, m := range sym.Scope.Members() { + tail, ok := strings.CutPrefix(m.Name, prefix) + if !ok { + continue + } + if n, err := strconv.Atoi(tail); err == nil { + existing.Taken[n] = true + } + } + } + return existing, nil +} + +// recordDefOwner is the case a record definition's caseName marks it for, +// "" when it carries none (a hand-written definition, reused by any case). +func recordDefOwner(def *symbols.Symbol) string { + if def.Scope == nil { + return "" + } + m, ok := def.Scope.LookupLocal("caseName") + if !ok { + return "" + } + u, ok := m.Decl.(*ast.Usage) + if !ok { + return "" + } + lit, ok := u.Value.(*ast.LiteralString) + if !ok { + return "" + } + return source.StringValue(lit.Value) +} + +// specializesOne reports whether def specializes want among its supertypes. +func specializesOne(sem *semantics.Model, def, want *symbols.Symbol) bool { + for _, sup := range sem.AllSupertypes(def) { + if sup == want { + return true + } + } + return def == want +} + +// recordAttributes are the features a record definition declares, by name, +// each with whether it is a reference and the name of its declared type. +func recordAttributes(idx *symbols.Index, sem *semantics.Model, def *symbols.Symbol) map[string]record.Feature { + attrs := map[string]record.Feature{} + if def.Scope == nil { + return attrs + } + for _, m := range def.Scope.Members() { + if m.Name == "" || !m.IsFeature() { + continue + } + f := record.Feature{} + if u, ok := m.Decl.(*ast.Usage); ok && u.IsReference { + f.Ref = true + } + if types := sem.DeclaredFeatureTypes(m); len(types) > 0 { + f.TypeFQN = idx.GetFQN(types[0]) + } + attrs[m.Name] = f + } + return attrs +} + +// recordSpelling renders the values the records cannot spell themselves, each +// through the context it was made in. +func (s *Session) recordSpelling(contexts map[*runtime.Context]bool) record.Spelling { + contextOf := func(v runtime.Value) *runtime.Context { + if v.Kind == runtime.ValInstance || v.Kind == runtime.ValVariant { + for ctx := range contexts { + if _, ok := ctx.Instance(v.Instance); ok { + return ctx + } + } + } + for ctx := range contexts { + return ctx + } + return s.rtCtx + } + return record.Spelling{ + ObjectUsage: func(v runtime.Value) string { + ctx := contextOf(v) + if ctx == nil { + return "" + } + var usage string + if inst, ok := ctx.Instance(v.Instance); ok { + usage = ctx.OccurrenceUsage(inst) + } else if v.Kind == runtime.ValVariant { + usage = s.symbolIndex().GetFQN(v.Variant()) + } + if usage == "" || len(s.symbolIndex().LookupQualified(usage)) != 1 { + return "" + } + return usage + }, + Text: func(v runtime.Value) string { + if ctx := contextOf(v); ctx != nil { + return objectText(ctx, v) + } + return runtime.FormatValue(v) + }, + Unset: func(v runtime.Value) bool { + if ctx := contextOf(v); ctx != nil { + return ctx.HoldsNoValue(v) + } + return v.Kind == runtime.ValNull + }, + } +} + +// recorded finishes a record verdict: the run's own report, then what the +// model gained, the count of rows skipped, or the failure that left it +// untouched. +func (s *Session) recorded(verdict Verdict, res record.Result, err error, skipped int, reasons []string) Verdict { + for _, reason := range reasons { + verdict.Lines = append(verdict.Lines, " "+reason) + } + if err != nil { + return s.recordFailed(verdict, err) + } + switch len(res.Records) { + case 0: + verdict.Lines = append(verdict.Lines, " nothing was recorded") + case 1: + verdict.Lines = append(verdict.Lines, fmt.Sprintf(" recorded %s (%s)", res.Records[0], res.Definition)) + default: + verdict.Lines = append(verdict.Lines, fmt.Sprintf(" recorded %d runs as %s … %s", len(res.Records), res.Records[0], shortName(res.Records[len(res.Records)-1]))) + } + if skipped > 0 { + verdict.Lines = append(verdict.Lines, fmt.Sprintf(" %d failed run(s) were not recorded", skipped)) + } + return verdict +} + +// recordFailed fails a run's verdict with the reason its record was not made. +func (s *Session) recordFailed(verdict Verdict, err error) Verdict { + verdict.Status = VerdictFails + verdict.Lines = append(verdict.Lines, errPrefix+"recording the run failed: "+err.Error()) + return verdict +} + +// shortName is the last segment of a qualified name. +func shortName(fqn string) string { + segs, ok := source.QualifiedNameSegments(fqn) + if !ok || len(segs) == 0 { + return fqn + } + return segs[len(segs)-1] +} + +// upperFirst capitalizes a name's leading letter. +func upperFirst(name string) string { + if name == "" { + return name + } + return strings.ToUpper(name[:1]) + name[1:] +} + +// submitRecord applies generated declarations as one submission: the record +// merge flag lets it fold into the package a loaded file declared, and any +// loss or error the submission makes restores the buffer as it was. +func (s *Session) submitRecord(src string) error { + before := append([]snippet{}, s.snippets...) + beforeErrors := s.errorCounts() + s.recordMerge = true + res, _, _ := s.submitEach([]SourceFile{{Text: src}}) + s.recordMerge = false + + problems := newProblems(beforeErrors, res.Diagnostics) + for _, drop := range s.recordDrops { + for _, name := range append(append([]string{}, drop.lost...), drop.gone...) { + problems = append(problems, fmt.Sprintf("recording would drop %s", name)) + } + } + if len(problems) == 0 { + return nil + } + s.rollbackSubmit(before) + return fmt.Errorf("the model was left unchanged: %s", strings.Join(problems, "; ")) +} + +// newProblems reports the error diagnostics a submission raised that the +// model did not already report: a message the model reported before is new +// only once its count grows. +func newProblems(before map[string]int, diagnostics []diag.Diagnostic) []string { + var problems []string + for _, d := range diagnostics { + if d.Severity != diag.SeverityError { + continue + } + if before[d.Message] > 0 { + before[d.Message]-- + } else { + problems = append(problems, d.Message) + } + } + return problems +} + +// errorCounts keys the error diagnostics the buffer already reports, by +// message: a merge can move the offsets an old error sits at, and a record's +// diagnostic is new only once it outnumbers what the model already reported. +func (s *Session) errorCounts() map[string]int { + counts := map[string]int{} + for _, d := range s.diagnostics() { + if d.Severity == diag.SeverityError { + counts[d.Message]++ + } + } + return counts +} + +// rollbackSubmit restores the snippets a failed record submission replaced and +// rebuilds over them as a submission does, so what the session holds — its +// objects and debugging sessions — is carried into the restored document. +func (s *Session) rollbackSubmit(before []snippet) { + s.snippets = before + s.version++ + s.rebuildOver(nil) + s.idxVersion = 0 + s.names = nil +} diff --git a/internal/frontend/repl/record_test.go b/internal/frontend/repl/record_test.go new file mode 100644 index 000000000..78690bed4 --- /dev/null +++ b/internal/frontend/repl/record_test.go @@ -0,0 +1,700 @@ +package repl + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Open-MBEE/OpenSysML/internal/syntax/diag" +) + +// recordModel declares cases %record runs: one binding its own subject, one +// without one, and an action a debugging session drives. +const recordModel = `package Demo { + private import ScalarValues::*; + part def Probe { + attribute t : Real = 3.0; + action tick { first start; then assign t := t + 1.0; then done; } + } + part probe : Probe; + analysis def Check { + subject s : Probe; + in gain : Real; + out x : Real = s.t + gain; + } + analysis def Bound { + out y : Real = 1.0 + 2.0; + } + analysis timed : Check { subject s = probe; in gain = 2.0; } + calc def Sum { in a : Real; return : Real = a; } +}` + +func recordSession(t *testing.T) *Session { + t.Helper() + s := NewSession() + s.now = func() time.Time { return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) } + if errs := errorDiagnostics(s.Submit(recordModel).Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + return s +} + +// A recorded run is written into the model beside the case's package and +// saved with it; re-recording in the saved model numbers the record on. +func TestRecordRunSavesAndRenumbers(t *testing.T) { + s := recordSession(t) + wants(t, run(t, s, "%record Demo::timed"), + "✓ Demo::timed", "x = 5.0", "recorded Records::timed_run1 (Records::TimedRun)") + text := s.text() + for _, want := range []string{ + "package Records", "part def TimedRun :> AnalysisRecords::AnalysisRun", + "part timed_run1 : TimedRun", "@AnalysisRecords::RecordedRun", + "caseName = \"Demo::timed\"", "attribute :>> gain = 2.0", + "attribute :>> x = 5.0", "runAt = \"2026-01-01T00:00:00Z\"", + "ref :>> 'subject' = Demo::probe", + } { + if !strings.Contains(text, want) { + t.Errorf("session text is missing %q:\n%s", want, text) + } + } + + path := filepath.Join(t.TempDir(), "model.sysml") + if _, _, err := s.runMeta("%save " + path); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(path); err != nil || !strings.Contains(string(data), "part timed_run1 : TimedRun") { + t.Fatalf("saved file lacks the record: %v", err) + } + + fresh := recordSession(t) + res := fresh.SubmitFiles([]SourceFile{{Name: path, Text: mustRead(t, path)}}) + if errs := errorDiagnostics(res.Diagnostics); len(errs) > 0 { + t.Fatalf("saved model has errors: %v", errs) + } + // A second record merges into the Records package the file declared. + fresh.SubmitFiles([]SourceFile{{Name: path, Text: mustRead(t, path)}}) + wants(t, run(t, fresh, "%record Demo::timed"), + "recorded Records::timed_run2") +} + +func mustRead(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +// `into` records into the package named, nesting it under a package already +// in the buffer. +func TestRecordIntoNestedPackage(t *testing.T) { + s := recordSession(t) + wants(t, run(t, s, "%record Demo::Bound into Demo::Log"), + "recorded Demo::Log::Bound_run1 (Demo::Log::BoundRun)") + if !strings.Contains(s.text(), "package Log") { + t.Errorf("the records package is missing:\n%s", s.text()) + } +} + +// A record that cannot be made leaves the buffer exactly as it was. +func TestRecordLeavesModelOnFailure(t *testing.T) { + for _, line := range []string{ + "%record Demo::Gone", // no such case + "%record Demo::Sum", // a calc is not a case + "%record Demo::Check(1.0)", // the run binds no subject + "%record Demo::Bound into Demo::Probe", // into names a part def + } { + s := recordSession(t) + before := s.text() + out := run(t, s, line) + if got := s.text(); got != before { + t.Errorf("%s changed the model:\nbefore:\n%s\nafter:\n%s\nout:%s", line, before, got, out) + } + } +} + +// Records a document query finds by their RecordedRun metadata and reads the +// objective they carry. +func TestRecordRunQueryable(t *testing.T) { + s := recordSession(t) + if errs := errorDiagnostics(s.Submit(`package Demo { + private import DocumentQueries::*; + private import KerML::Root::Element; + calc def RecordedRuns :> Query { + in root : Element; + Project(source = WhereMetadata( + source = Descendants(source = root, maxDepth = 10), + 'metadata' = "AnalysisRecords::RecordedRun"), + properties = ("name", "caseName", "kind")) + } +}`).Diagnostics); len(errs) > 0 { + t.Fatalf("query has errors: %v", errs) + } + run(t, s, "%record Demo::Bound into Demo::Log") + out := run(t, s, "%run-query RecordedRuns root=Demo") + wants(t, out, "Bound_run1", "Demo::Bound", "run") +} + +// A document query filters records by the features they carry and projects the +// values the run bound, so the run is readable as model data. +func TestRecordRunQueryableByFeature(t *testing.T) { + s := recordSession(t) + if errs := errorDiagnostics(s.Submit(`package Demo { + private import DocumentQueries::*; + private import KerML::Root::Element; + calc def TimedRuns :> Query { + in root : Element; + Project(source = WhereFeature( + source = WhereMetadata( + source = Descendants(source = root, maxDepth = 10), + 'metadata' = "AnalysisRecords::RecordedRun"), + 'feature' = "caseName", + operator = "=", + value = "Demo::timed"), + properties = ("name", "gain", "x")) + } +}`).Diagnostics); len(errs) > 0 { + t.Fatalf("query has errors: %v", errs) + } + run(t, s, "%record Demo::timed into Demo::Log") + out := run(t, s, "%run-query TimedRuns root=Demo") + wants(t, out, "timed_run1", "gain = 2.0", "x = 5.0") +} + +// The records land beside the package enclosing the case's: a case in a +// nested package's Records is made under its parent, and one nested in a +// part lands under a top-level Records. +func TestRecordPackageFollowsTheCasesPackage(t *testing.T) { + s := recordSession(t) + if errs := errorDiagnostics(s.Submit(`package A { + package Descent { + analysis def C { out k : ScalarValues::Real = 1.0; } + analysis c : C; + } +} +package P { + part def H { + analysis def Inner { out k : ScalarValues::Real = 1.0; } + analysis inner : Inner; + } +}`).Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + wants(t, run(t, s, "%record A::Descent::c"), "recorded A::Records::c_run1") + wants(t, run(t, s, "%record P::H::inner"), "recorded Records::inner_run1") +} + +// A record submission that would drop a declaration restores the buffer, and +// the objects and debugging sessions it holds, as they were. +func TestRecordFailureKeepsObjectsAndDebugSession(t *testing.T) { + s := recordSession(t) + run(t, s, "%instantiate Demo::probe") + wants(t, run(t, s, "%action Demo::Probe::tick #1"), "Started action executor") + // A differently-headed Records package cannot be merged into, so the + // record's package would supersede it and drop its member. + if errs := errorDiagnostics(s.Submit(`package 'Records' { + part keep : Demo::Probe; +}`).Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + before := s.text() + out := run(t, s, "%record Demo::timed") + wants(t, out, "recording the run failed", "drop") + if s.text() != before { + t.Error("a failed record changed the buffer:\n" + s.text()) + } + if s.actionExec == nil { + t.Fatal("the debugging session was ended by the failed record") + } + wants(t, run(t, s, "%step"), "Step") + if len(s.instances) == 0 { + t.Error("the instantiated object was lost by the failed record") + } +} + +// Recording a run does not end a debugging session over an unrelated +// declaration. +func TestRecordKeepsDebugSession(t *testing.T) { + s := recordSession(t) + run(t, s, "%instantiate Demo::probe") + wants(t, run(t, s, "%action Demo::Probe::tick #1"), "Started action executor") + run(t, s, "%record Demo::timed") + if s.actionExec == nil { + t.Fatal("the debugging session was ended by a record submission") + } + wants(t, run(t, s, "%step"), "Step") +} + +// A sweep that could not run records nothing and does not hold: its verdict is +// the sweep's own error, not a report over an empty table. +func TestRecordSweepErrorRecordsNothing(t *testing.T) { + s := recordSession(t) + before := s.text() + v := s.RecordSweep("Demo::timed", []string{"missing=1..3"}, "", "%record Demo::timed") + if v.Status == VerdictHolds { + t.Errorf("a sweep that could not run holds:\n%s", strings.Join(v.Lines, "\n")) + } + for _, line := range v.Lines { + if strings.Contains(line, "recorded") { + t.Errorf("a failed sweep reported a record: %q", line) + } + } + if s.text() != before { + t.Error("a failed sweep record changed the buffer") + } +} + +// `into` splits only outside quoted names, string literals and parentheses. +func TestSplitRecordArgsLeavesIntoInNames(t *testing.T) { + inv, into, err := splitRecordArgs("'step into space'") + if err != nil || into != "" || inv.name != "'step into space'" { + t.Errorf("'step into space': inv %+v, into %q, err %v", inv, into, err) + } + inv, into, err = splitRecordArgs(`c("go into it") into P`) + if err != nil || into != "P" || inv.name != "c" || inv.argText != `"go into it"` { + t.Errorf(`c("go into it") into P: inv %+v, into %q, err %v`, inv, into, err) + } +} + +// Each sweep row's values spell in its own context: instance ids restart per +// row, so one row's object means nothing read through another row's. +func TestRecordSweepSpellsObjectsInTheirOwnContext(t *testing.T) { + s := NewSession() + s.now = func() time.Time { return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) } + if errs := errorDiagnostics(s.Submit(`package Demo { + private import ScalarValues::*; + private import ControlFunctions::*; + part def Probe { attribute t : Real = 1.0; } + part a : Probe; + part b : Probe; + analysis def Pick { + subject s : Probe; + in n : Real; + out chosen : Probe = 'if'(n < 2.0, a, b); + } + analysis pick : Pick { subject s = a; in n = 1.0; } +}`).Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + v := s.RecordSweep("Demo::pick", []string{"n=1..2"}, "", "%record Demo::pick") + if v.Status != VerdictHolds { + t.Fatalf("record sweep: %v", v.Lines) + } + text := s.text() + for _, want := range []string{"ref :>> chosen = Demo::a;", "ref :>> chosen = Demo::b;"} { + if !strings.Contains(text, want) { + t.Errorf("recorded model is missing %q:\n%s", want, text) + } + } +} + +// Record numbers fill the gaps a package's earlier records leave rather than +// renumbering on from the first free prefix. +func TestRecordNumbersIntoTheGaps(t *testing.T) { + s := recordSession(t) + if errs := errorDiagnostics(s.Submit(`package Records { part timed_run2 : Demo::Probe; }`).Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + v := s.RecordSweep("Demo::timed", []string{"gain=1..2"}, "", "%record Demo::timed") + if v.Status != VerdictHolds { + t.Fatalf("record sweep: %v", v.Lines) + } + for _, want := range []string{"part timed_run1 :", "part timed_run3 :"} { + if !strings.Contains(s.text(), want) { + t.Errorf("recorded model is missing %q:\n%s", want, s.text()) + } + } +} + +// A Monte Carlo run whose declared output errors is not recorded; the report +// says which run and why. +func TestRecordMonteCarloSkipsOutputErrors(t *testing.T) { + s := NewSession() + s.now = func() time.Time { return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) } + if errs := errorDiagnostics(s.Submit(`package MC { + private import ScalarValues::*; + private import RandomFunctions::*; + part def Probe { + attribute t : Real; + action settle { first start; then assign t := uniform(1.0, 5.0); then done; } + } + individual def probe :> Probe; + analysis def Mc :> Simulation::MonteCarlo { + subject analysed : Probe; + perform action run ::> analysed.settle; + attribute :>> observed : Real = analysed.t; + return Mean : Real = mean; + out Bad : Real = 1.0 / 0.0; + } +}`).Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + run(t, s, "%instantiate MC::probe") + seed := uint64(7) + v := s.RecordMonteCarlo("MC::Mc MC::probe", 1, &seed, "", "%record MC::Mc") + out := strings.Join(v.Lines, "\n") + for _, want := range []string{"run 1 not recorded:", "sample not recorded:", "nothing recorded: no run completed"} { + if !strings.Contains(out, want) { + t.Errorf("the skipped run is not reported (%q missing):\n%s", want, out) + } + } + if strings.Contains(s.text(), "Mc_run") { + t.Errorf("the model holds a record of an unreadable run:\n%s", s.text()) + } +} + +// A Monte Carlo sample records each iteration under kind "runs" and its +// conclusion once under kind "sample", carrying the statistics and result +// the rows cannot. +func TestRecordMonteCarloRecordsTheSample(t *testing.T) { + s := monteCarloSession(t) + run(t, s, "%instantiate MC::probe") + seed := uint64(7) + v := s.RecordMonteCarlo("MC::Mc MC::probe", 3, &seed, "", "%record MC::Mc") + out := strings.Join(v.Lines, "\n") + if !strings.Contains(out, "recorded 4 runs") { + t.Errorf("three iterations and the sample were not recorded:\n%s", out) + } + text := s.text() + for _, want := range []string{ + `attribute :>> kind = "runs"`, `attribute :>> kind = "sample"`, + "attribute :>> Mean", "attribute :>> Deviation", "attribute :>> N = 3", + "attribute :>> iteration = 3", + } { + if !strings.Contains(text, want) { + t.Errorf("recorded model is missing %q:\n%s", want, text) + } + } + + path := filepath.Join(t.TempDir(), "model.sysml") + if _, _, err := s.runMeta("%save " + path); err != nil { + t.Fatal(err) + } + fresh := recordSession(t) + res := fresh.SubmitFiles([]SourceFile{{Name: path, Text: mustRead(t, path)}}) + if errs := errorDiagnostics(res.Diagnostics); len(errs) > 0 { + t.Fatalf("saved model has errors: %v", errs) + } +} + +// A sample no run of which completed records nothing and says why instead of +// reporting a generator error. +func TestRecordMonteCarloRecordsNothingWhenNoRunCompletes(t *testing.T) { + s := monteCarloSession(t) + run(t, s, "%instantiate MC::probe") + before := s.text() + seed := uint64(7) + v := s.RecordMonteCarlo("MC::Crashing MC::probe", 2, &seed, "", "%record MC::Crashing") + out := strings.Join(v.Lines, "\n") + if !strings.Contains(out, "nothing recorded: no run completed") { + t.Errorf("the empty sample is not explained:\n%s", out) + } + if strings.Contains(out, "record") && strings.Contains(out, "nothing to record") { + t.Errorf("the generator's own error reported instead:\n%s", out) + } + if s.text() != before { + t.Errorf("the model changed:\n%s", s.text()) + } +} + +// A sweep whose every row failed records nothing and says why instead of +// reporting a generator error. +func TestRecordSweepRecordsNothingWhenEveryRowFails(t *testing.T) { + s := recordSession(t) + if errs := errorDiagnostics(s.Submit(`package Demo { + analysis def Breakable { subject s : Probe; in n : Real; out x : Real = 3.0 / n; } + analysis breakable : Breakable { subject s = probe; } +}`).Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + before := s.text() + v := s.RecordSweep("Demo::breakable", []string{"n=0..0"}, "", "%record Demo::breakable") + out := strings.Join(v.Lines, "\n") + if !strings.Contains(out, "nothing recorded: every row failed") { + t.Errorf("the empty sweep is not explained:\n%s", out) + } + if strings.Contains(out, "nothing to record") { + t.Errorf("the generator's own error reported instead:\n%s", out) + } + if s.text() != before { + t.Errorf("the model changed:\n%s", s.text()) + } +} + +// A case whose name needs quoting records under quoted names; the saved model +// numbers the next record on. +func TestRecordRunQuotesNames(t *testing.T) { + s := NewSession() + s.now = func() time.Time { return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) } + const model = `package Demo { + private import ScalarValues::*; + analysis def Bound { out y : Real = 3.0; } + analysis 'fuel budget' : Bound; + }` + if errs := errorDiagnostics(s.Submit(model).Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + wants(t, run(t, s, "%record Demo::'fuel budget'"), + "recorded Records::fuel budget_run1 (Records::Fuel budgetRun)") + for _, want := range []string{ + "part def 'Fuel budgetRun' :> AnalysisRecords::AnalysisRun", + `attribute :>> caseName default = "Demo::fuel budget";`, + "part 'fuel budget_run1' : 'Fuel budgetRun'", + } { + if !strings.Contains(s.text(), want) { + t.Errorf("session text is missing %q:\n%s", want, s.text()) + } + } + + path := filepath.Join(t.TempDir(), "model.sysml") + if _, _, err := s.runMeta("%save " + path); err != nil { + t.Fatal(err) + } + fresh := recordSession(t) + res := fresh.SubmitFiles([]SourceFile{{Name: path, Text: mustRead(t, path)}}) + if errs := errorDiagnostics(res.Diagnostics); len(errs) > 0 { + t.Fatalf("saved model has errors: %v", errs) + } + fresh.SubmitFiles([]SourceFile{{Name: path, Text: mustRead(t, path)}}) + wants(t, run(t, fresh, "%record Demo::'fuel budget'"), + "recorded Records::fuel budget_run2") +} + +// A sibling case of the same short name does not take over the first case's +// record definition: its own definition is named from its owner, in the model +// and again after a save. +func TestRecordRunPrefixesASiblingsDefinition(t *testing.T) { + s := NewSession() + s.now = func() time.Time { return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) } + const model = `package Demo { + private import ScalarValues::*; + package A { + analysis def Check { out x : Real = 1.0; } + analysis check : Check; + } + package B { + analysis def Check { out x : Real = 2.0; } + analysis check : Check; + } + }` + if errs := errorDiagnostics(s.Submit(model).Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + wants(t, run(t, s, "%record Demo::A::check"), + "recorded Demo::Records::check_run1 (Demo::Records::CheckRun)") + wants(t, run(t, s, "%record Demo::B::check"), + "recorded Demo::Records::B_check_run1 (Demo::Records::B_checkRun)") + + path := filepath.Join(t.TempDir(), "model.sysml") + if _, _, err := s.runMeta("%save " + path); err != nil { + t.Fatal(err) + } + fresh := recordSession(t) + res := fresh.SubmitFiles([]SourceFile{{Name: path, Text: mustRead(t, path)}}) + if errs := errorDiagnostics(res.Diagnostics); len(errs) > 0 { + t.Fatalf("saved model has errors: %v", errs) + } + fresh.SubmitFiles([]SourceFile{{Name: path, Text: mustRead(t, path)}}) + wants(t, run(t, fresh, "%record Demo::B::check"), + "recorded Demo::Records::B_check_run2") +} + +// A second record settles a member the first left ScalarValue: the existing +// definition is reused and the new record carries the concrete value. +func TestRecordSettlesAnEarlierUnsetMember(t *testing.T) { + s := recordSession(t) + if errs := errorDiagnostics(s.Submit(`package Demo { + analysis def Settle { subject s : Probe; in m : Real[0..1]; out x : Real[0..1] = m; } + analysis settle : Settle { subject s = probe; } +}`).Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + wants(t, run(t, s, "%record Demo::settle"), "recorded Records::settle_run1") + if !strings.Contains(s.text(), "attribute x : ScalarValues::ScalarValue") { + t.Fatalf("the unset member is not ScalarValue:\n%s", s.text()) + } + wants(t, run(t, s, "%record Demo::settle(2.0)"), "recorded Records::settle_run2") + if !strings.Contains(s.text(), "attribute :>> x = 2.0;") { + t.Errorf("the settled run did not record:\n%s", s.text()) + } +} + +// A verification case's record carries the verdict its body decided and a +// VerdictRecord row for it, beside whatever the body's checks returned. +func TestRecordVerificationRun(t *testing.T) { + s := NewSession() + s.now = func() time.Time { return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) } + res := s.Submit(`package Demo { + private import ScalarValues::*; + part def Engine { attribute thrust : Real; } + part engine : Engine { attribute :>> thrust = 2800.0; } + verification def Fire { + subject e : Engine; + VerificationCases::PassIf(e.thrust >= 3000.0) + } + verification fire : Fire { subject e = engine; } +}`) + if errs := errorDiagnostics(res.Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + wants(t, run(t, s, "%record Demo::fire"), "recorded Records::fire_run1") + text := s.text() + for _, want := range []string{ + `attribute :>> verdict = "fail"`, + `attribute :>> kind = "verification"`, + `attribute :>> status = "fail"`, + } { + if !strings.Contains(text, want) { + t.Errorf("session text is missing %q:\n%s", want, text) + } + } + // The record parses and validates clean on reload. + path := filepath.Join(t.TempDir(), "demo.sysml") + if _, _, err := s.runMeta("%save " + path); err != nil { + t.Fatal(err) + } + fresh := NewSession() + if errs := errorDiagnostics(fresh.SubmitFiles([]SourceFile{{Name: path, Text: mustRead(t, path)}}).Diagnostics); len(errs) > 0 { + t.Fatalf("saved model has errors: %v", errs) + } +} + +// A diagnostic the model already reports is a record's fault only when the +// record adds an occurrence of it. +func TestNewProblemsCountsOccurrences(t *testing.T) { + err := func(msg string) diag.Diagnostic { + return diag.Diagnostic{Severity: diag.SeverityError, Message: msg} + } + warn := diag.Diagnostic{Severity: diag.SeverityWarning, Message: "w"} + before := map[string]int{"old": 2} + got := newProblems(before, []diag.Diagnostic{err("old"), err("old"), warn}) + if len(got) != 0 { + t.Errorf("reported %v for what the model already had", got) + } + if got := newProblems(map[string]int{"old": 1}, []diag.Diagnostic{err("old"), err("old")}); len(got) != 1 || got[0] != "old" { + t.Errorf("a second occurrence is the record's: %v", got) + } + if got := newProblems(map[string]int{"old": 1}, []diag.Diagnostic{err("new")}); len(got) != 1 || got[0] != "new" { + t.Errorf("a new message is the record's: %v", got) + } +} + +// An inout is one parameter: the record carries the value the run left in it +// as the member, and the value it was bound with as an In companion. +func TestRecordInoutRun(t *testing.T) { + s := NewSession() + s.now = func() time.Time { return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) } + res := s.Submit(`package Demo { + private import ScalarValues::*; + private import DocumentQueries::*; + private import KerML::Root::Element; + part def Probe; + part probe : Probe; + analysis def Doubling { + subject s : Probe; + inout counter : Integer = 3; + return doubled : Integer = counter * 2; + } + analysis tick : Doubling { subject s = probe; } + calc def Counts :> DocumentQueries::Query { + in root : Element; + Project(source = WhereMetadata( + source = Descendants(source = root, maxDepth = 10), + 'metadata' = "AnalysisRecords::RecordedRun"), + properties = ("counter", "counterIn", "doubled")) + } +}`) + if errs := errorDiagnostics(res.Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + wants(t, run(t, s, "%record Demo::tick into Demo::Log"), "recorded Demo::Log::tick_run1") + text := s.text() + for _, want := range []string{ + "attribute counter : ScalarValues::Integer;", + "attribute counterIn : ScalarValues::Integer;", + "attribute :>> counter = 3;", + "attribute :>> counterIn = 3;", + "attribute :>> doubled = 6;", + } { + if !strings.Contains(text, want) { + t.Errorf("session text is missing %q:\n%s", want, text) + } + } + wants(t, run(t, s, "%run-query Counts root=Demo"), + "counter = 3", "counterIn = 3", "doubled = 6") + path := filepath.Join(t.TempDir(), "demo.sysml") + if _, _, err := s.runMeta("%save " + path); err != nil { + t.Fatal(err) + } + fresh := NewSession() + if errs := errorDiagnostics(fresh.SubmitFiles([]SourceFile{{Name: path, Text: mustRead(t, path)}}).Diagnostics); len(errs) > 0 { + t.Fatalf("saved model has errors: %v", errs) + } +} + +// Two files reopening one package: the record goes into the file that already +// holds the target package, not the first file opening the shared ancestor. +func TestRecordMergesIntoTheFileHoldingTheTargetPackage(t *testing.T) { + s := NewSession() + s.now = func() time.Time { return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) } + res := s.SubmitFiles([]SourceFile{ + {Name: "one.sysml", Text: `package A { + private import ScalarValues::*; + package Cases { analysis def Bound { out y : Real = 1.0; } analysis check : Bound; } +}`}, + {Name: "two.sysml", Text: `package A { package Records { attribute keep : ScalarValues::Integer; } }`}, + }) + if errs := errorDiagnostics(res.Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + wants(t, run(t, s, "%record A::Cases::check"), "recorded A::Records::check_run1") + if n := strings.Count(s.text(), "package Records"); n != 1 { + t.Fatalf("the record made a second A::Records:\n%s", s.text()) + } + if errs := errorDiagnostics(s.diagnostics()); len(errs) > 0 { + t.Fatalf("recording left errors: %v", errs) + } +} + +// A member the runs supply as Integer and Real alike settles to Real — the +// Integer literal stays valid under it — and a scalar-valued enum literal +// records as the literal it is. +func TestRecordRunSettlesNumericFamilyAndKeepsLiterals(t *testing.T) { + s := NewSession() + s.now = func() time.Time { return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) } + res := s.Submit(`package Demo { + private import ScalarValues::*; + part def Thing; + part t : Thing; + enum def Grade :> Integer { high = 3; low = 1; } + analysis def Mix { + subject s = t; + in n : Real; + return half : Real = if n > 2 ? 3 else n / 2.0; + out g : Grade = Grade::high; + } +}`) + if errs := errorDiagnostics(res.Diagnostics); len(errs) > 0 { + t.Fatalf("model has errors: %v", errs) + } + wants(t, run(t, s, "%record Demo::Mix(n=1.0)"), "recorded Records::Mix_run1") + wants(t, run(t, s, "%record Demo::Mix(n=3.0)"), "recorded Records::Mix_run2") + text := s.text() + for _, want := range []string{ + "attribute half : ScalarValues::Real;", + "attribute g : Demo::Grade;", + "attribute :>> half = 0.5;", + "attribute :>> half = 3;", + "attribute :>> g = Demo::Grade::high;", + } { + if !strings.Contains(text, want) { + t.Errorf("session text is missing %q:\n%s", want, text) + } + } + if errs := errorDiagnostics(s.diagnostics()); len(errs) > 0 { + t.Fatalf("recording left errors: %v", errs) + } +} diff --git a/internal/frontend/repl/session.go b/internal/frontend/repl/session.go index e6a577fee..63703da49 100644 --- a/internal/frontend/repl/session.go +++ b/internal/frontend/repl/session.go @@ -8,6 +8,7 @@ import ( "sort" "strings" "sync" + "time" "github.com/Open-MBEE/OpenSysML/internal/check/passes" "github.com/Open-MBEE/OpenSysML/internal/exec/analysis" @@ -166,6 +167,16 @@ type Session struct { // renderWidth is the width a text rendering's table is written to fit, 0 for // as wide as its widest cell. renderWidth int + + // toolVersion is what a recorded run's provenance names as its tool. + toolVersion string + // now is the clock a recorded run's timestamp is taken from. + now func() time.Time + // recordMerge lets a record submission merge into a loaded file's package. + recordMerge bool + // recordDrops are the drop reports of the last submission, which a record + // submission inspects for the loss it must not make. + recordDrops []dropReport } // unnamedObject is an object a later %instantiate of its name displaced. @@ -282,15 +293,28 @@ func (s *stateSession) selfOf() string { // NewSession returns a session over a fresh workspace. func NewSession() *Session { return &Session{ - ws: model.NewWorkspace(), - instances: make(map[string]*runtime.Instance), - budgets: runtime.DefaultBudgets(), - jobs: analysis.DefaultJobs(), - engines: engines.Default(), - verbosity: VerbosityNormal, + ws: model.NewWorkspace(), + instances: make(map[string]*runtime.Instance), + budgets: runtime.DefaultBudgets(), + jobs: analysis.DefaultJobs(), + engines: engines.Default(), + verbosity: VerbosityNormal, + toolVersion: "sysml dev", + now: time.Now, } } +// SetToolVersion names the tool a recorded run's provenance reports. +func (s *Session) SetToolVersion(tool string) { + s.toolVersion = tool +} + +// Text is the session's buffer as it was submitted: what %save writes back. +func (s *Session) Text() string { + defer s.reading()() + return s.text() +} + // enter takes the session for one command; the function returned leaves it. func (s *Session) enter() func() { s.mu.Lock() @@ -456,10 +480,13 @@ func (s *Session) acceptFrom(origin, src string) (declared []string, drops []dro // snippet it absorbed, so the names it replaces are its own, not just // the submitted ones — and is appended like any other submission so a // report still scopes to the tail of the buffer. - if merged, added, drop, ok := s.mergeSubmission(src, root, comments); ok { + if merged, added, drop, ok, inPlace := s.mergeSubmission(src, root, comments); ok { + drops = append(drops, drop) + if inPlace { + return declared, drops + } text, comments, mergedOwn = merged, "", added names = declaredNames(parser.New(source.New(docName, []byte(merged))).ParseFile()) - drops = append(drops, drop) } top := topLevelMembers(root) kept := s.snippets[:0] @@ -602,6 +629,10 @@ func isCommentOnly(src string) bool { // belongs to no file on disk. const sessionOrigin = "" +// SessionOrigin names the accumulated buffer in diagnostics, for callers that +// write the session's text as a document of their own. +const SessionOrigin = sessionOrigin + // joined is the buffer the session analyzes: every accepted submission, with a // submission that does not close its own text masked out so it cannot change how // the others parse. Masking is byte for byte, so every offset still locates the @@ -829,6 +860,7 @@ func (s *Session) submitEach(files []SourceFile) (res Result, byFile [][]string, byFile[i] = dropNotices(dropped) drops = append(drops, dropped...) } + s.recordDrops = drops joined := s.joined() offset := s.genOffset(joined) // A merge rewrote a snippet that was already accepted, so only the text the @@ -837,6 +869,32 @@ func (s *Session) submitEach(files []SourceFile) (res Result, byFile [][]string, if at, ok := firstText(own); ok { offset = at } + whole = s.rebuildOver(drops) + notices := append(dropNotices(drops), whole...) + // The diagnostics already carry their own "did you mean" hints. + diags := s.diagnostics() + members := s.sessionMembers() + res = Result{ + Members: members, + Declared: declared, + Diagnostics: diags, + // The unmasked buffer: masking is byte for byte, so offsets still land + // where they did, and a diagnostic echoes the line it is about. + Source: s.text(), + Offset: offset, + Origins: s.origins(), + own: own, + masked: s.maskedSpans(), + Notices: notices, + } + res.Blocked = s.blockedBy(res) + return res, byFile, whole +} + +// rebuildOver replaces the open document and everything derived from it — the +// runtime context, the resolutions held objects and debugging sessions were +// made against — after the snippets changed, reporting what it carried over. +func (s *Session) rebuildOver(drops []dropReport) []string { // What the session holds is recorded against the resolution that produced it // before the new text replaces that resolution, so what the new document does // not change can be told apart from what it does. @@ -855,29 +913,11 @@ func (s *Session) submitEach(files []SourceFile) (res Result, byFile [][]string, // why it records the version it holds. s.rtCtx = nil gone := goneNames(drops) - whole = s.carryOverObjects(over) + whole := s.carryOverObjects(over) whole = append(whole, s.dropStaleDebugSessions(gone, over)...) - notices := append(dropNotices(drops), whole...) s.rebindRestartedMachine() s.keepIdentitiesOf(over.prev) - // The diagnostics already carry their own "did you mean" hints. - diags := s.diagnostics() - members := s.sessionMembers() - res = Result{ - Members: members, - Declared: declared, - Diagnostics: diags, - // The unmasked buffer: masking is byte for byte, so offsets still land - // where they did, and a diagnostic echoes the line it is about. - Source: s.text(), - Offset: offset, - Origins: s.origins(), - own: own, - masked: s.maskedSpans(), - Notices: notices, - } - res.Blocked = s.blockedBy(res) - return res, byFile, whole + return whole } // fileSpan is where the current submission's text from the named file sits in the diff --git a/internal/frontend/repl/sweep.go b/internal/frontend/repl/sweep.go index 811a57cfa..f8ecba79a 100644 --- a/internal/frontend/repl/sweep.go +++ b/internal/frontend/repl/sweep.go @@ -108,19 +108,23 @@ func (s *Session) sweepFromText(invocation string, ranges []string, draws sweepD // satisfied holds; any failed run or unsatisfied objective fails it. The rows' // traces lead the report in plan order, as one run's trace leads its verdict. func (s *Session) sweepVerdict(inv analysisInvocation, specs []sweepSpec, draws sweepDraws) Verdict { - label := sweepLabel(inv, draws) table, plan, err := s.runSweep(inv, specs, draws) if err != nil { - return standing(unresolvedVerdict(label, err.Error()), plan) + return standing(unresolvedVerdict(sweepLabel(inv, draws), err.Error()), plan) } + return standing(s.sweepReport(inv, table, draws), plan) +} + +// sweepReport is the verdict a completed sweep's table reports. +func (s *Session) sweepReport(inv analysisInvocation, table runtime.SweepTable, draws sweepDraws) Verdict { status, rows := sweepStatus(table) - return standing(Verdict{ - Subject: label, + return Verdict{ + Subject: sweepLabel(inv, draws), Status: status, Lines: append(sweepTraces(table), sweepTableLines(table)...), Values: sweepValues(table, rows), Rows: rows, - }, plan) + } } // sweepTraces is what the rows' runs traced, in plan order, as trace lines print. @@ -247,6 +251,7 @@ func (s *Session) runSweep(inv analysisInvocation, specs []sweepSpec, draws swee Verdicts: result.Verdicts, Subject: result.Subject, Evaluations: result.Evaluations, + Inputs: result.Inputs, }, err } diff --git a/internal/translate/convert/convert.go b/internal/translate/convert/convert.go index d12a3a9cd..08e66228a 100644 --- a/internal/translate/convert/convert.go +++ b/internal/translate/convert/convert.go @@ -210,7 +210,12 @@ func ConvertWith(name string, data []byte, from, to Format, opts Options) ([]byt // declarations the parser could not read would be silently missing, so a broken // model is still rejected. func ConvertTolerant(name string, data []byte, from, to Format) ([]byte, *SyntaxError, error) { - return convert(name, data, from, to, true, Options{}) + return ConvertTolerantWith(name, data, from, to, Options{}) +} + +// ConvertTolerantWith is ConvertTolerant under non-default options. +func ConvertTolerantWith(name string, data []byte, from, to Format, opts Options) ([]byte, *SyntaxError, error) { + return convert(name, data, from, to, true, opts) } // ErrNoNotation reports an element no notation can be written for: one the diff --git a/internal/workspace/libs/stdlib.snapshot b/internal/workspace/libs/stdlib.snapshot index 3f52ed281..a550b8134 100644 Binary files a/internal/workspace/libs/stdlib.snapshot and b/internal/workspace/libs/stdlib.snapshot differ diff --git a/internal/workspace/libs/stdlib/OpenSysML Libraries/AnalysisRecords.sysml b/internal/workspace/libs/stdlib/OpenSysML Libraries/AnalysisRecords.sysml new file mode 100644 index 000000000..7ffb2f32e --- /dev/null +++ b/internal/workspace/libs/stdlib/OpenSysML Libraries/AnalysisRecords.sysml @@ -0,0 +1,83 @@ +library package AnalysisRecords { + /* NON-NORMATIVE OpenSysML run-record vocabulary; + * not part of the OMG SysML or KerML standard library. */ + + private import ScalarValues::*; + + /* RecordedRun is the metadata an analysis run, sweep or sample records + * itself with: when it ran, the tool and command that ran it, and the + * shape of the run it records. */ + metadata def RecordedRun { + /* runAt is the timestamp of the run, written in UTC. */ + attribute runAt : String; + /* tool names the program that made the run. */ + attribute tool : String; + /* command is the invocation text that made the run. */ + attribute command : String; + /* kind is the run shape recorded: "run", "trade", "sweep", "runs" or + * "sample" — the conclusion a Monte-Carlo sample made. */ + attribute kind : String; + } + + /* VerdictRecord is what one check of a recorded run decided: an + * objective's required conditions, an assertion in the case's body, + * or a verification case's own verdict and its subcases'. */ + part def VerdictRecord { + /* kind is "objective", "assertion", "verification" or "subcase". */ + attribute kind : String; + /* name names the objective or asserted constraint, the verification + * case, or spells an anonymous assertion's condition. */ + attribute name : String; + /* status is what the check decided. */ + attribute status : String; + /* detail is the violated condition of a failed check, or why an + * undecided one could not be evaluated. */ + attribute detail : String; + } + + /* EvaluationRecord is one application of a case's calc to an + * alternative, as a trade study's evaluation records it. */ + part def EvaluationRecord { + /* function is the qualified name of the calc applied. */ + attribute function : String; + /* alternative is what the calc was applied to. */ + attribute alternative : String; + /* score is the value the evaluation computed. */ + attribute score : Real; + /* result is the computed value's text. */ + attribute result : String; + /* selected marks the evaluation the case picked. */ + attribute selected : Boolean; + /* tied marks an evaluation computing what the selected did. */ + attribute tied : Boolean; + /* error is why an evaluation computed nothing. */ + attribute error : String; + } + + /* AnalysisRun is the record one run of an analysis case makes: the + * values its inputs bound and its outputs took, the object it ran on, + * and what each of its checks decided and alternatives evaluated. */ + part def AnalysisRun { + /* caseName is the qualified name of the case that ran. */ + attribute caseName : String; + /* kind is the run shape this record makes. */ + attribute kind : String; + /* 'objective' is the status of the run's objective verdict, + * "undecided" when the case declares none. */ + attribute 'objective' : String; + /* iteration is the run's position in a sweep or sample. */ + attribute iteration : Integer; + /* 'subject' is the object the run was made on. */ + ref part 'subject'; + /* subjectName is the subject object's text. */ + attribute subjectName : String; + /* verdict is what the verification body's verdict value decided — + * "pass", "fail", "inconclusive" or "error" — unset for a case + * that is not a verification. */ + attribute verdict : String; + /* verdicts are what the run's checks decided, in order. */ + part verdicts : VerdictRecord[0..*]; + /* evaluations are the calc applications the run made, in order. */ + part evaluations : EvaluationRecord[0..*]; + } +} diff --git a/mkdocs.yml b/mkdocs.yml index 3ad1fbfdc..1d9b3da14 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -183,6 +183,7 @@ nav: - Interfaces: manual/interfaces.md - Worked example: manual/worked-example.md - Requirements traceability examples: manual/traceability-examples.md + - Recording analysis runs: manual/recording-analysis-runs.md - Limitations and troubleshooting: manual/troubleshooting.md - Worked example output: manual/examples/observatory.md - Requirements example output: manual/examples/requirements.md diff --git a/packaging/man/man1/sysml.1 b/packaging/man/man1/sysml.1 index a35448b06..703c69718 100644 --- a/packaging/man/man1/sysml.1 +++ b/packaging/man/man1/sysml.1 @@ -60,6 +60,15 @@ Invoke this calculation and report its result, as \-calc "Fall(3, 4)" Run this analysis or verification case and report its outputs and verdict, as \-analysis "Pkg::Case(3.0) Pkg::part" (repeatable) .TP +.BR \-record\-run " \fIcall\fP" +Run this analysis case as \-analysis does and record the run into the model as +AnalysisRecords elements, one record per \-sweep value or \-runs run +(repeatable) +.TP +.BR \-record\-into " \fIpackage\fP" +Record \-record\-run runs into this package instead of a Records package +beside the case's +.TP .BR \-run\-query " \fIquery\fP" Execute this document query and report its rows, as \-run\-query "Heavy root=telescope" (repeatable) diff --git a/tests/hygiene/layering_test.go b/tests/hygiene/layering_test.go index 8f700217d..47673af95 100644 --- a/tests/hygiene/layering_test.go +++ b/tests/hygiene/layering_test.go @@ -76,6 +76,7 @@ var packageLayer = map[string]string{ "internal/exec/analysis": "exec", "internal/exec/analysis/enginewire": "exec", "internal/exec/analysis/modelform": "exec", + "internal/exec/analysis/record": "exec", "internal/exec/engines": "exec", "internal/exec/objref": "exec",