diff --git a/changes/unreleased/release-preparse-trees.fixed.md b/changes/unreleased/release-preparse-trees.fixed.md new file mode 100644 index 0000000000..412d2c5aa2 --- /dev/null +++ b/changes/unreleased/release-preparse-trees.fixed.md @@ -0,0 +1 @@ +- A submission's parse trees are let go once its files are accepted, instead of being held through the analysis for the root namespaces an unresolved reference would be looked up by (#1441); a reference left unresolved parses the text again. Validating a generated 12 086-element model peaks at 164 MiB of resident memory instead of 184, beside 0.11.0's 164 on the same toolchain (152 against 147 under `GOGC=25`), with the load's time and allocations unchanged. diff --git a/internal/frontend/repl/resolve_walk_test.go b/internal/frontend/repl/resolve_walk_test.go index c55251603d..2ca95759a5 100644 --- a/internal/frontend/repl/resolve_walk_test.go +++ b/internal/frontend/repl/resolve_walk_test.go @@ -4,18 +4,14 @@ import ( "fmt" "strings" "testing" - - "github.com/Open-MBEE/OpenSysML/internal/syntax/parser" - "github.com/Open-MBEE/OpenSysML/internal/syntax/source" ) -func resolvedParses(n int) []parsed { +func resolvedSources(n int) []SourceFile { var sb strings.Builder for i := 0; i < n; i++ { fmt.Fprintf(&sb, "package P%d { attribute a%d = %d; }\n", i, i, i) } - root := parser.New(source.New("typed", []byte(sb.String()))).ParseFile() - return []parsed{{root: root}} + return []SourceFile{{Name: "typed", Text: sb.String()}} } // A submission that leaves nothing unresolved looks for no root namespace, so @@ -24,9 +20,9 @@ func TestResolvingNothingDoesNotWalkTheText(t *testing.T) { s, dir := workspaceSession(t) search := &rootSearch{find: s.workspaceIndex().Declaring, where: "under " + dir, onDemand: true} cost := func(n int) float64 { - parses := resolvedParses(n) + sources := resolvedSources(n) return testing.AllocsPerRun(20, func() { - s.resolveReferences(Result{}, parses, search) + s.resolveReferences(Result{}, &submissionRoots{pending: sources}, search) }) } if small, large := cost(50), cost(100); large > small { diff --git a/internal/frontend/repl/session.go b/internal/frontend/repl/session.go index a6c24ca129..52a9dc7c6b 100644 --- a/internal/frontend/repl/session.go +++ b/internal/frontend/repl/session.go @@ -1139,7 +1139,8 @@ func (s *Session) submitEach(files []SourceFile) (res Result, byFile [][]string, // A root namespace the submission leaves unresolved may be declared by a // file beside a loaded one or under the workspace; it joins the submission. search := s.searchFor(load, files) - if ws, more := s.pullReferences(s.resolveReferences(res, parses, search), parses, search); ws.loaded || len(ws.notices) > 0 { + roots := &submissionRoots{pending: files} + if ws, more := s.pullReferences(s.resolveReferences(res, roots, search), roots, search); ws.loaded || len(ws.notices) > 0 { if ws.loaded { whole = append(whole, more...) // A loaded file may have replaced an earlier snippet, moving the diff --git a/internal/frontend/repl/workspace.go b/internal/frontend/repl/workspace.go index 0428b77fd2..8cd2873f8f 100644 --- a/internal/frontend/repl/workspace.go +++ b/internal/frontend/repl/workspace.go @@ -70,12 +70,12 @@ func (s *Session) doWorkspace() []string { } // workspaceLoad is what resolving references did: the files it accepted into -// the current generation, if any, their parses, what they declared, what +// the current generation, if any, their sources, what they declared, what // accepting them dropped, and the lines to report about it. type workspaceLoad struct { loaded bool files []string - parses []parsed + sources []SourceFile declared []string drops []dropReport notices []string @@ -115,12 +115,12 @@ func (s *Session) searchFor(load bool, files []SourceFile) *rootSearch { return nil } -// resolveReferences loads, for a submission whose text parsed to parses and -// analyzed to res, the files declaring the root namespaces its imports and -// qualified references leave unresolved, found where search looks. Imports -// come first: a qualified name an import would answer is not looked for until -// the imports have been (import Lib::*; then Inner::Thing with Lib::Inner). -func (s *Session) resolveReferences(res Result, parses []parsed, search *rootSearch) workspaceLoad { +// resolveReferences loads, for a submission of sources that analyzed to res, +// the files declaring the root namespaces its imports and qualified references +// leave unresolved, found where search looks. Imports come first: a qualified +// name an import would answer is not looked for until the imports have been +// (import Lib::*; then Inner::Thing with Lib::Inner). +func (s *Session) resolveReferences(res Result, roots *submissionRoots, search *rootSearch) workspaceLoad { if search == nil { return workspaceLoad{} } @@ -138,17 +138,10 @@ func (s *Session) resolveReferences(res Result, parses []parsed, search *rootSea return workspaceLoad{} } unresolved := func(name string) bool { return referenceBegins(subjects, name) } - var imported []string - for _, pre := range parses { - imported = append(imported, project.ImportedRoots(pre.root)...) - } + imported, referenced := roots.names(s) ws := s.loadDeclaring(s.undeclaredRoots(imported, unresolved), search) if !ws.loaded { // An import no file answers must not keep the qualified names waiting. - var referenced []string - for _, pre := range parses { - referenced = append(referenced, project.ReferencedRoots(pre.root)...) - } more := s.loadDeclaring(s.undeclaredRoots(referenced, unresolved), search) more.notices = append(ws.notices, more.notices...) ws = more @@ -156,12 +149,47 @@ func (s *Session) resolveReferences(res Result, parses []parsed, search *rootSea return ws } -// pullReferences carries on from what loading ws did for the text parsed to -// parses: the session is rebuilt over the files loaded, and the root +// submissionRoots is the root namespaces a submission's sources import and +// reference, by name: a source is parsed once, on first need, and its tree let go. +type submissionRoots struct { + pending []SourceFile + imported []string + referenced []string +} + +// names is the imported and the referenced root names, parsing what is pending. +func (r *submissionRoots) names(s *Session) (imported, referenced []string) { + if len(r.pending) > 0 { + for _, root := range s.parseRoots(r.pending) { + r.imported = append(r.imported, project.ImportedRoots(root)...) + r.referenced = append(r.referenced, project.ReferencedRoots(root)...) + } + r.pending = nil + } + return r.imported, r.referenced +} + +// add takes in the files a round loaded, to parse if a later round asks. +func (r *submissionRoots) add(sources []SourceFile) { r.pending = append(r.pending, sources...) } + +// parseRoots parses the sources for the root namespaces they import and reference. +func (s *Session) parseRoots(sources []SourceFile) []*ast.RootNamespace { + roots := make([]*ast.RootNamespace, len(sources)) + model.ParallelFor(s.jobs, len(sources), func(i int) { + roots[i] = preparseWithKind(sources[i].Name, sources[i].Text, sources[i].Kind).root + }) + return roots +} + +// pullReferences carries on from what loading ws did for the submission whose +// roots these are: the session is rebuilt over the files loaded, and the root // namespaces the text and those files leave unresolved in turn are loaded the // same way, round after round, until a round loads nothing. It reports what // the rounds did together, and what rebuilding did to the session. -func (s *Session) pullReferences(ws workspaceLoad, parses []parsed, search *rootSearch) (all workspaceLoad, whole []string) { +func (s *Session) pullReferences(ws workspaceLoad, roots *submissionRoots, search *rootSearch) (all workspaceLoad, whole []string) { + if roots == nil { + roots = &submissionRoots{} + } // A root two files declare is noticed by every round that wants it; once is enough. noticed := map[string]bool{} notice := func(notices []string) { @@ -183,8 +211,8 @@ func (s *Session) pullReferences(ws workspaceLoad, parses []parsed, search *root whole = append(whole, more...) all.notices = append(append(all.notices, dropNotices(ws.drops)...), more...) notice(ws.notices) - parses = append(parses, ws.parses...) - ws = s.resolveReferences(s.generationResult(), parses, search) + roots.add(ws.sources) + ws = s.resolveReferences(s.generationResult(), roots, search) } notice(ws.notices) return all, whole @@ -318,13 +346,14 @@ func (s *Session) loadDeclaring(names []string, search *rootSearch) workspaceLoa out.notices = append(out.notices, errPrefix+err.Error()) return out } - out.parses = make([]parsed, len(read.files)) + parses := make([]parsed, len(read.files)) model.ParallelFor(s.jobs, len(read.files), func(i int) { - out.parses[i] = preparseWithKind(read.files[i].Name, read.files[i].Text, read.files[i].Kind) + parses[i] = preparseWithKind(read.files[i].Name, read.files[i].Text, read.files[i].Kind) }) + out.sources = read.files seen := map[string]bool{} for i, f := range read.files { - names, dropped := s.acceptParsed(f.Name, f.key(), fileKeyOf(f.Of), f.Text, out.parses[i], f.Kind) + names, dropped := s.acceptParsed(f.Name, f.key(), fileKeyOf(f.Of), f.Text, parses[i], f.Kind) s.snippets[len(s.snippets)-1].onDemand = search.onDemand if f.Of != "" { out.files = append(out.files, f.Of)