diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 0000000..606c80b --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2024-05-24 - Memoizing NLP Regex Compilation +**Learning:** NLP and clinical text evaluation repetitively instantiates the same compiled regexes and lexicons if missing memoization, leading to heavy redundant object compilation overhead. +**Action:** Always memoize deterministic lexicon object/regex instantiations (e.g. using @functools.lru_cache) and wrap return types in immutables to avoid performance bottlenecks in loops/text processing pipelines. diff --git a/openmed/openmed/clinical/context.py b/openmed/openmed/clinical/context.py index 9fd11df..b761125 100644 --- a/openmed/openmed/clinical/context.py +++ b/openmed/openmed/clinical/context.py @@ -35,7 +35,9 @@ from __future__ import annotations +import functools import re +import types from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass, replace from datetime import date @@ -154,7 +156,9 @@ class _CompiledContextLexicon: backward_context_cues: frozenset[str] +@functools.lru_cache(maxsize=32) def _compiled_context_lexicon(language: str | None = None) -> _CompiledContextLexicon: + """Returns a memoized compiled lexicon to avoid repeated regex compilation.""" lexicon = get_clinical_cue_lexicon(language) token_boundaries = lexicon.token_boundaries return _CompiledContextLexicon( @@ -195,7 +199,7 @@ def _compiled_context_lexicon(language: str | None = None) -> _CompiledContextLe ), token_boundaries=token_boundaries, ), - category_by_text=_cue_category_lookup(lexicon), + category_by_text=types.MappingProxyType(_cue_category_lookup(lexicon)), backward_context_cues=frozenset( _normalize_cue_text(cue) for cue in lexicon.backward ),