diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 0000000..73bcc6c --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2024-05-14 - Clinical Regex Compilation Overhead +**Learning:** `_compiled_context_lexicon` in `openmed.clinical.context` builds an array of complex regular expressions across large clinical cue lexicons (historical, negation, uncertainty, etc). Calling it repeatedly inside functions that evaluate every single span (like `scan_context_cues`, `_scope_bounds`) caused severe performance bottlenecks because it recompiled the regexes on every call. +**Action:** Always memoize deterministic, heavily-invoked regex compilations or lexicon generations using `@functools.lru_cache` in text processing pipelines (especially in `openmed.clinical`) to avoid repeated, expensive string evaluations. diff --git a/openmed/openmed/clinical/context.py b/openmed/openmed/clinical/context.py index 9fd11df..8e28234 100644 --- a/openmed/openmed/clinical/context.py +++ b/openmed/openmed/clinical/context.py @@ -37,6 +37,7 @@ import re from collections.abc import Iterable, Iterator, Mapping, Sequence +from functools import lru_cache from dataclasses import dataclass, replace from datetime import date from typing import Any, Literal @@ -154,6 +155,10 @@ class _CompiledContextLexicon: backward_context_cues: frozenset[str] + +# Cache regex compilations for clinical cues to prevent severe performance +# bottlenecks during repeated span evaluations. +@lru_cache(maxsize=32) def _compiled_context_lexicon(language: str | None = None) -> _CompiledContextLexicon: lexicon = get_clinical_cue_lexicon(language) token_boundaries = lexicon.token_boundaries