Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,17 @@

### Bug Fixes

- **#1514**: Markdown path links resolve against the note's own path at resolution
time, the way wikilinks do, instead of at parse time from the file's location
on disk. The parser had derived the note's project path with `relative_to` on the
filesystem path, which raised for content parsed from anywhere outside the project
root (a hosted note read from object storage, for one) and gave a wrong base for
any other temporary location. The graph now stores the path as authored
(`../guides/Guide.md`, `./same.md`, `/root.md`); both resolvers turn it into a
project path from the source note, and background resolution keys path targets
by their source note. Wikilinks spelled `[[../x.md]]` or `[[./x.md]]` resolve
by the same rule.

- **#1558**: `search_notes(search_all_projects=True)` and `projects=[...]` rank merged
full-text hits by score strength instead of raw value. SQLite bm25 scores are
negative with lower meaning better, so sorting raw values put the weakest hit first
Expand Down
12 changes: 10 additions & 2 deletions docs/MARKDOWN_RELATIONS.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,16 @@ Markdown links work too.

These are exact file paths. Basic Memory does not guess a title, add `.md`, apply
filename aliases, or search another project when the target is missing. The graph
stores a normalized project-root target such as `/guides/Getting Started.md`;
missing targets remain unresolved and can resolve when indexed later.
stores the path as the author wrote it, relative to the note: `../guides/Getting
Started.md`, `./same.md` (a bare `same.md` is stored with the `./` mark), or a
rooted `/guides/Getting Started.md`. Resolution turns it into a project path
against the note's own location, so a note parsed from remote storage or moved
later resolves the same way; missing targets remain unresolved and can resolve
when indexed later.

Wikilinks spelled as explicit paths follow the same rule: `[[../guides/Getting
Started.md]]` and `[[./same.md]]` resolve relative to the note and only to that
exact file. Other wikilinks keep their title, permalink, and alias resolution.

External URLs, `mailto:` and `file:` links, fragment-only links, paths that escape
the project, images, and links inside code do not create relations. Ordinary
Expand Down
13 changes: 13 additions & 0 deletions src/basic_memory/indexing/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,19 @@ class IndexFrontmatterWriteResult:
content: str


@dataclass(frozen=True, slots=True)
class RelationTargetRequest:
"""One relation target to resolve, and the note a path target is relative to.

Identity targets (titles, permalinks, external ids) mean the same thing from
every note, so they carry no source and one lookup serves them all. A path
target means one file per source note, so its source path is part of the key.
"""

link_text: str
source_path: str | None = None


@dataclass(frozen=True, slots=True)
class IndexedRelation:
"""One parsed outgoing relation waiting for generation-owned publication."""
Expand Down
51 changes: 42 additions & 9 deletions src/basic_memory/indexing/relation_resolution.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@

from basic_memory import db
from basic_memory.indexing.accepted_note_search import accepted_search_content_from_markdown
from basic_memory.indexing.models import IndexFileJobStatus
from basic_memory.indexing.models import IndexFileJobStatus, RelationTargetRequest
from basic_memory.markdown.path_links import is_path_target
from basic_memory.models import Entity
from basic_memory.repository.relation_repository import (
PendingRelationSearchRefresh,
Expand Down Expand Up @@ -194,10 +195,10 @@ class RelationTargetBatchResolver(Protocol):

async def resolve_relation_targets(
self,
link_texts: Sequence[str],
requests: Sequence[RelationTargetRequest],
*,
session: AsyncSession,
) -> Mapping[str, ResolvedRelationTarget | None]:
) -> Mapping[RelationTargetRequest, ResolvedRelationTarget | None]:
"""Resolve strict link targets without per-target database round-trips."""


Expand Down Expand Up @@ -257,6 +258,16 @@ async def count_unresolved_relations(self) -> int:
async with db.scoped_session(self.session_maker) as session:
return len(await self.relation_repository.find_unresolved_relations(session))

async def _source_paths(
self,
session: AsyncSession,
relations: Sequence[UnresolvedRelation],
) -> dict[EntityId, str]:
"""Return the project path of every note that carries one of ``relations``."""
source_ids = sorted({relation.from_id for relation in relations})
sources = await self.entity_repository.find_by_ids(session, source_ids)
return {source.id: source.file_path for source in sources}

async def resolve_relations(
self,
entity_id: EntityId | None = None,
Expand All @@ -280,15 +291,25 @@ async def resolve_relations(
count=len(unresolved_relations),
)

target_names = list(
dict.fromkeys(relation.to_name for relation in unresolved_relations)
# A path target (``./``, ``../``, ``/``) names a file relative to the note
# that carries it, so it is keyed by that note's path as well; identity
# targets stay keyed by text alone and resolve once for every source.
source_paths = (
await self._source_paths(session, unresolved_relations)
if any(is_path_target(relation.to_name) for relation in unresolved_relations)
else {}
)
resolved_targets_by_link_text = (
requests = list(
dict.fromkeys(
_target_request(relation, source_paths) for relation in unresolved_relations
)
)
resolved_targets_by_request = (
await self.target_resolver.resolve_relation_targets(
target_names,
requests,
session=session,
)
if target_names
if requests
else {}
)

Expand All @@ -300,7 +321,7 @@ async def resolve_relations(
f"from_id={relation.from_id} "
f"to_name={relation.to_name}"
)
resolved_entity = resolved_targets_by_link_text[relation.to_name]
resolved_entity = resolved_targets_by_request[_target_request(relation, source_paths)]
if resolved_entity is None or resolved_entity.id == relation.from_id:
continue

Expand Down Expand Up @@ -406,6 +427,18 @@ async def resolve_relations(
return affected_entity_ids


def _target_request(
relation: UnresolvedRelation,
source_paths: Mapping[EntityId, str],
) -> RelationTargetRequest:
return RelationTargetRequest(
link_text=relation.to_name,
source_path=(
source_paths.get(relation.from_id) if is_path_target(relation.to_name) else None
),
)


@dataclass(frozen=True, slots=True)
class ResolveRelationsJobRequest:
"""Queue-neutral request shape for resolving one project's forward references."""
Expand Down
35 changes: 13 additions & 22 deletions src/basic_memory/markdown/entity_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from markdown_it import MarkdownIt

from basic_memory.markdown.plugins import observation_plugin, relation_plugin
from basic_memory.markdown.path_links import markdown_link_target
from basic_memory.markdown.path_links import markdown_link_path
from basic_memory.markdown.schemas import (
EntityFrontmatter,
EntityMarkdown,
Expand Down Expand Up @@ -155,7 +155,7 @@ class EntityContent:
relations: list[Relation] = field(default_factory=list)


def parse(content: str, *, source_path: str | None = None) -> EntityContent:
def parse(content: str) -> EntityContent:
"""Parse markdown content into EntityMarkdown."""

# Parse content for observations and relations using markdown-it
Expand All @@ -165,16 +165,16 @@ def parse(content: str, *, source_path: str | None = None) -> EntityContent:
if content:
for token in md.parse(content):
# MarkdownIt owns link syntax, including escapes, reference links and
# code exclusion. Rooted targets retain exact project-path semantics
# through deferred resolution without changing the authored body.
if source_path is not None:
for child in token.children or []:
if child.type == "link_open":
href = child.attrGet("href")
assert isinstance(href, str)
target = markdown_link_target(href, source_path) if href else None
if target is not None:
relations.append(Relation(type="links_to", target=target))
# code exclusion. A Markdown link is recorded as the path its author
# wrote; it resolves against the note's own path later, like any other
# relation, so parsing needs no idea of where the bytes came from.
for child in token.children or []:
if child.type == "link_open":
href = child.attrGet("href")
assert isinstance(href, str)
target = markdown_link_path(href) if href else None
if target is not None:
relations.append(Relation(type="links_to", target=target))
# check for observations and relations
if token.meta:
if "observation" in token.meta:
Expand Down Expand Up @@ -361,16 +361,7 @@ async def parse_markdown_content(
or (isinstance(semantic_setting, str) and semantic_setting.lower() == "false")
)
entity_content = (
parse(
post.content,
source_path=(
file_path.relative_to(self.base_path).as_posix()
if file_path.is_absolute()
else file_path.as_posix()
),
)
if parse_semantics
else EntityContent(content=post.content)
parse(post.content) if parse_semantics else EntityContent(content=post.content)
)

# The parser reports only a qualifier the author plainly meant: an unknown kind,
Expand Down
54 changes: 48 additions & 6 deletions src/basic_memory/markdown/path_links.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,36 @@
"""Project-local targets carried by ordinary Markdown links."""
"""Project-local file targets: Markdown links and path-shaped wikilinks.

A path target names one file by where it sits, relative to the note that carries
it. It never resolves through a title, a permalink, a filename alias or another
project. The parser records the author's spelling; resolution turns it into a
project-root path once the note's own path is known, which is a database fact and
not a filesystem one.
"""

from pathlib import PurePosixPath
from urllib.parse import unquote, urlsplit

PATH_TARGET_PREFIXES = ("/", "./", "../")


def is_path_target(target: str) -> bool:
"""Whether a relation target names a project file by path rather than identity.

Rooted (``/``) and explicitly relative (``./``, ``../``) spellings are paths in
Markdown links and wikilinks alike.
"""
return target.startswith(PATH_TARGET_PREFIXES)


def markdown_link_target(href: str, source_path: str) -> str | None:
"""Return a project-root path, excluding URLs and paths escaping the project."""
def markdown_link_path(href: str) -> str | None:
"""Return the authored project path of a Markdown link, or None for anything else.

URLs, ``mailto:`` and ``file:`` links, fragment-only links, and paths carrying
backslashes or null bytes are prose, not relations. Percent-encoding is decoded
and the query and fragment are dropped. The result keeps the author's spelling
relative to the note; a bare relative path is marked ``./`` so the stored
target says it is a path and not a title.
"""
try:
parsed = urlsplit(href)
except ValueError:
Expand All @@ -14,10 +39,27 @@ def markdown_link_target(href: str, source_path: str) -> str | None:
if parsed.scheme or parsed.netloc or not parsed.path:
return None
path = unquote(parsed.path)
if "\\" in path or "\x00" in path:
if "\\" in path or "\x00" in path or path.endswith("/"):
# Backslashes and null bytes are not project paths; a trailing slash
# names a folder, and only files carry relations.
return None
return path if is_path_target(path) else f"./{path}"


def resolve_project_path(target: str, source_path: str | None) -> str | None:
"""Resolve a path target against the note that carries it, as a project-root path.

``source_path`` is the note's own project path, such as ``notes/source.md``.
Rooted targets ignore it. Relative targets resolve against its folder, or
against the project root when no source is known. ``.`` and ``..`` segments
collapse, and a target that climbs past the root names no project file.
"""
if not is_path_target(target):
return None
parts = [] if path.startswith("/") else list(PurePosixPath(source_path).parent.parts)
for part in path.split("/"):
parts: list[str] = []
if not target.startswith("/") and source_path:
parts = list(PurePosixPath(source_path).parent.parts)
for part in target.split("/"):
if part in {"", "."}:
continue
if part == "..":
Expand Down
46 changes: 34 additions & 12 deletions src/basic_memory/services/bulk_link_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
from sqlalchemy.ext.asyncio import AsyncSession

from basic_memory.config import BasicMemoryConfig
from basic_memory.indexing.models import RelationTargetRequest
from basic_memory.markdown.path_links import is_path_target, resolve_project_path
from basic_memory.models import Entity, Project
from basic_memory.repository.entity_repository import EntityRepository, file_path_alias
from basic_memory.repository.project_repository import ProjectRepository
Expand All @@ -30,22 +32,37 @@ class RelationTargetReference:
original: str
identifier: str
explicitly_qualified: bool
# The note a path target is relative to; identity targets carry none.
source_path: str | None = None

@classmethod
def parse(cls, link_text: str) -> "RelationTargetReference":
def parse(cls, link_text: str, source_path: str | None = None) -> "RelationTargetReference":
"""Normalize wikilink syntax once for the whole bulk-resolution pass."""
if link_text.startswith("/"):
return cls(original=link_text, identifier=link_text, explicitly_qualified=False)
clean_text, _ = normalize_link_text(link_text)
if is_path_target(clean_text):
return cls(
original=link_text,
identifier=clean_text,
explicitly_qualified=False,
source_path=source_path,
)
return cls(
original=link_text,
identifier=normalize_project_reference(clean_text),
explicitly_qualified="::" in clean_text,
)

@classmethod
def from_request(cls, request: RelationTargetRequest) -> "RelationTargetReference":
return cls.parse(request.link_text, request.source_path)

@property
def is_path(self) -> bool:
return is_path_target(self.identifier)

def project_path(self) -> tuple[str | None, str]:
"""Return a possible project prefix and its remaining target path."""
if "/" not in self.identifier:
if self.is_path or "/" not in self.identifier:
return None, self.identifier

project_prefix, remainder = self.identifier.split("/", 1)
Expand Down Expand Up @@ -208,10 +225,11 @@ def resolve(self, target: RelationTargetReference) -> Entity | None:
"""Resolve one parsed target without additional I/O."""
current_index = self.entity_indexes[self.current_project_id]

# Rooted Markdown targets are file identities, never title/permalink or
# cross-project guesses, including while their target is still absent.
if target.identifier.startswith("/"):
return current_index.by_file_path.get(target.identifier[1:])
# Path targets are file identities relative to their source note, never
# title, permalink or cross-project guesses, including while absent.
if target.is_path:
project_path = resolve_project_path(target.identifier, target.source_path)
return current_index.by_file_path.get(project_path[1:]) if project_path else None

try:
external_id = str(uuid_mod.UUID(target.identifier))
Expand Down Expand Up @@ -332,13 +350,14 @@ class BulkLinkResolver:

async def resolve_relation_targets(
self,
link_texts: Sequence[str],
requests: Sequence[RelationTargetRequest],
*,
session: AsyncSession,
) -> dict[str, Entity | None]:
) -> dict[RelationTargetRequest, Entity | None]:
"""Resolve unique relation targets with I/O bounded by referenced projects."""
unique_requests = tuple(dict.fromkeys(requests))
targets = tuple(
RelationTargetReference.parse(link_text) for link_text in dict.fromkeys(link_texts)
RelationTargetReference.from_request(request) for request in unique_requests
)
if not targets:
return {}
Expand All @@ -350,4 +369,7 @@ async def resolve_relation_targets(
app_config=self.app_config,
session=session,
)
return {target.original: snapshot.resolve(target) for target in targets}
return {
request: snapshot.resolve(target)
for request, target in zip(unique_requests, targets, strict=True)
}
Loading
Loading