Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 26 additions & 17 deletions cacheseek/reuse/exact_prefix/telefuser_lingbot.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,27 +50,36 @@
from .types import ActionKey


def make_full_kv_config(*, break_even_k: int = 1) -> WorldKVConfig:
"""Full-length KV (local_attn_size=-1) => materialization needs the entire
prefix."""
return WorldKVConfig(
window_chunks=1_000_000, sink_chunks=1, break_even_k=break_even_k
)


def make_rolling_config(
def make_world_kv_config(
*,
local_attn_size: int = 7, # latent frames, incl. sink (matches TeleFuser pipeline config)
sink_size: int = 3, # latent frames
chunk_size: int = 3, # latent frames per chunk
break_even_k: int = 1,
) -> WorldKVConfig:
"""Rolling window: materialization needs only the sink chunk plus the chunks
covering the most recent (L-S) frames -- O(W) rather than O(K)."""
recent_frames = max(local_attn_size - sink_size, 1)
"""Build WorldKVConfig from TeleFuser LingBot KV geometry.

Uses local_attn_size=-1 for full-length KV. Positive values for
rolling KV.

Rolling window: materialization needs only the sink chunk plus the chunks
covering the most recent (L-S) frames -- O(W) rather than O(K).
"""
if local_attn_size == -1:
window_chunks = 1_000_000
sink_chunks = 1
elif local_attn_size > 0:
recent_frames = max(local_attn_size - sink_size, 1)

window_chunks=-(-recent_frames // chunk_size) # ceil
sink_chunks=-(-sink_size // chunk_size)
else:
raise ValueError(f"local_attn_size must be -1 for full KV and positive "
f"for rolling, got {local_attn_size}")

return WorldKVConfig(
window_chunks=-(-recent_frames // chunk_size), # ceil
sink_chunks=-(-sink_size // chunk_size),
window_chunks=window_chunks,
sink_chunks=sink_chunks,
break_even_k=break_even_k,
)

Expand Down Expand Up @@ -222,8 +231,8 @@ def set_resume_depth(self, depth: int) -> None:
rt = self._rt
global_end = (depth + 1) * rt.chunk_size * rt.frame_tokens
for kv in rt.self_kv_cache:
kv["global_end_index"][0] = global_end
kv["local_end_index"][0] = self._local_end_tokens
kv["global_end_index"] = global_end
kv["local_end_index"] = self._local_end_tokens


class LingBotWorldKVBinding:
Expand Down Expand Up @@ -376,7 +385,7 @@ def on_chunk_finalized(
# In rolling mode this chunk's clean KV sits at the physical tail
# [local_end-ct : local_end] (logical position idx*ct holds only in
# full-length mode); local_end was just advanced by the clean rewrite.
e = int(kv["local_end_index"][0])
e = int(kv["local_end_index"])
s = e - ct
payload.append(
(
Expand Down
8 changes: 6 additions & 2 deletions examples/exact_prefix_reuse/e2e_telefuser_lingbot.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@
from cacheseek.reuse.exact_prefix import NamespaceForest, WorldKVManager
from cacheseek.reuse.exact_prefix.telefuser_lingbot import (
LingBotWorldKVBinding,
make_rolling_config,
make_world_kv_config,
)
from cacheseek.stores import InMemoryTierStore

Expand Down Expand Up @@ -246,7 +246,11 @@ def fresh_stack():
forest = NamespaceForest()
mgr = WorldKVManager(
forest, shared_store,
make_rolling_config(local_attn_size=cfg.local_attn_size, sink_size=cfg.sink_size, chunk_size=3),
make_world_kv_config(
local_attn_size=cfg.local_attn_size,
sink_size=cfg.sink_size,
chunk_size=3
),
)
return forest, mgr

Expand Down
4 changes: 2 additions & 2 deletions tests/test_world_kv_telefuser_binding.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
)
from cacheseek.reuse.exact_prefix.telefuser_lingbot import ( # noqa: E402
LingBotWorldKVBinding,
make_rolling_config,
make_world_kv_config,
)

# Tiny geometry: chunk=3 frames, frame_tokens=2; window L=7 frames including
Expand Down Expand Up @@ -159,7 +159,7 @@ def run_session(binding, runtime, session, *, snapshots: list | None = None):

def make_stack():
forest = NamespaceForest()
cfg = make_rolling_config(local_attn_size=LOCAL_ATTN, sink_size=SINK, chunk_size=CHUNK_FRAMES)
cfg = make_world_kv_config(local_attn_size=LOCAL_ATTN, sink_size=SINK, chunk_size=CHUNK_FRAMES)
return forest, WorldKVManager(forest, InMemoryTierStore(), cfg)


Expand Down