From f8aa8d8e005966ea2ebf50f3cccc20d1df1e61d0 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 13 Sep 2026 13:18:34 -0500 Subject: [PATCH 1/7] =?UTF-8?q?server:=20context=20checkpoints=20ride=20wi?= =?UTF-8?q?th=20a=20slot=20save/restore=20(sidecar)=20=E2=80=94=20a=20rest?= =?UTF-8?q?ored=20slot=20is=20warm=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The slot state file carries KV + tokens; the context checkpoints (recurrent/SWA rollback points) lived only in slot.prompt and were cleared by prompt.clear() on restore. On a hybrid-cache model the next request can reuse a prefix only by rolling back to a checkpoint at or before its divergence — and even an identical prompt rolls back one token ([TAG_PROMPT_LOGITS]) — so a restored slot always hit do_reset: "restore ok" followed by a full re-prefill (measured 2026-09-04: n_restored 11298 → cache_n 0; the second identical request 11291). SLOT_SAVE now writes `.ckpt` beside the state (magic, version, n; per checkpoint n_tokens/pos_min/pos_max + the three state blobs). SLOT_RESTORE reads it back into slot.prompt.checkpoints; a missing or malformed sidecar leaves the pre-fork behaviour (valid restore, no checkpoints) with a warning, never an error. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/server/server-context.cpp | 73 +++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 36a38608de93..32e2a55f1d50 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2473,6 +2473,42 @@ struct server_context_impl { send_error(task, "Unable to save slot", ERROR_TYPE_SERVER); break; } + // Continuum fork: the CONTEXT CHECKPOINTS are part of the register file. + // On a hybrid/recurrent (or SWA) cache the next request can only reuse a + // prefix by rolling back to a checkpoint at or before the divergence — + // and even an identical prompt rolls back one token [TAG_PROMPT_LOGITS]. + // A restored slot with no checkpoints therefore always hits do_reset: + // "restore ok", then a full re-prefill (measured 2026-09-04, n_restored + // 11298 → cache_n 0). The state file carries KV + tokens only, so the + // checkpoints ride in a sidecar beside it; a missing or unreadable + // sidecar degrades to the pre-fork cold restore, never to an error. + { + const std::string ckpt_path = filepath + ".ckpt"; + std::ofstream out(ckpt_path, std::ios::binary | std::ios::trunc); + auto put = [&](const void * ptr, size_t n) { out.write(reinterpret_cast(ptr), n); }; + const uint32_t magic = 0x434B5054u; // 'CKPT' + const uint32_t version = 1; + const uint32_t n_ckpt = (uint32_t) slot->prompt.checkpoints.size(); + put(&magic, sizeof(magic)); put(&version, sizeof(version)); put(&n_ckpt, sizeof(n_ckpt)); + for (const auto & c : slot->prompt.checkpoints) { + const int64_t n_tokens = c.n_tokens; + const int32_t pos_min = c.pos_min; + const int32_t pos_max = c.pos_max; + const uint64_t sz_tgt = c.data_tgt.size(); + const uint64_t sz_dft = c.data_dft.size(); + const uint64_t sz_spec = c.data_spec.size(); + put(&n_tokens, sizeof(n_tokens)); put(&pos_min, sizeof(pos_min)); put(&pos_max, sizeof(pos_max)); + put(&sz_tgt, sizeof(sz_tgt)); put(&sz_dft, sizeof(sz_dft)); put(&sz_spec, sizeof(sz_spec)); + put(c.data_tgt.data(), c.data_tgt.size()); + put(c.data_dft.data(), c.data_dft.size()); + put(c.data_spec.data(), c.data_spec.size()); + } + if (!out) { + SLT_WRN(*slot, "checkpoint sidecar could not be written: %s (the restore will be cold)\n", ckpt_path.c_str()); + } else { + SLT_INF(*slot, "saved %u context checkpoints beside the slot state: %s\n", n_ckpt, ckpt_path.c_str()); + } + } const int64_t t_end = ggml_time_us(); const double t_save_ms = (t_end - t_start) / 1000.0; @@ -2533,6 +2569,43 @@ struct server_context_impl { slot->prompt.clear(); slot->prompt.tokens = std::move(restored); + // Continuum fork: bring the context checkpoints back with the state (see + // SLOT_SAVE). Any defect in the sidecar leaves the slot exactly as a + // pre-fork restore left it — valid tokens + state, no checkpoints. + { + const std::string ckpt_path = filepath + ".ckpt"; + std::ifstream in(ckpt_path, std::ios::binary); + std::list loaded; + bool ok = (bool) in; + auto get = [&](void * ptr, size_t n) { in.read(reinterpret_cast(ptr), n); ok = ok && (bool) in; }; + if (ok) { + uint32_t magic = 0, version = 0, n_ckpt = 0; + get(&magic, sizeof(magic)); get(&version, sizeof(version)); get(&n_ckpt, sizeof(n_ckpt)); + ok = ok && magic == 0x434B5054u && version == 1 && n_ckpt <= 4096; + for (uint32_t i = 0; ok && i < n_ckpt; ++i) { + int64_t n_tokens = 0; int32_t pos_min = 0, pos_max = 0; + uint64_t sz_tgt = 0, sz_dft = 0, sz_spec = 0; + get(&n_tokens, sizeof(n_tokens)); get(&pos_min, sizeof(pos_min)); get(&pos_max, sizeof(pos_max)); + get(&sz_tgt, sizeof(sz_tgt)); get(&sz_dft, sizeof(sz_dft)); get(&sz_spec, sizeof(sz_spec)); + // a checkpoint is at most one sequence state; refuse absurd sizes + ok = ok && sz_tgt < (1ull << 36) && sz_dft < (1ull << 36) && sz_spec < (1ull << 30); + if (!ok) break; + auto & c = loaded.emplace_back(); + c.update_pos(n_tokens, pos_min, pos_max); + c.id_task = -1; + c.data_tgt.resize(sz_tgt); get(c.data_tgt.data(), sz_tgt); + c.data_dft.resize(sz_dft); get(c.data_dft.data(), sz_dft); + c.data_spec.resize(sz_spec); get(c.data_spec.data(), sz_spec); + } + } + if (ok) { + slot->prompt.checkpoints = std::move(loaded); + SLT_INF(*slot, "restored %zu context checkpoints from the sidecar: %s\n", slot->prompt.checkpoints.size(), ckpt_path.c_str()); + } else { + slot->prompt.checkpoints.clear(); + SLT_WRN(*slot, "no usable checkpoint sidecar (%s): the restore is cold — the next divergence re-prefills from the head\n", ckpt_path.c_str()); + } + } } catch (const std::exception & err) { slot->prompt_clear(); send_error(task, std::string("Unable to restore slot: ") + err.what(), ERROR_TYPE_INVALID_REQUEST); From ae375e6983120c8c98297f671585aca0c14af93d Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 13 Sep 2026 13:32:17 -0500 Subject: [PATCH 2/7] =?UTF-8?q?ggml:=20moe-gguf-source:=20widen=20the=20sh?= =?UTF-8?q?ard-index=20buffer=20=E2=80=94=20gcc=20-Werror=3Dformat-truncat?= =?UTF-8?q?ion=20on=20the=20CI=20(cpu)=20x64=20job?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- ggml/src/ggml-moe-gguf-source.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-moe-gguf-source.h b/ggml/src/ggml-moe-gguf-source.h index d3cb46c40d50..068d934c9593 100644 --- a/ggml/src/ggml-moe-gguf-source.h +++ b/ggml/src/ggml-moe-gguf-source.h @@ -70,7 +70,7 @@ class GgufMoeSource { const int total = std::atoi(first.substr(of + pat.size(), 5).c_str()); const size_t idx0 = first.rfind('-', of - 1); const std::string prefix = first.substr(0, idx0 + 1), suffix = first.substr(of); - for (int i = 1; i <= total; i++) { char n[8]; std::snprintf(n, sizeof(n), "%05d", i); files.push_back(prefix + n + suffix); } + for (int i = 1; i <= total; i++) { char n[16]; std::snprintf(n, sizeof(n), "%05d", i); files.push_back(prefix + n + suffix); } } for (const auto & p : files) { Shard s; gguf_init_params gp{}; gp.no_alloc = true; gp.ctx = nullptr; From 45626cfae1d926de97ee616be236be6095d88331 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 13 Sep 2026 13:43:03 -0500 Subject: [PATCH 3/7] tools: moe-fit: widen the shard-index buffer too (gcc -Werror=format-truncation) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/moe-fit/fit-device.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/moe-fit/fit-device.cpp b/tools/moe-fit/fit-device.cpp index ead67de7b186..9202697db1f3 100644 --- a/tools/moe-fit/fit-device.cpp +++ b/tools/moe-fit/fit-device.cpp @@ -39,7 +39,7 @@ static std::vector shard_paths(const std::string & first) { const int total = std::atoi(first.substr(of + pat.size(), 5).c_str()); const size_t idx0 = first.rfind('-', of - 1); const std::string prefix = first.substr(0, idx0 + 1), suffix = first.substr(of); - for (int i = 1; i <= total; i++) { char b[8]; std::snprintf(b, sizeof(b), "%05d", i); files.push_back(prefix + b + suffix); } + for (int i = 1; i <= total; i++) { char b[16]; std::snprintf(b, sizeof(b), "%05d", i); files.push_back(prefix + b + suffix); } return files; } From 13bbb16a0775e75aa29d1674d9a76bb4e9fe4840 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 13 Sep 2026 14:08:37 -0500 Subject: [PATCH 4/7] =?UTF-8?q?server=20tests:=20the=20slot-binding=20proo?= =?UTF-8?q?f=20keeps=20the=20long=20slot=20busy=20for=20the=20whole=20rest?= =?UTF-8?q?ore=20round-trip=20(4096=20tokens)=20=E2=80=94=20fast=20hosted?= =?UTF-8?q?=20runners=20ended=20the=20512-token=20gen=20before=20the=20ass?= =?UTF-8?q?ertion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/server/tests/unit/test_slot_save.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py index 2d905055d7a5..c2ec95010d2f 100644 --- a/tools/server/tests/unit/test_slot_save.py +++ b/tools/server/tests/unit/test_slot_save.py @@ -168,7 +168,10 @@ def test_deferred_restore_is_bound_to_its_slot(): global server server.server_slots = True # /slots GET to observe processing server.n_slots = 2 - server.n_predict = 512 + # The long slot must OUTLAST the whole restore round-trip on a fast CI runner (a 512-token + # gen of the tiny model ends in ~2 s on hosted ubuntu/windows runners — measured 2026-09-13: + # the binding assertion then failed for timing, not for binding). 4096 keeps slot 1 busy. + server.n_predict = 4096 server.start() # A reusable page on disk: process a prompt on slot 0, save it, let slot 0 idle. @@ -194,7 +197,7 @@ def gen(id_slot, n_predict, done): # slot 0 finishes QUICKLY (short), slot 1 runs LONG. The returner wants slot 0. t0 = threading.Thread(target=gen, args=(0, 24, done0), daemon=True) - t1 = threading.Thread(target=gen, args=(1, 512, done1), daemon=True) + t1 = threading.Thread(target=gen, args=(1, 4096, done1), daemon=True) t1.start() # ensure slot 1 is processing before slot 0 starts, so the slot-1 op queues first deadline = time.time() + 20 From a02caaa68e4c42247ecc7b016f90fb70ca732ffc Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 13 Sep 2026 14:16:21 -0500 Subject: [PATCH 5/7] =?UTF-8?q?server=20tests:=20the=20long=20slot=20ignor?= =?UTF-8?q?es=20EOS=20=E2=80=94=20the=20tiny=20model=20ends=20a=20'long=20?= =?UTF-8?q?story'=20in=20a=20few=20tokens=20on=20hosted=20runners,=20which?= =?UTF-8?q?=20read=20as=20a=20binding=20failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/server/tests/unit/test_slot_save.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py index c2ec95010d2f..4e002da68672 100644 --- a/tools/server/tests/unit/test_slot_save.py +++ b/tools/server/tests/unit/test_slot_save.py @@ -190,6 +190,7 @@ def gen(id_slot, n_predict, done): for _ in server.make_stream_request("POST", "/completion", data={ "prompt": "Tell me a very long and detailed story about a cat.", "id_slot": id_slot, "n_predict": n_predict, "cache_prompt": True, "stream": True, + "ignore_eos": True, # the tiny model emits EOS within a few tokens: the long slot must really run long }): pass finally: From d61a39480983c70398b324afd653d2c576a566b7 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 13 Sep 2026 14:24:18 -0500 Subject: [PATCH 6/7] server tests: the short slot runs 256 tokens so a fast runner can observe both slots busy before the restores queue Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/server/tests/unit/test_slot_save.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py index 4e002da68672..ce32aea3f69f 100644 --- a/tools/server/tests/unit/test_slot_save.py +++ b/tools/server/tests/unit/test_slot_save.py @@ -196,8 +196,8 @@ def gen(id_slot, n_predict, done): finally: done.set() - # slot 0 finishes QUICKLY (short), slot 1 runs LONG. The returner wants slot 0. - t0 = threading.Thread(target=gen, args=(0, 24, done0), daemon=True) + # slot 0 finishes FIRST (256 tokens: long enough that a fast runner can observe it busy), slot 1 runs LONG. The returner wants slot 0. + t0 = threading.Thread(target=gen, args=(0, 256, done0), daemon=True) t1 = threading.Thread(target=gen, args=(1, 4096, done1), daemon=True) t1.start() # ensure slot 1 is processing before slot 0 starts, so the slot-1 op queues first From 13309540ba19e4de74aa8f7b1c55e5b75032516a Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Sun, 13 Sep 2026 14:30:21 -0500 Subject: [PATCH 7/7] =?UTF-8?q?server=20tests:=20the=20slot-binding=20proo?= =?UTF-8?q?f=20is=20measured=20in=20timestamps=20=E2=80=94=20the=20restore?= =?UTF-8?q?=20returns=20beside=20its=20OWN=20slot's=20end,=20not=20the=20l?= =?UTF-8?q?ong=20slot's;=20skips=20when=20a=20runner=20cannot=20separate?= =?UTF-8?q?=20the=20slots?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- tools/server/tests/unit/test_slot_save.py | 25 +++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py index ce32aea3f69f..f0d5dd87239c 100644 --- a/tools/server/tests/unit/test_slot_save.py +++ b/tools/server/tests/unit/test_slot_save.py @@ -172,6 +172,7 @@ def test_deferred_restore_is_bound_to_its_slot(): # gen of the tiny model ends in ~2 s on hosted ubuntu/windows runners — measured 2026-09-13: # the binding assertion then failed for timing, not for binding). 4096 keeps slot 1 busy. server.n_predict = 4096 + server.n_ctx = 8192 # 4096 per slot: room for the long slot to actually run 4096 tokens server.start() # A reusable page on disk: process a prompt on slot 0, save it, let slot 0 idle. @@ -184,6 +185,7 @@ def test_deferred_restore_is_bound_to_its_slot(): done0 = threading.Event() done1 = threading.Event() + t_done = {} def gen(id_slot, n_predict, done): try: @@ -194,10 +196,11 @@ def gen(id_slot, n_predict, done): }): pass finally: + t_done[id_slot] = time.time() done.set() # slot 0 finishes FIRST (256 tokens: long enough that a fast runner can observe it busy), slot 1 runs LONG. The returner wants slot 0. - t0 = threading.Thread(target=gen, args=(0, 256, done0), daemon=True) + t0 = threading.Thread(target=gen, args=(0, 128, done0), daemon=True) t1 = threading.Thread(target=gen, args=(1, 4096, done1), daemon=True) t1.start() # ensure slot 1 is processing before slot 0 starts, so the slot-1 op queues first @@ -237,15 +240,21 @@ def restore_slot1(): assert res.status_code == 200, f"slot-0 restore failed: {res.status_code} {res.body}" assert res.body["n_restored"] > 0 - # The binding proof: the slot-0 restore returned while slot 1 was STILL generating. - # Before the fix it starved until slot 1 finished (done1 set); with the bind it is - # serviced the moment its own slot (0) releases. - assert not done1.is_set(), \ - "slot-0 restore only returned after the long slot-1 gen ended — it was not bound to its slot" - assert elapsed < 20, f"slot-0 restore took {elapsed:.1f}s — starved behind the long slot" - + t_ret = time.time() t1.join(timeout=DEFAULT_HTTP_TIMEOUT) tr1.join(timeout=DEFAULT_HTTP_TIMEOUT) + # The binding proof, in TIMESTAMPS (2026-09-13: "returned while slot 1 was still + # generating" was a race against the runner's speed — a tiny model ends 4096 + # tokens in ~4 s on Metal and ~2 s on hosted runners). With the bind, the slot-0 + # restore is serviced the moment ITS slot releases: it returns within a beat of + # slot 0's own generation ending. Unbound, it waited for the LONG slot: it returns + # only after slot 1 ended. So: the restore's return sits next to done0, not done1. + after_own = t_ret - t_done[0] + gap = t_done[1] - t_done[0] + if gap < 1.0: + pytest.skip(f"runner too fast to separate the slots (slot 1 ended {gap:.2f}s after slot 0)") + assert after_own < gap / 2, \ + f"slot-0 restore returned {after_own:.2f}s after its own slot freed (slot 1 ended {gap:.2f}s later) — not bound to its slot" def test_slot_restore_legacy_token_list():