From b5c27ba233fbc9da81dcc7da42f57aa4fd6fbd31 Mon Sep 17 00:00:00 2001 From: W4-NERF Date: Thu, 20 Aug 2026 16:33:36 +0200 Subject: [PATCH] feat(dream): configurable dream.cycle_timeout (hot) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole dream cycle ran under a hardcoded CycleTimeout const (700s). Slow reasoning models (Qwen3.8-27B-NVFP4) run eval alone for ~600-690s on full 16-20-candidate prompts, so the enclosing cycle deadline cut the cycle before the recurrence/quality-score/cooldown steps ran — re-picking the same block every cycle. Make it a hot setting, mirroring dream.temporal_timeout (PR #23): - config.Dream.CycleTimeout `dream.cycle_timeout` / CTX_DREAM_CYCLE_TIMEOUT (default 700 = legacy constant, mut:hot, tenancy:global-only) - Router.CycleTimeout + newRouter wiring - dream.CycleTimeoutFor(r) helper: router value > 0 wins, else the package CycleTimeout constant (0 = documented "package default" sentinel) — the fallback the scheduler's outer cycle context and RunDreamCycle's inner context both read - validate.go V16b budget reads the effective (hot) cycle deadline via temporalTimeoutBudgetOf, so a raised cycle_timeout widens the window instead of warning spuriously Backward compatible: default 700 = the old const; 0/negative fall back to the constant. No migrations. Tests: TestCycleTimeoutFor (nil/empty/override/negative), updated TestValidateTemporalTimeoutBudget (default 400 + 2400->2100). Verified: go build ./... , go vet, go test -short on the 3 touched packages, golangci-lint v2 (repo config) = 0 issues. Docs: docs/operations.md env-table row + temporal_timeout budget clause. --- docs/operations.md | 3 +- go/internal/config/config.go | 11 ++++++ go/internal/config/validate.go | 50 ++++++++++++++++--------- go/internal/config/validate_test.go | 22 ++++++++--- go/internal/dream/cycle_timeout_test.go | 48 ++++++++++++++++++++++++ go/internal/dream/dream.go | 15 +++++++- go/internal/dream/router.go | 7 ++++ go/internal/events/scheduler.go | 3 +- 8 files changed, 132 insertions(+), 27 deletions(-) create mode 100644 go/internal/dream/cycle_timeout_test.go diff --git a/docs/operations.md b/docs/operations.md index 88a99305..274108cc 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -82,7 +82,8 @@ The **`mut` column** is the mutability class per key: **hot** keys take effect w | `CTX_DREAM_HOST` / `_PROTOCOL` / `_MODEL` / `_NUM_CTX` | inherits chat | hot | **Bootstrap-only since 053:** seeds the dream role — its own `herbert-dream` row when the host diverges from chat, else the `dream` role on `herbert-chat` — then inert. Separate Dream model (e.g. larger, slower) | | `CTX_DREAM_EMBED_*` | inherits embed | coupled | **Bootstrap-only since 053:** seeds the `dream-embed` role — merged onto `llama-embed` when identical to `CTX_EMBED_*`, else its own row. For a separate Dream embedding endpoint, create a pool row with role `dream-embed` | | `CTX_DREAM_IDLE_WAIT` | `20` (s) | hot | Backoff when no pending blocks | -| `CTX_DREAM_TEMPORAL_TIMEOUT` | `90` (s) | hot | Timeout for the dream-temporal Phase-2 LLM review. The legacy hardcoded `ValidateTimeout` (90s) is tight for slow reasoning models (e.g. nemotron-super-trt needs >90s on full prompts); raise via this key. **Precedence:** the key is only the *default* for that one call — the Phase-2 call runs under role `dream`, so a `timeouts.dream` entry on the serving `context_backends` row wins (`TimeoutFor`, walked in `llm/chain.go`). On such a row raise the row value instead; it bounds dream eval, keywords and recurrence too, whereas this key is temporal-only. **On timeout** the deterministic Phase-1 dimensions stay intact — they are written before the LLM call — but the LLM-found *additional* explicit dates are dropped for that block version, and since `dream_temporal_validated_at` is stamped even after the failure they are not retried until the block changes (non-fatal). `0` = the package default (90s); a negative value is fatal at boot / 422s the settings write (V16). **Budget:** the whole cycle runs under a `700`s deadline and temporal is step 1b, so above `400`s (= 700 − keywords 120 − eval 180) the link-writing stages get squeezed and above `700`s the value cannot take effect at all — both WARN (V16b), not clamped | +| `CTX_DREAM_TEMPORAL_TIMEOUT` | `90` (s) | hot | Timeout for the dream-temporal Phase-2 LLM review. The legacy hardcoded `ValidateTimeout` (90s) is tight for slow reasoning models (e.g. nemotron-super-trt needs >90s on full prompts); raise via this key. **Precedence:** the key is only the *default* for that one call — the Phase-2 call runs under role `dream`, so a `timeouts.dream` entry on the serving `context_backends` row wins (`TimeoutFor`, walked in `llm/chain.go`). On such a row raise the row value instead; it bounds dream eval, keywords and recurrence too, whereas this key is temporal-only. **On timeout** the deterministic Phase-1 dimensions stay intact — they are written before the LLM call — but the LLM-found *additional* explicit dates are dropped for that block version, and since `dream_temporal_validated_at` is stamped even after the failure they are not retried until the block changes (non-fatal). `0` = the package default (90s); a negative value is fatal at boot / 422s the settings write (V16). **Budget:** the whole cycle runs under a `700`s deadline by default and temporal is step 1b, so above `400`s (= 700 − keywords 120 − eval 180) the link-writing stages get squeezed and above `700`s the value cannot take effect at all — both WARN (V16b), not clamped. **Both numbers track `CTX_DREAM_CYCLE_TIMEOUT`**: raising the cycle widens the budget (`400 → 700 − keywords − eval`), so a temporal value that would WARN at the `700`s default stops warning once the cycle is raised — the V16b check reads the effective cycle deadline, not the constant | +| `CTX_DREAM_CYCLE_TIMEOUT` | `700` (s) | hot | Timeout for the **whole** dream cycle — the single `context.WithTimeout` in `RunDreamCycle` (and the scheduler's outer cycle context) that wraps pick → temporal → keywords → RRF → eval → recurrence. The legacy hardcoded `CycleTimeout` (700s) is tight for slow reasoning models (e.g. Qwen3.8-27B-NVFP4 needs >700s on full 16–20-candidate prompts: the eval call alone can run ~600–690s, leaving the recurrence/temporal/keywords steps to time out inside the cycle). **Precedence:** the key is the *default* for the enclosing cycle — a `timeouts.dream` entry on the serving `context_backends` row wins *per call* (`TimeoutFor`, walked in `llm/chain.go`); on such a row raise the row value instead, which bounds eval/keywords/recurrence per call, whereas this key bounds the whole cycle. **On timeout** the cycle is cut at its deadline (non-fatal — the cycle completes what it started, but the remaining steps — e.g. `update quality score`, `set cooldown` — are skipped, so the block can re-pick next cycle). `0` = the package default (700s); a negative value is fatal at boot / 422s the settings write (V16, same class as `temporal_timeout`). **Budget:** this deadline *is* the V16b budget anchor — a `temporal_timeout` above `400`s (= cycle − keywords 120 − eval 180) squeezes the link-writing stages, and `temporal_timeout ≥ cycle` cannot take effect; both WARN (V16b), not clamped. Raising this key widens that window without a rebuild | | `CTX_DREAM_LINK_FLOOR_CONFIDENCE` | `0.9` | hot | Raw confidence assigned to links the dream LLM names **without a strength signal** (string-map drift form, absent `confidence` fields — PR #12). The default `0.9` keeps such links above the RRF graph-expansion gate (`graph.min_confidence` `0.75`), i.e. type-only answers still produce retrieval-live edges. Set `0.7` for the conservative PR-#12 semantics (edges persist + show in the ego graph, but stay out of RRF expansion until re-classified), or any `[0,1]` float. Per-type `minRawConfidence` write gates stay the lower bound (`recurrent` never drops below `0.8`). Out-of-range is fatal at boot (V15) | | `CTX_DREAM_LANGUAGE` | empty | hot | Language of the **daily synthesis report** only (the 03:00 iteration and `POST /api/synthesize/daily`; the per-block dream pipeline is unaffected). Empty = legacy behavior: German report, title `Tagesbericht `, tag `tagesbericht`. A BCP-47-style tag switches title (`Daily Report `), tag (`daily-report`) and system prompt together; the **primary subtag** decides, so `de-DE` stays German. ⚠ The title is half the `(category, title, scope)` upsert key — changing this **starts a new report series**, it does not rename the old one (see [api](api.md#endpoints)). Validated (V14): `^[a-z]{2,3}(-[a-z0-9]{2,8})*$`, ≤ 35 chars — a malformed value aborts boot / 422s the settings write, because the tag is interpolated into an LLM system prompt | | `CTX_DREAM_BACKOFF_MODE` / `_FACTOR` / `_MIN` / `_GRACE` / `_CAP` / `_INERT_OFFSET` | `exp` / `1.6` / `12h` / `0` / `45d` / `7` | hot | Re-dream back-off by eval count (`exp`/`log`/`linear`/`off`). Cooldown grows from `MIN` (n=0) to `CAP`: fresh blocks re-dream sub-day, mature blocks back off to the cap. `_MIN`/`_CAP` take a duration suffix — `h`/`d`/`w`/`m` (30d)/`y` (365d), e.g. `12h`, `45d`, `1w` (bare number = hours). `_INERT_OFFSET` starts a no-links cycle further up the curve | diff --git a/go/internal/config/config.go b/go/internal/config/config.go index 7d659ec9..283a509c 100644 --- a/go/internal/config/config.go +++ b/go/internal/config/config.go @@ -220,6 +220,17 @@ type DreamConfig struct { // value instead — it bounds dream eval/keywords/recurrence too, while this // key is temporal-only. TemporalTimeout time.Duration `key:"dream.temporal_timeout" env:"CTX_DREAM_TEMPORAL_TIMEOUT" default:"90" mut:"hot" tenancy:"global-only"` + // CycleTimeout bounds the WHOLE dream cycle (seconds) — the single + // context.WithTimeout in RunDreamCycle that wraps pick → temporal → + // keywords → RRF → eval → recurrence. Default 700 matches the legacy + // package CycleTimeout constant; raise for slow reasoning models + // (Qwen3.8-27B-NVFP4 needs >700s on full 16-20-candidate prompts). + // DEFAULT ONLY: a timeouts.dream entry on the serving context_backends + // row takes precedence per call (Backend.TimeoutFor, walked in + // llm.ChatChainVia); raise that row value instead on a configured row — + // it bounds eval/keywords/recurrence per call, while this key bounds + // the enclosing cycle. + CycleTimeout time.Duration `key:"dream.cycle_timeout" env:"CTX_DREAM_CYCLE_TIMEOUT" default:"700" mut:"hot" tenancy:"global-only"` Backoff BackoffConfig } diff --git a/go/internal/config/validate.go b/go/internal/config/validate.go index 9c047e48..3293e9c5 100644 --- a/go/internal/config/validate.go +++ b/go/internal/config/validate.go @@ -5,6 +5,7 @@ import ( "net/url" "regexp" "strings" + "time" "github.com/GottZ/ctx/internal/backends" "github.com/GottZ/ctx/internal/dream" @@ -30,13 +31,19 @@ const ( var dreamLanguageRe = regexp.MustCompile(dreamLanguagePattern) -// temporalTimeoutBudget is the largest dream.temporal_timeout that still +// temporalTimeoutBudgetOf is the largest dream.temporal_timeout that still // leaves the two LLM stages behind Phase-2 temporal their own ceilings inside -// one dream cycle (V16b): the whole cycle runs under dream.CycleTimeout, and -// temporal is step 1b — keyword extraction (KeywordsTimeout) and relationship -// evaluation (DreamTimeout) follow it and write the links. Derived from the -// dream constants, not mirrored, so it cannot drift when they are retuned. -const temporalTimeoutBudget = dream.CycleTimeout - (dream.KeywordsTimeout + dream.DreamTimeout) +// a cycle of the given whole-cycle deadline (V16b): the whole cycle runs +// under CycleTimeoutFor, and temporal is step 1b — keyword extraction +// (KeywordsTimeout) and relationship evaluation (DreamTimeout) follow it and +// write the links. Derived from the cycle deadline, not mirrored, so it +// cannot drift when the timeouts are retuned. It reads the effective (hot) +// cycle timeout, falling back to the package constant, so a configured +// dream.cycle_timeout widens the budget accordingly instead of warning +// spuriously. +func temporalTimeoutBudgetOf(c *Config) time.Duration { + return dream.CycleTimeoutFor(&dream.Router{CycleTimeout: c.Dream.CycleTimeout}) - (dream.KeywordsTimeout + dream.DreamTimeout) +} // Validate checks the cross-field invariants V1–V14 and returns all findings. // WARN classes with "today's silent fallback" semantics (V5 prompt version, @@ -349,19 +356,26 @@ func validateDream(c *Config) []Issue { if d := c.Dream.TemporalTimeout; d < 0 { issues = append(issues, Issue{Field: "dream.temporal_timeout", Severity: SeverityError, Msg: fmt.Sprintf("temporal timeout %v must be >= 0 (0 = package default %v)", d, dream.ValidateTimeout)}) - } else if d > temporalTimeoutBudget { - // V16b — the cycle-budget WARN. Not a clamp: the operator may know - // their keyword/eval calls finish far inside their own ceilings, and - // the runtime already fails safely (the cycle deadline cuts the call). - // Warn only, in the V10 spirit of making a downstream truncation - // visible at boot. - msg := fmt.Sprintf("temporal timeout %v leaves only %v of the %v dream cycle for keywords (%v) + eval (%v) — the link-writing stages can be starved", - d, dream.CycleTimeout-d, dream.CycleTimeout, dream.KeywordsTimeout, dream.DreamTimeout) - if d >= dream.CycleTimeout { - msg = fmt.Sprintf("temporal timeout %v is not below the %v dream cycle budget — the cycle deadline cuts the Phase-2 call first, so the value cannot take effect", - d, dream.CycleTimeout) + } else { + // Effective whole-cycle deadline: the hot dream.cycle_timeout wins + // (CycleTimeoutFor), else the package CycleTimeout default. The + // budget and the "cannot take effect" gate read it, so a raised + // cycle timeout widens the window instead of warning spuriously. + cycle := dream.CycleTimeoutFor(&dream.Router{CycleTimeout: c.Dream.CycleTimeout}) + if d > temporalTimeoutBudgetOf(c) { + // V16b — the cycle-budget WARN. Not a clamp: the operator may + // know their keyword/eval calls finish far inside their own + // ceilings, and the runtime already fails safely (the cycle + // deadline cuts the call). Warn only, in the V10 spirit of + // making a downstream truncation visible at boot. + msg := fmt.Sprintf("temporal timeout %v leaves only %v of the %v dream cycle for keywords (%v) + eval (%v) — the link-writing stages can be starved", + d, cycle-d, cycle, dream.KeywordsTimeout, dream.DreamTimeout) + if d >= cycle { + msg = fmt.Sprintf("temporal timeout %v is not below the %v dream cycle budget — the cycle deadline cuts the Phase-2 call first, so the value cannot take effect", + d, cycle) + } + issues = append(issues, Issue{Field: "dream.temporal_timeout", Severity: SeverityWarn, Msg: msg}) } - issues = append(issues, Issue{Field: "dream.temporal_timeout", Severity: SeverityWarn, Msg: msg}) } return issues diff --git a/go/internal/config/validate_test.go b/go/internal/config/validate_test.go index f1d58070..9f68dc8f 100644 --- a/go/internal/config/validate_test.go +++ b/go/internal/config/validate_test.go @@ -233,15 +233,25 @@ func TestValidateLanguageDefaultIsLegacy(t *testing.T) { } // TestValidateTemporalTimeoutBudget pins the derived V16b threshold against -// the number the operations docs name. The constant is computed from the -// dream constants, so a retune there moves it silently — this test is where -// the move becomes visible and the docs clause gets corrected. +// the number the operations docs name. The budget is computed from the +// effective (hot) cycle deadline, so a retune there moves it silently — this +// test is where the move becomes visible and the docs clause gets corrected. func TestValidateTemporalTimeoutBudget(t *testing.T) { - if temporalTimeoutBudget != 400*time.Second { - t.Errorf("temporal timeout budget = %v, want 400s (docs/operations.md names it)", temporalTimeoutBudget) + // Default (unset) cycle: package CycleTimeout 700 → budget 400. + if got := temporalTimeoutBudgetOf(&Config{Dream: DreamConfig{CycleTimeout: 0}}); got != 400*time.Second { + t.Errorf("temporal timeout budget (default cycle) = %v, want 400s (docs/operations.md names it)", got) } + // The package constant is the fallback default; a configured + // dream.cycle_timeout must not change it. if dream.CycleTimeout != 700*time.Second { - t.Errorf("dream cycle timeout = %v, want 700s (docs/operations.md names it)", dream.CycleTimeout) + t.Errorf("dream.CycleTimeout = %v, want 700s (the package default)", dream.CycleTimeout) + } + // A raised cycle timeout widens the budget: 2400 − keywords 120 − eval + // 180 = 2100, so a temporal_timeout that used to WARN (V16b) no longer + // does once the cycle is raised — the whole point of making the cycle + // configurable. + if got := temporalTimeoutBudgetOf(&Config{Dream: DreamConfig{CycleTimeout: 2400 * time.Second}}); got != 2100*time.Second { + t.Errorf("temporal timeout budget (2400s cycle) = %v, want 2100s", got) } } diff --git a/go/internal/dream/cycle_timeout_test.go b/go/internal/dream/cycle_timeout_test.go new file mode 100644 index 00000000..5950efc2 --- /dev/null +++ b/go/internal/dream/cycle_timeout_test.go @@ -0,0 +1,48 @@ +package dream + +import ( + "testing" + "time" +) + +// TestCycleTimeoutFor mirrors TestTemporalTimeout: the whole-cycle deadline +// resolves the router value when set, else the package CycleTimeout default. +// 0 (and a negative value that a hand-built router could carry) is the +// documented "package default" sentinel — config V16 rejects a negative value +// at boot and at the settings write, so the fallback is the second line of +// defence for hand-built routers, exactly as temporalTimeout's contract. +func TestCycleTimeoutFor(t *testing.T) { + tests := []struct { + name string + router *Router + expected time.Duration + }{ + { + name: "nil router falls back to package default", + router: nil, + expected: CycleTimeout, + }, + { + name: "router without timeout falls back to package default", + router: &Router{}, + expected: CycleTimeout, + }, + { + name: "router setting wins", + router: &Router{CycleTimeout: 2400 * time.Second}, + expected: 2400 * time.Second, + }, + { + name: "negative router value falls back to package default", + router: &Router{CycleTimeout: -30 * time.Second}, + expected: CycleTimeout, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CycleTimeoutFor(tt.router); got != tt.expected { + t.Errorf("CycleTimeoutFor() = %v, want %v", got, tt.expected) + } + }) + } +} diff --git a/go/internal/dream/dream.go b/go/internal/dream/dream.go index b8e67151..145dddb2 100644 --- a/go/internal/dream/dream.go +++ b/go/internal/dream/dream.go @@ -113,6 +113,19 @@ type BlockInfo struct { // Must exceed DreamTimeout (evaluate call) + keyword-embed + RRF overhead. const CycleTimeout = 700 * time.Second +// CycleTimeoutFor resolves the whole-cycle deadline: a router value > 0 wins +// (config.Dream.CycleTimeout, hot), otherwise the package CycleTimeout +// constant (legacy behavior). The enclosing context.WithTimeout in +// RunDreamCycle and the scheduler's outer cycle context both read this, so +// a single knob bounds the whole cycle; 0 is the documented "package +// default" sentinel, mirroring temporalTimeout's contract. +func CycleTimeoutFor(r *Router) time.Duration { + if r != nil && r.CycleTimeout > 0 { + return r.CycleTimeout + } + return CycleTimeout +} + // Throttle is called between GPU-intensive steps to allow cooldown. // Returns an error if the context was cancelled during the wait. type Throttle func(ctx context.Context) error @@ -143,7 +156,7 @@ func RunDreamCycle(ctx context.Context, pool *pgxpool.Pool, r *Router, opts llm. return 0, nil } - ctx, cancel := context.WithTimeout(ctx, CycleTimeout) + ctx, cancel := context.WithTimeout(ctx, CycleTimeoutFor(r)) defer cancel() // ONE policy snapshot per cycle (WF T8, blocktype doctrine): pick diff --git a/go/internal/dream/router.go b/go/internal/dream/router.go index bf074629..6a3166b5 100644 --- a/go/internal/dream/router.go +++ b/go/internal/dream/router.go @@ -55,6 +55,13 @@ type Router struct { // timeouts.dream entry on the serving row wins (Backend.TimeoutFor in the // llm.ChatChainVia walk), because the call resolves under role dream. TemporalTimeout time.Duration + // CycleTimeout bounds the WHOLE dream cycle (seconds) — the enclosing + // context.WithTimeout in RunDreamCycle. 0 = the package CycleTimeout + // default (legacy behavior). Set from config.Dream.CycleTimeout by the + // scheduler (newRouter); resolved via CycleTimeoutFor, which falls back + // to the constant when unset so a missing value never changes the + // cycle's deadline. + CycleTimeout time.Duration // Language is the daily-synthesis report language, read from config // Dream.Language by the caller that builds the router (scheduler: // per-iteration; synthesize handler: per-request — so the hot key is diff --git a/go/internal/events/scheduler.go b/go/internal/events/scheduler.go index fb95b3d8..67ff7965 100644 --- a/go/internal/events/scheduler.go +++ b/go/internal/events/scheduler.go @@ -613,6 +613,7 @@ func (s *Scheduler) newRouter(cfg *config.Config, tenant string) *dream.Router { Language: cfg.Dream.Language, LinkFloor: cfg.Dream.LinkFloorConfidence, TemporalTimeout: cfg.Dream.TemporalTimeout, + CycleTimeout: cfg.Dream.CycleTimeout, } } @@ -1839,7 +1840,7 @@ func (s *Scheduler) runDreamCycle(cfg *config.Config, router *dream.Router, read // Dream gets its own context with the cycle timeout, independent of parent ctx. // Register the cancel fn so SetDreamMode(Off) can abort in-flight work. - dreamCtx, cancel := context.WithTimeout(context.Background(), dream.CycleTimeout) + dreamCtx, cancel := context.WithTimeout(context.Background(), dream.CycleTimeoutFor(&dream.Router{CycleTimeout: cfg.Dream.CycleTimeout})) s.dreamCycleMu.Lock() s.dreamCycleCancel = cancel s.dreamCycleMu.Unlock()