diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml new file mode 100644 index 0000000..c353b65 --- /dev/null +++ b/.github/workflows/cli.yml @@ -0,0 +1,38 @@ +name: cli + +on: + pull_request: + paths: ["cli/**", "serve/**", "fold/**", "Cargo.toml", ".github/workflows/cli.yml"] + push: + branches: [main] + paths: ["cli/**", "serve/**", "fold/**", "Cargo.toml", ".github/workflows/cli.yml"] + +concurrency: + group: cli-${{ github.ref }} + cancel-in-progress: true + +jobs: + scaffold: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: Swatinem/rust-cache@v2 + + - name: Build the CLI + run: cargo build -p bogkit + + - name: bog-serve tests + run: cargo test -p bog-serve + + - name: Golden OpenAPI docs validate against the spec + run: pipx run openapi-spec-validator serve/tests/golden/template.json serve/tests/golden/search.json + + - name: Scaffold an embedded project + run: cargo run -p bogkit -- new ci-embedded --kind embedded + + - name: Scaffold a server project + run: cargo run -p bogkit -- new ci-server --kind server + + - name: Scaffolded projects compile + run: cargo check -p ci-embedded -p ci-server diff --git a/Cargo.lock b/Cargo.lock index f152980..33eb4a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -354,6 +354,34 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bog-serve" +version = "0.0.0" +dependencies = [ + "anny", + "axum", + "fold", + "http-body-util", + "schemars", + "serde", + "serde_json", + "serde_urlencoded", + "tempfile", + "tokio", + "tokio-stream", + "tower", +] + +[[package]] +name = "bogkit" +version = "0.0.0" +dependencies = [ + "anyhow", + "clap", + "serde_json", + "ureq", +] + [[package]] name = "brotli" version = "8.0.4" @@ -934,6 +962,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.16.0" @@ -2479,6 +2513,26 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "regex" version = "1.13.0" @@ -2625,6 +2679,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.3", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2640,6 +2719,19 @@ dependencies = [ "fold", ] +[[package]] +name = "search-server" +version = "0.0.0" +dependencies = [ + "anny", + "bog-serve", + "ese", + "fold", + "schemars", + "serde", + "serde_json", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -2711,6 +2803,17 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "serde_json" version = "1.0.150" @@ -3155,6 +3258,18 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "tokio-tungstenite" version = "0.29.0" @@ -3167,6 +3282,19 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "tower" version = "0.5.3" diff --git a/Cargo.toml b/Cargo.toml index 6415c3f..556a4b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["fold", "anny", "ese", "examples/*"] +members = ["fold", "anny", "ese", "cli", "serve", "examples/*"] # hoisted from fold; applies workspace-wide [profile.release] diff --git a/README.md b/README.md index 951a620..28cd25c 100644 --- a/README.md +++ b/README.md @@ -2,18 +2,29 @@ This repo contains some of the tooling we've been working on for building Bog style databases. We've collected these tools and examples in one cargo workspace, so you can start building immediately. -The best way to create your project is to run this terminal command in the root of this repo: +The best way to create your project is the `bogkit` CLI, from the root of this repo: ```console -$ ./scripts/new-project.sh [project-name] -``` +$ cargo run -p bogkit -- new [project-name] +``` + +This creates a new binary crate in `examples/[project-name]` with local path dependencies on the workspace crates. Two flavors: -This creates a new binary crate in `examples/[project-name]`, wires it into the workspace, and adds local path dependencies on `fold`, `anny`, and `ese` (though you may not necessarily use all of these). +- `--kind server` (the default) — a fold pipeline served over HTTP by `bog-serve`: writes, reads, search, and a live OpenAPI doc, all generated from the pipeline itself. +- `--kind embedded` — a plain Rust binary using fold directly (what `scripts/new-project.sh` used to produce; the script still works). Run your project with: ```console -$ cargo run -p [project-name] +$ cargo run -p bogkit -- dev -p [project-name] +``` + +`dev` wraps `cargo run` with the bogkit conventions: a stable data dir in `~/.bogkit/data/[project-name]` (pass `--fresh` to wipe it) and `$PORT` (default 7877). For server projects, explore the API with: + +```console +$ cargo run -p bogkit -- api # pretty-prints the running server's /openapi.json +$ curl localhost:7877/views/total +$ curl -N localhost:7877/watch # server-sent events, one per commit ``` ## Documentation @@ -45,6 +56,12 @@ ESE, our first take on a compiler oriented approach to static embedding. It’s ### Approximate Nearest Neighbors... yeah (ANNy) This is a very fast crate for creating HNSWs. +### bog-serve +Serve any fold pipeline over HTTP with the API generated from the pipeline itself: the input type describes the write routes (via schemars), the named terminal sinks describe the read routes, and the OpenAPI doc is assembled from the same values the router dispatches with — so it can't drift. Atomic batches, hybrid search, SSE watch streams, and custom routes included. See `serve/` and the crate rustdocs (`cargo doc --open -p bog-serve`). + +### bogkit CLI +Scaffolding and a dev runner for BogKit projects (`cli/`): `bogkit new`, `bogkit dev`, `bogkit api`. + ### Examples In this directory you'll find a few examples that show bog style databases in various use cases. @@ -52,6 +69,7 @@ In this directory you'll find a few examples that show bog style databases in va - `timeseries` — weather readings bucketed into hourly and daily aggregates, updated incrementally. `cargo run -p timeseries` - `chat` — a chat backend where fold is the source of truth and every update is broadcast to clients over a websocket. `cargo run -p chat`, then open http://localhost:3000 - `search` — text search three ways over one document stream: BM25 keyword search, HNSW semantic search over ese embeddings, and hybrid rank fusion. A good base for agent memory or document search projects. `cargo run -p search` +- `search-server` — the search example served over HTTP by bog-serve: the same pipeline plus generated CRUD/search routes, a custom `/search/hybrid` fusion endpoint, and a live OpenAPI doc. `cargo run -p search-server`, then `curl localhost:7877/openapi.json` ## More about Bog Bog is a database runtime that makes every attempt to do as much work as possible as early as possible, to make reads incredibly fast. This means compiling queries into functions that eagerly update their output as mutations occur. diff --git a/cli/Cargo.toml b/cli/Cargo.toml new file mode 100644 index 0000000..a2296dc --- /dev/null +++ b/cli/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "bogkit" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +anyhow = "1" +clap = { version = "4", features = ["derive"] } +serde_json = "1" +ureq = { version = "2", features = ["json"] } diff --git a/cli/src/api.rs b/cli/src/api.rs new file mode 100644 index 0000000..6111821 --- /dev/null +++ b/cli/src/api.rs @@ -0,0 +1,19 @@ +use anyhow::Context; + +/// Fetch and pretty-print a running server's OpenAPI document. +pub fn run(port: Option) -> anyhow::Result<()> { + // same resolution order as the server itself: flag, $PORT, default + let port = port + .or_else(|| std::env::var("PORT").ok().and_then(|p| p.parse().ok())) + .unwrap_or(7877); + let url = format!("http://localhost:{port}/openapi.json"); + + let doc: serde_json::Value = ureq::get(&url) + .call() + .with_context(|| format!("fetching {url} — is the server running? (`bogkit dev`)"))? + .into_json() + .context("parsing the OpenAPI document")?; + + println!("{}", serde_json::to_string_pretty(&doc)?); + Ok(()) +} diff --git a/cli/src/dev.rs b/cli/src/dev.rs new file mode 100644 index 0000000..d05deff --- /dev/null +++ b/cli/src/dev.rs @@ -0,0 +1,55 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, bail}; + +use crate::workspace; + +pub fn run(project: Option<&str>, fresh: bool) -> anyhow::Result<()> { + let root = workspace::root()?; + let name = match project { + Some(name) => name.to_string(), + None => infer_project(&root)?, + }; + + let data_dir = data_dir(&name)?; + if fresh && data_dir.exists() { + std::fs::remove_dir_all(&data_dir).context("wiping data dir for --fresh")?; + } + std::fs::create_dir_all(&data_dir).context("creating data dir")?; + println!("data dir: {}", data_dir.display()); + + // Inherit stdio so the project owns the terminal (the search example's + // interactive loop, server logs, ...). We only collect the exit status. + let status = Command::new("cargo") + .args(["run", "-p", &name]) + .current_dir(&root) + .env("BOG_DATA_DIR", &data_dir) + .env("PORT", std::env::var("PORT").unwrap_or_else(|_| "7877".into())) + .status() + .context("running cargo")?; + + if !status.success() { + // mirror the project's exit code so `bogkit dev` composes in scripts + std::process::exit(status.code().unwrap_or(1)); + } + Ok(()) +} + +/// When run from inside examples/, that project is the target — +/// scaffolded crates are named after their directory. +fn infer_project(root: &Path) -> anyhow::Result { + let cwd = std::env::current_dir()?; + let rel = cwd.strip_prefix(root.join("examples")).ok(); + match rel.and_then(|rel| rel.iter().next()) { + Some(first) => Ok(first.to_string_lossy().into_owned()), + None => bail!("run from inside examples/ or pass -p "), + } +} + +/// Stable per-project data dir: ~/.bogkit/data/. Persistent across +/// runs by default; `--fresh` wipes it. +fn data_dir(name: &str) -> anyhow::Result { + let home = std::env::home_dir().context("cannot determine home directory")?; + Ok(home.join(".bogkit").join("data").join(name)) +} diff --git a/cli/src/main.rs b/cli/src/main.rs new file mode 100644 index 0000000..2876714 --- /dev/null +++ b/cli/src/main.rs @@ -0,0 +1,67 @@ +//! The bogkit CLI: scaffold and run BogKit projects. +//! +//! Phase 0 (see docs/bog-cli-plan.md): `bogkit new --kind embedded` matches +//! scripts/new-project.sh, and `bogkit dev` wraps `cargo run` with the +//! data-dir conventions the server flavor will rely on in phase 1. + +mod api; +mod dev; +mod new; +mod workspace; + +use clap::{Parser, Subcommand, ValueEnum}; + +#[derive(Parser)] +#[command(name = "bogkit", version, about = "Scaffold and run BogKit projects")] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Create a new project in examples/ + New { + /// Project name (also the crate and directory name) + name: String, + + /// Which flavor of project to create + #[arg(long, value_enum, default_value_t = Kind::Server)] + kind: Kind, + }, + + /// Run a project with bogkit conventions ($BOG_DATA_DIR, $PORT) + Dev { + /// Project to run; inferred when run from inside examples/ + #[arg(short, long)] + project: Option, + + /// Wipe the project's data dir before running + #[arg(long)] + fresh: bool, + }, + + /// Fetch and pretty-print the running server's OpenAPI document + Api { + /// Port the server listens on (default: $PORT, then 7877) + #[arg(short, long)] + port: Option, + }, +} + +#[derive(Clone, Copy, ValueEnum)] +pub enum Kind { + /// A plain Rust binary using fold directly + Embedded, + /// A fold program served over HTTP by bog-serve + Server, +} + +fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + match cli.command { + Command::New { name, kind } => new::run(&name, kind), + Command::Dev { project, fresh } => dev::run(project.as_deref(), fresh), + Command::Api { port } => api::run(port), + } +} diff --git a/cli/src/new.rs b/cli/src/new.rs new file mode 100644 index 0000000..42f487c --- /dev/null +++ b/cli/src/new.rs @@ -0,0 +1,117 @@ +use std::fs; + +use anyhow::{Context, bail}; + +use crate::{Kind, workspace}; + +// Templates are embedded in the binary (not read from the repo) so `bogkit +// new` keeps working once the CLI is installed standalone. `{name}` is +// substituted with plain str::replace — these aren't format! strings. + +const MANIFEST_EMBEDDED: &str = r#"[package] +name = "{name}" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +anny = { path = "../../anny" } +ese = { path = "../../ese", features = ["dim-512", "quant-8"] } +fold = { path = "../../fold" } +serde = { version = "1", features = ["derive"] } +"#; + +const MAIN_EMBEDDED: &str = r#"fn main() { + println!("welcome to bog kit. start hacking in examples/{name}/src/main.rs"); +} +"#; + +const MANIFEST_SERVER: &str = r#"[package] +name = "{name}" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +bog-serve = { path = "../../serve" } +fold = { path = "../../fold" } +schemars = "1" +serde = { version = "1", features = ["derive"] } +"#; + +const MAIN_SERVER: &str = r#"//! A fold database served over HTTP. +//! +//! The pipeline below fans every inserted `Entry` out to two views; the +//! server generates the whole API from it. Once running, explore with: +//! +//! curl localhost:7877/openapi.json +//! curl -X POST localhost:7877/insert \ +//! -H 'content-type: application/json' -d '{"text": "hello"}' +//! curl localhost:7877/views/total +//! +//! Add fields to `Entry` or sinks to the pipeline, and the API (and its +//! OpenAPI doc) follow automatically. + +use bog_serve::App; +use fold::pipeline::terminal; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// What `POST /insert` accepts. `JsonSchema` puts its shape in the doc. +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +struct Entry { + text: String, +} + +fn main() { + App::stream( + // $BOG_DATA_DIR when run via `bogkit dev`, ./bog.db otherwise + bog_serve::data_dir(), + ( + terminal::Count::new("total"), + terminal::Bag::::new("entries"), + ), + ) + .run() +} +"#; + +pub fn run(name: &str, kind: Kind) -> anyhow::Result<()> { + validate_name(name)?; + + let project_dir = workspace::root()?.join("examples").join(name); + if project_dir.exists() { + bail!("{} already exists", project_dir.display()); + } + + let (manifest, main_rs) = match kind { + Kind::Embedded => (MANIFEST_EMBEDDED, MAIN_EMBEDDED), + Kind::Server => (MANIFEST_SERVER, MAIN_SERVER), + }; + + fs::create_dir_all(project_dir.join("src")).context("creating project directories")?; + fs::write(project_dir.join("Cargo.toml"), manifest.replace("{name}", name))?; + fs::write(project_dir.join("src/main.rs"), main_rs.replace("{name}", name))?; + + println!("created {}", project_dir.display()); + println!("run it with: bogkit dev -p {name}"); + if let Kind::Server = kind { + println!("then explore: curl localhost:7877/openapi.json"); + } + Ok(()) +} + +/// Same rules as scripts/new-project.sh: start with a letter or number, +/// then letters, numbers, '-' or '_'. +fn validate_name(name: &str) -> anyhow::Result<()> { + let mut chars = name.chars(); + let first_ok = chars.next().is_some_and(|c| c.is_ascii_alphanumeric()); + let rest_ok = chars.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'); + if !(first_ok && rest_ok) { + bail!( + "project name must start with a letter or number and contain only \ + letters, numbers, '-' or '_'" + ); + } + Ok(()) +} diff --git a/cli/src/workspace.rs b/cli/src/workspace.rs new file mode 100644 index 0000000..ae98972 --- /dev/null +++ b/cli/src/workspace.rs @@ -0,0 +1,25 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, bail}; + +/// Walk up from the current directory to the bog-kit workspace root: the +/// nearest ancestor whose Cargo.toml declares `[workspace]`. +pub fn root() -> anyhow::Result { + let cwd = std::env::current_dir().context("cannot read current directory")?; + for dir in cwd.ancestors() { + if is_workspace_root(dir) { + return Ok(dir.to_path_buf()); + } + } + bail!( + "not inside the bog-kit workspace — standalone projects arrive once \ + fold is published to crates.io" + ); +} + +fn is_workspace_root(dir: &Path) -> bool { + match std::fs::read_to_string(dir.join("Cargo.toml")) { + Ok(manifest) => manifest.lines().any(|l| l.trim() == "[workspace]"), + Err(_) => false, + } +} diff --git a/docs/bog-cli-plan.md b/docs/bog-cli-plan.md new file mode 100644 index 0000000..46257ac --- /dev/null +++ b/docs/bog-cli-plan.md @@ -0,0 +1,255 @@ +# BogKit CLI + local dev server — development plan + +> **Status (Aug 2026):** phases 0–3 are built and tested, plus typed custom +> routes (`.get`/`.post` with schemas captured at registration; POST runs +> in fold's `try_wtx`, so `Err` rolls the transaction back). Deferred: +> the crates.io dual-mode switch (blocked on fold publishing), +> `KeyedRanked` view coverage, persistent commit seqs (waiting on the +> delta log), and `Last-Event-ID` resume (unnecessary under /watch's +> level semantics — documented instead). + +Goal: a `bogkit` CLI that scaffolds and runs BogKit projects locally, in two +flavors that both stay first-class. (The name `bog` is reserved for a future +tool that is an actual compiler.) + +- **embedded** — a plain Rust binary using fold directly (what + `scripts/new-project.sh` produces today) +- **server** — a fold program served over HTTP by a new `bog-serve` crate, + with an auto-generated API and OpenAPI doc + +The server flavor also lifts a hard embedded limitation: fjall's single-writer +database means exactly one process can open a fold store, so the serve process +becomes the shared access point — any number of clients, across devices and +sandboxes, each reading a consistent snapshot. `rtx` already takes `&self` +(pinned snapshot per call) while `wtx` takes `&mut self`, so reads can run +genuinely in parallel behind an RwLock; writes stay serialized, which is +fold's model anyway. + +Cloud deployment is explicitly out of scope for this plan. Everything here +must be independently useful with zero cloud behind it: `bogkit new` + +`bogkit dev` should give any bogkit project an instant local HTTP API. The cloud story +later hangs off the same contract (`/openapi.json`, `/healthz`, `/schema`, +`$PORT`, `$BOG_DATA_DIR`), so nothing here is throwaway. + +## New workspace members + +``` +cli/ binary crate `bogkit` — scaffolding + dev runner +serve/ library crate `bog-serve` — the HTTP layer over fold +``` + +Templates live inside the CLI binary (`include_str!`-style embedding, not +files copied from the repo), so `bogkit new` works from any directory once +the CLI itself is installed — required for the post-crates.io world. + +## Command surface (v0) + +``` +bogkit new [--kind server|embedded] scaffold a project (default: server) +bogkit dev [-p ] run a project locally +bogkit api [-p ] fetch & pretty-print /openapi.json from the running server +``` + +- `bogkit new`, run inside this workspace, keeps today's behavior: creates + `examples/` with path deps (`examples/*` is already a workspace + member, so no manifest edits needed). Run elsewhere, it creates a + standalone cargo project with crates.io deps — gated until fold/anny/ese + publish (see "crates.io transition"). +- `bogkit dev` is `cargo run` plus conventions: picks a stable per-project + data dir (`~/.bogkit/data/` — persistent by default, `--fresh` to wipe), + sets `$BOG_DATA_DIR`/`$PORT`, and for server projects prints the URL and + route table on startup. +- `scripts/new-project.sh` stays until the CLI reaches parity, then becomes + a one-line shim calling `bogkit new --kind embedded` for a release or two. + +Name validation, collision checks, and the `--kind embedded` template body +are ports of the existing script — same rules, same output. + +## `bog-serve` architecture + +The API surface of any fold program is exactly (input type at the front, +named terminal sinks at the back); the closures in between never appear in +the API. That makes generation tractable: + +### App builder + +```rust +bog_serve::App::stream(data_dir, pipeline).run() // Stream +bog_serve::App::keyed(data_dir, pipeline).run() // KeyedStream +``` + +`run()` generalizes the chat example's plumbing: one plain thread owns the +fold stream (single-writer), an mpsc feeds it writes, a `tokio::watch` +publishes commit notifications, axum serves. No async database code. + +### Write routes (from the stream flavor + input type) + +Input types require `Serialize + DeserializeOwned + schemars::JsonSchema`. + +- `Stream`: `POST /insert`, `POST /remove` (full record — fold's actual + retraction contract), `POST /batch` +- `KeyedStream`: `PUT /docs/{key}`, `DELETE /docs/{key}`, `POST /batch` +- `/batch` maps to a single `wtx` — atomicity across all views is a feature, + documented in the generated OpenAPI description. +- Every write response carries a monotonic commit `seq`. + +### Read routes (from an `ApiSurface` trait) + +A trait implemented per terminal sink, composed over tuples the same way +readers already mirror the pipeline structure: + +```rust +trait ApiSurface { + fn routes(&self) -> ...; // axum routes, closed over reader access + fn openapi(&self) -> ...; // path + schema fragments +} +``` + +Sink names become paths: + +| Sink | Routes | +|---|---| +| `Count` | `GET /views/{name}` → `{ value }` | +| `Bag` | `GET /views/{name}?limit&offset` → `[[T, multiplicity]]` | +| `Table` | `GET /views/{name}`, `GET /views/{name}/{key}` | +| `Multimap` | `GET /views/{name}/{key}` → `[V]` | +| `Bm25` | `GET /views/{name}/search?q&k` → scored hits | +| `Hnsw` | `POST /views/{name}/search` (vector body; `?q=` text form only if a query encoder is registered) | +| `Histogram` / `Stats` / `Ranked` | corresponding `GET`s | + +The HNSW text-query hole is real and handled explicitly: the text→vector map +lives in a user closure upstream where the server can't see it, so text +search requires opt-in registration (e.g. `.query_encoder(ese::encode_single)` +on the sink or serve config). Otherwise the route accepts raw vectors only. + +### Platform plumbing (the future cloud contract) + +- `GET /openapi.json` — assembled at startup from the same `ApiSurface` + values that build the router, so doc and behavior cannot drift +- `GET /healthz` +- `GET /schema` — fingerprint: hash of (input type JSON schema + sink + names/types). Locally: detects data-dir/pipeline mismatch at startup with + a clear error instead of a fjall surprise. Later: the deploy-safety check. +- `GET /watch` — SSE, events carry commit `seq`, resumable via + `Last-Event-ID`. v1 emits per-commit notification events; per-view + payloads (`/views/{name}/watch`) are phase 3. +- Bearer-token middleware, **off by default locally**, enabled by env var. + +### Escape hatch for custom routes + +Non-negotiable (the search example's RRF hybrid endpoint is exactly this): + +```rust +App::keyed(dir, pipeline) + .route("/search/hybrid", get(|readers, params| { ... })) + .run() +``` + +Custom handlers get snapshot access via a closure where the concrete reader +tuple type is inferred — the same trick the examples' macros use, but +captured by the builder's generics so users never name the type. If custom +logic can't coexist with generated routes, power users abandon the crate and +the OpenAPI guarantee is lost. + +### Required fold changes (small, coordinated) + +1. Sink name accessors — sinks store their name; expose it (`fn name()`) so + `ApiSurface` can build paths and the fingerprint. +2. Possibly a marker/metadata trait on terminal readers so tuple composition + of `ApiSurface` doesn't need one impl per tuple arity per sink + combination. To be settled in a short design spike (phase 1, first task). +3. Nothing else: `wtx`/`rtx` and the single-writer model are used as-is. + +Land these before the crates.io publish if possible — cheaper than a +point release right after. + +## crates.io transition (~2 weeks out) + +- Until publish: templates emit path deps; `bogkit new` only supports + in-workspace mode (matching today's script). Standalone mode exists behind + a flag but errors with a friendly "fold isn't on crates.io yet". +- On publish: templates carry both dep forms; the CLI picks path deps when + cwd is inside this workspace, versioned deps otherwise. In-workspace mode + stays supported forever (hackathon flow, contributor flow). +- `bog-serve` and the `bogkit` CLI should publish in the same wave as fold, so + a standalone server project resolves entirely from crates.io. +- Version pinning: templates pin the minor version of fold/anny/ese/bog-serve + that the CLI was built against. +- Watch item: ese is a heavy dependency (embedded model). First build of a + standalone project will be slow; `bogkit new` should say so, and the embedded + template should keep ese optional/commented like the current script keeps + it merely available. + +## Phases + +### Phase 0 — CLI skeleton + embedded parity (small) + +`bogkit new --kind embedded` reproduces the script exactly (same validation, +same manifest, same starter main.rs); `bogkit dev` runs it. clap, no config +file yet. + +**Exit:** `bogkit new foo --kind embedded && bogkit dev -p foo` works from the +workspace root; CI job scaffolds and `cargo check`s the result. + +### Phase 1 — `bog-serve` MVP on `Stream` (the meat) + +Design spike on the `ApiSurface`/reader-metadata question first (with fold +changes from it landed), then: App builder + ingest thread, write routes + +`/batch`, `ApiSurface` for `Count`/`Bag`/`Table`, `/openapi.json`, +`/healthz`, `/schema`, commit seqs. Server template (a small +Count+Bag starter, served) and `bogkit new --kind server` + `bogkit dev`. + +**Exit:** `bogkit new foo && bogkit dev` gives a working HTTP API over the starter +pipeline; `/openapi.json` validates against the OpenAPI spec; integration +test spawns the server and exercises every generated route; batch atomicity +covered by a test that reads mid-batch state and observes all-or-nothing. + +### Phase 2 — keyed streams, search, custom routes + +`App::keyed` with upsert/remove-by-key routes; `ApiSurface` for `Bm25` and +`Hnsw` (+ `query_encoder`); `/watch` SSE (commit events); the custom-route +escape hatch. **Dogfood milestone:** port `examples/search` to `bog-serve` — +generated BM25/HNSW/table routes plus a custom `/search/hybrid` — kept in +the repo as the reference server example. + +**Exit:** the ported search example serves hybrid search over HTTP with a +correct OpenAPI doc, custom route included; forget-by-key over HTTP +demonstrably removes from every index. + +### Phase 3 — ironclad + +- Golden OpenAPI snapshots per template + the search port (drift = CI fail) +- Error model: consistent JSON error shape, correct status codes, + deserialization failures reported with the offending field +- Concurrency: hammer test — many writers + readers + one SSE subscriber; + seqs strictly monotonic, snapshots never torn +- `/schema` mismatch on startup produces the clear error, not corruption +- Remaining sinks (`Histogram`, `Stats`, `Ranked`, `Multimap`, + `InvertedIndex`) covered by `ApiSurface` +- Per-view `/views/{name}/watch` payload streaming +- Docs: rustdoc for `bog-serve`, README section replacing the + new-project.sh instructions +- crates.io switch flipped when fold publishes + +**Exit:** the bar for starting cloud work — a stranger can `cargo install +bogkit`, `bogkit new`, `bogkit dev`, and drive the whole API from curl with +only the OpenAPI doc for guidance. + +## Deferred (noted so they don't creep in) + +Auth beyond a static local token, MCP surface generation, msgpack/CBOR +negotiation, delta-log export, remote builds, `bogkit deploy` — all +cloud-phase. + +## Open questions + +1. Tuple-arity blowup in `ApiSurface` composition — macro-generate impls up + to arity N (fold presumably already does this for `Push` on tuples), or + a different composition shape? Phase 1 spike decides. +2. `Bag` route semantics for non-trivially-large bags — pagination is in the + table above, but is offset-pagination over an LSM iterator acceptable, or + do we need cursor tokens? Fine to ship offset first. +3. Does `bogkit dev` watch-and-rebuild (`cargo watch` style)? Nice, not + phase 0–3 critical. +4. Server template default port and data-dir conventions — proposed: + `PORT=7877`, `~/.bogkit/data/`; confirm before phase 1 lands. diff --git a/docs/phase-0-3-summary.md b/docs/phase-0-3-summary.md new file mode 100644 index 0000000..2091b79 --- /dev/null +++ b/docs/phase-0-3-summary.md @@ -0,0 +1,91 @@ +# bogkit CLI + bog-serve: local HTTP serving for fold pipelines + +Implements phases 0–3 of `docs/bog-cli-plan.md`: scaffold a BogKit project, +serve any fold pipeline over HTTP with the API generated from the pipeline +itself, and an OpenAPI doc that cannot drift from behavior. + +## New crates + +### `cli/` — the `bogkit` CLI + +- `bogkit new [--kind server|embedded]` — scaffolds into `examples/`; + embedded reproduces `scripts/new-project.sh` exactly; server (default) is a + fold pipeline served by bog-serve +- `bogkit dev [-p name] [--fresh]` — `cargo run` plus conventions: stable data + dir (`~/.bogkit/data/`), `$BOG_DATA_DIR`, `$PORT` (default 7877), + project inferred from cwd +- `bogkit api [--port]` — pretty-prints the running server's `/openapi.json` +- templates embedded in the binary (works standalone once published); + `bog` name reserved for the future compiler + +### `serve/` — `bog-serve` + +Core idea: a fold program's API surface is exactly (input type at the front, +named terminal sinks at the back). Everything between is closures that never +appear in the API — so the whole HTTP layer is generated. + +- `App` (unkeyed `Stream`) and `KeyedApp` (`KeyedStream`) builders; user + `main` stays a plain fn +- writes: `POST /insert|/remove` (unkeyed), `PUT|GET|DELETE /docs/{key}` + (keyed), `POST /batch` — every batch is one atomic `wtx`; every write + returns a monotonic commit `seq` +- reads: `GET /views/{name}[/{key}]` dispatched through a `Views` trait + implemented on fold's reader types (orphan-rule friendly, tuple-composed); + all sink kinds covered: count, bag, table, stats, histogram, ranked, + multimap, inverted index, bm25, hnsw +- search: `GET /views/{name}/search?q=&k=` (text) and + `POST {vector, k}` (raw vector); `TextQuery` wrapper opts an HNSW view into + text queries by registering the pipeline's encoder — encoder/index dim + mismatch is a compile error +- live: `GET /watch` (SSE, one `{"seq"}` event per commit) and + `GET /views/{name}/watch?desc&limit` (fresh view payload per commit — e.g. + a live top-10) +- custom routes, fully typed: `.get(path, |readers, params: Q| -> Result)` + reads on one consistent snapshot (same reader tuple as `rtx`); + `.post(path, |tx, body: B| -> Result)` runs inside a fold write + transaction where `Err` **rolls the whole transaction back** — atomic + check-and-set over HTTP. `Q`/`B`/`T` schemas are captured at registration + (before type erasure), so custom routes appear fully typed in + `/openapi.json` and doc/behavior cannot drift +- docs: `/openapi.json` (validates against the OpenAPI 3.1 spec) and + `/schema` (pipeline fingerprint) assembled from the same values the router + dispatches with +- safety: schema fingerprint persisted beside the data dir; reopening with a + changed pipeline refuses to start with a clear error +- errors: uniform `{"error": ...}` JSON with field-level serde detail +- concurrency: `RwLock` mirrors fold's model (`rtx: &self` / `wtx: &mut + self`) — parallel snapshot readers, one writer, multi-client access to a + previously single-process database + +## fold changes (small, coordinated) + +- all terminal readers expose `name()` (enables generic dispatch/docs) +- `Hnsw` shared state: `Rc` → `Arc` — pipelines with vector + indexes are now `Send + Sync` (servable across threads); also removes a + latent double-borrow panic +- `pub use fjall;` re-export (downstream layers name `Snapshot` without + version skew) + +## Examples / docs / CI + +- `examples/search-server` — the search example served: same pipeline, plus + generated CRUD/search routes and a custom `/search/hybrid` RRF endpoint +- README: bogkit CLI workflow, bog-serve + CLI sections, search-server entry +- `.github/workflows/cli.yml` — builds CLI, runs bog-serve tests, validates + golden OpenAPI docs against the spec, scaffolds both project kinds and + compiles them + +## Tests (51 total) + +- 30 HTTP integration tests driving every generated route in-process +- batch atomicity, forget-by-key removes from every index, error model, + fingerprint mismatch (`should_panic`) +- concurrency hammer: 8 writers × 25 + 8 torn-snapshot readers + SSE + subscriber; seqs unique and gapless, snapshots never torn +- golden OpenAPI snapshots (`UPDATE_GOLDEN=1` to re-record), spec-validated + +## Known gaps (deferred, tracked in the plan doc) + +- commit seq resets on restart (persistent seq arrives with the delta log) +- `KeyedRanked` has no `Views` impl yet +- crates.io dual-mode scaffolding blocked on fold publishing diff --git a/examples/search-server/Cargo.toml b/examples/search-server/Cargo.toml new file mode 100644 index 0000000..b703c75 --- /dev/null +++ b/examples/search-server/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "search-server" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +anny = { path = "../../anny" } +bog-serve = { path = "../../serve" } +ese = { path = "../../ese", features = ["dim-512", "quant-8"] } +fold = { path = "../../fold" } +schemars = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/examples/search-server/src/main.rs b/examples/search-server/src/main.rs new file mode 100644 index 0000000..0227afc --- /dev/null +++ b/examples/search-server/src/main.rs @@ -0,0 +1,131 @@ +//! The `search` example, served: text search three ways over one keyed +//! stream of documents, generated straight from the pipeline by bog-serve. +//! A good base for a shared agent memory reachable over HTTP. +//! +//! Run it (`cargo run -p search-server`, or `bogkit dev -p search-server`), +//! then: +//! +//! # remember, correct, forget — every index updates atomically +//! curl -X PUT localhost:7877/docs/1 -H 'content-type: application/json' \ +//! -d '"the postgres database was slow because of a missing index"' +//! curl -X DELETE localhost:7877/docs/1 +//! +//! # search: keyword, semantic, hybrid +//! curl 'localhost:7877/views/bm25/search?q=postgres+slow' +//! curl 'localhost:7877/views/vecs/search?q=database+performance' +//! curl 'localhost:7877/search/hybrid?q=database+performance' +//! +//! # write-if-absent, atomically (409 if the id is taken) +//! curl -X POST localhost:7877/remember_once -H 'content-type: application/json' \ +//! -d '{"id": 7, "text": "the standup moved to 9:30"}' +//! +//! # everything else +//! curl localhost:7877/openapi.json +//! curl -N localhost:7877/watch +//! +//! The pipeline is the search example's, unchanged: BM25 over the text, an +//! HNSW graph over ese embeddings (computed inside the pipeline by a Map, +//! so retraction cancels cleanly), and an id -> text table. `TextQuery` +//! hands the server the same encoder for query time, which is what turns +//! `?q=` on into the vector view. + +use anny::metric::Cosine; +use bog_serve::{KeyedApp, TextQuery}; +use fold::pipeline::{Keyed, Map, terminal}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use std::collections::HashMap; + +const DIM: usize = ese::DIMENSIONS; + +/// Reciprocal-rank-fusion constant: dampens the head so one list can't +/// dominate. 60 is the value from the original RRF paper. +const RRF_K: f64 = 60.0; + +fn main() { + KeyedApp::stream( + bog_serve::data_dir(), + ( + // keyword: tokenized text, ranked by BM25 relevance + terminal::search::Bm25::new("bm25"), + // semantic: ese embeds the text right here in the pipeline; + // TextQuery registers the same encoding for query time + Map::new( + |d: &Keyed| Keyed::new(d.key, ese::encode_single(&d.val)), + TextQuery::new( + terminal::search::Hnsw::::new("vecs", Cosine, 42), + |q| ese::encode_single(q), + ), + ), + // id -> text, for showing hits + terminal::Table::new("docs"), + ), + ) + // reciprocal rank fusion of both indexes — logic that lives outside any + // sink, which is exactly what custom routes are for. The readers are + // the same tuple an rtx closure gets, on one consistent snapshot; the + // params and response types put real schemas in /openapi.json. + .get("/search/hybrid", |(bm25, vecs, docs), p: HybridParams| { + // rank-based fusion sidesteps the incomparable score scales + // (BM25 relevance vs cosine distance) + let mut fused: HashMap = HashMap::new(); + for (rank, hit) in bm25.search(&p.q, 10).iter().enumerate() { + *fused.entry(hit.val).or_default() += 1.0 / (RRF_K + rank as f64 + 1.0); + } + for (rank, hit) in vecs.inner().search(&vecs.encode(&p.q)).iter().enumerate() { + *fused.entry(hit.val).or_default() += 1.0 / (RRF_K + rank as f64 + 1.0); + } + let mut fused: Vec<(u64, f64)> = fused.into_iter().collect(); + fused.sort_by(|a, b| b.1.total_cmp(&a.1)); + fused.truncate(p.k); + + Ok(fused + .into_iter() + .map(|(id, score)| HybridHit { + id, + score, + text: docs.get(&id), + }) + .collect::>()) + }) + // a custom write: remember a fact only if its id is free. Err rolls the + // whole transaction back, so racing agents can't clobber each other — + // atomic check-and-set over HTTP. + .post("/remember_once", |tx, m: Memory| { + if tx.contains(&m.id) { + return Err((409, format!("memory {} already exists", m.id))); + } + tx.upsert(&m.id, &m.text); + Ok(json!({ "remembered": m.id })) + }) + .run() +} + +/// Query for `/search/hybrid`. Field types become the documented (and +/// enforced) query parameters: `?q=...&k=5`. +#[derive(Deserialize, JsonSchema)] +struct HybridParams { + q: String, + #[serde(default = "default_k")] + k: usize, +} + +fn default_k() -> usize { + 3 +} + +/// One fused hit; the schema of `/search/hybrid`'s response items. +#[derive(Serialize, JsonSchema)] +struct HybridHit { + id: u64, + score: f64, + text: Option, +} + +/// Body for `POST /remember_once`. +#[derive(Deserialize, JsonSchema)] +struct Memory { + id: u64, + text: String, +} diff --git a/fold/src/lib.rs b/fold/src/lib.rs index 768c6e2..d980783 100644 --- a/fold/src/lib.rs +++ b/fold/src/lib.rs @@ -58,5 +58,9 @@ pub mod pipeline; pub mod stream; +// Re-exported so layers generic over readers (e.g. bog-serve) can name +// fjall types like `Snapshot` without depending on fjall separately. +pub use fjall; + #[cfg(test)] mod tests; diff --git a/fold/src/pipeline/terminal/histogram.rs b/fold/src/pipeline/terminal/histogram.rs index 80a7604..468bb88 100644 --- a/fold/src/pipeline/terminal/histogram.rs +++ b/fold/src/pipeline/terminal/histogram.rs @@ -131,6 +131,7 @@ where HistogramReader { tx, ks: self.ks.clone().unwrap(), + name: self.name.clone(), _p: PhantomData, } } @@ -140,9 +141,17 @@ where pub struct HistogramReader<'tx, R: Readable, T> { tx: &'tx R, ks: fjall::SingleWriterTxKeyspace, + name: String, _p: PhantomData, } +impl<'tx, R: Readable, T> HistogramReader<'tx, R, T> { + /// The sink name this reader serves, as given to [`Histogram::new`]. + pub fn name(&self) -> &str { + &self.name + } +} + impl<'tx, R: Readable, T: Score> HistogramReader<'tx, R, T> { /// The number of live records across all buckets. pub fn total(&self) -> i64 { diff --git a/fold/src/pipeline/terminal/mod.rs b/fold/src/pipeline/terminal/mod.rs index 6a784d7..92118eb 100644 --- a/fold/src/pipeline/terminal/mod.rs +++ b/fold/src/pipeline/terminal/mod.rs @@ -107,9 +107,15 @@ impl Count { pub struct CountReader<'tx, R: Readable> { tx: &'tx R, ks: fjall::SingleWriterTxKeyspace, + name: String, } impl CountReader<'_, R> { + /// The sink name this reader serves, as given to [`Count::new`]. + pub fn name(&self) -> &str { + &self.name + } + /// The current count (0 if nothing was ever inserted). pub fn get(&self) -> i64 { self.tx @@ -154,6 +160,7 @@ impl Push for Count { CountReader { tx, ks: self.ks.clone().unwrap(), + name: self.name.clone(), } } } @@ -190,9 +197,17 @@ impl Bag { pub struct BagReader<'tx, R: Readable, D> { tx: &'tx R, ks: fjall::SingleWriterTxKeyspace, + name: String, _p: PhantomData, } +impl<'tx, R: Readable, D> BagReader<'tx, R, D> { + /// The sink name this reader serves, as given to [`Bag::new`]. + pub fn name(&self) -> &str { + &self.name + } +} + impl<'tx, R: Readable, D: DeserializeOwned> BagReader<'tx, R, D> { /// Iterate all `(element, multiplicity)` pairs, ordered by the element's /// `postcard` encoding. Multiplicities are always positive. @@ -262,6 +277,7 @@ impl Push for Bag { BagReader { tx, ks: self.ks.clone().unwrap(), + name: self.name.clone(), _p: PhantomData, } } @@ -305,9 +321,17 @@ impl InvertedIndex { pub struct InvertedIndexReader<'tx, R: Readable, K, V> { tx: &'tx R, ks: fjall::SingleWriterTxKeyspace, + name: String, _p: PhantomData<(K, V)>, } +impl<'tx, R: Readable, K, V> InvertedIndexReader<'tx, R, K, V> { + /// The sink name this reader serves, as given to [`InvertedIndex::new`]. + pub fn name(&self) -> &str { + &self.name + } +} + impl<'tx, R: Readable, K: DeserializeOwned, V: Serialize> InvertedIndexReader<'tx, R, K, V> { /// All keys posted under exactly `q` (empty if none). pub fn search(&self, q: &V) -> Vec { @@ -361,6 +385,7 @@ where InvertedIndexReader { tx, ks: self.ks.clone().unwrap(), + name: self.name.clone(), _p: PhantomData, } } @@ -395,9 +420,17 @@ impl Multimap { pub struct MultimapReader<'tx, R: Readable, K, V> { tx: &'tx R, ks: fjall::SingleWriterTxKeyspace, + name: String, _p: PhantomData<(K, V)>, } +impl<'tx, R: Readable, K, V> MultimapReader<'tx, R, K, V> { + /// The sink name this reader serves, as given to [`Multimap::new`]. + pub fn name(&self) -> &str { + &self.name + } +} + impl<'tx, R: Readable, K: Serialize, V: DeserializeOwned> MultimapReader<'tx, R, K, V> { /// All values posted under `key` (empty if none), ordered by the /// value's `postcard` encoding. @@ -452,6 +485,7 @@ where MultimapReader { tx, ks: self.ks.clone().unwrap(), + name: self.name.clone(), _p: PhantomData, } } diff --git a/fold/src/pipeline/terminal/ranked.rs b/fold/src/pipeline/terminal/ranked.rs index bfa79f5..3e16036 100644 --- a/fold/src/pipeline/terminal/ranked.rs +++ b/fold/src/pipeline/terminal/ranked.rs @@ -191,6 +191,7 @@ impl Push> for R RankedReader { tx, ks: self.ks.clone().unwrap(), + name: self.name.clone(), _p: PhantomData, } } @@ -200,9 +201,17 @@ impl Push> for R pub struct RankedReader<'tx, R: Readable, S, V> { tx: &'tx R, ks: fjall::SingleWriterTxKeyspace, + name: String, _p: PhantomData<(S, V)>, } +impl<'tx, R: Readable, S, V> RankedReader<'tx, R, S, V> { + /// The sink name this reader serves, as given to [`Ranked::new`]. + pub fn name(&self) -> &str { + &self.name + } +} + impl<'tx, R: Readable, S: Score, V: Clone + DeserializeOwned> RankedReader<'tx, R, S, V> { /// The lowest-scored record, if any. pub fn min(&self) -> Option> { @@ -319,6 +328,7 @@ where KeyedRankedReader { tx, ks: self.ks.clone().unwrap(), + name: self.name.clone(), _p: PhantomData, } } @@ -328,9 +338,17 @@ where pub struct KeyedRankedReader<'tx, R: Readable, K, S, V> { tx: &'tx R, ks: fjall::SingleWriterTxKeyspace, + name: String, _p: PhantomData<(K, S, V)>, } +impl<'tx, R: Readable, K, S, V> KeyedRankedReader<'tx, R, K, S, V> { + /// The sink name this reader serves, as given to [`KeyedRanked::new`]. + pub fn name(&self) -> &str { + &self.name + } +} + impl<'tx, R, K, S, V> KeyedRankedReader<'tx, R, K, S, V> where R: Readable, diff --git a/fold/src/pipeline/terminal/search/hnsw.rs b/fold/src/pipeline/terminal/search/hnsw.rs index 4a11d1e..182c65e 100644 --- a/fold/src/pipeline/terminal/search/hnsw.rs +++ b/fold/src/pipeline/terminal/search/hnsw.rs @@ -1,4 +1,4 @@ -use std::{cell::RefCell, rc::Rc}; +use std::sync::{Arc, Mutex}; use anny::metric::{Metric, Scalar}; use fjall::Readable; @@ -19,6 +19,11 @@ fn decode_vector(bytes: &[u8]) -> // persisted rows to anny's ephemeral node ids; `stale` marks the graph as // diverged from the store (an aborted transaction cannot un-mutate it), to // be rebuilt from the persisted vectors on next use. +// +// Shared behind Arc (not Rc) so readers can be used from +// multiple threads — e.g. parallel snapshot readers behind an HTTP layer. +// Searches serialize on the lock; the lock cost is noise next to the +// graph traversal. struct State< K, T, @@ -141,7 +146,7 @@ pub struct Hnsw< ks: Option, metric: M, seed: u64, - state: Rc>>, + state: Arc>>, // encoded key -> (key, latest embedding, net delta this tx) pending: FxHashMap, (K, [T; DIM], i64)>, vec_buf: Vec, @@ -174,7 +179,7 @@ where ks: None, metric, seed, - state: Rc::new(RefCell::new(State { + state: Arc::new(Mutex::new(State { index: anny::hnsw::Hnsw::new(metric, seed), ids: FxHashMap::default(), keys: FxHashMap::default(), @@ -208,7 +213,7 @@ where fn init(&mut self, init: &mut PipelineInitCtx<'_>) { let ks = init.keyspace(&self.name); // recover the graph from the vectors persisted by earlier runs - self.state.borrow_mut().rebuild( + self.state.lock().unwrap().rebuild( self.metric, self.seed, init.snapshot().iter(&ks).map(|kv| { @@ -235,7 +240,7 @@ where return; } let ks = self.ks.clone().unwrap(); - let mut state = self.state.borrow_mut(); + let mut state = self.state.lock().unwrap(); if state.stale { // the previous transaction aborted: this one sees clean // committed state, so resync the graph before applying @@ -268,16 +273,17 @@ where fn abort(&mut self) { self.pending.clear(); // graph mutations from any mid-tx flush cannot be undone in place - self.state.borrow_mut().stale = true; + self.state.lock().unwrap().stale = true; } fn reader<'tx, R: Readable>(&self, tx: &'tx R) -> Self::Reader<'tx, R> { HnswReader { tx, ks: self.ks.clone().unwrap(), + name: self.name.clone(), metric: self.metric, seed: self.seed, - state: Rc::clone(&self.state), + state: Arc::clone(&self.state), } } } @@ -298,9 +304,10 @@ pub struct HnswReader< > { tx: &'tx R, ks: fjall::SingleWriterTxKeyspace, + name: String, metric: M, seed: u64, - state: Rc>>, + state: Arc>>, } impl< @@ -322,11 +329,16 @@ where T: Scalar + DeserializeOwned, M: Metric + Copy, { + /// The sink name this reader serves, as given to [`Hnsw::new`]. + pub fn name(&self) -> &str { + &self.name + } + fn with_state( &self, f: impl FnOnce(&mut State) -> Ret, ) -> Ret { - let mut state = self.state.borrow_mut(); + let mut state = self.state.lock().unwrap(); if state.stale { let entries = self.tx.iter(&self.ks).map(|kv| { let (k, v) = kv.into_inner().unwrap(); diff --git a/fold/src/pipeline/terminal/search/mod.rs b/fold/src/pipeline/terminal/search/mod.rs index fbc883d..ede8b95 100644 --- a/fold/src/pipeline/terminal/search/mod.rs +++ b/fold/src/pipeline/terminal/search/mod.rs @@ -235,6 +235,7 @@ where Bm25Reader { tx, ks: self.ks.clone().unwrap(), + name: self.name.clone(), tok: self.tok.clone(), k1: self.k1, b: self.b, @@ -247,12 +248,20 @@ where pub struct Bm25Reader<'tx, R: Readable, K, T> { tx: &'tx R, ks: fjall::SingleWriterTxKeyspace, + name: String, tok: T, k1: f64, b: f64, _p: PhantomData, } +impl<'tx, R: Readable, K, T> Bm25Reader<'tx, R, K, T> { + /// The sink name this reader serves, as given to [`Bm25::new`]. + pub fn name(&self) -> &str { + &self.name + } +} + impl<'tx, R: Readable, K: DeserializeOwned, T: Fn(&str, &mut Vec)> Bm25Reader<'tx, R, K, T> { fn stats(&self) -> (i64, i64) { self.tx diff --git a/fold/src/pipeline/terminal/stats.rs b/fold/src/pipeline/terminal/stats.rs index d0520ba..9210fed 100644 --- a/fold/src/pipeline/terminal/stats.rs +++ b/fold/src/pipeline/terminal/stats.rs @@ -103,6 +103,7 @@ impl f64> Push for Stats { StatsReader { tx, ks: self.ks.clone().unwrap(), + name: self.name.clone(), } } } @@ -111,9 +112,15 @@ impl f64> Push for Stats { pub struct StatsReader<'tx, R: Readable> { tx: &'tx R, ks: fjall::SingleWriterTxKeyspace, + name: String, } impl<'tx, R: Readable> StatsReader<'tx, R> { + /// The sink name this reader serves, as given to [`Stats::new`]. + pub fn name(&self) -> &str { + &self.name + } + fn get(&self) -> (i64, f64, f64) { self.tx .get(&self.ks, [0]) diff --git a/fold/src/pipeline/terminal/table.rs b/fold/src/pipeline/terminal/table.rs index 3cdb06b..b625a2e 100644 --- a/fold/src/pipeline/terminal/table.rs +++ b/fold/src/pipeline/terminal/table.rs @@ -99,6 +99,7 @@ where TableReader { tx, ks: self.ks.clone().unwrap(), + name: self.name.clone(), _p: PhantomData, } } @@ -108,9 +109,17 @@ where pub struct TableReader<'tx, R: Readable, K, V> { tx: &'tx R, ks: fjall::SingleWriterTxKeyspace, + name: String, _p: PhantomData<(K, V)>, } +impl<'tx, R: Readable, K, V> TableReader<'tx, R, K, V> { + /// The sink name this reader serves, as given to [`Table::new`]. + pub fn name(&self) -> &str { + &self.name + } +} + impl<'tx, R: Readable, K: Serialize, V: DeserializeOwned> TableReader<'tx, R, K, V> { fn with_key(&self, key: &K, f: impl FnOnce(&Self, &[u8]) -> T) -> T { thread_local! { diff --git a/fold/src/stream/keyed.rs b/fold/src/stream/keyed.rs index cd8360f..cc56200 100644 --- a/fold/src/stream/keyed.rs +++ b/fold/src/stream/keyed.rs @@ -67,17 +67,33 @@ where /// Open (or create) the store at `path` and initialize the pipeline; /// see [`Stream::new`]. pub fn new(path: impl AsRef, pipeline: P) -> Self { - let inner = Stream::new(path, pipeline); + Self::try_new(path, pipeline).unwrap() + } + + /// Fallible [`new`](KeyedStream::new); see [`Stream::try_new`] for the + /// meaning of [`fjall::Error::Locked`]. + pub fn try_new(path: impl AsRef, pipeline: P) -> Result { + let inner = Stream::try_new(path, pipeline)?; let table = inner .store() - .keyspace("keyed_root", fjall::KeyspaceCreateOptions::default) - .unwrap(); - KeyedStream { + .keyspace("keyed_root", fjall::KeyspaceCreateOptions::default)?; + Ok(KeyedStream { inner, table, key_buf: Default::default(), val_buf: Default::default(), - } + }) + } + + /// Destroy all persisted state — the primary-key table and every sink — + /// and re-initialize over the empty store; see [`Stream::reset`]. + pub fn reset(&mut self) { + self.inner.reset(); + self.table = self + .inner + .store() + .keyspace("keyed_root", fjall::KeyspaceCreateOptions::default) + .unwrap(); } /// Run a write transaction over the table and the pipeline: every @@ -97,6 +113,25 @@ where }) } + /// Run a fallible write transaction: commits only if `f` returns `Ok`, + /// rolls back completely on `Err`; see [`Stream::try_wtx`]. + pub fn try_wtx( + &mut self, + f: impl FnOnce(&mut KeyedTx<'_, '_, '_, K, D, P>) -> Result, + ) -> Result { + let table = self.table.clone(); + let key_buf = &mut self.key_buf; + let val_buf = &mut self.val_buf; + self.inner.try_wtx(move |tx| { + f(&mut KeyedTx { + tx, + table, + key_buf, + val_buf, + }) + }) + } + /// Run a read transaction over one consistent snapshot across all /// sinks; see [`Stream::rtx`]. pub fn rtx(&self, f: impl for<'tx> FnOnce(P::Reader<'tx, fjall::Snapshot>) -> R) -> R { diff --git a/fold/src/stream/unkeyed.rs b/fold/src/stream/unkeyed.rs index 636332b..9c2c93a 100644 --- a/fold/src/stream/unkeyed.rs +++ b/fold/src/stream/unkeyed.rs @@ -23,19 +23,56 @@ impl> Stream { /// resolving each named node's keyspace. /// /// # Panics - /// Panics if the store cannot be opened or if two nodes claim the same - /// name. - pub fn new(path: impl AsRef, mut pipeline: P) -> Self { - let store = fjall::SingleWriterTxDatabase::builder(path).open().unwrap(); + /// Panics if the store cannot be opened (see [`try_new`](Stream::try_new) + /// for a recoverable form) or if two nodes claim the same name. + pub fn new(path: impl AsRef, pipeline: P) -> Self { + Self::try_new(path, pipeline).unwrap() + } + + /// Fallible [`new`](Stream::new): open (or create) the store at `path` + /// and initialize the pipeline. + /// + /// The store is exclusively locked per process. When another process + /// already holds it this returns [`fjall::Error::Locked`] — the signal + /// for spawn-or-connect setups to fall back to whatever server the lock + /// holder runs (fjall retries the lock internally for a few hundred + /// milliseconds first). + /// + /// # Panics + /// Panics if two nodes claim the same name. + pub fn try_new(path: impl AsRef, mut pipeline: P) -> Result { + let store = fjall::SingleWriterTxDatabase::builder(path).open()?; let mut init = PipelineInitCtx::new(&store); pipeline.init(&mut init); - Stream { + Ok(Stream { pipeline, store, _p: PhantomData, + }) + } + + /// Destroy every keyspace in the store and re-initialize the pipeline + /// over the now-empty state, as if the stream had been opened on a + /// fresh directory. + /// + /// This is for recovering a data dir whose persisted state no longer + /// matches the pipeline (e.g. the sink structure changed between runs): + /// all data is lost, but the store stays open and the process keeps its + /// lock. Sinks recover their in-memory state from the empty store, so + /// readers created afterwards see a blank slate. + pub fn reset(&mut self) { + for name in self.store.list_keyspace_names() { + let ks = self + .store + .inner() + .keyspace(&name, fjall::KeyspaceCreateOptions::default) + .unwrap(); + self.store.inner().delete_keyspace(ks).unwrap(); } + let mut init = PipelineInitCtx::new(&self.store); + self.pipeline.init(&mut init); } /// Run a write transaction: every delta pushed through the [`Tx`] handle @@ -70,6 +107,45 @@ impl> Stream { r } + /// Run a fallible write transaction: commits only if `f` returns `Ok`. + /// + /// On `Err` the whole transaction rolls back — the store is untouched + /// (even for deltas pushed before the error) and pipeline nodes reset + /// their pending state — and the error is returned. This is the + /// building block for check-and-set workflows: read mid-transaction + /// via [`Tx::rtx`], bail with `Err` to abort, return `Ok` to commit. + /// + /// Panics roll back exactly as in [`wtx`](Stream::wtx). + pub fn try_wtx( + &mut self, + f: impl FnOnce(&mut Tx<'_, '_, D, P>) -> Result, + ) -> Result { + let mut wtx = WriteTx::new(self.store.write_tx()); + + let r = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + f(&mut Tx { + pipeline: &mut self.pipeline, + tx: &mut wtx, + _p: PhantomData, + }) + })) { + Ok(Ok(r)) => r, + Ok(Err(e)) => { + // fjall tx rolls back on drop + self.pipeline.abort(); + return Err(e); + } + Err(p) => { + self.pipeline.abort(); + std::panic::resume_unwind(p); + } + }; + + self.pipeline.commit(&mut wtx); + wtx.commit(); + Ok(r) + } + /// Run a read transaction over one consistent snapshot across all sinks. /// /// The closure receives the pipeline's reader, which mirrors its sink diff --git a/fold/src/tests/mod.rs b/fold/src/tests/mod.rs index d2eec0c..8e5129a 100644 --- a/fold/src/tests/mod.rs +++ b/fold/src/tests/mod.rs @@ -11,6 +11,9 @@ mod hnsw; #[cfg(test)] mod keyed_stream; +#[cfg(test)] +mod open; + #[cfg(test)] mod retain; @@ -23,6 +26,9 @@ mod scored; #[cfg(test)] mod terminals; +#[cfg(test)] +mod try_wtx; + /// A path in the system temp dir, cleared of any previous test run's state. pub(crate) fn fresh_db(name: &str) -> std::path::PathBuf { let path = std::env::temp_dir().join(name); diff --git a/fold/src/tests/open.rs b/fold/src/tests/open.rs new file mode 100644 index 0000000..d9bd2c9 --- /dev/null +++ b/fold/src/tests/open.rs @@ -0,0 +1,63 @@ +//! Open-time behavior: the per-process store lock and full-store reset. + +use super::fresh_db; +use crate::{pipeline::*, stream::*}; + +#[test] +fn second_open_returns_locked() { + let path = fresh_db("open_locked.db"); + let _held = Stream::new(&path, terminal::Bag::::new("bag")); + match Stream::try_new(&path, terminal::Bag::::new("bag")) { + Err(crate::fjall::Error::Locked) => {} + Ok(_) => panic!("second open of a locked store unexpectedly succeeded"), + Err(e) => panic!("expected Locked, got {e:?}"), + } +} + +#[test] +fn reset_wipes_and_reinitializes() { + let path = fresh_db("reset.db"); + let mut st = Stream::new( + &path, + (terminal::Count::new("n"), terminal::Bag::new("bag")), + ); + st.wtx(|tx| { + tx.insert(&"a".to_string()); + tx.insert(&"b".to_string()); + }); + st.rtx(|(n, bag)| { + assert_eq!(n.get(), 2); + assert_eq!(bag.iter().count(), 2); + }); + + st.reset(); + st.rtx(|(n, bag)| { + assert_eq!(n.get(), 0); + assert_eq!(bag.iter().count(), 0); + }); + + // the stream stays fully usable after a reset + st.wtx(|tx| tx.insert(&"c".to_string())); + st.rtx(|(n, bag)| { + assert_eq!(n.get(), 1); + assert!(bag.contains(&"c".to_string())); + }); +} + +#[test] +fn keyed_reset_wipes_table_and_sinks() { + let path = fresh_db("reset_keyed.db"); + let mut st = KeyedStream::new(&path, terminal::Table::new("rows")); + st.wtx(|tx| { + tx.upsert(&1u32, &"alice".to_string()); + }); + assert!(st.contains(&1)); + + st.reset(); + assert_eq!(st.get(&1), None); + + st.wtx(|tx| { + tx.upsert(&2u32, &"bob".to_string()); + }); + assert_eq!(st.get(&2), Some("bob".to_string())); +} diff --git a/fold/src/tests/try_wtx.rs b/fold/src/tests/try_wtx.rs new file mode 100644 index 0000000..ec4d154 --- /dev/null +++ b/fold/src/tests/try_wtx.rs @@ -0,0 +1,59 @@ +use super::fresh_db; +use crate::pipeline::terminal; +use crate::stream::{KeyedStream, Stream}; + +#[test] +fn err_rolls_back_completely() { + let mut st = Stream::new( + fresh_db("try_wtx.db"), + ( + terminal::Count::new("total"), + terminal::Bag::::new("bag"), + ), + ); + + // deltas pushed before the Err must not survive + let out: Result<(), &str> = st.try_wtx(|tx| { + tx.insert(&"doomed".to_string()); + tx.insert(&"also doomed".to_string()); + Err("changed my mind") + }); + assert_eq!(out, Err("changed my mind")); + st.rtx(|(count, bag)| { + assert_eq!(count.get(), 0); + assert_eq!(bag.iter().count(), 0); + }); + + // Ok commits, and the stream is healthy after a prior rollback + let out: Result = st.try_wtx(|tx| { + tx.insert(&"kept".to_string()); + Ok(7) + }); + assert_eq!(out, Ok(7)); + st.rtx(|(count, bag)| { + assert_eq!(count.get(), 1); + assert!(bag.contains(&"kept".to_string())); + }); +} + +#[test] +fn keyed_check_and_set() { + let mut st = KeyedStream::new( + fresh_db("try_wtx_keyed.db"), + terminal::Table::::new("rows"), + ); + + let mut claim = |key: u32, val: &str| { + st.try_wtx(|tx| { + if tx.contains(&key) { + return Err("taken"); + } + tx.upsert(&key, &val.to_string()); + Ok(()) + }) + }; + + assert_eq!(claim(1, "first"), Ok(())); + assert_eq!(claim(1, "second"), Err("taken")); + assert_eq!(st.get(&1), Some("first".to_string())); +} diff --git a/serve/Cargo.toml b/serve/Cargo.toml new file mode 100644 index 0000000..d4d12e4 --- /dev/null +++ b/serve/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "bog-serve" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +anny = { path = "../anny" } +axum = "0.8" +fold = { path = "../fold" } +schemars = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_urlencoded = "0.7" +tokio = { version = "1", features = ["rt-multi-thread", "net", "time"] } +tokio-stream = { version = "0.1", features = ["sync"] } + +[dev-dependencies] +http-body-util = "0.1" +tempfile = "3" +# macros spelled out here: the workspace unifies tokio features today, but +# this crate's tests must also build standalone once published +tokio = { version = "1", features = ["macros"] } +tower = { version = "0.5", features = ["util"] } diff --git a/serve/src/http.rs b/serve/src/http.rs new file mode 100644 index 0000000..8a9664b --- /dev/null +++ b/serve/src/http.rs @@ -0,0 +1,868 @@ +//! Generic axum handlers over any served pipeline. +//! +//! Reads are identical for plain and keyed streams, so they're generic over +//! [`ViewSource`]. Writes differ by stream flavor (raw insert/remove vs +//! upsert/remove-by-key) and stay per-kind. + +use std::convert::Infallible; +use std::sync::{Arc, RwLock}; + +use axum::body::Bytes; +use axum::extract::rejection::{JsonRejection, QueryRejection}; +use axum::extract::{Path, Query, RawQuery, State}; +use axum::http::StatusCode; +use axum::response::sse::{Event, KeepAlive, Sse}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post, put}; +use axum::{Json, Router}; +use fold::fjall::Snapshot; +use fold::pipeline::{Keyed, Push}; +use fold::stream::{KeyedStream, Stream}; +use schemars::JsonSchema; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use tokio::sync::watch; +use tokio_stream::StreamExt; + +use crate::openapi::{CustomDoc, WriteStyle}; +use crate::to_value; +use crate::views::{ViewQuery, ViewRead, Views, parse_key, schema_of}; + +const DEFAULT_LIMIT: usize = 100; +const DEFAULT_K: usize = 10; + +/// Read access shared by every view route, implemented per stream flavor. +trait Rtx: Send + Sync + 'static { + type Reader<'tx>: Views + where + Self: 'tx; + + fn rtx(&self, f: impl for<'tx> FnOnce(Self::Reader<'tx>) -> R) -> R; + + /// Fsync all committed state; see [`fold::stream::Stream::checkpoint`]. + fn checkpoint(&mut self); + + /// Wipe all persisted state and re-initialize; see + /// [`fold::stream::Stream::reset`]. + fn reset(&mut self); +} + +trait CustomRead: Rtx +where + D: Clone + 'static, + P: Push + 'static, +{ + fn custom_read(&self, h: &ReadHandler, query: &str) -> Result; +} + +impl Rtx for Stream +where + D: Clone + Send + Sync + 'static, + P: Push + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + type Reader<'tx> = P::Reader<'tx, Snapshot>; + + fn rtx(&self, f: impl for<'tx> FnOnce(Self::Reader<'tx>) -> R) -> R { + Stream::rtx(self, f) + } + + fn checkpoint(&mut self) { + Stream::checkpoint(self) + } + + fn reset(&mut self) { + Stream::reset(self) + } +} + +impl CustomRead for Stream +where + D: Clone + Send + Sync + 'static, + P: Push + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + fn custom_read(&self, h: &ReadHandler, query: &str) -> Result { + Stream::rtx(self, |r| h(r, query)) + } +} + +impl Rtx for KeyedStream +where + K: Clone + Send + Sync + Serialize + 'static, + V: Clone + Send + Sync + Serialize + DeserializeOwned + 'static, + P: Push> + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + type Reader<'tx> = P::Reader<'tx, Snapshot>; + + fn rtx(&self, f: impl for<'tx> FnOnce(Self::Reader<'tx>) -> R) -> R { + KeyedStream::rtx(self, f) + } + + fn checkpoint(&mut self) { + KeyedStream::checkpoint(self) + } + + fn reset(&mut self) { + KeyedStream::reset(self) + } +} + +impl CustomRead, P> for KeyedStream +where + K: Clone + Send + Sync + Serialize + 'static, + V: Clone + Send + Sync + Serialize + DeserializeOwned + 'static, + P: Push> + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + fn custom_read( + &self, + h: &ReadHandler, P>, + query: &str, + ) -> Result { + KeyedStream::rtx(self, |r| h(r, query)) + } +} + +pub(crate) type ReadHandler = Arc< + dyn for<'tx> Fn(

>::Reader<'tx, Snapshot>, &str) -> Result + + Send + + Sync, +>; + +pub(crate) type WriteHandler = + Arc Result + Send + Sync>; + +pub(crate) struct CustomRoute, S> { + pub doc: CustomDoc, + pub action: CustomAction, +} + +pub(crate) enum CustomAction, S> { + Read(ReadHandler), + Write(WriteHandler), +} + +pub(crate) type StreamCustom = CustomRoute>; +pub(crate) type KeyedCustom = CustomRoute, P, KeyedStream>; + +pub(crate) fn custom_get(path: String, handler: F) -> CustomRoute +where + D: Clone, + P: Push + 'static, + Q: DeserializeOwned + JsonSchema, + T: Serialize + JsonSchema, + F: for<'tx> Fn(P::Reader<'tx, Snapshot>, Q) -> Result + Send + Sync + 'static, +{ + CustomRoute { + doc: CustomDoc { + path, + method: "get", + params: Some(schema_of::()), + body: None, + response: schema_of::(), + }, + action: CustomAction::Read(Arc::new(move |readers, raw_query| { + let q: Q = serde_urlencoded::from_str(raw_query) + .map_err(|e| (400, format!("invalid query: {e}")))?; + handler(readers, q).map(to_value) + })), + } +} + +pub(crate) fn custom_write(path: String, run: F) -> CustomRoute +where + D: Clone, + P: Push, + B: DeserializeOwned + JsonSchema, + T: Serialize + JsonSchema, + F: Fn(&mut S, B) -> Result + Send + Sync + 'static, +{ + CustomRoute { + doc: CustomDoc { + path, + method: "post", + params: None, + body: Some(schema_of::()), + response: schema_of::(), + }, + action: CustomAction::Write(Arc::new(move |stream, raw| { + let body: B = + serde_json::from_slice(raw).map_err(|e| (400, format!("invalid body: {e}")))?; + run(stream, body).map(to_value) + })), + } +} + +struct Docs { + openapi: Value, + schema: Value, +} + +trait ViewSource: Send + Sync + 'static { + fn view(&self, view: &str, q: &ViewQuery) -> (u64, ViewRead); + fn docs(&self) -> &Docs; + fn subscribe(&self) -> watch::Receiver; +} + +struct Shared { + inner: RwLock>, + docs: Docs, + notify: watch::Sender, +} + +struct Inner { + stream: S, + seq: u64, +} + +impl ViewSource for Shared { + fn view(&self, view: &str, q: &ViewQuery) -> (u64, ViewRead) { + let inner = self.inner.read().unwrap(); + (inner.seq, inner.stream.rtx(|r| r.read(view, q))) + } + + fn docs(&self) -> &Docs { + &self.docs + } + + fn subscribe(&self) -> watch::Receiver { + self.notify.subscribe() + } +} + +/// Everything [`run`](crate::App::run) needs beyond the router itself: +/// hooks into the shared stream for the idle watchdog, type-erased so the +/// serve loop stays non-generic. +pub(crate) struct Lifecycle { + /// The pipeline's schema fingerprint (also persisted in the `.schema` + /// sidecar), for the discovery sidecar. + pub fingerprint: String, + /// Drains in-flight writes (takes the write lock) and fsyncs all + /// committed state. + pub checkpoint: Box, + /// Live SSE subscriptions (`/watch`, `/views/{name}/watch`); the idle + /// watchdog won't exit while any are connected. + pub watchers: Box usize + Send + Sync>, +} + +impl Lifecycle { + fn new(shared: Arc>, fingerprint: String) -> Self { + let cp = shared.clone(); + Lifecycle { + fingerprint, + checkpoint: Box::new(move || cp.inner.write().unwrap().stream.checkpoint()), + watchers: Box::new(move || shared.notify.receiver_count()), + } + } +} + +fn shared_from( + mut stream: S, + custom: &[CustomRoute], + db_path: &std::path::Path, + input_schema: &Value, + style: WriteStyle<'_>, + drift: crate::SchemaDrift, +) -> (Arc>, String) +where + D: Clone, + P: Push, +{ + let specs = stream.rtx(|r| { + let mut specs = Vec::new(); + r.specs(&mut specs); + specs + }); + let custom_docs: Vec<_> = custom.iter().map(|c| c.doc.clone()).collect(); + let schema = crate::openapi::schema_doc(input_schema, &specs, style); + let fingerprint = schema["fingerprint"].as_str().unwrap().to_string(); + check_fingerprint(db_path, &fingerprint, drift, &mut stream); + let shared = Arc::new(Shared { + docs: Docs { + openapi: crate::openapi::openapi_doc(input_schema, &specs, style, &custom_docs), + schema, + }, + notify: watch::channel(0).0, + inner: RwLock::new(Inner { stream, seq: 0 }), + }); + (shared, fingerprint) +} + +fn attach_custom( + mut app: Router, + shared: Arc>, + custom: Vec>, +) -> Router +where + D: Clone + 'static, + P: Push + 'static, + S: Rtx + CustomRead, +{ + for route in custom { + let path = route.doc.path.clone(); + match route.action { + CustomAction::Read(handler) => { + let shared = shared.clone(); + app = app.route( + &path, + get(move |RawQuery(query): RawQuery| { + let shared = shared.clone(); + let handler = handler.clone(); + async move { + let inner = shared.inner.read().unwrap(); + let seq = inner.seq; + let out = inner + .stream + .custom_read(&handler, query.as_deref().unwrap_or("")); + drop(inner); + respond_custom(seq, out) + } + }), + ); + } + CustomAction::Write(handler) => { + let shared = shared.clone(); + app = app.route( + &path, + post(move |body: Bytes| { + let shared = shared.clone(); + let handler = handler.clone(); + async move { + let mut inner = shared.inner.write().unwrap(); + match handler(&mut inner.stream, &body) { + Ok(data) => { + inner.seq += 1; + let seq = inner.seq; + drop(inner); + shared.notify.send_replace(seq); + Json(json!({ "seq": seq, "data": data })).into_response() + } + Err((code, msg)) => { + drop(inner); + error( + StatusCode::from_u16(code) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), + msg, + ) + } + } + } + }), + ); + } + } + } + app +} + +pub(crate) fn router( + stream: Stream, + custom: Vec>, + db_path: &std::path::Path, + drift: crate::SchemaDrift, +) -> (Router, Lifecycle) +where + D: Clone + Send + Sync + DeserializeOwned + JsonSchema + 'static, + P: Push + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + let input_schema = schema_of::(); + let (shared, fingerprint) = shared_from::( + stream, + &custom, + db_path, + &input_schema, + WriteStyle::Unkeyed, + drift, + ); + let app = read_routes::>>() + .route("/insert", post(insert::)) + .route("/remove", post(remove::)) + .route("/batch", post(batch::)) + .with_state(shared.clone()); + let lifecycle = Lifecycle::new(shared.clone(), fingerprint); + (attach_custom(app, shared, custom), lifecycle) +} + +pub(crate) fn router_keyed( + stream: KeyedStream, + custom: Vec>, + db_path: &std::path::Path, + drift: crate::SchemaDrift, +) -> (Router, Lifecycle) +where + K: Clone + Send + Sync + Serialize + DeserializeOwned + JsonSchema + 'static, + V: Clone + Send + Sync + Serialize + DeserializeOwned + JsonSchema + 'static, + P: Push> + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + let input_schema = schema_of::(); + let key_schema = schema_of::(); + let (shared, fingerprint) = shared_from::, P, _>( + stream, + &custom, + db_path, + &input_schema, + WriteStyle::Keyed { + key_schema: &key_schema, + }, + drift, + ); + let app = read_routes::>>() + .route( + "/docs/{key}", + put(put_doc::) + .delete(delete_doc::) + .get(get_doc::), + ) + .route("/batch", post(batch_keyed::)) + .with_state(shared.clone()); + let lifecycle = Lifecycle::new(shared.clone(), fingerprint); + (attach_custom(app, shared, custom), lifecycle) +} + +/// Run one write on `stream`, bump seq, notify `/watch`. +/// +/// `send_replace`, not `send`: `send()` refuses to store when no client is +/// connected yet, and late subscribers must still see the latest seq. +fn commit(shared: &Shared, f: impl FnOnce(&mut S)) -> u64 { + let mut inner = shared.inner.write().unwrap(); + f(&mut inner.stream); + inner.seq += 1; + let seq = inner.seq; + drop(inner); + shared.notify.send_replace(seq); + seq +} + +fn commit_with(shared: &Shared, f: impl FnOnce(&mut S) -> R) -> (u64, R) { + let mut inner = shared.inner.write().unwrap(); + let out = f(&mut inner.stream); + inner.seq += 1; + let seq = inner.seq; + drop(inner); + shared.notify.send_replace(seq); + (seq, out) +} + +async fn insert( + State(shared): State>>>, + body: Result, JsonRejection>, +) -> Response +where + D: Clone + Send + Sync + DeserializeOwned + 'static, + P: Push + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + let data = match require_json(body) { + Ok(d) => d, + Err(resp) => return resp, + }; + let seq = commit(&shared, |stream| stream.wtx(|tx| tx.insert(&data))); + Json(json!({ "seq": seq })).into_response() +} + +async fn remove( + State(shared): State>>>, + body: Result, JsonRejection>, +) -> Response +where + D: Clone + Send + Sync + DeserializeOwned + 'static, + P: Push + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + let data = match require_json(body) { + Ok(d) => d, + Err(resp) => return resp, + }; + let seq = commit(&shared, |stream| stream.wtx(|tx| tx.remove(&data))); + Json(json!({ "seq": seq })).into_response() +} + +#[derive(Deserialize)] +#[serde(tag = "op", rename_all = "lowercase")] +enum Op { + Insert { data: D }, + Remove { data: D }, +} + +async fn batch( + State(shared): State>>>, + body: Result>>, JsonRejection>, +) -> Response +where + D: Clone + Send + Sync + DeserializeOwned + 'static, + P: Push + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + let ops = match require_json(body) { + Ok(ops) => ops, + Err(resp) => return resp, + }; + let applied = ops.len(); + let seq = commit(&shared, |stream| { + stream.wtx(|tx| { + for op in &ops { + match op { + Op::Insert { data } => tx.insert(data), + Op::Remove { data } => tx.remove(data), + } + } + }) + }); + Json(json!({ "seq": seq, "applied": applied })).into_response() +} + +async fn put_doc( + State(shared): State>>>, + Path(raw): Path, + body: Result, JsonRejection>, +) -> Response +where + K: Clone + Send + Sync + Serialize + DeserializeOwned + 'static, + V: Clone + Send + Sync + Serialize + DeserializeOwned + 'static, + P: Push> + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + let data = match require_json(body) { + Ok(d) => d, + Err(resp) => return resp, + }; + let Some(key) = parse_key::(&raw) else { + return error(StatusCode::BAD_REQUEST, key_parse_msg(&raw)); + }; + let (seq, old) = commit_with(&shared, |stream| stream.wtx(|tx| tx.upsert(&key, &data))); + Json(json!({ "seq": seq, "replaced": old.is_some() })).into_response() +} + +async fn delete_doc( + State(shared): State>>>, + Path(raw): Path, +) -> Response +where + K: Clone + Send + Sync + Serialize + DeserializeOwned + 'static, + V: Clone + Send + Sync + Serialize + DeserializeOwned + 'static, + P: Push> + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + let Some(key) = parse_key::(&raw) else { + return error(StatusCode::BAD_REQUEST, key_parse_msg(&raw)); + }; + let (seq, old) = commit_with(&shared, |stream| stream.wtx(|tx| tx.remove(&key))); + Json(json!({ "seq": seq, "removed": old.is_some() })).into_response() +} + +async fn get_doc( + State(shared): State>>>, + Path(raw): Path, +) -> Response +where + K: Clone + Send + Sync + Serialize + DeserializeOwned + 'static, + V: Clone + Send + Sync + Serialize + DeserializeOwned + 'static, + P: Push> + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + let Some(key) = parse_key::(&raw) else { + return error(StatusCode::BAD_REQUEST, key_parse_msg(&raw)); + }; + let inner = shared.inner.read().unwrap(); + let seq = inner.seq; + let record = inner.stream.get(&key); + drop(inner); + match record { + Some(data) => Json(json!({ "seq": seq, "data": data })).into_response(), + None => error(StatusCode::NOT_FOUND, "not found"), + } +} + +#[derive(Deserialize)] +#[serde(tag = "op", rename_all = "lowercase")] +enum KeyedOp { + Upsert { key: K, data: V }, + Remove { key: K }, +} + +async fn batch_keyed( + State(shared): State>>>, + body: Result>>, JsonRejection>, +) -> Response +where + K: Clone + Send + Sync + Serialize + DeserializeOwned + 'static, + V: Clone + Send + Sync + Serialize + DeserializeOwned + 'static, + P: Push> + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + let ops = match require_json(body) { + Ok(ops) => ops, + Err(resp) => return resp, + }; + let applied = ops.len(); + let (seq, ()) = commit_with(&shared, |stream| { + stream.wtx(|tx| { + for op in &ops { + match op { + KeyedOp::Upsert { key, data } => { + tx.upsert(key, data); + } + KeyedOp::Remove { key } => { + tx.remove(key); + } + } + } + }) + }); + Json(json!({ "seq": seq, "applied": applied })).into_response() +} + +fn key_parse_msg(raw: &str) -> String { + format!("cannot parse {raw:?} as this stream's key type") +} + +fn read_routes() -> Router> { + Router::new() + .route("/healthz", get(async || "ok")) + .route("/views/{name}", get(view_list::)) + .route( + "/views/{name}/search", + get(search_get::).post(search_post::), + ) + .route("/views/{name}/watch", get(watch_view::)) + .route("/views/{name}/{key}", get(view_key::)) + .route("/openapi.json", get(serve_openapi::)) + .route("/schema", get(serve_schema::)) + .route("/watch", get(watch_sse::)) +} + +#[derive(Deserialize)] +struct Page { + limit: Option, + offset: Option, + desc: Option, +} + +impl Page { + fn parts(&self) -> (usize, usize, bool) { + ( + self.limit.unwrap_or(DEFAULT_LIMIT), + self.offset.unwrap_or(0), + self.desc.unwrap_or(false), + ) + } + + fn query(&self) -> ViewQuery { + let (limit, offset, desc) = self.parts(); + ViewQuery::list(limit, offset, desc) + } +} + +async fn view_list( + State(shared): State>, + Path(name): Path, + page: Result, QueryRejection>, +) -> Response { + let page = match require_query(page) { + Ok(p) => p, + Err(resp) => return resp, + }; + respond(shared.view(&name, &page.query())) +} + +async fn view_key( + State(shared): State>, + Path((name, key)): Path<(String, String)>, + page: Result, QueryRejection>, +) -> Response { + let page = match require_query(page) { + Ok(p) => p, + Err(resp) => return resp, + }; + let (limit, offset, desc) = page.parts(); + respond(shared.view(&name, &ViewQuery::point_page(key, limit, offset, desc))) +} + +#[derive(Deserialize)] +struct SearchParams { + q: Option, + k: Option, +} + +async fn search_get( + State(shared): State>, + Path(name): Path, + params: Result, QueryRejection>, +) -> Response { + let p = match require_query(params) { + Ok(p) => p, + Err(resp) => return resp, + }; + let q = ViewQuery::search(p.q, None, p.k.unwrap_or(DEFAULT_K)); + respond(shared.view(&name, &q)) +} + +#[derive(Deserialize)] +struct SearchBody { + vector: Value, + k: Option, +} + +async fn search_post( + State(shared): State>, + Path(name): Path, + body: Result, JsonRejection>, +) -> Response { + let body = match require_json(body) { + Ok(b) => b, + Err(resp) => return resp, + }; + let q = ViewQuery::search(None, Some(body.vector), body.k.unwrap_or(DEFAULT_K)); + respond(shared.view(&name, &q)) +} + +async fn watch_view( + State(shared): State>, + Path(name): Path, + page: Result, QueryRejection>, +) -> Response { + let page = match require_query(page) { + Ok(p) => p, + Err(resp) => return resp, + }; + let q = page.query(); + match shared.view(&name, &q).1 { + ViewRead::NotFound => return error(StatusCode::NOT_FOUND, "not found"), + ViewRead::BadRequest(msg) => return error(StatusCode::BAD_REQUEST, msg), + ViewRead::Data(_) => {} + } + + let events = tokio_stream::wrappers::WatchStream::new(shared.subscribe()).map(move |_| { + let (seq, out) = shared.view(&name, &q); + let data = match out { + ViewRead::Data(data) => data, + _ => Value::Null, + }; + Ok::<_, Infallible>( + Event::default() + .id(seq.to_string()) + .data(json!({ "seq": seq, "data": data }).to_string()), + ) + }); + Sse::new(events) + .keep_alive(KeepAlive::default()) + .into_response() +} + +async fn serve_openapi(State(shared): State>) -> Response { + Json(shared.docs().openapi.clone()).into_response() +} + +async fn serve_schema(State(shared): State>) -> Response { + Json(shared.docs().schema.clone()).into_response() +} + +async fn watch_sse(State(shared): State>) -> impl IntoResponse { + let events = tokio_stream::wrappers::WatchStream::new(shared.subscribe()).map(|seq| { + Ok::<_, Infallible>( + Event::default() + .id(seq.to_string()) + .data(json!({ "seq": seq }).to_string()), + ) + }); + Sse::new(events).keep_alive(KeepAlive::default()) +} + +fn respond((seq, outcome): (u64, ViewRead)) -> Response { + match outcome { + ViewRead::Data(data) => Json(json!({ "seq": seq, "data": data })).into_response(), + ViewRead::NotFound => error(StatusCode::NOT_FOUND, "not found"), + ViewRead::BadRequest(msg) => error(StatusCode::BAD_REQUEST, msg), + } +} + +#[allow(clippy::result_large_err)] +fn require_json(body: Result, JsonRejection>) -> Result { + match body { + Ok(Json(v)) => Ok(v), + Err(rej) => Err(error( + StatusCode::BAD_REQUEST, + format!("invalid body: {}", rej.body_text()), + )), + } +} + +#[allow(clippy::result_large_err)] +fn require_query(query: Result, QueryRejection>) -> Result { + match query { + Ok(Query(v)) => Ok(v), + Err(rej) => Err(error( + StatusCode::BAD_REQUEST, + format!("invalid query: {}", rej.body_text()), + )), + } +} + +fn respond_custom(seq: u64, out: Result) -> Response { + match out { + Ok(data) => Json(json!({ "seq": seq, "data": data })).into_response(), + Err((code, msg)) => error( + StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), + msg, + ), + } +} + +fn error(status: StatusCode, msg: impl Into) -> Response { + (status, Json(json!({ "error": msg.into() }))).into_response() +} + +/// Guard a data dir against pipeline drift. The schema fingerprint is +/// persisted in a sidecar file (`.schema`) on first open; a +/// mismatch on a later open means the pipeline's input type or sink +/// structure changed, and reads through the new pipeline could silently +/// misinterpret persisted state. What happens then is the caller's choice +/// ([`SchemaDrift`](crate::SchemaDrift)): refuse to start loudly, or wipe +/// the store and rebuild from empty. +fn check_fingerprint( + db_path: &std::path::Path, + fingerprint: &str, + drift: crate::SchemaDrift, + stream: &mut S, +) { + let mut marker = db_path.as_os_str().to_owned(); + marker.push(".schema"); + + match std::fs::read_to_string(&marker) { + Ok(stored) if stored.trim() == fingerprint => return, + Ok(stored) => match drift { + crate::SchemaDrift::Panic => panic!( + "pipeline changed since this data dir was written\n\ + \n\ + data dir: {}\n\ + stored fingerprint: {}\n\ + current fingerprint: {}\n\ + \n\ + The input type or sink structure no longer matches the persisted\n\ + state. Either restore the previous pipeline, or start fresh\n\ + (`bogkit dev --fresh`, or delete the data dir and its .schema file).", + db_path.display(), + stored.trim(), + fingerprint, + ), + crate::SchemaDrift::WipeAndRebuild => { + eprintln!( + "bog-serve: pipeline changed since {} was written \ + (stored {}, current {fingerprint}); wiping and rebuilding", + db_path.display(), + stored.trim(), + ); + stream.reset(); + } + }, + Err(_) => {} + } + if let Err(e) = std::fs::write(&marker, fingerprint) { + eprintln!("bog-serve: could not persist schema fingerprint: {e}"); + } +} diff --git a/serve/src/lib.rs b/serve/src/lib.rs new file mode 100644 index 0000000..9f4bf4a --- /dev/null +++ b/serve/src/lib.rs @@ -0,0 +1,520 @@ +//! Serve a fold pipeline over HTTP, with the API generated from the +//! pipeline itself. +//! +//! The API surface of any fold program is exactly two things: the input +//! type at the front (which can describe itself via +//! [schemars](https://docs.rs/schemars)) and the named terminal sinks at +//! the back (which readers dispatch to via the [`Views`] trait). Everything +//! between is user closures that never appear in the API — so the whole +//! HTTP layer is generated, and the OpenAPI doc is assembled from the same +//! values the router dispatches with. +//! +//! ```no_run +//! use bog_serve::{App, NoParams}; +//! use fold::pipeline::terminal; +//! use schemars::JsonSchema; +//! use serde::{Deserialize, Serialize}; +//! use serde_json::json; +//! +//! #[derive(Clone, Serialize, Deserialize, JsonSchema)] +//! struct Entry { +//! text: String, +//! } +//! +//! App::stream( +//! bog_serve::data_dir(), +//! ( +//! terminal::Count::new("total"), +//! terminal::Bag::::new("entries"), +//! ), +//! ) +//! // custom routes are typed; their schemas land in /openapi.json. +//! // GET handlers read one consistent snapshot: +//! .get("/summary", |(count, _entries), _: NoParams| { +//! Ok(json!({ "total": count.get() })) +//! }) +//! // POST handlers run inside one write transaction; Err rolls it back: +//! .post("/insert_nonempty", |tx, e: Entry| { +//! if e.text.is_empty() { +//! return Err((422, "empty entries rejected".into())); +//! } +//! tx.insert(&e); +//! Ok(json!("ok")) +//! }) +//! .run() +//! ``` +//! +//! Generated routes: +//! +//! - unkeyed ([`App`]): `POST /insert`, `POST /remove`, `POST /batch` +//! (`[{"op": "insert"|"remove", "data": ...}]`) — each one atomic +//! transaction +//! - keyed ([`KeyedApp`]): `PUT|GET|DELETE /docs/{key}`, `POST /batch` +//! (`[{"op": "upsert"|"remove", "key": ..., "data": ...}]`) +//! - `GET /views/{name}` — read the sink with that name (`?limit`/`?offset` +//! paginate list-shaped views; `?desc=true` lists ordered views +//! highest-first) +//! - `GET /views/{name}/{key}` — point lookup on keyed views +//! - `GET|POST /views/{name}/search` — ranked search on searchable views +//! (`?q=&k=` text, or `{"vector": [...], "k": n}` by raw vector) +//! - `GET /watch` — SSE, one `{"seq": n}` event per commit +//! - `GET /views/{name}/watch` — SSE, the view's fresh payload after every +//! commit (`?limit`/`?desc` shape the read — e.g. a live top-10) +//! - `GET /openapi.json`, `GET /schema`, `GET /healthz` +//! +//! Every write response carries the commit `seq`; every read response +//! carries the `seq` its snapshot reflects. The seq is in-memory: it +//! orders reads against writes within one server run and restarts at 0 +//! with the process. +//! +//! # Daemon mode +//! +//! The store is exclusively locked per process, so this server is the +//! natural shared access point for multiple client processes (agents, +//! CLIs). Three builder options support running it as a spawn-on-demand +//! daemon: +//! +//! - [`bind`](App::bind) — listen on a unix domain socket instead of TCP +//! ([`Bind::Unix`]). +//! - [`idle_timeout`](App::idle_timeout) — checkpoint the store and exit 0 +//! after a quiet period, so daemons don't accumulate. +//! - [`on_schema_drift`](App::on_schema_drift) — wipe and rebuild instead +//! of panicking when the pipeline changed +//! ([`SchemaDrift::WipeAndRebuild`]), for stores holding derived state. +//! +//! While running, the server advertises itself in a discovery sidecar +//! (see [`sidecar_path`]). The spawn-or-connect protocol for clients: +//! try the sidecar's address; if that fails, spawn the server; if the +//! spawned server dies because the store is locked +//! ([`fjall::Error::Locked`](fold::fjall::Error) from +//! [`Stream::try_new`](fold::stream::Stream::try_new) — someone else won +//! the race), re-read the sidecar and connect. +//! +//! Custom [`App::get`]/[`KeyedApp::get`] handlers receive the pipeline's +//! readers plus a typed query string; [`App::post`]/[`KeyedApp::post`] +//! handlers receive the write transaction plus a typed body — `Err` rolls +//! the transaction back. Request and response schemas are captured at +//! registration so custom routes appear fully typed in `/openapi.json`. + +mod http; +mod openapi; +mod search; +mod views; + +pub use search::{TextQuery, TextQueryReader, VectorSearch}; +pub use views::{SearchMode, ViewKind, ViewQuery, ViewRead, ViewSpec, Views}; + +use fold::fjall::Snapshot; +use fold::pipeline::{Keyed, Push}; +use fold::stream::{KeyedStream, Stream}; +use http::{CustomRoute, KeyedCustom}; +use schemars::JsonSchema; +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_json::Value; + +/// Typed "no query parameters" for custom GET routes that take none: +/// `.get("/path", |readers, _: NoParams| ...)`. +#[derive(serde::Deserialize, JsonSchema)] +pub struct NoParams {} + +/// Where [`run`](App::run) listens. Defaults to TCP on `$PORT` (7877). +#[derive(Clone, Debug)] +pub enum Bind { + /// TCP on `0.0.0.0:port`. + Tcp(u16), + /// A unix domain socket at this path (created on bind, removed on + /// graceful shutdown; a stale file from a crashed run is replaced). + Unix(std::path::PathBuf), +} + +/// What to do when the data dir was written by a pipeline whose schema +/// fingerprint no longer matches (see the `/schema` route). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum SchemaDrift { + /// Refuse to start, loudly. The right default for durable data. + #[default] + Panic, + /// Delete all persisted state and start empty. The right mode for + /// derived state (caches, search indices) that the writers can simply + /// re-feed after an upgrade. + WipeAndRebuild, +} + +/// Options consumed by [`run`](App::run); see the builder methods on +/// [`App`]/[`KeyedApp`]. +#[derive(Default)] +struct ServeOpts { + bind: Option, + idle_timeout: Option, + drift: SchemaDrift, +} + +pub(crate) fn to_value(t: T) -> Value { + serde_json::to_value(t).expect("custom route Ok type serializes to JSON") +} + +/// A fold [`Stream`] wrapped in a generated HTTP server. +pub struct App> { + stream: Stream, + custom: Vec>>, + db_path: std::path::PathBuf, + opts: ServeOpts, +} + +impl App +where + D: Clone + Send + Sync + DeserializeOwned + JsonSchema + 'static, + P: Push + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + /// Open (or create) the database at `path` and wrap `pipeline` in a + /// server. Reopening the same path with the same pipeline resumes + /// prior state. + pub fn stream(path: impl AsRef, pipeline: P) -> Self { + App { + stream: Stream::new(&path, pipeline), + custom: Vec::new(), + db_path: path.as_ref().to_path_buf(), + opts: ServeOpts::default(), + } + } + + /// Where [`run`](App::run) listens; overrides `$PORT`. + pub fn bind(mut self, bind: Bind) -> Self { + self.opts.bind = Some(bind); + self + } + + /// Exit cleanly after this long without a request (and with no live SSE + /// watchers): drains writes, checkpoints the store, removes the + /// discovery sidecar and socket file, and exits 0. For daemons spawned + /// on demand; see the crate docs. + pub fn idle_timeout(mut self, timeout: std::time::Duration) -> Self { + self.opts.idle_timeout = Some(timeout); + self + } + + /// What to do when the data dir's schema fingerprint doesn't match this + /// pipeline; the default is [`SchemaDrift::Panic`]. + pub fn on_schema_drift(mut self, drift: SchemaDrift) -> Self { + self.opts.drift = drift; + self + } + + /// Typed custom GET: readers + query struct in, `Ok` value in the + /// `{seq, data}` envelope. Both schemas land in `/openapi.json`. + pub fn get(mut self, path: impl Into, handler: F) -> Self + where + Q: DeserializeOwned + JsonSchema, + T: Serialize + JsonSchema, + F: for<'tx> Fn(P::Reader<'tx, Snapshot>, Q) -> Result + + Send + + Sync + + 'static, + { + self.custom.push(http::custom_get(path.into(), handler)); + self + } + + /// Typed custom POST over a write transaction. `Err` rolls the whole + /// transaction back (see crate-level docs). + pub fn post(mut self, path: impl Into, handler: F) -> Self + where + B: DeserializeOwned + JsonSchema, + T: Serialize + JsonSchema, + F: for<'g, 'tx> Fn(&mut fold::stream::Tx<'g, 'tx, D, P>, B) -> Result + + Send + + Sync + + 'static, + { + self.custom.push(http::custom_write( + path.into(), + move |stream: &mut Stream, body| stream.try_wtx(|tx| handler(tx, body)), + )); + self + } + + /// Every generated route as an [`axum::Router`], no listener attached. + /// + /// # Panics + /// If the data dir was written by a different pipeline (schema + /// fingerprint mismatch) and the drift mode is [`SchemaDrift::Panic`] + /// — see the `/schema` route. + pub fn into_router(self) -> axum::Router { + http::router(self.stream, self.custom, &self.db_path, self.opts.drift).0 + } + + /// Serve blocking forever on the configured [`bind`](App::bind) + /// (default: TCP `0.0.0.0:$PORT`, port 7877), or until the configured + /// [`idle_timeout`](App::idle_timeout) exits the process. + pub fn run(mut self) { + let opts = std::mem::take(&mut self.opts); + let db_path = self.db_path.clone(); + let (router, lifecycle) = http::router(self.stream, self.custom, &self.db_path, opts.drift); + serve_blocking(router, lifecycle, opts, &db_path) + } +} + +/// A fold [`KeyedStream`] wrapped in a generated HTTP server: writes are +/// upsert/remove by primary key, and replacing or deleting a record +/// retracts the old one from every view automatically. +pub struct KeyedApp>> { + stream: KeyedStream, + custom: Vec>, + db_path: std::path::PathBuf, + opts: ServeOpts, +} + +impl KeyedApp +where + K: Clone + Send + Sync + Serialize + DeserializeOwned + JsonSchema + 'static, + V: Clone + Send + Sync + Serialize + DeserializeOwned + JsonSchema + 'static, + P: Push> + Send + Sync + 'static, + for<'tx> P::Reader<'tx, Snapshot>: Views, +{ + /// Open (or create) the database at `path` and wrap `pipeline` — which + /// receives [`Keyed`]`` deltas — in a server. + pub fn stream(path: impl AsRef, pipeline: P) -> Self { + KeyedApp { + stream: KeyedStream::new(&path, pipeline), + custom: Vec::new(), + db_path: path.as_ref().to_path_buf(), + opts: ServeOpts::default(), + } + } + + /// Where [`run`](KeyedApp::run) listens; see [`App::bind`]. + pub fn bind(mut self, bind: Bind) -> Self { + self.opts.bind = Some(bind); + self + } + + /// Exit cleanly when idle; see [`App::idle_timeout`]. + pub fn idle_timeout(mut self, timeout: std::time::Duration) -> Self { + self.opts.idle_timeout = Some(timeout); + self + } + + /// Schema-drift handling; see [`App::on_schema_drift`]. + pub fn on_schema_drift(mut self, drift: SchemaDrift) -> Self { + self.opts.drift = drift; + self + } + + /// Typed custom GET; see [`App::get`]. + pub fn get(mut self, path: impl Into, handler: F) -> Self + where + Q: DeserializeOwned + JsonSchema, + T: Serialize + JsonSchema, + F: for<'tx> Fn(P::Reader<'tx, Snapshot>, Q) -> Result + + Send + + Sync + + 'static, + { + self.custom.push(http::custom_get(path.into(), handler)); + self + } + + /// Typed custom POST over the keyed write transaction; see [`App::post`]. + pub fn post(mut self, path: impl Into, handler: F) -> Self + where + B: DeserializeOwned + JsonSchema, + T: Serialize + JsonSchema, + F: for<'a, 'g, 'tx> Fn( + &mut fold::stream::KeyedTx<'a, 'g, 'tx, K, V, P>, + B, + ) -> Result + + Send + + Sync + + 'static, + { + self.custom.push(http::custom_write( + path.into(), + move |stream: &mut KeyedStream, body| stream.try_wtx(|tx| handler(tx, body)), + )); + self + } + + /// Every generated route as an [`axum::Router`], no listener attached. + /// + /// # Panics + /// If the data dir was written by a different pipeline (schema + /// fingerprint mismatch) and the drift mode is [`SchemaDrift::Panic`] + /// — see the `/schema` route. + pub fn into_router(self) -> axum::Router { + http::router_keyed(self.stream, self.custom, &self.db_path, self.opts.drift).0 + } + + /// Serve blocking forever; see [`App::run`]. + pub fn run(mut self) { + let opts = std::mem::take(&mut self.opts); + let db_path = self.db_path.clone(); + let (router, lifecycle) = + http::router_keyed(self.stream, self.custom, &self.db_path, opts.drift); + serve_blocking(router, lifecycle, opts, &db_path) + } +} + +/// Where state lives: `$BOG_DATA_DIR` (set by `bogkit dev`) or `./bog.db`. +pub fn data_dir() -> std::path::PathBuf { + std::env::var_os("BOG_DATA_DIR") + .map(Into::into) + .unwrap_or_else(|| "bog.db".into()) +} + +/// Where a running server advertises itself: a `.serve.json` +/// sibling of the data dir (like the `.schema` sidecar) holding +/// `{"pid", "bind": {"tcp": port} | {"unix": path}, "fingerprint", +/// "version"}`. +/// +/// The sidecar is written after the listener binds and removed on graceful +/// (idle-timeout) shutdown. It can outlive a crashed server: treat it as +/// advisory and a dead `pid` as "not running" — the store's file lock, not +/// this file, is what guarantees at most one server. +pub fn sidecar_path(db_path: impl AsRef) -> std::path::PathBuf { + let mut p = db_path.as_ref().as_os_str().to_owned(); + p.push(".serve.json"); + p.into() +} + +/// Wall-clock idle tracking: the request middleware stamps it, the watchdog +/// reads it. Millisecond resolution is plenty for multi-minute timeouts. +struct Activity { + start: std::time::Instant, + last_ms: std::sync::atomic::AtomicU64, +} + +impl Activity { + fn new() -> Self { + Activity { + start: std::time::Instant::now(), + last_ms: 0.into(), + } + } + + fn touch(&self) { + let ms = self.start.elapsed().as_millis() as u64; + self.last_ms + .store(ms, std::sync::atomic::Ordering::Relaxed); + } + + fn idle_for(&self) -> std::time::Duration { + let now = self.start.elapsed().as_millis() as u64; + let last = self.last_ms.load(std::sync::atomic::Ordering::Relaxed); + std::time::Duration::from_millis(now.saturating_sub(last)) + } +} + +fn serve_blocking( + router: axum::Router, + lifecycle: http::Lifecycle, + opts: ServeOpts, + db_path: &std::path::Path, +) { + let bind = opts.bind.unwrap_or_else(|| { + Bind::Tcp( + std::env::var("PORT") + .ok() + .and_then(|p| p.parse().ok()) + .unwrap_or(7877), + ) + }); + + let activity = std::sync::Arc::new(Activity::new()); + let touch = activity.clone(); + let router = router.layer(axum::middleware::from_fn( + move |req: axum::extract::Request, next: axum::middleware::Next| { + touch.touch(); + next.run(req) + }, + )); + + let sidecar = sidecar_path(db_path); + // files a graceful shutdown must remove so clients don't chase a ghost + let mut cleanup = vec![sidecar.clone()]; + if let Bind::Unix(path) = &bind { + cleanup.push(path.clone()); + } + + let rt = tokio::runtime::Runtime::new().expect("starting tokio runtime"); + rt.block_on(async move { + match &bind { + Bind::Tcp(port) => { + let listener = tokio::net::TcpListener::bind(("0.0.0.0", *port)) + .await + .unwrap_or_else(|e| panic!("binding port {port}: {e}")); + write_sidecar(&sidecar, &bind, &lifecycle.fingerprint); + if let Some(timeout) = opts.idle_timeout { + tokio::spawn(watchdog(activity, timeout, lifecycle, cleanup)); + } + println!("bog-serve on http://localhost:{port} — routes at /openapi.json"); + axum::serve(listener, router).await.expect("serving"); + } + Bind::Unix(path) => { + // a leftover socket from a crashed run refuses rebinding; + // the store's file lock already guarantees we're alone here + let _ = std::fs::remove_file(path); + let listener = tokio::net::UnixListener::bind(path) + .unwrap_or_else(|e| panic!("binding {}: {e}", path.display())); + write_sidecar(&sidecar, &bind, &lifecycle.fingerprint); + if let Some(timeout) = opts.idle_timeout { + tokio::spawn(watchdog(activity, timeout, lifecycle, cleanup)); + } + println!( + "bog-serve on unix socket {} — routes at /openapi.json", + path.display() + ); + axum::serve(listener, router).await.expect("serving"); + } + } + }); +} + +fn write_sidecar(path: &std::path::Path, bind: &Bind, fingerprint: &str) { + let bind = match bind { + Bind::Tcp(port) => serde_json::json!({ "tcp": port }), + Bind::Unix(path) => serde_json::json!({ "unix": path }), + }; + let doc = serde_json::json!({ + "pid": std::process::id(), + "bind": bind, + "fingerprint": fingerprint, + "version": env!("CARGO_PKG_VERSION"), + }); + if let Err(e) = std::fs::write(path, doc.to_string()) { + eprintln!("bog-serve: could not write discovery sidecar: {e}"); + } +} + +/// Exit the process once no request has arrived for `timeout` and no SSE +/// watcher is connected: drain in-flight writes, checkpoint the store (so +/// the next open replays a minimal journal), remove the sidecar and socket +/// file, exit 0. In-flight response bodies are cut — spawn-or-connect +/// clients must treat a dropped connection as "daemon gone, respawn". +async fn watchdog( + activity: std::sync::Arc, + timeout: std::time::Duration, + lifecycle: http::Lifecycle, + cleanup: Vec, +) { + let period = (timeout / 10).clamp( + std::time::Duration::from_secs(1), + std::time::Duration::from_secs(60), + ); + let mut interval = tokio::time::interval(period); + loop { + interval.tick().await; + if activity.idle_for() >= timeout && (lifecycle.watchers)() == 0 { + break; + } + } + eprintln!("bog-serve: idle for {timeout:?}, shutting down"); + tokio::task::spawn_blocking(lifecycle.checkpoint) + .await + .expect("checkpoint on idle shutdown"); + for path in &cleanup { + let _ = std::fs::remove_file(path); + } + std::process::exit(0); +} diff --git a/serve/src/openapi.rs b/serve/src/openapi.rs new file mode 100644 index 0000000..de88652 --- /dev/null +++ b/serve/src/openapi.rs @@ -0,0 +1,404 @@ +//! Assemble the OpenAPI document and the `/schema` fingerprint from the +//! input type's JSON schema plus the pipeline's view specs — the same +//! values the router dispatches with, so doc and behavior cannot drift. + +use serde_json::{Value, json}; + +use crate::views::{Listing, ViewKind, ViewSpec}; + +/// How this server writes: raw deltas, or upsert/remove by primary key. +#[derive(Clone, Copy)] +pub(crate) enum WriteStyle<'a> { + Unkeyed, + Keyed { key_schema: &'a Value }, +} + +/// What a custom route contributes to the OpenAPI doc — captured at +/// registration time, while the handler's types are still known, then +/// carried alongside the type-erased handler. +#[derive(Clone)] +pub(crate) struct CustomDoc { + pub path: String, + /// `"get"` | `"post"`. + pub method: &'static str, + /// JSON schema of the query-parameter struct (GET routes). + pub params: Option, + /// JSON schema of the request body (POST routes). + pub body: Option, + /// JSON schema of the handler's Ok type. + pub response: Value, +} + +/// Flatten a flat object schema's properties into OpenAPI query-parameter +/// entries. Schemas without listed properties (e.g. a HashMap catch-all) +/// yield an empty list. +fn query_params(schema: &Value) -> Vec { + let required: Vec<&str> = schema["required"] + .as_array() + .map(|a| a.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + schema["properties"] + .as_object() + .map(|props| { + props + .iter() + .map(|(name, s)| { + json!({ + "name": name, + "in": "query", + "required": required.contains(&name.as_str()), + "schema": s, + }) + }) + .collect() + }) + .unwrap_or_default() +} + +/// Reads respond `{ "seq": , "data": }`. +fn envelope(data: Value) -> Value { + json!({ + "type": "object", + "properties": { "seq": { "type": "integer" }, "data": data }, + "required": ["seq", "data"], + }) +} + +/// A JSON response wrapped in OpenAPI's content/media-type nesting. +fn json_response(desc: &str, schema: Value) -> Value { + json!({ + "200": { + "description": desc, + "content": { "application/json": { "schema": schema } }, + } + }) +} + +/// A write endpoint: POST a body, get back the committed seq. +fn write_op(summary: &str, body_schema: &Value) -> Value { + let seq = json!({ + "type": "object", + "properties": { "seq": { "type": "integer" } }, + "required": ["seq"], + }); + json!({ + "post": { + "summary": summary, + "requestBody": { + "required": true, + "content": { "application/json": { "schema": body_schema } }, + }, + "responses": json_response("committed atomically", seq), + } + }) +} + +fn page_params() -> Value { + json!([ + { "name": "limit", "in": "query", "schema": { "type": "integer" } }, + { "name": "offset", "in": "query", "schema": { "type": "integer" } }, + { "name": "desc", "in": "query", "schema": { "type": "boolean" }, + "description": "list highest-first (ordered views only)" }, + ]) +} + +pub(crate) fn openapi_doc( + input: &Value, + specs: &[ViewSpec], + style: WriteStyle<'_>, + custom: &[CustomDoc], +) -> Value { + let mut paths = serde_json::Map::new(); + + match style { + WriteStyle::Unkeyed => { + paths.insert( + "/insert".into(), + write_op("insert one record into the pipeline", input), + ); + paths.insert( + "/remove".into(), + write_op( + "retract one record: every view rolls back as if it was never inserted", + input, + ), + ); + let batch_body = json!({ + "type": "array", + "items": { + "type": "object", + "properties": { + "op": { "type": "string", "enum": ["insert", "remove"] }, + "data": input, + }, + "required": ["op", "data"], + }, + }); + paths.insert( + "/batch".into(), + write_op( + "apply a mix of inserts and removes in one atomic transaction", + &batch_body, + ), + ); + } + WriteStyle::Keyed { key_schema } => { + let key_param = json!([{ + "name": "key", "in": "path", "required": true, + "schema": { "type": "string" }, + "description": "the key, as JSON or a bare string", + }]); + paths.insert( + "/docs/{key}".into(), + json!({ + "put": { + "summary": "insert or replace the record under this key; \ + the old record is retracted from every view", + "parameters": key_param, + "requestBody": { + "required": true, + "content": { "application/json": { "schema": input } }, + }, + "responses": json_response("committed", json!({ + "type": "object", + "properties": { + "seq": { "type": "integer" }, + "replaced": { "type": "boolean" }, + }, + "required": ["seq", "replaced"], + })), + }, + "delete": { + "summary": "remove by key, retracting the record from every view", + "parameters": key_param, + "responses": json_response("committed", json!({ + "type": "object", + "properties": { + "seq": { "type": "integer" }, + "removed": { "type": "boolean" }, + }, + "required": ["seq", "removed"], + })), + }, + "get": { + "summary": "the current record under this key", + "parameters": key_param, + "responses": json_response("found", envelope(input.clone())), + }, + }), + ); + let batch_body = json!({ + "type": "array", + "items": { + "type": "object", + "properties": { + "op": { "type": "string", "enum": ["upsert", "remove"] }, + "key": key_schema, + "data": input, + }, + "required": ["op", "key"], + }, + }); + paths.insert( + "/batch".into(), + write_op( + "apply a mix of upserts and removes in one atomic transaction", + &batch_body, + ), + ); + } + } + + for spec in specs { + match spec.kind.listing() { + Listing::SearchOnly | Listing::PointOnly => {} + listing => { + let (data, summary) = match listing { + Listing::Scalar | Listing::Object => { + (spec.item_schema.clone(), "read this view".to_string()) + } + Listing::Page => ( + json!({ "type": "array", "items": spec.item_schema }), + format!("list this {} view (paginated)", spec.kind.as_str()), + ), + Listing::PointOnly | Listing::SearchOnly => unreachable!(), + }; + let mut get = json!({ + "summary": summary, + "responses": json_response("one consistent snapshot", envelope(data)), + }); + if listing != Listing::Scalar { + get["parameters"] = page_params(); + } + paths.insert(format!("/views/{}", spec.name), json!({ "get": get })); + + paths.insert( + format!("/views/{}/watch", spec.name), + json!({ "get": { + "summary": "server-sent events: this view's fresh payload after every commit \ + (?limit/?desc shape the read)", + "responses": { "200": { "description": "text/event-stream" } }, + } }), + ); + } + } + + if spec.keyed { + let key_param = json!({ + "name": "key", "in": "path", "required": true, + "schema": { "type": "string" }, + "description": "the key, as JSON or a bare string", + }); + let ranked = spec.kind == ViewKind::KeyedRanked; + let data = if ranked { + json!({ "type": "array", "items": spec.item_schema }) + } else { + spec.item_schema.clone() + }; + let mut params = vec![key_param]; + if ranked && let Value::Array(extra) = page_params() { + params.extend(extra); + } + paths.insert( + format!("/views/{}/{{key}}", spec.name), + json!({ + "get": { + "summary": "point-read one key", + "parameters": params, + "responses": json_response( + "the current value under this key", + envelope(data), + ), + } + }), + ); + } + + if let Some(mode) = spec.search { + let hits = envelope(json!({ "type": "array", "items": spec.item_schema })); + let mut ops = serde_json::Map::new(); + if mode.has_text() { + ops.insert( + "get".into(), + json!({ + "summary": "text search, ranked", + "parameters": [ + { "name": "q", "in": "query", "required": true, + "schema": { "type": "string" } }, + { "name": "k", "in": "query", + "schema": { "type": "integer", "default": 10 } }, + ], + "responses": json_response("hits, best first", hits.clone()), + }), + ); + } + if mode.has_vector() { + ops.insert( + "post".into(), + json!({ + "summary": "nearest-neighbor search by raw vector", + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { + "type": "object", + "properties": { + "vector": { "type": "array", "items": { "type": "number" } }, + "k": { "type": "integer", "default": 10 }, + }, + "required": ["vector"], + } } }, + }, + "responses": json_response("hits, nearest first", hits), + }), + ); + } + paths.insert(format!("/views/{}/search", spec.name), Value::Object(ops)); + } + } + + for doc in custom { + let mut op = serde_json::Map::new(); + op.insert("summary".into(), json!("custom route")); + if let Some(params) = &doc.params { + let params = query_params(params); + if !params.is_empty() { + op.insert("parameters".into(), json!(params)); + } + } + if let Some(body) = &doc.body { + op.insert( + "requestBody".into(), + json!({ + "required": true, + "content": { "application/json": { "schema": body } }, + }), + ); + } + op.insert( + "responses".into(), + json_response( + "handler result in the seq envelope", + envelope(doc.response.clone()), + ), + ); + paths.insert(doc.path.clone(), json!({ doc.method: Value::Object(op) })); + } + + paths.insert( + "/watch".into(), + json!({ "get": { + "summary": "server-sent events: one {\"seq\": n} event per commit", + "responses": { "200": { "description": "text/event-stream" } }, + } }), + ); + paths.insert( + "/healthz".into(), + json!({ "get": { "summary": "liveness probe", "responses": { "200": { "description": "ok" } } } }), + ); + + json!({ + "openapi": "3.1.0", + "info": { + "title": "bog-serve", + "description": "auto-generated HTTP API over a fold pipeline", + "version": "0.0.0", + }, + "paths": paths, + }) +} + +/// The `/schema` document: the machine-comparable identity of this server's +/// pipeline. Two builds serving the same input type and sink structure +/// produce the same fingerprint; a mismatch against a data dir's previous +/// fingerprint is how deploys will detect incompatible pipeline changes. +pub(crate) fn schema_doc(input: &Value, specs: &[ViewSpec], style: WriteStyle<'_>) -> Value { + let views: Vec = specs + .iter() + .map(|s| { + json!({ + "name": s.name, "kind": s.kind.as_str(), "keyed": s.keyed, + "search": s.search.map(|m| m.as_str()), "item": s.item_schema, + }) + }) + .collect(); + let write = match style { + WriteStyle::Unkeyed => json!("unkeyed"), + WriteStyle::Keyed { key_schema } => json!({ "keyed": key_schema }), + }; + let body = json!({ "input": input, "views": views, "write": write }); + // serde_json maps iterate in sorted key order, so this string is a + // canonical encoding — safe to hash + let fingerprint = format!("{:016x}", fnv1a(body.to_string().as_bytes())); + json!({ "fingerprint": fingerprint, "input": input, "views": views, "write": write }) +} + +/// FNV-1a, 64-bit: tiny, dependency-free, and stable across builds and +/// platforms — unlike std's DefaultHasher, which explicitly is not. +fn fnv1a(bytes: &[u8]) -> u64 { + let mut hash: u64 = 0xcbf29ce484222325; + for &b in bytes { + hash ^= b as u64; + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} diff --git a/serve/src/search.rs b/serve/src/search.rs new file mode 100644 index 0000000..9b06efa --- /dev/null +++ b/serve/src/search.rs @@ -0,0 +1,267 @@ +//! [`Views`] for the search sinks, and the [`TextQuery`] wrapper that adds +//! query-time text encoding to a vector index. +//! +//! BM25 is text-in, text-searched — its `Views` impl is direct. HNSW is +//! vector-searched, and the text→vector mapping lives in a user `Map` +//! closure *upstream* of the sink, where no generic layer can see it. So +//! text search over HNSW is opt-in: wrap the sink in +//! [`TextQuery::new(hnsw, encoder)`](TextQuery), handing the server the +//! same encoder the pipeline uses. Without the wrapper the view still +//! serves raw-vector searches (`POST /views/{name}/search`). + +use fold::pipeline::terminal::search::{Bm25Reader, HnswReader}; +use fold::pipeline::{Push, Scored}; +use fold::stream::{PipelineInitCtx, Readable, WriteTx}; +use schemars::JsonSchema; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::{Value, json}; + +use crate::views::{SearchMode, ViewKind, ViewQuery, ViewRead, ViewSpec, Views, schema_of, spec}; + +/// One search hit: `{ "score": , "key": }`. +fn hit_schema(key_schema: Value) -> Value { + json!({ + "type": "object", + "properties": { + "score": { "type": "number" }, + "key": key_schema, + }, + "required": ["score", "key"], + }) +} + +fn hits_json(hits: &[Scored], k: usize) -> Value { + Value::Array( + hits.iter() + .take(k) + .map(|h| json!({ "score": h.score, "key": h.val })) + .collect(), + ) +} + +impl Views for Bm25Reader<'_, R, K, T> +where + R: Readable, + K: Serialize + DeserializeOwned + JsonSchema, + T: Fn(&str, &mut Vec), +{ + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.name(), + ViewKind::Bm25, + false, + Some(SearchMode::Text), + hit_schema(schema_of::()), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + if view != self.name() { + return ViewRead::NotFound; + } + match q { + ViewQuery::Search { + q: Some(text), k, .. + } => ViewRead::Data(hits_json(&self.search(text, *k), *k)), + ViewQuery::Search { + vector: Some(_), .. + } => ViewRead::BadRequest("bm25 searches text: GET .../search?q=...".into()), + ViewQuery::Search { .. } => ViewRead::BadRequest("q required".into()), + _ => ViewRead::BadRequest(format!( + "this view is searched: GET /views/{view}/search?q=..." + )), + } + } +} + +/// Typed vector search behind a JSON boundary — the shared surface between +/// a bare [`HnswReader`] and one wrapped in [`TextQuery`]. `Query` is the +/// embedding type (`[T; DIM]`), so a wrapper's encoder output is checked +/// against the index it feeds at compile time. +pub trait VectorSearch { + type Query; + + fn name(&self) -> &str; + /// Parse a JSON value (`[0.1, 0.2, ...]`) as a query vector. + fn parse(&self, v: &Value) -> Result; + /// Up to `k` nearest hits, ascending by distance. + fn hits(&self, q: &Self::Query, k: usize) -> Value; + fn key_schema(&self) -> Value; +} + +impl< + R, + K, + T, + M, + const DIM: usize, + const M0: usize, + const TOP_K: usize, + const EF_SEARCH: usize, + const EF_BUILD: usize, + const MAX_LEVEL: usize, +> VectorSearch for HnswReader<'_, R, K, T, M, DIM, M0, TOP_K, EF_SEARCH, EF_BUILD, MAX_LEVEL> +where + R: Readable, + K: Clone + Serialize + DeserializeOwned + JsonSchema, + T: anny::metric::Scalar + Copy + DeserializeOwned, + M: anny::metric::Metric + Copy, + M::Out: Serialize, +{ + type Query = [T; DIM]; + + fn name(&self) -> &str { + self.name() + } + + fn parse(&self, v: &Value) -> Result<[T; DIM], String> { + let vec: Vec = serde_json::from_value(v.clone()) + .map_err(|e| format!("vector must be an array of numbers: {e}"))?; + let got = vec.len(); + <[T; DIM]>::try_from(vec).map_err(|_| format!("vector must have {DIM} dims, got {got}")) + } + + fn hits(&self, q: &[T; DIM], k: usize) -> Value { + hits_json(&self.search(q), k) + } + + fn key_schema(&self) -> Value { + schema_of::() + } +} + +/// Serve a search request against any [`VectorSearch`] index, with an +/// optional text encoder for `?q=` queries. +fn vector_read( + index: &S, + encoder: Option S::Query>, + view: &str, + q: &ViewQuery, +) -> ViewRead { + if view != index.name() { + return ViewRead::NotFound; + } + match q { + ViewQuery::Search { + q: Some(text), k, .. + } => match encoder { + Some(encode) => ViewRead::Data(index.hits(&encode(text), *k)), + None => ViewRead::BadRequest( + "this view searches by vector: POST {\"vector\": [...]} — or wrap the sink \ + in TextQuery to enable ?q=" + .into(), + ), + }, + ViewQuery::Search { + vector: Some(raw), + k, + .. + } => match index.parse(raw) { + Ok(vec) => ViewRead::Data(index.hits(&vec, *k)), + Err(msg) => ViewRead::BadRequest(msg), + }, + ViewQuery::Search { .. } => ViewRead::BadRequest("a q or vector query is required".into()), + _ => ViewRead::BadRequest(format!("this view is searched: POST /views/{view}/search")), + } +} + +impl Views for I { + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.name(), + ViewKind::Hnsw, + false, + Some(SearchMode::Vector), + hit_schema(self.key_schema()), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + vector_read(self, None, view, q) + } +} + +/// Wrap a vector sink with the encoder that turns query text into an +/// embedding, enabling `GET /views/{name}/search?q=...`. +/// +/// The encoder is a plain `fn` pointer (e.g. `|q| ese::encode_single(q)`), +/// so it must not capture state — the same purity fold requires of the +/// pipeline's own embedding `Map`. Pass the *same* encoding both places or +/// query vectors won't live in the document vector space. +/// +/// Transparent to the pipeline: pushes, commits, and retraction all +/// delegate to the wrapped sink. +pub struct TextQuery { + inner: S, + encoder: fn(&str) -> E, +} + +impl TextQuery { + pub fn new(inner: S, encoder: fn(&str) -> E) -> Self { + TextQuery { inner, encoder } + } +} + +impl, E> Push for TextQuery { + type Reader<'tx, R: Readable + 'tx> = TextQueryReader, E>; + + fn init(&mut self, init: &mut PipelineInitCtx<'_>) { + self.inner.init(init); + } + + fn push(&mut self, tx: &mut WriteTx<'_>, data: &D, delta: isize) { + self.inner.push(tx, data, delta); + } + + fn commit(&mut self, tx: &mut WriteTx<'_>) { + self.inner.commit(tx); + } + + fn abort(&mut self) { + self.inner.abort(); + } + + fn reader<'tx, R: Readable>(&self, tx: &'tx R) -> Self::Reader<'tx, R> { + TextQueryReader { + inner: self.inner.reader(tx), + encoder: self.encoder, + } + } +} + +/// Read handle for [`TextQuery`]: the wrapped reader plus the encoder. +pub struct TextQueryReader { + inner: I, + encoder: fn(&str) -> E, +} + +impl TextQueryReader { + /// The wrapped reader, for custom routes that search directly. + pub fn inner(&self) -> &I { + &self.inner + } + + /// Encode query text the way this view's searches do. + pub fn encode(&self, text: &str) -> E { + (self.encoder)(text) + } +} + +impl Views for TextQueryReader +where + I: VectorSearch, +{ + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.inner.name(), + ViewKind::Hnsw, + false, + Some(SearchMode::TextAndVector), + hit_schema(self.inner.key_schema()), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + vector_read(&self.inner, Some(self.encoder), view, q) + } +} diff --git a/serve/src/views.rs b/serve/src/views.rs new file mode 100644 index 0000000..a70ccfd --- /dev/null +++ b/serve/src/views.rs @@ -0,0 +1,703 @@ +//! The [`Views`] trait: uniform, JSON-shaped read access to a pipeline's +//! terminal sinks. +//! +//! A fold pipeline's reader mirrors its sink structure — a lone sink yields +//! its reader, tuple branches yield tuples of readers, and operators in +//! between are already erased. Implementing `Views` on each sink reader +//! (fold's types, our trait — the orphan rule allows it) and on tuples of +//! `Views` lets one generic HTTP handler dispatch `GET /views/{name}` to +//! whichever sink claims that name, with no per-pipeline code. + +use fold::pipeline::Score; +use fold::pipeline::terminal::{ + BagReader, CountReader, HistogramReader, InvertedIndexReader, KeyedRankedReader, + MultimapReader, RankedReader, StatsReader, TableReader, +}; +use fold::stream::Readable; +use schemars::JsonSchema; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::{Value, json}; + +/// What one sink contributes to the OpenAPI doc and `/schema` fingerprint. +pub struct ViewSpec { + pub name: String, + pub kind: ViewKind, + /// Whether the view supports point lookup at `/views/{name}/{key}`. + pub keyed: bool, + pub search: Option, + /// JSON schema of one item as this view returns it. + pub item_schema: Value, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ViewKind { + Count, + Bag, + Table, + Stats, + Histogram, + Ranked, + KeyedRanked, + Multimap, + InvertedIndex, + Bm25, + Hnsw, +} + +impl ViewKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Count => "count", + Self::Bag => "bag", + Self::Table => "table", + Self::Stats => "stats", + Self::Histogram => "histogram", + Self::Ranked => "ranked", + Self::KeyedRanked => "keyed_ranked", + Self::Multimap => "multimap", + Self::InvertedIndex => "inverted_index", + Self::Bm25 => "bm25", + Self::Hnsw => "hnsw", + } + } + + pub(crate) fn listing(self) -> Listing { + match self { + Self::Count | Self::Stats => Listing::Scalar, + Self::Histogram => Listing::Object, + Self::Bag | Self::Table | Self::Ranked => Listing::Page, + Self::Multimap | Self::InvertedIndex | Self::KeyedRanked => Listing::PointOnly, + Self::Bm25 | Self::Hnsw => Listing::SearchOnly, + } + } +} + +/// How `GET /views/{name}` is documented and served. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Listing { + /// One object, no pagination (count, stats). + Scalar, + /// One object whose innards paginate (histogram buckets). + Object, + /// A paginated array. + Page, + /// No listing GET; point lookup only. + PointOnly, + /// No listing GET; search only. + SearchOnly, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum SearchMode { + Text, + Vector, + TextAndVector, +} + +impl SearchMode { + pub fn as_str(self) -> &'static str { + match self { + Self::Text => "text", + Self::Vector => "vector", + Self::TextAndVector => "text+vector", + } + } + + pub(crate) fn has_text(self) -> bool { + matches!(self, Self::Text | Self::TextAndVector) + } + + pub(crate) fn has_vector(self) -> bool { + matches!(self, Self::Vector | Self::TextAndVector) + } +} + +/// A read request already routed to `/views/{name}[/{key}|/search]`. +pub enum ViewQuery { + List { + limit: usize, + offset: usize, + desc: bool, + }, + Point { + key: String, + limit: usize, + offset: usize, + desc: bool, + }, + Search { + q: Option, + vector: Option, + k: usize, + }, +} + +impl ViewQuery { + pub fn list(limit: usize, offset: usize, desc: bool) -> Self { + Self::List { + limit, + offset, + desc, + } + } + + pub fn point(key: String) -> Self { + Self::Point { + key, + limit: 100, + offset: 0, + desc: false, + } + } + + pub fn point_page(key: String, limit: usize, offset: usize, desc: bool) -> Self { + Self::Point { + key, + limit, + offset, + desc, + } + } + + pub fn search(q: Option, vector: Option, k: usize) -> Self { + Self::Search { q, vector, k } + } +} + +/// Outcome of asking one pipeline branch to serve a read. +pub enum ViewRead { + /// No sink in this branch has that name (or the key holds nothing). + NotFound, + /// The view exists but the request doesn't fit it. + BadRequest(String), + Data(Value), +} + +/// Read dispatch and self-description for a pipeline's readers. +pub trait Views { + /// Append a spec for every sink in this branch, in pipeline order. + fn specs(&self, out: &mut Vec); + + /// Serve `q` if a sink in this branch is named `view`. + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead; +} + +pub(crate) fn spec( + name: impl Into, + kind: ViewKind, + keyed: bool, + search: Option, + item_schema: Value, +) -> ViewSpec { + ViewSpec { + name: name.into(), + kind, + keyed, + search, + item_schema, + } +} + +/// JSON schema for `T`, with all subschemas inlined. +/// +/// Inlining matters: these schemas are embedded deep inside the OpenAPI +/// document, where a schemars-default `{"$ref": "#/$defs/..."}` would +/// point at the *document* root and dangle. Recursive types cannot be +/// inlined — API DTOs shouldn't be recursive. +pub(crate) fn schema_of() -> Value { + let mut settings = schemars::generate::SchemaSettings::default(); + settings.inline_subschemas = true; + let schema = settings.into_generator().into_root_schema_for::(); + serde_json::to_value(schema).unwrap() +} + +/// Parse a raw URL path segment as a view's key type: first as JSON +/// (`7`, `"quoted"`, `[1,2]`), then as a bare string — so `/views/t/7` +/// and `/views/t/alice` both do what they look like. +pub(crate) fn parse_key(raw: &str) -> Option { + serde_json::from_str(raw) + .ok() + .or_else(|| serde_json::from_value(Value::String(raw.to_string())).ok()) +} + +fn not_searchable() -> ViewRead { + ViewRead::BadRequest("this view is not searchable".into()) +} + +fn page( + iter: I, + offset: usize, + limit: usize, + desc: bool, + row: impl Fn(T) -> Value, +) -> Vec +where + I: DoubleEndedIterator, +{ + if desc { + iter.rev().skip(offset).take(limit).map(row).collect() + } else { + iter.skip(offset).take(limit).map(row).collect() + } +} + +impl Views for CountReader<'_, R> { + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.name(), + ViewKind::Count, + false, + None, + json!({ + "type": "object", + "properties": { "value": { "type": "integer" } }, + "required": ["value"], + }), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + if view != self.name() { + return ViewRead::NotFound; + } + match q { + ViewQuery::Search { .. } => not_searchable(), + ViewQuery::Point { .. } => { + ViewRead::BadRequest("count views have no key lookup".into()) + } + ViewQuery::List { .. } => ViewRead::Data(json!({ "value": self.get() })), + } + } +} + +impl Views for BagReader<'_, R, D> +where + D: Serialize + DeserializeOwned + JsonSchema, +{ + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.name(), + ViewKind::Bag, + false, + None, + json!({ + "type": "object", + "properties": { + "value": schema_of::(), + "count": { "type": "integer" }, + }, + "required": ["value", "count"], + }), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + if view != self.name() { + return ViewRead::NotFound; + } + match q { + ViewQuery::Search { .. } => not_searchable(), + ViewQuery::Point { .. } => ViewRead::BadRequest("bag views have no key lookup".into()), + ViewQuery::List { limit, offset, .. } => { + let items: Vec = self + .iter() + .skip(*offset) + .take(*limit) + .map(|(value, count)| json!({ "value": value, "count": count })) + .collect(); + ViewRead::Data(Value::Array(items)) + } + } + } +} + +impl Views for TableReader<'_, R, K, V> +where + K: Serialize + DeserializeOwned + JsonSchema, + V: Serialize + DeserializeOwned + JsonSchema, +{ + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.name(), + ViewKind::Table, + true, + None, + json!({ + "type": "object", + "properties": { + "key": schema_of::(), + "value": schema_of::(), + }, + "required": ["key", "value"], + }), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + if view != self.name() { + return ViewRead::NotFound; + } + match q { + ViewQuery::Search { .. } => not_searchable(), + ViewQuery::Point { key: raw, .. } => match parse_key::(raw) { + None => { + ViewRead::BadRequest(format!("cannot parse {raw:?} as this table's key type")) + } + Some(key) => match self.get(&key) { + Some(value) => ViewRead::Data(json!({ "key": key, "value": value })), + None => ViewRead::NotFound, + }, + }, + ViewQuery::List { limit, offset, .. } => { + let items: Vec = self + .iter() + .skip(*offset) + .take(*limit) + .map(|(key, value)| json!({ "key": key, "value": value })) + .collect(); + ViewRead::Data(Value::Array(items)) + } + } + } +} + +impl Views for StatsReader<'_, R> { + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.name(), + ViewKind::Stats, + false, + None, + json!({ + "type": "object", + "properties": { + "count": { "type": "integer" }, + "sum": { "type": "number" }, + "mean": { "type": ["number", "null"] }, + "variance": { "type": ["number", "null"] }, + "stddev": { "type": ["number", "null"] }, + }, + "required": ["count", "sum", "mean", "variance", "stddev"], + }), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + if view != self.name() { + return ViewRead::NotFound; + } + match q { + ViewQuery::Search { .. } => not_searchable(), + ViewQuery::Point { .. } => { + ViewRead::BadRequest("stats views have no key lookup".into()) + } + ViewQuery::List { .. } => ViewRead::Data(json!({ + "count": self.count(), + "sum": self.sum(), + "mean": self.mean(), + "variance": self.variance(), + "stddev": self.stddev(), + })), + } + } +} + +impl Views for HistogramReader<'_, R, T> +where + R: Readable, + T: Score + Serialize + JsonSchema, +{ + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.name(), + ViewKind::Histogram, + false, + None, + json!({ + "type": "object", + "properties": { + "total": { "type": "integer" }, + "buckets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "bucket": schema_of::(), + "count": { "type": "integer" }, + }, + "required": ["bucket", "count"], + }, + }, + }, + "required": ["total", "buckets"], + }), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + if view != self.name() { + return ViewRead::NotFound; + } + match q { + ViewQuery::Search { .. } => not_searchable(), + ViewQuery::Point { .. } => { + ViewRead::BadRequest("histogram views have no key lookup".into()) + } + ViewQuery::List { + limit, + offset, + desc, + } => { + let row = |(bucket, count): (T, i64)| json!({ "bucket": bucket, "count": count }); + let buckets = page(self.iter(), *offset, *limit, *desc, row); + ViewRead::Data(json!({ "total": self.total(), "buckets": buckets })) + } + } + } +} + +impl Views for RankedReader<'_, R, S, V> +where + R: Readable, + S: Score + Serialize + JsonSchema, + V: Clone + Serialize + DeserializeOwned + JsonSchema, +{ + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.name(), + ViewKind::Ranked, + false, + None, + json!({ + "type": "object", + "properties": { + "score": schema_of::(), + "value": schema_of::(), + "count": { "type": "integer" }, + }, + "required": ["score", "value", "count"], + }), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + if view != self.name() { + return ViewRead::NotFound; + } + match q { + ViewQuery::Search { .. } => not_searchable(), + ViewQuery::Point { .. } => { + ViewRead::BadRequest("ranked views have no key lookup".into()) + } + ViewQuery::List { + limit, + offset, + desc, + } => { + let row = |(scored, count): (fold::pipeline::Scored, i64)| json!({ "score": scored.score, "value": scored.val, "count": count }); + ViewRead::Data(Value::Array(page(self.iter(), *offset, *limit, *desc, row))) + } + } + } +} + +impl Views for KeyedRankedReader<'_, R, K, S, V> +where + R: Readable, + K: Serialize + DeserializeOwned + JsonSchema, + S: Score + Serialize + JsonSchema, + V: Clone + Serialize + DeserializeOwned + JsonSchema, +{ + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.name(), + ViewKind::KeyedRanked, + true, + None, + json!({ + "type": "object", + "properties": { + "score": schema_of::(), + "value": schema_of::(), + "count": { "type": "integer" }, + }, + "required": ["score", "value", "count"], + }), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + if view != self.name() { + return ViewRead::NotFound; + } + match q { + ViewQuery::Search { .. } => not_searchable(), + ViewQuery::List { .. } => ViewRead::BadRequest(format!( + "keyed ranked views are point lookups: GET /views/{view}/{{key}}" + )), + ViewQuery::Point { + key: raw, + limit, + offset, + desc, + } => match parse_key::(raw) { + None => { + ViewRead::BadRequest(format!("cannot parse {raw:?} as this view's key type")) + } + Some(key) => { + let row = |(scored, count): (fold::pipeline::Scored, i64)| json!({ "score": scored.score, "value": scored.val, "count": count }); + ViewRead::Data(Value::Array(page( + self.iter(&key), + *offset, + *limit, + *desc, + row, + ))) + } + }, + } + } +} + +impl Views for MultimapReader<'_, R, K, V> +where + R: Readable, + K: Serialize + DeserializeOwned + JsonSchema, + V: Serialize + DeserializeOwned + JsonSchema, +{ + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.name(), + ViewKind::Multimap, + true, + None, + json!({ + "type": "object", + "properties": { + "key": schema_of::(), + "values": { "type": "array", "items": schema_of::() }, + }, + "required": ["key", "values"], + }), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + if view != self.name() { + return ViewRead::NotFound; + } + match q { + ViewQuery::Search { .. } => not_searchable(), + ViewQuery::List { .. } => ViewRead::BadRequest(format!( + "multimap views are point lookups: GET /views/{view}/{{key}}" + )), + ViewQuery::Point { key: raw, .. } => match parse_key::(raw) { + None => { + ViewRead::BadRequest(format!("cannot parse {raw:?} as this view's key type")) + } + // set semantics: an absent key and an empty posting list are + // the same thing, so this is a 200 with [], not a 404 + Some(key) => ViewRead::Data(json!({ "key": key, "values": self.get(&key) })), + }, + } + } +} + +impl Views for InvertedIndexReader<'_, R, K, V> +where + R: Readable, + K: Serialize + DeserializeOwned + JsonSchema, + V: Serialize + DeserializeOwned + JsonSchema, +{ + fn specs(&self, out: &mut Vec) { + out.push(spec( + self.name(), + ViewKind::InvertedIndex, + true, + None, + json!({ + "type": "object", + "properties": { + "value": schema_of::(), + "keys": { "type": "array", "items": schema_of::() }, + }, + "required": ["value", "keys"], + }), + )); + } + + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + if view != self.name() { + return ViewRead::NotFound; + } + match q { + ViewQuery::Search { .. } => not_searchable(), + ViewQuery::List { .. } => ViewRead::BadRequest(format!( + "inverted index views are point lookups: GET /views/{view}/{{value}}" + )), + ViewQuery::Point { key: raw, .. } => match parse_key::(raw) { + None => { + ViewRead::BadRequest(format!("cannot parse {raw:?} as this view's value type")) + } + Some(value) => { + let keys: Vec = self.search(&value); + ViewRead::Data(json!({ "value": value, "keys": keys })) + } + }, + } + } +} + +// Fan-out branches: readers of a tuple pipeline are a tuple of readers. +// Try each element in order; the first non-NotFound answer wins (sink +// names are unique pipeline-wide, so at most one element ever answers). +// Arities match fold's tuple Push impls (fold/src/pipeline/tuple.rs). +macro_rules! impl_views_tuple { + ($($name:ident $idx:tt),+) => { + impl<$($name: Views),+> Views for ($($name,)+) { + fn specs(&self, out: &mut Vec) { + $(self.$idx.specs(out);)+ + } + fn read(&self, view: &str, q: &ViewQuery) -> ViewRead { + $( + match self.$idx.read(view, q) { + ViewRead::NotFound => {} + hit => return hit, + } + )+ + ViewRead::NotFound + } + } + }; +} + +impl_views_tuple!(A 0); +impl_views_tuple!(A 0, B 1); +impl_views_tuple!(A 0, B 1, C 2); +impl_views_tuple!(A 0, B 1, C 2, D 3); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4, F 5); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4, F 5, G 6); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4, F 5, G 6, H 7); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4, F 5, G 6, H 7, I 8); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4, F 5, G 6, H 7, I 8, J 9); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4, F 5, G 6, H 7, I 8, J 9, K 10); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4, F 5, G 6, H 7, I 8, J 9, K 10, L 11); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4, F 5, G 6, H 7, I 8, J 9, K 10, L 11, M 12); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4, F 5, G 6, H 7, I 8, J 9, K 10, L 11, M 12, N 13); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4, F 5, G 6, H 7, I 8, J 9, K 10, L 11, M 12, N 13, O 14); +impl_views_tuple!(A 0, B 1, C 2, D 3, E 4, F 5, G 6, H 7, I 8, J 9, K 10, L 11, M 12, N 13, O 14, P 15); + +#[cfg(test)] +mod tests { + use super::parse_key; + + #[test] + fn keys_parse_as_json_then_bare_string() { + assert_eq!(parse_key::("7"), Some(7)); + assert_eq!(parse_key::("alice"), Some("alice".to_string())); + // JSON wins when it parses: a quoted segment is the string inside + assert_eq!(parse_key::("\"alice\""), Some("alice".to_string())); + assert_eq!(parse_key::("alice"), None); + } +} diff --git a/serve/tests/all_sinks.rs b/serve/tests/all_sinks.rs new file mode 100644 index 0000000..707f07e --- /dev/null +++ b/serve/tests/all_sinks.rs @@ -0,0 +1,285 @@ +//! Phase-3 coverage: every remaining sink kind served, the JSON error +//! model, ordered listings, per-view watch streams, and the schema +//! fingerprint guard. + +use axum::body::Body; +use axum::http::{Request, StatusCode, header}; +use bog_serve::App; +use fold::pipeline::{KeyBy, Keyed, Map, ScoreBy, Scored, terminal}; +use http_body_util::BodyExt; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use tokio_stream::StreamExt; +use tower::ServiceExt; + +mod common; +use common::send; + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +struct Reading { + station: String, + temp: i64, +} + +/// One pipeline exercising every non-search sink kind at once. +fn all_sinks_router() -> axum::Router { + let dir = tempfile::tempdir().unwrap().keep(); + App::stream( + dir, + ( + terminal::Count::new("total"), + terminal::Stats::new("temp_stats", |r: &Reading| r.temp as f64), + ScoreBy::new( + |r: &Reading| r.temp, + ( + terminal::Ranked::new("by_temp"), + terminal::Histogram::new("temp_hist", |t: &i64| (t / 10) * 10), + ), + ), + KeyBy::new( + |r: &Reading| r.station.clone(), + terminal::Multimap::new("by_station"), + ), + Map::new( + |r: &Reading| Keyed::new(r.station.clone(), (r.temp / 10) * 10), + terminal::InvertedIndex::::new("stations_by_bucket"), + ), + Map::new( + |r: &Reading| Keyed::new(r.station.clone(), Scored::new(r.temp, ())), + terminal::KeyedRanked::new("by_station_rank"), + ), + ), + ) + // custom POST for the rollback test: inserts, then rejects hot + // readings — the Err must un-insert + .post("/insert_checked", |tx, r: Reading| { + tx.insert(&r); + if r.temp > 100 { + return Err((422, format!("{} is implausibly hot", r.temp))); + } + Ok(json!({ "accepted": r.station })) + }) + .into_router() +} + +fn reading(station: &str, temp: i64) -> Value { + json!({ "station": station, "temp": temp }) +} + +async fn seed(router: &axum::Router) { + let ops: Vec = [("alpha", 12), ("alpha", 18), ("beta", 25), ("beta", 31)] + .iter() + .map(|(s, t)| json!({ "op": "insert", "data": reading(s, *t) })) + .collect(); + let (status, _) = send(router, "POST", "/batch", Some(json!(ops))).await; + assert_eq!(status, StatusCode::OK); +} + +#[tokio::test] +async fn every_sink_kind_serves() { + let router = all_sinks_router(); + seed(&router).await; + + let (_, body) = send(&router, "GET", "/views/total", None).await; + assert_eq!(body["data"]["value"], 4); + + // stats: one object with the moments + let (_, body) = send(&router, "GET", "/views/temp_stats", None).await; + assert_eq!(body["data"]["count"], 4); + assert_eq!(body["data"]["sum"], 86.0); + assert_eq!(body["data"]["mean"], 21.5); + + // ranked ascending, and ?desc=true for top-first + let (_, body) = send(&router, "GET", "/views/by_temp", None).await; + assert_eq!(body["data"][0]["score"], 12); + let (_, body) = send(&router, "GET", "/views/by_temp?desc=true&limit=1", None).await; + assert_eq!(body["data"][0]["score"], 31); + assert_eq!(body["data"][0]["value"]["station"], "beta"); + + // histogram: decade buckets plus the total + let (_, body) = send(&router, "GET", "/views/temp_hist", None).await; + assert_eq!(body["data"]["total"], 4); + assert_eq!( + body["data"]["buckets"], + json!([ + { "bucket": 10, "count": 2 }, + { "bucket": 20, "count": 1 }, + { "bucket": 30, "count": 1 }, + ]) + ); + + // multimap: all readings posted under a station + let (_, body) = send(&router, "GET", "/views/by_station/alpha", None).await; + assert_eq!(body["data"]["values"].as_array().unwrap().len(), 2); + + // inverted index: stations posted under a temp bucket + let (_, body) = send(&router, "GET", "/views/stations_by_bucket/10", None).await; + assert_eq!(body["data"]["keys"], json!(["alpha"])); + + // point-lookup views explain themselves on a bare GET + let (status, body) = send(&router, "GET", "/views/by_station", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body["error"].as_str().unwrap().contains("point lookups")); + + // keyed ranked: per-station score-ordered list; ?desc=true is top-first + let (status, _) = send(&router, "GET", "/views/by_station_rank", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + let (_, body) = send( + &router, + "GET", + "/views/by_station_rank/alpha?desc=true&limit=1", + None, + ) + .await; + assert_eq!(body["data"][0]["score"], 18); +} + +#[tokio::test] +async fn retraction_rolls_back_every_sink() { + let router = all_sinks_router(); + seed(&router).await; + + send(&router, "POST", "/remove", Some(reading("beta", 31))).await; + + let (_, body) = send(&router, "GET", "/views/temp_stats", None).await; + assert_eq!(body["data"]["count"], 3); + assert_eq!(body["data"]["sum"], 55.0); + + let (_, body) = send(&router, "GET", "/views/by_temp?desc=true&limit=1", None).await; + assert_eq!(body["data"][0]["score"], 25, "31 retracted from ranked"); + + let (_, body) = send(&router, "GET", "/views/temp_hist", None).await; + assert_eq!(body["data"]["total"], 3); + + let (_, body) = send(&router, "GET", "/views/stations_by_bucket/30", None).await; + assert_eq!(body["data"]["keys"], json!([]), "posting deleted"); +} + +#[tokio::test] +async fn errors_are_json_with_field_detail() { + let router = all_sinks_router(); + + // missing field: serde's message names it + let (status, body) = send(&router, "POST", "/insert", Some(json!({ "station": "x" }))).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + let msg = body["error"].as_str().unwrap(); + assert!(msg.contains("temp"), "field named in: {msg}"); + + // syntactically broken body + let req = Request::builder() + .method("POST") + .uri("/insert") + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from("{not json")) + .unwrap(); + let resp = router.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let bytes = resp.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).expect("error body is JSON"); + assert!(body["error"].is_string()); + + // bad query parameter + let (status, body) = send(&router, "GET", "/views/by_temp?limit=abc", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body["error"].as_str().unwrap().contains("invalid query")); +} + +#[tokio::test] +async fn custom_post_rolls_back_on_err() { + let router = all_sinks_router(); + seed(&router).await; // 4 readings, seq 1 + + // rejected after the insert was already pushed: everything rolls back + let (status, body) = send( + &router, + "POST", + "/insert_checked", + Some(reading("volcano", 999)), + ) + .await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + assert!(body["error"].as_str().unwrap().contains("implausibly hot")); + let (_, body) = send(&router, "GET", "/views/total", None).await; + assert_eq!( + body["data"]["value"], 4, + "rolled-back insert must not count" + ); + assert_eq!(body["seq"], 1, "no commit, no seq"); + + // accepted: commits like any generated write + let (status, body) = send( + &router, + "POST", + "/insert_checked", + Some(reading("delta", 21)), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["seq"], 2); + let (_, body) = send(&router, "GET", "/views/total", None).await; + assert_eq!(body["data"]["value"], 5); +} + +#[tokio::test] +async fn view_watch_streams_fresh_payloads() { + let router = all_sinks_router(); + seed(&router).await; // seq 1 + + let req = Request::builder() + .uri("/views/total/watch") + .body(Body::empty()) + .unwrap(); + let resp = router.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let mut events = resp.into_body().into_data_stream(); + + let first = tokio::time::timeout(std::time::Duration::from_secs(5), events.next()) + .await + .expect("initial event") + .unwrap() + .unwrap(); + let first = String::from_utf8_lossy(&first); + assert!( + first.contains("\"seq\":1") && first.contains("\"value\":4"), + "got: {first}" + ); + + // a commit through a different clone of the router pushes a fresh payload + send(&router, "POST", "/insert", Some(reading("gamma", 7))).await; + let second = tokio::time::timeout(std::time::Duration::from_secs(5), events.next()) + .await + .expect("post-commit event") + .unwrap() + .unwrap(); + let second = String::from_utf8_lossy(&second); + assert!( + second.contains("\"seq\":2") && second.contains("\"value\":5"), + "got: {second}" + ); + + // unknown views 404 immediately instead of hanging a stream + let (status, _) = send(&router, "GET", "/views/nope/watch", None).await; + assert_eq!(status, StatusCode::NOT_FOUND); +} + +#[tokio::test] +#[should_panic(expected = "pipeline changed")] +async fn changed_pipeline_refuses_stale_data_dir() { + let dir = tempfile::tempdir().unwrap().keep(); + + // first open records the fingerprint; drop closes the store. + // (Count accepts any input type, so D needs the turbofish here.) + let first = App::::stream(&dir, terminal::Count::new("total")).into_router(); + drop(first); + + // same dir, same input type, different sink structure: must refuse + let _ = App::stream( + &dir, + ( + terminal::Count::new("total"), + terminal::Bag::::new("extras"), + ), + ) + .into_router(); +} diff --git a/serve/tests/common/mod.rs b/serve/tests/common/mod.rs new file mode 100644 index 0000000..7de4009 --- /dev/null +++ b/serve/tests/common/mod.rs @@ -0,0 +1,32 @@ +use axum::body::Body; +use axum::http::{Request, StatusCode, header}; +use http_body_util::BodyExt; +use serde_json::Value; +use tower::ServiceExt; + +pub async fn send( + router: &axum::Router, + method: &str, + path: &str, + body: Option, +) -> (StatusCode, Value) { + let req = match body { + Some(v) => Request::builder() + .method(method) + .uri(path) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(v.to_string())) + .unwrap(), + None => Request::builder() + .method(method) + .uri(path) + .body(Body::empty()) + .unwrap(), + }; + let resp = router.clone().oneshot(req).await.unwrap(); + let status = resp.status(); + let bytes = resp.into_body().collect().await.unwrap().to_bytes(); + let value = serde_json::from_slice(&bytes) + .unwrap_or_else(|_| Value::String(String::from_utf8_lossy(&bytes).into_owned())); + (status, value) +} diff --git a/serve/tests/daemon.rs b/serve/tests/daemon.rs new file mode 100644 index 0000000..0da7710 --- /dev/null +++ b/serve/tests/daemon.rs @@ -0,0 +1,182 @@ +//! Daemon-mode behavior: schema-drift handling, unix-socket serving with +//! the discovery sidecar, and idle shutdown (exercised in a child process, +//! since a graceful idle exit terminates the process). + +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use bog_serve::{App, Bind, SchemaDrift, sidecar_path}; +use fold::pipeline::terminal; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +mod common; +use common::send; + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +struct Entry { + id: u64, + text: String, +} + +fn tmp() -> PathBuf { + tempfile::tempdir().unwrap().keep() +} + +fn entry(id: u64, text: &str) -> Value { + json!({ "id": id, "text": text }) +} + +#[tokio::test] +async fn schema_drift_panics_by_default() { + let db = tmp().join("db"); + drop( + App::stream( + &db, + ( + terminal::Count::new("total"), + terminal::Bag::::new("entries"), + ), + ) + .into_router(), + ); + + // same data dir, different sink structure => fingerprint mismatch + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + App::::stream(&db, terminal::Bag::::new("entries")).into_router() + })); + assert!(panicked.is_err()); +} + +#[tokio::test] +async fn schema_drift_wipe_and_rebuild() { + let db = tmp().join("db"); + { + let router = App::stream( + &db, + ( + terminal::Count::new("total"), + terminal::Bag::::new("entries"), + ), + ) + .into_router(); + send(&router, "POST", "/insert", Some(entry(1, "peat"))).await; + send(&router, "POST", "/insert", Some(entry(2, "moss"))).await; + } + + // drop the bag sink but keep "total": without the wipe, the old count + // of 2 would still be visible through the same-named sink + let router = App::::stream(&db, terminal::Count::new("total")) + .on_schema_drift(SchemaDrift::WipeAndRebuild) + .into_router(); + let (status, body) = send(&router, "GET", "/views/total", None).await; + assert_eq!(status, axum::http::StatusCode::OK); + assert_eq!(body["data"]["value"], 0); + + let (_, body) = send(&router, "POST", "/insert", Some(entry(3, "fen"))).await; + assert_eq!(body["seq"], 1); + drop(router); + + // the marker was rewritten: the new pipeline reopens fine in Panic mode + let router = App::::stream(&db, terminal::Count::new("total")).into_router(); + let (_, body) = send(&router, "GET", "/views/total", None).await; + assert_eq!(body["data"]["value"], 1); +} + +/// One blocking HTTP/1.1 request over a unix socket, returning the raw +/// response text. +fn uds_get(sock: &Path, path: &str) -> String { + let mut s = std::os::unix::net::UnixStream::connect(sock).unwrap(); + write!( + s, + "GET {path} HTTP/1.1\r\nhost: localhost\r\nconnection: close\r\n\r\n" + ) + .unwrap(); + let mut buf = String::new(); + s.read_to_string(&mut buf).unwrap(); + buf +} + +fn wait_for(what: &str, timeout: Duration, mut ready: impl FnMut() -> bool) { + let start = Instant::now(); + while !ready() { + assert!(start.elapsed() < timeout, "timed out waiting for {what}"); + std::thread::sleep(Duration::from_millis(50)); + } +} + +#[test] +fn uds_serving_and_sidecar() { + let dir = tmp(); + let db = dir.join("db"); + let sock = dir.join("serve.sock"); + let (db2, sock2) = (db.clone(), sock.clone()); + std::thread::spawn(move || { + App::stream(&db2, terminal::Bag::::new("entries")) + .bind(Bind::Unix(sock2)) + .run() + }); + wait_for("socket", Duration::from_secs(10), || sock.exists()); + + let resp = uds_get(&sock, "/healthz"); + assert!(resp.starts_with("HTTP/1.1 200"), "response: {resp}"); + + let sidecar: Value = + serde_json::from_str(&std::fs::read_to_string(sidecar_path(&db)).unwrap()).unwrap(); + assert_eq!(sidecar["pid"], std::process::id()); + assert_eq!(sidecar["bind"]["unix"], sock.to_str().unwrap()); + assert!(sidecar["fingerprint"].is_string()); + assert!(sidecar["version"].is_string()); +} + +/// Not a test of its own: the daemon half of `idle_exit_...`, run in a +/// child process because a clean idle shutdown exits the process. Without +/// the env var (the normal test run) it's a no-op. +#[test] +fn idle_exit_child() { + let Ok(db) = std::env::var("BOG_DAEMON_TEST_DB") else { + return; + }; + App::stream(&db, terminal::Bag::::new("entries")) + .bind(Bind::Unix(PathBuf::from(format!("{db}.sock")))) + .idle_timeout(Duration::from_secs(3)) + .run(); + unreachable!("the idle timeout should have exited the process"); +} + +#[test] +fn idle_exit_checkpoints_and_cleans_up() { + let dir = tmp(); + let db = dir.join("db"); + let sock = PathBuf::from(format!("{}.sock", db.display())); + + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["idle_exit_child", "--exact", "--nocapture"]) + .env("BOG_DAEMON_TEST_DB", &db) + .spawn() + .unwrap(); + + wait_for("child socket", Duration::from_secs(15), || sock.exists()); + let resp = uds_get(&sock, "/healthz"); + assert!(resp.starts_with("HTTP/1.1 200"), "response: {resp}"); + + let start = Instant::now(); + let status = loop { + if let Some(status) = child.try_wait().unwrap() { + break status; + } + assert!( + start.elapsed() < Duration::from_secs(30), + "child did not exit; killing" + ); + std::thread::sleep(Duration::from_millis(100)); + }; + assert!(status.success(), "idle exit status: {status}"); + assert!(!sock.exists(), "socket file not cleaned up"); + assert!( + !sidecar_path(&db).exists(), + "discovery sidecar not cleaned up" + ); +} diff --git a/serve/tests/golden.rs b/serve/tests/golden.rs new file mode 100644 index 0000000..c82d426 --- /dev/null +++ b/serve/tests/golden.rs @@ -0,0 +1,100 @@ +//! Golden OpenAPI snapshots: the generated doc for each reference pipeline +//! shape is pinned to a file. Drift fails CI; intended changes are +//! re-recorded with `UPDATE_GOLDEN=1 cargo test -p bog-serve --test golden`. + +use std::path::Path; + +use anny::metric::Cosine; +use bog_serve::{App, KeyedApp, NoParams, TextQuery}; +use fold::pipeline::{Keyed, Map, terminal}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +mod common; +use common::send; + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +struct Entry { + text: String, +} + +/// The `bogkit new --kind server` template pipeline, verbatim. +fn template_router() -> axum::Router { + let dir = tempfile::tempdir().unwrap().keep(); + App::stream( + dir, + ( + terminal::Count::new("total"), + terminal::Bag::::new("entries"), + ), + ) + .into_router() +} + +/// The search-server shape (toy 4-dim embedder standing in for ese). +fn search_router() -> axum::Router { + fn embed(s: &str) -> [f32; 4] { + let mut v = [0.0f32; 4]; + for (i, b) in s.bytes().enumerate() { + v[i % 4] += b as f32; + } + v + } + let dir = tempfile::tempdir().unwrap().keep(); + KeyedApp::stream( + dir, + ( + terminal::search::Bm25::new("bm25"), + Map::new( + |d: &Keyed| Keyed::new(d.key, embed(&d.val)), + TextQuery::new( + terminal::search::Hnsw::::new("vecs", Cosine, 42), + embed, + ), + ), + terminal::Table::new("docs"), + ), + ) + .get("/search/hybrid", |_readers, _: NoParams| Ok(Value::Null)) + .into_router() +} + +async fn openapi(router: axum::Router) -> Value { + let (_, doc) = send(&router, "GET", "/openapi.json", None).await; + doc +} + +fn check_golden(name: &str, doc: &Value) { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/golden") + .join(name); + let pretty = serde_json::to_string_pretty(doc).unwrap() + "\n"; + + if std::env::var_os("UPDATE_GOLDEN").is_some() { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, &pretty).unwrap(); + return; + } + + let stored = std::fs::read_to_string(&path).unwrap_or_else(|_| { + panic!("missing golden {name} — record it with UPDATE_GOLDEN=1 cargo test -p bog-serve --test golden") + }); + let stored: Value = serde_json::from_str(&stored).unwrap(); + assert_eq!( + &stored, doc, + "\nthe generated OpenAPI doc for {name} drifted from its golden file.\n\ + If this change is intended, re-record with:\n\ + UPDATE_GOLDEN=1 cargo test -p bog-serve --test golden\n" + ); +} + +#[tokio::test] +async fn template_openapi_matches_golden() { + check_golden("template.json", &openapi(template_router()).await); +} + +#[tokio::test] +async fn search_openapi_matches_golden() { + check_golden("search.json", &openapi(search_router()).await); +} diff --git a/serve/tests/golden/search.json b/serve/tests/golden/search.json new file mode 100644 index 0000000..3d1cacc --- /dev/null +++ b/serve/tests/golden/search.json @@ -0,0 +1,598 @@ +{ + "info": { + "description": "auto-generated HTTP API over a fold pipeline", + "title": "bog-serve", + "version": "0.0.0" + }, + "openapi": "3.1.0", + "paths": { + "/batch": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "items": { + "properties": { + "data": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "string", + "type": "string" + }, + "key": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uint64", + "minimum": 0, + "title": "uint64", + "type": "integer" + }, + "op": { + "enum": [ + "upsert", + "remove" + ], + "type": "string" + } + }, + "required": [ + "op", + "key" + ], + "type": "object" + }, + "type": "array" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "seq": { + "type": "integer" + } + }, + "required": [ + "seq" + ], + "type": "object" + } + } + }, + "description": "committed atomically" + } + }, + "summary": "apply a mix of upserts and removes in one atomic transaction" + } + }, + "/docs/{key}": { + "delete": { + "parameters": [ + { + "description": "the key, as JSON or a bare string", + "in": "path", + "name": "key", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "removed": { + "type": "boolean" + }, + "seq": { + "type": "integer" + } + }, + "required": [ + "seq", + "removed" + ], + "type": "object" + } + } + }, + "description": "committed" + } + }, + "summary": "remove by key, retracting the record from every view" + }, + "get": { + "parameters": [ + { + "description": "the key, as JSON or a bare string", + "in": "path", + "name": "key", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "string", + "type": "string" + }, + "seq": { + "type": "integer" + } + }, + "required": [ + "seq", + "data" + ], + "type": "object" + } + } + }, + "description": "found" + } + }, + "summary": "the current record under this key" + }, + "put": { + "parameters": [ + { + "description": "the key, as JSON or a bare string", + "in": "path", + "name": "key", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "string", + "type": "string" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "replaced": { + "type": "boolean" + }, + "seq": { + "type": "integer" + } + }, + "required": [ + "seq", + "replaced" + ], + "type": "object" + } + } + }, + "description": "committed" + } + }, + "summary": "insert or replace the record under this key; the old record is retracted from every view" + } + }, + "/healthz": { + "get": { + "responses": { + "200": { + "description": "ok" + } + }, + "summary": "liveness probe" + } + }, + "/search/hybrid": { + "get": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "AnyValue" + }, + "seq": { + "type": "integer" + } + }, + "required": [ + "seq", + "data" + ], + "type": "object" + } + } + }, + "description": "handler result in the seq envelope" + } + }, + "summary": "custom route" + } + }, + "/views/bm25/search": { + "get": { + "parameters": [ + { + "in": "query", + "name": "q", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "k", + "schema": { + "default": 10, + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "items": { + "properties": { + "key": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uint64", + "minimum": 0, + "title": "uint64", + "type": "integer" + }, + "score": { + "type": "number" + } + }, + "required": [ + "score", + "key" + ], + "type": "object" + }, + "type": "array" + }, + "seq": { + "type": "integer" + } + }, + "required": [ + "seq", + "data" + ], + "type": "object" + } + } + }, + "description": "hits, best first" + } + }, + "summary": "text search, ranked" + } + }, + "/views/docs": { + "get": { + "parameters": [ + { + "in": "query", + "name": "limit", + "schema": { + "type": "integer" + } + }, + { + "in": "query", + "name": "offset", + "schema": { + "type": "integer" + } + }, + { + "description": "list highest-first (ordered views only)", + "in": "query", + "name": "desc", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "items": { + "properties": { + "key": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uint64", + "minimum": 0, + "title": "uint64", + "type": "integer" + }, + "value": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "string", + "type": "string" + } + }, + "required": [ + "key", + "value" + ], + "type": "object" + }, + "type": "array" + }, + "seq": { + "type": "integer" + } + }, + "required": [ + "seq", + "data" + ], + "type": "object" + } + } + }, + "description": "one consistent snapshot" + } + }, + "summary": "list this table view (paginated)" + } + }, + "/views/docs/watch": { + "get": { + "responses": { + "200": { + "description": "text/event-stream" + } + }, + "summary": "server-sent events: this view's fresh payload after every commit (?limit/?desc shape the read)" + } + }, + "/views/docs/{key}": { + "get": { + "parameters": [ + { + "description": "the key, as JSON or a bare string", + "in": "path", + "name": "key", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": { + "key": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uint64", + "minimum": 0, + "title": "uint64", + "type": "integer" + }, + "value": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "string", + "type": "string" + } + }, + "required": [ + "key", + "value" + ], + "type": "object" + }, + "seq": { + "type": "integer" + } + }, + "required": [ + "seq", + "data" + ], + "type": "object" + } + } + }, + "description": "the current value under this key" + } + }, + "summary": "point-read one key" + } + }, + "/views/vecs/search": { + "get": { + "parameters": [ + { + "in": "query", + "name": "q", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "k", + "schema": { + "default": 10, + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "items": { + "properties": { + "key": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uint64", + "minimum": 0, + "title": "uint64", + "type": "integer" + }, + "score": { + "type": "number" + } + }, + "required": [ + "score", + "key" + ], + "type": "object" + }, + "type": "array" + }, + "seq": { + "type": "integer" + } + }, + "required": [ + "seq", + "data" + ], + "type": "object" + } + } + }, + "description": "hits, best first" + } + }, + "summary": "text search, ranked" + }, + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "k": { + "default": 10, + "type": "integer" + }, + "vector": { + "items": { + "type": "number" + }, + "type": "array" + } + }, + "required": [ + "vector" + ], + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "items": { + "properties": { + "key": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uint64", + "minimum": 0, + "title": "uint64", + "type": "integer" + }, + "score": { + "type": "number" + } + }, + "required": [ + "score", + "key" + ], + "type": "object" + }, + "type": "array" + }, + "seq": { + "type": "integer" + } + }, + "required": [ + "seq", + "data" + ], + "type": "object" + } + } + }, + "description": "hits, nearest first" + } + }, + "summary": "nearest-neighbor search by raw vector" + } + }, + "/watch": { + "get": { + "responses": { + "200": { + "description": "text/event-stream" + } + }, + "summary": "server-sent events: one {\"seq\": n} event per commit" + } + } + } +} diff --git a/serve/tests/golden/template.json b/serve/tests/golden/template.json new file mode 100644 index 0000000..c2edc7c --- /dev/null +++ b/serve/tests/golden/template.json @@ -0,0 +1,319 @@ +{ + "info": { + "description": "auto-generated HTTP API over a fold pipeline", + "title": "bog-serve", + "version": "0.0.0" + }, + "openapi": "3.1.0", + "paths": { + "/batch": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "items": { + "properties": { + "data": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "text": { + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "Entry", + "type": "object" + }, + "op": { + "enum": [ + "insert", + "remove" + ], + "type": "string" + } + }, + "required": [ + "op", + "data" + ], + "type": "object" + }, + "type": "array" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "seq": { + "type": "integer" + } + }, + "required": [ + "seq" + ], + "type": "object" + } + } + }, + "description": "committed atomically" + } + }, + "summary": "apply a mix of inserts and removes in one atomic transaction" + } + }, + "/healthz": { + "get": { + "responses": { + "200": { + "description": "ok" + } + }, + "summary": "liveness probe" + } + }, + "/insert": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "text": { + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "Entry", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "seq": { + "type": "integer" + } + }, + "required": [ + "seq" + ], + "type": "object" + } + } + }, + "description": "committed atomically" + } + }, + "summary": "insert one record into the pipeline" + } + }, + "/remove": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "text": { + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "Entry", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "seq": { + "type": "integer" + } + }, + "required": [ + "seq" + ], + "type": "object" + } + } + }, + "description": "committed atomically" + } + }, + "summary": "retract one record: every view rolls back as if it was never inserted" + } + }, + "/views/entries": { + "get": { + "parameters": [ + { + "in": "query", + "name": "limit", + "schema": { + "type": "integer" + } + }, + { + "in": "query", + "name": "offset", + "schema": { + "type": "integer" + } + }, + { + "description": "list highest-first (ordered views only)", + "in": "query", + "name": "desc", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "items": { + "properties": { + "count": { + "type": "integer" + }, + "value": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "text": { + "type": "string" + } + }, + "required": [ + "text" + ], + "title": "Entry", + "type": "object" + } + }, + "required": [ + "value", + "count" + ], + "type": "object" + }, + "type": "array" + }, + "seq": { + "type": "integer" + } + }, + "required": [ + "seq", + "data" + ], + "type": "object" + } + } + }, + "description": "one consistent snapshot" + } + }, + "summary": "list this bag view (paginated)" + } + }, + "/views/entries/watch": { + "get": { + "responses": { + "200": { + "description": "text/event-stream" + } + }, + "summary": "server-sent events: this view's fresh payload after every commit (?limit/?desc shape the read)" + } + }, + "/views/total": { + "get": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "data": { + "properties": { + "value": { + "type": "integer" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "seq": { + "type": "integer" + } + }, + "required": [ + "seq", + "data" + ], + "type": "object" + } + } + }, + "description": "one consistent snapshot" + } + }, + "summary": "read this view" + } + }, + "/views/total/watch": { + "get": { + "responses": { + "200": { + "description": "text/event-stream" + } + }, + "summary": "server-sent events: this view's fresh payload after every commit (?limit/?desc shape the read)" + } + }, + "/watch": { + "get": { + "responses": { + "200": { + "description": "text/event-stream" + } + }, + "summary": "server-sent events: one {\"seq\": n} event per commit" + } + } + } +} diff --git a/serve/tests/hammer.rs b/serve/tests/hammer.rs new file mode 100644 index 0000000..c4ad039 --- /dev/null +++ b/serve/tests/hammer.rs @@ -0,0 +1,154 @@ +//! Concurrency hammer: many writers and readers at once, plus a /watch +//! subscriber. Asserts the three properties the RwLock design promises: +//! every write gets a unique monotonic seq, no reader ever observes a torn +//! snapshot, and the watch feed is nondecreasing. + +use std::time::Duration; + +use axum::body::Body; +use axum::http::{Request, header}; +use bog_serve::{App, NoParams}; +use fold::pipeline::terminal; +use http_body_util::BodyExt; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use tokio_stream::StreamExt; +use tower::ServiceExt; + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +struct Item { + n: u64, +} + +const WRITERS: u64 = 8; +const WRITES_EACH: u64 = 25; +const TOTAL: u64 = WRITERS * WRITES_EACH; + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn writers_readers_and_watchers_agree() { + let dir = tempfile::tempdir().unwrap().keep(); + let router = App::stream( + dir, + ( + terminal::Count::new("total"), + terminal::Bag::::new("items"), + ), + ) + // the torn-snapshot detector: two sinks read in ONE rtx must agree. + // If a reader could ever interleave with a half-applied write, the + // count and the bag would diverge here. + .get("/invariant", |(count, items), _: NoParams| { + let count = count.get(); + let bag_total: i64 = items.iter().map(|(_, mult)| mult).sum(); + if count == bag_total { + Ok(json!({ "consistent": true, "count": count })) + } else { + Err((500, format!("torn snapshot: count={count} bag={bag_total}"))) + } + }) + .into_router(); + + // subscribe to /watch before any writes so the feed spans the whole run + let watch_resp = router + .clone() + .oneshot( + Request::builder() + .uri("/watch") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let mut watch_events = watch_resp.into_body().into_data_stream(); + + // writers: each inserts its own range, collecting the returned seqs + let mut tasks = Vec::new(); + for w in 0..WRITERS { + let router = router.clone(); + tasks.push(tokio::spawn(async move { + let mut seqs = Vec::new(); + for i in 0..WRITES_EACH { + let body = json!({ "n": w * WRITES_EACH + i }).to_string(); + let req = Request::builder() + .method("POST") + .uri("/insert") + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(body)) + .unwrap(); + let resp = router.clone().oneshot(req).await.unwrap(); + assert!(resp.status().is_success()); + let bytes = resp.into_body().collect().await.unwrap().to_bytes(); + let v: Value = serde_json::from_slice(&bytes).unwrap(); + seqs.push(v["seq"].as_u64().unwrap()); + } + seqs + })); + } + + // readers: hammer the invariant route the whole time + let mut readers = Vec::new(); + for _ in 0..8 { + let router = router.clone(); + readers.push(tokio::spawn(async move { + loop { + let req = Request::builder() + .uri("/invariant") + .body(Body::empty()) + .unwrap(); + let resp = router.clone().oneshot(req).await.unwrap(); + assert!( + resp.status().is_success(), + "invariant route reported a torn snapshot" + ); + let bytes = resp.into_body().collect().await.unwrap().to_bytes(); + let v: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(v["data"]["consistent"], true); + if v["data"]["count"].as_i64().unwrap() as u64 >= TOTAL { + return; + } + tokio::task::yield_now().await; + } + })); + } + + // collect writer seqs: every one unique, together exactly 1..=TOTAL + let mut all_seqs = Vec::new(); + for t in tasks { + all_seqs.extend(t.await.unwrap()); + } + all_seqs.sort_unstable(); + assert_eq!( + all_seqs, + (1..=TOTAL).collect::>(), + "seqs must be unique and gapless" + ); + + for r in readers { + tokio::time::timeout(Duration::from_secs(30), r) + .await + .expect("readers finish") + .unwrap(); + } + + // watch feed: nondecreasing seqs, reaching the final one + let mut last = 0; + loop { + let chunk = tokio::time::timeout(Duration::from_secs(10), watch_events.next()) + .await + .expect("watch event") + .unwrap() + .unwrap(); + for line in String::from_utf8_lossy(&chunk).lines() { + if let Some(data) = line.strip_prefix("data: ") { + let v: Value = serde_json::from_str(data).unwrap(); + let seq = v["seq"].as_u64().unwrap(); + assert!(seq >= last, "watch went backwards: {last} -> {seq}"); + last = seq; + } + } + if last >= TOTAL { + break; + } + } +} diff --git a/serve/tests/http.rs b/serve/tests/http.rs new file mode 100644 index 0000000..8c8257f --- /dev/null +++ b/serve/tests/http.rs @@ -0,0 +1,176 @@ +//! Drive every generated route through the router in-process: tower's +//! `oneshot` sends one request through the service without a listener. + +use axum::http::StatusCode; +use bog_serve::App; +use fold::pipeline::{KeyBy, terminal}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +mod common; +use common::send; + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +struct Entry { + id: u64, + text: String, +} + +/// Every phase-1 sink kind, plus an operator (KeyBy) to prove readers pass +/// through operators untouched. +fn test_router() -> axum::Router { + let dir = tempfile::tempdir().unwrap().keep(); + App::stream( + dir, + ( + terminal::Count::new("total"), + terminal::Bag::::new("entries"), + KeyBy::new(|e: &Entry| e.id, terminal::Table::new("by_id")), + ), + ) + .into_router() +} + +fn entry(id: u64, text: &str) -> Value { + json!({ "id": id, "text": text }) +} + +#[tokio::test] +async fn healthz() { + let router = test_router(); + let (status, body) = send(&router, "GET", "/healthz", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, Value::String("ok".into())); +} + +#[tokio::test] +async fn writes_flow_to_every_view() { + let router = test_router(); + + let (status, body) = send(&router, "POST", "/insert", Some(entry(1, "peat"))).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["seq"], 1); + send(&router, "POST", "/insert", Some(entry(2, "moss"))).await; + let (_, body) = send(&router, "POST", "/remove", Some(entry(1, "peat"))).await; + assert_eq!(body["seq"], 3); + + // count: retraction rolled entry 1 back out + let (status, body) = send(&router, "GET", "/views/total", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["seq"], 3); + assert_eq!(body["data"]["value"], 1); + + // bag: only entry 2 remains + let (_, body) = send(&router, "GET", "/views/entries", None).await; + let items = body["data"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["value"]["text"], "moss"); + assert_eq!(items[0]["count"], 1); + + // table through the KeyBy operator: point lookup by key + let (status, body) = send(&router, "GET", "/views/by_id/2", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["data"]["value"]["text"], "moss"); +} + +#[tokio::test] +async fn batch_commits_atomically_or_rejects_wholly() { + let router = test_router(); + + let ops = json!([ + { "op": "insert", "data": entry(1, "a") }, + { "op": "insert", "data": entry(2, "b") }, + { "op": "remove", "data": entry(1, "a") }, + ]); + let (status, body) = send(&router, "POST", "/batch", Some(ops)).await; + assert_eq!(status, StatusCode::OK); + // three ops, ONE transaction, one seq + assert_eq!(body["seq"], 1); + assert_eq!(body["applied"], 3); + let (_, body) = send(&router, "GET", "/views/total", None).await; + assert_eq!(body["data"]["value"], 1); + + // a malformed op anywhere rejects the whole batch before any write + let bad = json!([ + { "op": "insert", "data": entry(3, "c") }, + { "op": "bogus" }, + ]); + let (status, _) = send(&router, "POST", "/batch", Some(bad)).await; + assert!(status.is_client_error()); + let (_, body) = send(&router, "GET", "/views/total", None).await; + assert_eq!( + body["data"]["value"], 1, + "rejected batch must write nothing" + ); + assert_eq!(body["seq"], 1, "rejected batch must not commit"); +} + +#[tokio::test] +async fn read_errors() { + let router = test_router(); + send(&router, "POST", "/insert", Some(entry(1, "a"))).await; + + let (status, _) = send(&router, "GET", "/views/nope", None).await; + assert_eq!(status, StatusCode::NOT_FOUND, "unknown view"); + + let (status, _) = send(&router, "GET", "/views/by_id/999", None).await; + assert_eq!(status, StatusCode::NOT_FOUND, "known view, absent key"); + + let (status, _) = send(&router, "GET", "/views/by_id/abc", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "unparseable key for u64"); + + let (status, _) = send(&router, "GET", "/views/entries/1", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "bags have no key lookup"); +} + +#[tokio::test] +async fn pagination() { + let router = test_router(); + for id in 0..5 { + send(&router, "POST", "/insert", Some(entry(id, "x"))).await; + } + + let (_, body) = send(&router, "GET", "/views/entries?limit=2", None).await; + assert_eq!(body["data"].as_array().unwrap().len(), 2); + + let (_, body) = send(&router, "GET", "/views/entries?limit=10&offset=4", None).await; + assert_eq!(body["data"].as_array().unwrap().len(), 1); +} + +#[tokio::test] +async fn openapi_reflects_the_pipeline() { + let router = test_router(); + let (status, doc) = send(&router, "GET", "/openapi.json", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(doc["openapi"], "3.1.0"); + + let paths = doc["paths"].as_object().unwrap(); + for p in [ + "/insert", + "/remove", + "/batch", + "/views/total", + "/views/entries", + "/views/by_id", + "/views/by_id/{key}", + "/healthz", + ] { + assert!(paths.contains_key(p), "missing path {p}"); + } + + // the write body documents the actual input type + let input = + &doc["paths"]["/insert"]["post"]["requestBody"]["content"]["application/json"]["schema"]; + assert!(input["properties"]["id"].is_object()); + assert!(input["properties"]["text"].is_object()); +} + +#[tokio::test] +async fn schema_fingerprint_is_stable_across_builds() { + let (_, a) = send(&test_router(), "GET", "/schema", None).await; + let (_, b) = send(&test_router(), "GET", "/schema", None).await; + assert!(a["fingerprint"].as_str().unwrap().len() == 16); + assert_eq!(a["fingerprint"], b["fingerprint"]); + assert_eq!(a["views"].as_array().unwrap().len(), 3); +} diff --git a/serve/tests/keyed_http.rs b/serve/tests/keyed_http.rs new file mode 100644 index 0000000..4587ca5 --- /dev/null +++ b/serve/tests/keyed_http.rs @@ -0,0 +1,363 @@ +//! Keyed apps end to end: CRUD by key, search three ways, custom routes, +//! and the phase-2 exit criterion — forgetting a document over HTTP +//! removes it from every index. + +use anny::metric::Cosine; +use axum::body::Body; +use axum::http::{Request, StatusCode, header}; +use bog_serve::{KeyedApp, TextQuery}; +use fold::pipeline::{Keyed, Map, terminal}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use tokio_stream::StreamExt; +use tower::ServiceExt; + +mod common; +use common::send; + +#[derive(Deserialize, JsonSchema)] +struct TopDocParams { + q: String, +} + +#[derive(Serialize, JsonSchema)] +struct TopDoc { + id: u64, + text: Option, +} + +#[derive(Deserialize, JsonSchema)] +struct Claim { + key: u64, + text: String, +} + +/// Deterministic toy embedder standing in for ese: 4 dims, char-bucket +/// counts, L2-normalized so cosine distances behave. +fn embed(s: &str) -> [f32; 4] { + let mut v = [0.0f32; 4]; + for (i, b) in s.bytes().enumerate() { + v[i % 4] += b as f32; + } + let norm = v.iter().map(|x| x * x).sum::().sqrt().max(1e-6); + v.map(|x| x / norm) +} + +/// The dogfood shape: BM25 + HNSW (text-queryable) + doc table, all fed by +/// one keyed stream of `id -> text`. +fn test_router() -> axum::Router { + let dir = tempfile::tempdir().unwrap().keep(); + KeyedApp::stream( + dir, + ( + terminal::search::Bm25::new("bm25"), + Map::new( + |d: &Keyed| Keyed::new(d.key, embed(&d.val)), + TextQuery::new( + terminal::search::Hnsw::::new("vecs", Cosine, 42), + embed, + ), + ), + terminal::Table::new("docs"), + ), + ) + // typed custom GET: query struct in, response struct out — both + // schemas land in /openapi.json + .get("/top_doc", |(bm25, _vecs, docs), p: TopDocParams| { + Ok(bm25.search(&p.q, 1).into_iter().next().map(|hit| TopDoc { + id: hit.val, + text: docs.get(&hit.val), + })) + }) + // typed custom POST: atomic check-and-set — Err rolls the whole + // transaction back, so a taken key is never overwritten + .post("/claim", |tx, c: Claim| { + if tx.contains(&c.key) { + return Err((409, format!("key {} taken", c.key))); + } + tx.upsert(&c.key, &c.text); + Ok(json!({ "claimed": c.key })) + }) + .into_router() +} + +async fn seed(router: &axum::Router) { + for (id, text) in [ + (1, "the postgres database was slow"), + (2, "deployed the api to kubernetes"), + (3, "the user prefers rust for backends"), + ] { + let (status, _) = send(router, "PUT", &format!("/docs/{id}"), Some(json!(text))).await; + assert_eq!(status, StatusCode::OK); + } +} + +#[tokio::test] +async fn crud_by_key() { + let router = test_router(); + + let (status, body) = send(&router, "PUT", "/docs/1", Some(json!("hello bog"))).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["replaced"], false); + + let (_, body) = send(&router, "GET", "/docs/1", None).await; + assert_eq!(body["data"], "hello bog"); + + // upsert replaces and reports it + let (_, body) = send(&router, "PUT", "/docs/1", Some(json!("hello again"))).await; + assert_eq!(body["replaced"], true); + let (_, body) = send(&router, "GET", "/views/docs/1", None).await; + assert_eq!(body["data"]["value"], "hello again"); + + let (_, body) = send(&router, "DELETE", "/docs/1", None).await; + assert_eq!(body["removed"], true); + let (status, _) = send(&router, "GET", "/docs/1", None).await; + assert_eq!(status, StatusCode::NOT_FOUND); + + // deleting an absent key commits but reports removed: false + let (_, body) = send(&router, "DELETE", "/docs/99", None).await; + assert_eq!(body["removed"], false); +} + +#[tokio::test] +async fn search_three_ways() { + let router = test_router(); + seed(&router).await; + + // bm25: GET ?q= + let (status, body) = send(&router, "GET", "/views/bm25/search?q=kubernetes", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["data"][0]["key"], 2); + + // hnsw by text (via the TextQuery encoder) + let (status, body) = send( + &router, + "GET", + "/views/vecs/search?q=deployed%20the%20api%20to%20kubernetes&k=1", + None, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["data"][0]["key"], 2, "self-similarity must win"); + + // hnsw by raw vector (POST) + let vector: Vec = embed("deployed the api to kubernetes").to_vec(); + let (status, body) = send( + &router, + "POST", + "/views/vecs/search", + Some(json!({ "vector": vector, "k": 1 })), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["data"][0]["key"], 2); + + // wrong dimensionality is a 400, not a panic + let (status, body) = send( + &router, + "POST", + "/views/vecs/search", + Some(json!({ "vector": [1.0, 2.0] })), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body["error"].as_str().unwrap().contains("4 dims")); + + // non-searchable views say so + let (status, _) = send(&router, "GET", "/views/docs/search?q=x", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn forgetting_removes_from_every_index() { + let router = test_router(); + seed(&router).await; + + // present everywhere before + let (_, body) = send(&router, "GET", "/views/bm25/search?q=kubernetes", None).await; + assert_eq!(body["data"][0]["key"], 2); + + let (_, body) = send(&router, "DELETE", "/docs/2", None).await; + assert_eq!(body["removed"], true); + + // bm25: no hit for its terms + let (_, body) = send(&router, "GET", "/views/bm25/search?q=kubernetes", None).await; + assert!( + body["data"].as_array().unwrap().is_empty(), + "bm25 must forget: {body}" + ); + + // hnsw: doc 2 gone from the graph (its own text no longer finds it) + let (_, body) = send( + &router, + "GET", + "/views/vecs/search?q=deployed%20the%20api%20to%20kubernetes", + None, + ) + .await; + let keys: Vec = body["data"] + .as_array() + .unwrap() + .iter() + .map(|h| h["key"].as_u64().unwrap()) + .collect(); + assert!(!keys.contains(&2), "hnsw must forget: {keys:?}"); + + // table and primary store: gone + let (status, _) = send(&router, "GET", "/views/docs/2", None).await; + assert_eq!(status, StatusCode::NOT_FOUND); + let (status, _) = send(&router, "GET", "/docs/2", None).await; + assert_eq!(status, StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn keyed_batch_is_one_transaction() { + let router = test_router(); + seed(&router).await; + + let ops = json!([ + { "op": "upsert", "key": 4, "data": "a brand new memory" }, + { "op": "remove", "key": 1 }, + ]); + let (status, body) = send(&router, "POST", "/batch", Some(ops)).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["applied"], 2); + let seq = body["seq"].as_u64().unwrap(); + + let (_, body) = send(&router, "GET", "/docs/4", None).await; + assert_eq!(body["data"], "a brand new memory"); + assert_eq!(body["seq"].as_u64().unwrap(), seq, "one tx, one seq"); + let (status, _) = send(&router, "GET", "/docs/1", None).await; + assert_eq!(status, StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn custom_route_reads_the_same_snapshot() { + let router = test_router(); + seed(&router).await; + + let (status, body) = send(&router, "GET", "/top_doc?q=kubernetes", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["data"]["id"], 2); + assert_eq!(body["data"]["text"], "deployed the api to kubernetes"); + + // missing required param: rejected by the typed extractor, field named + let (status, body) = send(&router, "GET", "/top_doc", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body["error"].as_str().unwrap().contains("q")); +} + +#[tokio::test] +async fn custom_post_is_atomic_check_and_set() { + let router = test_router(); + seed(&router).await; // seq 3 + + let claim = json!({ "key": 9, "text": "the deploy runs at midnight" }); + let (status, body) = send(&router, "POST", "/claim", Some(claim)).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["data"]["claimed"], 9); + assert_eq!(body["seq"], 4, "successful claim commits"); + + // second claim: refused, and the whole transaction rolled back + let steal = json!({ "key": 9, "text": "overwritten!" }); + let (status, body) = send(&router, "POST", "/claim", Some(steal)).await; + assert_eq!(status, StatusCode::CONFLICT); + assert!(body["error"].as_str().unwrap().contains("taken")); + + let (_, body) = send(&router, "GET", "/docs/9", None).await; + assert_eq!( + body["data"], "the deploy runs at midnight", + "original survives" + ); + assert_eq!(body["seq"], 4, "rejected claim must not commit a seq"); + + // the rolled-back text was never indexed anywhere + let (_, body) = send(&router, "GET", "/views/bm25/search?q=overwritten", None).await; + assert!(body["data"].as_array().unwrap().is_empty()); + + // malformed body: rejected before any transaction, field named + let (status, body) = send(&router, "POST", "/claim", Some(json!({ "key": 10 }))).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body["error"].as_str().unwrap().contains("text")); +} + +#[tokio::test] +async fn watch_emits_commit_events() { + let router = test_router(); + seed(&router).await; // seq is now 3 + + let req = Request::builder() + .uri("/watch") + .body(Body::empty()) + .unwrap(); + let resp = router.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert!( + resp.headers()[header::CONTENT_TYPE] + .to_str() + .unwrap() + .starts_with("text/event-stream") + ); + + // the current seq arrives immediately on connect + let mut body = resp.into_body().into_data_stream(); + let first = tokio::time::timeout(std::time::Duration::from_secs(5), body.next()) + .await + .expect("an event within 5s") + .unwrap() + .unwrap(); + let text = String::from_utf8_lossy(&first); + assert!(text.contains("{\"seq\":3}"), "got: {text}"); +} + +#[tokio::test] +async fn keyed_openapi_and_schema() { + let router = test_router(); + + let (_, doc) = send(&router, "GET", "/openapi.json", None).await; + let paths = doc["paths"].as_object().unwrap(); + for p in [ + "/docs/{key}", + "/batch", + "/views/bm25/search", + "/views/vecs/search", + "/views/docs/{key}", + "/top_doc", + "/claim", + "/watch", + ] { + assert!(paths.contains_key(p), "missing path {p}"); + } + // vecs is text+vector: both operations documented + assert!(paths["/views/vecs/search"].get("get").is_some()); + assert!(paths["/views/vecs/search"].get("post").is_some()); + // bm25 is text-only + assert!(paths["/views/bm25/search"].get("post").is_none()); + + // custom routes are fully typed in the doc: the GET documents its + // query params, the POST its body and response schemas + let top_doc = &paths["/top_doc"]["get"]; + assert_eq!(top_doc["parameters"][0]["name"], "q"); + assert_eq!(top_doc["parameters"][0]["required"], true); + let claim_body = + &paths["/claim"]["post"]["requestBody"]["content"]["application/json"]["schema"]; + assert!(claim_body["properties"]["key"].is_object()); + assert!(claim_body["properties"]["text"].is_object()); + let claim_resp = + &paths["/claim"]["post"]["responses"]["200"]["content"]["application/json"]["schema"]; + assert_eq!(claim_resp["properties"]["seq"]["type"], "integer"); + + let (_, schema) = send(&router, "GET", "/schema", None).await; + assert_eq!( + schema["write"]["keyed"]["type"], "integer", + "u64 key schema" + ); + let kinds: Vec<&str> = schema["views"] + .as_array() + .unwrap() + .iter() + .map(|v| v["kind"].as_str().unwrap()) + .collect(); + assert_eq!(kinds, ["bm25", "hnsw", "table"]); +}