basic cli and local http server setup

.github/workflows/cli.yml

@@ -0,0 +1,38 @@
+name: cli
+
+on:
+  pull_request:
+    paths: ["cli/**", "serve/**", "fold/**", "Cargo.toml", ".github/workflows/cli.yml"]
+  push:
+    branches: [main]
+    paths: ["cli/**", "serve/**", "fold/**", "Cargo.toml", ".github/workflows/cli.yml"]
+
+concurrency:
+  group: cli-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  scaffold:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Build the CLI
+        run: cargo build -p bogkit
+
+      - name: bog-serve tests
+        run: cargo test -p bog-serve
+
+      - name: Golden OpenAPI docs validate against the spec
+        run: pipx run openapi-spec-validator serve/tests/golden/template.json serve/tests/golden/search.json
+
+      - name: Scaffold an embedded project
+        run: cargo run -p bogkit -- new ci-embedded --kind embedded
+
+      - name: Scaffold a server project
+        run: cargo run -p bogkit -- new ci-server --kind server
+
+      - name: Scaffolded projects compile
+        run: cargo check -p ci-embedded -p ci-server

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "3"
-members = ["fold", "anny", "ese", "examples/*"]
+members = ["fold", "anny", "ese", "cli", "serve", "examples/*"]
 
 # hoisted from fold; applies workspace-wide
 [profile.release]

README.md

@@ -2,18 +2,29 @@
 
 This repo contains some of the tooling we've been working on for building Bog style databases. We've collected these tools and examples in one cargo workspace, so you can start building immediately. 
 
-The best way to create your project is to run this terminal command in the root of this repo:
+The best way to create your project is the `bogkit` CLI, from the root of this repo:
 
 ```console
-$ ./scripts/new-project.sh [project-name]
-``` 
+$ cargo run -p bogkit -- new [project-name]
+```
+
+This creates a new binary crate in `examples/[project-name]` with local path dependencies on the workspace crates. Two flavors:
 
-This creates a new binary crate in `examples/[project-name]`, wires it into the workspace, and adds local path dependencies on `fold`, `anny`, and `ese` (though you may not necessarily use all of these).
+- `--kind server` (the default) — a fold pipeline served over HTTP by `bog-serve`: writes, reads, search, and a live OpenAPI doc, all generated from the pipeline itself.
+- `--kind embedded` — a plain Rust binary using fold directly (what `scripts/new-project.sh` used to produce; the script still works).
 
 Run your project with:
 
 ```console
-$ cargo run -p [project-name]
+$ cargo run -p bogkit -- dev -p [project-name]
+```
+
+`dev` wraps `cargo run` with the bogkit conventions: a stable data dir in `~/.bogkit/data/[project-name]` (pass `--fresh` to wipe it) and `$PORT` (default 7877). For server projects, explore the API with:
+
+```console
+$ cargo run -p bogkit -- api          # pretty-prints the running server's /openapi.json
+$ curl localhost:7877/views/total
+$ curl -N localhost:7877/watch        # server-sent events, one per commit
 ```
 
 ## Documentation
@@ -45,13 +56,20 @@ ESE, our first take on a compiler oriented approach to static embedding. It’s
 ### Approximate Nearest Neighbors... yeah (ANNy)
 This is a very fast crate for creating HNSWs.
 
+### bog-serve
+Serve any fold pipeline over HTTP with the API generated from the pipeline itself: the input type describes the write routes (via schemars), the named terminal sinks describe the read routes, and the OpenAPI doc is assembled from the same values the router dispatches with — so it can't drift. Atomic batches, hybrid search, SSE watch streams, and custom routes included. See `serve/` and the crate rustdocs (`cargo doc --open -p bog-serve`).
+
+### bogkit CLI
+Scaffolding and a dev runner for BogKit projects (`cli/`): `bogkit new`, `bogkit dev`, `bogkit api`.
+
 ### Examples
 In this directory you'll find a few examples that show bog style databases in various use cases.
 
 - `starter` — the smallest possible fold database: a persistent count and bag, with inserts, reads, and retraction. `cargo run -p starter`
 - `timeseries` — weather readings bucketed into hourly and daily aggregates, updated incrementally. `cargo run -p timeseries`
 - `chat` — a chat backend where fold is the source of truth and every update is broadcast to clients over a websocket. `cargo run -p chat`, then open http://localhost:3000
 - `search` — text search three ways over one document stream: BM25 keyword search, HNSW semantic search over ese embeddings, and hybrid rank fusion. A good base for agent memory or document search projects. `cargo run -p search`
+- `search-server` — the search example served over HTTP by bog-serve: the same pipeline plus generated CRUD/search routes, a custom `/search/hybrid` fusion endpoint, and a live OpenAPI doc. `cargo run -p search-server`, then `curl localhost:7877/openapi.json`
 
 ## More about Bog
 Bog is a database runtime that makes every attempt to do as much work as possible as early as possible, to make reads incredibly fast. This means compiling queries into functions that eagerly update their output as mutations occur.

cli/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "bogkit"
+version = "0.0.0"
+edition = "2024"
+publish = false
+
+[dependencies]
+anyhow = "1"
+clap = { version = "4", features = ["derive"] }
+serde_json = "1"
+ureq = { version = "2", features = ["json"] }

cli/src/api.rs

@@ -0,0 +1,19 @@
+use anyhow::Context;
+
+/// Fetch and pretty-print a running server's OpenAPI document.
+pub fn run(port: Option<u16>) -> anyhow::Result<()> {
+    // same resolution order as the server itself: flag, $PORT, default
+    let port = port
+        .or_else(|| std::env::var("PORT").ok().and_then(|p| p.parse().ok()))
+        .unwrap_or(7877);
+    let url = format!("http://localhost:{port}/openapi.json");
+
+    let doc: serde_json::Value = ureq::get(&url)
+        .call()
+        .with_context(|| format!("fetching {url} — is the server running? (`bogkit dev`)"))?
+        .into_json()
+        .context("parsing the OpenAPI document")?;
+
+    println!("{}", serde_json::to_string_pretty(&doc)?);
+    Ok(())
+}

cli/src/dev.rs

@@ -0,0 +1,55 @@
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+use anyhow::{Context, bail};
+
+use crate::workspace;
+
+pub fn run(project: Option<&str>, fresh: bool) -> anyhow::Result<()> {
+    let root = workspace::root()?;
+    let name = match project {
+        Some(name) => name.to_string(),
+        None => infer_project(&root)?,
+    };
+
+    let data_dir = data_dir(&name)?;
+    if fresh && data_dir.exists() {
+        std::fs::remove_dir_all(&data_dir).context("wiping data dir for --fresh")?;
+    }
+    std::fs::create_dir_all(&data_dir).context("creating data dir")?;
+    println!("data dir: {}", data_dir.display());
+
+    // Inherit stdio so the project owns the terminal (the search example's
+    // interactive loop, server logs, ...). We only collect the exit status.
+    let status = Command::new("cargo")
+        .args(["run", "-p", &name])
+        .current_dir(&root)
+        .env("BOG_DATA_DIR", &data_dir)
+        .env("PORT", std::env::var("PORT").unwrap_or_else(|_| "7877".into()))
+        .status()
+        .context("running cargo")?;
+
+    if !status.success() {
+        // mirror the project's exit code so `bogkit dev` composes in scripts
+        std::process::exit(status.code().unwrap_or(1));
+    }
+    Ok(())
+}
+
+/// When run from inside examples/<name>, that project is the target —
+/// scaffolded crates are named after their directory.
+fn infer_project(root: &Path) -> anyhow::Result<String> {
+    let cwd = std::env::current_dir()?;
+    let rel = cwd.strip_prefix(root.join("examples")).ok();
+    match rel.and_then(|rel| rel.iter().next()) {
+        Some(first) => Ok(first.to_string_lossy().into_owned()),
+        None => bail!("run from inside examples/<project> or pass -p <project>"),
+    }
+}
+
+/// Stable per-project data dir: ~/.bogkit/data/<name>. Persistent across
+/// runs by default; `--fresh` wipes it.
+fn data_dir(name: &str) -> anyhow::Result<PathBuf> {
+    let home = std::env::home_dir().context("cannot determine home directory")?;
+    Ok(home.join(".bogkit").join("data").join(name))
+}