From f2193f730f96206687ff1370b821f0acfe887010 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 9 Nov 2026 04:21:50 +0000 Subject: [PATCH 1/7] video + gameserver: finish both plugins, mark available video: pin the published runner by digest (RUNNER_PULL_REF), run it under a local name, tag :stable in the workflow, label the package with its source, and check anonymous pulls in CI. `video setup` (now in core) pulls the pinned digest or builds from source. C0MPUTE_DOCKER overrides the container CLI. Speech gateway exposes router() so POST /v1/videos is tested end to end on loopback with a fake runner that writes a real MP4. gameserver (spec v2): secrets never ride in the offer (sealed handoff to the winning worker); backups every 30 min and at stop, encrypted to the buyer's X25519 key; signed status records on c0mpute/gameserver/v1; `gameserver watch` probes the port (Minecraft status, Valheim A2S, Factorio UDP + status), mirrors backups and fails over to another worker from the latest one; hard expiry via timeout + --init + --rm plus a boot/sweep watchdog. Runtime trait lets the loopback e2e run without docker. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/video-runner.yml | 45 +- apps/web/src/app/plugins/page.tsx | 6 + apps/web/src/lib/plugins.ts | 2 + node/crates/c0mpute-cli/src/gameserver.rs | 80 +- node/crates/c0mpute-cli/src/main.rs | 85 +- node/crates/c0mpute-core/src/container.rs | 57 + node/crates/c0mpute-core/src/gameserver.rs | 1650 ++++++++++++++--- node/crates/c0mpute-core/src/handoff.rs | 251 +++ node/crates/c0mpute-core/src/lib.rs | 2 + .../crates/c0mpute-core/src/speech_gateway.rs | 43 +- node/crates/c0mpute-core/src/video.rs | 120 +- .../c0mpute-core/tests/fixtures/tiny.mp4 | Bin 0 -> 1542 bytes .../c0mpute-core/tests/gameserver_e2e.rs | 396 ++++ node/crates/c0mpute-core/tests/video_e2e.rs | 279 +++ plugins/gameserver/module.toml | 59 +- plugins/video/install.sh | 12 +- plugins/video/module.toml | 21 +- plugins/video/runner/Dockerfile | 2 +- plugins/video/runner/README.md | 8 +- 19 files changed, 2755 insertions(+), 363 deletions(-) create mode 100644 node/crates/c0mpute-core/src/container.rs create mode 100644 node/crates/c0mpute-core/src/handoff.rs create mode 100644 node/crates/c0mpute-core/tests/fixtures/tiny.mp4 create mode 100644 node/crates/c0mpute-core/tests/gameserver_e2e.rs create mode 100644 node/crates/c0mpute-core/tests/video_e2e.rs diff --git a/.github/workflows/video-runner.yml b/.github/workflows/video-runner.yml index 40e1aff4..76a8b244 100644 --- a/.github/workflows/video-runner.yml +++ b/.github/workflows/video-runner.yml @@ -6,8 +6,19 @@ # torch image is several GB and not worth building to learn that a validation # rule changed. # -# The image the node pulls is the tag in plugins/video/module.toml under -# [module.dispatch]; keep the two in step. +# Tags on every master build: :stable (what operators pull by hand), +# :candidate (kept for older nodes) and :. Nodes pull by DIGEST, the one +# pinned in plugins/video/module.toml [module.dispatch] and RUNNER_DIGEST in +# node/crates/c0mpute-core/src/video.rs (a unit test keeps those two equal). +# The job summary prints the new digest; bump both pins to it after a runner +# change. +# +# Visibility: the package must be PUBLIC for outside operators to pull it. +# GHCR creates org packages private and has no API to change that, so it is +# one manual click by an org admin: github.com/orgs/profullstack/packages -> +# c0mpute-video-runner -> Package settings -> Change visibility -> Public. +# The last step below checks an anonymous pull and warns until it works. +# Until then `c0mpute video setup` builds the same image from source. name: video runner on: @@ -81,15 +92,21 @@ jobs: EXTRA: ${{ inputs.tag }} run: | set -eu - tags="${IMAGE}:candidate,${IMAGE}:${GITHUB_SHA}" + tags="${IMAGE}:stable,${IMAGE}:candidate,${IMAGE}:${GITHUB_SHA}" if [ -n "${EXTRA:-}" ]; then tags="${tags},${IMAGE}:${EXTRA}" fi echo "tags=${tags}" >> "$GITHUB_OUTPUT" - name: Build and push + id: push uses: docker/build-push-action@v6 with: + # Links the package to this public repo (and its README/license). + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.description=c0mpute video runner (Wan 2.2, LTX-2.3) + org.opencontainers.image.licenses=MIT # The context is plugins/video so module.toml — the runner's model # registry — is part of the image. context: plugins/video @@ -103,3 +120,25 @@ jobs: # Proves the thing we just pushed can parse the manifest, encode and # probe. Needs no GPU, so a hosted runner can do it. run: docker run --rm "${IMAGE}:${GITHUB_SHA}" selftest + + - name: Digest to pin, and can a stranger pull it? + env: + DIGEST: ${{ steps.push.outputs.digest }} + run: | + set -u + { + echo "### video runner published" + echo "" + echo "Pin this in plugins/video/module.toml and RUNNER_DIGEST in video.rs:" + echo "" + echo " ${IMAGE}@${DIGEST}" + } >> "$GITHUB_STEP_SUMMARY" + # Anonymous pull: a fresh docker config with no credentials. + export DOCKER_CONFIG="$(mktemp -d)" + if docker manifest inspect "${IMAGE}@${DIGEST}" >/dev/null 2>&1; then + echo "Anonymous pull: OK (package is public)." >> "$GITHUB_STEP_SUMMARY" + else + msg="ghcr.io package c0mpute-video-runner is not publicly pullable. An org admin must set it Public once: github.com/orgs/profullstack/packages -> c0mpute-video-runner -> Package settings -> Change visibility. Until then nodes build the runner from source." + echo "::warning::${msg}" + echo "Anonymous pull: FAILED. ${msg}" >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/apps/web/src/app/plugins/page.tsx b/apps/web/src/app/plugins/page.tsx index 07369c32..1dc50997 100644 --- a/apps/web/src/app/plugins/page.tsx +++ b/apps/web/src/app/plugins/page.tsx @@ -186,6 +186,12 @@ function PluginCard({ p }: { p: PluginManifest }) { + {typeof p.requirements?.hardware === "string" && ( +

+ requires: {p.requirements.hardware} +

+ )} + {p.status === "partial" && p.statusNote && (

partial: {p.statusNote} diff --git a/apps/web/src/lib/plugins.ts b/apps/web/src/lib/plugins.ts index be09bf39..1fc20167 100644 --- a/apps/web/src/lib/plugins.ts +++ b/apps/web/src/lib/plugins.ts @@ -113,6 +113,8 @@ export interface PluginManifest { os?: string[]; arch?: string[]; capabilities?: string[]; + /** What a node needs to serve it, in words (GPU, docker, public IP). */ + hardware?: string; }; workloads?: Record; models?: Record; diff --git a/node/crates/c0mpute-cli/src/gameserver.rs b/node/crates/c0mpute-cli/src/gameserver.rs index 51d7deb0..0bdcc76f 100644 --- a/node/crates/c0mpute-cli/src/gameserver.rs +++ b/node/crates/c0mpute-cli/src/gameserver.rs @@ -34,12 +34,16 @@ enum Sub { /// Restore a world: the hash `gameserver stop` printed, or a .tar of the game's data dir. #[arg(long)] world: Option, - /// Valheim server password (5-32 letters/digits; peers on the job topic can see it). + /// Valheim server password (5-32 letters/digits). Sent sealed to the + /// winning worker after the auction, never in the public offer. #[arg(long)] password: Option, /// Server name / MOTD. #[arg(long)] name: Option, + /// Minutes between live world backups, sealed to your key (0 = only at stop/expiry). + #[arg(long, default_value_t = gameserver::DEFAULT_BACKUP_MINS)] + backup_mins: u32, /// Most you pay for the whole reservation, USD. #[arg(long, default_value_t = 1.0)] max_price: f64, @@ -56,6 +60,32 @@ enum Sub { #[arg(long)] json: bool, }, + /// Watch a server: probe its port, mirror its backups, and if it stays + /// down for --fail-after-secs, start the same world on another worker. + Watch { + id: String, + #[arg(long, default_value_t = 30)] + probe_every_secs: u64, + #[arg(long, default_value_t = 300)] + fail_after_secs: u64, + /// Only probe and mirror backups; never start a replacement. + #[arg(long)] + no_failover: bool, + /// Most you pay for a replacement, USD. + #[arg(long, default_value_t = 1.0)] + max_price: f64, + /// Only accept replacement bids from these worker peer ids. Repeatable. + #[arg(long)] + worker: Vec, + }, + /// Show a server's probe log (what `watch` recorded). + Health { + id: String, + #[arg(long, default_value_t = 20)] + last: usize, + #[arg(long)] + json: bool, + }, /// Stop a server early (or collect an expired one's backup) and download its world. Stop { id: String, @@ -85,7 +115,7 @@ fn left(expires_at: u64) -> String { pub async fn run(args: &[String]) -> Result<()> { let cli = GameserverCli::try_parse_from(std::iter::once("c0mpute gameserver".to_string()).chain(args.iter().cloned())).unwrap_or_else(|e| e.exit()); match cli.cmd { - Sub::Start { game, hours, ram, cpus, accept_eula, world, password, name, max_price, worker, timeout_secs, json } => { + Sub::Start { game, hours, ram, cpus, accept_eula, world, password, name, backup_mins, max_price, worker, timeout_secs, json } => { let game = Game::parse(&game).with_context(|| format!("unknown game {game:?}: minecraft, factorio or valheim"))?; let req = StartRequest { game, @@ -96,8 +126,10 @@ pub async fn run(args: &[String]) -> Result<()> { world, password, server_name: name, + backup_mins, max_price_usd: max_price, workers: worker, + exclude: Vec::new(), timeout: Duration::from_secs(timeout_secs.clamp(60, 3600)), }; eprintln!("looking for a worker that hosts {} ...", game.name()); @@ -107,6 +139,7 @@ pub async fn run(args: &[String]) -> Result<()> { } else { println!("{} {} is up at {}", r.game.name(), r.id, r.address); println!(" worker {} ${} stops in {}", r.worker, r.price_usd, left(r.expires_at)); + println!(" keep it healthy (probe, mirror backups, fail over): c0mpute gameserver watch {}", r.id); println!(" stop early (and keep the world): c0mpute gameserver stop {}", r.id); } Ok(()) @@ -121,15 +154,50 @@ pub async fn run(args: &[String]) -> Result<()> { println!("no servers rented from this machine"); } for r in all { - let status = match (r.stopped_at, &r.backup) { - (Some(_), Some(b)) => format!("stopped world {b}"), - (Some(_), None) => "stopped".into(), - (None, _) => left(r.expires_at), + let status = match (r.stopped_at, &r.backup, &r.superseded_by) { + (_, _, Some(next)) => format!("failed over to {next}"), + (Some(_), Some(b), _) => format!("stopped world {b}"), + (Some(_), None, _) => "stopped".into(), + (None, _, _) => match &r.latest_backup { + Some(b) => format!("{} backup {}", left(r.expires_at), &b[..12.min(b.len())]), + None => left(r.expires_at), + }, }; println!("{:20} {:10} {:28} {}", r.id, r.game.name(), r.address, status); } Ok(()) } + Sub::Watch { id, probe_every_secs, fail_after_secs, no_failover, max_price, worker } => { + let opts = gameserver::WatchOptions { + probe_every: Duration::from_secs(probe_every_secs.clamp(5, 3600)), + fail_after: Duration::from_secs(fail_after_secs.clamp(30, 86_400)), + failover: !no_failover, + max_price_usd: max_price, + workers: worker, + ..Default::default() + }; + eprintln!("watching {id} (probe every {}s, fail over after {}s down)", opts.probe_every.as_secs(), opts.fail_after.as_secs()); + match gameserver::watch(&id, opts).await? { + gameserver::WatchEnd::Ended(r) => println!("{} ended (reservation over or stopped)", r.id), + gameserver::WatchEnd::FailedOver { from, to } => println!("{} failed over to {} at {}", from.id, to.id, to.address), + } + Ok(()) + } + Sub::Health { id, last, json } => { + let log = gameserver::health_log(&gameserver::buyer_dir(), &id); + let tail = &log[log.len().saturating_sub(last)..]; + if json { + println!("{}", serde_json::to_string(tail)?); + return Ok(()); + } + if tail.is_empty() { + println!("no probes recorded for {id} (run `c0mpute gameserver watch {id}`)"); + } + for p in tail { + println!("{} {} {:16} {:>5}ms {}", p.ts, if p.ok { "up " } else { "DOWN" }, p.method, p.rtt_ms, p.detail); + } + Ok(()) + } Sub::Stop { id, timeout_secs, json } => { let r = gameserver::stop(&id, Duration::from_secs(timeout_secs.clamp(30, 3600))).await?; if json { diff --git a/node/crates/c0mpute-cli/src/main.rs b/node/crates/c0mpute-cli/src/main.rs index 0395e71f..7332ecce 100644 --- a/node/crates/c0mpute-cli/src/main.rs +++ b/node/crates/c0mpute-cli/src/main.rs @@ -2345,62 +2345,6 @@ async fn run_stt(cmd: SttCmd) -> Result<()> { } } -/// Build the runner image from a shallow clone of the public repo. -/// -/// The fallback for `c0mpute video setup` when the registry will not serve -/// the image. Slower than a pull (it downloads the Python stack rather than -/// finished layers) but it needs no registry account, which is the point. -async fn build_video_runner() -> Result<()> { - use c0mpute_core::video; - - for bin in ["docker", "git"] { - if tokio::process::Command::new(bin) - .arg("--version") - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .status() - .await - .is_err() - { - anyhow::bail!("building the runner needs both docker and git; {bin} is missing"); - } - } - - // A fresh directory per run: a half-finished clone from a previous - // attempt would make `git clone` fail rather than resume. - let checkout = std::env::temp_dir().join(format!("c0mpute-video-src-{}", std::process::id())); - let _ = tokio::fs::remove_dir_all(&checkout).await; - - println!("cloning {} (shallow)", video::RUNNER_SOURCE_REPO); - let status = tokio::process::Command::new("git") - .args(["clone", "--depth", "1", video::RUNNER_SOURCE_REPO]) - .arg(&checkout) - .status() - .await - .context("could not run git")?; - if !status.success() { - anyhow::bail!("git clone failed with {status}"); - } - - println!("building {} (several minutes)", video::RUNNER_IMAGE); - let args = video::build_argv(&checkout, video::RUNNER_IMAGE); - let status = tokio::process::Command::new("docker") - .args(&args) - .status() - .await - .context("could not run docker build")?; - - // Clean up whether or not the build worked: the checkout is scratch and - // a stale one in /tmp is worse than no cache at all. - let _ = tokio::fs::remove_dir_all(&checkout).await; - - if !status.success() { - anyhow::bail!("docker build failed with {status}"); - } - println!("built {}", video::RUNNER_IMAGE); - Ok(()) -} - /// `c0mpute video` — the surface `plugins/video/module.toml` declares. /// /// Everything here talks to the runner image rather than to the network: a @@ -2411,29 +2355,8 @@ async fn run_video(cmd: VideoCmd) -> Result<()> { match cmd { VideoCmd::Setup { build } => { - let mut have_image = false; - if !build { - println!("pulling {} (this is a large image)", video::RUNNER_IMAGE); - let status = tokio::process::Command::new("docker") - .args(["pull", video::RUNNER_IMAGE]) - .status() - .await - .context("could not run docker; install a container runtime first")?; - have_image = status.success(); - if !have_image { - // A registry is the kind of dependency that is private by - // default, rate-limited or down, and none of those are a - // reason a node with a card cannot serve video. The repo - // is public, so build the same image from source. - println!(); - println!("could not pull the runner image (private, rate-limited or offline)."); - println!("building it from source instead — same Dockerfile, same result."); - println!(); - } - } - if !have_image { - build_video_runner().await?; - } + let how = video::setup(build, &std::env::temp_dir(), &|line: &str| println!("{line}")).await?; + println!("runner image ready as {} ({how:?})", video::RUNNER_IMAGE); match video::readiness(video::RUNNER_IMAGE) { None => { println!("this node can serve video jobs"); @@ -2460,7 +2383,7 @@ async fn run_video(cmd: VideoCmd) -> Result<()> { } if selftest { println!("running the runner selftest in the container"); - let status = tokio::process::Command::new("docker") + let status = c0mpute_core::container::command() .args(["run", "--rm", video::RUNNER_IMAGE, "selftest"]) .status() .await @@ -2475,7 +2398,7 @@ async fn run_video(cmd: VideoCmd) -> Result<()> { // The image carries the manifest it was built from, so this is // the model set that node would actually serve — not what this // CLI happened to be compiled with. - let out = tokio::process::Command::new("docker") + let out = c0mpute_core::container::command() .args([ "run", "--rm", diff --git a/node/crates/c0mpute-core/src/container.rs b/node/crates/c0mpute-core/src/container.rs new file mode 100644 index 00000000..b1513ce2 --- /dev/null +++ b/node/crates/c0mpute-core/src/container.rs @@ -0,0 +1,57 @@ +//! The container CLI the container workloads (`video`, `gameserver`) drive. +//! +//! `docker` by default. `C0MPUTE_DOCKER` names another binary with the same +//! command line: `podman`, a wrapper that adds `--context`, or a test double +//! that stands in for the daemon in the loopback end-to-end tests. + +use std::ffi::OsString; +use std::path::PathBuf; + +/// Environment variable that overrides the container CLI. +pub const ENV: &str = "C0MPUTE_DOCKER"; + +/// The container CLI to run: `$C0MPUTE_DOCKER`, else `docker`. +pub fn cli() -> OsString { + match std::env::var_os(ENV) { + Some(v) if !v.is_empty() => v, + _ => OsString::from("docker"), + } +} + +/// Where [`cli`] resolves: the path itself when it names a file, else the +/// first match on `PATH`. `None` means there is no container runtime. +pub fn which() -> Option { + let bin = cli(); + let as_path = PathBuf::from(&bin); + if as_path.components().count() > 1 { + return as_path.is_file().then_some(as_path); + } + std::env::var_os("PATH").and_then(|paths| { + std::env::split_paths(&paths) + .map(|dir| dir.join(&bin)) + .find(|candidate| candidate.is_file()) + }) +} + +/// A `tokio` command for the container CLI. +pub fn command() -> tokio::process::Command { + tokio::process::Command::new(cli()) +} + +/// A blocking command for the container CLI (readiness probes only). +pub fn command_sync() -> std::process::Command { + std::process::Command::new(cli()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_to_docker() { + // The variable is not set in the unit-test process. + if std::env::var_os(ENV).is_none() { + assert_eq!(cli(), OsString::from("docker")); + } + } +} diff --git a/node/crates/c0mpute-core/src/gameserver.rs b/node/crates/c0mpute-core/src/gameserver.rs index 0213a842..ee396cb4 100644 --- a/node/crates/c0mpute-core/src/gameserver.rs +++ b/node/crates/c0mpute-core/src/gameserver.rs @@ -4,14 +4,34 @@ //! A start job is an ordinary auction whose job is short: the winner creates //! a container from a pinned, well-known image (`docker create`, argv only, //! never a shell) with memory, CPU and pid limits, a named volume for the -//! world and a published port, optionally restores a world backup the buyer -//! serves by hash, starts it, and returns `address:port` and the expiry. The -//! container then runs until the reservation ends: the worker's reaper stops -//! it at expiry. `gameserver.stop` (bid on only by the worker holding the -//! server, and only with the stop token whose sha256 the start job carried) -//! ends it early. Either way the worker tars the world out of the volume, -//! stores it as 1 MiB content-addressed chunks plus a manifest, and hands the -//! manifest hash back: `start --world ` restores it on any worker. +//! world and a published port, optionally restores a world the buyer serves +//! by hash, starts it, and returns `address:port` and the expiry. +//! +//! Secrets never ride in the offer. The offer carries the buyer's fresh +//! X25519 key; after the accept the winner announces its own and the buyer +//! seals the Valheim password and the restore key to it ([`crate::handoff`]). +//! +//! Worlds are encrypted. A world the buyer restores is published as AES-GCM +//! chunks under a fresh key that only the winning worker receives. Every +//! backup a worker takes is sealed to the buyer's long-term backup key +//! (X25519 → HKDF → AES-256-GCM per 1 MiB chunk): the manifest hash in a +//! receipt or a status record is useless to anyone else. +//! +//! While a server runs the worker backs the world up every `backup_mins` +//! (default 30) and gossips a status record (container running, latest +//! backup hash) on [`STATUS_TOPIC`], signed by its peer key. The buyer's +//! `gameserver watch` probes the game port (Minecraft status ping, Valheim +//! Steam A2S query, Factorio UDP reachability), logs every probe, mirrors +//! each new backup to its own machine, and when the server fails its checks +//! for `fail_after` it starts the same world on another worker from the +//! latest backup. No coordinator anywhere. +//! +//! Expiry does not depend on the worker being online: the container runs +//! under `timeout` (with `--init`) so it exits by itself when the +//! reservation ends, and `--rm` removes it; the named volume keeps the world +//! until the worker, when it next runs, backs it up and removes it. At boot +//! and on every sweep the worker also kills any labelled container whose +//! reservation has passed. //! //! Only opt-in nodes serve it (`--roles ...,gameserver`): it holds a port and //! RAM for hours. A node advertises it only when `docker info` works and it @@ -20,13 +40,15 @@ //! The operator must open the port range in the firewall. use std::collections::{HashMap, HashSet}; -use std::net::IpAddr; +use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use anyhow::{Context, Result, bail}; +use aes_gcm::aead::{Aead, KeyInit}; +use aes_gcm::{Aes256Gcm, Nonce}; +use anyhow::{Context, Result, anyhow, bail}; use c0mpute_net::topics::{JobOffer, JobStatus}; use c0mpute_net::{Libp2pNetwork, Network}; use c0mpute_proto::Hash; @@ -35,27 +57,43 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::sync::{Semaphore, mpsc}; +use tokio::sync::{Semaphore, broadcast, mpsc}; use tracing::{info, warn}; +use x25519_dalek::StaticSecret; use crate::runner::{AcceptedJob, WorkerLoad, publish_receipt}; +use crate::temporal::crypto; pub const WORKLOAD_START: &str = "gameserver.start"; pub const WORKLOAD_STOP: &str = "gameserver.stop"; pub const ROLE_TAG: &str = "c0mpute:role:gameserver"; +/// Where workers gossip [`StatusRecord`]s. +pub const STATUS_TOPIC: &str = "c0mpute/gameserver/v1"; +pub const SPEC_VERSION: u32 = 2; pub const MAX_HOURS: u32 = 72; pub const MAX_RAM_GB: u32 = 32; pub const MAX_CPUS: f64 = 8.0; -/// World backups are stored and moved as chunks of this size. +/// World backups are stored and moved as chunks of this size (plaintext). pub const CHUNK_BYTES: usize = 1024 * 1024; +/// AES-GCM tag on every chunk. +const TAG_BYTES: usize = 16; pub const MAX_WORLD_BYTES: u64 = 4 * 1024 * 1024 * 1024; +pub const DEFAULT_BACKUP_MINS: u32 = 30; +pub const MIN_BACKUP_MINS: u32 = 5; +pub const MAX_BACKUP_MINS: u32 = 720; +pub const ENC_SCHEME: &str = "x25519-hkdf-sha256-aes256gcm-chunked/v1"; const MANIFEST_KIND: &str = "c0mpute-gameserver-world"; +const MANIFEST_VERSION: u32 = 2; const MAX_MANIFEST_BYTES: usize = 1024 * 1024; /// Start/stop jobs handled at once (the servers themselves are not jobs). const JOB_SLOTS: usize = 2; /// Stopped-server records (and their backup hash) are kept this long. const KEEP_STOPPED_SECS: u64 = 7 * 86_400; -const REAP_EVERY: Duration = Duration::from_secs(30); +pub const REAP_EVERY: Duration = Duration::from_secs(30); +/// How long a worker waits for the buyer's sealed secrets after the accept. +const HANDOFF_WAIT: Duration = Duration::from_secs(120); +/// `timeout -k`: after the TERM at expiry, KILL this many seconds later. +const EXPIRY_KILL_AFTER_SECS: u64 = 90; // ── games ─────────────────────────────────────────────────────────────── @@ -104,6 +142,16 @@ impl Game { Game::Valheim => "lloesche/valheim-server:sha-0ab6cdcc16dd@sha256:fde7829b12d39203e155b6209689b70234f8a27d12c4d48baf890cc2ae2c2df3", } } + /// The pinned image's own ENTRYPOINT + CMD (read from each image config + /// at the digest above). The container runs it under `timeout`, so the + /// server exits at expiry whether or not the worker is online. + pub fn command(self) -> &'static [&'static str] { + match self { + Game::Minecraft => &["/image/scripts/start"], + Game::Factorio => &["/docker-entrypoint.sh"], + Game::Valheim => &["/usr/local/sbin/bootstrap"], + } + } /// Where the image keeps its world; the named volume mounts here. pub fn data_dir(self) -> &'static str { match self { @@ -140,7 +188,8 @@ impl Game { // ── specs ─────────────────────────────────────────────────────────────── -/// `spec_inline` of a `gameserver.start` offer. +/// `spec_inline` of a `gameserver.start` offer. Everything in it is public: +/// every subscriber of the job topic can read it. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct StartSpec { pub v: u32, @@ -153,16 +202,25 @@ pub struct StartSpec { pub accept_eula: bool, /// `sha256:` of the stop token only the buyer holds. pub stop_hash: String, - /// Manifest hash (blake3 hex) of a world backup to restore. + /// The buyer's fresh X25519 key for this job's [`crate::handoff`]. + pub client_pubkey: String, + /// The buyer's long-term X25519 key every backup is sealed to. + pub backup_pubkey: String, + /// Minutes between live backups; 0 = only at stop or expiry. + #[serde(default = "default_backup_mins")] + pub backup_mins: u32, + /// Manifest hash (blake3 hex) of an encrypted world to restore; its key + /// arrives in the sealed [`StartSecrets`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub world: Option, - /// Valheim only (it requires one). Visible to peers on the job topic. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub password: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub server_name: Option, } +fn default_backup_mins() -> u32 { + DEFAULT_BACKUP_MINS +} + fn is_hex(s: &str, len: usize) -> bool { s.len() == len && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) } @@ -173,8 +231,8 @@ fn valid_sha256_ref(s: &str) -> bool { impl StartSpec { pub fn validate(&self) -> std::result::Result<(), String> { - if self.v != 1 { - return Err(format!("unsupported gameserver spec version {}", self.v)); + if self.v != SPEC_VERSION { + return Err(format!("unsupported gameserver spec version {} (this node speaks {SPEC_VERSION})", self.v)); } if !(1..=MAX_HOURS).contains(&self.hours) { return Err(format!("hours must be 1..={MAX_HOURS}")); @@ -191,15 +249,15 @@ impl StartSpec { if !valid_sha256_ref(&self.stop_hash) { return Err("stop_hash must be sha256:<64 lowercase hex>".into()); } + if !is_hex(&self.client_pubkey, 64) || !is_hex(&self.backup_pubkey, 64) { + return Err("client_pubkey and backup_pubkey must be X25519 keys (64 hex)".into()); + } + if self.backup_mins != 0 && !(MIN_BACKUP_MINS..=MAX_BACKUP_MINS).contains(&self.backup_mins) { + return Err(format!("backup_mins must be 0 or {MIN_BACKUP_MINS}..={MAX_BACKUP_MINS}")); + } if self.world.as_deref().is_some_and(|w| !is_hex(w, 64)) { return Err("world must be a 64-hex manifest hash".into()); } - match (self.game, &self.password) { - (Game::Valheim, Some(p)) if (5..=32).contains(&p.len()) && p.chars().all(|c| c.is_ascii_alphanumeric()) => {} - (Game::Valheim, _) => return Err("valheim needs a password of 5-32 letters or digits".into()), - (_, Some(_)) => return Err("only valheim takes a password".into()), - _ => {} - } if let Some(n) = &self.server_name && (n.is_empty() || n.len() > 32 || !n.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, ' ' | '-' | '_' | '.'))) { @@ -209,6 +267,51 @@ impl StartSpec { } } +/// What the buyer seals to the winning worker after the accept. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct StartSecrets { + pub v: u32, + /// Valheim only (it requires one). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + /// Hex AES-256 key of the world named in the spec. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub world_key: Option, +} + +impl StartSecrets { + pub fn validate(&self, spec: &StartSpec) -> std::result::Result<(), String> { + if self.v != 1 { + return Err("unsupported secrets version".into()); + } + match (spec.game, &self.password) { + (Game::Valheim, Some(p)) if valid_password(p) => {} + (Game::Valheim, _) => return Err("valheim needs a password of 5-32 letters or digits".into()), + (_, Some(_)) => return Err("only valheim takes a password".into()), + _ => {} + } + match (&spec.world, &self.world_key) { + (Some(_), Some(k)) if is_hex(k, 64) => {} + (Some(_), _) => return Err("a world restore needs its 32-byte key".into()), + (None, Some(_)) => return Err("a world key without a world".into()), + (None, None) => {} + } + Ok(()) + } + + pub fn world_key(&self) -> Result> { + self.world_key.as_deref().map(key_from_hex).transpose() + } +} + +pub fn valid_password(p: &str) -> bool { + (5..=32).contains(&p.len()) && p.chars().all(|c| c.is_ascii_alphanumeric()) +} + +fn key_from_hex(h: &str) -> Result<[u8; 32]> { + hex::decode(h)?.try_into().map_err(|_| anyhow!("key must be 32 bytes")) +} + /// `spec_inline` of a `gameserver.stop` offer. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct StopSpec { @@ -264,12 +367,14 @@ pub struct ContainerPlan { pub cpus: f64, /// First host port; the game's ports map onto consecutive host ports. pub host_port: u16, + /// Seconds the server may run before it stops itself (the reservation). + pub run_secs: u64, pub env: Vec<(String, String)>, pub labels: Vec<(String, String)>, } impl ContainerPlan { - pub fn new(id: &str, spec: &StartSpec, host_port: u16, expires: u64) -> Self { + pub fn new(id: &str, spec: &StartSpec, secrets: &StartSecrets, host_port: u16, now: u64, expires: u64) -> Self { let memory_mb = u64::from(spec.ram_gb) * 1024; let name = format!("c0mpute-{id}"); let mut env: Vec<(String, String)> = Vec::new(); @@ -285,7 +390,7 @@ impl ContainerPlan { Game::Valheim => { env.push(("SERVER_NAME".into(), label)); env.push(("WORLD_NAME".into(), "c0mpute".into())); - env.push(("SERVER_PASS".into(), spec.password.clone().unwrap_or_default())); + env.push(("SERVER_PASS".into(), secrets.password.clone().unwrap_or_default())); env.push(("SERVER_PUBLIC".into(), "false".into())); env.push(("SERVER_PORT".into(), "2456".into())); } @@ -296,13 +401,28 @@ impl ContainerPlan { ("c0mpute.gameserver.game".into(), spec.game.name().into()), ("c0mpute.gameserver.expires".into(), expires.to_string()), ]; - Self { volume: name.clone(), name, game: spec.game, memory_mb, cpus: spec.cpus, host_port, env, labels } + Self { + volume: name.clone(), + name, + game: spec.game, + memory_mb, + cpus: spec.cpus, + host_port, + run_secs: expires.saturating_sub(now).max(1), + env, + labels, + } } /// `docker create` argv (without the leading `docker`). Every value is its /// own argument: nothing passes through a shell. + /// + /// The image's own command runs under `timeout ` behind + /// `--init`, and `--rm` removes the container when it exits: the server + /// ends at expiry even if this worker is offline then. The world lives + /// in the named volume, which `--rm` keeps. pub fn create_argv(&self) -> Vec { - let mut a: Vec = vec!["create".into(), "--name".into(), self.name.clone()]; + let mut a: Vec = vec!["create".into(), "--name".into(), self.name.clone(), "--rm".into(), "--init".into()]; for (k, v) in [ ("--restart", "no".to_string()), ("--memory", format!("{}m", self.memory_mb)), @@ -313,6 +433,7 @@ impl ContainerPlan { ("--log-opt", "max-size=10m".into()), ("--log-opt", "max-file=2".into()), ("--stop-timeout", "60".into()), + ("--entrypoint", "timeout".into()), ] { a.push(k.into()); a.push(v); @@ -332,12 +453,41 @@ impl ContainerPlan { a.push(format!("{k}={v}")); } a.push(self.game.image().into()); + // Arguments to `timeout`: TERM at expiry, KILL if still alive after. + for x in ["-s".to_string(), "TERM".into(), "-k".into(), format!("{EXPIRY_KILL_AFTER_SECS}s"), format!("{}s", self.run_secs)] { + a.push(x); + } + a.extend(self.game.command().iter().map(|s| s.to_string())); a } } +/// What a worker needs from a container runtime. [`Docker`] in production; +/// the loopback end-to-end tests supply a double. +#[async_trait::async_trait] +pub trait Runtime: Send + Sync + 'static { + /// Create (not start) the container and its volume. + async fn create(&self, plan: &ContainerPlan) -> Result<()>; + /// Extract a world tar (entries under the data dir's name) into the + /// created container's volume. + async fn restore(&self, plan: &ContainerPlan, tar: &Path) -> Result<()>; + async fn start(&self, name: &str) -> Result<()>; + async fn running(&self, name: &str) -> bool; + /// Ask the server to write its world to disk (best effort). + async fn flush(&self, name: &str, game: Game); + async fn stop(&self, name: &str) -> Result<()>; + /// Write a tar of the volume's world to `out`. Works whether or not the + /// server container still exists. + async fn tar_world(&self, volume: &str, game: Game, out: &Path) -> Result<()>; + /// Remove the container (if it is still there) and its volume. + async fn remove(&self, name: &str, volume: &str); + /// Containers this runtime labelled as game servers, with their expiry. + async fn labelled(&self) -> Result>; + async fn kill(&self, name: &str); +} + async fn docker(args: &[&str], timeout: Duration) -> Result { - let out = tokio::time::timeout(timeout, tokio::process::Command::new("docker").args(args).stdin(Stdio::null()).kill_on_drop(true).output()) + let out = tokio::time::timeout(timeout, crate::container::command().args(args).stdin(Stdio::null()).kill_on_drop(true).output()) .await .with_context(|| format!("docker {} timed out", args.first().unwrap_or(&"")))? .context("run docker")?; @@ -348,7 +498,124 @@ async fn docker(args: &[&str], timeout: Duration) -> Result { } fn docker_sync_ok(args: &[&str]) -> bool { - std::process::Command::new("docker").args(args).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success()) + crate::container::command_sync().args(args).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success()) +} + +/// `docker run` argv that tars a world volume to stdout with the game's own +/// (already pulled) image: no network, the volume read-only. +pub fn tar_argv(volume: &str, game: Game) -> Vec { + let dir = game.data_dir(); + vec![ + "run".into(), + "--rm".into(), + "--network".into(), + "none".into(), + "--entrypoint".into(), + "tar".into(), + "-v".into(), + format!("{volume}:{dir}:ro"), + game.image().into(), + "-C".into(), + "/".into(), + "-cf".into(), + "-".into(), + dir.trim_start_matches('/').into(), + ] +} + +/// The real container runtime: the `docker` CLI (or `$C0MPUTE_DOCKER`). +pub struct Docker; + +#[async_trait::async_trait] +impl Runtime for Docker { + async fn create(&self, plan: &ContainerPlan) -> Result<()> { + let argv = plan.create_argv(); + let args: Vec<&str> = argv.iter().map(String::as_str).collect(); + docker(&args, Duration::from_secs(1800)).await.map(drop) + } + + async fn restore(&self, plan: &ContainerPlan, tar: &Path) -> Result<()> { + // `docker cp - :/` extracts the tar (entries `data/...`) into the volume. + let f = std::fs::File::open(tar).with_context(|| format!("open {}", tar.display()))?; + let out = crate::container::command() + .args(["cp", "-", &format!("{}:/", plan.name)]) + .stdin(Stdio::from(f)) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .output() + .await + .context("run docker cp")?; + if !out.status.success() { + bail!("docker cp into {}: {}", plan.name, String::from_utf8_lossy(&out.stderr).trim()); + } + Ok(()) + } + + async fn start(&self, name: &str) -> Result<()> { + docker(&["start", name], Duration::from_secs(120)).await.map(drop) + } + + async fn running(&self, name: &str) -> bool { + docker(&["inspect", "-f", "{{.State.Running}}", name], Duration::from_secs(30)).await.is_ok_and(|s| s == "true") + } + + async fn flush(&self, name: &str, game: Game) { + // itzg's image ships rcon-cli with RCON on; the others autosave. + if game == Game::Minecraft { + let _ = docker(&["exec", name, "rcon-cli", "save-all", "flush"], Duration::from_secs(60)).await; + } + } + + async fn stop(&self, name: &str) -> Result<()> { + docker(&["stop", "-t", "60", name], Duration::from_secs(120)).await.map(drop) + } + + async fn tar_world(&self, volume: &str, game: Game, out: &Path) -> Result<()> { + let f = std::fs::File::create(out).with_context(|| format!("create {}", out.display()))?; + let argv = tar_argv(volume, game); + let res = tokio::time::timeout( + Duration::from_secs(1800), + crate::container::command().args(&argv).stdin(Stdio::null()).stdout(Stdio::from(f)).stderr(Stdio::piped()).kill_on_drop(true).output(), + ) + .await + .context("tar of the world timed out")? + .context("run docker")?; + if !res.status.success() { + bail!("tar of volume {volume}: {}", String::from_utf8_lossy(&res.stderr).trim()); + } + Ok(()) + } + + async fn remove(&self, name: &str, volume: &str) { + let _ = docker(&["rm", "-f", name], Duration::from_secs(60)).await; + let _ = docker(&["volume", "rm", volume], Duration::from_secs(60)).await; + } + + async fn labelled(&self) -> Result> { + let out = docker(&["ps", "-a", "--filter", "label=c0mpute.gameserver=1", "--format", "{{.Names}}\t{{.Label \"c0mpute.gameserver.expires\"}}"], Duration::from_secs(30)).await?; + Ok(parse_labelled(&out)) + } + + async fn kill(&self, name: &str) { + let _ = docker(&["kill", name], Duration::from_secs(30)).await; + let _ = docker(&["rm", "-f", name], Duration::from_secs(30)).await; + } +} + +/// `docker ps --format '{{.Names}}\t{{.Label ...expires}}'` → (name, expiry). +/// A container without a readable expiry counts as already expired. +pub fn parse_labelled(out: &str) -> Vec<(String, u64)> { + out.lines() + .filter_map(|l| { + let mut it = l.split('\t'); + let name = it.next()?.trim(); + if name.is_empty() { + return None; + } + Some((name.to_string(), it.next().and_then(|e| e.trim().parse().ok()).unwrap_or(0))) + }) + .collect() } // ── public address ────────────────────────────────────────────────────── @@ -483,10 +750,15 @@ pub async fn ensure_images(cfg: &crate::config::GameserverConfig) { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct Backup { - /// Blake3 hex of the world manifest. + /// Blake3 hex of the (encrypted) world manifest. pub manifest: String, + /// Plaintext tar size. pub bytes: u64, + /// sha256 of the plaintext tar. pub sha256: String, + /// When it was taken (unix seconds). + #[serde(default)] + pub at: u64, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] @@ -502,8 +774,15 @@ pub struct ServerState { pub expires_at: u64, pub stop_hash: String, pub buyer: String, + /// Every backup is sealed to this key. + #[serde(default)] + pub backup_pubkey: String, + /// Seconds between live backups (0 = none). + #[serde(default)] + pub backup_every_secs: u64, #[serde(default)] pub stopped_at: Option, + /// The latest backup (live, or the final one at stop/expiry). #[serde(default)] pub backup: Option, #[serde(default)] @@ -527,6 +806,26 @@ pub fn due_for_stop(servers: &HashMap, now: u64) -> Vec, now: u64) -> Vec { + let mut ids: Vec = servers + .values() + .filter(|s| { + let last = s.backup.as_ref().map(|b| b.at).unwrap_or(s.started_at).max(s.started_at); + s.running() && s.backup_every_secs > 0 && now < s.expires_at && now >= last + s.backup_every_secs + }) + .map(|s| s.id.clone()) + .collect(); + ids.sort(); + ids +} + +/// Labelled containers whose reservation has passed: killed on sight, at +/// boot and on every sweep, whatever the state file says. +pub fn expired_containers(labelled: &[(String, u64)], now: u64) -> Vec { + labelled.iter().filter(|(_, exp)| now >= *exp).map(|(n, _)| n.clone()).collect() +} + /// Can this node take `spec` next to what it already runs? pub fn has_capacity(cfg: &crate::config::GameserverConfig, total_ram_mb: u64, servers: &HashMap, spec: &StartSpec) -> std::result::Result<(), String> { if !cfg.games.contains(&spec.game) { @@ -567,6 +866,55 @@ fn port_bindable(p: u16) -> bool { std::net::TcpListener::bind(("0.0.0.0", p)).is_ok() && std::net::UdpSocket::bind(("0.0.0.0", p)).is_ok() } +// ── world encryption ──────────────────────────────────────────────────── + +/// How a world's chunks are encrypted. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct WorldEnc { + pub scheme: String, + /// Present on backups: the worker's one-time X25519 key; the buyer's + /// backup key and this give the chunk key. Absent on a world the buyer + /// publishes for a restore: its key travels in the sealed secrets. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub epk: Option, +} + +impl WorldEnc { + pub fn handed_off() -> Self { + Self { scheme: ENC_SCHEME.into(), epk: None } + } +} + +/// A fresh random world key. +pub fn random_key() -> [u8; 32] { + rand::random() +} + +/// The chunk key of a backup sealed to `recipient_pub`: derived from the +/// worker's one-time secret and the recipient's key (worker side), or the +/// recipient's secret and the one-time `epk` (buyer side). +pub fn backup_key(secret: &StaticSecret, peer_pub_hex: &str, epk_hex: &str, recipient_hex: &str) -> Result<[u8; 32]> { + crypto::session_key(secret, peer_pub_hex, &format!("gameserver-backup:{epk_hex}:{recipient_hex}")) +} + +fn chunk_nonce(i: u64) -> [u8; 12] { + let mut n = [0u8; 12]; + n[4..].copy_from_slice(&i.to_be_bytes()); + n +} + +/// Chunk `i` under `key`. The key is single-use (one per world), so the +/// index is a safe nonce, and it pins every chunk to its position. +pub fn seal_chunk(key: &[u8; 32], i: u64, plain: &[u8]) -> Result> { + let c = Aes256Gcm::new_from_slice(key).map_err(|_| anyhow!("key"))?; + c.encrypt(Nonce::from_slice(&chunk_nonce(i)), plain).map_err(|_| anyhow!("encrypt chunk")) +} + +pub fn open_chunk(key: &[u8; 32], i: u64, sealed: &[u8]) -> Result> { + let c = Aes256Gcm::new_from_slice(key).map_err(|_| anyhow!("key"))?; + c.decrypt(Nonce::from_slice(&chunk_nonce(i)), sealed).map_err(|_| anyhow!("world chunk {i} does not decrypt (wrong key or tampered)")) +} + // ── world backups ─────────────────────────────────────────────────────── #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] @@ -576,28 +924,34 @@ pub struct WorldManifest { pub game: Game, /// A tar of this directory (entries start with its name, e.g. `data/`). pub data_dir: String, + /// Plaintext tar size. pub bytes: u64, + /// sha256 of the plaintext tar, checked after decryption. pub sha256: String, - /// Blake3 hex of each chunk, in order. + pub enc: WorldEnc, + /// Blake3 hex of each encrypted chunk, in order. pub chunks: Vec, } impl WorldManifest { - pub fn new(game: Game) -> Self { - Self { v: 1, kind: MANIFEST_KIND.into(), game, data_dir: game.data_dir().into(), bytes: 0, sha256: String::new(), chunks: Vec::new() } + pub fn new(game: Game, enc: WorldEnc) -> Self { + Self { v: MANIFEST_VERSION, kind: MANIFEST_KIND.into(), game, data_dir: game.data_dir().into(), bytes: 0, sha256: String::new(), enc, chunks: Vec::new() } } pub fn to_bytes(&self) -> Vec { serde_json::to_vec(self).unwrap_or_default() } pub fn from_bytes(b: &[u8]) -> Result { - let m: Self = serde_json::from_slice(b).context("world manifest is not JSON")?; + let m: Self = serde_json::from_slice(b).context("world manifest is not a v2 JSON manifest")?; m.check()?; Ok(m) } pub fn check(&self) -> Result<()> { - if self.v != 1 || self.kind != MANIFEST_KIND { + if self.v != MANIFEST_VERSION || self.kind != MANIFEST_KIND { bail!("not a c0mpute game world manifest"); } + if self.enc.scheme != ENC_SCHEME || self.enc.epk.as_deref().is_some_and(|k| !is_hex(k, 64)) { + bail!("unknown world encryption"); + } if self.data_dir != self.game.data_dir() { bail!("manifest data_dir does not match {}", self.game.name()); } @@ -612,10 +966,11 @@ impl WorldManifest { } } -/// Read `r` to the end into 1 MiB chunks in the store; returns the manifest -/// (not yet stored). Chunks are announced so any peer can fetch them. -pub async fn chunk_into_store(store: &ChunkStore, net: Option<&Libp2pNetwork>, game: Game, mut r: R) -> Result { - let mut m = WorldManifest::new(game); +/// Read `r` to the end, encrypt it under `key` in 1 MiB chunks, and put the +/// chunks in the store; returns the manifest (not yet stored). Chunks are +/// announced so any peer can fetch them; only the key holder can read them. +pub async fn chunk_into_store(store: &ChunkStore, net: Option<&Libp2pNetwork>, game: Game, mut r: R, key: &[u8; 32], enc: WorldEnc) -> Result { + let mut m = WorldManifest::new(game, enc); let mut sha = Sha256::new(); let mut buf = vec![0u8; CHUNK_BYTES]; loop { @@ -635,7 +990,8 @@ pub async fn chunk_into_store(store: &ChunkStor bail!("world is over {} GiB", MAX_WORLD_BYTES >> 30); } sha.update(&buf[..filled]); - let h = store.put(&buf[..filled]).await?; + let sealed = seal_chunk(key, m.chunks.len() as u64, &buf[..filled])?; + let h = store.put(&sealed).await?; if let Some(net) = net { let _ = net.announce(&h).await; } @@ -663,18 +1019,32 @@ async fn fetch_bytes(net: &Libp2pNetwork, store: &ChunkStore, hash: &str, provid { return Ok(b); } - crate::gateway_util::fetch_verified(net, hash, provider, max, Duration::from_secs(60)).await.map_err(|e| anyhow::anyhow!("fetch {hash} from {provider}: {e:?}")) + let b = crate::gateway_util::fetch_verified(net, hash, provider, max, Duration::from_secs(60)).await.map_err(|e| anyhow!("fetch {hash} from {provider}: {e:?}"))?; + // Keep a copy: a buyer mirroring backups must be able to restore after + // the worker that took them is gone. + let _ = store.put(&b).await; + Ok(b) } -/// Fetch a world (manifest, then each chunk) from `provider` (or any peer -/// that announced it), verifying every chunk and the whole tar's sha256, -/// and write the tar to `out` as it arrives. -pub async fn fetch_world(net: &Libp2pNetwork, store: &ChunkStore, manifest_hash: &str, provider: &str, mut out: W) -> Result { +/// Fetch an encrypted world (manifest, then each chunk) from `provider` (or +/// a local copy), verify every chunk's hash, decrypt it with the key +/// `key_of` gives for the manifest, check the plaintext sha256, and write the +/// tar to `out` as it arrives. +pub async fn fetch_world( + net: &Libp2pNetwork, + store: &ChunkStore, + manifest_hash: &str, + provider: &str, + key_of: impl FnOnce(&WorldManifest) -> Result<[u8; 32]>, + mut out: W, +) -> Result { let mb = fetch_bytes(net, store, manifest_hash, provider, MAX_MANIFEST_BYTES).await.context("world manifest")?; let m = WorldManifest::from_bytes(&mb)?; + let key = key_of(&m)?; let mut sha = Sha256::new(); - for c in &m.chunks { - let b = fetch_bytes(net, store, c, provider, CHUNK_BYTES).await?; + for (i, c) in m.chunks.iter().enumerate() { + let sealed = fetch_bytes(net, store, c, provider, CHUNK_BYTES + TAG_BYTES).await?; + let b = open_chunk(&key, i as u64, &sealed)?; sha.update(&b); out.write_all(&b).await?; } @@ -685,6 +1055,32 @@ pub async fn fetch_world(net: &Libp2pNetwork, Ok(m) } +// ── status records ────────────────────────────────────────────────────── + +/// What a worker gossips about each server it holds, every sweep. The +/// gossipsub message is signed by the worker's peer key (strict validation), +/// so a watcher trusts a record only when its source is the rental's worker. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct StatusRecord { + pub t: String, + pub v: u32, + pub id: String, + pub worker: String, + pub game: Game, + pub address: String, + /// The container is up, per the worker's runtime. + pub running: bool, + pub expires_at: u64, + #[serde(default)] + pub stopped_at: Option, + /// Latest backup: sealed to the buyer, so the hash is safe to gossip. + #[serde(default)] + pub backup: Option, + pub ts: u64, +} + +pub const STATUS_KIND: &str = "gs_status"; + // ── worker ────────────────────────────────────────────────────────────── pub struct Worker { @@ -692,28 +1088,36 @@ pub struct Worker { store: ChunkStore, cfg: crate::config::GameserverConfig, host: String, + state_dir: PathBuf, state_path: PathBuf, + rt: Arc, servers: Mutex>, - /// Servers a start, stop or reap is working on right now. + /// Servers a start, stop, backup or reap is working on right now. busy: Mutex>, } impl Worker { pub fn new(net: Arc, store: ChunkStore, cfg: crate::config::GameserverConfig, host: String, state_dir: &Path) -> Arc { + Self::with_runtime(net, store, cfg, host, state_dir, Arc::new(Docker)) + } + + pub fn with_runtime(net: Arc, store: ChunkStore, cfg: crate::config::GameserverConfig, host: String, state_dir: &Path, rt: Arc) -> Arc { let state_path = state_dir.join("servers.json"); let servers: HashMap = std::fs::read(&state_path).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default(); - Arc::new(Self { net, store, cfg, host, state_path, servers: Mutex::new(servers), busy: Mutex::new(HashSet::new()) }) + Arc::new(Self { net, store, cfg, host, state_dir: state_dir.to_path_buf(), state_path, rt, servers: Mutex::new(servers), busy: Mutex::new(HashSet::new()) }) + } + + pub fn net(&self) -> &Libp2pNetwork { + &self.net } - fn snapshot(&self) -> HashMap { + pub fn snapshot(&self) -> HashMap { self.servers.lock().map(|g| g.clone()).unwrap_or_default() } fn save(&self) { let snap = self.snapshot(); - if let Some(dir) = self.state_path.parent() { - let _ = std::fs::create_dir_all(dir); - } + let _ = std::fs::create_dir_all(&self.state_dir); let tmp = self.state_path.with_extension("tmp"); if std::fs::write(&tmp, serde_json::to_vec_pretty(&snap).unwrap_or_default()).is_ok() { let _ = std::fs::rename(&tmp, &self.state_path); @@ -729,6 +1133,10 @@ impl Worker { } } + fn scratch(&self, what: &str, id: &str) -> PathBuf { + self.state_dir.join(format!("{what}-{id}.tar")) + } + pub fn accepts_start(&self, offer: &JobOffer) -> bool { let Some(spec) = offer.spec_inline.as_ref().and_then(|v| serde_json::from_value::(v.clone()).ok()) else { return false; @@ -757,20 +1165,25 @@ impl Worker { async fn start(&self, job: &AcceptedJob) -> Result { let spec: StartSpec = serde_json::from_value(job.accept.spec_inline.clone().context("accept lacks spec_inline")?)?; - spec.validate().map_err(|e| anyhow::anyhow!(e))?; + spec.validate().map_err(|e| anyhow!(e))?; + // The password and the restore key come sealed, after the accept. + let raw = crate::handoff::receive(&self.net, WORKLOAD_START, &job.accept, &spec.client_pubkey, HANDOFF_WAIT).await.context("receive the sealed secrets")?; + let secrets: StartSecrets = serde_json::from_slice(&raw).context("sealed secrets are not JSON")?; + drop(raw); + secrets.validate(&spec).map_err(|e| anyhow!(e))?; let id = server_id(&job.accept.job_id); // Reserve RAM and ports under the lock, so two starts cannot both take the last slot. let (state, plan) = { - let mut servers = self.servers.lock().map_err(|_| anyhow::anyhow!("state lock poisoned"))?; + let mut servers = self.servers.lock().map_err(|_| anyhow!("state lock poisoned"))?; if servers.contains_key(&id) { bail!("server {id} already exists"); } - has_capacity(&self.cfg, crate::text::local_ram_mb(), &servers, &spec).map_err(|e| anyhow::anyhow!(e))?; + has_capacity(&self.cfg, crate::text::local_ram_mb(), &servers, &spec).map_err(|e| anyhow!(e))?; let used: HashSet = servers.values().filter(|s| s.running()).flat_map(|s| s.ports()).collect(); let port = allocate_ports(self.cfg.port_min, self.cfg.port_max, spec.game.ports().len(), &used, port_bindable).context("no free port in the range")?; let started = now_secs(); let expires = expires_at(started, spec.hours); - let plan = ContainerPlan::new(&id, &spec, port, expires); + let plan = ContainerPlan::new(&id, &spec, &secrets, port, started, expires); let state = ServerState { id: id.clone(), game: spec.game, @@ -783,6 +1196,8 @@ impl Worker { expires_at: expires, stop_hash: spec.stop_hash.clone(), buyer: job.accept.buyer_peer_id.clone(), + backup_pubkey: spec.backup_pubkey.clone(), + backup_every_secs: u64::from(spec.backup_mins) * 60, stopped_at: None, backup: None, backup_error: None, @@ -791,11 +1206,10 @@ impl Worker { (state, plan) }; self.claim(&id); - let res = self.create_and_start(&plan, &spec, &job.accept.buyer_peer_id).await; + let res = self.create_and_start(&id, &plan, &spec, &secrets, &job.accept.buyer_peer_id).await; self.release(&id); if let Err(e) = res { - let _ = docker(&["rm", "-f", &plan.name], Duration::from_secs(60)).await; - let _ = docker(&["volume", "rm", &plan.volume], Duration::from_secs(60)).await; + self.rt.remove(&plan.name, &plan.volume).await; if let Ok(mut s) = self.servers.lock() { s.remove(&id); } @@ -807,35 +1221,60 @@ impl Worker { "v": 1, "ok": true, "id": id, "game": spec.game.name(), "address": state.address, "ports": spec.game.ports().iter().enumerate().map(|(i, (p, proto))| json!({"host": plan.host_port + i as u16, "container": p, "proto": proto.as_str()})).collect::>(), "expires_at": state.expires_at, "image": spec.game.image(), "restored": spec.world, + "backup_mins": spec.backup_mins, })) } - async fn create_and_start(&self, plan: &ContainerPlan, spec: &StartSpec, buyer: &str) -> Result<()> { - let argv = plan.create_argv(); - let args: Vec<&str> = argv.iter().map(String::as_str).collect(); - docker(&args, Duration::from_secs(1800)).await?; + async fn create_and_start(&self, id: &str, plan: &ContainerPlan, spec: &StartSpec, secrets: &StartSecrets, buyer: &str) -> Result<()> { + self.rt.create(plan).await?; if let Some(world) = &spec.world { - // `docker cp - :/` extracts the tar (entries `data/...`) into the volume. - let mut child = tokio::process::Command::new("docker") - .args(["cp", "-", &format!("{}:/", plan.name)]) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .context("spawn docker cp")?; - let stdin = child.stdin.take().context("docker cp stdin")?; - let m = fetch_world(&self.net, &self.store, world, buyer, stdin).await.context("restore world")?; - if m.game != spec.game { - bail!("world is a {} world, not {}", m.game.name(), spec.game.name()); + let key = secrets.world_key()?.context("world key missing")?; + let tmp = self.scratch("restore", id); + let _ = tokio::fs::create_dir_all(&self.state_dir).await; + let res = async { + let f = tokio::fs::File::create(&tmp).await?; + let m = fetch_world(&self.net, &self.store, world, buyer, |m| { + if m.enc.epk.is_some() { + bail!("a restore world must be encrypted under a handed-off key"); + } + Ok(key) + }, f) + .await + .context("restore world")?; + if m.game != spec.game { + bail!("world is a {} world, not {}", m.game.name(), spec.game.name()); + } + self.rt.restore(plan, &tmp).await } - let out = child.wait_with_output().await?; - if !out.status.success() { - bail!("docker cp into {}: {}", plan.name, String::from_utf8_lossy(&out.stderr).trim()); + .await; + let _ = tokio::fs::remove_file(&tmp).await; + res?; + } + self.rt.start(&plan.name).await + } + + /// Back the world up now (live if the server runs), seal it to the + /// buyer, and record it as the latest backup. + pub async fn backup_now(&self, id: &str) -> Result { + let s = self.snapshot().get(id).cloned().with_context(|| format!("no server {id}"))?; + if !self.claim(id) { + bail!("server {id} is busy"); + } + let res = self.backup(&s).await; + self.release(id); + if let Ok(mut servers) = self.servers.lock() + && let Some(e) = servers.get_mut(id) + { + match &res { + Ok(b) => { + e.backup = Some(b.clone()); + e.backup_error = None; + } + Err(err) => e.backup_error = Some(format!("{err:#}")), } } - docker(&["start", &plan.name], Duration::from_secs(120)).await?; - Ok(()) + self.save(); + res } /// Stop a server, back its world up into the store, remove the container @@ -849,11 +1288,11 @@ impl Worker { bail!("server {id} is busy"); } let res = async { - let _ = docker(&["stop", "-t", "60", &s.container], Duration::from_secs(120)).await; + // Already gone when it stopped itself at expiry (`--rm`). + let _ = self.rt.stop(&s.container).await; let backup = self.backup(&s).await; if backup.is_ok() { - let _ = docker(&["rm", "-f", &s.container], Duration::from_secs(60)).await; - let _ = docker(&["volume", "rm", &s.volume], Duration::from_secs(60)).await; + self.rt.remove(&s.container, &s.volume).await; } backup } @@ -873,28 +1312,33 @@ impl Worker { } async fn backup(&self, s: &ServerState) -> Result { - let mut child = tokio::process::Command::new("docker") - .args(["cp", &format!("{}:{}", s.container, s.game.data_dir()), "-"]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .context("spawn docker cp")?; - let stdout = child.stdout.take().context("docker cp stdout")?; - let m = chunk_into_store(&self.store, Some(&self.net), s.game, stdout).await?; - let out = child.wait_with_output().await?; - if !out.status.success() { - bail!("docker cp out of {}: {}", s.container, String::from_utf8_lossy(&out.stderr).trim()); + if !is_hex(&s.backup_pubkey, 64) { + bail!("server {} has no backup key", s.id); + } + if self.rt.running(&s.container).await { + self.rt.flush(&s.container, s.game).await; + } + let _ = tokio::fs::create_dir_all(&self.state_dir).await; + let tmp = self.scratch("backup", &s.id); + let res = async { + self.rt.tar_world(&s.volume, s.game, &tmp).await?; + let e = crypto::keypair(); + let key = backup_key(&e.secret, &s.backup_pubkey, &e.public_hex, &s.backup_pubkey)?; + let f = tokio::fs::File::open(&tmp).await?; + let m = chunk_into_store(&self.store, Some(&self.net), s.game, f, &key, WorldEnc { scheme: ENC_SCHEME.into(), epk: Some(e.public_hex.clone()) }).await?; + let manifest = put_manifest(&self.store, &self.net, &m).await?; + Ok::<_, anyhow::Error>(Backup { manifest, bytes: m.bytes, sha256: m.sha256, at: now_secs() }) } - let manifest = put_manifest(&self.store, &self.net, &m).await?; - info!(id = %s.id, %manifest, bytes = m.bytes, "gameserver: world backed up"); - Ok(Backup { manifest, bytes: m.bytes, sha256: m.sha256 }) + .await; + let _ = tokio::fs::remove_file(&tmp).await; + let b = res?; + info!(id = %s.id, manifest = %b.manifest, bytes = b.bytes, "gameserver: world backed up (sealed to the buyer)"); + Ok(b) } async fn stop(&self, job: &AcceptedJob) -> Result { let spec: StopSpec = serde_json::from_value(job.accept.spec_inline.clone().context("accept lacks spec_inline")?)?; - spec.validate().map_err(|e| anyhow::anyhow!(e))?; + spec.validate().map_err(|e| anyhow!(e))?; let s = self.snapshot().get(&spec.id).cloned().context("unknown server")?; if s.stop_hash != sha256_ref(spec.token.as_bytes()) { bail!("stop token does not match"); @@ -903,15 +1347,34 @@ impl Worker { Ok(json!({"v": 1, "ok": true, "id": spec.id, "game": s.game.name(), "backup": backup})) } - /// Stop servers whose reservation ended; forget old stopped ones. + /// Kill labelled containers whose reservation passed. Runs at boot (the + /// worker may have been offline at expiry) and on every sweep. + pub async fn sweep_expired(&self) -> Vec { + let Ok(labelled) = self.rt.labelled().await else { return Vec::new() }; + let dead = expired_containers(&labelled, now_secs()); + for name in &dead { + warn!(container = %name, "gameserver: reservation passed; killing"); + self.rt.kill(name).await; + } + dead + } + + /// Kill expired containers, stop + back up servers whose reservation + /// ended, take due live backups, forget old stopped ones. pub async fn reap_once(&self) { let now = now_secs(); + self.sweep_expired().await; for id in due_for_stop(&self.snapshot(), now) { info!(%id, "gameserver: reservation ended, stopping"); if let Err(e) = self.stop_and_backup(&id).await { warn!(%id, err = %format!("{e:#}"), "gameserver: stop at expiry failed"); } } + for id in due_for_backup(&self.snapshot(), now) { + if let Err(e) = self.backup_now(&id).await { + warn!(%id, err = %format!("{e:#}"), "gameserver: live backup failed"); + } + } let mut changed = false; if let Ok(mut s) = self.servers.lock() { let before = s.len(); @@ -923,6 +1386,38 @@ impl Worker { } } + /// The status record for one server. + pub async fn status(&self, s: &ServerState) -> StatusRecord { + StatusRecord { + t: STATUS_KIND.into(), + v: 1, + id: s.id.clone(), + worker: self.net.peer_id().to_base58(), + game: s.game, + address: s.address.clone(), + running: s.running() && self.rt.running(&s.container).await, + expires_at: s.expires_at, + stopped_at: s.stopped_at, + backup: s.backup.clone(), + ts: now_secs(), + } + } + + /// Gossip a status record for every server held (stopped ones for a day, + /// so a watcher learns the final backup). + pub async fn publish_status(&self) { + let now = now_secs(); + for s in self.snapshot().values() { + if s.stopped_at.is_some_and(|t| now.saturating_sub(t) > 86_400) { + continue; + } + let r = self.status(s).await; + if let Ok(b) = serde_json::to_vec(&r) { + let _ = self.net.publish(STATUS_TOPIC, b).await; + } + } + } + async fn publish_result(&self, v: &Value) -> Result { let h = self.store.put(&serde_json::to_vec(v)?).await?; self.net.announce(&h).await?; @@ -956,8 +1451,42 @@ fn spawn_runner(worker: Arc, slots: Arc, stop: bool) -> mpsc: tx } -/// Supervisor entry: pull images, then subscribe to start and stop offers and -/// run the expiry reaper. Returns without subscribing when the node cannot host. +/// Run the sweep (expiry, live backups) and gossip status every `every`. +pub fn spawn_reaper(worker: Arc, every: Duration) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let _ = worker.net.subscribe(STATUS_TOPIC).await; + loop { + worker.reap_once().await; + worker.publish_status().await; + tokio::time::sleep(every).await; + } + }) +} + +/// Subscribe a worker to start and stop offers. +pub fn subscribe(worker: Arc, tags: Vec, pricing: Option) { + let slots = Arc::new(Semaphore::new(JOB_SLOTS)); + let load = WorkerLoad::new(slots.clone(), JOB_SLOTS); + let w = worker.clone(); + let start_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_start(o)); + let w = worker.clone(); + let stop_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_stop(o)); + crate::dispatch::run_worker_subscriber_with_load( + worker.net.clone(), + WORKLOAD_START.into(), + tags.clone(), + Some(spawn_runner(worker.clone(), slots.clone(), false)), + Some(start_filter), + pricing, + Some(load.clone()), + ); + // Stopping is part of the rent: no price floor. + crate::dispatch::run_worker_subscriber_with_load(worker.net.clone(), WORKLOAD_STOP.into(), tags, Some(spawn_runner(worker.clone(), slots, true)), Some(stop_filter), None, Some(load)); +} + +/// Supervisor entry: kill expired leftovers, pull images, then subscribe to +/// start and stop offers and run the sweep. Returns without subscribing when +/// the node cannot host. pub async fn run_worker(net: Arc, store: ChunkStore, config: crate::Config, state_dir: PathBuf) { if !docker_sync_ok(&["info", "--format", "{{.ServerVersion}}"]) { info!("gameserver: docker is not usable here; not serving game servers"); @@ -968,14 +1497,10 @@ pub async fn run_worker(net: Arc, store: ChunkStore, config: crat return; }; let worker = Worker::new(net.clone(), store, config.gameserver.clone(), host, &state_dir); + // Boot watchdog: anything whose reservation passed while we were down dies now. + worker.sweep_expired().await; // Enforce expiries even before the images finish pulling. - let reaper = worker.clone(); - tokio::spawn(async move { - loop { - reaper.reap_once().await; - tokio::time::sleep(REAP_EVERY).await; - } - }); + spawn_reaper(worker.clone(), REAP_EVERY); ensure_images(&config.gameserver).await; let ready = probe(&config); if !ready.ready() { @@ -986,29 +1511,255 @@ pub async fn run_worker(net: Arc, store: ChunkStore, config: crat *g = Some((Instant::now(), ready.clone())); } let tags = crate::capabilities::tags_from_config(&config); - let slots = Arc::new(Semaphore::new(JOB_SLOTS)); - let load = WorkerLoad::new(slots.clone(), JOB_SLOTS); - let w = worker.clone(); - let start_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_start(o)); - let w = worker.clone(); - let stop_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_stop(o)); info!(games = ?ready.games, host = ?ready.host, "gameserver: serving"); - crate::dispatch::run_worker_subscriber_with_load( - net.clone(), - WORKLOAD_START.into(), - tags.clone(), - Some(spawn_runner(worker.clone(), slots.clone(), false)), - Some(start_filter), - Some(config.pricing.clone()), - Some(load.clone()), - ); - // Stopping is part of the rent: no price floor. - crate::dispatch::run_worker_subscriber_with_load(net, WORKLOAD_STOP.into(), tags, Some(spawn_runner(worker, slots, true)), Some(stop_filter), None, Some(load)); + subscribe(worker, tags, Some(config.pricing.clone())); +} + +// ── health probes ─────────────────────────────────────────────────────── + +/// One probe of a server's game port. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ProbeResult { + pub ts: u64, + pub ok: bool, + /// `minecraft-status`, `a2s-info` or `udp-reachable`. + pub method: String, + pub rtt_ms: u64, + pub detail: String, +} + +fn put_varint(out: &mut Vec, mut v: u32) { + loop { + let mut b = (v & 0x7f) as u8; + v >>= 7; + if v != 0 { + b |= 0x80; + } + out.push(b); + if v == 0 { + break; + } + } +} + +async fn read_varint(r: &mut R) -> Result { + let mut v = 0u32; + for i in 0..5 { + let b = r.read_u8().await?; + v |= u32::from(b & 0x7f) << (7 * i); + if b & 0x80 == 0 { + return Ok(v); + } + } + bail!("varint too long") +} + +fn read_varint_slice(b: &[u8], at: &mut usize) -> Result { + let mut v = 0u32; + for i in 0..5 { + let x = *b.get(*at).context("truncated varint")?; + *at += 1; + v |= u32::from(x & 0x7f) << (7 * i); + if x & 0x80 == 0 { + return Ok(v); + } + } + bail!("varint too long") +} + +/// Minecraft Java "server list ping": handshake (next state 1) + status request. +pub fn minecraft_status_request(host: &str, port: u16) -> Vec { + let mut body = Vec::new(); + put_varint(&mut body, 0x00); + put_varint(&mut body, 767); + put_varint(&mut body, host.len() as u32); + body.extend_from_slice(host.as_bytes()); + body.extend_from_slice(&port.to_be_bytes()); + put_varint(&mut body, 1); + let mut pkt = Vec::new(); + put_varint(&mut pkt, body.len() as u32); + pkt.extend(body); + pkt.extend([0x01, 0x00]); + pkt +} + +/// A status response packet carrying `json` (what a server answers). +pub fn minecraft_status_response(json: &str) -> Vec { + let mut body = Vec::new(); + put_varint(&mut body, 0x00); + put_varint(&mut body, json.len() as u32); + body.extend_from_slice(json.as_bytes()); + let mut pkt = Vec::new(); + put_varint(&mut pkt, body.len() as u32); + pkt.extend(body); + pkt +} + +/// Parse a status response body (after the length prefix) into its JSON. +pub fn parse_minecraft_status(body: &[u8]) -> Result { + let mut at = 0; + if read_varint_slice(body, &mut at)? != 0 { + bail!("not a status response"); + } + let n = read_varint_slice(body, &mut at)? as usize; + let s = body.get(at..at + n).context("truncated status")?; + let v: Value = serde_json::from_slice(s).context("status is not JSON")?; + if !v.is_object() { + bail!("status is not an object"); + } + Ok(v) +} + +async fn minecraft_status(addr: SocketAddr, host: &str, timeout: Duration) -> Result { + tokio::time::timeout(timeout, async { + let mut s = tokio::net::TcpStream::connect(addr).await?; + s.write_all(&minecraft_status_request(host, addr.port())).await?; + let len = read_varint(&mut s).await? as usize; + if len == 0 || len > 256 * 1024 { + bail!("status packet length {len}"); + } + let mut body = vec![0u8; len]; + s.read_exact(&mut body).await?; + let v = parse_minecraft_status(&body)?; + let online = v["players"]["online"].as_u64().unwrap_or(0); + let version = v["version"]["name"].as_str().unwrap_or("?").to_string(); + Ok(format!("{version}, {online} online")) + }) + .await + .context("timed out")? +} + +/// Steam A2S_INFO request (Valheim answers on its query port, game port + 1). +pub const A2S_INFO: &[u8] = b"\xFF\xFF\xFF\xFFTSource Engine Query\0"; + +async fn a2s_info(addr: SocketAddr, timeout: Duration) -> Result { + tokio::time::timeout(timeout, async { + let bind: SocketAddr = if addr.is_ipv4() { "0.0.0.0:0".parse()? } else { "[::]:0".parse()? }; + let sock = tokio::net::UdpSocket::bind(bind).await?; + sock.connect(addr).await?; + let mut q = A2S_INFO.to_vec(); + let mut buf = vec![0u8; 1400]; + for _ in 0..2 { + sock.send(&q).await?; + let n = sock.recv(&mut buf).await?; + let r = &buf[..n]; + if n < 5 || r[..4] != [0xFF; 4] { + bail!("not an A2S answer"); + } + match r[4] { + // Challenge: repeat the query with it appended. + 0x41 if n >= 9 => { + q = A2S_INFO.to_vec(); + q.extend_from_slice(&r[5..9]); + } + 0x49 => return Ok(format!("a2s info ({n} bytes)")), + other => bail!("unexpected A2S answer 0x{other:02x}"), + } + } + bail!("no A2S info after the challenge") + }) + .await + .context("timed out")? +} + +/// Factorio has no cheap unauthenticated query. A connected UDP socket learns +/// of a closed port from the ICMP "port unreachable" (ECONNREFUSED on recv); +/// silence means "not refused". Weak on its own, so a watcher also requires +/// a fresh worker status record saying the container runs. +async fn udp_reachable(addr: SocketAddr, timeout: Duration) -> Result { + let bind: SocketAddr = if addr.is_ipv4() { "0.0.0.0:0".parse()? } else { "[::]:0".parse()? }; + let sock = tokio::net::UdpSocket::bind(bind).await?; + sock.connect(addr).await?; + sock.send(&[0u8]).await?; + let mut buf = [0u8; 64]; + match tokio::time::timeout(timeout, sock.recv(&mut buf)).await { + Ok(Err(e)) if e.kind() == std::io::ErrorKind::ConnectionRefused => bail!("port unreachable"), + Ok(Err(e)) => bail!("{e}"), + Ok(Ok(_)) => Ok("answered".into()), + Err(_) => Ok("not refused".into()), + } +} + +fn split_address(address: &str) -> Option<(String, u16)> { + let (h, p) = address.rsplit_once(':')?; + Some((h.trim_start_matches('[').trim_end_matches(']').to_string(), p.parse().ok()?)) +} + +/// Probe a server's game port the way its players' clients would. +pub async fn probe_server(game: Game, address: &str, timeout: Duration) -> ProbeResult { + let started = Instant::now(); + let method = match game { + Game::Minecraft => "minecraft-status", + Game::Valheim => "a2s-info", + Game::Factorio => "udp-reachable", + }; + let res: Result = async { + let (host, port) = split_address(address).context("address is not host:port")?; + let mut addrs = tokio::net::lookup_host((host.as_str(), port)).await?; + let addr = addrs.next().context("address does not resolve")?; + match game { + Game::Minecraft => minecraft_status(addr, &host, timeout).await, + Game::Valheim => a2s_info(SocketAddr::new(addr.ip(), port.checked_add(1).context("port")?), timeout).await, + Game::Factorio => udp_reachable(addr, timeout).await, + } + } + .await; + let (ok, detail) = match res { + Ok(d) => (true, d), + Err(e) => (false, format!("{e:#}")), + }; + ProbeResult { ts: now_secs(), ok, method: method.into(), rtt_ms: started.elapsed().as_millis() as u64, detail } +} + +/// Decides when a server has been down long enough to fail over. +#[derive(Clone, Debug)] +pub struct HealthTracker { + fail_after_secs: u64, + first_fail: Option, + pub last_ok: Option, +} + +impl HealthTracker { + pub fn new(fail_after: Duration) -> Self { + Self { fail_after_secs: fail_after.as_secs().max(1), first_fail: None, last_ok: None } + } + /// Record one check; `true` once checks have failed without a pass for + /// `fail_after`. + pub fn observe(&mut self, now: u64, ok: bool) -> bool { + if ok { + self.first_fail = None; + self.last_ok = Some(now); + return false; + } + let since = *self.first_fail.get_or_insert(now); + now.saturating_sub(since) >= self.fail_after_secs + } + pub fn reset(&mut self) { + self.first_fail = None; + } +} + +/// Is the server healthy, given a probe and the worker's latest status +/// record (and when it arrived)? A record that says the container is down +/// always fails it; for Factorio, whose probe is weak, a missing or stale +/// record fails it too. +pub fn judge(game: Game, probe: &ProbeResult, status: Option<(&StatusRecord, u64)>, now: u64) -> bool { + if let Some((s, _)) = status + && (!s.running || s.stopped_at.is_some()) + { + return false; + } + if game == Game::Factorio { + let fresh = status.is_some_and(|(s, seen)| s.running && now.saturating_sub(seen) <= 180); + return probe.ok && fresh; + } + probe.ok } // ── buyer ─────────────────────────────────────────────────────────────── -/// What the buyer keeps about a server it rented (`gameserver ls`). +/// What the buyer keeps about a server it rented (`gameserver ls`). The file +/// is mode 0600 and never leaves this machine. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct Rental { pub id: String, @@ -1025,6 +1776,28 @@ pub struct Rental { pub stopped_at: Option, #[serde(default)] pub backup: Option, + // What a failover needs to start the same server elsewhere. + #[serde(default)] + pub ram_gb: u32, + #[serde(default)] + pub cpus: f64, + #[serde(default)] + pub accept_eula: bool, + #[serde(default)] + pub password: Option, + #[serde(default)] + pub server_name: Option, + #[serde(default)] + pub backup_mins: u32, + /// The newest backup mirrored to `worlds/.tar` on this machine. + #[serde(default)] + pub latest_backup: Option, + /// The rental this one replaced after a failover. + #[serde(default)] + pub replaces: Option, + /// The rental that replaced this one after a failover. + #[serde(default)] + pub superseded_by: Option, } pub fn buyer_dir() -> PathBuf { @@ -1035,47 +1808,95 @@ pub fn load_rentals(dir: &Path) -> Vec { std::fs::read(dir.join("rentals.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default() } -pub fn save_rentals(dir: &Path, rentals: &[Rental]) -> Result<()> { - std::fs::create_dir_all(dir)?; - let p = dir.join("rentals.json"); - std::fs::write(&p, serde_json::to_vec_pretty(rentals)?)?; +fn write_private(p: &Path, bytes: &[u8]) -> Result<()> { + if let Some(d) = p.parent() { + std::fs::create_dir_all(d)?; + } + let tmp = p.with_extension("tmp"); + std::fs::write(&tmp, bytes)?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let _ = std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o600)); + std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600))?; } + std::fs::rename(&tmp, p)?; Ok(()) } +pub fn save_rentals(dir: &Path, rentals: &[Rental]) -> Result<()> { + write_private(&dir.join("rentals.json"), &serde_json::to_vec_pretty(rentals)?) +} + +fn upsert_rental(dir: &Path, r: &Rental) -> Result<()> { + let mut all = load_rentals(dir); + match all.iter_mut().find(|x| x.id == r.id) { + Some(x) => *x = r.clone(), + None => all.push(r.clone()), + } + save_rentals(dir, &all) +} + +/// The buyer's long-term backup key (`

/backup.key`, 0600), created on +/// first use. Every worker seals its backups to the public half. +pub fn backup_keypair(dir: &Path) -> Result { + let p = dir.join("backup.key"); + if let Ok(h) = std::fs::read_to_string(&p) { + let secret = crypto::secret_from_hex(h.trim()).context("backup.key is not a 32-byte hex key")?; + let public_hex = hex::encode(x25519_dalek::PublicKey::from(&secret).as_bytes()); + return Ok(crypto::KeyPair { secret, public_hex }); + } + let kp = crypto::keypair(); + write_private(&p, hex::encode(kp.secret.to_bytes()).as_bytes())?; + Ok(kp) +} + pub struct StartRequest { pub game: Game, pub hours: u32, pub ram_gb: u32, pub cpus: f64, pub accept_eula: bool, - /// A manifest hash from an earlier stop, or a `.tar` laid out like a backup. + /// A manifest hash from an earlier backup, or a `.tar` laid out like a backup. pub world: Option, pub password: Option, pub server_name: Option, + /// Minutes between live backups (0 = only at stop/expiry). + pub backup_mins: u32, pub max_price_usd: f64, pub workers: Vec, + /// Never use these workers (a failover excludes the one that failed). + pub exclude: Vec, pub timeout: Duration, } -async fn auction(net: Arc, workload: &str, spec: Value, caps: Vec, workers: Vec, max_price: f64, timeout: Duration) -> Result<(crate::buyer::AuctionOutcome, Value)> { - let a = crate::JobAuction::new(workload, spec) +async fn auction( + net: Arc, + workload: &str, + spec: Value, + caps: Vec, + workers: Vec, + exclude: Vec, + max_price: f64, + timeout: Duration, + notify: Option>, +) -> Result<(crate::buyer::AuctionOutcome, Value)> { + let mut a = crate::JobAuction::new(workload, spec) .with_required_capabilities(caps) .with_allowed_bidders(workers) + .with_excluded_bidders(exclude) .with_max_price_usd(max_price) .with_bid_window(Duration::from_secs(4)) .with_deadline(timeout) .with_receipt_timeout(timeout); + if let Some(tx) = notify { + a = a.with_accept_notify(tx); + } let o = crate::run_auction(net.clone(), a).await?; if o.receipt.status != JobStatus::Completed { bail!("worker {} reported {:?}", o.accepted_bid.bidder_peer_id, o.receipt.status); } let hash = o.receipt.output_hash.clone().context("worker returned no result hash")?; - let bytes = crate::gateway_util::fetch_verified(&net, &hash, &o.accepted_bid.bidder_peer_id, 64 * 1024, Duration::from_secs(30)).await.map_err(|e| anyhow::anyhow!("fetch result: {e:?}"))?; + let bytes = crate::gateway_util::fetch_verified(&net, &hash, &o.accepted_bid.bidder_peer_id, 64 * 1024, Duration::from_secs(30)).await.map_err(|e| anyhow!("fetch result: {e:?}"))?; let v: Value = serde_json::from_slice(&bytes)?; if v["ok"] != json!(true) { bail!("worker result: {v}"); @@ -1083,54 +1904,76 @@ async fn auction(net: Arc, workload: &str, spec: Value, caps: Vec Ok((o, v)) } -/// Put a world into the local store for the worker to fetch; returns the manifest hash. -async fn publish_world(store: &ChunkStore, net: &Libp2pNetwork, dir: &Path, game: Game, world: &str) -> Result { +/// Encrypt a world under a fresh key and put it in the local store for the +/// worker to fetch; returns the manifest hash and the key. +async fn publish_world(store: &ChunkStore, net: &Libp2pNetwork, dir: &Path, game: Game, world: &str) -> Result<(String, [u8; 32])> { let (tar, saved_manifest) = if is_hex(world, 64) { (dir.join("worlds").join(format!("{world}.tar")), Some(dir.join("worlds").join(format!("{world}.json")))) } else { (PathBuf::from(world), None) }; let f = tokio::fs::File::open(&tar).await.with_context(|| format!("open world {}", tar.display()))?; - let m = chunk_into_store(store, Some(net), game, f).await?; - if let Some(p) = saved_manifest { - let want = WorldManifest::from_bytes(&std::fs::read(&p).with_context(|| format!("read {}", p.display()))?)?; - if want != m { + let key = random_key(); + let m = chunk_into_store(store, Some(net), game, f, &key, WorldEnc::handed_off()).await?; + if let Some(p) = saved_manifest + && let Ok(b) = std::fs::read(&p) + { + // The manifest saved beside the tar names its game and plaintext sha256. + let saved: Value = serde_json::from_slice(&b).with_context(|| format!("read {}", p.display()))?; + if saved["sha256"].as_str() != Some(m.sha256.as_str()) { bail!("{} does not match its manifest", tar.display()); } - if want.game != game { - bail!("that world is a {} world", want.game.name()); + if saved["game"].as_str().is_some_and(|g| g != game.name()) { + bail!("that world is a {} world", saved["game"].as_str().unwrap_or("?")); } } - put_manifest(store, net, &m).await + Ok((put_manifest(store, net, &m).await?, key)) } -/// Rent a server: auction it, wait for the address, remember the rental. -pub async fn start(req: StartRequest) -> Result { - let token = { - let b: [u8; 32] = rand::random(); - hex::encode(b) - }; - let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; - let dir = buyer_dir(); - let world = match &req.world { - Some(w) => Some(publish_world(&sup.store, &sup.libp2p, &dir, req.game, w).await?), - None => None, +/// Rent a server on `net` (a booted buyer node), remembering it under `dir`. +pub async fn start_with(net: Arc, store: &ChunkStore, dir: &Path, req: StartRequest) -> Result { + let token = hex::encode(rand::random::<[u8; 32]>()); + let backup = backup_keypair(dir)?; + let (world, world_key) = match &req.world { + Some(w) => { + let (h, k) = publish_world(store, &net, dir, req.game, w).await?; + (Some(h), Some(hex::encode(k))) + } + None => (None, None), }; + let kp = crate::handoff::keypair(); let spec = StartSpec { - v: 1, + v: SPEC_VERSION, game: req.game, hours: req.hours, ram_gb: req.ram_gb, cpus: req.cpus, accept_eula: req.accept_eula, stop_hash: sha256_ref(token.as_bytes()), + client_pubkey: kp.public_hex.clone(), + backup_pubkey: backup.public_hex.clone(), + backup_mins: req.backup_mins, world, - password: req.password.clone(), server_name: req.server_name.clone(), }; - spec.validate().map_err(|e| anyhow::anyhow!(e))?; - tokio::time::sleep(Duration::from_secs(5)).await; - let (o, v) = auction(sup.libp2p.clone(), WORKLOAD_START, serde_json::to_value(&spec)?, vec![ROLE_TAG.into(), req.game.tag()], req.workers.clone(), req.max_price_usd, req.timeout).await?; + spec.validate().map_err(|e| anyhow!(e))?; + let secrets = StartSecrets { v: 1, password: req.password.clone(), world_key }; + secrets.validate(&spec).map_err(|e| anyhow!(e))?; + let mut handoff = crate::handoff::BuyerHandoff::spawn(net.clone(), WORKLOAD_START, kp, serde_json::to_vec(&secrets)?)?; + let res = auction( + net.clone(), + WORKLOAD_START, + serde_json::to_value(&spec)?, + vec![ROLE_TAG.into(), req.game.tag()], + req.workers.clone(), + req.exclude.clone(), + req.max_price_usd, + req.timeout, + Some(handoff.notify()?), + ) + .await; + handoff.finish(); + let (o, v) = res?; let rental = Rental { id: v["id"].as_str().context("result has no id")?.to_string(), game: req.game, @@ -1143,65 +1986,278 @@ pub async fn start(req: StartRequest) -> Result { price_usd: o.accepted_bid.price_usd, stopped_at: None, backup: None, + ram_gb: req.ram_gb, + cpus: req.cpus, + accept_eula: req.accept_eula, + password: req.password, + server_name: req.server_name, + backup_mins: req.backup_mins, + latest_backup: None, + replaces: None, + superseded_by: None, }; - let mut all = load_rentals(&dir); - all.push(rental.clone()); - save_rentals(&dir, &all)?; + upsert_rental(dir, &rental)?; Ok(rental) } -/// Stop a rental early (or collect the backup of one that expired) and -/// download its world to `/worlds/.tar`. -pub async fn stop(id: &str, timeout: Duration) -> Result { - let dir = buyer_dir(); - let mut all = load_rentals(&dir); - let idx = all.iter().position(|r| r.id == id).with_context(|| format!("no rental {id} on this machine (see `c0mpute gameserver ls`)"))?; - let r = all[idx].clone(); +/// Rent a server: boot a buyer node, auction it, wait for the address, +/// remember the rental. +pub async fn start(req: StartRequest) -> Result { let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; tokio::time::sleep(Duration::from_secs(5)).await; + start_with(sup.libp2p.clone(), &sup.store, &buyer_dir(), req).await +} + +/// Download, decrypt and keep a backup as `/worlds/.tar` (+ its +/// manifest as `.json`). +pub async fn download_backup(net: &Libp2pNetwork, store: &ChunkStore, dir: &Path, manifest: &str, provider: &str) -> Result { + let key = backup_keypair(dir)?; + let worlds = dir.join("worlds"); + tokio::fs::create_dir_all(&worlds).await?; + let tmp = worlds.join(format!("{manifest}.partial")); + let f = tokio::fs::File::create(&tmp).await?; + let res = fetch_world(net, store, manifest, provider, |m| { + let epk = m.enc.epk.as_deref().context("backup has no one-time key; it was not sealed to us")?; + backup_key(&key.secret, epk, epk, &key.public_hex) + }, f) + .await; + let m = match res { + Ok(m) => m, + Err(e) => { + let _ = tokio::fs::remove_file(&tmp).await; + return Err(e.context("download world backup")); + } + }; + tokio::fs::rename(&tmp, worlds.join(format!("{manifest}.tar"))).await?; + tokio::fs::write(worlds.join(format!("{manifest}.json")), m.to_bytes()).await?; + Ok(m) +} + +/// Stop a rental early (or collect the backup of one that expired) and +/// download its world to `/worlds/.tar`. +pub async fn stop_with(net: Arc, store: &ChunkStore, dir: &Path, id: &str, timeout: Duration) -> Result { + let all = load_rentals(dir); + let r = all.iter().find(|r| r.id == id).cloned().with_context(|| format!("no rental {id} on this machine (see `c0mpute gameserver ls`)"))?; let spec = StopSpec { v: 1, id: r.id.clone(), token: r.token.clone() }; - let (_, v) = auction(sup.libp2p.clone(), WORKLOAD_STOP, serde_json::to_value(&spec)?, vec![ROLE_TAG.into()], vec![r.worker.clone()], 0.0, timeout).await?; + let (_, v) = auction(net.clone(), WORKLOAD_STOP, serde_json::to_value(&spec)?, vec![ROLE_TAG.into()], vec![r.worker.clone()], vec![], 0.0, timeout, None).await?; let backup: Option = serde_json::from_value(v["backup"].clone()).ok().flatten(); let mut updated = r.clone(); updated.stopped_at = Some(now_secs()); if let Some(b) = &backup { - let worlds = dir.join("worlds"); - tokio::fs::create_dir_all(&worlds).await?; - let tmp = worlds.join(format!("{}.partial", b.manifest)); - let f = tokio::fs::File::create(&tmp).await?; - let m = fetch_world(&sup.libp2p, &sup.store, &b.manifest, &r.worker, f).await.context("download world backup")?; - tokio::fs::rename(&tmp, worlds.join(format!("{}.tar", b.manifest))).await?; - tokio::fs::write(worlds.join(format!("{}.json", b.manifest)), m.to_bytes()).await?; + download_backup(&net, store, dir, &b.manifest, &r.worker).await?; updated.backup = Some(b.manifest.clone()); + updated.latest_backup = Some(b.manifest.clone()); } - all[idx] = updated.clone(); - save_rentals(&dir, &all)?; + upsert_rental(dir, &updated)?; Ok(updated) } +pub async fn stop(id: &str, timeout: Duration) -> Result { + let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; + tokio::time::sleep(Duration::from_secs(5)).await; + stop_with(sup.libp2p.clone(), &sup.store, &buyer_dir(), id, timeout).await +} + +/// `gameserver watch` knobs. +#[derive(Clone, Debug)] +pub struct WatchOptions { + pub probe_every: Duration, + pub probe_timeout: Duration, + /// Fail over after the server has failed every check for this long. + pub fail_after: Duration, + pub failover: bool, + pub max_price_usd: f64, + pub workers: Vec, + pub start_timeout: Duration, + /// Return after the first failover instead of watching the replacement. + pub stop_after_failover: bool, +} + +impl Default for WatchOptions { + fn default() -> Self { + Self { + probe_every: Duration::from_secs(30), + probe_timeout: Duration::from_secs(5), + fail_after: Duration::from_secs(5 * 60), + failover: true, + max_price_usd: 1.0, + workers: Vec::new(), + start_timeout: Duration::from_secs(900), + stop_after_failover: false, + } + } +} + +/// How a watch ended. +#[derive(Clone, Debug, PartialEq)] +pub enum WatchEnd { + /// The reservation ended or the server was stopped. + Ended(Rental), + /// It failed over and `stop_after_failover` was set. + FailedOver { from: Rental, to: Rental }, +} + +fn log_probe(dir: &Path, id: &str, p: &ProbeResult) { + let d = dir.join("health"); + let _ = std::fs::create_dir_all(&d); + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(d.join(format!("{id}.jsonl"))) { + use std::io::Write; + let _ = writeln!(f, "{}", serde_json::to_string(p).unwrap_or_default()); + } +} + +/// Read a server's probe log. +pub fn health_log(dir: &Path, id: &str) -> Vec { + std::fs::read_to_string(dir.join("health").join(format!("{id}.jsonl"))).unwrap_or_default().lines().filter_map(|l| serde_json::from_str(l).ok()).collect() +} + +/// Watch a rental: probe its port every `probe_every` (logged to +/// `/health/.jsonl`), mirror each backup its worker announces, and +/// when it has failed every check for `fail_after`, start the same game on +/// another worker from the newest mirrored backup. +pub async fn watch_with(net: Arc, store: &ChunkStore, dir: &Path, id: &str, opts: WatchOptions) -> Result { + net.subscribe(STATUS_TOPIC).await?; + let mut rx = net.messages(); + let mut rental = load_rentals(dir).into_iter().find(|r| r.id == id).with_context(|| format!("no rental {id} on this machine"))?; + let mut health = HealthTracker::new(opts.fail_after); + let mut status: Option<(StatusRecord, u64)> = None; + let mut tick = tokio::time::interval(opts.probe_every); + loop { + if now_secs() >= rental.expires_at || rental.stopped_at.is_some() { + return Ok(WatchEnd::Ended(rental)); + } + tokio::select! { + m = rx.recv() => { + let msg = match m { + Ok(m) => m, + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(_) => bail!("network closed"), + }; + if msg.topic != STATUS_TOPIC || msg.source.map(|p| p.to_base58()).as_deref() != Some(rental.worker.as_str()) { + continue; + } + let Ok(rec) = serde_json::from_slice::(&msg.data) else { continue }; + if rec.t != STATUS_KIND || rec.id != rental.id || rec.worker != rental.worker { + continue; + } + if let Some(b) = &rec.backup + && rental.latest_backup.as_deref() != Some(b.manifest.as_str()) + { + match download_backup(&net, store, dir, &b.manifest, &rental.worker).await { + Ok(_) => { + info!(id = %rental.id, manifest = %b.manifest, "gameserver watch: backup mirrored"); + rental.latest_backup = Some(b.manifest.clone()); + upsert_rental(dir, &rental)?; + } + Err(e) => warn!(id = %rental.id, err = %format!("{e:#}"), "gameserver watch: could not mirror backup"), + } + } + status = Some((rec, now_secs())); + } + _ = tick.tick() => { + let p = probe_server(rental.game, &rental.address, opts.probe_timeout).await; + let now = now_secs(); + let ok = judge(rental.game, &p, status.as_ref().map(|(s, at)| (s, *at)), now); + log_probe(dir, &rental.id, &ProbeResult { ok, ..p.clone() }); + if !health.observe(now, ok) || !opts.failover { + continue; + } + warn!(id = %rental.id, worker = %rental.worker, "gameserver watch: failing checks; failing over"); + match failover(net.clone(), store, dir, &rental, &opts).await { + Ok(next) => { + let from = rental.clone(); + if opts.stop_after_failover { + return Ok(WatchEnd::FailedOver { from, to: next }); + } + info!(from = %from.id, to = %next.id, address = %next.address, "gameserver watch: failed over"); + rental = next; + status = None; + health = HealthTracker::new(opts.fail_after); + } + Err(e) => { + warn!(id = %rental.id, err = %format!("{e:#}"), "gameserver watch: failover failed; retrying next check"); + health.reset(); + } + } + } + } + } +} + +pub async fn watch(id: &str, opts: WatchOptions) -> Result { + let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; + tokio::time::sleep(Duration::from_secs(5)).await; + watch_with(sup.libp2p.clone(), &sup.store, &buyer_dir(), id, opts).await +} + +/// Start `old`'s game on a different worker from its newest mirrored backup +/// for the rest of its reservation, and link the two rentals. +pub async fn failover(net: Arc, store: &ChunkStore, dir: &Path, old: &Rental, opts: &WatchOptions) -> Result { + let left = old.expires_at.saturating_sub(now_secs()); + if left == 0 { + bail!("the reservation already ended"); + } + let req = StartRequest { + game: old.game, + hours: left.div_ceil(3600).clamp(1, u64::from(MAX_HOURS)) as u32, + ram_gb: old.ram_gb.max(old.game.min_ram_gb()), + cpus: if old.cpus > 0.0 { old.cpus } else { 2.0 }, + accept_eula: old.accept_eula, + world: old.latest_backup.clone(), + password: old.password.clone(), + server_name: old.server_name.clone(), + backup_mins: if old.backup_mins == 0 { 0 } else { old.backup_mins.clamp(MIN_BACKUP_MINS, MAX_BACKUP_MINS) }, + max_price_usd: opts.max_price_usd, + workers: opts.workers.clone(), + exclude: vec![old.worker.clone()], + timeout: opts.start_timeout, + }; + if req.world.is_none() { + warn!(id = %old.id, "gameserver failover: no backup mirrored yet; the replacement starts a fresh world"); + } + let mut next = start_with(net, store, dir, req).await?; + next.replaces = Some(old.id.clone()); + upsert_rental(dir, &next)?; + let mut prev = old.clone(); + prev.superseded_by = Some(next.id.clone()); + upsert_rental(dir, &prev)?; + Ok(next) +} + #[cfg(test)] mod tests { use super::*; + fn key() -> String { + crypto::keypair().public_hex + } + fn spec(game: Game) -> StartSpec { StartSpec { - v: 1, + v: SPEC_VERSION, game, hours: 4, ram_gb: 4, cpus: 2.0, accept_eula: true, stop_hash: sha256_ref(b"t"), + client_pubkey: key(), + backup_pubkey: key(), + backup_mins: 30, world: None, - password: (game == Game::Valheim).then(|| "hunter22".into()), server_name: None, } } + fn secrets(game: Game) -> StartSecrets { + StartSecrets { v: 1, password: (game == Game::Valheim).then(|| "hunter22".into()), world_key: None } + } + #[test] fn spec_validation() { for g in Game::ALL { assert!(spec(g).validate().is_ok(), "{g:?}"); + assert!(secrets(g).validate(&spec(g)).is_ok(), "{g:?}"); } let bad = |f: &dyn Fn(&mut StartSpec)| { let mut s = spec(Game::Minecraft); @@ -1218,18 +2274,19 @@ mod tests { assert!(bad(&|s| s.cpus = 64.0)); assert!(bad(&|s| s.stop_hash = "sha256:XYZ".into())); assert!(bad(&|s| s.world = Some("../etc".into()))); - assert!(bad(&|s| s.password = Some("hunter22".into())), "only valheim takes a password"); assert!(bad(&|s| s.server_name = Some("$(rm -rf /)".into()))); - assert!(bad(&|s| s.v = 2)); - // Factorio does not need the EULA flag; Valheim needs a password. + assert!(bad(&|s| s.v = 1), "v1 offers put the password in public"); + assert!(bad(&|s| s.client_pubkey = "nope".into())); + assert!(bad(&|s| s.backup_pubkey = String::new())); + assert!(bad(&|s| s.backup_mins = 1)); + assert!(bad(&|s| s.backup_mins = MAX_BACKUP_MINS + 1)); + assert!(!bad(&|s| s.backup_mins = 0), "0 turns live backups off"); + // Factorio does not need the EULA flag. let mut f = spec(Game::Factorio); f.accept_eula = false; assert!(f.validate().is_ok()); let mut v = spec(Game::Valheim); - v.password = Some("abc".into()); - assert!(v.validate().is_err()); v.ram_gb = 2; - v.password = Some("hunter22".into()); assert!(v.validate().is_err(), "valheim needs 4 GB"); // Unknown games never parse. let mut raw = serde_json::to_value(spec(Game::Minecraft)).unwrap(); @@ -1239,6 +2296,23 @@ mod tests { assert_eq!(Game::parse("terraria"), None); } + #[test] + fn secrets_never_ride_in_the_offer() { + // There is no password field in a start spec at all. + let raw = serde_json::to_string(&spec(Game::Valheim)).unwrap(); + assert!(!raw.contains("password") && !raw.contains("hunter22"), "{raw}"); + // Secrets are checked against the spec. + let v = spec(Game::Valheim); + assert!(StartSecrets { v: 1, password: None, world_key: None }.validate(&v).is_err(), "valheim needs one"); + assert!(StartSecrets { v: 1, password: Some("abc".into()), world_key: None }.validate(&v).is_err()); + assert!(StartSecrets { v: 1, password: Some("hunter22".into()), world_key: None }.validate(&spec(Game::Factorio)).is_err()); + let mut w = spec(Game::Minecraft); + w.world = Some("a".repeat(64)); + assert!(StartSecrets { v: 1, password: None, world_key: None }.validate(&w).is_err(), "a restore needs its key"); + assert!(StartSecrets { v: 1, password: None, world_key: Some("b".repeat(64)) }.validate(&w).is_ok()); + assert!(StartSecrets { v: 1, password: None, world_key: Some("b".repeat(64)) }.validate(&spec(Game::Minecraft)).is_err()); + } + #[test] fn stop_spec_validation() { let ok = StopSpec { v: 1, id: server_id("job"), token: "a".repeat(64) }; @@ -1249,9 +2323,9 @@ mod tests { } #[test] - fn docker_argv_has_limits_ports_volume_and_no_shell() { + fn docker_argv_has_limits_ports_volume_expiry_and_no_shell() { let s = spec(Game::Minecraft); - let plan = ContainerPlan::new("gs-0123456789abcdef", &s, 27500, 1_000); + let plan = ContainerPlan::new("gs-0123456789abcdef", &s, &secrets(Game::Minecraft), 27500, 1_000, 1_000 + 4 * 3600); let a = plan.create_argv(); let pair = |k: &str, v: &str| a.windows(2).any(|w| w[0] == k && w[1] == v); assert_eq!(a[0], "create"); @@ -1262,25 +2336,48 @@ mod tests { assert!(pair("--pids-limit", "1024")); assert!(pair("--restart", "no")); assert!(pair("--security-opt", "no-new-privileges")); + assert!(pair("--stop-timeout", "60")); assert!(pair("-p", "27500:25565/tcp")); assert!(pair("-v", "c0mpute-gs-0123456789abcdef:/data")); assert!(pair("-e", "EULA=TRUE")); assert!(pair("-e", "MEMORY=3328M")); - assert!(pair("--label", "c0mpute.gameserver.expires=1000")); - assert_eq!(a.last().unwrap(), Game::Minecraft.image()); + assert!(pair("--label", &format!("c0mpute.gameserver.expires={}", 1_000 + 4 * 3600))); + // Hard expiry without the worker: --rm + --init + timeout around the image's own command. + assert!(a.contains(&"--rm".to_string()) && a.contains(&"--init".to_string())); + assert!(pair("--entrypoint", "timeout")); + let img = a.iter().position(|x| x == Game::Minecraft.image()).unwrap(); + assert_eq!(&a[img + 1..], &["-s", "TERM", "-k", "90s", "14400s", "/image/scripts/start"]); assert!(!a.iter().any(|x| x == "sh" || x == "-c" || x == "--privileged" || x.starts_with("--network=host"))); - // Valheim maps two consecutive UDP ports and passes its password as one argument. - let v = ContainerPlan::new("gs-0123456789abcdef", &spec(Game::Valheim), 27510, 1).create_argv(); + // Valheim maps two consecutive UDP ports; its password comes from the sealed secrets. + let v = ContainerPlan::new("gs-0123456789abcdef", &spec(Game::Valheim), &secrets(Game::Valheim), 27510, 1, 2).create_argv(); assert!(v.windows(2).any(|w| w[0] == "-p" && w[1] == "27510:2456/udp")); assert!(v.windows(2).any(|w| w[0] == "-p" && w[1] == "27511:2457/udp")); assert!(v.windows(2).any(|w| w[0] == "-e" && w[1] == "SERVER_PASS=hunter22")); assert!(v.windows(2).any(|w| w[0] == "-v" && w[1] == "c0mpute-gs-0123456789abcdef:/config")); + assert_eq!(v.last().unwrap(), "/usr/local/sbin/bootstrap"); // Images are pinned by digest. for g in Game::ALL { assert!(g.image().contains("@sha256:"), "{g:?}"); + assert!(!g.command().is_empty()); } } + #[test] + fn tar_argv_reads_the_volume_read_only_with_no_network() { + let a = tar_argv("c0mpute-gs-x", Game::Factorio); + assert!(a.windows(2).any(|w| w[0] == "--network" && w[1] == "none")); + assert!(a.windows(2).any(|w| w[0] == "-v" && w[1] == "c0mpute-gs-x:/factorio:ro")); + assert!(a.windows(2).any(|w| w[0] == "--entrypoint" && w[1] == "tar")); + assert_eq!(a.last().unwrap(), "factorio"); + } + + #[test] + fn labelled_listing_and_expired_containers() { + let l = parse_labelled("c0mpute-gs-a\t100\nc0mpute-gs-b\t9999999999\nc0mpute-gs-c\t\n\n"); + assert_eq!(l, vec![("c0mpute-gs-a".into(), 100), ("c0mpute-gs-b".into(), 9_999_999_999), ("c0mpute-gs-c".into(), 0)]); + assert_eq!(expired_containers(&l, 1_000), vec!["c0mpute-gs-a".to_string(), "c0mpute-gs-c".to_string()]); + } + fn state(id: &str, ram_mb: u64, port: u16, expires: u64, stopped: Option) -> ServerState { ServerState { id: id.into(), @@ -1294,6 +2391,8 @@ mod tests { expires_at: expires, stop_hash: sha256_ref(b"t"), buyer: "b".into(), + backup_pubkey: key(), + backup_every_secs: 1800, stopped_at: stopped, backup: None, backup_error: None, @@ -1301,7 +2400,7 @@ mod tests { } #[test] - fn expiry() { + fn expiry_and_backup_schedule() { assert_eq!(expires_at(100, 4), 100 + 4 * 3600); let mut m = HashMap::new(); m.insert("a".into(), state("a", 1024, 1, 1_000, None)); @@ -1310,6 +2409,14 @@ mod tests { assert!(due_for_stop(&m, 999).is_empty()); assert_eq!(due_for_stop(&m, 1_000), vec!["a".to_string()]); assert_eq!(due_for_stop(&m, 9_999), vec!["a".to_string(), "b".to_string()]); + // Live backups: every 30 min from the start, then from the last backup. + assert!(due_for_backup(&m, 1_799).is_empty()); + assert_eq!(due_for_backup(&m, 1_800), vec!["b".to_string()], "a is past expiry, c stopped"); + m.get_mut("b").unwrap().backup = Some(Backup { manifest: "x".into(), bytes: 1, sha256: "s".into(), at: 1_800 }); + assert!(due_for_backup(&m, 3_599).is_empty()); + assert_eq!(due_for_backup(&m, 3_600), vec!["b".to_string()]); + m.get_mut("b").unwrap().backup_every_secs = 0; + assert!(due_for_backup(&m, 4_000).is_empty(), "0 = no live backups"); } #[test] @@ -1327,7 +2434,6 @@ mod tests { assert!(has_capacity(&cfg, 0, &m, &spec(Game::Minecraft)).unwrap_err().contains("slots")); m.get_mut("b").unwrap().stopped_at = Some(1); assert!(has_capacity(&cfg, 0, &m, &spec(Game::Minecraft)).is_ok(), "stopped servers free their slot"); - // Default budget is half the box. let half = crate::config::GameserverConfig { max_total_ram_gb: 0, ..cfg.clone() }; assert!(has_capacity(&half, 6 * 1024, &HashMap::new(), &spec(Game::Minecraft)).is_err()); assert!(has_capacity(&half, 16 * 1024, &HashMap::new(), &spec(Game::Minecraft)).is_ok()); @@ -1347,11 +2453,14 @@ mod tests { assert_eq!(public_host(Some("localhost"), Some("http://box.local"), Some("/ip4/127.0.0.1/tcp/1")), None); assert_eq!(public_host(None, None, None), None); assert_eq!(host_port_str("2001:db8::1", 5), "[2001:db8::1]:5"); + assert_eq!(split_address("[2001:db8::1]:5"), Some(("2001:db8::1".into(), 5))); + assert_eq!(split_address("1.2.3.4:27500"), Some(("1.2.3.4".into(), 27500))); + assert_eq!(split_address("nope"), None); } #[test] fn manifest_checks() { - let mut m = WorldManifest::new(Game::Factorio); + let mut m = WorldManifest::new(Game::Factorio, WorldEnc::handed_off()); m.bytes = CHUNK_BYTES as u64 + 1; m.sha256 = sha256_ref(b"x"); m.chunks = vec!["a".repeat(64), "b".repeat(64)]; @@ -1365,24 +2474,147 @@ mod tests { let mut huge = m.clone(); huge.bytes = MAX_WORLD_BYTES + 1; assert!(huge.check().is_err()); + let mut plain = m.clone(); + plain.enc.scheme = "none".into(); + assert!(plain.check().is_err(), "unencrypted worlds are refused"); + let mut v1 = serde_json::to_value(&m).unwrap(); + v1["v"] = json!(1); + v1.as_object_mut().unwrap().remove("enc"); + assert!(WorldManifest::from_bytes(&serde_json::to_vec(&v1).unwrap()).is_err()); } #[tokio::test] - async fn chunking_round_trips_through_the_store() { + async fn encrypted_chunking_round_trips_and_only_the_key_reads_it() { let dir = tempfile::tempdir().unwrap(); let store = ChunkStore::open(dir.path()).await.unwrap(); let data: Vec = (0..(CHUNK_BYTES * 2 + 17)).map(|i| (i % 251) as u8).collect(); - let m = chunk_into_store(&store, None, Game::Minecraft, &data[..]).await.unwrap(); + let k = random_key(); + let m = chunk_into_store(&store, None, Game::Minecraft, &data[..], &k, WorldEnc::handed_off()).await.unwrap(); m.check().unwrap(); assert_eq!(m.chunks.len(), 3); assert_eq!(m.bytes, data.len() as u64); assert_eq!(m.sha256, sha256_ref(&data)); let mut back = Vec::new(); - for c in &m.chunks { - back.extend(store.get(&Hash::from_hex(c).unwrap()).await.unwrap()); + for (i, c) in m.chunks.iter().enumerate() { + let sealed = store.get(&Hash::from_hex(c).unwrap()).await.unwrap(); + assert_ne!(&sealed[..64], &data[i * CHUNK_BYTES..i * CHUNK_BYTES + 64], "chunks are stored encrypted"); + assert!(open_chunk(&random_key(), i as u64, &sealed).is_err(), "another key cannot read it"); + assert!(open_chunk(&k, i as u64 + 1, &sealed).is_err(), "a chunk cannot move position"); + back.extend(open_chunk(&k, i as u64, &sealed).unwrap()); } assert_eq!(back, data); - let empty = chunk_into_store(&store, None, Game::Minecraft, &b""[..]).await.unwrap(); + let empty = chunk_into_store(&store, None, Game::Minecraft, &b""[..], &k, WorldEnc::handed_off()).await.unwrap(); assert!(empty.chunks.is_empty() && empty.check().is_ok()); } + + #[test] + fn backups_are_sealed_to_the_buyer_key() { + let buyer = crypto::keypair(); + let e = crypto::keypair(); + let worker_side = backup_key(&e.secret, &buyer.public_hex, &e.public_hex, &buyer.public_hex).unwrap(); + let buyer_side = backup_key(&buyer.secret, &e.public_hex, &e.public_hex, &buyer.public_hex).unwrap(); + assert_eq!(worker_side, buyer_side); + let eve = crypto::keypair(); + assert_ne!(worker_side, backup_key(&eve.secret, &e.public_hex, &e.public_hex, &buyer.public_hex).unwrap()); + } + + #[test] + fn backup_key_file_is_created_once_and_private() { + let dir = tempfile::tempdir().unwrap(); + let a = backup_keypair(dir.path()).unwrap(); + let b = backup_keypair(dir.path()).unwrap(); + assert_eq!(a.public_hex, b.public_hex); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(dir.path().join("backup.key")).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + } + } + + #[test] + fn minecraft_status_codec() { + let req = minecraft_status_request("play.example.com", 25565); + // length-prefixed handshake, then the 1-byte status request. + assert_eq!(&req[req.len() - 2..], &[0x01, 0x00]); + assert_eq!(req[0] as usize, req.len() - 3); + let resp = minecraft_status_response(r#"{"version":{"name":"1.21"},"players":{"online":3,"max":10}}"#); + let mut at = 0; + let len = read_varint_slice(&resp, &mut at).unwrap() as usize; + let v = parse_minecraft_status(&resp[at..at + len]).unwrap(); + assert_eq!(v["players"]["online"], 3); + assert!(parse_minecraft_status(&[0x01, 0x00]).is_err()); + assert!(parse_minecraft_status(&[0x00, 0x05, b'{']).is_err()); + let mut big = Vec::new(); + put_varint(&mut big, 300); + assert_eq!(big, vec![0xAC, 0x02]); + } + + #[test] + fn health_tracker_fails_over_only_after_a_sustained_outage() { + let mut h = HealthTracker::new(Duration::from_secs(300)); + assert!(!h.observe(0, true)); + assert!(!h.observe(30, false)); + assert!(!h.observe(200, false)); + assert!(!h.observe(250, true), "one pass resets the outage"); + assert!(!h.observe(300, false)); + assert!(!h.observe(599, false)); + assert!(h.observe(600, false)); + assert_eq!(h.last_ok, Some(250)); + } + + #[test] + fn judging_uses_the_signed_status_record() { + let up = ProbeResult { ts: 0, ok: true, method: "x".into(), rtt_ms: 1, detail: String::new() }; + let down = ProbeResult { ok: false, ..up.clone() }; + let rec = StatusRecord { t: STATUS_KIND.into(), v: 1, id: "gs-x".into(), worker: "w".into(), game: Game::Minecraft, address: "a:1".into(), running: true, expires_at: 9, stopped_at: None, backup: None, ts: 0 }; + assert!(judge(Game::Minecraft, &up, None, 0)); + assert!(!judge(Game::Minecraft, &down, Some((&rec, 0)), 0)); + let stopped = StatusRecord { running: false, ..rec.clone() }; + assert!(!judge(Game::Minecraft, &up, Some((&stopped, 0)), 0), "the worker says it is down"); + // Factorio's probe is weak: it also needs a fresh "running" record. + assert!(!judge(Game::Factorio, &up, None, 0)); + assert!(judge(Game::Factorio, &up, Some((&rec, 100)), 200)); + assert!(!judge(Game::Factorio, &up, Some((&rec, 0)), 500), "stale record"); + } + + #[tokio::test] + async fn probes_answer_for_live_servers_and_fail_for_dead_ports() { + // A Minecraft-status responder on loopback. + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut s, _)) = l.accept().await { + tokio::spawn(async move { + let mut buf = [0u8; 512]; + let _ = s.read(&mut buf).await; + let _ = s.write_all(&minecraft_status_response(r#"{"version":{"name":"fake"},"players":{"online":0}}"#)).await; + }); + } + }); + let p = probe_server(Game::Minecraft, &format!("127.0.0.1:{port}"), Duration::from_secs(3)).await; + assert!(p.ok, "{p:?}"); + assert_eq!(p.method, "minecraft-status"); + // An A2S responder (with a challenge round) on game port + 1. + let game = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + let gport = game.local_addr().unwrap().port(); + let q = tokio::net::UdpSocket::bind(("127.0.0.1", gport + 1)).await; + if let Ok(q) = q { + tokio::spawn(async move { + let mut buf = [0u8; 256]; + while let Ok((n, from)) = q.recv_from(&mut buf).await { + let reply: Vec = if n == A2S_INFO.len() { b"\xFF\xFF\xFF\xFFA\x01\x02\x03\x04".to_vec() } else { b"\xFF\xFF\xFF\xFFI\x11fake".to_vec() }; + let _ = q.send_to(&reply, from).await; + } + }); + let p = probe_server(Game::Valheim, &format!("127.0.0.1:{gport}"), Duration::from_secs(3)).await; + assert!(p.ok, "{p:?}"); + } + drop(game); + // A closed port: refused (TCP) / unreachable (UDP). + let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); + assert!(!probe_server(Game::Minecraft, &format!("127.0.0.1:{closed}"), Duration::from_secs(2)).await.ok); + let uclosed = std::net::UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); + assert!(!probe_server(Game::Factorio, &format!("127.0.0.1:{uclosed}"), Duration::from_secs(2)).await.ok); + } } diff --git a/node/crates/c0mpute-core/src/handoff.rs b/node/crates/c0mpute-core/src/handoff.rs new file mode 100644 index 00000000..86f50c17 --- /dev/null +++ b/node/crates/c0mpute-core/src/handoff.rs @@ -0,0 +1,251 @@ +//! Hand a small secret to the one worker that won an auction. +//! +//! Gossip topics are readable by every subscriber, so a secret (a game +//! server password, the key a world backup is encrypted with) can never ride +//! in an offer. Instead the offer carries the buyer's fresh X25519 public key +//! and, after the accept: +//! +//! ```text +//! worker → handoff_ready its own fresh X25519 key (gossip-signed by the worker) +//! buyer → handoff_secret the secret, sealed to it (gossip-signed by the buyer) +//! ``` +//! +//! The key is X25519 → HKDF-SHA256 → AES-256-GCM (`temporal::crypto`), bound +//! to the workload and job id. The buyer seals only to the bidder it +//! accepted, and only on a `handoff_ready` whose gossip source is that +//! bidder; the worker opens only a secret whose gossip source is the offer's +//! buyer. Gossipsub runs with strict signing, so a source cannot be forged. +//! Every other node sees two public keys and ciphertext. +//! +//! The same exchange `webhook.deliver` does for its request, cut down to one +//! direction and a payload of at most [`MAX_SECRET`] bytes. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; +use c0mpute_net::topics::{JobAccept, JobBid, job_topic}; +use c0mpute_net::{GossipMessage, Libp2pNetwork}; +use serde::{Deserialize, Serialize}; +use tokio::sync::{broadcast, mpsc}; +use tokio::task::JoinHandle; +use tracing::debug; + +use crate::temporal::crypto; + +pub use crate::temporal::crypto::KeyPair; + +/// A sealed secret must fit one gossip message with room to spare. +pub const MAX_SECRET: usize = 16 * 1024; +const REPUBLISH: Duration = Duration::from_secs(2); + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Ready { + t: String, + job_id: String, + worker_peer_id: String, + client_pubkey: String, + worker_pubkey: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Sealed { + t: String, + job_id: String, + worker_peer_id: String, + sealed: String, +} + +/// A fresh key pair for one handoff. +pub fn keypair() -> KeyPair { + crypto::keypair() +} + +/// `true` for 64 lowercase hex characters (an X25519 public key). +pub fn valid_pubkey(s: &str) -> bool { + s.len() == 64 && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// The shared key, bound to the workload and the job. +pub fn key(secret: &x25519_dalek::StaticSecret, peer_pub_hex: &str, workload: &str, job_id: &str) -> Result<[u8; 32]> { + crypto::session_key(secret, peer_pub_hex, &format!("handoff:{workload}:{job_id}")) +} + +fn from(msg: &GossipMessage, peer: &str) -> bool { + msg.source.map(|p| p.to_base58()).as_deref() == Some(peer) +} + +/// Worker side: announce a fresh key until the buyer's sealed secret +/// arrives (or `wait` passes), and return the opened secret. +pub async fn receive(net: &Libp2pNetwork, workload: &str, accept: &JobAccept, client_pubkey: &str, wait: Duration) -> Result> { + if !valid_pubkey(client_pubkey) { + bail!("the offer's client key is not an X25519 public key"); + } + let topic = job_topic(workload); + let me = net.peer_id().to_base58(); + let mut rx = net.messages(); + let kp = crypto::keypair(); + let k = key(&kp.secret, client_pubkey, workload, &accept.job_id)?; + let ready = serde_json::to_vec(&Ready { + t: "handoff_ready".into(), + job_id: accept.job_id.clone(), + worker_peer_id: me.clone(), + client_pubkey: client_pubkey.to_string(), + worker_pubkey: kp.public_hex.clone(), + })?; + let deadline = Instant::now() + wait; + let mut next = Instant::now(); + loop { + if Instant::now() >= deadline { + bail!("the buyer never sent the sealed secret"); + } + if Instant::now() >= next { + let _ = net.publish(&topic, ready.clone()).await; + next = Instant::now() + REPUBLISH; + } + let left = next.min(deadline).saturating_duration_since(Instant::now()); + let msg = match tokio::time::timeout(left, rx.recv()).await { + Ok(Ok(m)) => m, + Ok(Err(broadcast::error::RecvError::Closed)) => bail!("network closed"), + _ => continue, + }; + if msg.topic != topic || msg.data.len() > MAX_SECRET * 2 { + continue; + } + let Ok(s) = serde_json::from_slice::(&msg.data) else { continue }; + if s.t != "handoff_secret" || s.job_id != accept.job_id || s.worker_peer_id != me || !from(&msg, &accept.buyer_peer_id) { + continue; + } + match crypto::open(&k, &s.sealed) { + Ok(plain) if plain.len() <= MAX_SECRET => return Ok(plain), + Ok(_) => bail!("sealed secret too large"), + Err(e) => debug!(job_id = %accept.job_id, err = %e, "handoff: unreadable secret ignored"), + } + } +} + +/// Buyer side of one auction's handoff. Pass [`BuyerHandoff::notify`] to +/// `JobAuction::with_accept_notify`; drop (or [`BuyerHandoff::finish`]) it +/// once the receipt is in. +pub struct BuyerHandoff { + notify: Option>, + task: JoinHandle<()>, +} + +impl BuyerHandoff { + /// Start listening for the winner's key. `kp` is the pair whose public + /// half went into the offer. + pub fn spawn(net: Arc, workload: &str, kp: KeyPair, secret: Vec) -> Result { + if secret.len() > MAX_SECRET { + bail!("secret is {} bytes; a handoff carries at most {MAX_SECRET}", secret.len()); + } + let topic = job_topic(workload); + let workload = workload.to_string(); + let mut rx = net.messages(); + let (tx, mut acc_rx) = mpsc::unbounded_channel::<(String, JobBid)>(); + let task = tokio::spawn(async move { + let mut winners: HashMap = HashMap::new(); + let mut readies: HashMap = HashMap::new(); + // job_id → sealed message, republished until the task ends. + let mut sent: HashMap> = HashMap::new(); + let mut tick = tokio::time::interval(REPUBLISH); + let seal = |r: &Ready| -> Result> { + let k = key(&kp.secret, &r.worker_pubkey, &workload, &r.job_id)?; + Ok(serde_json::to_vec(&Sealed { + t: "handoff_secret".into(), + job_id: r.job_id.clone(), + worker_peer_id: r.worker_peer_id.clone(), + sealed: crypto::seal(&k, &secret)?, + })?) + }; + loop { + tokio::select! { + Some((job_id, bid)) = acc_rx.recv() => { + if let Some(r) = readies.remove(&job_id) + && r.worker_peer_id == bid.bidder_peer_id + && let Ok(m) = seal(&r) + { + let _ = net.publish(&topic, m.clone()).await; + sent.insert(job_id.clone(), m); + } + winners.insert(job_id, bid.bidder_peer_id); + } + m = rx.recv() => { + let msg = match m { + Ok(m) => m, + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(_) => return, + }; + if msg.topic != topic { + continue; + } + let Ok(r) = serde_json::from_slice::(&msg.data) else { continue }; + // Seal only to the bidder we accepted, on a key it signed. + if r.t != "handoff_ready" || r.client_pubkey != kp.public_hex || !from(&msg, &r.worker_peer_id) || sent.contains_key(&r.job_id) || !valid_pubkey(&r.worker_pubkey) { + continue; + } + match winners.get(&r.job_id) { + Some(w) if *w == r.worker_peer_id => { + if let Ok(m) = seal(&r) { + let _ = net.publish(&topic, m.clone()).await; + sent.insert(r.job_id.clone(), m); + } + } + Some(_) => debug!(job_id = %r.job_id, "handoff: key from a worker we did not accept ignored"), + None if readies.len() < 64 => { readies.insert(r.job_id.clone(), r); } + None => {} + } + } + _ = tick.tick() => { + for m in sent.values() { + let _ = net.publish(&topic, m.clone()).await; + } + } + } + } + }); + Ok(Self { notify: Some(tx), task }) + } + + /// The accept notifier for `JobAuction::with_accept_notify`. + pub fn notify(&mut self) -> Result> { + self.notify.take().context("handoff notifier already taken") + } + + pub fn finish(self) {} +} + +impl Drop for BuyerHandoff { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn both_sides_derive_one_key_bound_to_the_job() { + let buyer = keypair(); + let worker = keypair(); + let b = key(&buyer.secret, &worker.public_hex, "gameserver.start", "job-1").unwrap(); + let w = key(&worker.secret, &buyer.public_hex, "gameserver.start", "job-1").unwrap(); + assert_eq!(b, w); + assert_ne!(b, key(&buyer.secret, &worker.public_hex, "gameserver.start", "job-2").unwrap()); + assert_ne!(b, key(&buyer.secret, &worker.public_hex, "webhook.deliver", "job-1").unwrap()); + let sealed = crypto::seal(&b, b"hunter22").unwrap(); + assert!(!sealed.contains("hunter22")); + assert_eq!(crypto::open(&w, &sealed).unwrap(), b"hunter22"); + let eve = keypair(); + assert!(crypto::open(&key(&eve.secret, &buyer.public_hex, "gameserver.start", "job-1").unwrap(), &sealed).is_err()); + } + + #[test] + fn pubkeys_are_checked() { + assert!(valid_pubkey(&keypair().public_hex)); + assert!(!valid_pubkey("zz")); + assert!(!valid_pubkey(&"A".repeat(64))); + } +} diff --git a/node/crates/c0mpute-core/src/lib.rs b/node/crates/c0mpute-core/src/lib.rs index 0bc26f3f..d1c9a35e 100644 --- a/node/crates/c0mpute-core/src/lib.rs +++ b/node/crates/c0mpute-core/src/lib.rs @@ -8,6 +8,7 @@ pub mod ads; pub mod buyer; pub mod capabilities; pub mod config; +pub mod container; pub mod crawl; pub mod diffusion; pub mod hosting; @@ -16,6 +17,7 @@ pub mod dispatch; pub mod embed; pub mod gameserver; pub(crate) mod gateway_util; +pub mod handoff; pub mod jobs_gateway; pub mod net_guard; pub mod ocr; diff --git a/node/crates/c0mpute-core/src/speech_gateway.rs b/node/crates/c0mpute-core/src/speech_gateway.rs index abb0bcb7..e9783b63 100644 --- a/node/crates/c0mpute-core/src/speech_gateway.rs +++ b/node/crates/c0mpute-core/src/speech_gateway.rs @@ -132,9 +132,32 @@ pub async fn serve(bind: std::net::SocketAddr, token: String, max_price_usd: f64 } let supervisor = Supervisor::boot_buyer(Config::default()).await?; + let penalties = crate::penalty_box::PenaltyBox::persistent( + crate::config::data_dir() + .unwrap_or_else(|| std::path::PathBuf::from("./c0mpute-data")) + .join("gateway-penalties.json"), + ); + let app = router(supervisor.libp2p, supervisor.store, token, max_price_usd, penalties); + let listener = tokio::net::TcpListener::bind(bind) + .await + .with_context(|| format!("bind c0mpute speech gateway at {bind}"))?; + tracing::info!(%bind, "c0mpute speech gateway listening"); + axum::serve(listener, app).await?; + Ok(()) +} + +/// The gateway's routes over an already booted buyer node. `serve` uses it; +/// so do the loopback end-to-end tests, with their own node and listener. +pub fn router( + net: Arc, + store: ChunkStore, + token: String, + max_price_usd: f64, + penalties: crate::penalty_box::PenaltyBox, +) -> Router { let state = GatewayState { - net: supervisor.libp2p, - store: supervisor.store, + net, + store, token: Arc::from(token), max_price_usd, // Let one multi-chunk readback fan out across up to six network @@ -145,13 +168,9 @@ pub async fn serve(bind: std::net::SocketAddr, token: String, max_price_usd: f64 ocr_slots: Arc::new(Semaphore::new(MAX_CONCURRENT_OCR_JOBS)), embed_slots: Arc::new(Semaphore::new(MAX_CONCURRENT_EMBED_JOBS)), sessions: crate::speech_session::SessionStore::new(), - penalties: crate::penalty_box::PenaltyBox::persistent( - crate::config::data_dir() - .unwrap_or_else(|| std::path::PathBuf::from("./c0mpute-data")) - .join("gateway-penalties.json"), - ), + penalties, }; - let app = Router::new() + Router::new() .route("/healthz", get(healthz)) .route("/v1/voices", get(voices)) .route("/v1/tts", post(synthesize)) @@ -171,13 +190,7 @@ pub async fn serve(bind: std::net::SocketAddr, token: String, max_price_usd: f64 .route("/v1/sessions/{session}", delete(close_session)) .layer(DefaultBodyLimit::max(MAX_AUDIO_BYTES)) .layer(TraceLayer::new_for_http()) - .with_state(state); - let listener = tokio::net::TcpListener::bind(bind) - .await - .with_context(|| format!("bind c0mpute speech gateway at {bind}"))?; - tracing::info!(%bind, "c0mpute speech gateway listening"); - axum::serve(listener, app).await?; - Ok(()) + .with_state(state) } async fn healthz() -> Json { diff --git a/node/crates/c0mpute-core/src/video.rs b/node/crates/c0mpute-core/src/video.rs index b78ce87c..0afefc5c 100644 --- a/node/crates/c0mpute-core/src/video.rs +++ b/node/crates/c0mpute-core/src/video.rs @@ -43,9 +43,25 @@ pub const WORKLOAD_IMG2VID: &str = "video.img2vid"; /// are all present — see [`readiness`]. pub const ROLE_TAG: &str = "c0mpute:role:video"; -/// The image the runner lives in. Kept in step with -/// `plugins/video/module.toml` [module.dispatch]. -pub const RUNNER_IMAGE: &str = "ghcr.io/profullstack/c0mpute-video-runner:candidate"; +/// Where `.github/workflows/video-runner.yml` publishes the runner. +pub const RUNNER_REPOSITORY: &str = "ghcr.io/profullstack/c0mpute-video-runner"; + +/// The stable tag the workflow moves on every master build. +pub const RUNNER_TAG: &str = "stable"; + +/// The published runner this release trusts, pinned by digest: a tag that +/// moves cannot change what a node runs. Bump it (and +/// `plugins/video/module.toml` [module.dispatch]) to the digest the workflow +/// prints in its job summary after a runner change. +pub const RUNNER_DIGEST: &str = "sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c"; + +/// What `c0mpute video setup` pulls: the repository at the pinned digest. +pub const RUNNER_PULL_REF: &str = "ghcr.io/profullstack/c0mpute-video-runner@sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c"; + +/// The local name the node runs. Setup tags the pinned pull as this, or +/// builds it from source under this name when the pull fails, so the worker +/// never depends on how the image arrived. +pub const RUNNER_IMAGE: &str = "c0mpute/video-runner:stable"; /// Where a node can build the runner itself when it cannot pull it. /// @@ -302,6 +318,71 @@ pub fn build_argv(checkout: &Path, image: &str) -> Vec { ] } +/// How `c0mpute video setup` got the runner image. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SetupSource { + /// Pulled at the pinned digest and tagged as [`RUNNER_IMAGE`]. + Pulled, + /// The pull failed (or `--build`): built from a clone of the source repo. + Built, +} + +/// The source repo to build from: `$C0MPUTE_VIDEO_SOURCE_REPO` (the same +/// override `install.sh` takes), else [`RUNNER_SOURCE_REPO`]. +pub fn source_repo() -> String { + std::env::var("C0MPUTE_VIDEO_SOURCE_REPO").ok().filter(|s| !s.is_empty()).unwrap_or_else(|| RUNNER_SOURCE_REPO.to_string()) +} + +async fn run_status(cmd: &mut tokio::process::Command, what: &str) -> Result { + Ok(cmd + .stdin(std::process::Stdio::null()) + .status() + .await + .with_context(|| format!("could not run {what}"))? + .success()) +} + +/// Put the runner image on this node as [`RUNNER_IMAGE`]: pull the pinned +/// digest and tag it, or, when the registry will not serve it (private, +/// rate-limited, offline) or `build_only`, shallow-clone the public repo +/// into `scratch` and build the same Dockerfile. `say` gets progress lines. +pub async fn setup(build_only: bool, scratch: &Path, say: &(dyn Fn(&str) + Sync)) -> Result { + if !build_only { + say(&format!("pulling {RUNNER_PULL_REF} (this is a large image)")); + let pulled = run_status(crate::container::command().args(["pull", RUNNER_PULL_REF]), "docker; install a container runtime first").await?; + if pulled && run_status(crate::container::command().args(["tag", RUNNER_PULL_REF, RUNNER_IMAGE]), "docker tag").await? { + return Ok(SetupSource::Pulled); + } + // A registry is the kind of dependency that is private by default, + // rate-limited or down, and none of those are a reason a node with a + // card cannot serve video. The repo is public: build it. + say("could not pull the runner image (private, rate-limited or offline)."); + say("building it from source instead: same Dockerfile, same runner."); + } + // A fresh directory per run: a half-finished clone from an earlier + // attempt would make `git clone` fail rather than resume. + let checkout = scratch.join(format!("c0mpute-video-src-{}", std::process::id())); + let _ = fs::remove_dir_all(&checkout).await; + let repo = source_repo(); + say(&format!("cloning {repo} (shallow)")); + let cloned = run_status(tokio::process::Command::new("git").args(["clone", "--depth", "1", &repo]).arg(&checkout), "git (building the runner needs git)").await; + let built = match cloned { + Ok(true) => { + say(&format!("building {RUNNER_IMAGE} (several minutes)")); + run_status(crate::container::command().args(build_argv(&checkout, RUNNER_IMAGE)), "docker build").await + } + Ok(false) => Err(anyhow::anyhow!("git clone of {repo} failed")), + Err(e) => Err(e), + }; + // The checkout is scratch whether or not the build worked. + let _ = fs::remove_dir_all(&checkout).await; + if !built? { + bail!("docker build of the runner failed"); + } + say(&format!("built {RUNNER_IMAGE}")); + Ok(SetupSource::Built) +} + /// The runner command for a workload type. pub fn command_for(workload: &str) -> Option<&'static str> { match workload { @@ -335,7 +416,7 @@ impl NotReady { /// Probe the host. Blocking, and only called at boot or from the CLI. pub fn readiness(image: &str) -> Option { - if which("docker").is_none() { + if crate::container::which().is_none() { return Some(NotReady::NoDocker); } if !nvidia_gpu_present() { @@ -369,14 +450,6 @@ pub fn readiness_cached(image: &str) -> Option { verdict } -fn which(bin: &str) -> Option { - std::env::var_os("PATH").and_then(|paths| { - std::env::split_paths(&paths) - .map(|dir| dir.join(bin)) - .find(|candidate| candidate.is_file()) - }) -} - fn nvidia_gpu_present() -> bool { std::process::Command::new("nvidia-smi") .args(["--query-gpu=name", "--format=csv,noheader"]) @@ -386,7 +459,7 @@ fn nvidia_gpu_present() -> bool { } fn image_present(image: &str) -> bool { - std::process::Command::new("docker") + crate::container::command_sync() .args(["image", "inspect", image]) .output() .map(|out| out.status.success()) @@ -539,7 +612,7 @@ async fn owner_of(_path: &Path) -> Result<(u32, u32)> { async fn run_container(run: &ContainerRun) -> Result<()> { let args = docker_argv(run); info!(image = %run.image, command = %run.command, "video runner: starting container"); - let mut child = tokio::process::Command::new("docker") + let mut child = crate::container::command() .args(&args) .stdin(std::process::Stdio::null()) .kill_on_drop(true) @@ -557,7 +630,7 @@ async fn run_container(run: &ContainerRun) -> Result<()> { Err(_) => { // kill_on_drop reaps `docker run`, but the container it started is // a child of the daemon and would keep the GPU busy. - let _ = tokio::process::Command::new("docker") + let _ = crate::container::command() .args(["kill", &run.container_name]) .output() .await; @@ -746,6 +819,23 @@ mod tests { assert_eq!(readiness_cached(RUNNER_IMAGE).is_none(), fresh); } + #[test] + fn the_runner_is_pinned_by_digest_and_matches_the_manifest() { + assert_eq!(RUNNER_PULL_REF, format!("{RUNNER_REPOSITORY}@{RUNNER_DIGEST}")); + let digest = RUNNER_DIGEST.strip_prefix("sha256:").unwrap(); + assert!(digest.len() == 64 && digest.bytes().all(|b| b.is_ascii_hexdigit())); + // The local name is not a registry reference: nothing can pull a + // different image under it. + assert!(!RUNNER_IMAGE.contains("ghcr.io") && !RUNNER_IMAGE.contains('@')); + let manifest: toml::Value = + toml::from_str(include_str!("../../../../plugins/video/module.toml")).unwrap(); + assert_eq!( + manifest["module"]["dispatch"]["image"].as_str(), + Some(RUNNER_PULL_REF), + "plugins/video/module.toml [module.dispatch] image must be the pinned digest" + ); + } + #[test] fn weights_live_under_the_cache_root() { let dir = weights_dir(Path::new("/var/c0mpute")); diff --git a/node/crates/c0mpute-core/tests/fixtures/tiny.mp4 b/node/crates/c0mpute-core/tests/fixtures/tiny.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b245e00c6506f65775a42915f154695e3c4f31c GIT binary patch literal 1542 zcmZux&ub(_6s}1$1o7Y@3sHiUZVn!jNq0{&u7So#2u4uwVn7g@>gt+wGu>U4s+weG zPYZg`s|PP0JTD01NiX6La#-;n*nkH`SQJ6gQ!YAwuRF8J1|R8q-+S+Sud3dw31f^8 z{KOPaSB%Xu()c2AEW^rdhK$YMsI)$2jFr`Koz$0q#Zi)+rQY>v>k%24OiaGBv}b4`AzkQD>zhPDfTnJpfobu9~9FRz!rX zW^HUGvx+z*6n}DLOWxwnP3O}R_-p6gY#_w>jD`kHaA2JH(m$7A9cg9 z8wLR{i*y|IH``#dE>EY>TVU;WGWNvuXSRF@{ps0jt&=cTYc1cRFk#|NWOA zp8w{b^K;65$_Qtm=@|4arQTljxTeODEIZ%ypYR25k?;zxB?&-g`7^zQ*zGnn_w7X- zKB-$D@#pt|#VGRv`t3?Jn!?Xroe}10g)!8|hq6|*4`KDL*HmyN6HPL0&d{s@n$@FT z!CV|zg(1&Bm3r{r1&^v;8kVTD#BBZvOEnHI%P&sjcE+e+zKG3)jXQ zZ|=RizR6!X*hMT;5`kT9CZ!s99tA;fJqjXX=iZy`jg6C&lP*0fU5dJ^?coO1?dHBJ z!O_MSx^~-Kid4ur7p2-2h2gcvXy5mQckI8HpA3p^C#km$}uINRS$GZ&g_q4(QhSRwg}wJgVXV zM-bU*TRqwg!74U!@>}wEpTKKJs3FU?G< zWZKcc(DLAMWRsBWXhIPSj$hx5wB zo4hcX5#9qvh+5(Z anyhow::Result> { + self.0.get(hash).await + } +} + +/// A world: 2.5 MiB of incompressible bytes, so it spans three chunks. +fn world(seed: u64) -> Vec { + let mut x = seed | 1; + let mut v = Vec::with_capacity(2_621_440); + while v.len() < 2_621_440 { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + v.extend_from_slice(&x.to_le_bytes()); + } + v +} + +#[derive(Default)] +struct Fake { + seed: Vec, + plans: Mutex>, + vols: Mutex>>, + live: Mutex>>, + labelled: Mutex>, + killed: Mutex>, + restored: Mutex>, +} + +impl Fake { + fn new(seed: Vec) -> Arc { + Arc::new(Self { seed, ..Default::default() }) + } + fn plan(&self, name: &str) -> Option { + self.plans.lock().unwrap().iter().rev().find(|p| p.name == name).cloned() + } + /// The server process dies (host trouble, OOM, the worker went away). + fn crash(&self, name: &str) { + if let Some(h) = self.live.lock().unwrap().remove(name) { + h.abort(); + } + } +} + +#[async_trait::async_trait] +impl Runtime for Fake { + async fn create(&self, plan: &ContainerPlan) -> anyhow::Result<()> { + self.plans.lock().unwrap().push(plan.clone()); + self.vols.lock().unwrap().entry(plan.volume.clone()).or_insert_with(|| self.seed.clone()); + let exp = plan.labels.iter().find(|(k, _)| k == "c0mpute.gameserver.expires").and_then(|(_, v)| v.parse().ok()).unwrap_or(0); + self.labelled.lock().unwrap().push((plan.name.clone(), exp)); + Ok(()) + } + async fn restore(&self, plan: &ContainerPlan, tar: &Path) -> anyhow::Result<()> { + self.vols.lock().unwrap().insert(plan.volume.clone(), std::fs::read(tar)?); + self.restored.lock().unwrap().push(plan.volume.clone()); + Ok(()) + } + async fn start(&self, name: &str) -> anyhow::Result<()> { + let plan = self.plan(name).ok_or_else(|| anyhow::anyhow!("no container {name}"))?; + let listener = tokio::net::TcpListener::bind(("127.0.0.1", plan.host_port)).await?; + let h = tokio::spawn(async move { + while let Ok((mut s, _)) = listener.accept().await { + tokio::spawn(async move { + let mut buf = [0u8; 512]; + let _ = s.read(&mut buf).await; + let _ = s.write_all(&gameserver::minecraft_status_response(r#"{"version":{"name":"fake 1.21"},"players":{"online":1,"max":8}}"#)).await; + }); + } + }); + self.live.lock().unwrap().insert(name.to_string(), h); + Ok(()) + } + async fn running(&self, name: &str) -> bool { + self.live.lock().unwrap().get(name).is_some_and(|h| !h.is_finished()) + } + async fn flush(&self, _name: &str, _game: Game) {} + async fn stop(&self, name: &str) -> anyhow::Result<()> { + self.crash(name); + Ok(()) + } + async fn tar_world(&self, volume: &str, _game: Game, out: &Path) -> anyhow::Result<()> { + let v = self.vols.lock().unwrap().get(volume).cloned().ok_or_else(|| anyhow::anyhow!("no volume {volume}"))?; + std::fs::write(out, v)?; + Ok(()) + } + async fn remove(&self, name: &str, volume: &str) { + self.crash(name); + self.vols.lock().unwrap().remove(volume); + self.labelled.lock().unwrap().retain(|(n, _)| n != name); + } + async fn labelled(&self) -> anyhow::Result> { + Ok(self.labelled.lock().unwrap().clone()) + } + async fn kill(&self, name: &str) { + self.crash(name); + self.killed.lock().unwrap().push(name.to_string()); + self.labelled.lock().unwrap().retain(|(n, _)| n != name); + } +} + +async fn node(dir: &Path, bootstrap: Vec) -> (Arc, ChunkStore) { + let store = ChunkStore::open(dir.join("store")).await.unwrap(); + let net = Libp2pNetwork::spawn( + NetworkConfig::for_dir(dir.join("id")) + .with_listen(vec!["/ip4/127.0.0.1/tcp/0".parse().unwrap()]) + .with_local_source(Arc::new(StoreSource(store.clone()))) + .with_bootstrap(bootstrap), + ) + .await + .unwrap(); + (Arc::new(net), store) +} + +async fn wait_for_listen(net: &Libp2pNetwork) -> libp2p::Multiaddr { + for _ in 0..50 { + if let Ok(addrs) = net.listen_addrs().await + && let Some(a) = addrs.into_iter().next() + { + return a; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("no listen addr after 2.5s"); +} + +fn cfg(port_min: u16) -> c0mpute_core::config::GameserverConfig { + c0mpute_core::config::GameserverConfig { port_min, port_max: port_min + 9, max_servers: 2, max_total_ram_gb: 16, max_hours: 24, ..Default::default() } +} + +/// A gameserver worker over `rt`, serving on loopback. +async fn worker(dir: &Path, bootstrap: Vec, port_min: u16, rt: Arc) -> Arc { + let (net, store) = node(dir, bootstrap).await; + let w = gameserver::Worker::with_runtime(net, store, cfg(port_min), "127.0.0.1".into(), &dir.join("state"), rt); + let tags = vec![gameserver::ROLE_TAG.to_string(), Game::Minecraft.tag(), Game::Valheim.tag()]; + gameserver::subscribe(w.clone(), tags, Some(PricingConfig::default())); + w +} + +/// Everything a bystander node sees on the job and status topics. +async fn bystander(dir: &Path, bootstrap: Vec) -> (Arc, Arc>>) { + let (net, _) = node(dir, bootstrap).await; + net.subscribe(&job_topic(gameserver::WORKLOAD_START)).await.unwrap(); + net.subscribe(gameserver::STATUS_TOPIC).await.unwrap(); + let seen = Arc::new(Mutex::new(Vec::new())); + let mut rx = net.messages(); + let s = seen.clone(); + tokio::spawn(async move { + while let Ok(m) = rx.recv().await { + let mut g = s.lock().unwrap(); + g.extend_from_slice(&m.data); + g.push(b'\n'); + } + }); + (net, seen) +} + +fn req(game: Game, password: Option<&str>, workers: Vec) -> StartRequest { + StartRequest { + game, + hours: 2, + ram_gb: game.min_ram_gb(), + cpus: 1.0, + accept_eula: true, + world: None, + password: password.map(str::to_string), + server_name: None, + backup_mins: 30, + max_price_usd: 100.0, + workers, + exclude: Vec::new(), + timeout: Duration::from_secs(120), + } +} + +fn contains(hay: &[u8], needle: &[u8]) -> bool { + hay.windows(needle.len()).any(|w| w == needle) +} + +/// Valheim: the password reaches only the winning worker (sealed), the +/// world comes back sealed to the buyer's key, and the stop downloads and +/// decrypts it. A bystander sees neither. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn secrets_are_sealed_and_backups_readable_only_by_the_buyer() { + let _ = tracing_subscriber::fmt().with_env_filter("c0mpute_core=info,c0mpute_net=warn").with_test_writer().try_init(); + let tmp = tempdir().unwrap(); + let rt = Fake::new(world(7)); + let a = worker(&tmp.path().join("a"), vec![], 41100, rt.clone()).await; + let a_addr = wait_for_listen(worker_net(&a)).await.with(Protocol::P2p(worker_net(&a).peer_id())); + let a_id = worker_net(&a).peer_id().to_base58(); + let (eve, seen) = bystander(&tmp.path().join("eve"), vec![a_addr.clone()]).await; + let (buyer, buyer_store) = node(&tmp.path().join("buyer"), vec![a_addr.clone()]).await; + let dir = tmp.path().join("buyer-data"); + tokio::time::sleep(Duration::from_secs(2)).await; + + let rental = gameserver::start_with(buyer.clone(), &buyer_store, &dir, req(Game::Valheim, Some("hunter22pw"), vec![a_id.clone()])).await.expect("valheim start"); + assert_eq!(rental.worker, a_id); + assert!(rental.address.starts_with("127.0.0.1:")); + let plan = rt.plan(&format!("c0mpute-{}", rental.id)).unwrap(); + assert!(plan.env.iter().any(|(k, v)| k == "SERVER_PASS" && v == "hunter22pw"), "the worker got the password through the handoff"); + assert!(plan.run_secs > 7000 && plan.run_secs <= 7200, "the container stops itself at expiry: {}", plan.run_secs); + + // A live backup, announced in a status record the bystander can see. + let live = a.backup_now(&rental.id).await.unwrap(); + a.publish_status().await; + + let stopped = gameserver::stop_with(buyer.clone(), &buyer_store, &dir, &rental.id, Duration::from_secs(60)).await.expect("stop"); + let manifest = stopped.backup.clone().expect("stop returns a backup"); + assert_ne!(manifest, live.manifest, "the final backup is a new one"); + let tar = std::fs::read(dir.join("worlds").join(format!("{manifest}.tar"))).unwrap(); + assert!(tar == world(7), "the buyer decrypted exactly the world"); + assert!(rt.vols.lock().unwrap().is_empty(), "the volume is removed after a good backup"); + + // Eve has every gossip message and can fetch every chunk, and learns nothing. + tokio::time::sleep(Duration::from_millis(500)).await; + let wire = seen.lock().unwrap().clone(); + assert!(contains(&wire, b"handoff_ready") && contains(&wire, b"handoff_secret"), "eve saw the sealed exchange"); + assert!(contains(&wire, live.manifest.as_bytes()), "eve saw the backup hash in a status record"); + assert!(!contains(&wire, b"hunter22pw"), "the password leaked onto gossip"); + let mb = eve.fetch_from_peer(worker_net(&a).peer_id(), &c0mpute_proto::ChunkRequest { chunk_hash: Hash::from_hex(&manifest).unwrap(), shard_index: None }).await.unwrap(); + let m = gameserver::WorldManifest::from_bytes(&mb).unwrap(); + assert!(m.enc.epk.is_some()); + let first = eve.fetch_from_peer(worker_net(&a).peer_id(), &c0mpute_proto::ChunkRequest { chunk_hash: Hash::from_hex(&m.chunks[0]).unwrap(), shard_index: None }).await.unwrap(); + assert!(!contains(&first, &world(7)[..64]), "chunks are ciphertext"); + assert!(gameserver::open_chunk(&gameserver::random_key(), 0, &first).is_err()); + + // The rentals file (token, password) is private. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!(std::fs::metadata(dir.join("rentals.json")).unwrap().permissions().mode() & 0o777, 0o600); + } +} + +/// Minecraft on worker A: the buyer's watch probes it, mirrors A's backup, +/// and when the server stops answering it starts the same world on worker B +/// from that backup (handed to B under a fresh key). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn watch_fails_over_to_another_worker_from_the_latest_backup() { + let tmp = tempdir().unwrap(); + let rt_a = Fake::new(world(11)); + let rt_b = Fake::new(world(99)); + let a = worker(&tmp.path().join("a"), vec![], 41200, rt_a.clone()).await; + let a_addr = wait_for_listen(worker_net(&a)).await.with(Protocol::P2p(worker_net(&a).peer_id())); + let a_id = worker_net(&a).peer_id().to_base58(); + let b = worker(&tmp.path().join("b"), vec![a_addr.clone()], 41210, rt_b.clone()).await; + let b_id = worker_net(&b).peer_id().to_base58(); + let (buyer, buyer_store) = node(&tmp.path().join("buyer"), vec![a_addr.clone()]).await; + let dir = tmp.path().join("buyer-data"); + tokio::time::sleep(Duration::from_secs(3)).await; + + let rental = gameserver::start_with(buyer.clone(), &buyer_store, &dir, req(Game::Minecraft, None, vec![a_id.clone()])).await.expect("minecraft start"); + assert_eq!(rental.worker, a_id); + let p = gameserver::probe_server(Game::Minecraft, &rental.address, Duration::from_secs(3)).await; + assert!(p.ok && p.detail.contains("fake 1.21"), "{p:?}"); + + let opts = WatchOptions { + probe_every: Duration::from_secs(1), + probe_timeout: Duration::from_secs(1), + fail_after: Duration::from_secs(4), + max_price_usd: 100.0, + start_timeout: Duration::from_secs(120), + stop_after_failover: true, + ..Default::default() + }; + let watch = { + let (buyer, store, dir, id) = (buyer.clone(), buyer_store.clone(), dir.clone(), rental.id.clone()); + tokio::spawn(async move { gameserver::watch_with(buyer, &store, &dir, &id, opts).await }) + }; + + // A's 30-minute backup, then its status records until the watch mirrors it. + let backup = a.backup_now(&rental.id).await.unwrap(); + let mut mirrored = false; + for _ in 0..40 { + a.publish_status().await; + tokio::time::sleep(Duration::from_millis(500)).await; + if gameserver::load_rentals(&dir).iter().any(|r| r.id == rental.id && r.latest_backup.as_deref() == Some(backup.manifest.as_str())) { + mirrored = true; + break; + } + } + assert!(mirrored, "the watch mirrored A's backup"); + assert!(std::fs::read(dir.join("worlds").join(format!("{}.tar", backup.manifest))).unwrap() == world(11)); + assert!(gameserver::health_log(&dir, &rental.id).iter().any(|p| p.ok), "healthy probes were logged"); + + // The server stops answering (and A says nothing more). + rt_a.crash(&format!("c0mpute-{}", rental.id)); + let end = tokio::time::timeout(Duration::from_secs(150), watch).await.expect("watch returns").unwrap().expect("watch ok"); + let WatchEnd::FailedOver { from, to } = end else { panic!("expected a failover, got {end:?}") }; + assert_eq!(from.id, rental.id); + assert_eq!(to.worker, b_id, "the replacement runs on the other worker"); + assert_eq!(to.replaces.as_deref(), Some(rental.id.as_str())); + let log = gameserver::health_log(&dir, &rental.id); + assert!(log.iter().any(|p| !p.ok), "failed probes were logged"); + + // B restored A's world (not its own seed) and is serving it. + let vol = format!("c0mpute-{}", to.id); + assert_eq!(rt_b.restored.lock().unwrap().as_slice(), &[vol.clone()]); + assert!(rt_b.vols.lock().unwrap().get(&vol).unwrap() == &world(11)); + assert!(gameserver::probe_server(Game::Minecraft, &to.address, Duration::from_secs(3)).await.ok); + let all = gameserver::load_rentals(&dir); + assert_eq!(all.iter().find(|r| r.id == rental.id).unwrap().superseded_by.as_deref(), Some(to.id.as_str())); + drop(b); +} + +/// The worker was offline when a reservation ended: at its next sweep it +/// kills the expired container, backs the world up from the volume (sealed +/// to the buyer) and removes the volume. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_expiry_missed_while_offline_is_enforced_at_the_next_sweep() { + let tmp = tempdir().unwrap(); + let state_dir = tmp.path().join("w").join("state"); + std::fs::create_dir_all(&state_dir).unwrap(); + let now = gameserver::now_secs(); + let id = gameserver::server_id("job-expired"); + let buyer_key = c0mpute_core::temporal::crypto::keypair(); + let st = ServerState { + id: id.clone(), + game: Game::Factorio, + container: format!("c0mpute-{id}"), + volume: format!("c0mpute-{id}"), + ram_mb: 2048, + host_port: 41300, + address: "127.0.0.1:41300".into(), + started_at: now - 7200, + expires_at: now - 60, + stop_hash: gameserver::sha256_ref(b"t"), + buyer: "buyer".into(), + backup_pubkey: buyer_key.public_hex.clone(), + backup_every_secs: 1800, + stopped_at: None, + backup: None, + backup_error: None, + }; + let servers: HashMap = [(id.clone(), st.clone())].into_iter().collect(); + std::fs::write(state_dir.join("servers.json"), serde_json::to_vec(&servers).unwrap()).unwrap(); + + let rt = Fake::new(world(5)); + rt.vols.lock().unwrap().insert(st.volume.clone(), world(5)); + rt.labelled.lock().unwrap().push((st.container.clone(), now - 60)); + rt.labelled.lock().unwrap().push(("c0mpute-gs-ffffffffffffffff".into(), now + 3600)); + let (net, store) = node(&tmp.path().join("w"), vec![]).await; + let w = gameserver::Worker::with_runtime(net, store.clone(), cfg(41300), "127.0.0.1".into(), &state_dir, rt.clone()); + + assert_eq!(w.sweep_expired().await, vec![st.container.clone()], "only the expired container dies"); + w.reap_once().await; + let after = w.snapshot().get(&id).cloned().unwrap(); + assert!(after.stopped_at.is_some()); + let b = after.backup.expect("backed up from the volume"); + assert!(rt.vols.lock().unwrap().get(&st.volume).is_none(), "volume removed"); + // The backup opens with the buyer's key and nobody else's. + let m = gameserver::WorldManifest::from_bytes(&store.get(&Hash::from_hex(&b.manifest).unwrap()).await.unwrap()).unwrap(); + let epk = m.enc.epk.clone().unwrap(); + let k = gameserver::backup_key(&buyer_key.secret, &epk, &epk, &buyer_key.public_hex).unwrap(); + let mut plain = Vec::new(); + for (i, c) in m.chunks.iter().enumerate() { + plain.extend(gameserver::open_chunk(&k, i as u64, &store.get(&Hash::from_hex(c).unwrap()).await.unwrap()).unwrap()); + } + assert!(plain == world(5)); + // Persisted: a restart does not stop it twice. + let saved: HashMap = serde_json::from_slice(&std::fs::read(state_dir.join("servers.json")).unwrap()).unwrap(); + assert!(saved[&id].stopped_at.is_some()); +} + +fn worker_net(w: &gameserver::Worker) -> &Libp2pNetwork { + w.net() +} diff --git a/node/crates/c0mpute-core/tests/video_e2e.rs b/node/crates/c0mpute-core/tests/video_e2e.rs new file mode 100644 index 00000000..90ad964f --- /dev/null +++ b/node/crates/c0mpute-core/tests/video_e2e.rs @@ -0,0 +1,279 @@ +//! The video buyer path end to end on loopback, with a fake runner. +//! +//! `C0MPUTE_DOCKER` points the node's container CLI at a shell script that +//! behaves like `docker run `: it finds the job directory the node +//! bind-mounted at /job, checks `job.json` is there, and writes a real (tiny) +//! H.264 MP4 as `out.mp4`. Everything else is the shipping code: the worker's +//! video runner and bid filter, the auction, the store and announce, and the +//! buyer gateway's `POST /v1/videos`, which answers `video/mp4`. +//! +//! The same fake stands in for `docker pull/tag/build` to prove +//! `c0mpute video setup` builds from source when the registry will not serve +//! the pinned image. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use c0mpute_core::pricing::PricingConfig; +use c0mpute_core::{dispatch, penalty_box::PenaltyBox, speech_gateway, video}; +use c0mpute_net::{ChunkSource, Libp2pNetwork, NetworkConfig}; +use c0mpute_proto::Hash; +use c0mpute_store::ChunkStore; +use libp2p::multiaddr::Protocol; +use tempfile::tempdir; + +const TOKEN: &str = "video-e2e-token-0123456789abcdef"; +const FAKE_DOCKER: &str = r#"#!/bin/sh +# Stand-in for the docker CLI (see video_e2e.rs). +echo "$*" >> "$FAKE_DIR/log" +case "$1" in + run) + jd="" + for a in "$@"; do + case "$a" in *:/job) jd="${a%:/job}" ;; esac + done + [ -n "$jd" ] && [ -f "$jd/job.json" ] || exit 3 + if grep -q BIGCLIP "$jd/job.json"; then cp "$FAKE_DIR/big.mp4" "$jd/out.mp4"; else cp "$FAKE_DIR/tiny.mp4" "$jd/out.mp4"; fi + exit 0 ;; + pull) [ -f "$FAKE_DIR/pull-ok" ] && exit 0; echo "denied" >&2; exit 1 ;; + build) [ -f "$5" ] && [ -d "$6" ] || exit 4; exit 0 ;; + *) exit 0 ;; +esac +"#; + +struct StoreSource(ChunkStore); + +#[async_trait::async_trait] +impl ChunkSource for StoreSource { + async fn read_chunk(&self, hash: &Hash) -> anyhow::Result> { + self.0.get(hash).await + } +} + +fn tiny_mp4() -> Vec { + std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/tiny.mp4")).unwrap() +} + +/// The tiny clip plus a top-level `free` box: still a valid MP4, `len` bytes. +fn big_mp4(len: usize) -> Vec { + let mut v = tiny_mp4(); + let pad = len - v.len(); + v.extend_from_slice(&(pad as u32).to_be_bytes()); + v.extend_from_slice(b"free"); + let mut x: u64 = 0x9E37_79B9_7F4A_7C15; + while v.len() < len { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + v.extend_from_slice(&x.to_le_bytes()); + } + v.truncate(len); + v +} + +/// Install the fake docker once per test binary. +fn fake_docker() -> &'static PathBuf { + static DIR: OnceLock = OnceLock::new(); + DIR.get_or_init(|| { + let dir = tempdir().unwrap().keep(); + std::fs::write(dir.join("tiny.mp4"), tiny_mp4()).unwrap(); + std::fs::write(dir.join("big.mp4"), big_mp4(6 * 1024 * 1024 + 123)).unwrap(); + let bin = dir.join("docker"); + std::fs::write(&bin, FAKE_DOCKER).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + // SAFETY: set once, before any thread of this binary reads them. + unsafe { + std::env::set_var(c0mpute_core::container::ENV, &bin); + std::env::set_var("FAKE_DIR", &dir); + } + dir + }) +} + +async fn node(dir: &Path, bootstrap: Vec) -> (Arc, ChunkStore) { + let store = ChunkStore::open(dir.join("store")).await.unwrap(); + let net = Libp2pNetwork::spawn( + NetworkConfig::for_dir(dir.join("id")) + .with_listen(vec!["/ip4/127.0.0.1/tcp/0".parse().unwrap()]) + .with_local_source(Arc::new(StoreSource(store.clone()))) + .with_bootstrap(bootstrap), + ) + .await + .unwrap(); + (Arc::new(net), store) +} + +async fn wait_for_listen(net: &Libp2pNetwork) -> libp2p::Multiaddr { + for _ in 0..50 { + if let Ok(addrs) = net.listen_addrs().await + && let Some(a) = addrs.into_iter().next() + { + return a; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("no listen addr after 2.5s"); +} + +/// A video worker (the real runner over the fake docker), a buyer node, and +/// the buyer gateway on a loopback port. Returns the gateway URL. +async fn network(tmp: &Path) -> (String, Arc) { + fake_docker(); + let (worker, worker_store) = node(&tmp.join("w"), vec![]).await; + let worker_addr = wait_for_listen(&worker).await.with(Protocol::P2p(worker.peer_id())); + let load = c0mpute_core::WorkerLoad::new(Arc::new(tokio::sync::Semaphore::new(1)), 1); + let runner = video::spawn_video_runner( + worker.clone(), + worker_store, + tmp.join("cache"), + video::WORKLOAD_TXT2VID.to_string(), + video::RUNNER_IMAGE.to_string(), + load.clone(), + ); + dispatch::run_worker_subscriber_with_load( + worker.clone(), + video::WORKLOAD_TXT2VID.to_string(), + vec![video::ROLE_TAG.to_string(), "c0mpute:gpu:nvidia".to_string()], + Some(runner), + None, + Some(PricingConfig::default()), + Some(load), + ); + + let (buyer, buyer_store) = node(&tmp.join("b"), vec![worker_addr]).await; + let app = speech_gateway::router(buyer, buyer_store, TOKEN.into(), 100.0, PenaltyBox::new()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + tokio::time::sleep(Duration::from_secs(2)).await; + (url, worker) +} + +async fn post_video(url: &str, prompt: &str) -> reqwest::Response { + reqwest::Client::new() + .post(format!("{url}/v1/videos")) + .bearer_auth(TOKEN) + .json(&serde_json::json!({ + "model": "wan2.2-ti2v-5b", + "prompt": prompt, + "frames": 9, + "steps": 2, + "width": 64, + "height": 64, + })) + .timeout(Duration::from_secs(180)) + .send() + .await + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn post_v1_videos_returns_the_runner_clip() { + let _ = tracing_subscriber::fmt().with_env_filter("c0mpute_core=info,c0mpute_net=warn").with_test_writer().try_init(); + let tmp = tempdir().unwrap(); + let (url, worker) = network(tmp.path()).await; + + // No token, no job. + let anon = reqwest::Client::new().post(format!("{url}/v1/videos")).json(&serde_json::json!({"model": "m", "prompt": "p"})).send().await.unwrap(); + assert_eq!(anon.status(), 401); + + let res = post_video(&url, "a neon alley in the rain").await; + assert_eq!(res.status(), 200, "{:?}", res.text().await); + assert_eq!(res.headers()["content-type"], "video/mp4"); + assert_eq!(res.headers()["x-c0mpute-worker-peer-id"], worker.peer_id().to_base58().as_str()); + let body = res.bytes().await.unwrap(); + assert!(body.as_ref() == tiny_mp4().as_slice(), "the gateway returns exactly the runner's out.mp4"); + + // The node ran the runner with the GPU, the job dir and the weights cache. + let log = std::fs::read_to_string(fake_docker().join("log")).unwrap(); + let run = log.lines().find(|l| l.starts_with("run ")).expect("docker run was called"); + assert!(run.contains("--gpus all") && run.contains(":/job") && run.contains(":/weights"), "{run}"); + assert!(run.contains(video::RUNNER_IMAGE) && run.contains("txt2vid"), "{run}"); + + // An invalid job is refused before any auction. + let bad = reqwest::Client::new() + .post(format!("{url}/v1/videos")) + .bearer_auth(TOKEN) + .json(&serde_json::json!({"model": "wan2.2-ti2v-5b", "prompt": "x", "frames": 9999})) + .send() + .await + .unwrap(); + assert_eq!(bad.status(), 400); +} + +/// A clip over one network message (6 MiB): the worker publishes it as +/// pieces and the gateway fetches and reassembles them (fix/large-results). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs PR #160 (fix/large-results): results over ~5 MiB travel as pieces"] +async fn a_clip_over_5_mib_reaches_the_buyer() { + let tmp = tempdir().unwrap(); + let (url, _) = network(tmp.path()).await; + let res = post_video(&url, "BIGCLIP a long take").await; + assert_eq!(res.status(), 200, "{:?}", res.text().await); + let body = res.bytes().await.unwrap(); + let want = std::fs::read(fake_docker().join("big.mp4")).unwrap(); + assert_eq!(body.len(), want.len()); + assert!(body.as_ref() == want.as_slice()); +} + +/// `c0mpute video setup`: the pinned pull is tried first and tagged as the +/// local name; when the registry refuses it, the runner is built from a +/// clone of the source repo instead. +#[tokio::test] +async fn setup_pulls_the_pinned_digest_or_builds_from_source() { + let dir = fake_docker().clone(); + let git_ok = std::process::Command::new("git").arg("--version").output().is_ok_and(|o| o.status.success()); + if !git_ok { + eprintln!("skipping: git is not installed"); + return; + } + // A local "source repo" laid out like this one. + let src = tempdir().unwrap(); + std::fs::create_dir_all(src.path().join("plugins/video/runner")).unwrap(); + std::fs::write(src.path().join("plugins/video/runner/Dockerfile"), "FROM scratch\n").unwrap(); + std::fs::write(src.path().join("plugins/video/module.toml"), "[module]\n").unwrap(); + let git = |args: &[&str]| { + let ok = std::process::Command::new("git") + .args(["-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "init.defaultBranch=master"]) + .args(args) + .current_dir(src.path()) + .output() + .unwrap(); + assert!(ok.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&ok.stderr)); + }; + git(&["init", "-q"]); + git(&["add", "."]); + git(&["commit", "-qm", "runner"]); + // SAFETY: only this test reads it. + unsafe { std::env::set_var("C0MPUTE_VIDEO_SOURCE_REPO", format!("file://{}", src.path().display())) }; + + let scratch = tempdir().unwrap(); + let said = std::sync::Mutex::new(Vec::::new()); + let say = |l: &str| said.lock().unwrap().push(l.to_string()); + + // The registry refuses the pull: build from source. + let _ = std::fs::remove_file(dir.join("pull-ok")); + let got = video::setup(false, scratch.path(), &say).await.expect("setup falls back to a build"); + assert_eq!(got, video::SetupSource::Built); + let log = std::fs::read_to_string(dir.join("log")).unwrap(); + assert!(log.contains(&format!("pull {}", video::RUNNER_PULL_REF)), "{log}"); + let build = log.lines().find(|l| l.starts_with("build ")).expect("docker build ran"); + assert!(build.contains(&format!("-t {}", video::RUNNER_IMAGE)) && build.contains("plugins/video/runner/Dockerfile"), "{build}"); + assert!(said.lock().unwrap().iter().any(|l| l.contains("building it from source"))); + assert_eq!(std::fs::read_dir(scratch.path()).unwrap().count(), 0, "the scratch checkout is removed"); + + // The pull works: tag the pinned digest as the local name, no build. + std::fs::write(dir.join("pull-ok"), "").unwrap(); + let got = video::setup(false, scratch.path(), &say).await.unwrap(); + assert_eq!(got, video::SetupSource::Pulled); + let log = std::fs::read_to_string(dir.join("log")).unwrap(); + assert!(log.contains(&format!("tag {} {}", video::RUNNER_PULL_REF, video::RUNNER_IMAGE)), "{log}"); + let _ = std::fs::remove_file(dir.join("pull-ok")); +} diff --git a/plugins/gameserver/module.toml b/plugins/gameserver/module.toml index 1df53102..31d6fcb1 100644 --- a/plugins/gameserver/module.toml +++ b/plugins/gameserver/module.toml @@ -1,26 +1,47 @@ # c0mpute plugin manifest. See dips/0006-module-model.md. # # Built in: workloads `gameserver.start` and `gameserver.stop` -# (node/crates/c0mpute-core/src/gameserver.rs, CLI c0mpute-cli/src/gameserver.rs). +# (node/crates/c0mpute-core/src/gameserver.rs and handoff.rs, CLI +# c0mpute-cli/src/gameserver.rs). # # Start is an ordinary auction with a short job: the winning worker runs # `docker create` (argv only, no shell) on a pinned image with --memory, -# --memory-swap, --cpus, --pids-limit 1024, no-new-privileges and --restart no, -# a named volume for the world and a published port block from its range, -# restores a world if asked, starts it and returns address:port plus the -# expiry. The server then runs for the reserved hours; the worker's reaper -# stops it at expiry. Images (multi-arch index digests): +# --memory-swap, --cpus, --pids-limit 1024, no-new-privileges, --restart no, +# --rm and --init, a named volume for the world and a published port block +# from its range, restores a world if asked, starts it and returns +# address:port plus the expiry. Images (multi-arch index digests): # minecraft itzg/minecraft-server:java21 (EULA=TRUE only with the buyer's --accept-eula) # factorio factoriotools/factorio:stable-2.1.24 # valheim lloesche/valheim-server (x86_64 only; buyer sets the password) # +# Secrets: nothing secret is in the offer (spec v2). The offer carries the +# buyer's fresh X25519 key; after the accept the winner announces its own and +# the buyer seals the Valheim password and any restore key to it +# (X25519 -> HKDF-SHA256 -> AES-256-GCM, gossip-signed both ways). +# # Stop: the start spec carries sha256(stop token); the token stays on the # buyer's machine until `gameserver stop`, and only the worker holding that -# server bids. On stop or expiry the worker tars the world out of the volume -# (`docker cp`), stores it as 1 MiB content-addressed chunks plus a manifest, -# announces them, and returns the manifest hash; the buyer downloads the tar -# (verified chunk by chunk and by sha256). `start --world ` serves it -# from the buyer's node and any worker restores it. No coordinator anywhere. +# server bids. +# +# Backups: every `backup_mins` (default 30) and at stop or expiry the worker +# tars the world out of the volume, encrypts it to the buyer's long-term +# X25519 backup key in 1 MiB AES-GCM chunks, and stores and announces the +# chunks plus a manifest. The manifest hash is in the stop receipt and in the +# worker's status record; only the buyer can read what it names. +# +# Health and failover: workers gossip a status record per server (container +# running, latest backup) on c0mpute/gameserver/v1, signed by their peer key. +# `c0mpute gameserver watch` probes the game port (Minecraft status ping, +# Valheim Steam A2S query, Factorio UDP reachability plus a fresh "running" +# record), logs every probe (`gameserver health`), mirrors each backup to the +# buyer's machine, and after `--fail-after-secs` (default 300) of failed checks +# starts the same world on another worker from the newest backup. No +# coordinator anywhere. +# +# Expiry: the server runs under `timeout ` behind --init and +# --rm, so it stops itself at expiry even if the worker is offline then. At +# boot and on every sweep the worker kills any labelled container past its +# expiry, then backs the world up from the volume and removes it. # # Worker: opt-in role (`c0mpute worker start --roles ...,gameserver`), never # default. Advertised only when `docker info` works, the node knows a public @@ -28,11 +49,8 @@ # C0MPUTE_P2P_EXTERNAL) and a pinned image has been pulled. The operator opens # [gameserver] port_min..port_max (default 27500-27599, TCP and UDP). # -# Not built: replicas / live failover, proof-of-serve (RCON or player -# heartbeat) and per-epoch settlement: a reservation is paid once, up front, -# at the price the auction cleared. Backups are readable by anyone who has the -# manifest hash (it is in the stop receipt), and a Valheim password rides in -# the public offer. +# Billing: one payment up front at the price the auction cleared. A failover +# is a new rental, paid the same way. [module] id = "gameserver" @@ -40,19 +58,19 @@ name = "Game server — Minecraft / Factorio / Valheim" version = "0.2.79" kind = "workload" category = "infrastructure" -status = "partial" -status_note = "Start, stop, expiry and world backup/restore work. Only opt-in workers with docker and a public IP whose firewall opens the port range can host, and few do yet. No replicas or proof-of-serve: one worker, paid once up front; if it goes offline the server is gone until it returns, and the world survives only in its last backup." +status = "available" builtin = true -description = "Rent a Minecraft, Factorio or Valheim server on a c0mpute worker for 1-72 hours: pinned docker images with RAM and CPU limits, address:port back in seconds, auto-stop at expiry, and the world backed up p2p by hash so any worker can restore it." +description = "Rent a Minecraft, Factorio or Valheim server on a c0mpute worker for 1-72 hours: pinned docker images with RAM and CPU limits, address:port back in seconds, a hard stop at expiry, world backups every 30 minutes encrypted to your key, and a watcher that probes the server and fails over to another worker from the latest backup." author = "Profullstack" license = "MIT" keywords = ["gameserver", "minecraft", "factorio", "valheim", "hosting", "docker"] [module.requirements] -c0mpute = ">=0.2.77" +c0mpute = ">=NEXT" os = ["linux"] arch = ["x86_64", "aarch64"] capabilities = ["container-runtime", "network:public-ip", "network:ingress"] +hardware = "opt-in worker role: docker + a public IP with the port range open (TCP and UDP)" [module.workloads] "gameserver.start" = { command = "start", validation = "result_schema" } @@ -72,3 +90,4 @@ max_total_ram_gb = 0 max_hours = 24 min_usd_per_gb_hour = 0.0 games = ["minecraft", "factorio", "valheim"] +backup_mins = 30 diff --git a/plugins/video/install.sh b/plugins/video/install.sh index 0ceb8e3d..744a8fe9 100755 --- a/plugins/video/install.sh +++ b/plugins/video/install.sh @@ -11,7 +11,11 @@ # Source: https://github.com/profullstack/c0mpute/tree/master/plugins/video set -eu -IMAGE="${C0MPUTE_VIDEO_IMAGE:-ghcr.io/profullstack/c0mpute-video-runner:candidate}" +# The published runner, pinned by digest (keep in step with module.toml +# [module.dispatch] and RUNNER_DIGEST in video.rs), and the local name the +# node runs, whichever way the image arrives. +PULL_REF="${C0MPUTE_VIDEO_PULL_REF:-ghcr.io/profullstack/c0mpute-video-runner@sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c}" +IMAGE="c0mpute/video-runner:stable" SOURCE_REPO="${C0MPUTE_VIDEO_SOURCE_REPO:-https://github.com/profullstack/c0mpute.git}" say() { printf '\033[1;36m→\033[0m %s\n' "$*"; } @@ -43,12 +47,12 @@ else fi if [ "$missing" -eq 0 ]; then - say "pulling $IMAGE (several GB, once)" - if ! docker pull "$IMAGE"; then + say "pulling $PULL_REF (several GB, once)" + if ! { docker pull "$PULL_REF" && docker tag "$PULL_REF" "$IMAGE"; }; then # The registry is a convenience, not a dependency: private, rate-limited # or offline is no reason a box with a card cannot serve video. This repo # is public, so build the same image from source. - warn "could not pull $IMAGE (private, rate-limited or offline)" + warn "could not pull $PULL_REF (private, rate-limited or offline)" if command -v git >/dev/null 2>&1; then say "building it from source instead (several minutes)" src=$(mktemp -d) diff --git a/plugins/video/module.toml b/plugins/video/module.toml index a8695933..45bce2b7 100644 --- a/plugins/video/module.toml +++ b/plugins/video/module.toml @@ -5,9 +5,15 @@ # published as the image below; the node's video runner invokes it per job and # reads files, never stdout. See that directory's README for the job contract. # -# The image below is a convenience: `c0mpute video setup` falls back to -# building it from this public repo when the registry will not serve it, so a -# node with a GPU never needs a registry account. +# The image below is pinned by digest (the same digest as RUNNER_DIGEST in +# node/crates/c0mpute-core/src/video.rs, checked by a unit test). +# `c0mpute video setup` pulls it and tags it locally as +# c0mpute/video-runner:stable, the name the worker runs. If the registry will +# not serve it, setup builds the same Dockerfile from this public repo, so a +# node with a GPU never needs a registry account. The workflow +# .github/workflows/video-runner.yml publishes :stable, :candidate and : +# on every master build and prints the new digest; bump both pins to it after +# a runner change. # # Backend: Diffusers, not ComfyUI. ComfyUI would mean shipping hand-authored # API graph JSON per model, which cannot be checked by reading it; the @@ -47,18 +53,19 @@ name = "Video — generation (Wan 2.2, LTX-2.3 with audio)" version = "0.2.79" kind = "workload" category = "ai" -status = "partial" -status_note = "Needs an NVIDIA GPU and the c0mpute video runner image." +status = "available" +builtin = true description = "Decentralized video generation on consumer/prosumer GPUs: Wan 2.2 text/image-to-video, and LTX-2.3 video with a synchronized soundtrack." author = "Profullstack" license = "MIT" keywords = ["video", "wan", "wan2.2", "ltx-2", "ltx-2.3", "audio", "txt2vid", "img2vid", "generation", "diffusion"] [module.requirements] -c0mpute = ">=0.1.0" +c0mpute = ">=NEXT" os = ["linux"] arch = ["x86_64"] capabilities = ["gpu:nvidia", "model-runtime", "container-runtime"] +hardware = "NVIDIA GPU (24 GiB VRAM for the default model, 48 GiB for the 14B and LTX models) + docker with the NVIDIA container runtime" [module.workloads] "video.txt2vid" = { command = "txt2vid", validation = "schema_and_ffprobe" } @@ -80,7 +87,7 @@ capabilities = ["gpu:nvidia", "model-runtime", "container-runtime"] [module.dispatch] mode = "container" -image = "ghcr.io/profullstack/c0mpute-video-runner:candidate" +image = "ghcr.io/profullstack/c0mpute-video-runner@sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c" [module.surfaces] cli = "c0mpute video" diff --git a/plugins/video/runner/Dockerfile b/plugins/video/runner/Dockerfile index c28a48cd..ce752640 100644 --- a/plugins/video/runner/Dockerfile +++ b/plugins/video/runner/Dockerfile @@ -11,7 +11,7 @@ # docker run --rm --gpus all \ # -v c0mpute-video-weights:/weights \ # -v "$PWD/job:/job" \ -# ghcr.io/profullstack/c0mpute-video-runner:candidate txt2vid +# c0mpute/video-runner:stable txt2vid # # No CUDA base image on purpose: the torch wheels from PyPI bring their own # CUDA runtime, so only the driver comes from the host via the NVIDIA diff --git a/plugins/video/runner/README.md b/plugins/video/runner/README.md index 056779dc..efe3267b 100644 --- a/plugins/video/runner/README.md +++ b/plugins/video/runner/README.md @@ -4,8 +4,12 @@ The container the `video` plugin dispatches to. One job per process: read `job.json`, generate a clip, encode an MP4, write a sidecar, exit non-zero on any failure. -Published as `ghcr.io/profullstack/c0mpute-video-runner`. The tag a node pulls -is the one in [`../module.toml`](../module.toml) under `[module.dispatch]`. +Published as `ghcr.io/profullstack/c0mpute-video-runner` with the tags +`:stable`, `:candidate` and `:` on every master build. A node +pulls by digest: the one in [`../module.toml`](../module.toml) under +`[module.dispatch]` (and `RUNNER_DIGEST` in `video.rs`), then tags it locally +as `c0mpute/video-runner:stable`, the name the worker runs. After a runner +change, copy the digest from the workflow's job summary into both pins. **The registry is a convenience, not a dependency.** `c0mpute video setup` pulls the image, and if the registry will not serve it — private package, From 457c53849d53aecb0ee0a51dca21cf712aab9318 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 9 Nov 2026 04:30:37 +0000 Subject: [PATCH 2/7] feat(train, infernet): p2p dataset/adapter pieces, multi-turn chats, native p2p chat train (infernet.text.train, `c0mpute train lora`): - The corpus goes out as content-addressed 1 MiB pieces from the buyer's chunk store; the worker pulls them over libp2p (InputRef::fetch_to_file, streamed) and checks the sha256 the offer names. Training events are published on the job topic (run_id, step events throttled to 4/s). The worker stores the adapter files as pieces behind an AdapterManifest and its receipt carries the manifest's content hash; the buyer fetches and keeps the adapter only if the weights match the manifest's sha256. Nothing needs a reachable HTTP address any more. - `c0mpute infernet text chat` (infernet.text.infer) is p2p the same way: adapter out as pieces (fetched on a worker cache miss), answer back as a chunk named by the receipt. - Compatibility: specs keep the HTTP fields optional. Workers before this release require dataset_url/adapter_url, so they never bid on a pieces-only job; `--legacy-http` makes the buyer also run the transfer server for them (receipt `sha256:` = the old HTTP path, checked as before). Old buyers' HTTP specs still parse and run on new workers. - Multi-turn chats: messages records are kept whole (every turn, in order) and prompt/completion becomes a one-exchange conversation, instead of collapsing to a "Brief me on: " record. Infernet's runner trains them with the loss on assistant turns (infernetprotocol/infernet-protocol#161) and reports formats:["messages"]; workers then advertise infernet:text:format:messages and buyers whose corpus has conversations require it. {title, text} records are unchanged. - module.toml: available. infernet (infernet.inference): - New built-in workload c0mpute-core/src/inference.rs: workers answer chat turns from their own OpenAI-compatible runtime (Ollama at 127.0.0.1:11434 by default, C0MPUTE_INFERENCE_URL for llama-server etc., `off` disables). Role `inference`, on by default, advertised only while /v1/models lists chat models (refreshed every minute): infernet:inference plus a tag per model. Answers are stored as chunks and checked against the receipt. - `c0mpute chat --p2p` (the default when not signed in) runs one auction per turn over libp2p, REPL included; the jobs bridge accepts it too. No c0mpute.com and no infernet binary on either side. - module.toml: available; temporal and text workers still need the infernet CLI for Infernet's own runtimes. Tests: text_train_e2e (3 MiB mixed corpus as pieces on loopback with a fake peft runner, gossip progress, adapter pieces verified, then adapter chat p2p), inference_e2e (multi-turn chat p2p via the bridge; also passed against the live Ollama on dev2), unit tests for specs, compat, conversion and tags. Co-Authored-By: Claude Opus 5.5 --- node/crates/c0mpute-cli/src/ai_chat.rs | 154 ++++- node/crates/c0mpute-cli/src/main.rs | 4 +- node/crates/c0mpute-cli/src/text.rs | 29 +- node/crates/c0mpute-cli/src/train.rs | 38 +- node/crates/c0mpute-core/src/capabilities.rs | 9 + node/crates/c0mpute-core/src/config.rs | 1 + node/crates/c0mpute-core/src/dispatch.rs | 7 +- node/crates/c0mpute-core/src/inference.rs | 358 ++++++++++ node/crates/c0mpute-core/src/jobs_gateway.rs | 12 +- node/crates/c0mpute-core/src/lib.rs | 1 + node/crates/c0mpute-core/src/pieces.rs | 50 ++ .../c0mpute-core/src/status_aggregator.rs | 4 +- node/crates/c0mpute-core/src/supervisor.rs | 26 +- node/crates/c0mpute-core/src/temporal/node.rs | 16 +- node/crates/c0mpute-core/src/text.rs | 616 ++++++++++++++++-- node/crates/c0mpute-core/src/train.rs | 113 +++- .../tests/fixtures/fake-infernet-text | 69 ++ .../c0mpute-core/tests/inference_e2e.rs | 146 +++++ .../c0mpute-core/tests/text_train_e2e.rs | 185 ++++++ .../crates/c0mpute-core/tests/workflow_e2e.rs | 3 +- node/crates/c0mpute-proto/src/lib.rs | 4 + plugins/infernet/module.toml | 34 +- plugins/train/module.toml | 45 +- 23 files changed, 1768 insertions(+), 156 deletions(-) create mode 100644 node/crates/c0mpute-core/src/inference.rs create mode 100755 node/crates/c0mpute-core/tests/fixtures/fake-infernet-text create mode 100644 node/crates/c0mpute-core/tests/inference_e2e.rs create mode 100644 node/crates/c0mpute-core/tests/text_train_e2e.rs diff --git a/node/crates/c0mpute-cli/src/ai_chat.rs b/node/crates/c0mpute-cli/src/ai_chat.rs index bce77603..ccc1f600 100644 --- a/node/crates/c0mpute-cli/src/ai_chat.rs +++ b/node/crates/c0mpute-cli/src/ai_chat.rs @@ -1,6 +1,10 @@ -//! `c0mpute chat`: chat with models on the c0mpute network through c0mpute.com. +//! `c0mpute chat`: chat with models on the c0mpute network, either peer to +//! peer (`--p2p`, and the default when not signed in: an `infernet.inference` +//! auction over libp2p, answered by a worker's own Ollama/llama-server; see +//! `c0mpute_core::inference`) or through c0mpute.com with a signed-in key. //! //! ```text +//! c0mpute chat --p2p "explain LoRA" # a worker answers over libp2p, no account //! c0mpute chat login # device sign-in: approve on c0mpute.com with CoinPay (DID required) //! c0mpute chat "explain LoRA" # one answer, streamed //! c0mpute chat # interactive session (TTY) @@ -52,6 +56,16 @@ struct Args { /// Print the full JSON response instead of streaming text. #[arg(long)] json: bool, + /// Ask a worker directly over libp2p (`infernet.inference`): no sign-in, + /// no c0mpute.com. The default when this machine is not signed in. + #[arg(long)] + p2p: bool, + /// With --p2p: the most you pay a worker for one answer, in USD. + #[arg(long, default_value_t = 0.01)] + max_price: f64, + /// With --p2p: seconds to wait for an answer. + #[arg(long, default_value_t = 300)] + timeout_s: u64, /// c0mpute site URL (default https://c0mpute.com, or $C0MPUTE_URL). #[arg(long)] url: Option, @@ -151,7 +165,10 @@ pub async fn run(args: &[String]) -> Result<()> { let a = Args::try_parse_from(std::iter::once("c0mpute chat".to_string()).chain(args.iter().cloned())) .unwrap_or_else(|e| e.exit()); let url = base_url(a.url.as_deref()); - let key = require_key()?; + let p2p = a.p2p || api_key().is_none(); + if p2p && !a.p2p { + eprintln!("(not signed in: asking a c0mpute worker peer to peer; `c0mpute chat login` uses c0mpute.com instead)"); + } let mut prompt = a.prompt.join(" "); let stdin = std::io::stdin(); @@ -171,9 +188,23 @@ pub async fn run(args: &[String]) -> Result<()> { if !std::io::stdin().is_terminal() { bail!("no prompt: pass it as an argument or pipe it on stdin"); } - return repl(&url, &key, &a, messages).await; + if p2p { + return P2pChat::boot(&a).await?.repl(&a, messages).await; + } + return repl(&url, &require_key()?, &a, messages).await; } messages.push(json!({"role": "user", "content": prompt})); + if p2p { + let chat = P2pChat::boot(&a).await?; + let (text, raw) = chat.ask(&a, a.model.clone(), &messages).await?; + if a.json { + println!("{}", serde_json::to_string_pretty(&raw)?); + } else { + println!("{text}"); + } + return Ok(()); + } + let key = require_key()?; let answer = complete(&url, &key, &a, &messages, a.json).await?; if a.json { println!("{answer}"); @@ -300,11 +331,107 @@ fn clone_args(a: &Args) -> Args { max_tokens: a.max_tokens, temperature: a.temperature, json: false, + p2p: a.p2p, + max_price: a.max_price, + timeout_s: a.timeout_s, url: a.url.clone(), prompt: vec![], } } +/// The `infernet.inference` spec for one turn. +fn p2p_spec(a: &Args, model: Option, messages: &[Value]) -> Result { + let mut spec = json!({"v": 1, "messages": messages}); + if let Some(m) = model { + spec["model"] = json!(m); + } + if let Some(t) = a.max_tokens { + spec["max_tokens"] = json!(t); + } + if let Some(t) = a.temperature { + spec["temperature"] = json!(t); + } + c0mpute_core::inference::parse_spec(&spec).map_err(|e| anyhow!(e))?; + Ok(spec) +} + +/// Chat with workers over libp2p: one buyer node for the whole session, one +/// `infernet.inference` auction per turn, answers fetched from the winner and +/// checked against its receipt. +struct P2pChat { + bridge: c0mpute_core::jobs_gateway::JobsBridge, +} + +impl P2pChat { + async fn boot(a: &Args) -> Result { + let sup = c0mpute_core::Supervisor::boot_buyer(c0mpute_core::Config::default()).await?; + // Any token: the bridge's HTTP side is never mounted here. + let token: String = std::iter::repeat_n('x', 32).collect(); + let bridge = c0mpute_core::jobs_gateway::JobsBridge::new(sup.libp2p, sup.store, token, a.max_price, Vec::new())?; + // Let the swarm reach peers before the first auction. + tokio::time::sleep(Duration::from_secs(5)).await; + Ok(Self { bridge }) + } + + async fn ask(&self, a: &Args, model: Option, messages: &[Value]) -> Result<(String, Value)> { + let spec = p2p_spec(a, model, messages)?; + let r = self + .bridge + .submit_once(c0mpute_core::inference::WORKLOAD, spec, Duration::from_secs(a.timeout_s.clamp(10, 900))) + .await + .context("no worker answered (none serving infernet.inference for this model, or all over --max-price)")?; + let v = r.get("result").cloned().unwrap_or(r.clone()); + if v["ok"] != json!(true) { + bail!("the worker could not answer: {}", v["error"].as_str().unwrap_or("?")); + } + Ok((v["text"].as_str().unwrap_or_default().to_string(), r)) + } + + async fn repl(&self, a: &Args, mut messages: Vec) -> Result<()> { + let mut model = a.model.clone(); + eprintln!("c0mpute chat (peer to peer) · {} · /model , /reset, /exit", model.as_deref().unwrap_or("any served model")); + let stdin = std::io::stdin(); + let base = messages.clone(); + loop { + eprint!("\x1b[32m> \x1b[0m"); + std::io::stderr().flush().ok(); + let mut line = String::new(); + if stdin.lock().read_line(&mut line)? == 0 { + eprintln!(); + return Ok(()); + } + let line = line.trim(); + match line { + "" => continue, + "/exit" | "/quit" => return Ok(()), + "/reset" => { + messages = base.clone(); + eprintln!("(new conversation)"); + continue; + } + _ if line.starts_with("/model") => { + let m = line.trim_start_matches("/model").trim(); + model = (!m.is_empty() && m != "auto").then(|| m.to_string()); + eprintln!("(model: {})", model.as_deref().unwrap_or("any served model")); + continue; + } + _ => {} + } + messages.push(json!({"role": "user", "content": line})); + match self.ask(a, model.clone(), &messages).await { + Ok((answer, _)) => { + println!("{answer}"); + messages.push(json!({"role": "assistant", "content": answer})); + } + Err(e) => { + messages.pop(); + eprintln!("error: {e:#}"); + } + } + } + } +} + async fn login(rest: &[String]) -> Result<()> { let mut url_flag = None; let mut it = rest.iter(); @@ -497,6 +624,27 @@ mod tests { assert_eq!(a.model.as_deref(), Some("qwen2.5:7b")); assert_eq!(a.system.as_deref(), Some("terse")); assert_eq!(a.prompt.join(" "), "what is 2+2"); + assert!(!a.p2p); + } + + #[test] + fn p2p_turn_is_a_valid_inference_spec() { + let a = Args::try_parse_from(["c0mpute chat", "--p2p", "--max-tokens", "64", "--temperature", "0.3", "hi"]).unwrap(); + assert!(a.p2p); + let msgs = vec![json!({"role": "system", "content": "terse"}), json!({"role": "user", "content": "hi"})]; + let spec = p2p_spec(&a, Some("qwen2.5:0.5b".into()), &msgs).unwrap(); + assert_eq!(spec["v"], 1); + assert_eq!(spec["model"], "qwen2.5:0.5b"); + assert_eq!(spec["max_tokens"], 64); + assert_eq!( + c0mpute_core::inference::required_capabilities(&spec), + vec!["infernet:inference".to_string(), "infernet:inference:model:qwen2.5:0.5b".to_string()] + ); + let any = p2p_spec(&a, None, &msgs).unwrap(); + assert!(any.get("model").is_none()); + // A turn the workers would refuse never leaves this machine. + let bad = Args::try_parse_from(["c0mpute chat", "--p2p", "--temperature", "9", "hi"]).unwrap(); + assert!(p2p_spec(&bad, None, &msgs).is_err()); } #[test] diff --git a/node/crates/c0mpute-cli/src/main.rs b/node/crates/c0mpute-cli/src/main.rs index 0395e71f..c06937a1 100644 --- a/node/crates/c0mpute-cli/src/main.rs +++ b/node/crates/c0mpute-cli/src/main.rs @@ -47,7 +47,7 @@ use clap::{Parser, Subcommand}; name = "c0mpute", version, about = "c0mpute.com — decentralized compute network", - long_about = "c0mpute.com CLI. Submit jobs, run a worker, manage modules.\n\nBuilt-in workloads:\n transcode (FFmpeg)\n tts (Piper, Kokoro, or Pocket, CPU)\n stt (faster-whisper, CPU)\n workflow-step (JMESPath transforms + guarded HTTP calls, for meshhook)\n ocr (Tesseract on the worker, images + PDFs)\n embed (fastembed bge-small-en-v1.5, CPU)\n render (Blender frames, Cycles on CPU or GPU)\n coinpay (DID + payments, peer CLI)\n infernet (AI inference, peer CLI)\n\n c0mpute coinpay reputation did claim\n c0mpute transcode preset list\n c0mpute tts setup --engine all\n c0mpute stt setup\n c0mpute speech serve\n c0mpute ocr scan.pdf --lang eng\n c0mpute embed \"first text\" \"second text\"\n c0mpute render scene.blend --frames 1-120 --out frames --mp4\n C0MPUTE_JOBS_TOKEN= c0mpute jobs serve\n c0mpute worker start --roles tts,stt\n c0mpute chat login\n c0mpute chat \"hello\"\n c0mpute infernet chat --model qwen2.5:7b \"hello\"\n c0mpute infernet model list --node \n c0mpute infernet reservation pricing\n c0mpute infernet text serve\n c0mpute infernet text train --job job.json --dataset corpus.jsonl --yes\n c0mpute infernet text chat --adapter ./adapter \"How should I price my SaaS?\"\n c0mpute train lora --base Qwen/Qwen2.5-0.5B-Instruct --dataset data.jsonl --yes\n c0mpute gameserver start minecraft --hours 4 --ram 4g --accept-eula\n c0mpute infernet temporal serve --model sha256: --train\n c0mpute infernet temporal predict --session s1 --input step.json\n c0mpute webhook serve --price 0.001\n c0mpute webhook send https://example.com/hook -d '{\"event\":\"paid\"}' --secret whsec_..." + long_about = "c0mpute.com CLI. Submit jobs, run a worker, manage modules.\n\nBuilt-in workloads:\n transcode (FFmpeg)\n tts (Piper, Kokoro, or Pocket, CPU)\n stt (faster-whisper, CPU)\n workflow-step (JMESPath transforms + guarded HTTP calls, for meshhook)\n ocr (Tesseract on the worker, images + PDFs)\n embed (fastembed bge-small-en-v1.5, CPU)\n render (Blender frames, Cycles on CPU or GPU)\n coinpay (DID + payments, peer CLI)\n inference (chat from your Ollama/llama-server, p2p)\n infernet (temporal + LoRA runtimes, peer CLI)\n\n c0mpute coinpay reputation did claim\n c0mpute transcode preset list\n c0mpute tts setup --engine all\n c0mpute stt setup\n c0mpute speech serve\n c0mpute ocr scan.pdf --lang eng\n c0mpute embed \"first text\" \"second text\"\n c0mpute render scene.blend --frames 1-120 --out frames --mp4\n C0MPUTE_JOBS_TOKEN= c0mpute jobs serve\n c0mpute worker start --roles tts,stt\n c0mpute chat --p2p \"hello\" (peer to peer, no account)\n c0mpute chat login\n c0mpute chat \"hello\"\n c0mpute infernet chat --model qwen2.5:7b \"hello\"\n c0mpute infernet model list --node \n c0mpute infernet reservation pricing\n c0mpute infernet text serve\n c0mpute infernet text train --job job.json --dataset corpus.jsonl --yes\n c0mpute infernet text chat --adapter ./adapter \"How should I price my SaaS?\"\n c0mpute train lora --base Qwen/Qwen2.5-0.5B-Instruct --dataset data.jsonl --yes\n c0mpute gameserver start minecraft --hours 4 --ram 4g --accept-eula\n c0mpute infernet temporal serve --model sha256: --train\n c0mpute infernet temporal predict --session s1 --input step.json\n c0mpute webhook serve --price 0.001\n c0mpute webhook send https://example.com/hook -d '{\"event\":\"paid\"}' --secret whsec_..." )] struct Cli { /// Override the config file location. @@ -1553,6 +1553,7 @@ async fn run_worker(cmd: WorkerCmd, config_path: &std::path::Path) -> Result<()> Role::Embed, Role::LiveStream, Role::Render, + Role::Inference, ] { if !cfg.roles.contains(&role) { cfg.roles.push(role); @@ -3897,6 +3898,7 @@ fn parse_role(s: &str) -> Option { "gameserver" | "gameservers" | "games" => Some(Role::Gameserver), "live-stream" | "live_stream" | "livestream" | "live" | "stream" => Some(Role::LiveStream), "render" | "blender" => Some(Role::Render), + "inference" | "chat" | "llm" => Some(Role::Inference), other => { eprintln!("warning: unknown role '{other}' ignored"); None diff --git a/node/crates/c0mpute-cli/src/text.rs b/node/crates/c0mpute-cli/src/text.rs index ef0badf2..1e1c9f56 100644 --- a/node/crates/c0mpute-cli/src/text.rs +++ b/node/crates/c0mpute-cli/src/text.rs @@ -57,7 +57,10 @@ enum Sub { worker: Vec, #[arg(long, default_value_t = 0.05)] max_price: f64, - /// How the worker reaches this machine's transfer server (use https off-LAN). + /// Also serve workers older than the peer-to-peer transport over HTTP from this machine. + #[arg(long)] + legacy_http: bool, + /// With --legacy-http: how an old worker reaches this machine's transfer server (use https off-LAN). #[arg(long)] public_url: Option, #[arg(long, default_value = "0.0.0.0:0")] @@ -91,7 +94,10 @@ enum Sub { gpu: bool, #[arg(long)] min_ram_gb: Option, - /// How the worker reaches this machine's transfer server (use https off-LAN). + /// Also serve workers older than the peer-to-peer transport over HTTP from this machine. + #[arg(long)] + legacy_http: bool, + /// With --legacy-http: how an old worker reaches this machine's transfer server (use https off-LAN). #[arg(long)] public_url: Option, #[arg(long, default_value = "0.0.0.0:0")] @@ -148,7 +154,7 @@ pub async fn run(args: &[String]) -> Result<()> { tokio::signal::ctrl_c().await?; Ok(()) } - Sub::Chat { adapter, question, no_search, search_url, max_new_tokens, worker, max_price, public_url, bind, timeout_s, bid_window_s, json: as_json, net } => { + Sub::Chat { adapter, question, no_search, search_url, max_new_tokens, worker, max_price, legacy_http, public_url, bind, timeout_s, bid_window_s, json: as_json, net } => { let question = question.join(" "); anyhow::ensure!(!question.trim().is_empty(), "ask a question: c0mpute infernet text chat --adapter ./adapter \"How should I price my SaaS?\""); let req = text::ChatRequest { @@ -158,15 +164,18 @@ pub async fn run(args: &[String]) -> Result<()> { max_new_tokens, workers: worker, max_price_usd: max_price, + legacy_http, public_url, bind, timeout_s, bid_window: Duration::from_secs(bid_window_s), }; - let n = c0mpute_core::temporal::node::spawn_node(&net_opts(&net, "client")).await?; + // The adapter goes out as pieces from this store, served over libp2p. + let store = c0mpute_store::ChunkStore::open(data_dir().join("store")).await?; + let n = c0mpute_core::temporal::node::spawn_node_with_store(&net_opts(&net, "client"), store.clone()).await?; tokio::time::sleep(Duration::from_secs(2)).await; eprintln!("asking a c0mpute worker ..."); - let out = text::chat(n, &req).await?; + let out = text::chat(n, &store, &req).await?; if as_json { println!("{}", serde_json::to_string(&out)?); } else { @@ -178,8 +187,8 @@ pub async fn run(args: &[String]) -> Result<()> { } Ok(()) } - Sub::Train { job, dataset, output, worker, max_price, gpu, min_ram_gb, public_url, bind, timeout_s, retries, bid_window_s, yes, json: as_json, net } => { - let req = TrainRequest { job_path: job, dataset_path: dataset, output, workers: worker, max_price_usd: max_price, gpu, min_ram_gb, public_url, bind, timeout_s, retries, bid_window: Duration::from_secs(bid_window_s) }; + Sub::Train { job, dataset, output, worker, max_price, gpu, min_ram_gb, legacy_http, public_url, bind, timeout_s, retries, bid_window_s, yes, json: as_json, net } => { + let req = TrainRequest { job_path: job, dataset_path: dataset, output, workers: worker, max_price_usd: max_price, gpu, min_ram_gb, legacy_http, public_url, bind, timeout_s, retries, bid_window: Duration::from_secs(bid_window_s) }; let (_, _, p) = text::load(&req)?; if !yes { eprintln!( @@ -198,12 +207,14 @@ pub async fn run(args: &[String]) -> Result<()> { } std::process::exit(3); } - let n = c0mpute_core::temporal::node::spawn_node(&net_opts(&net, "client")).await?; + // The corpus goes out as pieces from this store, served over libp2p. + let store = c0mpute_store::ChunkStore::open(data_dir().join("store")).await?; + let n = c0mpute_core::temporal::node::spawn_node_with_store(&net_opts(&net, "client"), store.clone()).await?; tokio::time::sleep(Duration::from_secs(2)).await; eprintln!("looking for a worker for {} ...", p.name); let mut total = None; let mut on_step = false; - let out = text::train(n, &req, |ev| { + let out = text::train(n, &store, &req, |ev| { if as_json { println!("{ev}"); return; diff --git a/node/crates/c0mpute-cli/src/train.rs b/node/crates/c0mpute-cli/src/train.rs index d3f132d4..2e0be198 100644 --- a/node/crates/c0mpute-cli/src/train.rs +++ b/node/crates/c0mpute-cli/src/train.rs @@ -8,7 +8,8 @@ use std::path::PathBuf; use std::time::Duration; use anyhow::{Context, Result}; -use c0mpute_core::temporal::node::{NetOptions, spawn_node}; +use c0mpute_core::temporal::node::{NetOptions, spawn_node_with_store}; +use c0mpute_store::ChunkStore; use c0mpute_core::text::{self, TrainRequest}; use c0mpute_core::train::{LoraOptions, normalize_dataset}; use clap::{Args, Parser, Subcommand}; @@ -39,8 +40,10 @@ enum Sub { /// Train a LoRA adapter on a remote worker and download it. /// /// The dataset is JSONL: {"text"} (optionally "title"), {"prompt","completion"} - /// or {"messages":[{role,content}]}. Workers run `infernet model train` - /// (peft, fp32 LoRA on CPU or CUDA); keep the base model small. + /// or {"messages":[{role,content}]} (multi-turn chats train on every turn, + /// loss on the assistant's). Workers run `infernet model train` (peft, fp32 + /// LoRA on CPU or CUDA); keep the base model small. The dataset and the + /// adapter move peer to peer over libp2p. Lora { /// Base model, a Hugging Face id (e.g. Qwen/Qwen2.5-0.5B-Instruct). #[arg(long)] @@ -81,7 +84,11 @@ enum Sub { gpu: bool, #[arg(long)] min_ram_gb: Option, - /// How the worker reaches this machine's transfer server (needed off-LAN: an https URL that forwards to --bind). + /// Also serve workers older than the peer-to-peer transport: the corpus and adapter then go + /// over HTTP from this machine (the worker must reach it: same LAN, or --public-url). + #[arg(long)] + legacy_http: bool, + /// With --legacy-http: how an old worker reaches this machine's transfer server (an https URL forwarding to --bind). #[arg(long)] public_url: Option, #[arg(long, default_value = "0.0.0.0:0")] @@ -125,6 +132,7 @@ pub async fn run(args: &[String]) -> Result<()> { max_price, gpu, min_ram_gb, + legacy_http, public_url, bind, timeout_s, @@ -161,6 +169,7 @@ pub async fn run(args: &[String]) -> Result<()> { max_price_usd: max_price, gpu, min_ram_gb, + legacy_http, public_url, bind, timeout_s, @@ -189,18 +198,23 @@ pub async fn run(args: &[String]) -> Result<()> { std::process::exit(3); } - let node = spawn_node(&NetOptions { - identity_dir: data_dir().join("client"), - listen: net.listen.clone(), - bootstrap: net.bootstrap.clone(), - default_bootstrap: !net.no_default_bootstrap, - }) + // The corpus goes out as pieces from this store, served over libp2p. + let store = ChunkStore::open(data_dir().join("store")).await?; + let node = spawn_node_with_store( + &NetOptions { + identity_dir: data_dir().join("client"), + listen: net.listen.clone(), + bootstrap: net.bootstrap.clone(), + default_bootstrap: !net.no_default_bootstrap, + }, + store.clone(), + ) .await?; tokio::time::sleep(Duration::from_secs(2)).await; - eprintln!("looking for a worker advertising infernet:text:train ..."); + eprintln!("looking for a worker advertising {} ...", text::required_tags(&corpus).join(" + ")); let mut total = None; let mut on_step = false; - let result = text::train(node, &req, |ev| { + let result = text::train(node, &store, &req, |ev| { if as_json { println!("{ev}"); return; diff --git a/node/crates/c0mpute-core/src/capabilities.rs b/node/crates/c0mpute-core/src/capabilities.rs index b5fdc697..b941db39 100644 --- a/node/crates/c0mpute-core/src/capabilities.rs +++ b/node/crates/c0mpute-core/src/capabilities.rs @@ -109,6 +109,15 @@ pub fn role_tags(config: &Config) -> Vec { } } Role::LiveStream => tags.push(crate::live::ROLE_TAG.to_string()), + Role::Inference => { + // Only while the local runtime lists chat models (refreshed + // every minute), one tag per model. + let models = crate::inference::ready_models(); + if !models.is_empty() { + tags.push(crate::inference::ROLE_TAG.to_string()); + tags.extend(models.iter().map(|m| crate::inference::model_tag(m))); + } + } Role::Render => { // Only where `blender` runs. The probe (a few seconds) runs // once at worker start; until it finishes, nothing is shown. diff --git a/node/crates/c0mpute-core/src/config.rs b/node/crates/c0mpute-core/src/config.rs index 03813976..71eb139d 100644 --- a/node/crates/c0mpute-core/src/config.rs +++ b/node/crates/c0mpute-core/src/config.rs @@ -314,6 +314,7 @@ impl Default for Config { Role::Embed, Role::LiveStream, Role::Render, + Role::Inference, ], update_channel: default_update_channel(), update_auto: default_auto_update(), diff --git a/node/crates/c0mpute-core/src/dispatch.rs b/node/crates/c0mpute-core/src/dispatch.rs index a0171a3d..6b3909ad 100644 --- a/node/crates/c0mpute-core/src/dispatch.rs +++ b/node/crates/c0mpute-core/src/dispatch.rs @@ -519,8 +519,8 @@ async fn handle_offer( /// Workload types this worker should listen for, derived from its /// configured roles. Each modality maps to a separate job topic. -/// `infernet.inference` is handled by the infernet peer binary, not -/// this daemon. +/// `infernet.inference` is served natively from the box's own +/// OpenAI-compatible runtime ([`crate::inference`]). pub fn workload_types_from_roles(roles: &[c0mpute_proto::Role]) -> Vec { let mut types = Vec::new(); if roles.contains(&c0mpute_proto::Role::Transcode) { @@ -555,6 +555,9 @@ pub fn workload_types_from_roles(roles: &[c0mpute_proto::Role]) -> Vec { if roles.contains(&c0mpute_proto::Role::Embed) { types.push(crate::embed::WORKLOAD.to_string()); } + if roles.contains(&c0mpute_proto::Role::Inference) { + types.push(crate::inference::WORKLOAD.to_string()); + } if roles.contains(&c0mpute_proto::Role::Render) { types.push(crate::render::WORKLOAD.to_string()); } diff --git a/node/crates/c0mpute-core/src/inference.rs b/node/crates/c0mpute-core/src/inference.rs new file mode 100644 index 00000000..dea76cab --- /dev/null +++ b/node/crates/c0mpute-core/src/inference.rs @@ -0,0 +1,358 @@ +//! `infernet.inference`: chat completions from open models, peer to peer, +//! served by the worker's own OpenAI-compatible runtime. +//! +//! A buyer (`c0mpute chat --p2p`, or the `c0mpute jobs serve` bridge) auctions +//! a spec of chat messages on `c0mpute/jobs/infernet.inference`. A worker +//! answers it from the local runtime it already runs (Ollama on +//! `127.0.0.1:11434` by default, or any OpenAI-compatible server such as +//! llama.cpp's `llama-server`, set with `C0MPUTE_INFERENCE_URL`), stores the +//! answer as a chunk and publishes a receipt with its hash; the buyer fetches +//! it from the worker and checks it. No c0mpute.com, no `infernet` binary on +//! either side. +//! +//! Workers advertise `infernet:inference` plus one +//! `infernet:inference:model:` tag per model the runtime lists at +//! `/v1/models`, and only while it answers; the list is refreshed every +//! minute. The runtime URL is operator configuration, never from a spec, so a +//! buyer cannot point a worker at another address. `C0MPUTE_INFERENCE_URL=off` +//! turns the role off. +//! +//! Answers are one-shot (no token streaming across the network yet). + +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use c0mpute_net::topics::JobStatus; +use c0mpute_net::{Libp2pNetwork, Network}; +use c0mpute_store::ChunkStore; +use serde::Deserialize; +use serde_json::{Value, json}; +use tokio::sync::mpsc; +use tracing::{info, warn}; + +use crate::runner::{AcceptedJob, WorkerLoad, publish_receipt}; + +pub const WORKLOAD: &str = "infernet.inference"; +/// Capability tag: this node answers chat completions. +pub const ROLE_TAG: &str = "infernet:inference"; +pub const DEFAULT_RUNTIME_URL: &str = "http://127.0.0.1:11434"; +pub const RUNTIME_URL_ENV: &str = "C0MPUTE_INFERENCE_URL"; +/// Concurrent answers per node: the local runtime is usually one GPU or a CPU. +pub const MAX_CONCURRENT: usize = 2; +pub const MAX_MESSAGES: usize = 64; +pub const MAX_PROMPT_BYTES: usize = 64 * 1024; +pub const MAX_TOKENS: u32 = 4096; +pub const DEFAULT_MAX_TOKENS: u32 = 512; +pub const MAX_RESULT_BYTES: usize = 256 * 1024; +const GENERATION_TIMEOUT: Duration = Duration::from_secs(300); +const REFRESH_EVERY: Duration = Duration::from_secs(60); + +pub fn model_tag(model: &str) -> String { + format!("{ROLE_TAG}:model:{model}") +} + +/// What a buyer asks for. +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub struct ChatSpec { + pub v: u32, + /// A model id the worker's runtime lists; None means any. + #[serde(default)] + pub model: Option, + pub messages: Vec, + #[serde(default)] + pub max_tokens: Option, + #[serde(default)] + pub temperature: Option, +} + +fn valid_model_id(m: &str) -> bool { + !m.is_empty() && m.len() <= 128 && m.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | ':' | '/' | '@')) +} + +pub fn parse_spec(spec: &Value) -> std::result::Result { + let s: ChatSpec = serde_json::from_value(spec.clone()).map_err(|e| format!("invalid {WORKLOAD} spec: {e}"))?; + if s.v != 1 { + return Err(format!("unsupported {WORKLOAD} spec version {}", s.v)); + } + if s.model.as_deref().is_some_and(|m| !valid_model_id(m)) { + return Err("model must be 1-128 characters of letters, digits and -_.:/@".into()); + } + if s.messages.is_empty() || s.messages.len() > MAX_MESSAGES { + return Err(format!("messages must hold 1..={MAX_MESSAGES} entries")); + } + let mut bytes = 0; + for m in &s.messages { + let role = m.get("role").and_then(Value::as_str).unwrap_or(""); + if !["system", "user", "assistant"].contains(&role) { + return Err("message role must be system, user or assistant".into()); + } + bytes += m.get("content").and_then(Value::as_str).ok_or("message content must be a string")?.len(); + } + if bytes > MAX_PROMPT_BYTES { + return Err(format!("messages exceed {MAX_PROMPT_BYTES} bytes")); + } + if s.messages.last().and_then(|m| m.get("role")).and_then(Value::as_str) == Some("assistant") { + return Err("the last message must be the user's (or system's), not the assistant's".into()); + } + if s.max_tokens.is_some_and(|t| !(1..=MAX_TOKENS).contains(&t)) { + return Err(format!("max_tokens must be 1..={MAX_TOKENS}")); + } + if s.temperature.is_some_and(|t| !(0.0..=2.0).contains(&t)) { + return Err("temperature must be 0..=2".into()); + } + Ok(s) +} + +/// Tags a worker must carry for `spec`. +pub fn required_capabilities(spec: &Value) -> Vec { + let mut t = vec![ROLE_TAG.to_string()]; + if let Ok(ChatSpec { model: Some(m), .. }) = parse_spec(spec) { + t.push(model_tag(&m)); + } + t +} + +/// A result the buyer accepts: `{v:1, ok:true, text, model}` or `{v:1, ok:false, error}`. +pub fn valid_result(value: &Value) -> bool { + value.get("v").and_then(Value::as_u64) == Some(1) + && match value.get("ok").and_then(Value::as_bool) { + Some(true) => value.get("text").is_some_and(Value::is_string) && value.get("model").is_some_and(Value::is_string), + Some(false) => value.get("error").is_some_and(Value::is_string), + None => false, + } +} + +// ─── the local runtime ──────────────────────────────────────────────────── + +/// The operator's runtime base URL, or None when the role is turned off. +pub fn runtime_url() -> Option { + match std::env::var(RUNTIME_URL_ENV) { + Ok(v) if matches!(v.trim().to_ascii_lowercase().as_str(), "off" | "0" | "false" | "no" | "") => None, + Ok(v) => Some(v.trim().trim_end_matches('/').to_string()), + Err(_) => Some(DEFAULT_RUNTIME_URL.to_string()), + } +} + +static READY: RwLock> = RwLock::new(Vec::new()); + +/// Models the runtime listed at the last probe (empty: not ready, advertise nothing). +pub fn ready_models() -> Vec { + READY.read().map(|m| m.clone()).unwrap_or_default() +} + +fn set_ready(models: Vec) { + if let Ok(mut m) = READY.write() { + *m = models; + } +} + +/// Model ids at `{base}/v1/models` (OpenAI shape, as Ollama and llama-server serve it). +pub async fn list_models(base: &str) -> Result> { + let c = reqwest::Client::builder().timeout(Duration::from_secs(5)).build()?; + let v: Value = c.get(format!("{base}/v1/models")).send().await?.error_for_status()?.json().await?; + let mut models: Vec = v["data"] + .as_array() + .into_iter() + .flatten() + .filter_map(|m| m["id"].as_str()) + .filter(|m| valid_model_id(m)) + // Embedding-only models cannot chat. + .filter(|m| !m.contains("embed")) + .map(String::from) + .collect(); + models.sort(); + models.dedup(); + Ok(models) +} + +/// Probe once now, then every minute, keeping [`ready_models`] current. +pub async fn refresh_loop(base: String) { + let mut last: Option> = None; + loop { + let models = list_models(&base).await.unwrap_or_default(); + if last.as_ref() != Some(&models) { + if models.is_empty() { + info!(runtime = %base, "inference: no OpenAI-compatible runtime with chat models; not advertising {WORKLOAD}"); + } else { + info!(runtime = %base, ?models, "inference: serving {WORKLOAD}"); + } + last = Some(models.clone()); + } + set_ready(models); + tokio::time::sleep(REFRESH_EVERY).await; + } +} + +/// Answer one spec from the runtime at `base`. Never errors: a failure is an +/// `ok: false` result the buyer can read. +pub async fn execute(base: &str, spec: &Value, ready: &[String]) -> Value { + let started = Instant::now(); + let blob = match run(base, spec, ready).await { + Ok(v) => v, + Err(e) => json!({"v": 1, "ok": false, "error": format!("{e:#}")}), + }; + let mut blob = blob; + blob["duration_ms"] = json!(started.elapsed().as_millis() as u64); + if serde_json::to_vec(&blob).map(|b| b.len()).unwrap_or(usize::MAX) > MAX_RESULT_BYTES { + return json!({"v": 1, "ok": false, "error": format!("answer is over {MAX_RESULT_BYTES} bytes")}); + } + blob +} + +async fn run(base: &str, spec: &Value, ready: &[String]) -> Result { + let s = parse_spec(spec).map_err(anyhow::Error::msg)?; + let model = match &s.model { + Some(m) if ready.iter().any(|r| r == m) => m.clone(), + Some(m) => anyhow::bail!("model {m} is not served here"), + None => ready.first().cloned().context("no model is served here")?, + }; + let mut body = json!({ + "model": model, + "messages": s.messages, + "max_tokens": s.max_tokens.unwrap_or(DEFAULT_MAX_TOKENS), + "stream": false, + }); + if let Some(t) = s.temperature { + body["temperature"] = json!(t); + } + let c = reqwest::Client::builder().timeout(GENERATION_TIMEOUT).build()?; + let resp = c.post(format!("{base}/v1/chat/completions")).json(&body).send().await.context("reach the local runtime")?; + let status = resp.status(); + let v: Value = resp.json().await.context("runtime answer is not JSON")?; + if !status.is_success() { + anyhow::bail!("runtime answered {status}: {}", v["error"]["message"].as_str().or(v["error"].as_str()).unwrap_or("?")); + } + let choice = &v["choices"][0]; + let text = choice["message"]["content"].as_str().context("runtime answer has no choices[0].message.content")?; + Ok(json!({ + "v": 1, "ok": true, "model": model, "text": text, + "finish_reason": choice["finish_reason"], "usage": v["usage"], + })) +} + +// ─── worker runner ──────────────────────────────────────────────────────── + +pub fn worker_load() -> WorkerLoad { + WorkerLoad::new(Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT)), MAX_CONCURRENT) +} + +/// Bid only on valid specs for a model served right now. +pub fn offer_filter() -> crate::dispatch::OfferFilter { + Arc::new(|offer| { + let ready = ready_models(); + !ready.is_empty() + && offer.spec_inline.as_ref().and_then(|v| parse_spec(v).ok()).is_some_and(|s| s.model.is_none_or(|m| ready.contains(&m))) + }) +} + +pub fn spawn_inference_runner(net: Arc, store: ChunkStore, base: String, load: WorkerLoad) -> mpsc::Sender { + let (tx, mut rx) = mpsc::channel::(MAX_CONCURRENT * 4); + tokio::spawn(async move { + while let Some(job) = rx.recv().await { + let Ok(slot) = load.compute_slots().acquire_owned().await else { return }; + let (net, store, base, load) = (net.clone(), store.clone(), base.clone(), load.clone()); + tokio::spawn(async move { + let _slot = slot; + let started = Instant::now(); + let job_id = job.accept.job_id.clone(); + let result: Result = async { + let spec = crate::workflow::resolve_spec(&net, &job).await?; + let answer = execute(&base, &spec, &ready_models()).await; + let bytes = serde_json::to_vec(&answer)?; + let hash = store.put(&bytes).await.context("store answer")?; + net.announce(&hash).await.context("announce answer")?; + Ok(hash.to_hex()) + } + .await; + match result { + Ok(hash) => { + load.record_job(started.elapsed()); + info!(%job_id, %hash, "inference: answered"); + publish_receipt(&net, &job.accept, Some(hash), JobStatus::Completed).await; + } + Err(e) => { + warn!(%job_id, err = %format!("{e:#}"), "inference: failed"); + publish_receipt(&net, &job.accept, None, JobStatus::Failed).await; + } + } + }); + } + }); + tx +} + +#[cfg(test)] +mod tests { + use super::*; + + fn spec(v: Value) -> std::result::Result { + parse_spec(&v) + } + + #[test] + fn spec_validation() { + let ok = json!({"v": 1, "messages": [{"role": "system", "content": "be brief"}, {"role": "user", "content": "hi"}]}); + assert!(spec(ok.clone()).is_ok()); + assert_eq!(required_capabilities(&ok), vec![ROLE_TAG.to_string()]); + let m = json!({"v": 1, "model": "qwen2.5:0.5b", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 64, "temperature": 0.2}); + assert!(spec(m.clone()).is_ok()); + assert_eq!(required_capabilities(&m), vec![ROLE_TAG.to_string(), "infernet:inference:model:qwen2.5:0.5b".to_string()]); + assert!(spec(json!({"v": 2, "messages": [{"role": "user", "content": "hi"}]})).unwrap_err().contains("version")); + assert!(spec(json!({"v": 1, "messages": []})).is_err()); + assert!(spec(json!({"v": 1, "messages": [{"role": "tool", "content": "x"}]})).is_err()); + assert!(spec(json!({"v": 1, "messages": [{"role": "user", "content": 3}]})).is_err()); + assert!(spec(json!({"v": 1, "messages": [{"role": "user", "content": "x".repeat(MAX_PROMPT_BYTES + 1)}]})).is_err()); + assert!(spec(json!({"v": 1, "messages": [{"role": "user", "content": "q"}, {"role": "assistant", "content": "a"}]})).unwrap_err().contains("last message")); + assert!(spec(json!({"v": 1, "model": "../etc passwd", "messages": [{"role": "user", "content": "q"}]})).is_err()); + assert!(spec(json!({"v": 1, "max_tokens": 0, "messages": [{"role": "user", "content": "q"}]})).is_err()); + assert!(spec(json!({"v": 1, "temperature": 3.0, "messages": [{"role": "user", "content": "q"}]})).is_err()); + } + + #[test] + fn results() { + assert!(valid_result(&json!({"v": 1, "ok": true, "text": "hi", "model": "m"}))); + assert!(valid_result(&json!({"v": 1, "ok": false, "error": "no"}))); + assert!(!valid_result(&json!({"v": 1, "ok": true}))); + assert!(!valid_result(&json!({"v": 2, "ok": false, "error": "no"}))); + } + + /// A stand-in OpenAI-compatible runtime on loopback. + pub(crate) async fn fake_runtime() -> String { + use axum::{Json, Router, routing::{get, post}}; + let app = Router::new() + .route("/v1/models", get(|| async { Json(json!({"object": "list", "data": [{"id": "tiny:1b"}, {"id": "nomic-embed-text"}]})) })) + .route( + "/v1/chat/completions", + post(|Json(b): Json| async move { + let last = b["messages"].as_array().and_then(|m| m.last()).and_then(|m| m["content"].as_str()).unwrap_or("").to_string(); + Json(json!({"choices": [{"message": {"role": "assistant", "content": format!("echo[{}]: {last}", b["model"].as_str().unwrap_or(""))}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2}})) + }), + ); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + format!("http://{addr}") + } + + #[tokio::test] + async fn answers_from_an_openai_compatible_runtime() { + let base = fake_runtime().await; + let models = list_models(&base).await.unwrap(); + assert_eq!(models, vec!["tiny:1b".to_string()], "embedding models are not chat models"); + let s = json!({"v": 1, "messages": [{"role": "user", "content": "ping"}]}); + let r = execute(&base, &s, &models).await; + assert!(valid_result(&r), "{r}"); + assert_eq!(r["text"], "echo[tiny:1b]: ping"); + assert_eq!(r["model"], "tiny:1b"); + let wrong = json!({"v": 1, "model": "big:70b", "messages": [{"role": "user", "content": "ping"}]}); + let r = execute(&base, &wrong, &models).await; + assert_eq!(r["ok"], false); + assert!(r["error"].as_str().unwrap().contains("not served")); + // Runtime down: an ok:false answer, not a crash. + let r = execute("http://127.0.0.1:1", &s, &models).await; + assert_eq!(r["ok"], false); + assert!(list_models("http://127.0.0.1:1").await.is_err()); + } +} diff --git a/node/crates/c0mpute-core/src/jobs_gateway.rs b/node/crates/c0mpute-core/src/jobs_gateway.rs index 38afa6f7..6267031f 100644 --- a/node/crates/c0mpute-core/src/jobs_gateway.rs +++ b/node/crates/c0mpute-core/src/jobs_gateway.rs @@ -5,8 +5,9 @@ //! winner's receipt, fetches the result by the receipt's hash from the //! winner, checks the bytes against that hash, and answers with the result. //! Accepts `workflow.step`, `crawl.fetch`, `ocr.extract` (input by hash), -//! `embed.text` and `blender.render-frame` (input pieces by hash; the bridge -//! fetches and checks the frames and returns them base64-encoded); +//! `embed.text`, `blender.render-frame` (input pieces by hash; the bridge +//! fetches and checks the frames and returns them base64-encoded) and +//! `infernet.inference` (chat completions); //! [`Workload`] is where more go. //! //! Same hardening as `c0mpute speech serve`: a required bearer token @@ -55,6 +56,7 @@ enum Workload { OcrExtract, EmbedText, RenderFrame, + Inference, } impl Workload { @@ -64,6 +66,7 @@ impl Workload { Workload::OcrExtract, Workload::EmbedText, Workload::RenderFrame, + Workload::Inference, ]; fn parse(name: &str) -> Option { @@ -76,6 +79,7 @@ impl Workload { Workload::OcrExtract => crate::ocr::WORKLOAD, Workload::EmbedText => crate::embed::WORKLOAD, Workload::RenderFrame => crate::render::WORKLOAD, + Workload::Inference => crate::inference::WORKLOAD, } } /// Reject a spec no worker could run before auctioning it. `workflow.step` @@ -87,6 +91,7 @@ impl Workload { Workload::OcrExtract => crate::ocr::parse_spec(spec).map(|_| ()), Workload::EmbedText => crate::embed::parse_spec(spec).map(|_| ()), Workload::RenderFrame => crate::render::parse_spec(spec).map(|_| ()), + Workload::Inference => crate::inference::parse_spec(spec).map(|_| ()), } } fn required_capabilities(self, spec: &Value) -> Vec { @@ -104,6 +109,7 @@ impl Workload { ] } Workload::RenderFrame => crate::render::required_capabilities_for(spec), + Workload::Inference => crate::inference::required_capabilities(spec), } } fn max_result_bytes(self) -> usize { @@ -112,6 +118,7 @@ impl Workload { Workload::OcrExtract => crate::ocr::MAX_RESULT_BYTES, Workload::EmbedText => crate::embed::MAX_RESULT_BYTES, Workload::RenderFrame => crate::render::MAX_RESULT_BYTES, + Workload::Inference => crate::inference::MAX_RESULT_BYTES, } } fn valid_result(self, spec: &Value, value: &Value) -> bool { @@ -126,6 +133,7 @@ impl Workload { }), Workload::RenderFrame => crate::render::parse_spec(spec) .is_ok_and(|parsed| crate::render::valid_result(&parsed, value)), + Workload::Inference => crate::inference::valid_result(value), } } } diff --git a/node/crates/c0mpute-core/src/lib.rs b/node/crates/c0mpute-core/src/lib.rs index 0bc26f3f..76dc2082 100644 --- a/node/crates/c0mpute-core/src/lib.rs +++ b/node/crates/c0mpute-core/src/lib.rs @@ -26,6 +26,7 @@ pub mod podcast; pub mod pricing; pub mod reachability; pub mod render; +pub mod inference; pub mod private; pub mod register; pub mod resources; diff --git a/node/crates/c0mpute-core/src/pieces.rs b/node/crates/c0mpute-core/src/pieces.rs index cdae9e1c..e7d47f74 100644 --- a/node/crates/c0mpute-core/src/pieces.rs +++ b/node/crates/c0mpute-core/src/pieces.rs @@ -334,6 +334,56 @@ impl InputRef { } } + /// The ref for stored [`Pieces`] (one piece: a single chunk). + pub fn of(p: &Pieces) -> Result { + let whole = Hash::from_hex(&p.hash).context("piece hash")?; + if p.pieces.len() <= 1 { + return Ok(Self::Single(whole)); + } + Ok(Self::Pieces { + whole, + size: p.size, + pieces: p + .pieces + .iter() + .map(|h| Hash::from_hex(h)) + .collect::, _>>() + .context("piece hash")?, + }) + } + + /// Fetch and verify the whole input into `dest` (streamed for pieces: + /// never whole in memory), renamed into place only once every check passed. + pub async fn fetch_to_file( + &self, + net: &Arc, + provider: Option, + max_bytes: usize, + dest: &Path, + ) -> Result<()> { + match self { + Self::Single(_) => { + let bytes = self.fetch(net, provider, max_bytes).await?; + let tmp = dest.with_extension(format!("part-{}", uuid::Uuid::new_v4())); + tokio::fs::write(&tmp, &bytes).await?; + tokio::fs::rename(&tmp, dest).await?; + Ok(()) + } + Self::Pieces { + whole, + size, + pieces, + } => { + if *size as usize > max_bytes { + bail!("the input is over the {max_bytes}-byte limit"); + } + fetch_pieces_to_file(net, provider, pieces, *size, *whole, INPUT_PIECE_BYTES, dest) + .await + .context("fetch the input pieces") + } + } + } + /// Fetch and verify the whole input into memory. pub async fn fetch( &self, diff --git a/node/crates/c0mpute-core/src/status_aggregator.rs b/node/crates/c0mpute-core/src/status_aggregator.rs index edf1cfe2..0bac1d0e 100644 --- a/node/crates/c0mpute-core/src/status_aggregator.rs +++ b/node/crates/c0mpute-core/src/status_aggregator.rs @@ -55,8 +55,8 @@ const DHT_CRAWL_INTERVAL: Duration = Duration::from_secs(30); /// Workload job topics we subscribe to. Gossipsub can't wildcard, so we /// enumerate the known workload types. `ffmpeg.transcode` is emitted by the -/// worker daemon; `infernet.inference` is emitted by the infernet peer -/// binary. Both always appear in the output (as zeros when idle) to mirror +/// worker daemon; `infernet.inference` by workers serving their +/// local OpenAI-compatible runtime. Both always appear in the output (as zeros when idle) to mirror /// the page's stub shape. // The temporal workloads (DIP 0026) are listed so seeds running the // aggregator also relay them for buyers that only know the seeds; their diff --git a/node/crates/c0mpute-core/src/supervisor.rs b/node/crates/c0mpute-core/src/supervisor.rs index 80e4f4a5..b56ebcf4 100644 --- a/node/crates/c0mpute-core/src/supervisor.rs +++ b/node/crates/c0mpute-core/src/supervisor.rs @@ -321,6 +321,29 @@ impl Supervisor { }); continue; } + // Chat completions from the operator's own OpenAI-compatible + // runtime. Probe it now and every minute; bid only while it lists + // chat models (the offer filter re-checks), advertise likewise. + if workload_type == crate::inference::WORKLOAD { + let Some(base) = crate::inference::runtime_url() else { + info!(%workload_type, "not subscribing: {} is off", crate::inference::RUNTIME_URL_ENV); + continue; + }; + tokio::spawn(crate::inference::refresh_loop(base.clone())); + let load = crate::inference::worker_load(); + let runner_tx = crate::inference::spawn_inference_runner(net.clone(), self.store.clone(), base, load.clone()); + info!(%workload_type, "subscribing to job topic"); + dispatch::run_worker_subscriber_with_load( + net.clone(), + workload_type, + tags.clone(), + Some(runner_tx), + Some(crate::inference::offer_filter()), + Some(self.config.pricing.clone()), + Some(load), + ); + continue; + } // Embeddings: provision fastembed in the background (never blocks // the worker), subscribe once a model answers. Same shape as TTS. if workload_type == crate::embed::WORKLOAD { @@ -655,7 +678,8 @@ pub fn which_ffmpeg() -> Option { None } -struct StoreSource(ChunkStore); +/// Serves chunk fetches from a [`ChunkStore`]. +pub struct StoreSource(pub ChunkStore); #[async_trait::async_trait] impl ChunkSource for StoreSource { diff --git a/node/crates/c0mpute-core/src/temporal/node.rs b/node/crates/c0mpute-core/src/temporal/node.rs index 4ec78e54..0cfdf37a 100644 --- a/node/crates/c0mpute-core/src/temporal/node.rs +++ b/node/crates/c0mpute-core/src/temporal/node.rs @@ -8,6 +8,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; use c0mpute_net::{Libp2pNetwork, Multiaddr, NetworkConfig, bootstrap}; +use c0mpute_store::ChunkStore; use tokio::sync::Semaphore; use tracing::info; @@ -27,6 +28,16 @@ pub struct NetOptions { } pub async fn spawn_node(o: &NetOptions) -> Result> { + spawn(o, None).await +} + +/// [`spawn_node`] that also answers chunk fetches from `store`, so peers can +/// pull pieces this node stored and announced (text-train datasets and adapters). +pub async fn spawn_node_with_store(o: &NetOptions, store: ChunkStore) -> Result> { + spawn(o, Some(store)).await +} + +async fn spawn(o: &NetOptions, store: Option) -> Result> { std::fs::create_dir_all(&o.identity_dir)?; let parse = |v: &[String]| -> Result> { v.iter().map(|s| s.parse::().with_context(|| format!("bad multiaddr {s}"))).collect() }; let mut boot = parse(&o.bootstrap)?; @@ -34,7 +45,10 @@ pub async fn spawn_node(o: &NetOptions) -> Result> { boot.extend(bootstrap::fetch_or_fallback(bootstrap::DEFAULT_BOOTSTRAP_URL, bootstrap::default_seeds()).await); } let listen = if o.listen.is_empty() { vec!["/ip4/0.0.0.0/tcp/0".to_string()] } else { o.listen.clone() }; - let cfg = NetworkConfig::for_dir(o.identity_dir.clone()).with_listen(parse(&listen)?).with_bootstrap(boot); + let mut cfg = NetworkConfig::for_dir(o.identity_dir.clone()).with_listen(parse(&listen)?).with_bootstrap(boot); + if let Some(store) = store { + cfg = cfg.with_local_source(Arc::new(crate::supervisor::StoreSource(store))); + } Ok(Arc::new(Libp2pNetwork::spawn(cfg).await?)) } diff --git a/node/crates/c0mpute-core/src/text.rs b/node/crates/c0mpute-core/src/text.rs index e673661d..e0cd458e 100644 --- a/node/crates/c0mpute-core/src/text.rs +++ b/node/crates/c0mpute-core/src/text.rs @@ -2,18 +2,26 @@ //! `infernet.text.infer` (answers from a trained adapter, further down). //! //! A buyer has a job file (`@infernetprotocol/training` job config, `kind: -//! "lora"`) and a JSONL corpus. `c0mpute infernet text train` serves the -//! corpus from a token-gated transfer server, auctions the job to workers that -//! advertise `infernet:text:train`, and shows the winner's training live: the -//! worker relays every training-protocol v1 event (`meta`, `step`, `eval`, -//! `checkpoint`, `done`) to the transfer server as it happens. The worker then -//! uploads the LoRA adapter, and the buyer keeps it only if its sha256 matches -//! the receipt's `output_hash`. +//! "lora"`) and a JSONL corpus. `c0mpute infernet text train` (and its front +//! door `c0mpute train lora`) stores the corpus as content-addressed pieces in +//! its own chunk store, auctions the job to workers that advertise +//! `infernet:text:train`, and shows the winner's training live: the worker +//! pulls the pieces over libp2p and publishes every training-protocol v1 +//! event (`meta`, `step`, `eval`, `checkpoint`, `done`) on the job topic as it +//! happens. The worker then stores the LoRA adapter as pieces behind an +//! [`AdapterManifest`] whose hash its receipt carries; the buyer fetches it +//! over libp2p and keeps it only if the weights' sha256 matches. Peer to peer +//! end to end: neither side needs a reachable HTTP address. +//! +//! Workers before the pieces transport take the corpus and return the adapter +//! over HTTP from the buyer's transfer server; buyers opt into serving them +//! with `--legacy-http` (see [`TextTrainSpec`]). //! //! The training itself is Infernet's (`infernet model train --job -//! --json`, the `peft` backend). c0mpute only moves the data, the progress and -//! the result, over the same offer → bid → accept → receipt protocol as every -//! other workload. One job per node at a time. +//! --json`, the `peft` backend); c0mpute hands it local file paths. c0mpute +//! only moves the data, the progress and the result, over the same offer → +//! bid → accept → receipt protocol as every other workload. One job per node +//! at a time. use std::collections::HashMap; use std::net::SocketAddr; @@ -23,7 +31,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use anyhow::{Context, Result, bail}; -use c0mpute_net::Libp2pNetwork; +use c0mpute_net::{Libp2pNetwork, Network, PeerId}; +use c0mpute_store::ChunkStore; use c0mpute_net::topics::JobStatus; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -36,7 +45,7 @@ use crate::buyer::JobAuction; use crate::runner::{AcceptedJob, publish_receipt}; use crate::temporal::client::auction_with_mesh_retry; use crate::temporal::http::{SlotEvent, TransferServer}; -use crate::temporal::node::{NetOptions, spawn_node}; +use crate::temporal::node::{NetOptions, spawn_node_with_store}; pub const TRAIN_WORKLOAD: &str = "infernet.text.train"; pub const KINDS: [&str; 2] = ["lora", "peft"]; @@ -49,19 +58,91 @@ pub fn tag(workload: &str) -> String { workload.replace('.', ":") } +/// Capability tag: this worker's runner trains `{"messages": [...]}` records +/// as whole multi-turn conversations (Infernet's peft runner reports +/// `formats: ["text", "messages"]`). Buyers whose corpus has conversations +/// require it, so an older runner (which would drop them) never wins. +pub const MESSAGES_FORMAT_TAG: &str = "infernet:text:format:messages"; +/// Largest adapter file a buyer accepts as pieces. +pub const MAX_ADAPTER_FILE_BYTES: usize = 2 << 30; +/// How long a worker keeps a finished adapter's pieces for the buyer to fetch. +const ADAPTER_PIECES_TTL: Duration = Duration::from_secs(6 * 3600); +/// Gossip message type a worker publishes each training event as. +pub const EVENT_MSG: &str = "text_train_event"; +/// At most one `step` event per this interval goes on the wire. +const STEP_EVENT_INTERVAL: Duration = Duration::from_millis(250); + /// `JobOffer.spec_inline` for `infernet.text.train`. +/// +/// Two transports, both checked against the same `dataset_hash` and adapter +/// hash: +/// +/// - **pieces (peer to peer, this version on)**: the buyer stores the corpus +/// as [`crate::pieces::INPUT_PIECE_BYTES`] pieces and names them in +/// `corpus_hash` / `corpus_size` / `corpus_pieces`; the worker pulls them +/// over libp2p. With `adapter_pieces` the worker stores the adapter files as +/// pieces too and its receipt's `output_hash` is the content hash of an +/// [`AdapterManifest`] chunk. With `run_id` it publishes every training +/// event on the job topic. Nothing needs to be dialable over HTTP. +/// - **HTTP (workers before this version)**: `dataset_url`, +/// `artifact_upload_url` and `events_url` on the buyer's transfer server; +/// the receipt's `output_hash` is `sha256:` of the adapter weights. A buyer +/// fills these only with `--legacy-http`. Older workers cannot parse a spec +/// without `dataset_url`, so they never bid on a pieces-only job. +/// +/// A worker that understands both prefers pieces. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct TextTrainSpec { /// The job config. The worker rewrites `dataset.url` and /// `output.destination_url` to local paths; everything else is Infernet's. pub job: Value, - pub dataset_url: String, /// `sha256:` of the corpus; the worker refuses a mismatch. pub dataset_hash: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub dataset_url: String, + #[serde(default, skip_serializing_if = "String::is_empty")] pub artifact_upload_url: String, - /// Where the worker posts each training event as it happens. + /// Where the worker posts each training event as it happens (HTTP). #[serde(default, skip_serializing_if = "Option::is_none")] pub events_url: Option, + /// Content hash of the corpus as stored pieces (see [`crate::pieces::InputRef`]). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub corpus_hash: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub corpus_size: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub corpus_pieces: Option>, + /// Return the adapter as pieces behind an [`AdapterManifest`]. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub adapter_pieces: bool, + /// Publish training events on the job topic tagged with this id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub run_id: Option, +} + +/// What a worker's receipt points at when the adapter travels as pieces. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct AdapterManifest { + pub v: u32, + /// `sha256:` of adapter_model.safetensors (the checkpoint's `model`). + pub adapter_sha256: String, + /// Each of [`ARTIFACTS`] the worker produced. + pub files: std::collections::BTreeMap, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ManifestFile { + pub hash: String, + pub size: u64, + pub pieces: Vec, +} + +/// A training event as published on the job topic. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct EventMsg { + pub t: String, + pub run_id: String, + pub event: Value, } fn is_http(u: &str) -> bool { @@ -82,11 +163,48 @@ impl TextTrainSpec { if !crate::temporal::is_sha256_ref(&self.dataset_hash) { bail!("dataset_hash must be sha256:<64 lowercase hex>"); } - if !is_http(&self.dataset_url) || !is_http(&self.artifact_upload_url) || self.events_url.as_deref().is_some_and(|u| !is_http(u)) { + let urls = [Some(self.dataset_url.as_str()), Some(self.artifact_upload_url.as_str()), self.events_url.as_deref()]; + if urls.into_iter().flatten().any(|u| !u.is_empty() && !is_http(u)) { bail!("dataset, upload and events URLs must be http(s)"); } + if self.corpus_ref().transpose()?.is_none() && self.dataset_url.is_empty() { + bail!("the spec names no corpus: corpus pieces or a dataset URL"); + } + if !self.adapter_pieces && self.artifact_upload_url.is_empty() { + bail!("the spec names no way back for the adapter: adapter_pieces or an upload URL"); + } + if self.run_id.as_ref().is_some_and(|r| r.is_empty() || r.len() > 64 || !r.chars().all(|c| c.is_ascii_alphanumeric() || c == '-')) { + bail!("run_id must be 1-64 letters, digits or '-'"); + } Ok(()) } + + /// The corpus as pieces, when the spec names them. + pub fn corpus_ref(&self) -> Option> { + let hash = self.corpus_hash.as_deref()?; + Some( + crate::pieces::InputRef::parse("corpus", hash, self.corpus_size, self.corpus_pieces.as_deref(), MAX_DATASET as usize) + .map_err(anyhow::Error::msg), + ) + } +} + +/// Capability tags a worker needs to train `corpus`. +pub fn required_tags(corpus: &[u8]) -> Vec { + let mut t = vec![tag(TRAIN_WORKLOAD)]; + if corpus_has_conversations(corpus) { + t.push(MESSAGES_FORMAT_TAG.into()); + } + t +} + +/// Does any record of a JSONL corpus carry `messages` (a conversation)? +pub fn corpus_has_conversations(corpus: &[u8]) -> bool { + corpus.split(|b| *b == b'\n').any(|line| { + // Cheap pre-check before parsing: most corpora are {title, text}. + line.windows(10).any(|w| w == b"\"messages\"") + && serde_json::from_slice::(line).is_ok_and(|v| v.get("messages").is_some_and(Value::is_array)) + }) } pub fn sha256_hex(b: &[u8]) -> String { @@ -126,6 +244,11 @@ pub fn worker_tags(runtime: &Value, ram_mb: u64) -> Vec { if runtime["cuda"] == json!(true) { t.push("infernet:text:device:cuda".into()); } + // Infernet's runner trains whole conversations from the release that + // reports it; older ones would silently keep only {title, text} records. + if runtime["formats"].as_array().is_some_and(|f| f.iter().any(|x| x == "messages")) { + t.push(MESSAGES_FORMAT_TAG.into()); + } for gb in crate::temporal::caps::RAM_BUCKETS_GB { if ram_mb >= gb * 1024 { t.push(format!("c0mpute:ram:{gb}gb+")); @@ -151,13 +274,32 @@ async fn download(url: &str, dst: &Path, want: &str) -> Result<()> { Ok(()) } +/// Fetch the corpus into `dst`: pieces over libp2p (from the buyer first) +/// when the spec names them, else the legacy HTTP URL. Either way the file's +/// sha256 must be the offer's `dataset_hash`. +pub async fn fetch_corpus(net: &Arc, buyer: Option, spec: &TextTrainSpec, dst: &Path) -> Result<()> { + match spec.corpus_ref() { + Some(r) => { + r?.fetch_to_file(net, buyer, MAX_DATASET as usize, dst).await.context("fetch corpus pieces")?; + let got = crate::temporal::worker::file_sha256(dst)?; + if got != spec.dataset_hash { + let _ = tokio::fs::remove_file(dst).await; + bail!("dataset hash {got} differs from the offer's {}", spec.dataset_hash); + } + Ok(()) + } + None => download(&spec.dataset_url, dst, &spec.dataset_hash).await, + } +} + /// Run one accepted job: fetch the corpus, run `infernet model train --job /// --json`, relay every event, return the adapter dir and its weights hash. -pub async fn run_train(infernet: &Path, job_dir: &Path, spec: &TextTrainSpec) -> Result<(PathBuf, String)> { +/// `buyer` is the peer the corpus pieces are fetched from first. +pub async fn run_train(infernet: &Path, job_dir: &Path, spec: &TextTrainSpec, net: &Arc, buyer: Option) -> Result<(PathBuf, String)> { spec.validate()?; tokio::fs::create_dir_all(job_dir).await?; let dataset = job_dir.join("dataset.jsonl"); - download(&spec.dataset_url, &dataset, &spec.dataset_hash).await.context("fetch dataset")?; + fetch_corpus(net, buyer, spec, &dataset).await.context("fetch dataset")?; let mut job = spec.job.clone(); job["dataset"] = json!({"url": "dataset.jsonl", "sha256": spec.dataset_hash.trim_start_matches("sha256:")}); job["output"] = json!({"destination_url": "adapter"}); @@ -188,13 +330,34 @@ pub async fn run_train(infernet: &Path, job_dir: &Path, spec: &TextTrainSpec) -> last }); - // Events go to the buyer from their own task so a slow link never stalls training. + // Events go to the buyer from their own task so a slow link never stalls + // training: on the job topic (p2p, `run_id`) and/or to the legacy HTTP URL. let (ev_tx, mut ev_rx) = mpsc::unbounded_channel::(); - let relay = spec.events_url.clone().map(|url| { + let relaying = spec.events_url.is_some() || spec.run_id.is_some(); + let relay = relaying.then(|| { + let (url, run_id, net) = (spec.events_url.clone(), spec.run_id.clone(), net.clone()); tokio::spawn(async move { let c = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().unwrap_or_default(); + let topic = c0mpute_net::topics::job_topic(TRAIN_WORKLOAD); + let mut last_step: Option = None; while let Some(ev) = ev_rx.recv().await { - if let Err(e) = c.post(&url).json(&ev).send().await { + if let Some(run_id) = &run_id { + let step = ev["type"] == "step"; + if !(step && last_step.is_some_and(|t| t.elapsed() < STEP_EVENT_INTERVAL)) { + if step { + last_step = Some(std::time::Instant::now()); + } + let msg = EventMsg { t: EVENT_MSG.into(), run_id: run_id.clone(), event: ev.clone() }; + if let Ok(b) = serde_json::to_vec(&msg) + && let Err(e) = net.publish(&topic, b).await + { + warn!(err = %e, "text: could not publish a training event"); + } + } + } + if let Some(url) = &url + && let Err(e) = c.post(url).json(&ev).send().await + { warn!(err = %e, "text: could not relay a training event"); } } @@ -215,7 +378,7 @@ pub async fn run_train(infernet: &Path, job_dir: &Path, spec: &TextTrainSpec) -> Some("done") => done = Some(ev.clone()), _ => {} } - if spec.events_url.is_some() { + if relaying { let _ = ev_tx.send(ev); } } @@ -253,21 +416,61 @@ async fn upload(base: &str, dir: &Path) -> Result<()> { Ok(()) } -pub fn spawn_train_runner(net: Arc, cache_root: PathBuf, infernet: PathBuf, busy: Arc, slots: Arc) -> mpsc::Sender { +/// Store the adapter files as pieces, then the [`AdapterManifest`] naming +/// them; returns the manifest's content hash (what the receipt carries). The +/// pieces are dropped from the store after [`ADAPTER_PIECES_TTL`]. +pub async fn store_adapter(net: &Arc, store: &ChunkStore, dir: &Path, adapter_sha256: &str) -> Result { + let mut files = std::collections::BTreeMap::new(); + let mut held = Vec::new(); + for f in ARTIFACTS { + let p = dir.join(f); + if !p.exists() { + continue; + } + let stored = crate::pieces::put_file_pieces(net, store, &p, MAX_ADAPTER_FILE_BYTES as u64, crate::pieces::INPUT_PIECE_BYTES).await.with_context(|| format!("store {f}"))?; + held.extend(stored.pieces.iter().cloned()); + files.insert(f.to_string(), ManifestFile { hash: stored.hash, size: stored.size, pieces: stored.pieces }); + } + if !files.contains_key("adapter_model.safetensors") { + bail!("no adapter_model.safetensors to store"); + } + let manifest = serde_json::to_vec(&AdapterManifest { v: 1, adapter_sha256: adapter_sha256.to_string(), files })?; + let hash = store.put(&manifest).await.context("store adapter manifest")?; + net.announce(&hash).await.context("announce adapter manifest")?; + held.push(hash.to_hex()); + let store = store.clone(); + tokio::spawn(async move { + tokio::time::sleep(ADAPTER_PIECES_TTL).await; + for h in held { + if let Ok(h) = c0mpute_proto::Hash::from_hex(&h) { + let _ = store.delete(&h).await; + } + } + }); + Ok(hash.to_hex()) +} + +pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root: PathBuf, infernet: PathBuf, busy: Arc, slots: Arc) -> mpsc::Sender { let (tx, mut rx) = mpsc::channel::(8); tokio::spawn(async move { while let Some(job) = rx.recv().await { let Ok(_slot) = slots.clone().acquire_owned().await else { return }; let accept = job.accept.clone(); busy.store(true, Ordering::Relaxed); + let job_dir = cache_root.join("text").join("jobs").join(&accept.job_id); let result: Result = async { let spec: TextTrainSpec = serde_json::from_value(accept.spec_inline.clone().context("accept lacks spec_inline")?)?; - let job_dir = cache_root.join("text").join("jobs").join(&accept.job_id); - let (dir, hash) = run_train(&infernet, &job_dir, &spec).await?; + let buyer = accept.buyer_peer_id.parse::().ok(); + let (dir, hash) = run_train(&infernet, &job_dir, &spec, &net, buyer).await?; + if spec.adapter_pieces { + // Receipt -> manifest -> pieces, all over libp2p. + return store_adapter(&net, &store, &dir, &hash).await; + } upload(&spec.artifact_upload_url, &dir).await.context("upload adapter")?; Ok(hash) } .await; + let _ = tokio::fs::remove_dir_all(&job_dir).await; busy.store(false, Ordering::Relaxed); match result { Ok(hash) => { @@ -309,9 +512,17 @@ const MAX_PROMPT_BYTES: usize = 32 * 1024; const ANSWER_SLOT: &str = "answer"; /// `JobOffer.spec_inline` for `infernet.text.infer`: one answer from a base -/// model + LoRA adapter. The adapter travels with the request (a bundle on -/// the buyer's transfer server) and is cached by hash, so any worker can -/// serve any adapter; the buyer does retrieval, the worker only generates. +/// model + LoRA adapter. The adapter travels with the request and is cached +/// by hash, so any worker can serve any adapter; the buyer does retrieval, +/// the worker only generates. +/// +/// Peer to peer (this version on): the buyer stores the adapter as pieces +/// behind an [`AdapterManifest`] (`adapter_manifest`), the worker fetches it +/// over libp2p on a cache miss, and with `answer_chunk` stores `answer.json` +/// as a chunk whose hash its receipt carries. Workers before this version need +/// `adapter_url` / `answer_upload_url` on the buyer's transfer server (the +/// buyer's `--legacy-http`); they cannot parse a spec without them, so they +/// never bid on a pieces-only request. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct TextInferSpec { /// `sha256:` of adapter_model.safetensors. @@ -320,9 +531,17 @@ pub struct TextInferSpec { pub messages: Vec, pub max_new_tokens: u32, /// `{"files": {"": ""}}` for the adapter, fetched only on a cache miss. + #[serde(default, skip_serializing_if = "String::is_empty")] pub adapter_url: String, /// Base URL the worker PUTs `answer.json` to. + #[serde(default, skip_serializing_if = "String::is_empty")] pub answer_upload_url: String, + /// Content hash of the adapter's [`AdapterManifest`] chunk. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub adapter_manifest: Option, + /// Return `answer.json` as a chunk (receipt `output_hash` = its content hash). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub answer_chunk: bool, } fn valid_repo_id(base: &str) -> bool { @@ -354,9 +573,18 @@ impl TextInferSpec { if !(1..=1024).contains(&self.max_new_tokens) { bail!("max_new_tokens must be 1..=1024"); } - if !is_http(&self.adapter_url) || !is_http(&self.answer_upload_url) { + if [&self.adapter_url, &self.answer_upload_url].iter().any(|u| !u.is_empty() && !is_http(u)) { bail!("adapter and answer URLs must be http(s)"); } + if self.adapter_manifest.as_deref().is_some_and(|h| c0mpute_proto::Hash::from_hex(h).is_err()) { + bail!("adapter_manifest must be a hex content hash"); + } + if self.adapter_manifest.is_none() && self.adapter_url.is_empty() { + bail!("the spec names no adapter: adapter_manifest or adapter_url"); + } + if !self.answer_chunk && self.answer_upload_url.is_empty() { + bail!("the spec names no way back for the answer: answer_chunk or an upload URL"); + } Ok(()) } } @@ -407,12 +635,23 @@ async fn fetch_adapter(url: &str, want: &str, dir: &Path) -> Result<()> { } /// Answer one request: adapter from the cache (or fetched), then `infernet model generate`. -pub async fn run_infer(infernet: &Path, cache_root: &Path, job_id: &str, spec: &TextInferSpec) -> Result> { +/// `buyer` is the peer adapter pieces are fetched from first. +pub async fn run_infer(infernet: &Path, cache_root: &Path, job_id: &str, spec: &TextInferSpec, net: &Arc, buyer: Option) -> Result> { spec.validate()?; let adapter = cache_root.join("text").join("adapters").join(spec.adapter_hash.trim_start_matches("sha256:")); let cached = crate::temporal::worker::file_sha256(&adapter.join("adapter_model.safetensors")).ok(); if cached.as_deref() != Some(spec.adapter_hash.as_str()) { - fetch_adapter(&spec.adapter_url, &spec.adapter_hash, &adapter).await.context("fetch adapter")?; + match &spec.adapter_manifest { + Some(m) => { + let _ = tokio::fs::remove_dir_all(&adapter).await; + let got = fetch_adapter_pieces(net, buyer, m, &adapter).await.context("fetch adapter pieces")?; + if got != spec.adapter_hash { + let _ = tokio::fs::remove_dir_all(&adapter).await; + bail!("adapter {got} differs from the offer's {}", spec.adapter_hash); + } + } + None => fetch_adapter(&spec.adapter_url, &spec.adapter_hash, &adapter).await.context("fetch adapter")?, + } } let dir = cache_root.join("text").join("infer").join(job_id); tokio::fs::create_dir_all(&dir).await?; @@ -434,7 +673,7 @@ pub async fn run_infer(infernet: &Path, cache_root: &Path, job_id: &str, spec: & Ok(serde_json::to_vec(&ans)?) } -pub fn spawn_infer_runner(net: Arc, cache_root: PathBuf, infernet: PathBuf, busy: Arc) -> mpsc::Sender { +pub fn spawn_infer_runner(net: Arc, store: ChunkStore, cache_root: PathBuf, infernet: PathBuf, busy: Arc) -> mpsc::Sender { let (tx, mut rx) = mpsc::channel::(16); tokio::spawn(async move { while let Some(job) = rx.recv().await { @@ -442,7 +681,13 @@ pub fn spawn_infer_runner(net: Arc, cache_root: PathBuf, infernet busy.store(true, Ordering::Relaxed); let result: Result = async { let spec: TextInferSpec = serde_json::from_value(accept.spec_inline.clone().context("accept lacks spec_inline")?)?; - let answer = run_infer(&infernet, &cache_root, &accept.job_id, &spec).await?; + let buyer = accept.buyer_peer_id.parse::().ok(); + let answer = run_infer(&infernet, &cache_root, &accept.job_id, &spec, &net, buyer).await?; + if spec.answer_chunk { + let hash = store.put(&answer).await.context("store answer")?; + net.announce(&hash).await.context("announce answer")?; + return Ok(hash.to_hex()); + } reqwest::Client::new().put(format!("{}/answer.json", spec.answer_upload_url)).body(answer.clone()).send().await?.error_for_status()?; Ok(sha256_hex(&answer)) } @@ -500,16 +745,18 @@ pub async fn serve(net_opts: &NetOptions, infernet: PathBuf, cache_root: PathBuf if infer { tags.push(tag(INFER_WORKLOAD)); } - let net = spawn_node(net_opts).await?; + // Adapter pieces live here until the buyer has fetched them. + let store = ChunkStore::open(cache_root.join("text").join("store")).await?; + let net = spawn_node_with_store(net_opts, store.clone()).await?; let hardware = json!({"version": env!("CARGO_PKG_VERSION"), "text_runtime": "peft", "ram_mb": ram_mb, "cuda": runtime["cuda"]}); tokio::spawn(crate::capabilities::advertise_loop(net.clone(), tags.clone(), hardware, crate::capabilities::local_bench_score(), crate::capabilities::DEFAULT_ADVERTISE_INTERVAL)); let busy = Arc::new(AtomicBool::new(false)); let slots = Arc::new(Semaphore::new(1)); - let tx = spawn_train_runner(net.clone(), cache_root.clone(), infernet.clone(), busy.clone(), slots); + let tx = spawn_train_runner(net.clone(), store.clone(), cache_root.clone(), infernet.clone(), busy.clone(), slots); crate::dispatch::run_worker_subscriber_filtered(net.clone(), TRAIN_WORKLOAD.into(), tags.clone(), Some(tx), Some(train_offer_filter(busy))); if infer { let ibusy = Arc::new(AtomicBool::new(false)); - let itx = spawn_infer_runner(net.clone(), cache_root, infernet, ibusy.clone()); + let itx = spawn_infer_runner(net.clone(), store.clone(), cache_root, infernet, ibusy.clone()); crate::dispatch::run_worker_subscriber_filtered(net.clone(), INFER_WORKLOAD.into(), tags.clone(), Some(itx), Some(infer_offer_filter(ibusy))); } info!(peer = %net.peer_id().to_base58(), ?tags, "text: serving"); @@ -529,6 +776,10 @@ pub struct TrainRequest { pub max_price_usd: f64, pub gpu: bool, pub min_ram_gb: Option, + /// Also serve the corpus and take the adapter over HTTP from this machine, + /// so workers older than the pieces transport can bid (they need to reach + /// `public_url` / `bind`). Off by default: the job is fully peer to peer. + pub legacy_http: bool, pub public_url: Option, pub bind: SocketAddr, pub timeout_s: u64, @@ -602,19 +853,43 @@ pub fn render(ev: &Value, total: &mut Option) -> Option { } /// Auction the job, stream its progress through `on_event`, verify and keep the adapter. -pub async fn train(net: Arc, req: &TrainRequest, mut on_event: impl FnMut(&Value) + Send) -> Result { +/// +/// Peer to peer: the corpus goes out as pieces from `store` (served by `net`), +/// progress arrives on the job topic, and the adapter comes back as pieces +/// named by the manifest the worker's receipt points at. With +/// `req.legacy_http` the buyer also runs the HTTP transfer server, so a worker +/// that predates pieces can win; its adapter then arrives over HTTP and is +/// checked against the receipt exactly as before. +pub async fn train(net: Arc, store: &ChunkStore, req: &TrainRequest, mut on_event: impl FnMut(&Value) + Send) -> Result { let (job, bytes, preview) = load(req)?; - let (ev_tx, mut ev_rx) = mpsc::unbounded_channel::(); - let server = TransferServer::start_with(req.bind, req.public_url.clone(), bytes, &ARTIFACTS, Some(ev_tx)).await?; - let spec = TextTrainSpec { + let stored = crate::pieces::put_bytes_pieces(&net, store, &bytes, crate::pieces::INPUT_PIECE_BYTES).await.context("store the corpus as pieces")?; + let run_id = uuid::Uuid::new_v4().to_string(); + let mut spec = TextTrainSpec { job, - dataset_url: server.dataset_url(), dataset_hash: preview.dataset_hash.clone(), - artifact_upload_url: server.upload_url(SLOT), - events_url: Some(server.events_url(SLOT)), + dataset_url: String::new(), + artifact_upload_url: String::new(), + events_url: None, + corpus_hash: Some(stored.hash.clone()), + corpus_size: (stored.pieces.len() > 1).then_some(stored.size), + corpus_pieces: (stored.pieces.len() > 1).then(|| stored.pieces.clone()), + adapter_pieces: true, + run_id: Some(run_id.clone()), + }; + let (ev_tx, mut ev_rx) = mpsc::unbounded_channel::(); + let server = if req.legacy_http { + let s = TransferServer::start_with(req.bind, req.public_url.clone(), bytes.clone(), &ARTIFACTS, Some(ev_tx)).await?; + spec.dataset_url = s.dataset_url(); + spec.artifact_upload_url = s.upload_url(SLOT); + spec.events_url = Some(s.events_url(SLOT)); + Some(s) + } else { + drop(ev_tx); + None }; spec.validate()?; - let mut tags = vec![tag(TRAIN_WORKLOAD)]; + let mut tags = required_tags(&bytes); + drop(bytes); if req.gpu { tags.push("infernet:text:device:cuda".into()); } @@ -622,6 +897,16 @@ pub async fn train(net: Arc, req: &TrainRequest, mut on_event: im let b = crate::temporal::caps::RAM_BUCKETS_GB.iter().copied().find(|b| *b >= min).unwrap_or(128); tags.push(format!("c0mpute:ram:{b}gb+")); } + // Training events published on the job topic for this run. + let topic = c0mpute_net::topics::job_topic(TRAIN_WORKLOAD); + let mut gossip = net.messages(); + let p2p_event = |m: &c0mpute_net::GossipMessage| -> Option { + if m.topic != topic { + return None; + } + let msg: EventMsg = serde_json::from_slice(&m.data).ok()?; + (msg.t == EVENT_MSG && msg.run_id == run_id).then_some(msg.event) + }; let spec_v = serde_json::to_value(&spec)?; let mut last_err = None; let mut outcome = None; @@ -642,11 +927,21 @@ pub async fn train(net: Arc, req: &TrainRequest, mut on_event: im tokio::select! { r = &mut auction => break r, Some((_, ev)) = ev_rx.recv() => on_event(&ev), + m = gossip.recv() => match m { + Ok(m) => if let Some(ev) = p2p_event(&m) { on_event(&ev) }, + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {} + Err(_) => {} + }, } }; while let Ok((_, ev)) = ev_rx.try_recv() { on_event(&ev); } + while let Ok(m) = gossip.try_recv() { + if let Some(ev) = p2p_event(&m) { + on_event(&ev); + } + } match res { Ok(o) if o.receipt.status == JobStatus::Completed => { outcome = Some(o); @@ -656,22 +951,84 @@ pub async fn train(net: Arc, req: &TrainRequest, mut on_event: im Err(e) => last_err = Some(e.context(format!("attempt {}", attempt + 1))), } } - let o = outcome.ok_or_else(|| last_err.unwrap_or_else(|| anyhow::anyhow!("training failed")))?; - let files: HashMap> = server.take(SLOT).await; - let weights = files.get("adapter_model.safetensors").context("worker did not upload adapter_model.safetensors")?; - let got = sha256_hex(weights); - let claimed = o.receipt.output_hash.clone().unwrap_or_default(); - if got != claimed { - bail!("adapter {got} does not match the receipt's {claimed}"); + let result = async { + let o = outcome.ok_or_else(|| last_err.unwrap_or_else(|| anyhow::anyhow!("training failed")))?; + let claimed = o.receipt.output_hash.clone().unwrap_or_default(); + let (got, transport) = if claimed.starts_with("sha256:") { + // A worker before pieces: it uploaded over HTTP. + let server = server.as_ref().context("the worker answered with an HTTP upload, but this job has no HTTP transfer server")?; + let files: HashMap> = server.take(SLOT).await; + let weights = files.get("adapter_model.safetensors").context("worker did not upload adapter_model.safetensors")?; + let got = sha256_hex(weights); + if got != claimed { + bail!("adapter {got} does not match the receipt's {claimed}"); + } + std::fs::create_dir_all(&req.output)?; + for (name, data) in &files { + std::fs::write(req.output.join(name), data)?; + } + (got, "http") + } else { + let worker = o.receipt.worker_peer_id.parse::().ok(); + (fetch_adapter_pieces(&net, worker, &claimed, &req.output).await?, "pieces") + }; + Ok(json!({ + "adapter_dir": req.output, "adapter_hash": got, "base_model": preview.base_model, "dataset_hash": preview.dataset_hash, + "transport": transport, + "receipt": {"job_id": o.receipt.job_id, "worker_peer_id": o.receipt.worker_peer_id, "output_hash": o.receipt.output_hash, "price_usd": o.accepted_bid.price_usd} + })) } - std::fs::create_dir_all(&req.output)?; - for (name, data) in &files { - std::fs::write(req.output.join(name), data)?; + .await; + // The corpus pieces have done their job (stored whole too when small). + for h in stored.pieces.iter().chain(std::iter::once(&stored.hash)) { + if let Ok(h) = c0mpute_proto::Hash::from_hex(h) { + let _ = store.delete(&h).await; + } + } + result +} + +/// Fetch the [`AdapterManifest`] a receipt names, then every file it lists, +/// into `out`; keep them only if adapter_model.safetensors hashes to the +/// manifest's `adapter_sha256`. Returns that hash. +pub async fn fetch_adapter_pieces(net: &Arc, worker: Option, manifest_hash: &str, out: &Path) -> Result { + let mref = crate::pieces::InputRef::parse("adapter_manifest", manifest_hash, None, None, 1 << 20).map_err(anyhow::Error::msg)?; + let manifest: AdapterManifest = serde_json::from_slice(&mref.fetch(net, worker, 1 << 20).await.context("fetch the adapter manifest")?).context("adapter manifest is not JSON")?; + if manifest.v != 1 || !crate::temporal::is_sha256_ref(&manifest.adapter_sha256) { + bail!("adapter manifest has an unknown version or a bad adapter_sha256"); + } + if !manifest.files.contains_key("adapter_model.safetensors") || manifest.files.keys().any(|k| !ARTIFACTS.contains(&k.as_str())) { + bail!("adapter manifest must list adapter_model.safetensors and only {ARTIFACTS:?}"); + } + let tmp = out.with_extension(format!("partial-{}", uuid::Uuid::new_v4())); + tokio::fs::create_dir_all(&tmp).await?; + let fetched: Result = async { + for (name, f) in &manifest.files { + let (size, pieces) = if f.pieces.len() > 1 { (Some(f.size), Some(f.pieces.as_slice())) } else { (None, None) }; + let r = crate::pieces::InputRef::parse("adapter", &f.hash, size, pieces, MAX_ADAPTER_FILE_BYTES).map_err(anyhow::Error::msg)?; + r.fetch_to_file(net, worker, MAX_ADAPTER_FILE_BYTES, &tmp.join(name)).await.with_context(|| format!("fetch {name}"))?; + } + let got = crate::temporal::worker::file_sha256(&tmp.join("adapter_model.safetensors"))?; + if got != manifest.adapter_sha256 { + bail!("adapter {got} does not match the manifest's {}", manifest.adapter_sha256); + } + Ok(got) + } + .await; + match fetched { + Ok(got) => { + tokio::fs::create_dir_all(out).await?; + for name in manifest.files.keys() { + tokio::fs::rename(tmp.join(name), out.join(name)).await?; + } + let _ = tokio::fs::remove_dir_all(&tmp).await; + Ok(got) + } + Err(e) => { + let _ = tokio::fs::remove_dir_all(&tmp).await; + Err(e) + } } - Ok(json!({ - "adapter_dir": req.output, "adapter_hash": got, "base_model": preview.base_model, "dataset_hash": preview.dataset_hash, - "receipt": {"job_id": o.receipt.job_id, "worker_peer_id": o.receipt.worker_peer_id, "output_hash": o.receipt.output_hash, "price_usd": o.accepted_bid.price_usd} - })) } // ── chat (buyer) ──────────────────────────────────────────────────────── @@ -723,6 +1080,8 @@ pub struct ChatRequest { pub max_new_tokens: u32, pub workers: Vec, pub max_price_usd: f64, + /// Also serve workers older than the pieces transport over HTTP. + pub legacy_http: bool, pub public_url: Option, pub bind: SocketAddr, pub timeout_s: u64, @@ -730,7 +1089,12 @@ pub struct ChatRequest { } /// Ask a worker; returns the answer, its sources and the receipt. -pub async fn chat(net: Arc, req: &ChatRequest) -> Result { +/// +/// Peer to peer: the adapter goes out as pieces from `store` (the worker +/// fetches them only on a cache miss) and the answer comes back as a chunk +/// named by the receipt. `req.legacy_http` adds the HTTP transfer server for +/// older workers. +pub async fn chat(net: Arc, store: &ChunkStore, req: &ChatRequest) -> Result { let (bundle, adapter_hash, manifest) = adapter_bundle(&req.adapter_dir)?; let base_model = manifest["base_model"].as_str().context("the adapter's manifest.json has no base_model")?.to_string(); let sources = match &req.search { @@ -742,14 +1106,24 @@ pub async fn chat(net: Arc, req: &ChatRequest) -> Result { messages.push(json!({"role": "system", "content": system})); } messages.push(json!({"role": "user", "content": chat_prompt(&req.question, &sources)})); - let server = TransferServer::start_with(req.bind, req.public_url.clone(), bundle, &ANSWER_ARTIFACTS, None).await?; - let spec = TextInferSpec { + let adapter_manifest = store_adapter(&net, store, &req.adapter_dir, &adapter_hash).await.context("store the adapter as pieces")?; + let mut spec = TextInferSpec { adapter_hash: adapter_hash.clone(), base_model: base_model.clone(), messages, max_new_tokens: req.max_new_tokens, - adapter_url: server.dataset_url(), - answer_upload_url: server.upload_url(ANSWER_SLOT), + adapter_url: String::new(), + answer_upload_url: String::new(), + adapter_manifest: Some(adapter_manifest), + answer_chunk: true, + }; + let server = if req.legacy_http { + let s = TransferServer::start_with(req.bind, req.public_url.clone(), bundle, &ANSWER_ARTIFACTS, None).await?; + spec.adapter_url = s.dataset_url(); + spec.answer_upload_url = s.upload_url(ANSWER_SLOT); + Some(s) + } else { + None }; spec.validate()?; let spec_v = serde_json::to_value(&spec)?; @@ -762,17 +1136,30 @@ pub async fn chat(net: Arc, req: &ChatRequest) -> Result { .with_deadline(Duration::from_secs(req.timeout_s + 120)) .with_receipt_timeout(Duration::from_secs(req.timeout_s + 60)) }; - let o = auction_with_mesh_retry(net, make).await?; + let o = auction_with_mesh_retry(net.clone(), make).await?; if o.receipt.status != JobStatus::Completed { bail!("worker {} reported {:?}", o.accepted_bid.bidder_peer_id, o.receipt.status); } - let files = server.take(ANSWER_SLOT).await; - let answer = files.get("answer.json").context("worker did not upload answer.json")?; - let got = sha256_hex(answer); - if Some(got.as_str()) != o.receipt.output_hash.as_deref() { - bail!("answer {got} does not match the receipt's {:?}", o.receipt.output_hash); + let claimed = o.receipt.output_hash.clone().unwrap_or_default(); + let answer = if claimed.starts_with("sha256:") { + // A worker before pieces: it uploaded over HTTP. + let server = server.as_ref().context("the worker answered over HTTP, but this request has no transfer server")?; + let files = server.take(ANSWER_SLOT).await; + let answer = files.get("answer.json").context("worker did not upload answer.json")?.clone(); + let got = sha256_hex(&answer); + if got != claimed { + bail!("answer {got} does not match the receipt's {claimed}"); + } + answer + } else { + // The fetch checks the bytes against the receipt's content hash. + let r = crate::pieces::InputRef::parse("answer", &claimed, None, None, 1 << 20).map_err(anyhow::Error::msg)?; + r.fetch(&net, o.receipt.worker_peer_id.parse::().ok(), 1 << 20).await.context("fetch the answer")? + }; + let ans: Value = serde_json::from_slice(&answer)?; + if ans["adapter"].as_str().is_some_and(|a| a != adapter_hash) { + bail!("the worker answered with adapter {} instead of {adapter_hash}", ans["adapter"]); } - let ans: Value = serde_json::from_slice(answer)?; Ok(json!({ "text": ans["text"], "sources": sources, "base_model": base_model, "adapter_hash": adapter_hash, "tokens": ans["tokens"], "seconds": ans["seconds"], @@ -792,7 +1179,33 @@ mod tests { max_new_tokens: 200, adapter_url: "http://127.0.0.1:1/t/x/dataset.json".into(), answer_upload_url: "http://127.0.0.1:1/t/x/a/answer".into(), + adapter_manifest: None, + answer_chunk: false, + } + } + + #[test] + fn p2p_infer_specs() { + let s = TextInferSpec { + adapter_url: String::new(), + answer_upload_url: String::new(), + adapter_manifest: Some(c0mpute_proto::Hash::of(b"m").to_hex()), + answer_chunk: true, + ..infer_spec() + }; + s.validate().unwrap(); + let v = serde_json::to_value(&s).unwrap(); + assert!(v.get("adapter_url").is_none()); + #[derive(Deserialize)] + #[allow(dead_code)] + struct OldInfer { + adapter_url: String, + answer_upload_url: String, } + assert!(serde_json::from_value::(v).is_err(), "old workers never bid on a pieces-only chat"); + assert!(TextInferSpec { adapter_manifest: None, ..s.clone() }.validate().is_err()); + assert!(TextInferSpec { answer_chunk: false, ..s.clone() }.validate().is_err()); + assert!(TextInferSpec { adapter_manifest: Some("zz".into()), ..s }.validate().is_err()); } #[test] @@ -847,9 +1260,74 @@ mod tests { dataset_hash: format!("sha256:{}", "a".repeat(64)), artifact_upload_url: "http://127.0.0.1:1/t/x/a/adapter".into(), events_url: Some("http://127.0.0.1:1/t/x/e/adapter".into()), + corpus_hash: None, + corpus_size: None, + corpus_pieces: None, + adapter_pieces: false, + run_id: None, } } + fn p2p_spec() -> TextTrainSpec { + TextTrainSpec { + dataset_url: String::new(), + artifact_upload_url: String::new(), + events_url: None, + corpus_hash: Some(c0mpute_proto::Hash::of(b"corpus").to_hex()), + adapter_pieces: true, + run_id: Some("0e1f2a3b-run".into()), + ..spec() + } + } + + #[test] + fn p2p_specs_validate_and_old_workers_cannot_parse_them() { + let s = p2p_spec(); + s.validate().unwrap(); + let v = serde_json::to_value(&s).unwrap(); + // No HTTP fields on the wire at all. + assert!(v.get("dataset_url").is_none() && v.get("artifact_upload_url").is_none() && v.get("events_url").is_none()); + // A worker before pieces requires dataset_url, so it never bids. + #[derive(Deserialize)] + #[allow(dead_code)] + struct OldSpec { + job: Value, + dataset_url: String, + dataset_hash: String, + artifact_upload_url: String, + } + assert!(serde_json::from_value::(v.clone()).is_err()); + assert_eq!(serde_json::from_value::(v).unwrap(), s); + // A legacy spec (an old buyer) still parses and validates. + let old = json!({"job": spec().job, "dataset_url": "http://h/d", "dataset_hash": spec().dataset_hash, "artifact_upload_url": "http://h/u"}); + serde_json::from_value::(old).unwrap().validate().unwrap(); + + let mut bad = p2p_spec(); + bad.corpus_hash = None; + assert!(bad.validate().unwrap_err().to_string().contains("no corpus")); + let mut bad = p2p_spec(); + bad.adapter_pieces = false; + assert!(bad.validate().unwrap_err().to_string().contains("way back")); + let mut bad = p2p_spec(); + bad.corpus_size = Some(3 << 20); + bad.corpus_pieces = Some(vec![c0mpute_proto::Hash::of(b"x").to_hex()]); + assert!(bad.validate().is_err(), "3 MiB cannot be one piece"); + let mut bad = p2p_spec(); + bad.run_id = Some("no spaces".into()); + assert!(bad.validate().is_err()); + } + + #[test] + fn conversations_require_the_messages_format_tag() { + let text = b"{\"title\":\"a\",\"text\":\"b\"}\n{\"title\":\"c\",\"text\":\"the word \\\"messages\\\" in text\"}\n"; + assert_eq!(required_tags(text), vec!["infernet:text:train".to_string()]); + let chat = b"{\"title\":\"a\",\"text\":\"b\"}\n{\"messages\":[{\"role\":\"user\",\"content\":\"hi\"},{\"role\":\"assistant\",\"content\":\"yo\"}]}\n"; + assert_eq!(required_tags(chat), vec!["infernet:text:train".to_string(), MESSAGES_FORMAT_TAG.to_string()]); + let t = worker_tags(&json!({"cuda": false, "formats": ["text", "messages"]}), 0); + assert!(t.contains(&MESSAGES_FORMAT_TAG.to_string())); + assert!(!worker_tags(&json!({"cuda": false}), 0).contains(&MESSAGES_FORMAT_TAG.to_string())); + } + #[test] fn validates_kind_model_hash_and_urls() { assert!(spec().validate().is_ok()); diff --git a/node/crates/c0mpute-core/src/train.rs b/node/crates/c0mpute-core/src/train.rs index c8c5b98a..1f725a40 100644 --- a/node/crates/c0mpute-core/src/train.rs +++ b/node/crates/c0mpute-core/src/train.rs @@ -2,22 +2,27 @@ //! training c0mpute already runs as `infernet.text.train` ([`crate::text`]). //! //! It does not train anything itself. It checks the buyer's dataset, turns -//! the common JSONL shapes into the `{title, text}` corpus the worker's peft -//! runner reads, writes the job config from a handful of flags, and hands -//! both to [`crate::text::train`], which auctions the job, relays progress, -//! and keeps the adapter only if its sha256 matches the receipt. +//! the common JSONL shapes into the corpus the worker's peft runner reads, +//! writes the job config from a handful of flags, and hands both to +//! [`crate::text::train`], which auctions the job peer to peer, relays +//! progress, and keeps the adapter only if its sha256 matches. //! //! Accepted record shapes (one JSON object per line, mixing allowed): //! -//! - `{"text": "...", "title"?: "..."}`: used as is. -//! - `{"prompt": "...", "completion": "..."}`: `title` = prompt, `text` = completion. -//! - `{"messages": [{"role", "content"}, ...]}`: the last user turn becomes -//! `title`, the last assistant turn `text`; a system turn (the same in every -//! record) becomes the job's `system_prompt`. +//! - `{"text": "...", "title"?: "..."}`: kept as `{title, text}`; the worker +//! trains it as one `user: "Brief me on: "` / `assistant: <text>` +//! exchange, loss on the whole exchange (as before). +//! - `{"prompt": "...", "completion": "..."}`: a one-exchange conversation, +//! the prompt verbatim as the user turn. +//! - `{"messages": [{"role", "content"}, ...]}`: kept whole, every turn in +//! order. The worker renders it with the base model's chat template and +//! computes the loss on the assistant turns only. A system turn the same in +//! every record also becomes the job's `system_prompt` (the adapter's +//! default for chat); records without one get that prompt at train time. //! -//! The worker formats each record as one chat exchange -//! (`system`, `user: "Brief me on: <title>"`, `assistant: <text>`), so -//! multi-turn conversations train on their final exchange only. +//! Conversations need a runner that reports the `messages` dataset format +//! (Infernet with peft `formats: ["text", "messages"]`), so a corpus holding +//! any requires [`crate::text::MESSAGES_FORMAT_TAG`] of the worker. use anyhow::{Result, bail}; use serde::Serialize; @@ -69,15 +74,18 @@ pub fn normalize_dataset(bytes: &[u8]) -> Result<(Vec<u8>, DatasetReport)> { if !v.is_object() { bail!("line {n}: each line must be a JSON object"); } - let (title, body) = if let Some(t) = str_field(&v, "text") { + let (record, bytes) = if let Some(t) = str_field(&v, "text") { report.text += 1; - (str_field(&v, "title").unwrap_or("").to_string(), t.to_string()) + let title = str_field(&v, "title").unwrap_or(""); + (json!({"title": title, "text": t}), title.len() + t.len()) } else if let (Some(p), Some(c)) = (str_field(&v, "prompt"), str_field(&v, "completion")) { + // The prompt verbatim as the user turn (not a "Brief me on:" title). report.prompt_completion += 1; - (p.to_string(), c.to_string()) + (json!({"messages": [{"role": "user", "content": p}, {"role": "assistant", "content": c}]}), p.len() + c.len()) } else if let Some(msgs) = v.get("messages").and_then(Value::as_array) { - let mut user = None; - let mut assistant = None; + let mut turns = Vec::with_capacity(msgs.len()); + let mut bytes = 0; + let mut learns = false; for m in msgs { let role = m.get("role").and_then(Value::as_str).unwrap_or(""); let content = m.get("content").and_then(Value::as_str); @@ -90,27 +98,31 @@ pub fn normalize_dataset(bytes: &[u8]) -> Result<(Vec<u8>, DatasetReport)> { Some(s) if s != content => system_conflict = true, _ => {} }, - "user" => user = Some(content), - "assistant" => assistant = Some(content), + "user" => {} + "assistant" => learns |= !content.trim().is_empty(), other => bail!("line {n}: message role must be system, user or assistant, got {other:?}"), } + bytes += content.len(); + turns.push(json!({"role": role, "content": content})); } - let Some(a) = assistant.filter(|a| !a.trim().is_empty()) else { + if !learns { bail!("line {n}: messages need an assistant turn to learn from"); - }; + } report.messages += 1; - (user.unwrap_or("").to_string(), a.to_string()) + // Every turn, in order: the worker trains the whole conversation + // (loss on the assistant turns). + (json!({"messages": turns}), bytes) } else { bail!("line {n}: expected {{\"text\"}}, {{\"prompt\",\"completion\"}} or {{\"messages\"}}"); }; - if title.len() + body.len() > MAX_RECORD_BYTES { + if bytes > MAX_RECORD_BYTES { bail!("line {n}: record is over {} KiB", MAX_RECORD_BYTES >> 10); } report.records += 1; if report.records > MAX_RECORDS { bail!("dataset has more than {MAX_RECORDS} records"); } - serde_json::to_writer(&mut out, &json!({"title": title, "text": body}))?; + serde_json::to_writer(&mut out, &record)?; out.push(b'\n'); } if report.records < MIN_RECORDS { @@ -247,8 +259,48 @@ mod tests { assert_eq!(r.system_prompt.as_deref(), Some("be brief")); let lines: Vec<Value> = std::str::from_utf8(&out).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); assert_eq!(lines[0], json!({"title": "A", "text": "a body"})); - assert_eq!(lines[1], json!({"title": "Q?", "text": "Ans."})); - assert_eq!(lines[2], json!({"title": "hi", "text": "hello"})); + assert_eq!(lines[1], json!({"messages": [{"role": "user", "content": "Q?"}, {"role": "assistant", "content": "Ans."}]})); + assert_eq!( + lines[2], + json!({"messages": [{"role": "system", "content": "be brief"}, {"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}) + ); + assert!(crate::text::corpus_has_conversations(&out)); + } + + #[test] + fn multi_turn_chats_keep_every_turn_in_order() { + let chat = json!({"messages": [ + {"role": "system", "content": "You are a pricing coach."}, + {"role": "user", "content": "I sell a SaaS."}, + {"role": "assistant", "content": "What does it replace?"}, + {"role": "user", "content": "Spreadsheets."}, + {"role": "assistant", "content": "Price against the hours it saves."}, + ]}); + let src = format!("{chat}\n{chat}\n"); + let (out, r) = normalize_dataset(src.as_bytes()).unwrap(); + assert_eq!((r.records, r.messages), (2, 2)); + let first: Value = serde_json::from_str(std::str::from_utf8(&out).unwrap().lines().next().unwrap()).unwrap(); + // Nothing dropped, nothing reordered: both exchanges are there. + assert_eq!(first, chat); + assert_eq!(r.system_prompt.as_deref(), Some("You are a pricing coach.")); + // A record whose only assistant turn is empty teaches nothing. + let empty = b"{\"messages\":[{\"role\":\"user\",\"content\":\"q\"},{\"role\":\"assistant\",\"content\":\" \"}]}\n{\"text\":\"x\"}\n"; + assert!(normalize_dataset(empty).unwrap_err().to_string().contains("assistant")); + // Extra fields on a message are not carried to the worker. + let extra = b"{\"messages\":[{\"role\":\"user\",\"content\":\"q\",\"name\":\"n\"},{\"role\":\"assistant\",\"content\":\"a\"}]}\n{\"text\":\"x\"}\n"; + let (out, _) = normalize_dataset(extra).unwrap(); + assert!(!std::str::from_utf8(&out).unwrap().contains("\"name\"")); + // Text-only corpora stay {title, text} and need no format tag. + let (out, _) = normalize_dataset(b"{\"text\":\"a\"}\n{\"text\":\"b\",\"title\":\"t\"}\n").unwrap(); + assert!(!crate::text::corpus_has_conversations(&out)); + } + + #[test] + fn a_long_conversation_counts_toward_the_record_cap() { + let turn = "a".repeat(MAX_RECORD_BYTES / 4); + let msgs: Vec<Value> = (0..5).map(|i| json!({"role": if i % 2 == 0 { "user" } else { "assistant" }, "content": turn})).collect(); + let src = format!("{}\n{{\"text\":\"x\"}}\n", json!({"messages": msgs})); + assert!(normalize_dataset(src.as_bytes()).unwrap_err().to_string().contains("KiB")); } #[test] @@ -299,10 +351,15 @@ mod tests { // The same spec validation the worker runs. let spec = crate::text::TextTrainSpec { job: job.clone(), - dataset_url: "http://127.0.0.1:1/d".into(), + dataset_url: String::new(), dataset_hash: format!("sha256:{}", "a".repeat(64)), - artifact_upload_url: "http://127.0.0.1:1/u".into(), + artifact_upload_url: String::new(), events_url: None, + corpus_hash: Some(c0mpute_proto::Hash::of(b"corpus").to_hex()), + corpus_size: None, + corpus_pieces: None, + adapter_pieces: true, + run_id: Some("r1".into()), }; spec.validate().unwrap(); let o2 = LoraOptions { system_prompt: Some("flag".into()), ..o.clone() }; diff --git a/node/crates/c0mpute-core/tests/fixtures/fake-infernet-text b/node/crates/c0mpute-core/tests/fixtures/fake-infernet-text new file mode 100755 index 00000000..d83d4ea9 --- /dev/null +++ b/node/crates/c0mpute-core/tests/fixtures/fake-infernet-text @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Stand-in for the `infernet` CLI's peft text commands in c0mpute tests. + +Same command lines and JSON shapes as Infernet's peft backend +(`model runtime status --peft --json`, `model train --json --job <job.json>`), +no PyTorch. The "adapter" is deterministic bytes derived from the corpus the +worker handed over (a few MiB, so it travels as several pieces), so the +buyer's hash checks are real. Like the real runner it reads only local paths +and refuses a corpus whose sha256 differs from the job's. +""" +import hashlib, json, os, sys, time + +args = sys.argv[1:] + +if args[:3] == ["model", "runtime", "status"]: + print(json.dumps({"installed": True, "cuda": False, "torch": "fake", "transformers": "fake", "peft": "fake", + "formats": ["text", "messages"]})) + sys.exit(0) + +def flag(name): + return args[args.index(name) + 1] if name in args else None + +if args[:2] == ["model", "generate"]: + if "--input" not in args: + # What Infernet >= 0.1.80 prints for a bare call (c0mpute probes it). + print("usage: infernet model generate --adapter <dir> | --input <request.json>", file=sys.stderr) + sys.exit(2) + req = json.load(open(flag("--input"))) + weights = open(os.path.join(req["adapter_dir"], "adapter_model.safetensors"), "rb").read() + turns = " | ".join(f"{m['role']}={m['content']}" for m in req["messages"]) + print(json.dumps({"v": 1, "type": "answer", "adapter": "sha256:" + hashlib.sha256(weights).hexdigest(), + "text": f"[{req['base_model']}] {turns}", "tokens": 7, "seconds": 0.1})) + sys.exit(0) + +def emit(**ev): + print(json.dumps({"v": 1, **ev}), flush=True) + +if args[:2] == ["model", "train"]: + job_path = flag("--job") + base = os.path.dirname(os.path.abspath(job_path)) + job = json.load(open(job_path)) + url = job["dataset"]["url"] + if "://" in url: + emit(type="error", message="dataset.url must be a local path"); emit(type="done", reason="error"); sys.exit(1) + raw = open(os.path.join(base, url), "rb").read() + digest = hashlib.sha256(raw).hexdigest() + if digest != job["dataset"].get("sha256", digest): + emit(type="error", message="dataset sha256 mismatch"); emit(type="done", reason="error"); sys.exit(1) + docs = [json.loads(l) for l in raw.decode().splitlines() if l.strip()] + turns = sum(len(d.get("messages", [])) for d in docs) + emit(type="meta", base_model=job["base_model"], device="cpu", total_steps=3, documents=len(docs), conversation_turns=turns) + for s in (1, 2, 3): + emit(type="step", step=s, total_steps=3, loss=round(2.0 / s, 3), throughput=10.0) + time.sleep(0.3) + out = os.path.join(base, job["output"]["destination_url"]) + os.makedirs(out, exist_ok=True) + seed = hashlib.sha256(raw).digest() + weights = (seed * (2_621_440 // len(seed) + 1))[:2_621_440] + open(os.path.join(out, "adapter_model.safetensors"), "wb").write(weights) + open(os.path.join(out, "adapter_config.json"), "w").write(json.dumps({"r": job["hyperparams"]["lora_r"]})) + open(os.path.join(out, "manifest.json"), "w").write(json.dumps({ + "base_model": job["base_model"], "dataset_sha256": digest, "documents": len(docs), + "conversation_turns": turns, "system_prompt": job.get("system_prompt")})) + emit(type="checkpoint", step=3, model="sha256:" + hashlib.sha256(weights).hexdigest(), size_bytes=len(weights)) + emit(type="done", reason="complete") + sys.exit(0) + +print("fake-infernet-text: unsupported " + " ".join(args), file=sys.stderr) +sys.exit(2) diff --git a/node/crates/c0mpute-core/tests/inference_e2e.rs b/node/crates/c0mpute-core/tests/inference_e2e.rs new file mode 100644 index 00000000..54e91ff7 --- /dev/null +++ b/node/crates/c0mpute-core/tests/inference_e2e.rs @@ -0,0 +1,146 @@ +//! End-to-end: `infernet.inference` peer to peer on loopback libp2p. +//! +//! A worker runs the real inference runner and bid filter in front of an +//! OpenAI-compatible runtime; a buyer auctions chat turns through the jobs +//! bridge (what `c0mpute chat --p2p` does) and gets answers fetched from the +//! worker and checked against its receipt. No c0mpute.com, no `infernet` +//! binary. The runtime is a stand-in on loopback, unless +//! `C0MPUTE_INFERENCE_LIVE_URL` names a real one (e.g. Ollama at +//! http://127.0.0.1:11434), in which case that one answers. + +use std::sync::Arc; +use std::time::Duration; + +use axum::routing::{get, post}; +use axum::{Json, Router}; +use c0mpute_core::inference; +use c0mpute_core::jobs_gateway::JobsBridge; +use c0mpute_core::pricing::PricingConfig; +use c0mpute_core::dispatch; +use c0mpute_net::{ChunkSource, Libp2pNetwork, NetworkConfig}; +use c0mpute_proto::Hash; +use c0mpute_store::ChunkStore; +use libp2p::multiaddr::Protocol; +use serde_json::{Value, json}; +use tempfile::tempdir; + +struct StoreSource(ChunkStore); + +#[async_trait::async_trait] +impl ChunkSource for StoreSource { + async fn read_chunk(&self, hash: &Hash) -> anyhow::Result<Vec<u8>> { + self.0.get(hash).await + } +} + +async fn node(dir: &std::path::Path, bootstrap: Vec<libp2p::Multiaddr>) -> (Arc<Libp2pNetwork>, ChunkStore) { + let store = ChunkStore::open(dir.join("store")).await.unwrap(); + let net = Libp2pNetwork::spawn( + NetworkConfig::for_dir(dir.join("id")) + .with_listen(vec!["/ip4/127.0.0.1/tcp/0".parse().unwrap()]) + .with_local_source(Arc::new(StoreSource(store.clone()))) + .with_bootstrap(bootstrap), + ) + .await + .unwrap(); + (Arc::new(net), store) +} + +async fn wait_for_listen(net: &Libp2pNetwork) -> libp2p::Multiaddr { + for _ in 0..50 { + if let Ok(addrs) = net.listen_addrs().await + && let Some(a) = addrs.into_iter().next() + { + return a; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("no listen addr after 2.5s"); +} + +/// An OpenAI-compatible runtime that echoes the whole conversation it got. +async fn fake_runtime() -> String { + let app = Router::new() + .route("/v1/models", get(|| async { Json(json!({"object": "list", "data": [{"id": "tiny:1b"}]})) })) + .route( + "/v1/chat/completions", + post(|Json(b): Json<Value>| async move { + let turns: Vec<String> = b["messages"].as_array().unwrap().iter().map(|m| format!("{}={}", m["role"].as_str().unwrap(), m["content"].as_str().unwrap())).collect(); + Json(json!({"choices": [{"message": {"role": "assistant", "content": format!("[{}] {}", b["model"].as_str().unwrap(), turns.join(" | "))}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 5}})) + }), + ); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + format!("http://{addr}") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn chat_turns_are_answered_peer_to_peer() { + let _ = tracing_subscriber::fmt().with_env_filter("c0mpute_core=info,c0mpute_net=warn").with_test_writer().try_init(); + let live = std::env::var("C0MPUTE_INFERENCE_LIVE_URL").ok().filter(|s| !s.is_empty()); + let base = match &live { + Some(u) => u.trim_end_matches('/').to_string(), + None => fake_runtime().await, + }; + let tmp = tempdir().unwrap(); + + // ── worker: refresh loop, runner, bid filter ─────────────────────── + tokio::spawn(inference::refresh_loop(base.clone())); + let mut models = Vec::new(); + for _ in 0..50 { + models = inference::ready_models(); + if !models.is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert!(!models.is_empty(), "the runtime at {base} lists no chat models"); + let model = models[0].clone(); + let (worker, worker_store) = node(&tmp.path().join("w"), vec![]).await; + let worker_addr = wait_for_listen(&worker).await.with(Protocol::P2p(worker.peer_id())); + let load = inference::worker_load(); + let runner = inference::spawn_inference_runner(worker.clone(), worker_store, base.clone(), load.clone()); + let mut tags = vec![inference::ROLE_TAG.to_string(), "c0mpute:cpu".to_string()]; + tags.extend(models.iter().map(|m| inference::model_tag(m))); + dispatch::run_worker_subscriber_with_load(worker.clone(), inference::WORKLOAD.to_string(), tags, Some(runner), Some(inference::offer_filter()), Some(PricingConfig::default()), Some(load)); + + // ── buyer ────────────────────────────────────────────────────────── + let (buyer, buyer_store) = node(&tmp.path().join("b"), vec![worker_addr]).await; + let bridge = JobsBridge::new(buyer.clone(), buyer_store, "test-token-0123456789abcdef".into(), 0.0, vec![]).unwrap().with_bid_window(Duration::from_secs(2)); + tokio::time::sleep(Duration::from_secs(2)).await; + + // A multi-turn conversation, any model. + let spec = json!({"v": 1, "max_tokens": 32, "temperature": 0.0, "messages": [ + {"role": "system", "content": "Answer in one word."}, + {"role": "user", "content": "Name a primary color."}, + {"role": "assistant", "content": "Red."}, + {"role": "user", "content": "Another one?"}, + ]}); + let out = bridge.submit_once(inference::WORKLOAD, spec, Duration::from_secs(240)).await.expect("answered"); + assert_eq!(out["worker"], worker.peer_id().to_base58()); + let r = &out["result"]; + assert!(inference::valid_result(r), "{r}"); + assert_eq!(r["ok"], true, "{r}"); + assert_eq!(r["model"], model.as_str()); + let text = r["text"].as_str().unwrap(); + assert!(!text.trim().is_empty()); + if live.is_none() { + // The worker forwarded every turn, in order, to the runtime. + assert_eq!(text, "[tiny:1b] system=Answer in one word. | user=Name a primary color. | assistant=Red. | user=Another one?"); + } else { + eprintln!("live answer from {model}: {text}"); + } + + // Asking for the model by name works; a model nobody serves gets no bid. + let named = json!({"v": 1, "model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 8}); + let out = bridge.submit_once(inference::WORKLOAD, named, Duration::from_secs(240)).await.expect("named model"); + assert_eq!(out["result"]["model"], model.as_str()); + let missing = json!({"v": 1, "model": "not-served:9b", "messages": [{"role": "user", "content": "hi"}]}); + let err = bridge.submit_once(inference::WORKLOAD, missing, Duration::from_secs(8)).await.unwrap_err().to_string(); + assert!(err.contains("no_bids") || err.contains("timeout"), "{err}"); + // An invalid spec never leaves the buyer. + let bad = json!({"v": 1, "messages": [{"role": "tool", "content": "x"}]}); + assert!(bridge.submit_once(inference::WORKLOAD, bad, Duration::from_secs(5)).await.is_err()); +} diff --git a/node/crates/c0mpute-core/tests/text_train_e2e.rs b/node/crates/c0mpute-core/tests/text_train_e2e.rs new file mode 100644 index 00000000..0ad8df1e --- /dev/null +++ b/node/crates/c0mpute-core/tests/text_train_e2e.rs @@ -0,0 +1,185 @@ +//! End-to-end: `infernet.text.train` peer to peer on loopback libp2p. +//! +//! A worker (`text::serve`) and a buyer (`text::train`) with no HTTP between +//! them: the buyer stores a multi-MiB corpus (with multi-turn conversations) +//! as pieces, the worker pulls them over libp2p, publishes training events on +//! the job topic, stores the adapter as pieces behind a manifest its receipt +//! names, and the buyer fetches and verifies it. A fake `infernet` +//! (tests/fixtures/fake-infernet-text, no PyTorch) stands in for the peft +//! runner; it writes a deterministic adapter derived from the corpus it was +//! handed, so every hash check is real. Skips where python3 is missing. + +use std::path::PathBuf; +use std::time::Duration; + +use c0mpute_core::temporal::node::{NetOptions, spawn_node_with_store}; +use c0mpute_core::text::{self, TrainRequest}; +use c0mpute_core::train::{LoraOptions, normalize_dataset}; +use c0mpute_store::ChunkStore; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use tempfile::tempdir; + +fn fake_infernet() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/fake-infernet-text") +} + +fn python3() -> bool { + std::process::Command::new("python3").arg("--version").output().is_ok_and(|o| o.status.success()) +} + +fn net(dir: &std::path::Path, bootstrap: Vec<String>) -> NetOptions { + NetOptions { identity_dir: dir.to_path_buf(), listen: vec!["/ip4/127.0.0.1/tcp/0".into()], bootstrap, default_bootstrap: false } +} + +/// About 3 MiB of records: text, prompt/completion and multi-turn chats. +fn dataset() -> Vec<u8> { + let mut out = String::new(); + let mut i = 0u64; + while out.len() < 3 * 1024 * 1024 { + let filler: String = (0..40).map(|k| format!("w{}", (i * 31 + k) % 997)).collect::<Vec<_>>().join(" "); + let rec = match i % 3 { + 0 => json!({"title": format!("Topic {i}"), "text": filler}), + 1 => json!({"prompt": format!("Q{i}?"), "completion": filler}), + _ => json!({"messages": [ + {"role": "system", "content": "You are a pricing coach."}, + {"role": "user", "content": format!("I sell product {i}.")}, + {"role": "assistant", "content": "What does it replace?"}, + {"role": "user", "content": "Spreadsheets."}, + {"role": "assistant", "content": filler}, + ]}), + }; + out.push_str(&rec.to_string()); + out.push('\n'); + i += 1; + } + out.into_bytes() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn trains_peer_to_peer_with_pieces_and_conversations() { + if !python3() { + eprintln!("skipping: python3 is not installed (the fake infernet is a python script)"); + return; + } + let _ = tracing_subscriber::fmt().with_env_filter("c0mpute_core=info,c0mpute_net=warn").with_test_writer().try_init(); + let tmp = tempdir().unwrap(); + + // ── worker ───────────────────────────────────────────────────────── + let w = text::serve(&net(&tmp.path().join("w/id"), vec![]), fake_infernet(), tmp.path().join("w/cache")).await.expect("worker serves"); + assert!(w.tags.contains(&"infernet:text:train".to_string())); + assert!(w.tags.contains(&text::MESSAGES_FORMAT_TAG.to_string()), "the runner reports the messages format: {:?}", w.tags); + assert!(w.infer, "the fake answers chats (model generate)"); + assert!(w.tags.contains(&"infernet:text:infer".to_string())); + let wid = w.net.peer_id().to_base58(); + let waddr = format!("{}/p2p/{wid}", w.net.listen_addrs().await.unwrap()[0]); + + // ── buyer: what `c0mpute train lora` does ────────────────────────── + let raw = dataset(); + let (corpus, report) = normalize_dataset(&raw).unwrap(); + assert!(report.messages > 0 && report.text > 0 && report.prompt_completion > 0); + assert!(corpus.len() > 2 * 1024 * 1024, "several pieces"); + let opts = LoraOptions { base_model: "HuggingFaceTB/SmolLM2-135M-Instruct".into(), name: "coach".into(), ..Default::default() }; + let job = opts.job(&report); + let stage = tmp.path().join("stage"); + std::fs::create_dir_all(&stage).unwrap(); + std::fs::write(stage.join("corpus.jsonl"), &corpus).unwrap(); + std::fs::write(stage.join("job.json"), serde_json::to_vec(&job).unwrap()).unwrap(); + + let store = ChunkStore::open(tmp.path().join("b/store")).await.unwrap(); + let buyer = spawn_node_with_store(&net(&tmp.path().join("b/id"), vec![waddr]), store.clone()).await.unwrap(); + tokio::time::sleep(Duration::from_secs(2)).await; + + let req = TrainRequest { + job_path: stage.join("job.json"), + dataset_path: Some(stage.join("corpus.jsonl")), + output: tmp.path().join("adapter"), + workers: vec![], + max_price_usd: 1.0, + gpu: false, + min_ram_gb: None, + legacy_http: false, + public_url: None, + bind: "127.0.0.1:0".parse().unwrap(), + timeout_s: 120, + retries: 1, + bid_window: Duration::from_secs(2), + }; + let mut events: Vec<Value> = Vec::new(); + let out = text::train(buyer.clone(), &store, &req, |ev| events.push(ev.clone())).await.expect("trains"); + + // Peer to peer both ways. + assert_eq!(out["transport"], "pieces", "{out}"); + assert_eq!(out["receipt"]["worker_peer_id"], wid); + let manifest_hash = out["receipt"]["output_hash"].as_str().unwrap(); + assert!(!manifest_hash.starts_with("sha256:"), "the receipt names the adapter manifest"); + + // The adapter is exactly what the trainer derived from the corpus the + // buyer sent: the corpus arrived intact. + let seed = Sha256::digest(&corpus); + let want: Vec<u8> = seed.iter().copied().cycle().take(2_621_440).collect(); + let got = std::fs::read(tmp.path().join("adapter/adapter_model.safetensors")).unwrap(); + assert_eq!(got.len(), want.len()); + assert!(got == want, "adapter bytes differ"); + assert_eq!(out["adapter_hash"], format!("sha256:{}", hex::encode(Sha256::digest(&want)))); + let m: Value = serde_json::from_slice(&std::fs::read(tmp.path().join("adapter/manifest.json")).unwrap()).unwrap(); + assert_eq!(m["dataset_sha256"], hex::encode(seed)); + assert_eq!(m["system_prompt"], "You are a pricing coach."); + // Every conversation reached the trainer whole: five turns per chat, + // two per prompt/completion pair. + assert_eq!(m["conversation_turns"].as_u64().unwrap() as usize, report.messages * 5 + report.prompt_completion * 2); + assert!(std::fs::read(tmp.path().join("adapter/adapter_config.json")).is_ok()); + + // Progress came over gossip. + let kinds: Vec<&str> = events.iter().filter_map(|e| e["type"].as_str()).collect(); + assert!(kinds.contains(&"meta"), "{kinds:?}"); + assert!(kinds.contains(&"checkpoint"), "{kinds:?}"); + assert_eq!(kinds.last(), Some(&"done"), "{kinds:?}"); + + // The worker cleaned its job dir (corpus and adapter now live as pieces). + let left = std::fs::read_dir(tmp.path().join("w/cache/text/jobs")).map(|d| d.count()).unwrap_or(0); + assert_eq!(left, 0, "worker job dir removed"); + + // A tampered manifest hash is refused (nothing is written). + let bogus = c0mpute_proto::Hash::of(b"not a manifest").to_hex(); + let err = text::fetch_adapter_pieces(&buyer, Some(w.net.peer_id()), &bogus, &tmp.path().join("bad")).await; + assert!(err.is_err()); + assert!(!tmp.path().join("bad").exists()); + + // ── then chat with the adapter, also peer to peer ────────────────── + // (`c0mpute infernet text chat`: adapter out as pieces, answer back as a chunk) + let chat = text::ChatRequest { + adapter_dir: tmp.path().join("adapter"), + question: "How should I price it?".into(), + search: None, + max_new_tokens: 64, + workers: vec![], + max_price_usd: 1.0, + legacy_http: false, + public_url: None, + bind: "127.0.0.1:0".parse().unwrap(), + timeout_s: 60, + bid_window: Duration::from_secs(2), + }; + let ans = text::chat(buyer.clone(), &store, &chat).await.expect("chat answered"); + assert_eq!(ans["text"], "[HuggingFaceTB/SmolLM2-135M-Instruct] system=You are a pricing coach. | user=How should I price it?", "{ans}"); + assert_eq!(ans["adapter_hash"], out["adapter_hash"]); + assert!(!ans["receipt"]["output_hash"].as_str().unwrap().starts_with("sha256:"), "answer came back as a chunk"); + // The worker cached the adapter it fetched as pieces. + let cached = tmp.path().join("w/cache/text/adapters").join(out["adapter_hash"].as_str().unwrap().trim_start_matches("sha256:")); + assert_eq!(std::fs::read(cached.join("adapter_model.safetensors")).unwrap(), want); +} + +/// A corpus with conversations is never placed on a worker whose runner +/// cannot train them (it would silently drop them). +#[test] +fn conversations_need_a_runner_that_reports_messages() { + let chat = br#"{"messages":[{"role":"user","content":"a"},{"role":"assistant","content":"b"}]}"#; + let mut corpus = chat.to_vec(); + corpus.push(b'\n'); + let tags = text::required_tags(&corpus); + let old_runner = text::worker_tags(&json!({"installed": true, "cuda": false}), 8 * 1024); + assert!(!tags.iter().all(|t| old_runner.contains(t)), "an old runner lacks {}", text::MESSAGES_FORMAT_TAG); + let new_runner = text::worker_tags(&json!({"installed": true, "cuda": false, "formats": ["text", "messages"]}), 8 * 1024); + assert!(tags.iter().all(|t| new_runner.contains(t))); +} diff --git a/node/crates/c0mpute-core/tests/workflow_e2e.rs b/node/crates/c0mpute-core/tests/workflow_e2e.rs index 2d4bd188..89989baf 100644 --- a/node/crates/c0mpute-core/tests/workflow_e2e.rs +++ b/node/crates/c0mpute-core/tests/workflow_e2e.rs @@ -113,7 +113,8 @@ async fn transform_step_round_trips_through_the_bridge() { "crawl.fetch", "ocr.extract", "embed.text", - "blender.render-frame" + "blender.render-frame", + "infernet.inference" ]) ); diff --git a/node/crates/c0mpute-proto/src/lib.rs b/node/crates/c0mpute-proto/src/lib.rs index ed38a850..4d153e11 100644 --- a/node/crates/c0mpute-proto/src/lib.rs +++ b/node/crates/c0mpute-proto/src/lib.rs @@ -219,6 +219,10 @@ pub enum Role { /// Blender (Cycles on CPU, or on a GPU Blender finds). On by default; /// advertised only where `blender --version` works. Render, + /// `infernet.inference`: chat completions from the box's own + /// OpenAI-compatible runtime (Ollama by default, or llama-server). On by + /// default; advertised only while the runtime lists chat models. + Inference, } #[cfg(test)] diff --git a/plugins/infernet/module.toml b/plugins/infernet/module.toml index 2e2ed7d9..d585bf14 100644 --- a/plugins/infernet/module.toml +++ b/plugins/infernet/module.toml @@ -1,7 +1,19 @@ # c0mpute plugin manifest. See dips/0006-module-model.md. # -# Source code lives upstream at github.com/infernetprotocol/infernet-protocol, NOT -# this repo. This directory is a marketplace wrapper only. +# Two halves: +# +# - `infernet.inference` (chat) is NATIVE in the c0mpute binary since this +# release: buyers run `c0mpute chat --p2p "..."` (the default when not signed +# in to c0mpute.com) or POST it to `c0mpute jobs serve`; workers answer from +# their own OpenAI-compatible runtime (Ollama at 127.0.0.1:11434 by default, +# or llama-server etc. via C0MPUTE_INFERENCE_URL; `off` disables), role +# `inference`, on by default, advertised only while the runtime lists chat +# models (tags infernet:inference + infernet:inference:model:<id>). Answers +# come back as content-addressed chunks checked against the receipt. Neither +# side needs the infernet binary or c0mpute.com (node/crates/c0mpute-core/src/inference.rs). +# - The temporal and text (LoRA) workloads below still run Infernet's own +# runtimes on the worker, so those workers need the infernet CLI (installed by +# install.sh). Source: github.com/infernetprotocol/infernet-protocol. # # `version` is this wrapper's, kept in lockstep with c0mpute by # scripts/bump-version.sh. The installed infernet CLI's own version is what @@ -13,9 +25,8 @@ name = "Infernet Protocol — AI inference" version = "0.2.79" kind = "workload" category = "ai" -status = "partial" -status_note = "Chat runs through the separate infernet CLI, not the c0mpute binary." -description = "Decentralized AI inference workloads on c0mpute. Runs models on consumer/prosumer GPUs with model-hash and runtime-image verification. Also trains and serves temporal policies (CfC/LTC liquid networks, GRU baseline) and fine-tunes language models (LoRA) with live progress, on CPU or GPU." +status = "available" +description = "Chat with open models peer to peer from the c0mpute binary itself (`c0mpute chat --p2p`): workers answer from their own Ollama or llama.cpp server, no account or extra CLI. Also trains and serves temporal policies (CfC/LTC liquid networks, GRU baseline) and fine-tunes language models (LoRA) with live progress, on CPU or GPU." author = "Infernet Protocol" license = "MIT" homepage = "https://infernetprotocol.com" @@ -23,13 +34,15 @@ source = "https://github.com/infernetprotocol/infernet-protocol" keywords = ["ai", "llm", "inference", "gpu", "temporal-policy", "liquid-neural-network", "cfc", "lora", "fine-tuning"] [module.requirements] -c0mpute = ">=0.1.0" +c0mpute = ">=NEXT" os = ["linux", "darwin"] arch = ["x86_64", "aarch64"] capabilities = ["gpu:nvidia?", "model-runtime"] [module.workloads] -"infernet.inference" = { command = "chat", validation = "schema_and_spotcheck" } +# Native (no infernet binary): `c0mpute chat --p2p`; workers serve their local +# OpenAI-compatible runtime. Receipts carry the answer chunk's content hash. +"infernet.inference" = { command = "c0mpute chat --p2p", runs = "builtin", validation = "answer_chunk_hash" } # Temporal policies (liquid-agent/v1). Executed by Infernet >= 0.1.66 # (`infernet model train --job` / `infernet model serve --stdio`, which run # the infernet-liquid runtime the operator installs with @@ -67,6 +80,11 @@ runtime = "infernet:text:runtime:peft" devices = ["infernet:text:device:cpu", "infernet:text:device:cuda"] ram = "c0mpute:ram:<n>gb+" +[module.capabilities.inference] +# Tags a worker serving chat advertises; buyers naming a model require its tag. +role = "infernet:inference" +model = "infernet:inference:model:<id>" + [module.surfaces] -cli = "infernet" +cli = "c0mpute chat --p2p" # web = "c0mpute.com/infernet" # planned diff --git a/plugins/train/module.toml b/plugins/train/module.toml index 647ca9ce..57536aa4 100644 --- a/plugins/train/module.toml +++ b/plugins/train/module.toml @@ -8,20 +8,32 @@ # 1. checks the dataset: UTF-8 JSONL, one object per line as {text, title?}, # {prompt, completion} or {messages: [{role, content}]}; at least 2 # records, at most 1,000,000 records, 1 MiB per record, 512 MiB in all; -# 2. converts it to the {title, text} corpus the worker's peft runner reads -# (prompt -> title, completion -> text; for messages the last user and -# assistant turns, and a shared system turn becomes the system prompt); +# 2. converts it to the corpus the worker's peft runner reads: {title, text} +# stays a "Brief me on: <title>" exchange; prompt/completion and messages +# become whole conversations ({messages: [...]}, every turn in order) that +# the runner renders with the base model's chat template and trains with +# the loss on the assistant turns only; # 3. writes the @infernetprotocol/training job (kind "lora", base_model, # hyperparams lr / epochs / lora_r / seq_len / batch_size / max_steps); -# 4. auctions it through text::train, relays live progress, downloads the -# adapter and keeps it only if its sha256 matches the receipt; -# 5. prints how to use it: `c0mpute infernet text chat --adapter <out>`. +# 4. auctions it through text::train PEER TO PEER: the corpus goes out as +# content-addressed 1 MiB pieces the worker pulls over libp2p, progress +# comes back as gossip on the job topic, and the adapter comes back as +# pieces behind a manifest whose hash the receipt carries; the buyer keeps +# the adapter only if its sha256 matches. Nothing needs a reachable HTTP +# address (`--legacy-http` additionally serves workers from before the pieces transport over HTTP); +# 5. prints how to use it: `c0mpute infernet text chat --adapter <out>`, which +# is peer to peer the same way (adapter out as pieces, answer back as a chunk). # -# Worker side (unchanged): `c0mpute infernet text serve`, which needs Infernet -# >= 0.1.79 with the peft runtime (`infernet model runtime setup --peft --yes`: -# torch + transformers + peft in Infernet's venv). Training is fp32 LoRA on CPU -# or a CUDA GPU when torch sees one (buyers can require it with --gpu); no GPU -# is required, so keep base models small (0.5B-class trains on CPU in hours). +# Worker side: `c0mpute infernet text serve`. c0mpute moves the data and hands +# Infernet's trainer local file paths; the training itself needs Infernet with +# the peft runtime (`infernet model runtime setup --peft --yes`: torch + +# transformers + peft in Infernet's venv). A corpus with conversations goes +# only to workers whose runner reports the `messages` dataset format (tag +# infernet:text:format:messages, Infernet with infernetprotocol/infernet-protocol#161 +# "peft: train multi-turn conversations"); older runners would drop them. Training is +# fp32 LoRA on CPU or a CUDA GPU when torch sees one (buyers can require it with +# --gpu); no GPU is required, so keep base models small (0.5B-class trains on +# CPU in hours). # # No QLoRA: the peft runner loads the base model in fp32 with no 4-bit # quantization (no bitsandbytes), so `train.qlora` is not offered. @@ -32,22 +44,21 @@ name = "Fine-tune — LoRA training" version = "0.2.79" kind = "workload" category = "ai" -status = "partial" -status_note = "LoRA works end to end through infernet.text.train (`c0mpute train lora`). The buyer serves the corpus and receives the adapter over HTTP from its own machine, so the worker must reach it: same LAN, or pass --public-url to a forwarded https address. Each record trains as one 'Brief me on: <title>' exchange, so multi-turn chats keep only their last turn. No QLoRA (the peft runner trains fp32 LoRA)." +status = "available" builtin = true -description = "Fine-tune a LoRA adapter on a c0mpute worker from a JSONL dataset (text, prompt/completion or chat messages): live progress, adapter verified by hash, then chat with it." +description = "Fine-tune a LoRA adapter on a c0mpute worker from a JSONL dataset (text, prompt/completion or multi-turn chats, trained on every turn): dataset and adapter move peer to peer as verified pieces, live progress, then chat with it." author = "Profullstack" license = "MIT" keywords = ["train", "finetune", "lora", "peft", "adapter", "infernet"] [module.requirements] -c0mpute = ">=0.2.77" +c0mpute = ">=NEXT" os = ["linux", "darwin"] arch = ["x86_64", "aarch64"] -capabilities = ["bin:infernet", "infernet:text:runtime:peft", "gpu:cuda?"] +capabilities = ["bin:infernet", "infernet:text:runtime:peft", "infernet:text:format:messages?", "gpu:cuda?"] [module.workloads] -"train.lora" = { command = "lora", runs = "infernet.text.train", validation = "adapter_sha256_matches_receipt" } +"train.lora" = { command = "lora", runs = "infernet.text.train", validation = "adapter_sha256_matches_receipt_manifest" } [module.dispatch] mode = "subprocess" From ab307b86afb3f72f3033d0e1cd7c61c8fd513ced Mon Sep 17 00:00:00 2001 From: Anthony Ettinger <anthony@profullstack.com> Date: Mon, 9 Nov 2026 04:42:59 +0000 Subject: [PATCH 3/7] ads: pay through the shared payments module (one payment path) Ads billing no longer shells out to `coinpay wallet send`. Settlement lines become payments::Transfer values keyed `ads:<settlement hash>:<line>` and go through payments::pay_all on the node's one Rail (CoinPay payouts API via $C0MPUTE_COINPAY_PAYOUT_KEY, FakeRail in tests), with the shared idempotent payments ledger (`payments.json` per campaign) and the `[payments] max_usd_per_round` cap. A lost ads-ledger reference is healed from the shared ledger instead of paying twice. - Removed ads' own PaymentRail, DisabledRail, FakeRail and CoinpayCliRail. - `c0mpute ads settle --pay` needs both `[ads] payouts = true` and `[payments] enabled = true`; still a dry run otherwise. - Payout addresses are `<COIN>:<address>` (CoinPay currency code), mapped to payments::Payee by billing::payee_of. - Fee and split logic stay in ads. Docs and manifest notes updated. - Tests: billing unit tests (cap, idempotency, heal), billing e2e on payments::FakeRail still checks payouts equal budget spent. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- docs/ads.md | 42 ++- node/crates/c0mpute-cli/src/main.rs | 35 ++- node/crates/c0mpute-core/src/ads/billing.rs | 291 +++++++----------- node/crates/c0mpute-core/src/ads/manage.rs | 20 +- node/crates/c0mpute-core/src/ads/mod.rs | 4 +- node/crates/c0mpute-core/src/ads/stitch.rs | 4 +- node/crates/c0mpute-core/src/config.rs | 9 +- node/crates/c0mpute-core/src/podcast/show.rs | 4 +- .../c0mpute-core/tests/ads_billing_e2e.rs | 35 ++- plugins/ads-manager/module.toml | 4 +- plugins/ctv-ads/module.toml | 2 +- plugins/podcast-ads/module.toml | 4 +- 12 files changed, 210 insertions(+), 244 deletions(-) diff --git a/docs/ads.md b/docs/ads.md index 6776ec51..bccd81aa 100644 --- a/docs/ads.md +++ b/docs/ads.md @@ -11,7 +11,7 @@ Code: `node/crates/c0mpute-core/src/ads/`. Billing is built (see [Billing and payouts](#billing-and-payouts)): a campaign with a budget pays gateways and shows for verified impressions at -the second price, from the advertiser's own CoinPay wallet. Real payouts are +the second price, through the node's one payment rail. Real payouts are **off by default**. There is no OpenRTB auction. ## Advertisers @@ -89,13 +89,17 @@ the ad. Settlement is built on these chains (next section). ## Billing and payouts -The rail is CoinPay, the settlement adapter c0mpute already names everywhere, -through the `coinpay` CLI the `coinpay` plugin delegates to. Nothing in -c0mpute holds anyone's money; there is no clearing house and no ad server. -Code: `ads/billing.rs`. +Money moves through c0mpute's one payment path, `payments.rs` (shared with +hosting settlement): CoinPay payouts from the operator's own CoinPay business +account, with an idempotent ledger and a per-round cap. Nothing in c0mpute +holds anyone's money; there is no clearing house and no ad server. The fee +and split logic is ads-specific: `ads/billing.rs`. + +Payout addresses are `<COIN>:<address>`, where `COIN` is the CoinPay currency +code the payout is made in (`USDC_POL`, `SOL`, ...). 1. **Budget.** `--budget` (total, USD), `--daily-budget`, `--payer - <CHAIN>:<address>` (your paying wallet) and `--funding-ref` (a CoinPay + <COIN>:<address>` (your paying wallet) and `--funding-ref` (a CoinPay escrow or payment id as proof of funds) are signed into the campaign record. A campaign without a budget runs unpaid, as before. 2. **Cleared price.** Every impression receipt carries the price it cleared @@ -123,8 +127,11 @@ Code: `ads/billing.rs`. the running total spent), pays each line, and gossips the settlements so gateways learn the budget left and where their next claim starts. A gateway stops serving a campaign once settled spend plus its own - unsettled spend reaches the budget. Lines are idempotent: a paid line is - never sent again, a failed one is retried by the next `settle --pay`. + unsettled spend reaches the budget. Lines are idempotent: each is a + transfer keyed `ads:<settlement hash>:<line>` in the shared payments + ledger (`payments.json` next to the campaign), so a paid line is never + sent again; a failed one, or one over `[payments] max_usd_per_round`, is + retried by the next `settle --pay`. Split of each paid impression: **5% network fee** (the single fee c0mpute's market and Infernet take; operators keep 95%), **75% to the show** that @@ -139,18 +146,25 @@ min_claim_usd = "1.00" # Advertiser [ads] -payouts = true # allow `c0mpute ads settle --pay` to send money -fee_payout = "SOL:..." # network fee address; without it the fee is owed, not sent +payouts = true # allow `c0mpute ads settle --pay` to send money... +fee_payout = "USDC_POL:0x..." # network fee address; without it the fee is owed, not sent + +[payments] # ...through the node's payment rail, also off by default +enabled = true +rail = "coinpay" +max_usd_per_round = "10" +coinpay.recipient_email = "you@example.org" +# plus a CoinPay business API key with payouts:create in $C0MPUTE_COINPAY_PAYOUT_KEY ``` Shows add `payout = "SOL:..."` to `[ads]` in `show.toml` (or `c0mpute podcasting publish --ads pre --ads-payout SOL:...`). It lives in the signed `c0mpute-ads.json`, so no gateway can redirect a show's money. -What is not proven: `--pay` runs `coinpay wallet send --chain <CHAIN> --to -<address> --amount-fiat <usd> --fiat USD` from your wallet. Every test uses -an in-memory rail; real transfers have not been exercised, and they stay off -until `[ads] payouts = true`. A receipt is still a gateway's claim, not +What is not proven: `--pay` calls CoinPay `POST /api/payouts/create` once +per line. Every test uses the in-memory `payments::FakeRail`; real transfers +have not been exercised, and they stay off until both `[ads] payouts = true` +and `[payments] enabled = true`. A receipt is still a gateway's claim, not proof a person heard the ad: caps, dedupe and the self-dealing rules limit what a dishonest gateway can bill, they do not eliminate it. diff --git a/node/crates/c0mpute-cli/src/main.rs b/node/crates/c0mpute-cli/src/main.rs index 5bcae116..c7737d33 100644 --- a/node/crates/c0mpute-cli/src/main.rs +++ b/node/crates/c0mpute-cli/src/main.rs @@ -730,10 +730,10 @@ enum PodcastingCmd { /// Without it (or [ads]) no gateway ever inserts ads. #[arg(long, value_name = "BREAKS")] ads: Option<String>, - /// Where the show's share of ad revenue goes, <CHAIN>:<address> + /// Where the show's share of ad revenue goes, <COIN>:<address> /// (e.g. SOL:9xQe...). Same as `payout` in [ads]. Without it the show /// runs ads unpaid. - #[arg(long, value_name = "CHAIN:ADDRESS")] + #[arg(long, value_name = "COIN:ADDRESS")] ads_payout: Option<String>, /// Build the show folder only; do not publish. #[arg(long)] @@ -765,12 +765,14 @@ enum AdsCmd { json: bool, }, /// Verify gateways' payment claims for a campaign and (with --pay) settle - /// and pay them from your CoinPay wallet. Without --pay it only reports. + /// and pay them through the node's payment rail ([payments]). Without + /// --pay it only reports. Settle { /// Campaign name (local) or id. campaign: String, /// Sign settlements and send payouts. Also needs `[ads] payouts = - /// true` in config.toml: real money leaves your wallet. + /// true` and `[payments] enabled = true` in config.toml: real money + /// moves. #[arg(long)] pay: bool, /// How long to collect claims and receipts over gossip. @@ -818,9 +820,9 @@ struct AdsTargetArgs { /// Most the campaign pays for one UTC day of impressions, USD. #[arg(long)] daily_budget: Option<String>, - /// Your paying CoinPay wallet, <CHAIN>:<address> (claims paying out to it + /// Your paying CoinPay wallet, <COIN>:<address> (claims paying out to it /// are refused as self-dealing). - #[arg(long, value_name = "CHAIN:ADDRESS")] + #[arg(long, value_name = "COIN:ADDRESS")] payer: Option<String>, /// A CoinPay escrow or payment id that shows the budget is funded. #[arg(long)] @@ -3306,23 +3308,26 @@ async fn run_ads(cmd: AdsCmd) -> Result<()> { let cfg = Config::load_or_default(&config::default_config_path())?; let fee_payout = cfg.ads.fee_payout.clone().unwrap_or_default(); if !fee_payout.is_empty() && !billing::valid_payout(&fee_payout) { - bail!("[ads] fee_payout must be <CHAIN>:<address>"); + bail!("[ads] fee_payout must be <COIN>:<address>, e.g. \"USDC_POL:0x...\""); } - let coinpay; - let rail: Option<&dyn billing::PaymentRail> = if pay { - if !cfg.ads.payouts { + // Real money needs both switches: `[ads] payouts` (this feature) + // and `[payments] enabled` (the node's one payment rail). + let rail: Option<std::sync::Arc<dyn c0mpute_core::payments::Rail>> = if pay { + if !cfg.ads.payouts || !cfg.payments.enabled { bail!( - "--pay sends real money from your CoinPay wallet; turn it on with `[ads] payouts = true` in {}", + "--pay sends real money; it needs both `[ads] payouts = true` and `[payments] enabled = true` in {}", config::default_config_path().display() ); } - coinpay = billing::CoinpayCliRail::find() - .context("`coinpay` is not installed: c0mpute update installs it, then `c0mpute coinpay login`")?; - Some(&coinpay) + Some( + c0mpute_core::payments::rail_from_config(&cfg.payments)? + .context("[payments] is enabled but no rail was configured")?, + ) } else { None }; - let (round, ledger) = manage::settle(&record, &fee_payout, rail, wait(wait_secs)).await?; + let (round, ledger) = + manage::settle(&record, &fee_payout, rail.as_deref(), cfg.payments.round_cap(), wait(wait_secs)).await?; if json { let settled: Vec<_> = round .settled diff --git a/node/crates/c0mpute-core/src/ads/billing.rs b/node/crates/c0mpute-core/src/ads/billing.rs index 1a3ab4ce..58a2574e 100644 --- a/node/crates/c0mpute-core/src/ads/billing.rs +++ b/node/crates/c0mpute-core/src/ads/billing.rs @@ -25,7 +25,7 @@ //! caps, refuses self-dealing, and refuses a claim that would overspend. //! 5. **Settle.** The advertiser's node signs a [`Settlement`] (chained per //! campaign, carrying the running total spent), pays each line through a -//! [`PaymentRail`], and gossips the settlement so gateways learn how much +//! shared payments path ([`crate::payments`]), and gossips the settlement so gateways learn how much //! budget is left and where their next claim starts. //! //! Split of every paid impression ([`split`]): the c0mpute network fee is 5%, @@ -35,11 +35,10 @@ //! directly), the gateway is the publisher and gets 95%. //! //! Real money moves only with `[ads] payouts = true` on the advertiser's node -//! and `c0mpute ads settle --pay`; the default rail refuses to pay. +//! **and** `[payments] enabled = true` (the shared rail), and `c0mpute ads +//! settle --pay`. Otherwise settle only reports what it would pay. use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::path::PathBuf; -use std::sync::Mutex; use anyhow::{Context, Result, bail}; use c0mpute_proto::Hash; @@ -81,7 +80,7 @@ pub struct Budget { pub daily_micros: Option<u64>, /// Settlement adapter; "coinpay" is the only one with a rail today. pub rail: String, - /// The advertiser's paying wallet, `<CHAIN>:<address>`. A gateway or show + /// The advertiser's paying wallet, `<COIN>:<address>`. A gateway or show /// paid to this address is the advertiser paying itself and is refused. #[serde(default, skip_serializing_if = "Option::is_none")] pub payer: Option<String>, @@ -104,7 +103,7 @@ impl Budget { if let Some(p) = &self.payer && !valid_payout(p) { - bail!("payer must be <CHAIN>:<address>, e.g. SOL:9xQe..."); + bail!("payer must be <COIN>:<address>, e.g. SOL:9xQe..."); } if let Some(f) = &self.funding_ref && (f.is_empty() || f.len() > 128 || f.chars().any(|c| c.is_control() || c.is_whitespace())) @@ -119,9 +118,9 @@ fn valid_token(s: &str, max: usize) -> bool { !s.is_empty() && s.len() <= max && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') } -/// A payout address: `<CHAIN>:<address>`, e.g. `SOL:9xQe...`, `ETH:0xabc...`, -/// `BCH:bitcoincash:q...`. The chain is what `coinpay wallet send --chain` -/// takes. +/// A payout address: `<COIN>:<address>`, e.g. `USDC_POL:0xabc...`, +/// `SOL:9xQe...`. The coin is the CoinPay currency code the shared payments +/// rail pays in ([`payee_of`]). pub fn valid_payout(s: &str) -> bool { let Some((chain, addr)) = s.split_once(':') else { return false; @@ -153,7 +152,7 @@ pub struct Claim { pub campaign_id: String, /// Gateway key (hex), the same key that signed the batches. pub gateway: String, - /// Where the gateway's share goes, `<CHAIN>:<address>`. + /// Where the gateway's share goes, `<COIN>:<address>`. pub payout: String, pub from_seq: u64, pub to_seq: u64, @@ -269,7 +268,7 @@ pub struct Payout { pub role: Role, /// Recipient key (gateway or show owner); "" for the network fee. pub key: String, - /// `<CHAIN>:<address>`; "" when no network fee address is configured + /// `<COIN>:<address>`; "" when no network fee address is configured /// (the fee is owed, not sent). pub address: String, pub micros: u64, @@ -755,30 +754,68 @@ impl Ledger { Ok(s) } - /// Send every unpaid line that has an address. Idempotent per line: a - /// line with a reference is never sent again, and the rail gets - /// `<settlement hash>:<line>` as an idempotency key. Returns lines sent. - pub fn pay(&mut self, rail: &dyn PaymentRail, payer: Option<&str>) -> usize { - let mut sent = 0; - for st in self.lines.iter_mut().filter(|l| l.reference.is_none()) { + /// Send every unpaid line that has an address through the shared + /// payments path ([`crate::payments::pay_all`]): one [`Transfer`] per + /// line, keyed `ads:<settlement hash>:<line>`, so `shared` (the + /// payments ledger) and the rail both refuse to pay a line twice, and + /// the round stops at `cap`. `payer` is who pays (the campaign owner's + /// key). Returns lines sent. + pub async fn pay( + &mut self, + rail: &dyn crate::payments::Rail, + shared: &crate::payments::Ledger, + payer: &str, + cap: crate::payments::Micros, + ) -> usize { + use crate::payments::{Outcome, Transfer}; + let mut transfers = Vec::new(); + let mut index = Vec::new(); + for (i, st) in self.lines.iter_mut().enumerate().filter(|(_, l)| l.reference.is_none()) { let Some(s) = self.settlements.get(st.settlement as usize) else { continue; }; let Some(p) = s.payouts.get(st.line) else { continue; }; - if p.address.is_empty() { - st.error = Some("no address: owed, not sent".into()); + let Some(payee) = payee_of(&p.address) else { + st.error = Some(if p.address.is_empty() { + "no address: owed, not sent".into() + } else { + format!("not a payout address: {}", p.address) + }); continue; - } - let key = format!("{}:{}", s.hash(), st.line); - match rail.pay(payer, &p.address, p.micros, &key) { - Ok(r) => { - st.reference = Some(r); + }; + transfers.push(Transfer { + key: format!("ads:{}:{}", s.hash(), st.line), + payer: payer.to_string(), + payee, + amount: p.micros, + memo: format!("c0mpute ads {} settlement {} line {}", s.campaign_id, s.seq, st.line), + }); + index.push(i); + } + let outcomes = crate::payments::pay_all(rail, shared, &transfers, cap).await; + let by_key: HashMap<String, Outcome> = outcomes.into_iter().map(|(t, o)| (t.key, o)).collect(); + let entries = shared.entries(); + let mut sent = 0; + for (t, i) in transfers.iter().zip(index) { + let st = &mut self.lines[i]; + match by_key.get(&t.key) { + Some(Outcome::Paid(p)) => { + st.reference = Some(p.rail_ref.clone()); st.error = None; sent += 1; } - Err(e) => st.error = Some(e.to_string()), + // Paid in an earlier round whose ads ledger was not saved. + Some(Outcome::AlreadyPaid) => { + st.reference = Some(entries.get(&t.key).map(|e| e.rail_ref.clone()).unwrap_or_default()); + st.error = None; + } + Some(Outcome::OverBudget) => { + st.error = Some("over this round's payment cap or the payer's balance; next round retries".into()) + } + Some(Outcome::Failed(e)) => st.error = Some(e.clone()), + None => {} } } sent @@ -819,136 +856,20 @@ impl Ledger { } } -// ── rails ──────────────────────────────────────────────────────────────── - -/// Moves money. `pay` sends `micros` USD millionths to `to` -/// (`<CHAIN>:<address>`) and returns the rail's transfer id. -pub trait PaymentRail: Send + Sync { - fn name(&self) -> &str; - fn pay(&self, from: Option<&str>, to: &str, micros: u64, idempotency_key: &str) -> Result<String>; -} - -/// The default: refuses to move money. -pub struct DisabledRail; - -impl PaymentRail for DisabledRail { - fn name(&self) -> &str { - "disabled" - } - fn pay(&self, _: Option<&str>, _: &str, _: u64, _: &str) -> Result<String> { - bail!("payouts are disabled: set `[ads] payouts = true` and pass --pay to send real money") - } -} - -/// An in-memory rail for tests and dry runs: accounts with balances, every -/// transfer logged, idempotency keys honoured, overdrafts refused. -#[derive(Default)] -pub struct FakeRail { - pub balances: Mutex<HashMap<String, u64>>, - pub transfers: Mutex<Vec<(String, String, u64, String)>>, - keys: Mutex<HashMap<String, String>>, -} - -impl FakeRail { - pub fn fund(&self, account: &str, micros: u64) { - *self.balances.lock().unwrap().entry(account.to_string()).or_default() += micros; - } - pub fn balance(&self, account: &str) -> u64 { - self.balances.lock().unwrap().get(account).copied().unwrap_or(0) - } -} - -impl PaymentRail for FakeRail { - fn name(&self) -> &str { - "fake" - } - fn pay(&self, from: Option<&str>, to: &str, micros: u64, key: &str) -> Result<String> { - if let Some(r) = self.keys.lock().unwrap().get(key) { - return Ok(r.clone()); - } - let from = from.context("fake rail needs a payer account")?; - let mut bal = self.balances.lock().unwrap(); - let have = bal.get(from).copied().unwrap_or(0); - if have < micros { - bail!("insufficient funds: {from} has {have}, needs {micros}"); - } - bal.insert(from.to_string(), have - micros); - *bal.entry(to.to_string()).or_default() += micros; - let mut t = self.transfers.lock().unwrap(); - let r = format!("fake-{}", t.len()); - t.push((from.to_string(), to.to_string(), micros, r.clone())); - self.keys.lock().unwrap().insert(key.to_string(), r.clone()); - Ok(r) - } -} - -/// The real rail: `coinpay wallet send --chain <CHAIN> --to <address> -/// --amount-fiat <usd> --fiat USD` from the advertiser's own CoinPay wallet -/// (the `coinpay` plugin's CLI; it prompts for the wallet password unless -/// one is configured there). Nothing in c0mpute holds the funds. -pub struct CoinpayCliRail { - pub bin: PathBuf, - pub wallet_file: Option<PathBuf>, -} - -impl CoinpayCliRail { - /// `coinpay` on PATH (or `~/.c0mpute/bin`). - pub fn find() -> Option<Self> { - let bin = crate::ads::stitch::find_tool("coinpay")?; - Some(CoinpayCliRail { bin, wallet_file: None }) - } +// ── payouts ────────────────────────────────────────────────────────────── - /// The argv for one payout (without the binary). - pub fn args(&self, to: &str, micros: u64) -> Result<Vec<String>> { - if !valid_payout(to) { - bail!("not a payout address: {to:?}"); - } - let (chain, addr) = to.split_once(':').expect("checked"); - let mut a = vec![ - "wallet".into(), - "send".into(), - "--chain".into(), - chain.into(), - "--to".into(), - addr.into(), - "--amount-fiat".into(), - usd_decimal(micros), - "--fiat".into(), - "USD".into(), - ]; - if let Some(w) = &self.wallet_file { - a.push("--wallet-file".into()); - a.push(w.display().to_string()); - } - Ok(a) - } -} - -impl PaymentRail for CoinpayCliRail { - fn name(&self) -> &str { - "coinpay" - } - fn pay(&self, _from: Option<&str>, to: &str, micros: u64, key: &str) -> Result<String> { - let out = std::process::Command::new(&self.bin) - .args(self.args(to, micros)?) - .stdin(std::process::Stdio::inherit()) - .stderr(std::process::Stdio::inherit()) - .output() - .with_context(|| format!("run {}", self.bin.display()))?; - if !out.status.success() { - bail!("coinpay wallet send failed ({}) for {key}", out.status); - } - let text = String::from_utf8_lossy(&out.stdout); - // The CLI prints the transaction hash; keep the last token that looks - // like one, else the whole (trimmed) output. - let r = text - .split_whitespace() - .rev() - .find(|t| t.len() >= 16 && t.chars().all(|c| c.is_ascii_alphanumeric())) - .map(String::from) - .unwrap_or_else(|| text.trim().chars().take(200).collect()); - Ok(r) +/// `<COIN>:<address>` as a payments [`crate::payments::Payee`]: the part +/// before the first `:` is the CoinPay currency code (`SOL`, `USDC_POL`, +/// ...), the rest the wallet address. +pub fn payee_of(payout: &str) -> Option<crate::payments::Payee> { + if !valid_payout(payout) { + return None; } + let (coin, wallet) = payout.split_once(':')?; + Some(crate::payments::Payee { + wallet: wallet.to_string(), + coin: coin.to_string(), + }) } /// 1_234_500 -> "1.2345" (USD, trailing zeros dropped). @@ -1056,8 +977,8 @@ mod tests { assert_eq!(FEE_BPS + PUBLISHER_BPS + GATEWAY_BPS, BPS); } - #[test] - fn verify_prices_dedupes_and_settles() { + #[tokio::test] + async fn verify_prices_dedupes_and_settles() { let adv = Keypair::generate_ed25519(); let gw = Keypair::generate_ed25519(); let show_owner = crate::hosting::site::owner_hex(&Keypair::generate_ed25519()).unwrap(); @@ -1090,16 +1011,32 @@ mod tests { verify_claim(&c, &bids(&c), &ledger, &claim, &batches, &shows(&show_owner), "", NOW).unwrap_err(), ClaimError::AlreadySettled { next_seq: 3 } ); - // Paying through the fake rail moves exactly the gross. - let rail = FakeRail::default(); - rail.fund("SOL:advertiserwallet1", MICROS); - assert_eq!(ledger.pay(&rail, Some("SOL:advertiserwallet1")), 3); - assert_eq!(rail.balance("SOL:advertiserwallet1"), MICROS - 8_000); - assert_eq!(rail.balance("SOL:showwallet0001"), 6_000); - assert_eq!(rail.balance("SOL:gatewaywallet1"), 1_600); - assert_eq!(rail.balance("SOL:networkfee0001"), 400); - assert_eq!(ledger.pay(&rail, Some("SOL:advertiserwallet1")), 0, "never paid twice"); + // Paying through the shared fake rail moves exactly the gross. + use crate::payments::{FakeRail, Ledger as PayLedger, Rail}; + let rail = FakeRail::new(); + let shared = PayLedger::in_memory(); + rail.fund(&c.owner, MICROS); + // A round cap below the gross pays what fits and leaves the rest owed. + assert_eq!(ledger.pay(&rail, &shared, &c.owner, 6_000).await, 1); + assert_eq!(ledger.unpaid_micros(), 2_000); + assert!(ledger.lines.iter().any(|l| l.error.as_deref().is_some_and(|e| e.contains("cap")))); + assert_eq!(ledger.pay(&rail, &shared, &c.owner, MICROS).await, 2); + assert_eq!(rail.balance(&c.owner).await.unwrap(), Some(MICROS - 8_000)); + assert_eq!(rail.received("showwallet0001"), 6_000); + assert_eq!(rail.received("gatewaywallet1"), 1_600); + assert_eq!(rail.received("networkfee0001"), 400); + assert!(rail.payments().iter().all(|t| t.payee.coin == "SOL" && t.key.starts_with("ads:"))); + assert_eq!(ledger.pay(&rail, &shared, &c.owner, MICROS).await, 0, "never paid twice"); assert_eq!(ledger.unpaid_micros(), 0); + // An ads ledger that lost its references (crash before save) is + // healed from the shared ledger, not paid again. + let mut stale = ledger.clone(); + for l in &mut stale.lines { + l.reference = None; + } + assert_eq!(stale.pay(&rail, &shared, &c.owner, MICROS).await, 0); + assert_eq!(stale.unpaid_micros(), 0); + assert_eq!(rail.payments().len(), 3); // A later batch whose play was already paid in this claim is a duplicate. let mut log = ReceiptLog::open(None); @@ -1123,7 +1060,7 @@ mod tests { let s2 = ledger.settle(&c, v, Some(&adv), NOW).unwrap(); assert_eq!(s2.prev, s.hash()); assert_eq!(s2.spent_micros, 10_000); - ledger.pay(&rail, Some("SOL:advertiserwallet1")); + ledger.pay(&rail, &shared, &c.owner, MICROS).await; assert_eq!(ledger.unpaid_micros(), 100, "network fee owed, not sent"); } @@ -1286,7 +1223,7 @@ mod tests { } #[test] - fn payouts_budget_and_rail_args() { + fn payouts_budget_and_payees() { assert!(valid_payout("SOL:9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin")); assert!(valid_payout("BCH:bitcoincash:qpm2qsznhks23z7629mms6s4cwef74vcwvy22gdx6a")); assert!(valid_payout("USDC_POL:0x52908400098527886E0F7030069857D2E4169EE7")); @@ -1304,21 +1241,11 @@ mod tests { assert_eq!(usd_decimal(1_234_500), "1.2345"); assert_eq!(usd_decimal(5 * MICROS), "5"); assert_eq!(usd_decimal(1), "0.000001"); - let rail = CoinpayCliRail { - bin: "coinpay".into(), - wallet_file: None, - }; - assert_eq!( - rail.args("SOL:9xQeWvG816bUx9", 2_500_000).unwrap(), - ["wallet", "send", "--chain", "SOL", "--to", "9xQeWvG816bUx9", "--amount-fiat", "2.5", "--fiat", "USD"] - ); - assert!(rail.args("not an address", 1).is_err()); - assert!(DisabledRail.pay(None, "SOL:9xQeWvG816bUx9", 1, "k").is_err()); - let fake = FakeRail::default(); - fake.fund("a", 10); - assert!(fake.pay(Some("a"), "b", 11, "k1").is_err(), "overdraft refused"); - assert_eq!(fake.pay(Some("a"), "b", 10, "k2").unwrap(), "fake-0"); - assert_eq!(fake.pay(Some("a"), "b", 10, "k2").unwrap(), "fake-0", "idempotent"); - assert_eq!(fake.balance("b"), 10); + let p = payee_of("USDC_POL:0xabc0123456789def").unwrap(); + assert_eq!((p.coin.as_str(), p.wallet.as_str()), ("USDC_POL", "0xabc0123456789def")); + assert!(p.valid()); + assert_eq!(payee_of("SOL:9xQeWvG816bUx9").unwrap().coin, "SOL"); + assert!(payee_of("not an address").is_none()); + assert!(payee_of("").is_none()); } } diff --git a/node/crates/c0mpute-core/src/ads/manage.rs b/node/crates/c0mpute-core/src/ads/manage.rs index ab319d0c..381fb31f 100644 --- a/node/crates/c0mpute-core/src/ads/manage.rs +++ b/node/crates/c0mpute-core/src/ads/manage.rs @@ -536,17 +536,26 @@ pub async fn announce_settlements(ads: &Ads, ledger: &Ledger) { } } +/// The shared payments ledger for a local campaign: every transfer key paid +/// through [`crate::payments`], so a line is never paid twice even if the ads +/// ledger was not saved after a payment. +pub fn payments_ledger_path(name: &str) -> PathBuf { + campaign_dir(name).join("payments.json") +} + /// `c0mpute ads settle`: boot a buyer node, collect claims and receipts, -/// verify, and (with `pay`) settle and pay through `rail`. +/// verify, and (with a rail) settle and pay through the shared payments path, +/// at most `cap` this round. pub async fn settle( record: &CampaignRecord, fee_payout: &str, - rail: Option<&dyn billing::PaymentRail>, + rail: Option<&dyn crate::payments::Rail>, + cap: crate::payments::Micros, wait: Duration, ) -> Result<(SettleRound, Ledger)> { - let budget = record + record .budget - .clone() + .as_ref() .context("this campaign has no budget: c0mpute ads campaign update <name> --budget <usd>")?; let path = ledger_path(&record.name); let mut ledger = Ledger::load(&path, &record.campaign_id)?; @@ -562,7 +571,8 @@ pub async fn settle( let round = settle_round(&ads, &h, record, &bids, &mut ledger, owner, fee_payout, wait).await; if let Some(rail) = rail { ledger.save(&path)?; - ledger.pay(rail, budget.payer.as_deref()); + let shared = crate::payments::Ledger::open(payments_ledger_path(&record.name)); + ledger.pay(rail, &shared, &record.owner, cap).await; ledger.save(&path)?; for _ in 0..3 { announce_settlements(&ads, &ledger).await; diff --git a/node/crates/c0mpute-core/src/ads/mod.rs b/node/crates/c0mpute-core/src/ads/mod.rs index ac9ed366..e82a1918 100644 --- a/node/crates/c0mpute-core/src/ads/mod.rs +++ b/node/crates/c0mpute-core/src/ads/mod.rs @@ -133,7 +133,7 @@ pub struct Options { pub ffprobe: Option<PathBuf>, /// Absolute origin for URLs in VAST (else from the request's Host). pub public_origin: Option<String>, - /// Where this gateway is paid for the ads it serves, `<CHAIN>:<address>`. + /// Where this gateway is paid for the ads it serves, `<COIN>:<address>`. /// Without one it serves and signs receipts but never claims. pub payout: Option<String>, /// Smallest claim worth making, USD millionths. @@ -247,7 +247,7 @@ impl Ads { let payout = opts.payout.filter(|p| { let ok = billing::valid_payout(p); if !ok { - warn!(payout = %p, "ads: payout is not <CHAIN>:<address>; this gateway will not claim"); + warn!(payout = %p, "ads: payout is not <COIN>:<address>; this gateway will not claim"); } ok }); diff --git a/node/crates/c0mpute-core/src/ads/stitch.rs b/node/crates/c0mpute-core/src/ads/stitch.rs index 03a72f33..84ae54fe 100644 --- a/node/crates/c0mpute-core/src/ads/stitch.rs +++ b/node/crates/c0mpute-core/src/ads/stitch.rs @@ -49,7 +49,7 @@ pub struct ShowAds { /// Longest creative the show runs, seconds. #[serde(default = "default_max_ad_secs")] pub max_ad_secs: u64, - /// Where the show's share of ad revenue goes, `<CHAIN>:<address>` + /// Where the show's share of ad revenue goes, `<COIN>:<address>` /// ([`super::billing`]). Without one the show's impressions are not paid /// (and not charged to advertisers). #[serde(default, skip_serializing_if = "Option::is_none")] @@ -88,7 +88,7 @@ impl ShowAds { if let Some(p) = &s.payout && !super::billing::valid_payout(p) { - bail!("ads payout must be <CHAIN>:<address>"); + bail!("ads payout must be <COIN>:<address>"); } Ok(s) } diff --git a/node/crates/c0mpute-core/src/config.rs b/node/crates/c0mpute-core/src/config.rs index 03b8ec86..3f07c94a 100644 --- a/node/crates/c0mpute-core/src/config.rs +++ b/node/crates/c0mpute-core/src/config.rs @@ -77,18 +77,19 @@ pub struct AdsConfig { #[serde(default = "default_true")] pub podcast_insertion: bool, /// Gateway: where this gateway is paid for ads it serves on billed - /// campaigns, `<CHAIN>:<address>` (e.g. "SOL:9xQe..."). Without it the + /// campaigns, `<COIN>:<address>` (e.g. "SOL:9xQe..."). Without it the /// gateway serves and signs receipts but never claims payment. #[serde(default)] pub payout: Option<String>, /// Gateway: smallest claim worth making, USD. #[serde(default = "default_min_claim_usd")] pub min_claim_usd: String, - /// Advertiser: let `c0mpute ads settle --pay` send real money from your - /// CoinPay wallet. Off by default: settle only reports what it would pay. + /// Advertiser: let `c0mpute ads settle --pay` send real money through + /// the node's payment rail (also needs `[payments] enabled = true`). Off + /// by default: settle only reports what it would pay. #[serde(default)] pub payouts: bool, - /// Advertiser: where the 5% network fee is sent, `<CHAIN>:<address>`. + /// Advertiser: where the 5% network fee is sent, `<COIN>:<address>`. /// Without it the fee is recorded as owed and not sent. #[serde(default)] pub fee_payout: Option<String>, diff --git a/node/crates/c0mpute-core/src/podcast/show.rs b/node/crates/c0mpute-core/src/podcast/show.rs index 72be15e9..ad4f2a0c 100644 --- a/node/crates/c0mpute-core/src/podcast/show.rs +++ b/node/crates/c0mpute-core/src/podcast/show.rs @@ -76,7 +76,7 @@ pub struct AdsMeta { /// Lowest CPM in USD, e.g. "1.50". pub min_cpm: Option<String>, pub max_ad_secs: Option<u64>, - /// Where the show's ad revenue goes, `<CHAIN>:<address>` (e.g. + /// Where the show's ad revenue goes, `<COIN>:<address>` (e.g. /// "SOL:9xQe..."). Without it the show runs ads unpaid. pub payout: Option<String>, } @@ -118,7 +118,7 @@ impl AdsMeta { } if let Some(p) = &self.payout { if !crate::ads::billing::valid_payout(p.trim()) { - bail!("[ads] payout must be <CHAIN>:<address>, e.g. \"SOL:9xQe...\""); + bail!("[ads] payout must be <COIN>:<address>, e.g. \"SOL:9xQe...\""); } s.payout = Some(p.trim().to_string()); } diff --git a/node/crates/c0mpute-core/tests/ads_billing_e2e.rs b/node/crates/c0mpute-core/tests/ads_billing_e2e.rs index 43c2681c..2ef85780 100644 --- a/node/crates/c0mpute-core/tests/ads_billing_e2e.rs +++ b/node/crates/c0mpute-core/tests/ads_billing_e2e.rs @@ -22,7 +22,8 @@ use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; -use c0mpute_core::ads::billing::{Budget, ClaimError, FakeRail, Ledger, Role}; +use c0mpute_core::ads::billing::{Budget, ClaimError, Ledger, Role}; +use c0mpute_core::payments::{FakeRail, Ledger as PayLedger, PaymentsConfig, Rail}; use c0mpute_core::ads::campaign::{MICROS, Placement}; use c0mpute_core::ads::manage::{self, CampaignSpec}; use c0mpute_core::ads::receipt::{Event, Receipt}; @@ -252,8 +253,11 @@ async fn funded_campaign_two_gateways_claims_payouts_and_overspend() { let record = manage::record_from_spec(&spec, &adv_owner, creatives, 1).sign(&adv_key).unwrap(); let cid = record.campaign_id.clone(); let bids = BTreeMap::from([(record.seq, record.max_cpm_micros)]); - let rail = FakeRail::default(); - rail.fund(PAYER, MICROS); // the advertiser's wallet holds $1 + let rail = FakeRail::new(); + let shared = PayLedger::in_memory(); + let cap = PaymentsConfig::default().round_cap(); + let got = |payout: &str| rail.received(payout.split_once(':').unwrap().1); + rail.fund(&adv_owner, MICROS); // the advertiser holds $1 on the rail adv_ads.publish_campaign(record.clone()).await.unwrap(); { let (a, r) = (adv_ads.clone(), record.clone()); @@ -365,14 +369,19 @@ async fn funded_campaign_two_gateways_claims_payouts_and_overspend() { assert_eq!((line(&sb, Role::Gateway), line(&sb, Role::Network), line(&sb, Role::Publisher)), (4_600, 400, 3_000)); assert_eq!(sb.payouts.iter().find(|p| p.role == Role::Publisher).unwrap().key, pod_owner); - assert_eq!(ledger.pay(&rail, Some(PAYER)), 5); + assert_eq!(ledger.pay(&rail, &shared, &adv_owner, cap).await, 5); assert_eq!(ledger.unpaid_micros(), 0); - assert_eq!(rail.balance(PAYER), MICROS - 14_000, "budget spent = money moved"); - assert_eq!(rail.balance(GW_A), 5_700); - assert_eq!(rail.balance(GW_B), 4_600); - assert_eq!(rail.balance(SHOW), 3_000); - assert_eq!(rail.balance(FEE), 700); - assert_eq!(ledger.pay(&rail, Some(PAYER)), 0, "nothing is paid twice"); + assert_eq!(rail.balance(&adv_owner).await.unwrap().unwrap(), MICROS - 14_000, "budget spent = money moved"); + assert_eq!(got(GW_A), 5_700); + assert_eq!(got(GW_B), 4_600); + assert_eq!(got(SHOW), 3_000); + assert_eq!(got(FEE), 700); + assert_eq!(ledger.pay(&rail, &shared, &adv_owner, cap).await, 0, "nothing is paid twice"); + // Every line went through the one shared payments path and its ledger. + assert_eq!(shared.entries().len(), 5); + assert!(shared.entries().keys().all(|k| k.starts_with("ads:"))); + // With payments off (the default) there is no rail at all. + assert!(c0mpute_core::payments::rail_from_config(&PaymentsConfig::default()).unwrap().is_none()); // ── 4. gateways learn the budget left; overspend is refused ────────── { @@ -417,10 +426,10 @@ async fn funded_campaign_two_gateways_claims_payouts_and_overspend() { } ); assert_eq!(ledger.spent_micros, budget_total); - ledger.pay(&rail, Some(PAYER)); - assert_eq!(rail.balance(PAYER), MICROS - budget_total, "never more than the budget"); + ledger.pay(&rail, &shared, &adv_owner, cap).await; + assert_eq!(rail.balance(&adv_owner).await.unwrap().unwrap(), MICROS - budget_total, "never more than the budget"); assert_eq!( - rail.balance(GW_A) + rail.balance(GW_B) + rail.balance(SHOW) + rail.balance(FEE), + got(GW_A) + got(GW_B) + got(SHOW) + got(FEE), budget_total ); // The settlement chain links and every node can check it. diff --git a/plugins/ads-manager/module.toml b/plugins/ads-manager/module.toml index 51006e3b..ade7b096 100644 --- a/plugins/ads-manager/module.toml +++ b/plugins/ads-manager/module.toml @@ -42,9 +42,9 @@ version = "0.2.81" kind = "service" category = "applications" status = "partial" -status_note = "Campaigns, selection, receipts and billing work peer to peer and are tested end to end on a fake payment rail (budgets, second-price receipts, gateway claims, verified settlements, payouts and overspend refusal). Only the switch to real money remains: real payouts run `coinpay wallet send` from the advertiser's wallet and stay off until the advertiser sets `[ads] payouts = true` and runs `c0mpute ads settle --pay` (never exercised with real funds; moving third-party money needs Anthony's approval), and the 5% network fee is only owed, not sent, until `[ads] fee_payout` names c0mpute's fee wallet." +status_note = "Campaigns, selection, receipts and billing work peer to peer and are tested end to end on a fake payment rail (budgets, second-price receipts, gateway claims, verified settlements, payouts and overspend refusal). Only the switch to real money remains: real payouts go through the node's one payment rail (`payments.rs`: CoinPay payouts API, key in `$C0MPUTE_COINPAY_PAYOUT_KEY`) and stay off until the advertiser sets both `[ads] payouts = true` and `[payments] enabled = true` and runs `c0mpute ads settle --pay` (never exercised with real funds; moving third-party money needs Anthony's approval), and the 5% network fee is only owed, not sent, until `[ads] fee_payout` names c0mpute's fee wallet." builtin = true -description = "Peer-to-peer ad operations: advertisers sign campaigns with a budget that gossip to every gateway, creatives live on self-healing hosting copies, each gateway picks ads with an explainable second-price rule, gateways sign chained impression receipts and claim payment over them, and the advertiser verifies and pays gateways and shows from its own CoinPay wallet. No ad server, no clearing house." +description = "Peer-to-peer ad operations: advertisers sign campaigns with a budget that gossip to every gateway, creatives live on self-healing hosting copies, each gateway picks ads with an explainable second-price rule, gateways sign chained impression receipts and claim payment over them, and the advertiser verifies and pays gateways and shows through c0mpute's one payment rail. No ad server, no clearing house." author = "Profullstack" license = "MIT" source = "https://github.com/profullstack/c0mpute/tree/master/node/crates/c0mpute-core/src/ads" diff --git a/plugins/ctv-ads/module.toml b/plugins/ctv-ads/module.toml index 7ffc079d..e338e423 100644 --- a/plugins/ctv-ads/module.toml +++ b/plugins/ctv-ads/module.toml @@ -32,7 +32,7 @@ version = "0.2.81" kind = "service" category = "applications" status = "partial" -status_note = "VAST 4.2 / VMAP serving with signed, priced receipts works on every gateway, and gateways are paid per verified impression through ads-manager billing (tested end to end on a fake rail). Not built: OpenRTB bidding and live-stream (SCTE-35) insertion. Remaining switch: real payouts run `coinpay wallet send` from the advertiser's wallet and stay off until the advertiser sets `[ads] payouts = true` and runs `c0mpute ads settle --pay` (never exercised with real funds; moving third-party money needs Anthony's approval), and the 5% network fee is only owed, not sent, until `[ads] fee_payout` names c0mpute's fee wallet." +status_note = "VAST 4.2 / VMAP serving with signed, priced receipts works on every gateway, and gateways are paid per verified impression through ads-manager billing (tested end to end on a fake rail). Not built: OpenRTB bidding and live-stream (SCTE-35) insertion. Remaining switch: real payouts go through the node's one payment rail (`payments.rs`: CoinPay payouts API, key in `$C0MPUTE_COINPAY_PAYOUT_KEY`) and stay off until the advertiser sets both `[ads] payouts = true` and `[payments] enabled = true` and runs `c0mpute ads settle --pay` (never exercised with real funds; moving third-party money needs Anthony's approval), and the 5% network fee is only owed, not sent, until `[ads] fee_payout` names c0mpute's fee wallet." builtin = true description = "Video ads for connected TV and web players: every c0mpute gateway answers VAST 4.2 and VMAP requests from the gossiped campaign index, serves the video creative from hosting copies, and turns impression, quartile and click beacons into signed receipts." author = "Profullstack" diff --git a/plugins/podcast-ads/module.toml b/plugins/podcast-ads/module.toml index aeb00c1d..92787e6a 100644 --- a/plugins/podcast-ads/module.toml +++ b/plugins/podcast-ads/module.toml @@ -20,7 +20,7 @@ # impression receipt per ad. No ffmpeg, no fill or any failure: the clean # episode, with x-c0mpute-ads saying why. # -# Payout: a show names `payout = "<CHAIN>:<address>"` in [ads] (or +# Payout: a show names `payout = "<COIN>:<address>"` in [ads] (or # --ads-payout); it is inside the signed opt-in, so no gateway can redirect # it. Each paid impression on the show pays it 75% of the cleared price # (ads-manager billing). A show without a payout runs ads unpaid. @@ -35,7 +35,7 @@ version = "0.2.81" kind = "service" category = "applications" status = "partial" -status_note = "Dynamic ad insertion into opted-in podcasts works on gateways with ffmpeg, and shows that name a payout address earn 75% of each paid impression through ads-manager billing (tested end to end on a fake rail). Not built: a DAAST endpoint. Remaining switch: real payouts run `coinpay wallet send` from the advertiser's wallet and stay off until the advertiser sets `[ads] payouts = true` and runs `c0mpute ads settle --pay` (never exercised with real funds; moving third-party money needs Anthony's approval), and the 5% network fee is only owed, not sent, until `[ads] fee_payout` names c0mpute's fee wallet." +status_note = "Dynamic ad insertion into opted-in podcasts works on gateways with ffmpeg, and shows that name a payout address earn 75% of each paid impression through ads-manager billing (tested end to end on a fake rail). Not built: a DAAST endpoint. Remaining switch: real payouts go through the node's one payment rail (`payments.rs`: CoinPay payouts API, key in `$C0MPUTE_COINPAY_PAYOUT_KEY`) and stay off until the advertiser sets both `[ads] payouts = true` and `[payments] enabled = true` and runs `c0mpute ads settle --pay` (never exercised with real funds; moving third-party money needs Anthony's approval), and the 5% network fee is only owed, not sent, until `[ads] fee_payout` names c0mpute's fee wallet." builtin = true description = "Dynamic ad insertion for c0mpute-hosted podcasts: shows opt in with a signed flag (and payout address) in their own site, any gateway with ffmpeg splices a matching audio ad before, inside or after the episode, and the show is paid for each verified impression." author = "Profullstack" From 873b6ea3c1c84046f35b4e1ab388c4670267a531 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger <anthony@profullstack.com> Date: Mon, 9 Nov 2026 04:45:20 +0000 Subject: [PATCH 4/7] handoff: exchange dialable addresses and connect the buyer and the winner CI showed a failover worker that shared no connection with the buyer could not fetch the restore world (Failed to dial the requested peer). Both sides now put their addresses in the handoff and dial each other, so either side can be the one behind NAT. Also fixes a unit test slicing a short last chunk. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- node/crates/c0mpute-core/src/handoff.rs | 35 ++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/node/crates/c0mpute-core/src/handoff.rs b/node/crates/c0mpute-core/src/handoff.rs index 86f50c17..cf17fbb1 100644 --- a/node/crates/c0mpute-core/src/handoff.rs +++ b/node/crates/c0mpute-core/src/handoff.rs @@ -47,6 +47,9 @@ struct Ready { worker_peer_id: String, client_pubkey: String, worker_pubkey: String, + /// The worker's dialable addresses, so the buyer can connect directly. + #[serde(default)] + addrs: Vec<String>, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -55,6 +58,28 @@ struct Sealed { job_id: String, worker_peer_id: String, sealed: String, + /// The buyer's dialable addresses: a worker that must fetch from the + /// buyer (a world to restore) may share no connection with it yet. + #[serde(default)] + addrs: Vec<String>, +} + +const MAX_ADDRS: usize = 8; + +async fn my_addrs(net: &Libp2pNetwork) -> Vec<String> { + net.advertised_addrs().await.into_iter().take(MAX_ADDRS).map(|a| a.to_string()).collect() +} + +/// Learn `peer`'s addresses from a handoff message and open a direct +/// connection (either side may be the one that can dial out). +async fn link(net: &Libp2pNetwork, peer: &str, addrs: &[String]) { + let Ok(peer) = peer.parse::<c0mpute_net::PeerId>() else { return }; + for a in addrs.iter().take(MAX_ADDRS).filter(|a| a.len() <= 256) { + if let Ok(addr) = a.parse::<c0mpute_net::Multiaddr>() { + net.add_peer_address(peer, addr).await; + } + } + let _ = net.connect(peer).await; } /// A fresh key pair for one handoff. @@ -93,6 +118,7 @@ pub async fn receive(net: &Libp2pNetwork, workload: &str, accept: &JobAccept, cl worker_peer_id: me.clone(), client_pubkey: client_pubkey.to_string(), worker_pubkey: kp.public_hex.clone(), + addrs: my_addrs(net).await, })?; let deadline = Instant::now() + wait; let mut next = Instant::now(); @@ -118,7 +144,10 @@ pub async fn receive(net: &Libp2pNetwork, workload: &str, accept: &JobAccept, cl continue; } match crypto::open(&k, &s.sealed) { - Ok(plain) if plain.len() <= MAX_SECRET => return Ok(plain), + Ok(plain) if plain.len() <= MAX_SECRET => { + link(net, &accept.buyer_peer_id, &s.addrs).await; + return Ok(plain); + } Ok(_) => bail!("sealed secret too large"), Err(e) => debug!(job_id = %accept.job_id, err = %e, "handoff: unreadable secret ignored"), } @@ -150,6 +179,7 @@ impl BuyerHandoff { // job_id → sealed message, republished until the task ends. let mut sent: HashMap<String, Vec<u8>> = HashMap::new(); let mut tick = tokio::time::interval(REPUBLISH); + let addrs = my_addrs(&net).await; let seal = |r: &Ready| -> Result<Vec<u8>> { let k = key(&kp.secret, &r.worker_pubkey, &workload, &r.job_id)?; Ok(serde_json::to_vec(&Sealed { @@ -157,6 +187,7 @@ impl BuyerHandoff { job_id: r.job_id.clone(), worker_peer_id: r.worker_peer_id.clone(), sealed: crypto::seal(&k, &secret)?, + addrs: addrs.clone(), })?) }; loop { @@ -168,6 +199,7 @@ impl BuyerHandoff { { let _ = net.publish(&topic, m.clone()).await; sent.insert(job_id.clone(), m); + link(&net, &r.worker_peer_id, &r.addrs).await; } winners.insert(job_id, bid.bidder_peer_id); } @@ -191,6 +223,7 @@ impl BuyerHandoff { let _ = net.publish(&topic, m.clone()).await; sent.insert(r.job_id.clone(), m); } + link(&net, &r.worker_peer_id, &r.addrs).await; } Some(_) => debug!(job_id = %r.job_id, "handoff: key from a worker we did not accept ignored"), None if readies.len() < 64 => { readies.insert(r.job_id.clone(), r); } From 29b03b868bef8bc56d5896045a0a0389e9e82917 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger <anthony@profullstack.com> Date: Mon, 9 Nov 2026 04:56:24 +0000 Subject: [PATCH 5/7] hosting: release a copy only after peers see it; no failed receipt for a holder that just let go drop_site deleted chunks before announcing, so challengers that still listed the holder recorded failures (flaky hosting_market_e2e; could wrongly demote a preferred host). Announce first, garbage-collect after holder_ttl; a failed fetch from a peer that no longer lists the site yields no receipt. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- node/crates/c0mpute-core/src/hosting/mod.rs | 22 +++++++++++++++++-- .../c0mpute-core/tests/hosting_market_e2e.rs | 13 +++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/node/crates/c0mpute-core/src/hosting/mod.rs b/node/crates/c0mpute-core/src/hosting/mod.rs index f2f60433..505385bf 100644 --- a/node/crates/c0mpute-core/src/hosting/mod.rs +++ b/node/crates/c0mpute-core/src/hosting/mod.rs @@ -648,9 +648,22 @@ impl Hosting { return Ok(()); }; self.save_catalog().await; - self.collect_garbage(&rec, None).await; - info!(site = %site_id, "hosting: released a surplus copy"); + // Tell the network first, and keep the chunks (still served to + // fetches and challenges) until peers have certainly seen that we no + // longer hold the site. Deleting first made challengers that still + // listed us record a failure against a copy we had just let go. self.announce_holding().await; + info!(site = %site_id, "hosting: released a surplus copy"); + let h = self.clone(); + let grace = self.0.timing.holder_ttl; + tokio::spawn(async move { + tokio::time::sleep(grace).await; + // Taken back in the meantime: keep everything. + if h.0.catalog.read().await.sites.contains_key(&rec.site_id) { + return; + } + h.collect_garbage(&rec, None).await; + }); Ok(()) } @@ -856,6 +869,11 @@ impl Hosting { let started = Instant::now(); let got = tokio::time::timeout(self.0.timing.fetch_timeout, self.0.net.fetch_from_peer(pid, &req)).await; let ok = matches!(&got, Ok(Ok(bytes)) if Hash::of(bytes) == hash); + if !ok && !self.0.map.read().await.holders(&rec.site_id).iter().any(|p| p == holder) { + // The holder announced it let the copy go while we were asking: + // not a failed proof, just a stale target. No receipt. + bail!("{holder} no longer holds {}", rec.site_id); + } let rtt_ms = if ok { started.elapsed().as_millis().min(u32::MAX as u128) as u32 } else { 0 }; let receipt = Receipt::sign( kp, diff --git a/node/crates/c0mpute-core/tests/hosting_market_e2e.rs b/node/crates/c0mpute-core/tests/hosting_market_e2e.rs index b0959012..fa779d76 100644 --- a/node/crates/c0mpute-core/tests/hosting_market_e2e.rs +++ b/node/crates/c0mpute-core/tests/hosting_market_e2e.rs @@ -197,14 +197,19 @@ async fn preferred_hosts_hold_pass_challenges_and_get_paid() { } assert_eq!(publisher.preferred(&site).await, preferred); - // A challenge round: every holder passes. + // A challenge round: every holder passes. (A copy released moments ago + // may still be listed by the challenger; it keeps serving through its + // grace period, so it passes too rather than failing.) let receipts = publisher.challenge_round().await; - assert_eq!(receipts.len(), 3, "one challenge per holder: {receipts:?}"); - assert!(receipts.iter().all(|r| r.ok && r.verify() && r.challenger == owner_key)); + assert!(receipts.len() >= 3, "one challenge per holder: {receipts:?}"); + assert!(receipts.iter().all(|r| r.ok && r.verify() && r.challenger == owner_key), "{receipts:?}"); + for x in &preferred { + assert!(receipts.iter().any(|r| &r.holder == x), "preferred {x} was challenged"); + } let pd2 = pd.clone(); assert_eq!( std::fs::read_to_string(pd2.join("receipts.jsonl")).unwrap().lines().count(), - 3, + receipts.len(), "receipts are kept on disk for settlement" ); // The receipts reach other nodes over gossip. From 8fc34826aab7990b093b33f1cdff6242486809cd Mon Sep 17 00:00:00 2001 From: Anthony Ettinger <anthony@profullstack.com> Date: Mon, 9 Nov 2026 04:44:43 +0000 Subject: [PATCH 6/7] feat(secure-chat): p2p E2E messaging over mailboxes and direct delivery (DIP-0018) - c0mpute-secure-chat: addresses = hash of all four public keys, signed key/home records, sealed outer layer hiding sender and recipient, rotating recipient tags, replay guard, bounded mailbox store, rendezvous placement, and client state encrypted at rest. - c0mpute-net: /c0mpute/rpc/1.0.0 named-service request/response (RpcHandler, register_rpc, rpc, connected_peers). - c0mpute-core::dm: mailbox role (on by default, gateways too), gossip on c0mpute/dm/v1, direct delivery to a recipient's home node, store on 3 rendezvous-ranked mailboxes otherwise, fetch/ack, key publish and lookup by fingerprint. New Role::Mailbox. - c0mpute dm: send/pull/inbox/lookup/contacts/block/unblock wired to the network; old `c0mpute chat <cmd>` forms still work. - tests: unit tests plus dm_e2e (4 mailboxes, offline store-and-forward, ack deletes every copy, blocklist, fingerprint lookup, direct delivery). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- Cargo.lock | 1 + node/crates/c0mpute-cli/src/dm_cli.rs | 393 ++++++++ node/crates/c0mpute-cli/src/main.rs | 191 ++-- node/crates/c0mpute-core/Cargo.toml | 1 + node/crates/c0mpute-core/src/capabilities.rs | 1 + node/crates/c0mpute-core/src/config.rs | 1 + node/crates/c0mpute-core/src/dm.rs | 897 ++++++++++++++++++ node/crates/c0mpute-core/src/hosting/mod.rs | 2 +- node/crates/c0mpute-core/src/lib.rs | 1 + node/crates/c0mpute-core/src/supervisor.rs | 19 + node/crates/c0mpute-core/tests/dm_e2e.rs | 216 +++++ node/crates/c0mpute-net/src/lib.rs | 3 +- node/crates/c0mpute-net/src/swarm.rs | 272 +++++- node/crates/c0mpute-proto/src/lib.rs | 5 + .../c0mpute-secure-chat/src/identity.rs | 339 +++++++ node/crates/c0mpute-secure-chat/src/keys.rs | 11 +- node/crates/c0mpute-secure-chat/src/lib.rs | 32 +- .../crates/c0mpute-secure-chat/src/mailbox.rs | 421 ++++++++ .../crates/c0mpute-secure-chat/src/message.rs | 155 ++- .../c0mpute-secure-chat/src/placement.rs | 88 ++ node/crates/c0mpute-secure-chat/src/sealed.rs | 423 +++++++++ node/crates/c0mpute-secure-chat/src/state.rs | 268 ++++++ plugins/secure-chat/module.toml | 76 +- 23 files changed, 3607 insertions(+), 209 deletions(-) create mode 100644 node/crates/c0mpute-cli/src/dm_cli.rs create mode 100644 node/crates/c0mpute-core/src/dm.rs create mode 100644 node/crates/c0mpute-core/tests/dm_e2e.rs create mode 100644 node/crates/c0mpute-secure-chat/src/identity.rs create mode 100644 node/crates/c0mpute-secure-chat/src/mailbox.rs create mode 100644 node/crates/c0mpute-secure-chat/src/placement.rs create mode 100644 node/crates/c0mpute-secure-chat/src/sealed.rs create mode 100644 node/crates/c0mpute-secure-chat/src/state.rs diff --git a/Cargo.lock b/Cargo.lock index 5b9b7de0..246dc9ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -548,6 +548,7 @@ dependencies = [ "c0mpute-gateway", "c0mpute-net", "c0mpute-proto", + "c0mpute-secure-chat", "c0mpute-store", "c0mpute-transcode", "c0mpute-update", diff --git a/node/crates/c0mpute-cli/src/dm_cli.rs b/node/crates/c0mpute-cli/src/dm_cli.rs new file mode 100644 index 00000000..3a1b815d --- /dev/null +++ b/node/crates/c0mpute-cli/src/dm_cli.rs @@ -0,0 +1,393 @@ +//! `c0mpute dm`: the secure-chat client (DIP-0018). +//! +//! Network commands boot a short-lived libp2p client node (its own identity +//! under `<data_dir>/dm-client`), find mailbox nodes through the peers it +//! connects to, and talk to them over `/c0mpute/rpc/1.0.0`. When this +//! machine also runs a worker, the client signs a home record naming the +//! worker's peer id, so senders can deliver straight to that worker; the +//! client then reads the worker's spool on `dm pull`. +//! +//! The key password comes from `C0MPUTE_DM_PASSWORD` when set (scripts), +//! otherwise from a prompt. + +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use c0mpute_core::Config; +use c0mpute_core::config; +use c0mpute_core::dm::{self, Dm, Paths}; +use c0mpute_secure_chat::identity::{fingerprint_of, normalize_query}; +use c0mpute_secure_chat::mailbox::{Limits, Mailbox}; +use c0mpute_secure_chat::sealed::{MAX_TTL_SECS, pull_tags, seal}; +use c0mpute_secure_chat::state::{LocalState, Received, state_file_path}; +use c0mpute_secure_chat::{self as chat, ChatKey, HomeRecord, KeyFile, KeyRecord, PublicIdentity, unix_now}; + +use super::{ChatCmd, ContactsCmd, KeyCmd, prompt_new_password}; + +/// How long home and key records stay valid; refreshed on every network command. +const RECORD_SECS: u64 = 30 * 86_400; + +fn password(prompt: &str) -> Result<String> { + if let Ok(p) = std::env::var("C0MPUTE_DM_PASSWORD") + && !p.is_empty() + { + return Ok(p); + } + Ok(rpassword::prompt_password(prompt)?) +} + +struct Session { + key: ChatKey, + state: LocalState, + state_path: std::path::PathBuf, + cfg: Config, +} + +impl Session { + fn open() -> Result<Self> { + let path = chat::key_file_path()?; + let kf = chat::load_keyfile(&path).with_context(|| "no chat key yet: run `c0mpute dm keygen`".to_string())?; + let key = chat::decrypt_keyfile(&kf, &password("Chat key password: ")?)?; + let state_path = state_file_path()?; + let state = LocalState::load(&state_path, &key)?; + let cfg = Config::load_or_default(&config::default_config_path())?; + Ok(Self { + key, + state, + state_path, + cfg, + }) + } + + fn save(&self) -> Result<()> { + self.state.save(&self.state_path, &self.key) + } + + fn paths(&self) -> Paths { + Paths::under(&self.cfg.storage.root) + } + + /// If this machine has a worker identity, make that worker this user's + /// home: it accepts direct deliveries and gossips the record while up. + fn register_home(&self) -> Option<String> { + let dir = config::config_dir()?; + let peer = dm::worker_peer_id(&dir)?; + let rec = HomeRecord::sign(&self.key, &peer, unix_now(), RECORD_SECS).ok()?; + dm::write_home_record(&self.paths().homes, &rec).ok()?; + Some(peer) + } + + /// Resolve TO from local knowledge, then the network. + async fn resolve(&mut self, who: &str, net: Option<&Dm>) -> Result<PublicIdentity> { + if let Some(id) = self.state.resolve(who) { + return Ok(id); + } + let Some(net) = net else { + bail!("unknown recipient {who}: use an address, fingerprint, contact name, or c0dm1. identity"); + }; + if normalize_query(who).is_none() { + bail!("unknown contact {who}"); + } + match net.lookup(who).await { + Some(id) => { + self.state.remember(&id); + Ok(id) + } + None => bail!( + "no key record found for {who}. Ask them for their identity string (`c0mpute dm key show --full`)" + ), + } + } +} + +/// Boot the client node and wait briefly for mailboxes. +async fn connect(s: &Session) -> Result<Dm> { + let id_dir = config::data_dir() + .unwrap_or_else(|| std::path::PathBuf::from(".")) + .join("dm-client"); + let net = dm::boot_client(&id_dir).await?; + let d = Dm::start(net, dm::Options::default()).await?; + let n = d.wait_for_mailboxes(1, Duration::from_secs(20)).await; + if n == 0 { + eprintln!("warning: no mailbox nodes reachable yet (is a c0mpute worker or gateway reachable?)"); + } + // Publish our key record so others can find us by fingerprint. + if let Ok(rec) = KeyRecord::sign(&s.key, unix_now(), RECORD_SECS) { + let _ = d.publish_key(&rec).await; + } + Ok(d) +} + +pub async fn run_chat(cmd: ChatCmd) -> Result<()> { + match cmd { + ChatCmd::Keygen => keygen(), + ChatCmd::Key { cmd } => key_cmd(cmd), + ChatCmd::Restore { from_backup } => { + let dest = chat::key_file_path()?; + if dest.exists() { + eprintln!("A keyfile already exists at {}. Remove it first.", dest.display()); + bail!("keyfile already exists"); + } + let raw = std::fs::read_to_string(&from_backup)?; + let kf: KeyFile = serde_json::from_str(&raw)?; + // Verify the password decrypts correctly before saving. + chat::decrypt_keyfile(&kf, &password("Password: ")?)?; + chat::save_keyfile(&dest, &kf)?; + println!("keyfile restored to {}", dest.display()); + Ok(()) + } + ChatCmd::Send { + to, + message, + ttl_days, + sealed: _, + } => { + let mut s = Session::open()?; + s.register_home(); + let net = connect(&s).await?; + let rcpt = s.resolve(&to, Some(&net)).await?; + let ttl = (ttl_days * 86_400).clamp(60, MAX_TTL_SECS); + let sealed = seal(&s.key, &rcpt, &message, ttl, unix_now())?; + let report = net.send(&sealed, &rcpt.address(), ttl).await?; + s.save()?; + let name = s.state.name_of(&rcpt.address()); + match report.direct { + Some(peer) => println!("delivered directly to {name}'s node ({peer})"), + None => println!( + "{name} is offline: held on {} mailbox node(s) for up to {} days", + report.stored_on.len(), + ttl / 86_400 + ), + } + Ok(()) + } + ChatCmd::Pull => { + let mut s = Session::open()?; + s.register_home(); + let now = unix_now(); + let me = s.key.address(); + let mut new = 0usize; + let mut dropped = 0usize; + let mut bad = 0usize; + // 1. Direct deliveries on this machine's node. + let spool_dir = s.paths().spool; + if spool_dir.exists() { + let mut spool = Mailbox::open(&spool_dir, Limits::default(), now); + for tag in pull_tags(&me, now) { + for (id, bytes) in spool.fetch(&tag, now) { + let res = chat::SealedMessage::parse(&bytes) + .and_then(|m| s.state.receive(&s.key, &m, "direct", now)); + match res { + Ok(r) => { + tally(&r, &mut new, &mut dropped); + spool.ack(&tag, &id, r.ack()); + } + Err(e) => { + bad += 1; + eprintln!("skipped a direct message: {e}"); + } + } + } + } + } + // 2. Mailboxes. + let net = connect(&s).await?; + for f in net.fetch(&me).await { + match s.state.receive(&s.key, &f.msg, "mailbox", unix_now()) { + Ok(r) => { + tally(&r, &mut new, &mut dropped); + net.ack(&f, r.ack()).await; + } + Err(e) => { + bad += 1; + eprintln!("skipped a mailbox message: {e}"); + } + } + } + s.save()?; + println!( + "{new} new message(s){}{}", + if dropped > 0 { format!(", {dropped} from blocked senders dropped") } else { String::new() }, + if bad > 0 { format!(", {bad} rejected") } else { String::new() } + ); + if new > 0 { + print_inbox(&s.state, new); + } + Ok(()) + } + ChatCmd::Inbox { limit } => { + let s = Session::open()?; + if s.state.inbox.is_empty() { + println!("inbox empty (run `c0mpute dm pull`)"); + } else { + print_inbox(&s.state, limit); + } + Ok(()) + } + ChatCmd::Lookup { query } => { + let mut s = Session::open()?; + let id = match s.state.resolve(&query) { + Some(id) => id, + None => { + let net = connect(&s).await?; + s.resolve(&query, Some(&net)).await? + } + }; + s.save()?; + println!("address : {}", id.address()); + println!("fingerprint : {}", id.fingerprint()); + println!("identity : {}", id.to_share_string()); + Ok(()) + } + ChatCmd::Contacts { cmd } => match cmd.unwrap_or(ContactsCmd::Ls) { + ContactsCmd::Ls => { + let s = Session::open()?; + if s.state.contacts.is_empty() { + println!("no contacts (add one: c0mpute dm contacts add <address|fingerprint> --name alice)"); + } + for (addr, c) in &s.state.contacts { + let blocked = if s.state.blocked.contains(addr) { " [blocked]" } else { "" }; + println!("{:<16} {} {addr}{blocked}", c.name.as_deref().unwrap_or("-"), fingerprint_of(addr)); + } + Ok(()) + } + ContactsCmd::Add { who, name } => { + let mut s = Session::open()?; + let id = match s.state.resolve(&who) { + Some(id) => id, + None => { + let net = connect(&s).await?; + s.resolve(&who, Some(&net)).await? + } + }; + let addr = s.state.add_contact(id, name, unix_now()); + s.save()?; + println!("added {} ({})", s.state.name_of(&addr), fingerprint_of(&addr)); + Ok(()) + } + ContactsCmd::Rm { who } => { + let mut s = Session::open()?; + let id = s.state.resolve(&who).context("no such contact")?; + s.state.contacts.remove(&id.address()); + s.save()?; + println!("removed {}", fingerprint_of(&id.address())); + Ok(()) + } + }, + ChatCmd::Block { who } => { + let mut s = Session::open()?; + let addr = match s.state.resolve(&who) { + Some(id) => id.address(), + None => match normalize_query(&who) { + Some(q) if q.len() == 64 => q, + _ => bail!("block needs a full address, or a known contact/fingerprint"), + }, + }; + s.state.blocked.insert(addr.clone()); + s.save()?; + println!("blocked {}", fingerprint_of(&addr)); + Ok(()) + } + ChatCmd::Unblock { who } => { + let mut s = Session::open()?; + let addr = s + .state + .resolve(&who) + .map(|id| id.address()) + .or_else(|| normalize_query(&who).filter(|q| q.len() == 64)) + .context("unknown sender")?; + let was = s.state.blocked.remove(&addr); + s.save()?; + println!("{} {}", if was { "unblocked" } else { "was not blocked:" }, fingerprint_of(&addr)); + Ok(()) + } + } +} + +fn tally(r: &Received, new: &mut usize, dropped: &mut usize) { + match r { + Received::New(_) => *new += 1, + Received::Blocked(_) => *dropped += 1, + } +} + +fn print_inbox(state: &LocalState, limit: usize) { + let start = state.inbox.len().saturating_sub(limit); + for m in &state.inbox[start..] { + println!("[{}] {} ({}): {}", fmt_time(m.created_at), state.name_of(&m.from), m.via, m.text); + } +} + +fn fmt_time(t: u64) -> String { + let ago = unix_now().saturating_sub(t); + match ago { + 0..=59 => format!("{ago}s ago"), + 60..=3599 => format!("{}m ago", ago / 60), + 3600..=86_399 => format!("{}h ago", ago / 3600), + _ => format!("{}d ago", ago / 86_400), + } +} + +fn keygen() -> Result<()> { + let path = chat::key_file_path()?; + if path.exists() { + eprintln!( + "A keyfile already exists at {}.\n\ + Use `c0mpute dm key rotate` for how to replace it.", + path.display() + ); + bail!("keyfile already exists"); + } + let key = ChatKey::generate(); + let password = match std::env::var("C0MPUTE_DM_PASSWORD") { + Ok(p) if p.len() >= 10 => p, + _ => prompt_new_password()?, + }; + let kf = key.encrypt_to_keyfile(&password, None)?; + chat::save_keyfile(&path, &kf)?; + let id = key.public_identity(); + println!("keypair generated"); + println!(" address : {}", id.address()); + println!(" fingerprint : {}", id.fingerprint()); + println!(" keyfile : {}", path.display()); + println!(); + println!("Share your fingerprint (others look it up) or your full identity:"); + println!(" {}", id.to_share_string()); + println!(); + println!("── encrypted backup (save this somewhere safe) ──"); + println!("{}", serde_json::to_string_pretty(&kf)?); + println!("─────────────────────────────────────────────────"); + println!("restore with: c0mpute dm restore --from-backup <file>"); + Ok(()) +} + +fn key_cmd(cmd: KeyCmd) -> Result<()> { + match cmd { + KeyCmd::Show { full } => { + let kf = chat::load_keyfile(&chat::key_file_path()?)?; + // Public fields only: no password needed. + let id = PublicIdentity::from_keyfile(&kf); + println!("address : {}", id.address()); + println!("fingerprint : {}", id.fingerprint()); + if let Some(did) = &kf.did { + println!("did : {did}"); + } + if full { + println!("identity : {}", id.to_share_string()); + } + Ok(()) + } + KeyCmd::Export => { + let kf = chat::load_keyfile(&chat::key_file_path()?)?; + println!("{}", serde_json::to_string_pretty(&kf)?); + Ok(()) + } + KeyCmd::Rotate => { + println!("To replace your keypair (your address changes; tell your contacts):"); + println!(" 1. c0mpute dm pull (collect anything still queued)"); + println!(" 2. move ~/.config/c0mpute/chat.key and chat-state.enc aside"); + println!(" 3. c0mpute dm keygen"); + Ok(()) + } + } +} diff --git a/node/crates/c0mpute-cli/src/main.rs b/node/crates/c0mpute-cli/src/main.rs index 5bcae116..0281b154 100644 --- a/node/crates/c0mpute-cli/src/main.rs +++ b/node/crates/c0mpute-cli/src/main.rs @@ -22,6 +22,8 @@ //! c0mpute.com/coinpay, c0mpute.com/infernet. mod ai_chat; +mod dm_cli; +use dm_cli::run_chat; mod gameserver; mod peers; mod storage; @@ -39,7 +41,6 @@ use c0mpute_core::{ run_register, }; use c0mpute_proto::Role; -use c0mpute_secure_chat as chat; use clap::{Parser, Subcommand}; #[derive(Parser, Debug)] @@ -588,7 +589,7 @@ enum SpeechCmd { /// `c0mpute dm` subcommands, still accepted after `c0mpute chat` for old scripts. const SECURE_CHAT_CMDS: &[&str] = &[ - "keygen", "key", "restore", "send", "pull", "lookup", "contacts", "block", + "keygen", "key", "restore", "send", "pull", "inbox", "lookup", "contacts", "block", "unblock", ]; #[derive(Parser, Debug)] @@ -1063,31 +1064,67 @@ enum ChatCmd { #[arg(long = "from-backup")] from_backup: PathBuf, }, - /// Send an encrypted DM to a recipient DID. [v0.2: transport not yet live] + /// Send an end-to-end encrypted message. TO is a contact name, an + /// address, a fingerprint, or a `c0dm1.` identity string. Delivered + /// directly to the recipient's node when it is online, otherwise held on + /// three mailbox nodes for up to --ttl-days. Send { to: String, message: String, - /// Hide sender identity from relay nodes. - #[arg(long)] + /// Days mailboxes keep the message for an offline recipient (max 30). + #[arg(long, default_value_t = 30)] + ttl_days: u64, + /// Accepted for old scripts; every message is sealed now. + #[arg(long, hide = true)] sealed: bool, }, - /// Fetch queued messages from the relay. [v0.2: relay not yet live] + /// Fetch new messages (this machine's node spool and the mailboxes), + /// decrypt, verify, and delete them from the network. Pull, - /// Resolve a DID to its chat public key. [v0.2: DHT not yet live] - Lookup { did: String }, - /// List saved contacts. - Contacts, - /// Block a DID (messages dropped locally). - Block { did: String }, + /// Show received messages. + Inbox { + /// Most recent N messages. + #[arg(long, default_value_t = 50)] + limit: usize, + }, + /// Resolve a fingerprint or address to a full public key. + Lookup { query: String }, + /// Contacts (stored locally, encrypted under your key). Lists by default. + Contacts { + #[command(subcommand)] + cmd: Option<ContactsCmd>, + }, + /// Block a sender: their messages are dropped (and deleted) on receipt. + Block { who: String }, + /// Unblock a sender. + Unblock { who: String }, +} + +#[derive(Subcommand, Debug)] +enum ContactsCmd { + /// Add a contact by address, fingerprint, or `c0dm1.` identity string. + Add { + who: String, + #[arg(long)] + name: Option<String>, + }, + /// List contacts. + Ls, + /// Remove a contact. + Rm { who: String }, } #[derive(Subcommand, Debug)] enum KeyCmd { - /// Print the public key, fingerprint, and DID. - Show, + /// Print your address and fingerprint (and, with --full, the shareable identity). + Show { + /// Also print the `c0dm1.` identity string others can paste. + #[arg(long)] + full: bool, + }, /// Re-export the encrypted backup JSON (pipe to a file for safe storage). Export, - /// Generate a new keypair and re-announce (old key still valid until revoked). + /// How to replace your keypair. Rotate, } @@ -1363,12 +1400,12 @@ async fn run_app(cli: Cli) -> Result<()> { Cmd::Podcasting { cmd } => run_podcasting(cmd).await, Cmd::Podcasts { cmd } => run_podcasts(cmd).await, Cmd::Ads { cmd } => run_ads(cmd).await, - Cmd::Dm { cmd } => run_chat(cmd), + Cmd::Dm { cmd } => run_chat(cmd).await, // The old secure-chat forms (`c0mpute chat keygen|send|...`) keep working. Cmd::Chat { args } if args.first().is_some_and(|a| SECURE_CHAT_CMDS.contains(&a.as_str())) => { let dm = DmCli::try_parse_from(std::iter::once("c0mpute dm".to_string()).chain(args)) .unwrap_or_else(|e| e.exit()); - run_chat(dm.cmd) + run_chat(dm.cmd).await } Cmd::Chat { args } => ai_chat::run(&args).await, Cmd::Tui { args } => run_tui(&args), @@ -1625,6 +1662,7 @@ async fn run_worker(cmd: WorkerCmd, config_path: &std::path::Path) -> Result<()> Role::Embed, Role::LiveStream, Role::Render, + Role::Mailbox, ] { if !cfg.roles.contains(&role) { cfg.roles.push(role); @@ -3856,124 +3894,6 @@ async fn run_jobs(cmd: JobsCmd) -> Result<()> { // secure-chat plugin (in-process) // ──────────────────────────────────────────────────────────────────────── -fn run_chat(cmd: ChatCmd) -> Result<()> { - match cmd { - ChatCmd::Keygen => { - let path = chat::key_file_path()?; - if path.exists() { - eprintln!( - "A keyfile already exists at {}.\n\ - Use `c0mpute dm key rotate` to generate a new one, or\n\ - delete the file manually to start fresh.", - path.display() - ); - anyhow::bail!("keyfile already exists"); - } - let key = chat::ChatKey::generate(); - let password = prompt_new_password()?; - let kf = key.encrypt_to_keyfile(&password, None)?; - chat::save_keyfile(&path, &kf)?; - - println!("keypair generated"); - println!(" enc pubkey : {}", kf.pubkey_enc); - println!(" sig pubkey : {}", kf.pubkey_sig); - println!(" fingerprint : {}", key.fingerprint()); - println!(" keyfile : {}", path.display()); - println!(); - println!("── encrypted backup (save this somewhere safe) ──"); - println!("{}", serde_json::to_string_pretty(&kf)?); - println!("─────────────────────────────────────────────────"); - println!(); - println!("restore with: c0mpute dm restore --from-backup <file>"); - Ok(()) - } - - ChatCmd::Key { cmd } => match cmd { - KeyCmd::Show => { - let path = chat::key_file_path()?; - let kf = chat::load_keyfile(&path)?; - println!("enc pubkey : {}", kf.pubkey_enc); - println!("sig pubkey : {}", kf.pubkey_sig); - if let Some(did) = &kf.did { - println!("did : {did}"); - } - // Fingerprint needs decryption — show pubkey hash instead - use base64::prelude::*; - let enc_bytes = BASE64_URL_SAFE_NO_PAD.decode(&kf.pubkey_enc)?; - let fingerprint: String = enc_bytes[..8] - .iter() - .map(|b| format!("{b:02x}")) - .collect::<Vec<_>>() - .join(":"); - println!("fingerprint : {fingerprint}"); - Ok(()) - } - KeyCmd::Export => { - let path = chat::key_file_path()?; - let kf = chat::load_keyfile(&path)?; - println!("{}", serde_json::to_string_pretty(&kf)?); - Ok(()) - } - KeyCmd::Rotate => { - println!("[v0.2] key rotation requires DHT re-announcement (not yet live)"); - println!(" Generate a new keypair manually:"); - println!(" 1. Delete ~/.config/c0mpute/chat.key"); - println!(" 2. Run: c0mpute dm keygen"); - Ok(()) - } - }, - - ChatCmd::Restore { from_backup } => { - let dest = chat::key_file_path()?; - if dest.exists() { - eprintln!( - "A keyfile already exists at {}. Remove it first.", - dest.display() - ); - anyhow::bail!("keyfile already exists"); - } - let raw = std::fs::read_to_string(&from_backup)?; - let kf: chat::KeyFile = serde_json::from_str(&raw)?; - let password = rpassword::prompt_password("Password: ")?; - // Verify the password decrypts correctly before saving. - chat::decrypt_keyfile(&kf, &password)?; - chat::save_keyfile(&dest, &kf)?; - println!("keyfile restored to {}", dest.display()); - Ok(()) - } - - ChatCmd::Send { - to, - message: _, - sealed: _, - } => { - println!("[v0.2] direct send to {to} requires p2p transport (not yet live)"); - println!(" DHT key lookup and gossip relay land in v0.2."); - Ok(()) - } - - ChatCmd::Pull => { - println!("[v0.2] pull requires relay node support (not yet live)"); - Ok(()) - } - - ChatCmd::Lookup { did } => { - println!("[v0.2] DHT lookup for {did} not yet live"); - Ok(()) - } - - ChatCmd::Contacts => { - println!("[v0.2] contact list not yet implemented"); - Ok(()) - } - - ChatCmd::Block { did } => { - println!("[v0.2] block list for {did} not yet implemented"); - Ok(()) - } - } -} - fn prompt_new_password() -> Result<String> { loop { let pw = rpassword::prompt_password("New password (min 10 chars): ")?; @@ -4217,6 +4137,7 @@ fn parse_role(s: &str) -> Option<Role> { "gameserver" | "gameservers" | "games" => Some(Role::Gameserver), "live-stream" | "live_stream" | "livestream" | "live" | "stream" => Some(Role::LiveStream), "render" | "blender" => Some(Role::Render), + "mailbox" | "dm" | "secure-chat" | "chat-mailbox" => Some(Role::Mailbox), other => { eprintln!("warning: unknown role '{other}' ignored"); None diff --git a/node/crates/c0mpute-core/Cargo.toml b/node/crates/c0mpute-core/Cargo.toml index 2d6e5ddd..21ffe240 100644 --- a/node/crates/c0mpute-core/Cargo.toml +++ b/node/crates/c0mpute-core/Cargo.toml @@ -16,6 +16,7 @@ c0mpute-update = { workspace = true } c0mpute-doctor = { workspace = true } c0mpute-bench = { workspace = true } c0mpute-api = { workspace = true } +c0mpute-secure-chat = { workspace = true } axum = { workspace = true } tower-http = { workspace = true } rand = { workspace = true } diff --git a/node/crates/c0mpute-core/src/capabilities.rs b/node/crates/c0mpute-core/src/capabilities.rs index 24b5f772..8f71ecfa 100644 --- a/node/crates/c0mpute-core/src/capabilities.rs +++ b/node/crates/c0mpute-core/src/capabilities.rs @@ -109,6 +109,7 @@ pub fn role_tags(config: &Config) -> Vec<String> { } } Role::LiveStream => tags.push(crate::live::ROLE_TAG.to_string()), + Role::Mailbox => tags.push(crate::dm::ROLE_TAG.to_string()), Role::Render => { // Only where `blender` runs. The probe (a few seconds) runs // once at worker start; until it finishes, nothing is shown. diff --git a/node/crates/c0mpute-core/src/config.rs b/node/crates/c0mpute-core/src/config.rs index 03b8ec86..2dde546a 100644 --- a/node/crates/c0mpute-core/src/config.rs +++ b/node/crates/c0mpute-core/src/config.rs @@ -465,6 +465,7 @@ impl Default for Config { Role::Embed, Role::LiveStream, Role::Render, + Role::Mailbox, ], update_channel: default_update_channel(), update_auto: default_auto_update(), diff --git a/node/crates/c0mpute-core/src/dm.rs b/node/crates/c0mpute-core/src/dm.rs new file mode 100644 index 00000000..9340eaa8 --- /dev/null +++ b/node/crates/c0mpute-core/src/dm.rs @@ -0,0 +1,897 @@ +//! Secure-chat transport (DIP-0018): mailboxes, direct delivery and key +//! lookup over gossip plus `/c0mpute/rpc/1.0.0`. No coordinator anywhere. +//! +//! * Every mailbox node (the `mailbox` role, on by default; gateways too) +//! gossips "mailbox here" on [`TOPIC`] once a minute. From that every node +//! knows the mailbox set. +//! * A sender seals the message ([`c0mpute_secure_chat::sealed`]) and first +//! looks for the recipient's home node: a [`HomeRecord`] signed by the +//! recipient naming a libp2p peer, gossiped by that peer while it is up. +//! If the home answers, the message is delivered directly into its spool, +//! where the recipient's local `c0mpute dm` reads it. +//! * Otherwise the sealed message goes to the [`STORE_REPLICAS`] mailbox +//! nodes that rank highest for its recipient tag (rendezvous hashing, the +//! same on every node). They keep it up to 30 days, within size, count and +//! per-sender rate caps. The recipient polls the top [`PULL_REPLICAS`] for +//! each of its tags, opens what it finds, and acks; the ack reveals a +//! secret only the recipient could read, which deletes the message. +//! * Key lookup: users publish a self-signed [`KeyRecord`]; mailbox nodes +//! cache it (gossip plus a direct copy to the nodes ranked for its +//! fingerprint). A lookup by fingerprint or address asks those nodes and +//! checks the answer against what was asked: the address hashes every +//! public key, so a node cannot substitute another key. +//! +//! Mailbox nodes see recipient tags, sizes and the peer that handed them a +//! message; never sender identities, recipient addresses or content. + +use std::collections::{HashMap, VecDeque}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, anyhow, bail}; +use base64::prelude::*; +use c0mpute_net::{GossipMessage, Libp2pNetwork, PeerId, RpcHandler}; +use c0mpute_secure_chat::identity::{matches_query, normalize_query}; +use c0mpute_secure_chat::mailbox::{Limits, Mailbox, Put}; +use c0mpute_secure_chat::placement::{PULL_REPLICAS, STORE_REPLICAS, place, rank}; +use c0mpute_secure_chat::sealed::{Sealed, SealedMessage, is_hex32, pull_tags}; +use c0mpute_secure_chat::{HomeRecord, KeyRecord, PublicIdentity, unix_now}; +use serde::{Deserialize, Serialize}; +use tokio::sync::RwLock; +use tracing::{debug, info}; + +pub const TOPIC: &str = "c0mpute/dm/v1"; +pub const RPC_SERVICE: &str = "dm/1"; +pub const ROLE_TAG: &str = "c0mpute:role:mailbox"; +/// Key records a node caches. +const MAX_KEYS: usize = 20_000; +/// Tags per fetch request. +const MAX_FETCH_TAGS: usize = 16; +/// Direct deliveries accepted per sending peer per minute. +const DELIVER_PER_MIN: usize = 60; + +/// How often things happen. Tests shrink these. +#[derive(Clone, Debug)] +pub struct Timing { + pub announce_every: Duration, + /// A mailbox or home not heard from for this long is gone. + pub peer_ttl: Duration, + pub rpc_timeout: Duration, +} + +impl Default for Timing { + fn default() -> Self { + Self { + announce_every: Duration::from_secs(60), + peer_ttl: Duration::from_secs(200), + rpc_timeout: Duration::from_secs(20), + } + } +} + +/// Where a node keeps its secure-chat files, under its storage root. +#[derive(Clone, Debug)] +pub struct Paths { + /// Other people's sealed messages (the mailbox role). + pub mailbox: PathBuf, + /// Direct deliveries for the users whose home this node is. + pub spool: PathBuf, + /// Signed home records (`<address>.json`) the local `c0mpute dm` wrote. + pub homes: PathBuf, +} + +impl Paths { + pub fn under(root: &Path) -> Self { + let d = root.join("dm"); + Self { + mailbox: d.join("mailbox"), + spool: d.join("spool"), + homes: d.join("homes"), + } + } +} + +pub struct Options { + /// Hold messages for others (the mailbox role). + pub serve_mailbox: bool, + /// Persist under these paths; `None` = memory only, no home/spool. + pub paths: Option<Paths>, + pub limits: Limits, + pub timing: Timing, +} + +impl Default for Options { + fn default() -> Self { + Self { + serve_mailbox: false, + paths: None, + limits: Limits::default(), + timing: Timing::default(), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "t", rename_all = "snake_case")] +enum Gossip { + Mailbox { addrs: Vec<String> }, + Key { record: KeyRecord }, + Home { + record: HomeRecord, + /// Dial hints for the home peer (unsigned: the peer id in the + /// signed record is what authenticates the connection). + #[serde(default)] + addrs: Vec<String>, + }, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "t", rename_all = "snake_case")] +pub enum Request { + Store { msg: String, ack_hash: String, ttl: u64 }, + Fetch { tags: Vec<String> }, + Ack { tag: String, id: String, secret: String }, + Deliver { msg: String, ack_hash: String, ttl: u64 }, + Lookup { query: String }, + PublishKey { record: KeyRecord }, + Presence { address: String }, + Peers, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Item { + pub tag: String, + pub id: String, + pub msg: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct PeerInfo { + pub peer: String, + pub addrs: Vec<String>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "t", rename_all = "snake_case")] +pub enum Response { + Stored { duplicate: bool }, + Messages { items: Vec<Item> }, + Acked { deleted: bool }, + Delivered, + Key { record: Option<KeyRecord> }, + Presence { + record: Option<HomeRecord>, + #[serde(default)] + addrs: Vec<String>, + }, + Peers { peers: Vec<PeerInfo> }, + Ok, +} + +/// One message found by [`Dm::fetch`], with the mailboxes holding it. +#[derive(Clone, Debug)] +pub struct Fetched { + pub tag: String, + pub id: String, + pub msg: SealedMessage, + pub holders: Vec<String>, +} + +/// What [`Dm::send`] did. +#[derive(Clone, Debug, Default)] +pub struct SendReport { + /// The recipient's home node, when it took the message directly. + pub direct: Option<String>, + /// Mailboxes that stored it (empty after a direct delivery). + pub stored_on: Vec<String>, + /// Why direct delivery or individual mailboxes failed. + pub errors: Vec<String>, +} + +struct Inner { + net: Arc<Libp2pNetwork>, + me: String, + serves: bool, + limits: Limits, + timing: Timing, + paths: Option<Paths>, + mailbox: Mutex<Mailbox>, + /// Mailbox peers: last heard, dial addresses. + mailboxes: RwLock<HashMap<String, (Instant, Vec<String>)>>, + keys: RwLock<HashMap<String, KeyRecord>>, + /// Remote presence: address -> (record, last gossiped by its peer). + homes: RwLock<HashMap<String, (HomeRecord, Instant, Vec<String>)>>, + deliver_rate: Mutex<HashMap<String, VecDeque<u64>>>, + stopped: std::sync::atomic::AtomicBool, +} + +/// The secure-chat service: one per node (or per `c0mpute dm` client). +#[derive(Clone)] +pub struct Dm(Arc<Inner>); + +struct Handler(Dm); + +#[async_trait::async_trait] +impl RpcHandler for Handler { + async fn handle(&self, peer: PeerId, body: Vec<u8>) -> std::result::Result<Vec<u8>, String> { + let req: Request = serde_json::from_slice(&body).map_err(|e| format!("bad request: {e}"))?; + let resp = self.0.serve(&peer.to_base58(), req).await.map_err(|e| e.to_string())?; + serde_json::to_vec(&resp).map_err(|e| e.to_string()) + } +} + +fn b64d(s: &str) -> Result<Vec<u8>> { + BASE64_URL_SAFE_NO_PAD.decode(s).context("bad base64") +} + +fn b64e(b: &[u8]) -> String { + BASE64_URL_SAFE_NO_PAD.encode(b) +} + +impl Dm { + pub async fn start(net: Arc<Libp2pNetwork>, opts: Options) -> Result<Self> { + let now = unix_now(); + let mailbox = match (&opts.paths, opts.serve_mailbox) { + (Some(p), true) => Mailbox::open(&p.mailbox, opts.limits.clone(), now), + _ => Mailbox::new(opts.limits.clone()), + }; + let dm = Dm(Arc::new(Inner { + me: net.peer_id().to_base58(), + net: net.clone(), + serves: opts.serve_mailbox, + limits: opts.limits, + timing: opts.timing, + paths: opts.paths, + mailbox: Mutex::new(mailbox), + mailboxes: RwLock::new(HashMap::new()), + keys: RwLock::new(HashMap::new()), + homes: RwLock::new(HashMap::new()), + deliver_rate: Mutex::new(HashMap::new()), + stopped: std::sync::atomic::AtomicBool::new(false), + })); + net.subscribe(TOPIC).await.context("subscribe to dm topic")?; + net.register_rpc(RPC_SERVICE, Arc::new(Handler(dm.clone()))); + dm.spawn_gossip_reader(); + dm.spawn_announcer(); + if dm.0.serves { + info!("secure-chat: serving as a mailbox"); + } + Ok(dm) + } + + pub fn peer_id(&self) -> &str { + &self.0.me + } + + pub fn serves_mailbox(&self) -> bool { + self.0.serves + } + + pub fn stop(&self) { + self.0.stopped.store(true, std::sync::atomic::Ordering::Relaxed); + } + + /// Messages this node holds for others. + pub fn mailbox_count(&self) -> usize { + self.0.mailbox.lock().map(|m| m.count()).unwrap_or(0) + } + + // ── gossip ─────────────────────────────────────────────────────────── + + fn spawn_gossip_reader(&self) { + let dm = self.clone(); + let mut rx = self.0.net.messages(); + tokio::spawn(async move { + loop { + let msg: GossipMessage = match rx.recv().await { + Ok(m) => m, + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, + Err(_) => return, + }; + if msg.topic != TOPIC { + continue; + } + let Some(source) = msg.source else { continue }; + let Ok(g) = serde_json::from_slice::<Gossip>(&msg.data) else { + continue; + }; + dm.apply_gossip(source, g).await; + } + }); + } + + async fn apply_gossip(&self, source: PeerId, g: Gossip) { + let src = source.to_base58(); + let now = unix_now(); + match g { + Gossip::Mailbox { addrs } => { + let addrs: Vec<String> = addrs.into_iter().take(8).collect(); + for a in &addrs { + if let Ok(ma) = a.parse::<c0mpute_net::Multiaddr>() { + self.0.net.add_peer_address(source, ma).await; + } + } + self.0.mailboxes.write().await.insert(src, (Instant::now(), addrs)); + } + Gossip::Key { record } => { + if record.verify(now).is_ok() { + self.cache_key(record).await; + } + } + Gossip::Home { record, addrs } => { + // Presence counts only from the named peer itself. + if record.peer == src && record.verify(now).is_ok() { + self.learn_home(record, addrs).await; + } + } + } + } + + /// Remember a verified home record and teach libp2p how to dial its peer. + async fn learn_home(&self, record: HomeRecord, addrs: Vec<String>) { + let addrs: Vec<String> = addrs.into_iter().take(8).collect(); + if let Ok(pid) = record.peer.parse::<PeerId>() { + for a in &addrs { + if let Ok(ma) = a.parse::<c0mpute_net::Multiaddr>() { + self.0.net.add_peer_address(pid, ma).await; + } + } + } + let addr = record.address(); + self.0.homes.write().await.insert(addr, (record, Instant::now(), addrs)); + } + + async fn cache_key(&self, record: KeyRecord) { + let addr = record.address(); + let mut keys = self.0.keys.write().await; + if let Some(cur) = keys.get(&addr) + && cur.created_at >= record.created_at + { + return; + } + if keys.len() >= MAX_KEYS && !keys.contains_key(&addr) { + let now = unix_now(); + keys.retain(|_, r| r.expires_at > now); + if keys.len() >= MAX_KEYS + && let Some(oldest) = keys.iter().min_by_key(|(_, r)| r.expires_at).map(|(a, _)| a.clone()) + { + keys.remove(&oldest); + } + } + keys.insert(addr, record); + } + + async fn publish(&self, g: &Gossip) { + if let Ok(bytes) = serde_json::to_vec(g) + && let Err(e) = self.0.net.publish(TOPIC, bytes).await + { + debug!(err = %e, "dm: publish (mesh forming?)"); + } + } + + async fn my_addrs(&self) -> Vec<String> { + self.0 + .net + .advertised_addrs() + .await + .into_iter() + .map(|a| a.to_string()) + .filter(|a| !a.contains("/ip4/0.0.0.0") && !a.contains("/ip6/::/")) + .take(8) + .collect() + } + + /// Home records in `<homes>/` naming this node, verified and live. + pub fn local_homes(&self) -> Vec<HomeRecord> { + let Some(p) = &self.0.paths else { return Vec::new() }; + let now = unix_now(); + let Ok(rd) = std::fs::read_dir(&p.homes) else { return Vec::new() }; + rd.flatten() + .filter_map(|e| std::fs::read(e.path()).ok()) + .filter_map(|b| serde_json::from_slice::<HomeRecord>(&b).ok()) + .filter(|r| r.peer == self.0.me && r.verify(now).is_ok()) + .collect() + } + + /// Make this node the home of `record`'s address (writes it under + /// `homes/`, then gossips it). The `c0mpute dm` client does the same by + /// writing the file; the announcer picks it up. + pub async fn add_local_home(&self, record: &HomeRecord) -> Result<()> { + let p = self.0.paths.as_ref().ok_or_else(|| anyhow!("no data dir for home records"))?; + anyhow::ensure!(record.peer == self.0.me, "home record names another peer"); + record.verify(unix_now())?; + write_home_record(&p.homes, record)?; + let addrs = self.my_addrs().await; + self.publish(&Gossip::Home { + record: record.clone(), + addrs, + }) + .await; + Ok(()) + } + + /// Gossip once: mailbox presence and the homes this node serves. + pub async fn announce(&self) { + if self.0.serves { + let addrs = self.my_addrs().await; + self.0 + .mailboxes + .write() + .await + .insert(self.0.me.clone(), (Instant::now(), addrs.clone())); + self.publish(&Gossip::Mailbox { addrs }).await; + } + let homes = self.local_homes(); + if !homes.is_empty() { + let addrs = self.my_addrs().await; + for record in homes { + self.publish(&Gossip::Home { + record, + addrs: addrs.clone(), + }) + .await; + } + } + } + + fn spawn_announcer(&self) { + let dm = self.clone(); + tokio::spawn(async move { + loop { + if dm.0.stopped.load(std::sync::atomic::Ordering::Relaxed) { + return; + } + dm.announce().await; + let ttl = dm.0.timing.peer_ttl; + dm.0.mailboxes + .write() + .await + .retain(|_, (t, _)| t.elapsed() <= ttl); + dm.0.homes.write().await.retain(|_, (r, t, _)| t.elapsed() <= ttl && r.expires_at > unix_now()); + if let Ok(mut mb) = dm.0.mailbox.lock() { + mb.prune(unix_now()); + } + tokio::time::sleep(crate::hosting::jitter(dm.0.timing.announce_every)).await; + } + }); + } + + // ── serving ────────────────────────────────────────────────────────── + + async fn serve(&self, peer: &str, req: Request) -> Result<Response> { + let now = unix_now(); + match req { + Request::Store { msg, ack_hash, ttl } => { + if !self.0.serves { + bail!("not a mailbox"); + } + let bytes = b64d(&msg)?; + let r = self + .0 + .mailbox + .lock() + .map_err(|_| anyhow!("mailbox lock"))? + .put(peer, bytes, &ack_hash, ttl, now)?; + Ok(Response::Stored { + duplicate: r == Put::Duplicate, + }) + } + Request::Fetch { tags } => { + let mb = self.0.mailbox.lock().map_err(|_| anyhow!("mailbox lock"))?; + let mut items = Vec::new(); + let mut budget = self.0.limits.max_fetch_bytes; + for tag in tags.into_iter().take(MAX_FETCH_TAGS) { + for (id, bytes) in mb.fetch(&tag, now) { + if bytes.len() > budget && !items.is_empty() { + break; + } + budget = budget.saturating_sub(bytes.len()); + items.push(Item { + tag: tag.clone(), + id, + msg: b64e(&bytes), + }); + } + } + Ok(Response::Messages { items }) + } + Request::Ack { tag, id, secret } => { + let deleted = self + .0 + .mailbox + .lock() + .map_err(|_| anyhow!("mailbox lock"))? + .ack(&tag, &id, &secret); + Ok(Response::Acked { deleted }) + } + Request::Deliver { msg, ack_hash, ttl } => { + let p = self.0.paths.as_ref().ok_or_else(|| anyhow!("not a home node"))?; + let bytes = b64d(&msg)?; + let sealed = SealedMessage::parse(&bytes)?; + let mine = self + .local_homes() + .iter() + .any(|h| pull_tags(&h.address(), now).contains(&sealed.tag)); + if !mine { + bail!("no user here for that tag"); + } + { + let mut rate = self.0.deliver_rate.lock().map_err(|_| anyhow!("rate lock"))?; + let w = rate.entry(peer.to_string()).or_default(); + while w.front().is_some_and(|t| *t + 60 <= now) { + w.pop_front(); + } + if w.len() >= DELIVER_PER_MIN { + bail!("rate limited"); + } + w.push_back(now); + } + // The spool is shared with the local client process, which + // deletes what it has read: re-read it each time. + let mut spool = Mailbox::open(&p.spool, self.0.limits.clone(), now); + spool.put("direct", bytes, &ack_hash, ttl, now).map_err(|e| anyhow!("{e}"))?; + Ok(Response::Delivered) + } + Request::Lookup { query } => { + let q = normalize_query(&query).ok_or_else(|| anyhow!("bad query"))?; + let record = self + .0 + .keys + .read() + .await + .values() + .find(|r| matches_query(&r.identity, &q) && r.expires_at > now) + .cloned(); + Ok(Response::Key { record }) + } + Request::PublishKey { record } => { + record.verify(now)?; + self.cache_key(record).await; + Ok(Response::Ok) + } + Request::Presence { address } => { + let ttl = self.0.timing.peer_ttl; + let known = self + .0 + .homes + .read() + .await + .get(&address) + .filter(|(r, t, _)| t.elapsed() <= ttl && r.expires_at > now) + .map(|(r, _, a)| (r.clone(), a.clone())); + let (record, addrs) = match known { + Some((r, a)) => (Some(r), a), + None => match self.local_homes().into_iter().find(|h| h.address() == address) { + Some(r) => (Some(r), self.my_addrs().await), + None => (None, Vec::new()), + }, + }; + Ok(Response::Presence { record, addrs }) + } + Request::Peers => { + let ttl = self.0.timing.peer_ttl; + let mut peers: Vec<PeerInfo> = self + .0 + .mailboxes + .read() + .await + .iter() + .filter(|(_, (t, _))| t.elapsed() <= ttl) + .map(|(p, (_, a))| PeerInfo { + peer: p.clone(), + addrs: a.clone(), + }) + .take(256) + .collect(); + if self.0.serves && !peers.iter().any(|p| p.peer == self.0.me) { + peers.push(PeerInfo { + peer: self.0.me.clone(), + addrs: self.my_addrs().await, + }); + } + Ok(Response::Peers { peers }) + } + } + } + + // ── client ─────────────────────────────────────────────────────────── + + async fn call(&self, peer: &str, req: &Request) -> Result<Response> { + let pid: PeerId = peer.parse().map_err(|_| anyhow!("bad peer id {peer}"))?; + let body = serde_json::to_vec(req)?; + let out = tokio::time::timeout(self.0.timing.rpc_timeout, self.0.net.rpc(pid, RPC_SERVICE, body)) + .await + .map_err(|_| anyhow!("{peer}: timed out"))??; + Ok(serde_json::from_slice(&out)?) + } + + /// Live mailbox peers this node knows, itself included when it serves. + pub async fn mailboxes(&self) -> Vec<String> { + let ttl = self.0.timing.peer_ttl; + let mut v: Vec<String> = self + .0 + .mailboxes + .read() + .await + .iter() + .filter(|(_, (t, _))| t.elapsed() <= ttl) + .map(|(p, _)| p.clone()) + .collect(); + if self.0.serves && !v.contains(&self.0.me) { + v.push(self.0.me.clone()); + } + v + } + + /// Ask every connected peer for the mailboxes it knows (a client that + /// just joined learns the set without waiting for a gossip round). + pub async fn discover(&self) { + for pid in self.0.net.connected_peers().await { + let peer = pid.to_base58(); + if let Ok(Response::Peers { peers }) = self.call(&peer, &Request::Peers).await { + for p in peers.into_iter().take(256) { + let Ok(id) = p.peer.parse::<PeerId>() else { continue }; + for a in p.addrs.iter().take(8) { + if let Ok(ma) = a.parse::<c0mpute_net::Multiaddr>() { + self.0.net.add_peer_address(id, ma).await; + } + } + self.0 + .mailboxes + .write() + .await + .entry(p.peer) + .or_insert((Instant::now(), p.addrs)); + } + } + } + } + + /// Discover until at least `min` mailboxes are known or `wait` passes. + pub async fn wait_for_mailboxes(&self, min: usize, wait: Duration) -> usize { + let deadline = Instant::now() + wait; + loop { + let n = self.mailboxes().await.len(); + if n >= min || Instant::now() >= deadline { + return n; + } + self.discover().await; + let n = self.mailboxes().await.len(); + if n >= min { + return n; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + } + + /// The recipient's live home record, if any node has heard it lately. + pub async fn find_home(&self, address: &str) -> Option<HomeRecord> { + let now = unix_now(); + let ttl = self.0.timing.peer_ttl; + if let Some((r, t, _)) = self.0.homes.read().await.get(address) + && t.elapsed() <= ttl + && r.expires_at > now + { + return Some(r.clone()); + } + let peers = self.mailboxes().await; + for peer in place(address, &peers, PULL_REPLICAS) { + if peer == self.0.me { + continue; + } + if let Ok(Response::Presence { record: Some(r), addrs }) = self + .call( + &peer, + &Request::Presence { + address: address.to_string(), + }, + ) + .await + && r.address() == address + && r.verify(now).is_ok() + { + self.learn_home(r.clone(), addrs).await; + return Some(r); + } + } + None + } + + /// Deliver a sealed message: directly to the recipient's home node when + /// it is up, otherwise onto [`STORE_REPLICAS`] mailboxes. + pub async fn send(&self, sealed: &Sealed, recipient_address: &str, ttl: u64) -> Result<SendReport> { + let mut report = SendReport::default(); + let msg = b64e(&sealed.bytes); + if let Some(home) = self.find_home(recipient_address).await { + let req = Request::Deliver { + msg: msg.clone(), + ack_hash: sealed.ack_hash.clone(), + ttl, + }; + match self.call(&home.peer, &req).await { + Ok(Response::Delivered) => { + report.direct = Some(home.peer); + return Ok(report); + } + Ok(other) => report.errors.push(format!("home {}: unexpected {other:?}", home.peer)), + Err(e) => report.errors.push(format!("home {}: {e}", home.peer)), + } + } + let peers = self.mailboxes().await; + if peers.is_empty() { + bail!("no mailbox nodes known, and the recipient's home node is not reachable"); + } + let req = Request::Store { + msg, + ack_hash: sealed.ack_hash.clone(), + ttl, + }; + for peer in rank(&sealed.msg.tag, &peers) { + if report.stored_on.len() >= STORE_REPLICAS { + break; + } + match self.call(&peer, &req).await { + Ok(Response::Stored { .. }) => report.stored_on.push(peer), + Ok(other) => report.errors.push(format!("{peer}: unexpected {other:?}")), + Err(e) => report.errors.push(format!("{peer}: {e}")), + } + } + if report.stored_on.is_empty() { + bail!("no mailbox accepted the message: {}", report.errors.join("; ")); + } + Ok(report) + } + + /// Everything mailboxes hold for `address` (by its tags), deduplicated. + pub async fn fetch(&self, address: &str) -> Vec<Fetched> { + let tags = pull_tags(address, unix_now()); + let peers = self.mailboxes().await; + let mut by_peer: HashMap<String, Vec<String>> = HashMap::new(); + for tag in &tags { + for p in place(tag, &peers, PULL_REPLICAS) { + by_peer.entry(p).or_default().push(tag.clone()); + } + } + let mut found: HashMap<String, Fetched> = HashMap::new(); + for (peer, tags) in by_peer { + let Ok(Response::Messages { items }) = self.call(&peer, &Request::Fetch { tags: tags.clone() }).await else { + continue; + }; + for it in items { + if !tags.contains(&it.tag) || !is_hex32(&it.id) { + continue; + } + let Ok(bytes) = b64d(&it.msg) else { continue }; + if c0mpute_secure_chat::sealed::sha256_hex(&bytes) != it.id { + continue; + } + let Ok(msg) = SealedMessage::parse(&bytes) else { continue }; + if msg.tag != it.tag { + continue; + } + found + .entry(it.id.clone()) + .or_insert_with(|| Fetched { + tag: it.tag, + id: it.id, + msg, + holders: Vec::new(), + }) + .holders + .push(peer.clone()); + } + } + found.into_values().collect() + } + + /// Delete a fetched message from its mailboxes. Returns how many did. + pub async fn ack(&self, f: &Fetched, secret: &str) -> usize { + let req = Request::Ack { + tag: f.tag.clone(), + id: f.id.clone(), + secret: secret.to_string(), + }; + let mut n = 0; + for peer in &f.holders { + if let Ok(Response::Acked { deleted: true }) = self.call(peer, &req).await { + n += 1; + } + } + n + } + + /// Publish a key record: cache, gossip, and hand it to the mailboxes + /// ranked for its fingerprint (where lookups ask first). + pub async fn publish_key(&self, record: &KeyRecord) -> Result<usize> { + record.verify(unix_now())?; + self.cache_key(record.clone()).await; + self.publish(&Gossip::Key { record: record.clone() }).await; + let fp = record.address()[..16].to_string(); + let peers = self.mailboxes().await; + let req = Request::PublishKey { record: record.clone() }; + let mut n = 0; + for peer in place(&fp, &peers, STORE_REPLICAS) { + if peer == self.0.me || matches!(self.call(&peer, &req).await, Ok(Response::Ok)) { + n += 1; + } + } + Ok(n) + } + + /// Resolve a fingerprint or address to a verified public identity. + pub async fn lookup(&self, query: &str) -> Option<PublicIdentity> { + let q = normalize_query(query)?; + let now = unix_now(); + if let Some(r) = self + .0 + .keys + .read() + .await + .values() + .find(|r| matches_query(&r.identity, &q) && r.expires_at > now) + { + return Some(r.identity.clone()); + } + let peers = self.mailboxes().await; + let ranked = rank(&q[..16], &peers); + for peer in ranked.into_iter().take(8) { + if peer == self.0.me { + continue; + } + if let Ok(Response::Key { record: Some(r) }) = self.call(&peer, &Request::Lookup { query: q.clone() }).await + && matches_query(&r.identity, &q) + && r.verify(now).is_ok() + { + let id = r.identity.clone(); + self.cache_key(r).await; + return Some(id); + } + } + None + } +} + +/// Write a home record where the node's announcer reads it. +pub fn write_home_record(dir: &Path, record: &HomeRecord) -> Result<()> { + std::fs::create_dir_all(dir)?; + let path = dir.join(format!("{}.json", record.address())); + let tmp = path.with_extension("tmp"); + std::fs::write(&tmp, serde_json::to_vec_pretty(record)?)?; + std::fs::rename(&tmp, &path)?; + Ok(()) +} + +/// The worker's libp2p peer id on this machine (its identity lives in the +/// config dir), if it has ever run. +pub fn worker_peer_id(config_dir: &Path) -> Option<String> { + c0mpute_net::identity::existing_peer_id(config_dir).map(|p| p.to_base58()) +} + +/// A bare libp2p node for the `c0mpute dm` client: its own identity, the +/// usual bootstrap list (plus mDNS for a local worker), no chunk store. +pub async fn boot_client(identity_dir: &Path) -> Result<Arc<Libp2pNetwork>> { + std::fs::create_dir_all(identity_dir)?; + let bootstrap = c0mpute_net::bootstrap::fetch_or_fallback( + c0mpute_net::bootstrap::DEFAULT_BOOTSTRAP_URL, + c0mpute_net::bootstrap::default_seeds(), + ) + .await; + let cfg = c0mpute_net::NetworkConfig::for_dir(identity_dir).with_bootstrap(bootstrap); + Ok(Arc::new(Libp2pNetwork::spawn(cfg).await?)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wire_formats_are_tagged() { + let r = serde_json::to_string(&Request::Fetch { tags: vec!["a".into()] }).unwrap(); + assert!(r.contains("\"t\":\"fetch\"")); + let p: Response = serde_json::from_str("{\"t\":\"delivered\"}").unwrap(); + assert!(matches!(p, Response::Delivered)); + let paths = Paths::under(Path::new("/x")); + assert_eq!(paths.spool, Path::new("/x/dm/spool")); + } +} diff --git a/node/crates/c0mpute-core/src/hosting/mod.rs b/node/crates/c0mpute-core/src/hosting/mod.rs index f2f60433..596fa351 100644 --- a/node/crates/c0mpute-core/src/hosting/mod.rs +++ b/node/crates/c0mpute-core/src/hosting/mod.rs @@ -1103,7 +1103,7 @@ impl Hosting { } } -fn jitter(d: Duration) -> Duration { +pub(crate) fn jitter(d: Duration) -> Duration { let ms = d.as_millis() as u64; let spread = ms / 10; let nanos = std::time::SystemTime::now() diff --git a/node/crates/c0mpute-core/src/lib.rs b/node/crates/c0mpute-core/src/lib.rs index 44bdcbbb..7d899e01 100644 --- a/node/crates/c0mpute-core/src/lib.rs +++ b/node/crates/c0mpute-core/src/lib.rs @@ -10,6 +10,7 @@ pub mod capabilities; pub mod config; pub mod crawl; pub mod diffusion; +pub mod dm; pub mod music; pub mod hosting; pub mod live; diff --git a/node/crates/c0mpute-core/src/supervisor.rs b/node/crates/c0mpute-core/src/supervisor.rs index b56efc18..b9cdd31f 100644 --- a/node/crates/c0mpute-core/src/supervisor.rs +++ b/node/crates/c0mpute-core/src/supervisor.rs @@ -468,6 +468,25 @@ impl Supervisor { None }; + // Secure chat: mailbox nodes hold sealed messages for offline + // recipients; any node (gateways included) receives direct messages + // for this machine's own `c0mpute dm` user once it has a home record. + let roles = &self.config.roles; + let dm_mailbox = roles.contains(&c0mpute_proto::Role::Mailbox) || roles.contains(&c0mpute_proto::Role::Gateway); + match crate::dm::Dm::start( + self.libp2p.clone(), + crate::dm::Options { + serve_mailbox: dm_mailbox, + paths: Some(crate::dm::Paths::under(&self.config.storage.root)), + ..Default::default() + }, + ) + .await + { + Ok(_) => {} + Err(e) => tracing::warn!(err = %e, "secure-chat: not started"), + } + // Ads: every node that runs hosting keeps the campaign index and // relays c0mpute/ads/v1 (gossipsub forwards only through subscribed // peers); only gateways answer ad requests. diff --git a/node/crates/c0mpute-core/tests/dm_e2e.rs b/node/crates/c0mpute-core/tests/dm_e2e.rs new file mode 100644 index 00000000..97fe8967 --- /dev/null +++ b/node/crates/c0mpute-core/tests/dm_e2e.rs @@ -0,0 +1,216 @@ +//! Secure chat end to end on loopback libp2p, no coordinator: +//! +//! 1. Alice sends to Bob while Bob is offline: the sealed message lands on +//! exactly three of four mailbox nodes (rendezvous on Bob's tag). +//! 2. Bob comes online, pulls, decrypts, acks: every copy is deleted. +//! 3. A blocked sender's message is dropped on receipt (and still deleted). +//! 4. Bob looks up Alice by fingerprint from her self-published key record. +//! 5. Bob's home node is up: Alice's next message goes straight into its +//! spool, no mailbox involved, and Bob reads it from there. + +use std::sync::Arc; +use std::time::Duration; + +use c0mpute_core::dm::{Dm, Options, Paths, Timing}; +use c0mpute_net::{Libp2pNetwork, NetworkConfig}; +use c0mpute_secure_chat::mailbox::{Limits, Mailbox}; +use c0mpute_secure_chat::sealed::{pull_tags, seal}; +use c0mpute_secure_chat::state::{LocalState, Received}; +use c0mpute_secure_chat::{ChatKey, HomeRecord, KeyRecord, unix_now}; +use libp2p::multiaddr::Protocol; +use tempfile::tempdir; + +async fn net(dir: &std::path::Path, bootstrap: Vec<libp2p::Multiaddr>) -> Arc<Libp2pNetwork> { + Arc::new( + Libp2pNetwork::spawn( + NetworkConfig::for_dir(dir.join("id")) + .with_listen(vec!["/ip4/127.0.0.1/tcp/0".parse().unwrap()]) + .with_bootstrap(bootstrap), + ) + .await + .unwrap(), + ) +} + +async fn listen_addr(net: &Libp2pNetwork) -> libp2p::Multiaddr { + for _ in 0..50 { + if let Ok(addrs) = net.listen_addrs().await + && let Some(a) = addrs.into_iter().next() + { + return a.with(Protocol::P2p(net.peer_id())); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("no listen addr"); +} + +fn fast() -> Timing { + Timing { + announce_every: Duration::from_millis(500), + peer_ttl: Duration::from_secs(5), + rpc_timeout: Duration::from_secs(10), + } +} + +async fn wait_for<F, Fut>(what: &str, secs: u64, mut f: F) +where + F: FnMut() -> Fut, + Fut: std::future::Future<Output = bool>, +{ + let deadline = tokio::time::Instant::now() + Duration::from_secs(secs); + while tokio::time::Instant::now() < deadline { + if f().await { + return; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + panic!("timed out waiting for {what}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn offline_store_and_forward_then_direct_delivery() { + let _ = tracing_subscriber::fmt() + .with_env_filter("c0mpute_core=info,c0mpute_net=warn") + .with_test_writer() + .try_init(); + let tmp = tempdir().unwrap(); + let opts = |dir: &std::path::Path, serve: bool| Options { + serve_mailbox: serve, + paths: Some(Paths::under(dir)), + timing: fast(), + ..Default::default() + }; + + // Four mailbox nodes around a hub. + let hub_dir = tmp.path().join("m0"); + let hub_net = net(&hub_dir, vec![]).await; + let hub_addr = listen_addr(&hub_net).await; + let mut mailboxes = vec![Dm::start(hub_net, opts(&hub_dir, true)).await.unwrap()]; + for i in 1..4 { + let d = tmp.path().join(format!("m{i}")); + let n = net(&d, vec![hub_addr.clone()]).await; + mailboxes.push(Dm::start(n, opts(&d, true)).await.unwrap()); + } + + // Alice: a client node, no mailbox. + let alice_key = ChatKey::generate(); + let bob_key = ChatKey::generate(); + let ad = tmp.path().join("alice"); + let alice = Dm::start(net(&ad, vec![hub_addr.clone()]).await, opts(&ad, false)).await.unwrap(); + assert_eq!(alice.wait_for_mailboxes(4, Duration::from_secs(20)).await, 4); + // Every mailbox should see the full set before placement is compared. + for m in &mailboxes { + let m = m.clone(); + wait_for("mailbox gossip", 20, || { + let m = m.clone(); + async move { m.mailboxes().await.len() == 4 } + }) + .await; + } + + // 1. Bob is offline (no node at all): store-and-forward. + let now = unix_now(); + let s = seal(&alice_key, &bob_key.public_identity(), "are you there, bob?", 86_400, now).unwrap(); + let rep = alice.send(&s, &bob_key.address(), 86_400).await.unwrap(); + assert!(rep.direct.is_none()); + assert_eq!(rep.stored_on.len(), 3, "{rep:?}"); + let expected = c0mpute_secure_chat::placement::place(&s.msg.tag, &alice.mailboxes().await, 3); + let mut stored: Vec<String> = rep.stored_on.clone(); + stored.sort(); + let mut exp = expected.clone(); + exp.sort(); + assert_eq!(stored, exp, "stored on the rendezvous-ranked nodes"); + let holding: usize = mailboxes.iter().map(|m| m.mailbox_count()).sum(); + assert_eq!(holding, 3); + + // A blocked sender's message too. + let mallory_key = ChatKey::generate(); + let spam = seal(&mallory_key, &bob_key.public_identity(), "spam", 86_400, now).unwrap(); + alice.send(&spam, &bob_key.address(), 86_400).await.unwrap(); + assert_eq!(mailboxes.iter().map(|m| m.mailbox_count()).sum::<usize>(), 6); + + // Alice publishes her key record so others can look her up. + let published = alice + .publish_key(&KeyRecord::sign(&alice_key, now, 86_400).unwrap()) + .await + .unwrap(); + assert!(published >= 1); + + // 2. Bob comes online and pulls. + let bd = tmp.path().join("bob"); + let bob = Dm::start(net(&bd, vec![hub_addr.clone()]).await, opts(&bd, false)).await.unwrap(); + assert_eq!(bob.wait_for_mailboxes(4, Duration::from_secs(20)).await, 4); + let mut state = LocalState::default(); + state.blocked.insert(mallory_key.address()); + let fetched = bob.fetch(&bob_key.address()).await; + assert_eq!(fetched.len(), 2, "both messages found"); + let mut texts = Vec::new(); + let mut blocked = 0; + for f in &fetched { + let got = state.receive(&bob_key, &f.msg, "mailbox", unix_now()).unwrap(); + match &got { + Received::New(o) => texts.push((o.text.clone(), o.sender_address.clone())), + Received::Blocked(_) => blocked += 1, + } + assert_eq!(f.holders.len(), 3); + assert_eq!(bob.ack(f, got.ack()).await, 3, "every copy deleted"); + } + assert_eq!(texts, vec![("are you there, bob?".to_string(), alice_key.address())]); + assert_eq!(blocked, 1); + assert_eq!(mailboxes.iter().map(|m| m.mailbox_count()).sum::<usize>(), 0); + assert!(bob.fetch(&bob_key.address()).await.is_empty()); + // A replayed copy (say a mailbox kept one) is refused. + assert!(state.receive(&bob_key, &fetched[0].msg, "mailbox", unix_now()).is_err()); + + // 3. Lookup by fingerprint: resolved from the key record, verified. + let found = bob.lookup(&alice_key.public_identity().fingerprint()).await.expect("lookup"); + assert_eq!(found.address(), alice_key.address()); + assert!(bob.lookup(&ChatKey::generate().public_identity().fingerprint()).await.is_none()); + + // 4. Bob's home node: a running node on Bob's machine. + let hd = tmp.path().join("bob-home"); + let home = Dm::start(net(&hd, vec![hub_addr.clone()]).await, opts(&hd, false)).await.unwrap(); + let rec = HomeRecord::sign(&bob_key, home.peer_id(), unix_now(), 3600).unwrap(); + home.add_local_home(&rec).await.unwrap(); + // A home record for someone else's peer is refused. + let wrong = HomeRecord::sign(&bob_key, mailboxes[0].peer_id(), unix_now(), 3600).unwrap(); + assert!(home.add_local_home(&wrong).await.is_err()); + { + let alice = alice.clone(); + let addr = bob_key.address(); + wait_for("bob's presence to reach alice", 20, || { + let alice = alice.clone(); + let addr = addr.clone(); + async move { alice.find_home(&addr).await.is_some() } + }) + .await; + } + let s2 = seal(&alice_key, &bob_key.public_identity(), "direct this time", 86_400, unix_now()).unwrap(); + let rep = alice.send(&s2, &bob_key.address(), 86_400).await.unwrap(); + assert_eq!(rep.direct.as_deref(), Some(home.peer_id()), "{rep:?}"); + assert!(rep.stored_on.is_empty()); + assert_eq!(mailboxes.iter().map(|m| m.mailbox_count()).sum::<usize>(), 0); + // Bob's local client reads the home node's spool. + let now = unix_now(); + let mut spool = Mailbox::open(&Paths::under(&hd).spool, Limits::default(), now); + let mut direct = Vec::new(); + for tag in pull_tags(&bob_key.address(), now) { + for (id, bytes) in spool.fetch(&tag, now) { + let msg = c0mpute_secure_chat::SealedMessage::parse(&bytes).unwrap(); + let got = state.receive(&bob_key, &msg, "direct", now).unwrap(); + assert!(spool.ack(&tag, &id, got.ack())); + if let Received::New(o) = got { + direct.push(o.text); + } + } + } + assert_eq!(direct, vec!["direct this time".to_string()]); + assert_eq!(spool.count(), 0); + + // Someone with no home node: back to the mailboxes. + let carol = ChatKey::generate(); + let s3 = seal(&alice_key, &carol.public_identity(), "x", 3600, unix_now()).unwrap(); + let rep = alice.send(&s3, &carol.address(), 3600).await.unwrap(); + assert!(rep.direct.is_none()); + assert_eq!(rep.stored_on.len(), 3); +} diff --git a/node/crates/c0mpute-net/src/lib.rs b/node/crates/c0mpute-net/src/lib.rs index 778ba524..62132efa 100644 --- a/node/crates/c0mpute-net/src/lib.rs +++ b/node/crates/c0mpute-net/src/lib.rs @@ -13,6 +13,7 @@ //! Protocol IDs: //! /c0mpute/kad/1.0.0 — DHT (peer + content discovery) //! /c0mpute/chunk-fetch/1.0.0 — request/response for chunk bytes +//! /c0mpute/rpc/1.0.0 — small named-service request/response pub mod bootstrap; pub mod identity; @@ -28,7 +29,7 @@ use c0mpute_proto::{ChunkRequest, Hash}; pub use libp2p::{Multiaddr, PeerId, multiaddr::Protocol}; /// The libp2p this crate is built on, for identity and key types. pub use libp2p; -pub use swarm::{GossipMessage, Libp2pNetwork, NetworkConfig}; +pub use swarm::{GossipMessage, Libp2pNetwork, NetworkConfig, RpcHandler}; /// Trait that abstracts the underlying network so the rest of the node can be /// developed and tested without booting libp2p. diff --git a/node/crates/c0mpute-net/src/swarm.rs b/node/crates/c0mpute-net/src/swarm.rs index ccdec1cc..56978d00 100644 --- a/node/crates/c0mpute-net/src/swarm.rs +++ b/node/crates/c0mpute-net/src/swarm.rs @@ -5,6 +5,9 @@ //! - kad DHT under `/c0mpute/kad/1.0.0` for peer + content discovery //! - identify + ping for housekeeping //! - request/response under `/c0mpute/chunk-fetch/1.0.0` for chunk bytes +//! - request/response under `/c0mpute/rpc/1.0.0`: small named-service +//! calls between peers (secure-chat mailboxes use it); services register +//! an [`RpcHandler`] with [`Libp2pNetwork::register_rpc`] //! //! The swarm runs on a tokio task; the `Network` trait impl talks to it //! through mpsc channels. @@ -30,6 +33,20 @@ use crate::{ChunkSource, Network, identity as id_mod}; const KAD_PROTOCOL: StreamProtocol = StreamProtocol::new("/c0mpute/kad/1.0.0"); const FETCH_PROTOCOL: StreamProtocol = StreamProtocol::new("/c0mpute/chunk-fetch/1.0.0"); +const RPC_PROTOCOL: StreamProtocol = StreamProtocol::new("/c0mpute/rpc/1.0.0"); +/// Largest RPC request body accepted. +pub const RPC_MAX_REQUEST: u64 = 1024 * 1024; +/// Largest RPC response body accepted. +pub const RPC_MAX_RESPONSE: u64 = 8 * 1024 * 1024; + +/// Serves one named RPC service (see [`Libp2pNetwork::register_rpc`]). +#[async_trait] +pub trait RpcHandler: Send + Sync + 'static { + /// Answer a request from `peer`. `Err` is sent back as an error string. + async fn handle(&self, peer: PeerId, body: Vec<u8>) -> std::result::Result<Vec<u8>, String>; +} + +type RpcHandlers = Arc<std::sync::RwLock<HashMap<String, Arc<dyn RpcHandler>>>>; const IDENTIFY_PROTOCOL: &str = "/c0mpute/id/1.0.0"; /// What the swarm task accepts from the public API. @@ -78,6 +95,19 @@ enum Cmd { Addrs { reply: oneshot::Sender<Vec<Multiaddr>>, }, + Rpc { + peer: PeerId, + service: String, + body: Vec<u8>, + reply: oneshot::Sender<Result<Vec<u8>>>, + }, + RpcRespond { + channel: request_response::ResponseChannel<RpcResponse>, + resp: RpcResponse, + }, + ConnectedPeers { + reply: oneshot::Sender<Vec<PeerId>>, + }, Publish { topic: String, payload: Vec<u8>, @@ -150,6 +180,7 @@ pub struct Libp2pNetwork { cmd_tx: mpsc::Sender<Cmd>, gossip_tx: broadcast::Sender<GossipMessage>, peer_id: PeerId, + rpc_handlers: RpcHandlers, } impl Libp2pNetwork { @@ -162,7 +193,15 @@ impl Libp2pNetwork { let (cmd_tx, cmd_rx) = mpsc::channel::<Cmd>(64); let (gossip_tx, _) = broadcast::channel::<GossipMessage>(256); - let task = SwarmTask::new(keypair, config.local_source, gossip_tx.clone()).await?; + let rpc_handlers: RpcHandlers = Arc::new(std::sync::RwLock::new(HashMap::new())); + let task = SwarmTask::new( + keypair, + config.local_source, + gossip_tx.clone(), + rpc_handlers.clone(), + cmd_tx.downgrade(), + ) + .await?; for addr in &config.listen { task.swarm .lock() @@ -184,9 +223,50 @@ impl Libp2pNetwork { cmd_tx, gossip_tx, peer_id, + rpc_handlers, }) } + /// Serve `service` over `/c0mpute/rpc/1.0.0`. Replaces any handler + /// already registered under the name. + pub fn register_rpc(&self, service: &str, handler: Arc<dyn RpcHandler>) { + if let Ok(mut h) = self.rpc_handlers.write() { + h.insert(service.to_string(), handler); + } + } + + /// Call `service` on `peer` (dialing it if needed) and wait for the + /// answer. The request times out after 20 s. + pub async fn rpc(&self, peer: PeerId, service: &str, body: Vec<u8>) -> Result<Vec<u8>> { + if peer == self.peer_id { + let handler = self.rpc_handlers.read().ok().and_then(|h| h.get(service).cloned()); + return match handler { + Some(h) => h.handle(peer, body).await.map_err(|e| anyhow!("{e}")), + None => Err(anyhow!("{service}: not served here")), + }; + } + let (tx, rx) = oneshot::channel(); + self.cmd_tx + .send(Cmd::Rpc { + peer, + service: service.to_string(), + body, + reply: tx, + }) + .await + .map_err(|_| anyhow!("swarm task closed"))?; + rx.await.map_err(|_| anyhow!("swarm task closed"))? + } + + /// Peers with an open connection now. + pub async fn connected_peers(&self) -> Vec<PeerId> { + let (tx, rx) = oneshot::channel(); + if self.cmd_tx.send(Cmd::ConnectedPeers { reply: tx }).await.is_err() { + return Vec::new(); + } + rx.await.unwrap_or_default() + } + /// Publish bytes to a gossipsub topic. The topic must have been /// `subscribe`d first (libp2p subscribes-to-publish to participate /// in the mesh). @@ -364,6 +444,7 @@ struct C0mputeBehaviour { identify: identify::Behaviour, ping: ping::Behaviour, fetch: request_response::cbor::Behaviour<FetchRequest, FetchResponse>, + rpc: request_response::Behaviour<request_response::cbor::codec::Codec<RpcRequest, RpcResponse>>, gossipsub: gossipsub::Behaviour, mdns: mdns::tokio::Behaviour, } @@ -379,6 +460,58 @@ enum FetchResponse { NotFound, } +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] +struct RpcRequest { + service: String, + #[serde(with = "cbor_bytes")] + body: Vec<u8>, +} + +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] +enum RpcResponse { + Ok { + #[serde(with = "cbor_bytes")] + body: Vec<u8>, + }, + Err { + msg: String, + }, +} + +/// Encode `Vec<u8>` as a CBOR byte string (serde's default is an array of +/// integers, up to twice the size). +mod cbor_bytes { + use serde::{Deserializer, Serializer, de}; + + pub fn serialize<S: Serializer>(b: &[u8], s: S) -> Result<S::Ok, S::Error> { + s.serialize_bytes(b) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> { + struct V; + impl<'de> de::Visitor<'de> for V { + type Value = Vec<u8>; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("bytes") + } + fn visit_bytes<E: de::Error>(self, v: &[u8]) -> Result<Vec<u8>, E> { + Ok(v.to_vec()) + } + fn visit_byte_buf<E: de::Error>(self, v: Vec<u8>) -> Result<Vec<u8>, E> { + Ok(v) + } + fn visit_seq<A: de::SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<u8>, A::Error> { + let mut out = Vec::new(); + while let Some(b) = seq.next_element::<u8>()? { + out.push(b); + } + Ok(out) + } + } + d.deserialize_byte_buf(V) + } +} + // ──────────────────────────────────────────────────────────────────────── // Swarm task // ──────────────────────────────────────────────────────────────────────── @@ -395,6 +528,12 @@ struct SwarmTask { /// Track which chunk a fetch is for, so we can re-issue against another /// provider if the first one says NotFound. fetch_targets: HashMap<request_response::OutboundRequestId, Hash>, + /// Pending outbound RPC calls. + pending_rpcs: HashMap<request_response::OutboundRequestId, oneshot::Sender<Result<Vec<u8>>>>, + rpc_handlers: RpcHandlers, + /// For handler tasks to hand their answer back to the swarm loop (weak, + /// so the loop still ends when the last `Libp2pNetwork` drops). + cmd_weak: mpsc::WeakSender<Cmd>, /// Broadcast channel for gossipsub messages delivered to subscribed /// topics. Senders are upstream consumers that called `messages()`. gossip_tx: broadcast::Sender<GossipMessage>, @@ -405,6 +544,8 @@ impl SwarmTask { keypair: identity::Keypair, local_source: Option<Arc<dyn ChunkSource>>, gossip_tx: broadcast::Sender<GossipMessage>, + rpc_handlers: RpcHandlers, + cmd_weak: mpsc::WeakSender<Cmd>, ) -> Result<Self> { let swarm = libp2p::SwarmBuilder::with_existing_identity(keypair.clone()) .with_tokio() @@ -430,6 +571,13 @@ impl SwarmTask { [(FETCH_PROTOCOL, request_response::ProtocolSupport::Full)], request_response::Config::default(), ); + let rpc = request_response::Behaviour::with_codec( + request_response::cbor::codec::Codec::<RpcRequest, RpcResponse>::default() + .set_request_size_maximum(RPC_MAX_REQUEST) + .set_response_size_maximum(RPC_MAX_RESPONSE), + [(RPC_PROTOCOL, request_response::ProtocolSupport::Full)], + request_response::Config::default().with_request_timeout(Duration::from_secs(20)), + ); let gossipsub_cfg = gossipsub::ConfigBuilder::default() .heartbeat_interval(Duration::from_secs(1)) .validation_mode(gossipsub::ValidationMode::Strict) @@ -446,6 +594,7 @@ impl SwarmTask { identify, ping, fetch, + rpc, gossipsub, mdns, }) @@ -460,6 +609,9 @@ impl SwarmTask { pending_kad_lookups: HashMap::new(), pending_fetches: HashMap::new(), fetch_targets: HashMap::new(), + pending_rpcs: HashMap::new(), + rpc_handlers, + cmd_weak, gossip_tx, }) } @@ -575,6 +727,28 @@ impl SwarmTask { self.fetch_targets.insert(request_id, req.chunk_hash); debug!(%peer, chunk = %req.chunk_hash, "direct chunk fetch issued"); } + Cmd::Rpc { + peer, + service, + body, + reply, + } => { + let id = self + .swarm + .lock() + .await + .behaviour_mut() + .rpc + .send_request(&peer, RpcRequest { service, body }); + self.pending_rpcs.insert(id, reply); + } + Cmd::RpcRespond { channel, resp } => { + let _ = self.swarm.lock().await.behaviour_mut().rpc.send_response(channel, resp); + } + Cmd::ConnectedPeers { reply } => { + let peers = self.swarm.lock().await.connected_peers().copied().collect(); + let _ = reply.send(peers); + } Cmd::Publish { topic, payload, @@ -657,6 +831,56 @@ impl SwarmTask { let _ = reply.send(Err(anyhow!("fetch failed: {error}"))); } } + SwarmEvent::Behaviour(C0mputeBehaviourEvent::Rpc(request_response::Event::Message { + peer, message, .. + })) => match message { + request_response::Message::Request { request, channel, .. } => { + let handler = self + .rpc_handlers + .read() + .ok() + .and_then(|h| h.get(&request.service).cloned()); + match handler { + Some(h) => { + let weak = self.cmd_weak.clone(); + tokio::spawn(async move { + let resp = match h.handle(peer, request.body).await { + Ok(body) => RpcResponse::Ok { body }, + Err(msg) => RpcResponse::Err { msg }, + }; + if let Some(tx) = weak.upgrade() { + let _ = tx.send(Cmd::RpcRespond { channel, resp }).await; + } + }); + } + None => { + let resp = RpcResponse::Err { + msg: format!("{}: not served here", request.service), + }; + let _ = self.swarm.lock().await.behaviour_mut().rpc.send_response(channel, resp); + } + } + } + request_response::Message::Response { + request_id, response, .. + } => { + if let Some(reply) = self.pending_rpcs.remove(&request_id) { + let _ = reply.send(match response { + RpcResponse::Ok { body } => Ok(body), + RpcResponse::Err { msg } => Err(anyhow!("{msg}")), + }); + } + } + }, + SwarmEvent::Behaviour(C0mputeBehaviourEvent::Rpc(request_response::Event::OutboundFailure { + request_id, + error, + .. + })) => { + if let Some(reply) = self.pending_rpcs.remove(&request_id) { + let _ = reply.send(Err(anyhow!("rpc failed: {error}"))); + } + } SwarmEvent::Behaviour(C0mputeBehaviourEvent::Gossipsub( gossipsub::Event::Message { propagation_source, @@ -903,6 +1127,52 @@ mod tests { panic!("node never bound a listen addr"); } + struct Echo; + + #[async_trait] + impl RpcHandler for Echo { + async fn handle(&self, _peer: PeerId, body: Vec<u8>) -> std::result::Result<Vec<u8>, String> { + if body == b"fail" { + return Err("asked to fail".into()); + } + let mut out = b"echo:".to_vec(); + out.extend(body); + Ok(out) + } + } + + /// Named-service RPC between two nodes: answers, errors, unknown service. + #[tokio::test] + async fn two_node_rpc_roundtrip() { + let net_a = Libp2pNetwork::spawn( + NetworkConfig::for_dir(tempdir("rpc-a")).with_listen(vec!["/ip4/127.0.0.1/tcp/0".parse().unwrap()]), + ) + .await + .unwrap(); + net_a.register_rpc("echo", Arc::new(Echo)); + let a_full = wait_for_listen(&net_a) + .await + .with(libp2p::multiaddr::Protocol::P2p(net_a.peer_id())); + let net_b = Libp2pNetwork::spawn( + NetworkConfig::for_dir(tempdir("rpc-b")) + .with_listen(vec!["/ip4/127.0.0.1/tcp/0".parse().unwrap()]) + .with_bootstrap(vec![a_full]), + ) + .await + .unwrap(); + let big = vec![7u8; 200_000]; + let got = net_b.rpc(net_a.peer_id(), "echo", big.clone()).await.unwrap(); + assert_eq!(&got[..5], b"echo:"); + assert_eq!(&got[5..], &big[..]); + let err = net_b.rpc(net_a.peer_id(), "echo", b"fail".to_vec()).await.unwrap_err(); + assert!(err.to_string().contains("asked to fail"), "{err}"); + let err = net_b.rpc(net_a.peer_id(), "nope", vec![]).await.unwrap_err(); + assert!(err.to_string().contains("not served"), "{err}"); + assert!(net_b.connected_peers().await.contains(&net_a.peer_id())); + // Calling yourself runs the local handler. + assert_eq!(net_a.rpc(net_a.peer_id(), "echo", b"x".to_vec()).await.unwrap(), b"echo:x"); + } + /// gossipsub publish/subscribe across two nodes. #[tokio::test] async fn two_node_gossipsub_publish_receive() { diff --git a/node/crates/c0mpute-proto/src/lib.rs b/node/crates/c0mpute-proto/src/lib.rs index cccbaf73..bea9e900 100644 --- a/node/crates/c0mpute-proto/src/lib.rs +++ b/node/crates/c0mpute-proto/src/lib.rs @@ -223,6 +223,11 @@ pub enum Role { /// Blender (Cycles on CPU, or on a GPU Blender finds). On by default; /// advertised only where `blender --version` works. Render, + /// Secure-chat mailbox (DIP-0018): hold sealed messages for offline + /// recipients (bounded: 256 MiB, 30 days), cache self-published chat + /// keys, and receive direct messages for this machine's own `c0mpute dm` + /// user. On by default; needs nothing. + Mailbox, } #[cfg(test)] diff --git a/node/crates/c0mpute-secure-chat/src/identity.rs b/node/crates/c0mpute-secure-chat/src/identity.rs new file mode 100644 index 00000000..e3a6ad6e --- /dev/null +++ b/node/crates/c0mpute-secure-chat/src/identity.rs @@ -0,0 +1,339 @@ +//! Public identities, addresses, and the signed records nodes gossip. +//! +//! A user's address is their public key: the SHA-256 of all four public keys +//! (X25519, Ed25519, ML-KEM-768, ML-DSA-65), shown as 64 hex characters. The +//! short fingerprint is the first 8 bytes of the address as colon-separated +//! hex pairs. Because the address commits to every key, a record fetched from +//! any node is checked against the address (or fingerprint) the user typed: +//! nobody can answer a lookup with a different key. +//! +//! * [`KeyRecord`]: a self-published, self-signed copy of a public identity. +//! Gossiped and cached by mailbox nodes; that is the whole "directory". +//! * [`HomeRecord`]: "messages for me may be delivered directly to libp2p peer +//! P until T", signed by the user. The node named there accepts direct +//! deliveries for the address and gossips the record while it is online. + +use base64::prelude::*; +use ed25519_dalek::{Signature as EdSig, Signer, Verifier, VerifyingKey as EdVk}; +use ml_dsa::{KeyExport as _, MlDsa65, VerifyingKey as MlDsaVk}; +use ml_kem::{EncapsulationKey, MlKem768}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use x25519_dalek::PublicKey as X25519Public; + +use crate::keys::{ChatKey, KeyFile}; + +/// Longest a key or home record may claim to be valid for. +pub const MAX_RECORD_SECS: u64 = 31 * 86_400; +/// Clock skew tolerated on record and message timestamps. +pub const CLOCK_SKEW_SECS: u64 = 300; + +/// The four public keys of a chat identity (same encoding as the keyfile's +/// `pubkey_*` fields: base64url, no padding). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicIdentity { + pub pubkey_enc: String, + pub pubkey_kem: String, + pub pubkey_sig: String, + pub pubkey_ml_dsa: String, +} + +/// Parsed public keys, ready for crypto. +pub struct PublicKeys { + pub x25519: X25519Public, + pub ed25519: EdVk, + pub ml_kem: EncapsulationKey<MlKem768>, + pub ml_dsa: MlDsaVk<MlDsa65>, +} + +impl PublicIdentity { + pub fn of(key: &ChatKey) -> Self { + Self { + pubkey_enc: BASE64_URL_SAFE_NO_PAD.encode(key.x25519_public.as_bytes()), + pubkey_kem: BASE64_URL_SAFE_NO_PAD.encode(&key.ml_kem_ek().to_bytes()[..]), + pubkey_sig: BASE64_URL_SAFE_NO_PAD.encode(key.ed25519_public.as_bytes()), + pubkey_ml_dsa: BASE64_URL_SAFE_NO_PAD.encode(&key.ml_dsa_public.to_bytes()[..]), + } + } + + pub fn from_keyfile(kf: &KeyFile) -> Self { + Self { + pubkey_enc: kf.pubkey_enc.clone(), + pubkey_kem: kf.pubkey_kem.clone(), + pubkey_sig: kf.pubkey_sig.clone(), + pubkey_ml_dsa: kf.pubkey_ml_dsa.clone(), + } + } + + /// Decode and validate all four keys. + pub fn keys(&self) -> anyhow::Result<PublicKeys> { + let enc = BASE64_URL_SAFE_NO_PAD.decode(&self.pubkey_enc)?; + let x: [u8; 32] = enc.as_slice().try_into().map_err(|_| anyhow::anyhow!("bad x25519 key length"))?; + let sig = BASE64_URL_SAFE_NO_PAD.decode(&self.pubkey_sig)?; + let s: [u8; 32] = sig.as_slice().try_into().map_err(|_| anyhow::anyhow!("bad ed25519 key length"))?; + let ed25519 = EdVk::from_bytes(&s).map_err(|e| anyhow::anyhow!("bad ed25519 key: {e}"))?; + let kem = BASE64_URL_SAFE_NO_PAD.decode(&self.pubkey_kem)?; + let kem_arr: ml_kem::Key<EncapsulationKey<MlKem768>> = kem + .as_slice() + .try_into() + .map_err(|_| anyhow::anyhow!("bad ml-kem key length"))?; + let ml_kem = EncapsulationKey::<MlKem768>::new(&kem_arr).map_err(|_| anyhow::anyhow!("bad ml-kem key"))?; + let dsa = BASE64_URL_SAFE_NO_PAD.decode(&self.pubkey_ml_dsa)?; + let dsa_arr: ml_dsa::EncodedVerifyingKey<MlDsa65> = dsa + .as_slice() + .try_into() + .map_err(|_| anyhow::anyhow!("bad ml-dsa key length"))?; + let ml_dsa = MlDsaVk::<MlDsa65>::decode(&dsa_arr); + Ok(PublicKeys { + x25519: X25519Public::from(x), + ed25519, + ml_kem, + ml_dsa, + }) + } + + /// The address: hex SHA-256 over all four public keys. + pub fn address(&self) -> String { + let mut h = Sha256::new(); + h.update(b"c0mpute-dm-address-v1"); + for part in [&self.pubkey_enc, &self.pubkey_sig, &self.pubkey_kem, &self.pubkey_ml_dsa] { + h.update((part.len() as u32).to_be_bytes()); + h.update(part.as_bytes()); + } + hex::encode(h.finalize()) + } + + pub fn fingerprint(&self) -> String { + fingerprint_of(&self.address()) + } + + /// A single pasteable string carrying the whole public identity, for + /// sharing a key out of band (`c0mpute dm key show --full`). + pub fn to_share_string(&self) -> String { + format!( + "c0dm1.{}", + BASE64_URL_SAFE_NO_PAD.encode(serde_json::to_vec(self).unwrap_or_default()) + ) + } + + pub fn from_share_string(s: &str) -> anyhow::Result<Self> { + let body = s.trim().strip_prefix("c0dm1.").ok_or_else(|| anyhow::anyhow!("not a c0dm1. identity"))?; + let id: Self = serde_json::from_slice(&BASE64_URL_SAFE_NO_PAD.decode(body)?)?; + id.keys()?; + Ok(id) + } +} + +/// `aa:bb:cc:dd:ee:ff:00:11` from a 64-hex address. +pub fn fingerprint_of(address: &str) -> String { + address + .as_bytes() + .chunks(2) + .take(8) + .map(|c| String::from_utf8_lossy(c).into_owned()) + .collect::<Vec<_>>() + .join(":") +} + +/// What a user typed to name someone: a full address, or a fingerprint +/// (16 hex chars, colons optional). Lowercased, colons removed. +pub fn normalize_query(q: &str) -> Option<String> { + let s: String = q.trim().chars().filter(|c| *c != ':').collect::<String>().to_ascii_lowercase(); + let ok = (s.len() == 16 || s.len() == 64) && s.chars().all(|c| c.is_ascii_hexdigit()); + ok.then_some(s) +} + +/// Does `identity` answer `query` (normalized address or fingerprint)? +pub fn matches_query(identity: &PublicIdentity, query: &str) -> bool { + identity.address().starts_with(query) +} + +impl ChatKey { + pub fn public_identity(&self) -> PublicIdentity { + PublicIdentity::of(self) + } + + pub fn address(&self) -> String { + self.public_identity().address() + } + + /// Ed25519 + ML-DSA-65 signatures over `msg`, base64url. + pub fn sign_both(&self, msg: &[u8]) -> anyhow::Result<(String, String)> { + let ed: EdSig = self.ed25519_key.sign(msg); + let ml = ml_dsa::Signer::try_sign(&self.ml_dsa_key, msg).map_err(|e| anyhow::anyhow!("ml-dsa sign: {e}"))?; + Ok(( + BASE64_URL_SAFE_NO_PAD.encode(ed.to_bytes()), + BASE64_URL_SAFE_NO_PAD.encode(&ml.encode()[..]), + )) + } +} + +/// Check both signatures from [`ChatKey::sign_both`] against `keys`. +pub fn verify_both(keys: &PublicKeys, msg: &[u8], sig: &str, ml_dsa_sig: &str) -> anyhow::Result<()> { + let s = BASE64_URL_SAFE_NO_PAD.decode(sig)?; + let arr: [u8; 64] = s.as_slice().try_into().map_err(|_| anyhow::anyhow!("bad ed25519 signature length"))?; + keys.ed25519 + .verify(msg, &EdSig::from_bytes(&arr)) + .map_err(|_| anyhow::anyhow!("ed25519 signature invalid"))?; + let m = BASE64_URL_SAFE_NO_PAD.decode(ml_dsa_sig)?; + let enc: ml_dsa::EncodedSignature<MlDsa65> = + m.as_slice().try_into().map_err(|_| anyhow::anyhow!("bad ml-dsa signature length"))?; + let sig = ml_dsa::Signature::<MlDsa65>::decode(&enc).ok_or_else(|| anyhow::anyhow!("malformed ml-dsa signature"))?; + ml_dsa::Verifier::verify(&keys.ml_dsa, msg, &sig).map_err(|_| anyhow::anyhow!("ml-dsa signature invalid"))?; + Ok(()) +} + +/// A self-published public identity, signed by itself. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct KeyRecord { + pub v: u8, + pub identity: PublicIdentity, + pub created_at: u64, + pub expires_at: u64, + pub sig: String, + pub ml_dsa_sig: String, +} + +fn key_record_bytes(address: &str, created_at: u64, expires_at: u64) -> Vec<u8> { + format!("c0mpute-dm-keyrecord-v1|{address}|{created_at}|{expires_at}").into_bytes() +} + +impl KeyRecord { + pub fn sign(key: &ChatKey, now: u64, valid_secs: u64) -> anyhow::Result<Self> { + let identity = key.public_identity(); + let expires_at = now + valid_secs.min(MAX_RECORD_SECS); + let (sig, ml_dsa_sig) = key.sign_both(&key_record_bytes(&identity.address(), now, expires_at))?; + Ok(Self { + v: 1, + identity, + created_at: now, + expires_at, + sig, + ml_dsa_sig, + }) + } + + /// Signature valid, not expired, not from the future. + pub fn verify(&self, now: u64) -> anyhow::Result<()> { + anyhow::ensure!(self.v == 1, "unsupported key record version"); + anyhow::ensure!(self.created_at <= now + CLOCK_SKEW_SECS, "key record from the future"); + anyhow::ensure!(self.expires_at > now, "key record expired"); + anyhow::ensure!(self.expires_at - self.created_at <= MAX_RECORD_SECS, "key record validity too long"); + let keys = self.identity.keys()?; + verify_both( + &keys, + &key_record_bytes(&self.identity.address(), self.created_at, self.expires_at), + &self.sig, + &self.ml_dsa_sig, + ) + } + + pub fn address(&self) -> String { + self.identity.address() + } +} + +/// "Deliver my messages directly to libp2p peer `peer`", signed by the user. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct HomeRecord { + pub v: u8, + pub identity: PublicIdentity, + /// Base58 libp2p peer id of the node that receives for this address. + pub peer: String, + pub created_at: u64, + pub expires_at: u64, + pub sig: String, + pub ml_dsa_sig: String, +} + +fn home_record_bytes(address: &str, peer: &str, created_at: u64, expires_at: u64) -> Vec<u8> { + format!("c0mpute-dm-home-v1|{address}|{peer}|{created_at}|{expires_at}").into_bytes() +} + +impl HomeRecord { + pub fn sign(key: &ChatKey, peer: &str, now: u64, valid_secs: u64) -> anyhow::Result<Self> { + let identity = key.public_identity(); + let expires_at = now + valid_secs.min(MAX_RECORD_SECS); + let (sig, ml_dsa_sig) = key.sign_both(&home_record_bytes(&identity.address(), peer, now, expires_at))?; + Ok(Self { + v: 1, + identity, + peer: peer.to_string(), + created_at: now, + expires_at, + sig, + ml_dsa_sig, + }) + } + + pub fn verify(&self, now: u64) -> anyhow::Result<()> { + anyhow::ensure!(self.v == 1, "unsupported home record version"); + anyhow::ensure!(self.created_at <= now + CLOCK_SKEW_SECS, "home record from the future"); + anyhow::ensure!(self.expires_at > now, "home record expired"); + anyhow::ensure!(self.expires_at - self.created_at <= MAX_RECORD_SECS, "home record validity too long"); + let keys = self.identity.keys()?; + verify_both( + &keys, + &home_record_bytes(&self.identity.address(), &self.peer, self.created_at, self.expires_at), + &self.sig, + &self.ml_dsa_sig, + ) + } + + pub fn address(&self) -> String { + self.identity.address() + } + + /// The key record this home record implies (same identity), for caches. + pub fn identity(&self) -> &PublicIdentity { + &self.identity + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn address_and_fingerprint_are_stable_and_match_keyfile() { + let k = ChatKey::from_seed([7u8; 32]); + let id = k.public_identity(); + assert_eq!(id.address().len(), 64); + assert_eq!(id.address(), ChatKey::from_seed([7u8; 32]).address()); + assert_ne!(id.address(), ChatKey::from_seed([8u8; 32]).address()); + let fp = id.fingerprint(); + assert_eq!(fp.len(), 23); + assert_eq!(normalize_query(&fp).unwrap(), id.address()[..16]); + assert!(matches_query(&id, &normalize_query(&fp).unwrap())); + assert!(matches_query(&id, &normalize_query(&id.address().to_uppercase()).unwrap())); + assert!(normalize_query("zz").is_none()); + // Same identity from the (password-protected) keyfile's public fields. + let kf = k.encrypt_to_keyfile("correct horse battery", None).unwrap(); + assert_eq!(PublicIdentity::from_keyfile(&kf), id); + id.keys().unwrap(); + assert_eq!(PublicIdentity::from_share_string(&id.to_share_string()).unwrap(), id); + } + + #[test] + fn key_and_home_records_verify_and_reject_tampering() { + let k = ChatKey::generate(); + let now = 1_800_000_000; + let rec = KeyRecord::sign(&k, now, 86_400).unwrap(); + rec.verify(now).unwrap(); + assert!(rec.verify(now + 86_401).is_err(), "expired"); + let mut bad = rec.clone(); + bad.expires_at += 10; + assert!(bad.verify(now).is_err()); + // Someone else's identity with this signature. + let mut bad = rec.clone(); + bad.identity = ChatKey::generate().public_identity(); + assert!(bad.verify(now).is_err()); + + let home = HomeRecord::sign(&k, "12D3KooWpeer", now, 3600).unwrap(); + home.verify(now).unwrap(); + let mut bad = home.clone(); + bad.peer = "12D3KooWevil".into(); + assert!(bad.verify(now).is_err()); + assert!(home.verify(now - 10_000).is_err(), "from the future"); + } +} diff --git a/node/crates/c0mpute-secure-chat/src/keys.rs b/node/crates/c0mpute-secure-chat/src/keys.rs index db537b0d..e8ae8e74 100644 --- a/node/crates/c0mpute-secure-chat/src/keys.rs +++ b/node/crates/c0mpute-secure-chat/src/keys.rs @@ -125,13 +125,10 @@ impl ChatKey { self.ml_kem_dk.encapsulation_key() } - /// Hex fingerprint of the X25519 public key — 8 colon-separated byte pairs. + /// Short fingerprint: the first 8 bytes of the address (which hashes all + /// four public keys), as colon-separated hex pairs. pub fn fingerprint(&self) -> String { - self.x25519_public.as_bytes()[..8] - .iter() - .map(|b| format!("{b:02x}")) - .collect::<Vec<_>>() - .join(":") + self.public_identity().fingerprint() } /// Produce the password-encrypted `KeyFile` (backup blob). @@ -225,7 +222,7 @@ pub fn save_keyfile(path: &Path, kf: &KeyFile) -> anyhow::Result<()> { } /// Write `content` to `path` with 0o600 permissions (unix) or plain write (other). -fn write_secret_file(path: &Path, content: &[u8]) -> anyhow::Result<()> { +pub(crate) fn write_secret_file(path: &Path, content: &[u8]) -> anyhow::Result<()> { #[cfg(unix)] { use std::io::Write; diff --git a/node/crates/c0mpute-secure-chat/src/lib.rs b/node/crates/c0mpute-secure-chat/src/lib.rs index df71fe31..a4cc603d 100644 --- a/node/crates/c0mpute-secure-chat/src/lib.rs +++ b/node/crates/c0mpute-secure-chat/src/lib.rs @@ -1,11 +1,37 @@ //! E2E encrypted p2p messaging for the c0mpute network (DIP-0018). //! -//! v0.1 scope: key generation, password-encrypted backups, and DM -//! encrypt/decrypt. Transport (DHT key distribution, gossip relay, -//! store-and-forward) lands in v0.2. +//! * [`keys`]: hybrid classical + post-quantum keys from one seed, with a +//! password-encrypted keyfile. +//! * [`message`]: the signed, hybrid-encrypted DM envelope. +//! * [`identity`]: addresses (= public keys), fingerprints, and the signed +//! key and home records nodes gossip instead of a directory. +//! * [`sealed`]: the outer layer that hides sender and recipient from the +//! network; recipient tags; replay protection. +//! * [`mailbox`]: the bounded store-and-forward store every mailbox node runs. +//! * [`placement`]: rendezvous hashing that picks mailbox nodes per tag. +//! * [`state`]: the client's contacts, blocklist and inbox, encrypted at rest. +//! +//! The network service (gossip + libp2p request/response) lives in +//! `c0mpute_core::dm`. +pub mod identity; pub mod keys; +pub mod mailbox; pub mod message; +pub mod placement; +pub mod sealed; +pub mod state; +pub use identity::{HomeRecord, KeyRecord, PublicIdentity}; pub use keys::{ChatKey, KeyFile, decrypt_keyfile, key_file_path, load_keyfile, save_keyfile}; pub use message::{DmEnvelope, DmPlaintext, decrypt_dm, encrypt_dm}; +pub use sealed::{Sealed, SealedMessage}; +pub use state::LocalState; + +/// Seconds since the Unix epoch. +pub fn unix_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} diff --git a/node/crates/c0mpute-secure-chat/src/mailbox.rs b/node/crates/c0mpute-secure-chat/src/mailbox.rs new file mode 100644 index 00000000..3ba97fa7 --- /dev/null +++ b/node/crates/c0mpute-secure-chat/src/mailbox.rs @@ -0,0 +1,421 @@ +//! The store-and-forward mailbox a node keeps for other people. +//! +//! It holds opaque sealed messages keyed by recipient tag, each until its +//! TTL, its recipient acks it, or the node needs the room. Everything is +//! bounded: message size, TTL, bytes and count per tag, total bytes, and +//! messages per minute from each sending peer. Messages are persisted one +//! file per message (`<dir>/<tag>/<id>.msg`) so a restart loses nothing. +//! +//! The mailbox never sees a sender, a recipient address or any plaintext: +//! only the tag, the bytes, and the hash of the ack secret that deletes them. + +use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::sealed::{MAX_SEALED_BYTES, MAX_TTL_SECS, SealedMessage, is_hex32, sha256_hex}; + +#[derive(Clone, Debug)] +pub struct Limits { + pub max_msg_bytes: usize, + pub max_ttl_secs: u64, + pub max_bytes_per_tag: u64, + pub max_msgs_per_tag: usize, + pub max_total_bytes: u64, + /// Messages accepted per sending peer per minute. + pub per_sender_per_min: usize, + /// Most bytes one fetch returns (the rest come on the next pull). + pub max_fetch_bytes: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { + max_msg_bytes: MAX_SEALED_BYTES, + max_ttl_secs: MAX_TTL_SECS, + max_bytes_per_tag: 10 * 1024 * 1024, + max_msgs_per_tag: 500, + max_total_bytes: 256 * 1024 * 1024, + per_sender_per_min: 60, + max_fetch_bytes: 4 * 1024 * 1024, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Put { + Stored, + /// Already held (another sender's replica, or a retry). + Duplicate, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum PutError { + TooLarge, + BadMessage(String), + RateLimited, + TagFull, + MailboxFull, +} + +impl std::fmt::Display for PutError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PutError::TooLarge => write!(f, "message too large"), + PutError::BadMessage(m) => write!(f, "bad message: {m}"), + PutError::RateLimited => write!(f, "rate limited: too many messages from this peer"), + PutError::TagFull => write!(f, "this recipient's mailbox is full"), + PutError::MailboxFull => write!(f, "this node's mailbox is full"), + } + } +} + +impl std::error::Error for PutError {} + +#[derive(Clone, Debug)] +struct Entry { + bytes: Vec<u8>, + expires_at: u64, + ack_hash: String, +} + +#[derive(Serialize, Deserialize)] +struct Meta { + expires_at: u64, + ack_hash: String, +} + +pub struct Mailbox { + dir: Option<PathBuf>, + limits: Limits, + tags: HashMap<String, BTreeMap<String, Entry>>, + total: u64, + rate: HashMap<String, VecDeque<u64>>, +} + +impl Mailbox { + /// In memory only. + pub fn new(limits: Limits) -> Self { + Self { + dir: None, + limits, + tags: HashMap::new(), + total: 0, + rate: HashMap::new(), + } + } + + /// Persisted under `dir`; loads what is there and drops what expired. + pub fn open(dir: &Path, limits: Limits, now: u64) -> Self { + let mut mb = Self::new(limits); + mb.dir = Some(dir.to_path_buf()); + let _ = std::fs::create_dir_all(dir); + let Ok(tags) = std::fs::read_dir(dir) else { + return mb; + }; + for t in tags.flatten() { + let tag = t.file_name().to_string_lossy().into_owned(); + if !is_hex32(&tag) { + continue; + } + let Ok(files) = std::fs::read_dir(t.path()) else { + continue; + }; + for f in files.flatten() { + let path = f.path(); + let Some(id) = path + .file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.strip_suffix(".msg")) + .map(str::to_string) + else { + continue; + }; + match read_entry(&path) { + Some(e) if e.expires_at > now && sha256_hex(&e.bytes) == id => { + mb.total += e.bytes.len() as u64; + mb.tags.entry(tag.clone()).or_default().insert(id, e); + } + _ => { + let _ = std::fs::remove_file(&path); + } + } + } + } + mb + } + + pub fn limits(&self) -> &Limits { + &self.limits + } + + pub fn total_bytes(&self) -> u64 { + self.total + } + + pub fn count(&self) -> usize { + self.tags.values().map(|m| m.len()).sum() + } + + pub fn count_for(&self, tag: &str) -> usize { + self.tags.get(tag).map(|m| m.len()).unwrap_or(0) + } + + /// Accept a sealed message from `sender` (the libp2p peer that sent it). + pub fn put(&mut self, sender: &str, bytes: Vec<u8>, ack_hash: &str, ttl: u64, now: u64) -> Result<Put, PutError> { + if bytes.len() > self.limits.max_msg_bytes { + return Err(PutError::TooLarge); + } + let msg = SealedMessage::parse(&bytes).map_err(|e| PutError::BadMessage(e.to_string()))?; + if !is_hex32(ack_hash) { + return Err(PutError::BadMessage("bad ack hash".into())); + } + let id = sha256_hex(&bytes); + if self.tags.get(&msg.tag).is_some_and(|m| m.contains_key(&id)) { + return Ok(Put::Duplicate); + } + // Rate limit per sending peer over a sliding minute. + { + let window = self.rate.entry(sender.to_string()).or_default(); + while window.front().is_some_and(|t| *t + 60 <= now) { + window.pop_front(); + } + if window.len() >= self.limits.per_sender_per_min { + return Err(PutError::RateLimited); + } + } + self.prune(now); + let size = bytes.len() as u64; + let (tag_bytes, tag_count) = self + .tags + .get(&msg.tag) + .map(|m| (m.values().map(|e| e.bytes.len() as u64).sum::<u64>(), m.len())) + .unwrap_or((0, 0)); + if tag_count >= self.limits.max_msgs_per_tag || tag_bytes + size > self.limits.max_bytes_per_tag { + return Err(PutError::TagFull); + } + if self.total + size > self.limits.max_total_bytes { + return Err(PutError::MailboxFull); + } + self.rate.entry(sender.to_string()).or_default().push_back(now); + let ttl = ttl.clamp(60, self.limits.max_ttl_secs); + let entry = Entry { + bytes, + expires_at: now + ttl, + ack_hash: ack_hash.to_string(), + }; + if let Some(dir) = &self.dir { + write_entry(&dir.join(&msg.tag), &id, &entry); + } + self.total += size; + self.tags.entry(msg.tag).or_default().insert(id, entry); + // Forget idle senders so the rate table stays small. + if self.rate.len() > 10_000 { + self.rate.retain(|_, w| w.back().is_some_and(|t| *t + 60 > now)); + } + Ok(Put::Stored) + } + + /// Live messages for `tag`, oldest id order, up to the fetch byte cap. + pub fn fetch(&self, tag: &str, now: u64) -> Vec<(String, Vec<u8>)> { + let mut out = Vec::new(); + let mut bytes = 0usize; + if let Some(m) = self.tags.get(tag) { + for (id, e) in m { + if e.expires_at <= now { + continue; + } + if bytes + e.bytes.len() > self.limits.max_fetch_bytes && !out.is_empty() { + break; + } + bytes += e.bytes.len(); + out.push((id.clone(), e.bytes.clone())); + } + } + out + } + + /// Delete `id` if `secret` (hex) hashes to its ack hash. + pub fn ack(&mut self, tag: &str, id: &str, secret: &str) -> bool { + let Ok(raw) = hex::decode(secret) else { + return false; + }; + let ok = self + .tags + .get(tag) + .and_then(|m| m.get(id)) + .is_some_and(|e| e.ack_hash == sha256_hex(&raw)); + if ok { + self.remove(tag, id); + } + ok + } + + fn remove(&mut self, tag: &str, id: &str) { + if let Some(m) = self.tags.get_mut(tag) + && let Some(e) = m.remove(id) + { + self.total -= e.bytes.len() as u64; + if let Some(dir) = &self.dir { + let _ = std::fs::remove_file(dir.join(tag).join(format!("{id}.msg"))); + } + if m.is_empty() { + self.tags.remove(tag); + if let Some(dir) = &self.dir { + let _ = std::fs::remove_dir(dir.join(tag)); + } + } + } + } + + /// Drop expired messages. Returns how many went. + pub fn prune(&mut self, now: u64) -> usize { + let dead: Vec<(String, String)> = self + .tags + .iter() + .flat_map(|(t, m)| { + m.iter() + .filter(|(_, e)| e.expires_at <= now) + .map(move |(id, _)| (t.clone(), id.clone())) + }) + .collect(); + for (t, id) in &dead { + self.remove(t, id); + } + dead.len() + } +} + +fn write_entry(dir: &Path, id: &str, e: &Entry) { + let _ = std::fs::create_dir_all(dir); + let meta = Meta { + expires_at: e.expires_at, + ack_hash: e.ack_hash.clone(), + }; + let mut buf = serde_json::to_vec(&meta).unwrap_or_default(); + buf.push(b'\n'); + buf.extend_from_slice(&e.bytes); + let path = dir.join(format!("{id}.msg")); + let tmp = dir.join(format!("{id}.tmp")); + if std::fs::write(&tmp, &buf).is_ok() { + let _ = std::fs::rename(&tmp, &path); + } +} + +fn read_entry(path: &Path) -> Option<Entry> { + let raw = std::fs::read(path).ok()?; + let nl = raw.iter().position(|b| *b == b'\n')?; + let meta: Meta = serde_json::from_slice(&raw[..nl]).ok()?; + Some(Entry { + bytes: raw[nl + 1..].to_vec(), + expires_at: meta.expires_at, + ack_hash: meta.ack_hash, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::keys::ChatKey; + use crate::sealed::seal; + + const NOW: u64 = 1_800_000_000; + + fn msg_to(bob: &ChatKey) -> crate::sealed::Sealed { + seal(&ChatKey::from_seed([1; 32]), &bob.public_identity(), "hi", 3600, NOW).unwrap() + } + + #[test] + fn put_fetch_ack_and_ttl() { + let bob = ChatKey::from_seed([2; 32]); + let mut mb = Mailbox::new(Limits::default()); + let s = msg_to(&bob); + assert_eq!(mb.put("p1", s.bytes.clone(), &s.ack_hash, 3600, NOW), Ok(Put::Stored)); + assert_eq!(mb.put("p2", s.bytes.clone(), &s.ack_hash, 3600, NOW), Ok(Put::Duplicate)); + let got = mb.fetch(&s.msg.tag, NOW); + assert_eq!(got.len(), 1); + assert_eq!(got[0].0, s.id); + // Wrong secret: nothing deleted. + assert!(!mb.ack(&s.msg.tag, &s.id, &"00".repeat(32))); + assert_eq!(mb.count(), 1); + // TTL: gone after expiry. + assert!(mb.fetch(&s.msg.tag, NOW + 3601).is_empty()); + assert_eq!(mb.prune(NOW + 3601), 1); + assert_eq!(mb.total_bytes(), 0); + + // Ack with the secret only the recipient can read. + mb.put("p1", s.bytes.clone(), &s.ack_hash, 3600, NOW).unwrap(); + let o = crate::sealed::unseal(&bob, &s.msg, NOW).unwrap(); + assert!(mb.ack(&s.msg.tag, &s.id, &o.ack)); + assert_eq!(mb.count(), 0); + // TTL is capped at the maximum. + mb.put("p1", s.bytes.clone(), &s.ack_hash, 10 * MAX_TTL_SECS, NOW).unwrap(); + assert_eq!(mb.fetch(&s.msg.tag, NOW + MAX_TTL_SECS + 1).len(), 0); + } + + #[test] + fn caps_and_rate_limits() { + let bob = ChatKey::from_seed([3; 32]); + // Size cap and garbage. + let mut mb = Mailbox::new(Limits { + max_msg_bytes: 100, + ..Default::default() + }); + let s = msg_to(&bob); + assert_eq!(mb.put("p", s.bytes.clone(), &s.ack_hash, 3600, NOW), Err(PutError::TooLarge)); + let mut mb = Mailbox::new(Limits::default()); + assert!(matches!(mb.put("p", b"not json".to_vec(), &s.ack_hash, 3600, NOW), Err(PutError::BadMessage(_)))); + assert!(matches!(mb.put("p", s.bytes.clone(), "nothex", 3600, NOW), Err(PutError::BadMessage(_)))); + + // Per-sender rate limit, sliding minute. + let mut mb = Mailbox::new(Limits { + per_sender_per_min: 2, + ..Default::default() + }); + let m: Vec<_> = (0..4).map(|_| msg_to(&bob)).collect(); + assert_eq!(mb.put("spammer", m[0].bytes.clone(), &m[0].ack_hash, 3600, NOW), Ok(Put::Stored)); + assert_eq!(mb.put("spammer", m[1].bytes.clone(), &m[1].ack_hash, 3600, NOW + 1), Ok(Put::Stored)); + assert_eq!(mb.put("spammer", m[2].bytes.clone(), &m[2].ack_hash, 3600, NOW + 2), Err(PutError::RateLimited)); + assert_eq!(mb.put("someone-else", m[2].bytes.clone(), &m[2].ack_hash, 3600, NOW + 2), Ok(Put::Stored)); + assert_eq!(mb.put("spammer", m[3].bytes.clone(), &m[3].ack_hash, 3600, NOW + 61), Ok(Put::Stored)); + + // Per-tag count cap and total byte cap. + let mut mb = Mailbox::new(Limits { + max_msgs_per_tag: 2, + ..Default::default() + }); + for (i, s) in m.iter().enumerate() { + let r = mb.put("p", s.bytes.clone(), &s.ack_hash, 3600, NOW); + assert_eq!(r, if i < 2 { Ok(Put::Stored) } else { Err(PutError::TagFull) }); + } + let one = m[0].bytes.len() as u64; + let mut mb = Mailbox::new(Limits { + max_total_bytes: one * 2, + ..Default::default() + }); + let carol = ChatKey::from_seed([4; 32]); + let to_carol = msg_to(&carol); + mb.put("p", m[0].bytes.clone(), &m[0].ack_hash, 3600, NOW).unwrap(); + mb.put("p", m[1].bytes.clone(), &m[1].ack_hash, 3600, NOW).unwrap(); + assert_eq!(mb.put("p", to_carol.bytes.clone(), &to_carol.ack_hash, 3600, NOW), Err(PutError::MailboxFull)); + } + + #[test] + fn persists_across_restart_and_drops_expired() { + let dir = std::env::temp_dir().join(format!("c0dm-mb-{}", rand::random::<u64>())); + let bob = ChatKey::from_seed([5; 32]); + let a = msg_to(&bob); + let b = msg_to(&bob); + { + let mut mb = Mailbox::open(&dir, Limits::default(), NOW); + mb.put("p", a.bytes.clone(), &a.ack_hash, 3600, NOW).unwrap(); + mb.put("p", b.bytes.clone(), &b.ack_hash, 60, NOW).unwrap(); + } + let mb = Mailbox::open(&dir, Limits::default(), NOW + 100); + assert_eq!(mb.count(), 1, "the 60 s message expired across the restart"); + assert_eq!(mb.fetch(&a.msg.tag, NOW + 100)[0].0, a.id); + assert_eq!(mb.total_bytes(), a.bytes.len() as u64); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/node/crates/c0mpute-secure-chat/src/message.rs b/node/crates/c0mpute-secure-chat/src/message.rs index f15e7afe..817a1b89 100644 --- a/node/crates/c0mpute-secure-chat/src/message.rs +++ b/node/crates/c0mpute-secure-chat/src/message.rs @@ -6,6 +6,12 @@ //! read by an attacker. //! //! Signing: Ed25519 + ML-DSA-65 (FIPS 204). Both signatures must verify. +//! +//! Envelope v3 binds every field: the id is a hash over the header *and* the +//! hashes of both ciphertexts, the signatures cover the id, and verification +//! recomputes the id from the fields, so changing any byte (sender, recipient, +//! timestamp, TTL, nonce or ciphertext) fails verification. The header is also +//! the AES-GCM associated data. use aes_gcm::{ Aes256Gcm, Key, Nonce, @@ -30,7 +36,7 @@ use x25519_dalek::{PublicKey as X25519Public, StaticSecret as X25519Secret}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DmEnvelope { pub v: u8, - /// SHA-256 of `from|to|created_at|enc_nonce` (hex). + /// SHA-256 over the header and both ciphertext hashes (hex); see [`envelope_id`]. pub id: String, pub kind: String, pub from: String, @@ -42,7 +48,7 @@ pub struct DmEnvelope { /// Base64url ML-KEM-768 ciphertext (1088 bytes) for PQ shared-secret half. pub ml_kem_ct: String, pub created_at: u64, - /// Seconds the relay should keep this for an offline recipient (max 604800). + /// Seconds a mailbox should keep this for an offline recipient. pub ttl: u64, /// Base64url Ed25519 signature (64 bytes) over `id`. pub sig: String, @@ -70,8 +76,53 @@ pub fn encrypt_dm( sender_ml_dsa_key: &MlDsaSigningKey<MlDsa65>, ttl: u64, ) -> anyhow::Result<DmEnvelope> { - let created_at = unix_now(); + encrypt_dm_at( + from_did, + to_did, + text, + sender_x25519_secret, + recipient_x25519_public, + recipient_ml_kem_ek, + sender_ed25519_key, + sender_ml_dsa_key, + ttl, + unix_now(), + ) +} + +/// Current envelope version. +pub const ENVELOPE_VERSION: u8 = 3; + +fn header_aad(from: &str, to: &str, created_at: u64, ttl: u64) -> Vec<u8> { + format!("c0mpute-dm-v3|{from}|{to}|{created_at}|{ttl}").into_bytes() +} +/// The id an envelope must carry: SHA-256 over its header and the SHA-256 of +/// each ciphertext field. +pub fn envelope_id(e: &DmEnvelope) -> String { + let ct = hex::encode(Sha256::digest(e.ciphertext.as_bytes())); + let kem = hex::encode(Sha256::digest(e.ml_kem_ct.as_bytes())); + let canonical = format!( + "c0mpute-dm-id-v3|{}|{}|{}|{}|{}|{}|{}|{}", + e.kind, e.from, e.to, e.created_at, e.ttl, e.enc_nonce, kem, ct + ); + hex::encode(Sha256::digest(canonical.as_bytes())) +} + +/// [`encrypt_dm`] with an explicit creation time (seconds since the epoch). +#[allow(clippy::too_many_arguments)] +pub fn encrypt_dm_at( + from_did: &str, + to_did: &str, + text: &str, + sender_x25519_secret: &X25519Secret, + recipient_x25519_public: &X25519Public, + recipient_ml_kem_ek: &EncapsulationKey<MlKem768>, + sender_ed25519_key: &SigningKey, + sender_ml_dsa_key: &MlDsaSigningKey<MlDsa65>, + ttl: u64, + created_at: u64, +) -> anyhow::Result<DmEnvelope> { let plaintext_bytes = serde_json::to_vec(&DmPlaintext { text: text.to_string(), thread_id: None, @@ -94,17 +145,37 @@ pub fn encrypt_dm( let aes_key = Key::<Aes256Gcm>::from_slice(&sym_key); let cipher = Aes256Gcm::new(aes_key); let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let aad = header_aad(from_did, to_did, created_at, ttl); let ciphertext = cipher - .encrypt(&nonce, plaintext_bytes.as_ref()) + .encrypt( + &nonce, + aes_gcm::aead::Payload { + msg: plaintext_bytes.as_ref(), + aad: &aad, + }, + ) .map_err(|e| anyhow::anyhow!("aes-gcm encrypt: {e}"))?; let ml_kem_ct_b64 = BASE64_URL_SAFE_NO_PAD.encode(&ml_kem_ct[..]); let enc_nonce_b64 = BASE64_URL_SAFE_NO_PAD.encode(nonce.as_slice()); let ciphertext_b64 = BASE64_URL_SAFE_NO_PAD.encode(&ciphertext); - // Message ID (replay-resistance) - let canonical = format!("{from_did}|{to_did}|{created_at}|{enc_nonce_b64}"); - let id = hex::encode(Sha256::digest(canonical.as_bytes())); + let mut env = DmEnvelope { + v: ENVELOPE_VERSION, + id: String::new(), + kind: "dm".into(), + from: from_did.to_string(), + to: to_did.to_string(), + ciphertext: ciphertext_b64, + enc_nonce: enc_nonce_b64, + ml_kem_ct: ml_kem_ct_b64, + created_at, + ttl, + sig: String::new(), + ml_dsa_sig: String::new(), + }; + // Message ID (replay protection + what the signatures cover). + let id = envelope_id(&env); // Ed25519 signature (classical) let ed_sig: Ed25519Sig = Ed25519Signer::sign(sender_ed25519_key, id.as_bytes()); @@ -117,20 +188,10 @@ pub fn encrypt_dm( let ml_dsa_sig_encoded = ml_dsa_sig.encode(); let ml_dsa_sig_b64 = BASE64_URL_SAFE_NO_PAD.encode(&ml_dsa_sig_encoded[..]); - Ok(DmEnvelope { - v: 2, - id, - kind: "dm".into(), - from: from_did.to_string(), - to: to_did.to_string(), - ciphertext: ciphertext_b64, - enc_nonce: enc_nonce_b64, - ml_kem_ct: ml_kem_ct_b64, - created_at, - ttl, - sig: sig_b64, - ml_dsa_sig: ml_dsa_sig_b64, - }) + env.id = id; + env.sig = sig_b64; + env.ml_dsa_sig = ml_dsa_sig_b64; + Ok(env) } pub fn decrypt_dm( @@ -144,6 +205,7 @@ pub fn decrypt_dm( let ml_kem_ct_bytes = BASE64_URL_SAFE_NO_PAD.decode(&envelope.ml_kem_ct)?; anyhow::ensure!(nonce_bytes.len() == 12, "invalid AES-GCM nonce length"); + anyhow::ensure!(envelope.v == ENVELOPE_VERSION, "unsupported envelope version {}", envelope.v); // Hybrid KEM decapsulation let ecdh_ss = recipient_x25519_secret.diffie_hellman(sender_x25519_public); @@ -163,20 +225,30 @@ pub fn decrypt_dm( let aes_key = Key::<Aes256Gcm>::from_slice(&sym_key); let cipher = Aes256Gcm::new(aes_key); let nonce = Nonce::from_slice(&nonce_bytes); + let aad = header_aad(&envelope.from, &envelope.to, envelope.created_at, envelope.ttl); let plaintext_bytes = cipher - .decrypt(nonce, ciphertext.as_ref()) + .decrypt( + nonce, + aes_gcm::aead::Payload { + msg: ciphertext.as_ref(), + aad: &aad, + }, + ) .map_err(|_| anyhow::anyhow!("decryption failed — wrong key or corrupted message"))?; Ok(serde_json::from_slice(&plaintext_bytes)?) } -/// Verify both the Ed25519 and ML-DSA-65 signatures on a `DmEnvelope`. -/// Both must pass for the message to be considered authentic. +/// Verify both the Ed25519 and ML-DSA-65 signatures on a `DmEnvelope`, after +/// recomputing its id from the fields. Both must pass for the message to be +/// considered authentic. pub fn verify_envelope_sig( envelope: &DmEnvelope, sender_ed25519_public: &VerifyingKey, sender_ml_dsa_public: &MlDsaVerifyingKey<MlDsa65>, ) -> anyhow::Result<()> { + anyhow::ensure!(envelope.v == ENVELOPE_VERSION, "unsupported envelope version {}", envelope.v); + anyhow::ensure!(envelope_id(envelope) == envelope.id, "envelope id does not match its fields (tampered)"); // Ed25519 signature (classical) let sig_bytes = BASE64_URL_SAFE_NO_PAD.decode(&envelope.sig)?; anyhow::ensure!(sig_bytes.len() == 64, "invalid ed25519 signature length"); @@ -290,4 +362,39 @@ mod tests { let result = decrypt_dm(&envelope, &bob.x25519_secret, &alice.x25519_public, &eve.ml_kem_dk); assert!(result.is_err()); } + + #[test] + fn tampered_fields_fail_verification() { + let alice = ChatKey::generate(); + let bob = ChatKey::generate(); + let env = encrypt_dm( + "alice", "bob", "pay 5", &alice.x25519_secret, &bob.x25519_public, bob.ml_kem_ek(), + &alice.ed25519_key, &alice.ml_dsa_key, 3600, + ) + .unwrap(); + let ok = |e: &DmEnvelope| verify_envelope_sig(e, &alice.ed25519_public, &alice.ml_dsa_public).is_ok(); + assert!(ok(&env)); + let mut t = env.clone(); + t.created_at += 1; + assert!(!ok(&t)); + let mut t = env.clone(); + t.to = "eve".into(); + assert!(!ok(&t)); + let mut t = env.clone(); + t.ttl = 1; + assert!(!ok(&t)); + // Flip one ciphertext character. + let mut t = env.clone(); + let mut c: Vec<char> = t.ciphertext.chars().collect(); + c[3] = if c[3] == 'A' { 'B' } else { 'A' }; + t.ciphertext = c.into_iter().collect(); + assert!(!ok(&t)); + // Even with the id recomputed, the signatures no longer match. + t.id = envelope_id(&t); + assert!(!ok(&t)); + // And the AEAD rejects the header change on decrypt. + let mut t = env.clone(); + t.created_at += 1; + assert!(decrypt_dm(&t, &bob.x25519_secret, &alice.x25519_public, &bob.ml_kem_dk).is_err()); + } } diff --git a/node/crates/c0mpute-secure-chat/src/placement.rs b/node/crates/c0mpute-secure-chat/src/placement.rs new file mode 100644 index 00000000..d75bfd64 --- /dev/null +++ b/node/crates/c0mpute-secure-chat/src/placement.rs @@ -0,0 +1,88 @@ +//! Which mailbox nodes hold a key: rendezvous (highest random weight) +//! hashing, the same scheme hosting uses for site replicas. Sender and +//! recipient compute the same ranking from the same peer set with no +//! coordination; when the set changes, only the keys whose top nodes changed +//! move. + +use sha2::{Digest, Sha256}; + +/// Mailboxes a sender stores each message on. +pub const STORE_REPLICAS: usize = 3; +/// Mailboxes a recipient polls per tag: the store set plus two, so a pull +/// still finds messages when the peer set churned a little since the send. +pub const PULL_REPLICAS: usize = STORE_REPLICAS + 2; + +fn score(key: &str, peer: &str) -> [u8; 32] { + let mut h = Sha256::new(); + h.update(b"c0mpute-dm-rendezvous-v1|"); + h.update(key.as_bytes()); + h.update(b"|"); + h.update(peer.as_bytes()); + h.finalize().into() +} + +/// All `peers`, best-ranked first, for `key`. Duplicates are removed. +pub fn rank<S: AsRef<str>>(key: &str, peers: &[S]) -> Vec<String> { + let mut v: Vec<(String, [u8; 32])> = peers.iter().map(|p| (p.as_ref().to_string(), score(key, p.as_ref()))).collect(); + v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + v.dedup_by(|a, b| a.0 == b.0); + v.into_iter().map(|(p, _)| p).collect() +} + +/// The top `n` of [`rank`]. +pub fn place<S: AsRef<str>>(key: &str, peers: &[S], n: usize) -> Vec<String> { + let mut r = rank(key, peers); + r.truncate(n); + r +} + +#[cfg(test)] +mod tests { + use super::*; + + fn peers(n: usize) -> Vec<String> { + (0..n).map(|i| format!("12D3KooPeer{i:03}")).collect() + } + + #[test] + fn placement_is_deterministic_and_order_independent() { + let p = peers(20); + let mut shuffled = p.clone(); + shuffled.reverse(); + shuffled.rotate_left(7); + let a = place("tag-1", &p, STORE_REPLICAS); + let b = place("tag-1", &shuffled, STORE_REPLICAS); + assert_eq!(a, b); + assert_eq!(a.len(), 3); + // Different keys land on different sets (overwhelmingly). + let spread: std::collections::HashSet<Vec<String>> = + (0..50).map(|i| place(&format!("tag-{i}"), &p, 3)).collect(); + assert!(spread.len() > 40); + // Fewer peers than replicas: all of them. + assert_eq!(place("k", &peers(2), 3).len(), 2); + // Duplicates collapse. + let dup = vec!["a".to_string(), "a".to_string(), "b".to_string()]; + assert_eq!(rank("k", &dup).len(), 2); + } + + #[test] + fn churn_moves_only_affected_keys_and_pull_set_covers_store_set() { + let p = peers(20); + let mut grown = p.clone(); + grown.push("12D3KooNewcomer".into()); + let mut moved = 0; + for i in 0..200 { + let key = format!("tag-{i}"); + let before = place(&key, &p, STORE_REPLICAS); + let after_pull = place(&key, &grown, PULL_REPLICAS); + // A message stored before the newcomer joined is still found by + // a pull after: at most one slot shifts. + assert!(before.iter().all(|x| after_pull.contains(x)), "{key}"); + if place(&key, &grown, STORE_REPLICAS) != before { + moved += 1; + } + } + // Roughly 3/21 of keys should be affected. + assert!(moved < 60, "{moved} keys moved"); + } +} diff --git a/node/crates/c0mpute-secure-chat/src/sealed.rs b/node/crates/c0mpute-secure-chat/src/sealed.rs new file mode 100644 index 00000000..24f9b5e8 --- /dev/null +++ b/node/crates/c0mpute-secure-chat/src/sealed.rs @@ -0,0 +1,423 @@ +//! Sealed messages: what actually crosses the network. +//! +//! A [`DmEnvelope`] (signed, hybrid-encrypted, from [`crate::message`]) names +//! its sender and recipient, so it never travels bare. It is wrapped, with +//! the sender's public identity and a random ack secret, in a second +//! encryption layer to the recipient from a fresh ephemeral X25519 key plus a +//! fresh ML-KEM-768 encapsulation. The outer [`SealedMessage`] carries only: +//! +//! * `tag`: SHA-256 of the recipient address and the current epoch +//! ([`recipient_tag`]), the mailbox routing key. It changes every +//! [`EPOCH_SECS`], so a mailbox cannot link one recipient's messages across +//! epochs without already knowing the recipient's address; +//! * the ephemeral public key, KEM ciphertext, nonce and ciphertext. +//! +//! No sender, no timestamp, no plaintext length beyond the total size. The +//! mailbox stores the message under `id = SHA-256(bytes)` with `ack_hash = +//! SHA-256(ack secret)`; only the recipient can read the secret, so only the +//! recipient can delete the message. + +use std::collections::BTreeMap; + +use aes_gcm::{ + Aes256Gcm, Key, Nonce, + aead::{Aead, AeadCore, KeyInit, Payload}, +}; +use base64::prelude::*; +use hkdf::Hkdf; +use ml_kem::{Decapsulate, Encapsulate}; +use rand::RngCore; +use rand::rngs::OsRng; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use x25519_dalek::{PublicKey as X25519Public, StaticSecret as X25519Secret}; + +use crate::identity::{CLOCK_SKEW_SECS, PublicIdentity}; +use crate::keys::ChatKey; +use crate::message::{DmEnvelope, decrypt_dm, encrypt_dm_at, verify_envelope_sig}; + +/// Current sealed-message format. +pub const SEALED_VERSION: u8 = 1; +/// Recipient tags rotate this often. +pub const EPOCH_SECS: u64 = 7 * 86_400; +/// Longest a mailbox keeps a message, and the replay window. +pub const MAX_TTL_SECS: u64 = 30 * 86_400; +/// Default time a sender asks mailboxes to keep a message. +pub const DEFAULT_TTL_SECS: u64 = MAX_TTL_SECS; +/// Longest message text accepted. +pub const MAX_TEXT_BYTES: usize = 16 * 1024; +/// Largest sealed message on the wire (a 16 KiB text seals to about 40 KiB). +pub const MAX_SEALED_BYTES: usize = 64 * 1024; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SealedMessage { + pub v: u8, + /// Hex recipient tag (32 bytes). + pub tag: String, + /// Base64url ephemeral X25519 public key. + pub eph: String, + /// Base64url ML-KEM-768 ciphertext. + pub kem_ct: String, + /// Base64url AES-256-GCM nonce. + pub nonce: String, + /// Base64url AES-256-GCM ciphertext of [`SealedInner`]. + pub ct: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct SealedInner { + v: u8, + sender: PublicIdentity, + envelope: DmEnvelope, + /// Hex 32-byte secret whose SHA-256 the mailbox holds; revealing it + /// deletes the message. + ack: String, +} + +/// A sealed message ready to hand to a mailbox or a home node. +pub struct Sealed { + pub msg: SealedMessage, + pub bytes: Vec<u8>, + /// SHA-256 of `bytes` (hex): the mailbox id. + pub id: String, + /// SHA-256 of the ack secret (hex). + pub ack_hash: String, + /// The inner envelope id (what the recipient deduplicates on). + pub envelope_id: String, +} + +/// A message opened, verified and replay-checked. +#[derive(Clone, Debug)] +pub struct Opened { + pub sender: PublicIdentity, + pub sender_address: String, + pub text: String, + pub envelope_id: String, + pub created_at: u64, + /// Hex ack secret: send it to the mailbox to delete the message. + pub ack: String, +} + +pub fn epoch_of(t: u64) -> u64 { + t / EPOCH_SECS +} + +/// The mailbox routing key for `address` in `epoch`. +pub fn recipient_tag(address: &str, epoch: u64) -> String { + hex::encode(Sha256::digest(format!("c0mpute-dm-tag-v1|{address}|{epoch}").as_bytes())) +} + +/// Every tag a message to `address` could still be stored under at `now`: +/// from the epoch `MAX_TTL_SECS` ago through the next one (clock skew). +pub fn pull_tags(address: &str, now: u64) -> Vec<String> { + let first = epoch_of(now.saturating_sub(MAX_TTL_SECS + CLOCK_SKEW_SECS)); + let last = epoch_of(now + CLOCK_SKEW_SECS); + (first..=last).map(|e| recipient_tag(address, e)).collect() +} + +pub fn sha256_hex(b: &[u8]) -> String { + hex::encode(Sha256::digest(b)) +} + +fn outer_key(eph_ss: &[u8; 32], kem_ss: &[u8], eph_pub: &[u8; 32], tag: &str) -> [u8; 32] { + let mut ikm = Vec::with_capacity(64); + ikm.extend_from_slice(eph_ss); + ikm.extend_from_slice(kem_ss); + let hk = Hkdf::<Sha256>::new(Some(eph_pub), &ikm); + let mut k = [0u8; 32]; + hk.expand(format!("c0mpute-dm-seal-v1|{tag}").as_bytes(), &mut k) + .expect("32 bytes is a valid HKDF length"); + k +} + +fn outer_aad(tag: &str) -> Vec<u8> { + format!("c0mpute-dm-sealed-v{SEALED_VERSION}|{tag}").into_bytes() +} + +/// Encrypt, sign and seal `text` from `sender` to `recipient` at time `now`. +pub fn seal(sender: &ChatKey, recipient: &PublicIdentity, text: &str, ttl: u64, now: u64) -> anyhow::Result<Sealed> { + anyhow::ensure!(text.len() <= MAX_TEXT_BYTES, "message longer than {MAX_TEXT_BYTES} bytes"); + let ttl = ttl.clamp(60, MAX_TTL_SECS); + let rk = recipient.keys()?; + let sender_id = sender.public_identity(); + let to = recipient.address(); + let envelope = encrypt_dm_at( + &sender_id.address(), + &to, + text, + &sender.x25519_secret, + &rk.x25519, + &rk.ml_kem, + &sender.ed25519_key, + &sender.ml_dsa_key, + ttl, + now, + )?; + let mut ack = [0u8; 32]; + OsRng.fill_bytes(&mut ack); + let inner = SealedInner { + v: SEALED_VERSION, + sender: sender_id, + + envelope: envelope.clone(), + ack: hex::encode(ack), + }; + let inner_bytes = serde_json::to_vec(&inner)?; + + let tag = recipient_tag(&to, epoch_of(now)); + let eph = X25519Secret::random_from_rng(OsRng); + let eph_pub = X25519Public::from(&eph); + let eph_ss = eph.diffie_hellman(&rk.x25519); + let (kem_ct, kem_ss) = rk.ml_kem.encapsulate(); + let key = outer_key(eph_ss.as_bytes(), kem_ss.as_ref(), eph_pub.as_bytes(), &tag); + let cipher = Aes256Gcm::new(Key::<Aes256Gcm>::from_slice(&key)); + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let aad = outer_aad(&tag); + let ct = cipher + .encrypt( + &nonce, + Payload { + msg: &inner_bytes, + aad: &aad, + }, + ) + .map_err(|e| anyhow::anyhow!("seal: {e}"))?; + let msg = SealedMessage { + v: SEALED_VERSION, + tag, + eph: BASE64_URL_SAFE_NO_PAD.encode(eph_pub.as_bytes()), + kem_ct: BASE64_URL_SAFE_NO_PAD.encode(&kem_ct[..]), + nonce: BASE64_URL_SAFE_NO_PAD.encode(nonce.as_slice()), + ct: BASE64_URL_SAFE_NO_PAD.encode(&ct), + }; + let bytes = serde_json::to_vec(&msg)?; + anyhow::ensure!(bytes.len() <= MAX_SEALED_BYTES, "sealed message too large ({} bytes)", bytes.len()); + Ok(Sealed { + id: sha256_hex(&bytes), + ack_hash: sha256_hex(&ack), + envelope_id: envelope.id, + msg, + bytes, + }) +} + +impl SealedMessage { + /// Parse and sanity-check wire bytes (what a mailbox does: no keys). + pub fn parse(bytes: &[u8]) -> anyhow::Result<Self> { + anyhow::ensure!(bytes.len() <= MAX_SEALED_BYTES, "sealed message too large"); + let m: Self = serde_json::from_slice(bytes)?; + anyhow::ensure!(m.v == SEALED_VERSION, "unsupported sealed message version {}", m.v); + anyhow::ensure!(is_hex32(&m.tag), "bad tag"); + Ok(m) + } +} + +pub fn is_hex32(s: &str) -> bool { + s.len() == 64 && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// Decrypt and verify a sealed message addressed to `me`, without replay +/// checks (see [`open`]). +pub fn unseal(me: &ChatKey, msg: &SealedMessage, now: u64) -> anyhow::Result<Opened> { + anyhow::ensure!(msg.v == SEALED_VERSION, "unsupported sealed message version {}", msg.v); + let my_address = me.address(); + anyhow::ensure!( + pull_tags(&my_address, now).contains(&msg.tag), + "not addressed to this key (or outside the message window)" + ); + let eph = BASE64_URL_SAFE_NO_PAD.decode(&msg.eph)?; + let eph: [u8; 32] = eph.as_slice().try_into().map_err(|_| anyhow::anyhow!("bad ephemeral key"))?; + let eph_pub = X25519Public::from(eph); + let eph_ss = me.x25519_secret.diffie_hellman(&eph_pub); + let kem_ct = BASE64_URL_SAFE_NO_PAD.decode(&msg.kem_ct)?; + let kem_ss = me + .ml_kem_dk + .decapsulate_slice(&kem_ct) + .map_err(|_| anyhow::anyhow!("bad ml-kem ciphertext"))?; + let key = outer_key(eph_ss.as_bytes(), kem_ss.as_ref(), eph_pub.as_bytes(), &msg.tag); + let nonce = BASE64_URL_SAFE_NO_PAD.decode(&msg.nonce)?; + anyhow::ensure!(nonce.len() == 12, "bad nonce"); + let ct = BASE64_URL_SAFE_NO_PAD.decode(&msg.ct)?; + let cipher = Aes256Gcm::new(Key::<Aes256Gcm>::from_slice(&key)); + let aad = outer_aad(&msg.tag); + let inner_bytes = cipher + .decrypt(Nonce::from_slice(&nonce), Payload { msg: &ct, aad: &aad }) + .map_err(|_| anyhow::anyhow!("cannot open: not for this key, or tampered"))?; + let inner: SealedInner = serde_json::from_slice(&inner_bytes)?; + anyhow::ensure!(inner.v == SEALED_VERSION, "unsupported inner version"); + anyhow::ensure!(is_hex32(&inner.ack), "bad ack secret"); + + let env = &inner.envelope; + let sender_address = inner.sender.address(); + anyhow::ensure!(env.from == sender_address, "envelope sender does not match the enclosed identity"); + anyhow::ensure!(env.to == my_address, "envelope is addressed to someone else"); + let sk = inner.sender.keys()?; + verify_envelope_sig(env, &sk.ed25519, &sk.ml_dsa)?; + let plain = decrypt_dm(env, &me.x25519_secret, &sk.x25519, &me.ml_kem_dk)?; + Ok(Opened { + sender: inner.sender, + sender_address, + text: plain.text, + envelope_id: env.id.clone(), + created_at: env.created_at, + ack: inner.ack, + }) +} + +/// [`unseal`], then reject replays: a timestamp outside the window, or an +/// envelope id already seen. Records the id on success. +pub fn open(me: &ChatKey, msg: &SealedMessage, now: u64, guard: &mut ReplayGuard) -> anyhow::Result<Opened> { + let o = unseal(me, msg, now)?; + guard.check(&o.envelope_id, o.created_at, now)?; + guard.record(&o.envelope_id, o.created_at); + Ok(o) +} + +/// Envelope ids seen inside the replay window. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct ReplayGuard { + pub seen: BTreeMap<String, u64>, +} + +impl ReplayGuard { + pub fn check(&self, id: &str, created_at: u64, now: u64) -> anyhow::Result<()> { + anyhow::ensure!(created_at <= now + CLOCK_SKEW_SECS, "message timestamp is in the future"); + anyhow::ensure!( + created_at + MAX_TTL_SECS + CLOCK_SKEW_SECS >= now, + "message is older than the replay window" + ); + anyhow::ensure!(!self.seen.contains_key(id), "replayed message (already received)"); + Ok(()) + } + + pub fn record(&mut self, id: &str, created_at: u64) { + self.seen.insert(id.to_string(), created_at); + } + + /// Forget ids older than the window: their timestamps alone reject them. + pub fn prune(&mut self, now: u64) { + let floor = now.saturating_sub(MAX_TTL_SECS + 2 * CLOCK_SKEW_SECS); + self.seen.retain(|_, t| *t >= floor); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const NOW: u64 = 1_800_000_000; + + #[test] + fn seal_open_roundtrip_hides_metadata() { + let alice = ChatKey::generate(); + let bob = ChatKey::generate(); + let s = seal(&alice, &bob.public_identity(), "hi bob", 3600, NOW).unwrap(); + let wire = String::from_utf8(s.bytes.clone()).unwrap(); + assert!(!wire.contains(&alice.address()), "sender address visible on the wire"); + assert!(!wire.contains(&bob.address()), "recipient address visible on the wire"); + assert!(!wire.contains("hi bob")); + assert_eq!(s.msg.tag, recipient_tag(&bob.address(), epoch_of(NOW))); + assert_eq!(s.id, sha256_hex(&s.bytes)); + + let parsed = SealedMessage::parse(&s.bytes).unwrap(); + let mut guard = ReplayGuard::default(); + let o = open(&bob, &parsed, NOW + 10, &mut guard).unwrap(); + assert_eq!(o.text, "hi bob"); + assert_eq!(o.sender_address, alice.address()); + assert_eq!(sha256_hex(&hex::decode(&o.ack).unwrap()), s.ack_hash); + + // Replay of the same message is rejected. + let err = open(&bob, &parsed, NOW + 20, &mut guard).unwrap_err(); + assert!(err.to_string().contains("replayed"), "{err}"); + // Eve cannot open it. + let eve = ChatKey::generate(); + assert!(unseal(&eve, &parsed, NOW).is_err()); + } + + #[test] + fn tampering_and_stale_messages_are_rejected() { + let alice = ChatKey::generate(); + let bob = ChatKey::generate(); + let s = seal(&alice, &bob.public_identity(), "hello", 3600, NOW).unwrap(); + // Flip a ciphertext byte. + let mut t = s.msg.clone(); + let mut ct = BASE64_URL_SAFE_NO_PAD.decode(&t.ct).unwrap(); + ct[10] ^= 1; + t.ct = BASE64_URL_SAFE_NO_PAD.encode(ct); + assert!(unseal(&bob, &t, NOW).is_err()); + // Move it to another tag (AAD-bound). + let mut t = s.msg.clone(); + t.tag = recipient_tag(&bob.address(), epoch_of(NOW) + 1); + assert!(unseal(&bob, &t, NOW).is_err()); + // Past the window: tag no longer pulled, and the guard says too old. + assert!(unseal(&bob, &s.msg, NOW + MAX_TTL_SECS + EPOCH_SECS * 2).is_err()); + let guard = ReplayGuard::default(); + assert!(guard.check("x", NOW, NOW + MAX_TTL_SECS + 10 * CLOCK_SKEW_SECS).is_err()); + assert!(guard.check("x", NOW + 2 * CLOCK_SKEW_SECS, NOW).is_err(), "future"); + assert!(guard.check("x", NOW, NOW + 100).is_ok()); + } + + #[test] + fn forged_sender_is_rejected() { + // Mallory seals a message whose inner identity claims to be Alice but + // signs with her own key: the signatures do not verify. + let alice = ChatKey::generate(); + let bob = ChatKey::generate(); + let mallory = ChatKey::generate(); + let s = seal(&mallory, &bob.public_identity(), "trust me", 3600, NOW).unwrap(); + // Re-seal with the inner sender swapped for Alice's identity. + let mut forged = s.msg.clone(); + { + // Decrypt the outer layer as Bob would, swap the sender, re-encrypt. + let o = unseal(&bob, &forged, NOW).unwrap(); + assert_eq!(o.sender_address, mallory.address()); + } + let inner_forged = { + let rk = bob.public_identity().keys().unwrap(); + let env = crate::message::encrypt_dm_at( + &alice.address(), &bob.address(), "trust me", &mallory.x25519_secret, &rk.x25519, &rk.ml_kem, + &mallory.ed25519_key, &mallory.ml_dsa_key, 3600, NOW, + ) + .unwrap(); + SealedInner { v: 1, sender: alice.public_identity(), envelope: env, ack: "00".repeat(32) } + }; + let rk = bob.public_identity().keys().unwrap(); + let eph = X25519Secret::random_from_rng(OsRng); + let eph_pub = X25519Public::from(&eph); + let (kem_ct, kem_ss) = rk.ml_kem.encapsulate(); + let key = outer_key(eph.diffie_hellman(&rk.x25519).as_bytes(), kem_ss.as_ref(), eph_pub.as_bytes(), &forged.tag); + let cipher = Aes256Gcm::new(Key::<Aes256Gcm>::from_slice(&key)); + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let ct = cipher + .encrypt(&nonce, Payload { msg: &serde_json::to_vec(&inner_forged).unwrap(), aad: &outer_aad(&forged.tag) }) + .unwrap(); + forged.eph = BASE64_URL_SAFE_NO_PAD.encode(eph_pub.as_bytes()); + forged.kem_ct = BASE64_URL_SAFE_NO_PAD.encode(&kem_ct[..]); + forged.nonce = BASE64_URL_SAFE_NO_PAD.encode(nonce.as_slice()); + forged.ct = BASE64_URL_SAFE_NO_PAD.encode(ct); + let err = unseal(&bob, &forged, NOW).unwrap_err(); + assert!(err.to_string().contains("signature"), "{err}"); + } + + #[test] + fn tags_cover_the_whole_ttl() { + let addr = "ab".repeat(32); + let sent_at = NOW; + let tag = recipient_tag(&addr, epoch_of(sent_at)); + for later in [0, 3600, 7 * 86_400, MAX_TTL_SECS] { + assert!(pull_tags(&addr, sent_at + later).contains(&tag), "+{later}s"); + } + assert!(pull_tags(&addr, NOW).len() <= 7); + assert_ne!(recipient_tag(&addr, 1), recipient_tag(&addr, 2)); + assert!(is_hex32(&tag)); + } + + #[test] + fn oversized_text_is_refused() { + let a = ChatKey::generate(); + let b = ChatKey::generate(); + assert!(seal(&a, &b.public_identity(), &"x".repeat(MAX_TEXT_BYTES + 1), 3600, NOW).is_err()); + let s = seal(&a, &b.public_identity(), &"x".repeat(MAX_TEXT_BYTES), 3600, NOW).unwrap(); + assert!(s.bytes.len() <= MAX_SEALED_BYTES); + } +} diff --git a/node/crates/c0mpute-secure-chat/src/state.rs b/node/crates/c0mpute-secure-chat/src/state.rs new file mode 100644 index 00000000..3f443cd2 --- /dev/null +++ b/node/crates/c0mpute-secure-chat/src/state.rs @@ -0,0 +1,268 @@ +//! The client's local state: contacts, blocklist, cached keys, inbox and the +//! replay guard, encrypted at rest under a key derived from the chat seed +//! (AES-256-GCM, HKDF label `secure-chat-local-state-v1`). Without the key +//! password the file reveals nothing, not even who your contacts are. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use aes_gcm::{ + Aes256Gcm, Key, Nonce, + aead::{Aead, AeadCore, KeyInit}, +}; +use base64::prelude::*; +use hkdf::Hkdf; +use rand::rngs::OsRng; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use zeroize::Zeroizing; + +use crate::identity::{PublicIdentity, matches_query, normalize_query}; +use crate::keys::ChatKey; +use crate::sealed::{Opened, ReplayGuard, SealedMessage, open}; + +/// Inbox entries kept (oldest dropped first). +pub const MAX_INBOX: usize = 5_000; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Contact { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option<String>, + pub identity: PublicIdentity, + pub added_at: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct InboxEntry { + /// Envelope id. + pub id: String, + /// Sender address. + pub from: String, + pub text: String, + pub created_at: u64, + pub received_at: u64, + /// `direct` (pushed to this user's home node) or `mailbox`. + pub via: String, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct LocalState { + #[serde(default)] + pub contacts: BTreeMap<String, Contact>, + #[serde(default)] + pub blocked: BTreeSet<String>, + /// Identities learned from lookups and received messages. + #[serde(default)] + pub known: BTreeMap<String, PublicIdentity>, + #[serde(default)] + pub inbox: Vec<InboxEntry>, + #[serde(default)] + pub replay: ReplayGuard, +} + +#[derive(Serialize, Deserialize)] +struct EncryptedFile { + v: u8, + nonce: String, + ct: String, +} + +/// What happened to one received message. +#[derive(Debug)] +pub enum Received { + /// New message, now in the inbox. + New(Opened), + /// From a blocked sender: dropped (still acked so it is deleted). + Blocked(Opened), +} + +impl Received { + pub fn ack(&self) -> &str { + match self { + Received::New(o) | Received::Blocked(o) => &o.ack, + } + } +} + +impl ChatKey { + fn local_state_key(&self) -> Zeroizing<[u8; 32]> { + let hk = Hkdf::<Sha256>::new(None, self.seed.as_ref()); + let mut k = Zeroizing::new([0u8; 32]); + hk.expand(b"secure-chat-local-state-v1", k.as_mut()) + .expect("32 bytes is a valid HKDF length"); + k + } +} + +/// `chat-state.enc` next to `chat.key`. +pub fn state_file_path() -> anyhow::Result<PathBuf> { + Ok(crate::keys::key_file_path()?.with_file_name("chat-state.enc")) +} + +impl LocalState { + /// Load and decrypt; a missing file is an empty state. + pub fn load(path: &Path, key: &ChatKey) -> anyhow::Result<Self> { + let raw = match std::fs::read(path) { + Ok(r) => r, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Self::default()), + Err(e) => return Err(anyhow::anyhow!("read {}: {e}", path.display())), + }; + let f: EncryptedFile = serde_json::from_slice(&raw)?; + anyhow::ensure!(f.v == 1, "unsupported state file version"); + let k = key.local_state_key(); + let cipher = Aes256Gcm::new(Key::<Aes256Gcm>::from_slice(k.as_ref())); + let nonce = BASE64_URL_SAFE_NO_PAD.decode(&f.nonce)?; + anyhow::ensure!(nonce.len() == 12, "bad nonce"); + let plain = Zeroizing::new( + cipher + .decrypt(Nonce::from_slice(&nonce), BASE64_URL_SAFE_NO_PAD.decode(&f.ct)?.as_ref()) + .map_err(|_| anyhow::anyhow!("cannot decrypt {} with this key", path.display()))?, + ); + Ok(serde_json::from_slice(&plain)?) + } + + /// Encrypt and write atomically (0600). + pub fn save(&self, path: &Path, key: &ChatKey) -> anyhow::Result<()> { + let plain = Zeroizing::new(serde_json::to_vec(self)?); + let k = key.local_state_key(); + let cipher = Aes256Gcm::new(Key::<Aes256Gcm>::from_slice(k.as_ref())); + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let ct = cipher + .encrypt(&nonce, plain.as_ref()) + .map_err(|e| anyhow::anyhow!("encrypt state: {e}"))?; + let f = EncryptedFile { + v: 1, + nonce: BASE64_URL_SAFE_NO_PAD.encode(nonce.as_slice()), + ct: BASE64_URL_SAFE_NO_PAD.encode(ct), + }; + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir)?; + } + let tmp = path.with_extension("tmp"); + crate::keys::write_secret_file(&tmp, &serde_json::to_vec(&f)?)?; + std::fs::rename(&tmp, path)?; + Ok(()) + } + + /// Resolve what a user typed (contact name, address, fingerprint, or a + /// `c0dm1.` share string) from local knowledge only. + pub fn resolve(&self, query: &str) -> Option<PublicIdentity> { + if let Ok(id) = PublicIdentity::from_share_string(query) { + return Some(id); + } + if let Some(c) = self.contacts.values().find(|c| c.name.as_deref() == Some(query.trim())) { + return Some(c.identity.clone()); + } + let q = normalize_query(query)?; + self.contacts + .values() + .map(|c| &c.identity) + .chain(self.known.values()) + .find(|id| matches_query(id, &q)) + .cloned() + } + + pub fn remember(&mut self, id: &PublicIdentity) { + if self.known.len() >= 10_000 + && let Some(first) = self.known.keys().next().cloned() + { + self.known.remove(&first); + } + self.known.insert(id.address(), id.clone()); + } + + pub fn add_contact(&mut self, id: PublicIdentity, name: Option<String>, now: u64) -> String { + let addr = id.address(); + self.contacts.insert( + addr.clone(), + Contact { + name, + identity: id, + added_at: now, + }, + ); + addr + } + + /// Display name for an address: contact name, else its fingerprint. + pub fn name_of(&self, address: &str) -> String { + match self.contacts.get(address).and_then(|c| c.name.clone()) { + Some(n) => n, + None => crate::identity::fingerprint_of(address), + } + } + + /// Open a sealed message for `me`: verify, replay-check, apply the + /// blocklist, and file it in the inbox. + pub fn receive(&mut self, me: &ChatKey, msg: &SealedMessage, via: &str, now: u64) -> anyhow::Result<Received> { + self.replay.prune(now); + let o = open(me, msg, now, &mut self.replay)?; + if self.blocked.contains(&o.sender_address) { + return Ok(Received::Blocked(o)); + } + self.remember(&o.sender); + self.inbox.push(InboxEntry { + id: o.envelope_id.clone(), + from: o.sender_address.clone(), + text: o.text.clone(), + created_at: o.created_at, + received_at: now, + via: via.to_string(), + }); + if self.inbox.len() > MAX_INBOX { + let extra = self.inbox.len() - MAX_INBOX; + self.inbox.drain(..extra); + } + Ok(Received::New(o)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sealed::seal; + + const NOW: u64 = 1_800_000_000; + + #[test] + fn state_is_encrypted_at_rest_and_roundtrips() { + let dir = std::env::temp_dir().join(format!("c0dm-state-{}", rand::random::<u64>())); + let path = dir.join("chat-state.enc"); + let me = ChatKey::generate(); + let friend = ChatKey::generate(); + let mut st = LocalState::default(); + st.add_contact(friend.public_identity(), Some("carol".into()), NOW); + st.save(&path, &me).unwrap(); + let raw = std::fs::read_to_string(&path).unwrap(); + assert!(!raw.contains("carol")); + assert!(!raw.contains(&friend.address())); + let back = LocalState::load(&path, &me).unwrap(); + assert_eq!(back.contacts.len(), 1); + assert!(LocalState::load(&path, &ChatKey::generate()).is_err(), "wrong key"); + // Resolution by name, fingerprint, address. + assert_eq!(back.resolve("carol").unwrap().address(), friend.address()); + assert_eq!(back.resolve(&friend.public_identity().fingerprint()).unwrap().address(), friend.address()); + assert_eq!(back.resolve(&friend.address()).unwrap().address(), friend.address()); + assert!(back.resolve("dave").is_none()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn blocked_senders_are_dropped_and_replays_rejected() { + let bob = ChatKey::generate(); + let alice = ChatKey::generate(); + let mallory = ChatKey::generate(); + let mut st = LocalState::default(); + st.blocked.insert(mallory.address()); + + let good = seal(&alice, &bob.public_identity(), "hello", 3600, NOW).unwrap(); + let spam = seal(&mallory, &bob.public_identity(), "buy now", 3600, NOW).unwrap(); + assert!(matches!(st.receive(&bob, &good.msg, "mailbox", NOW).unwrap(), Received::New(_))); + assert!(matches!(st.receive(&bob, &spam.msg, "mailbox", NOW).unwrap(), Received::Blocked(_))); + assert_eq!(st.inbox.len(), 1); + assert_eq!(st.inbox[0].text, "hello"); + assert!(st.receive(&bob, &good.msg, "direct", NOW + 5).is_err(), "replay"); + assert_eq!(st.inbox.len(), 1); + assert!(st.known.contains_key(&alice.address())); + } +} diff --git a/plugins/secure-chat/module.toml b/plugins/secure-chat/module.toml index 68a78ca6..99836b31 100644 --- a/plugins/secure-chat/module.toml +++ b/plugins/secure-chat/module.toml @@ -1,19 +1,22 @@ -# c0mpute plugin manifest. See dips/0006-module-model.md. +# c0mpute plugin manifest. See dips/0006-module-model.md and +# dips/0018-secure-chat-plugin.md. # -# CANDIDATE plugin — implementation not yet scaffolded. -# See dips/0018-secure-chat-plugin.md for the full PRD. +# Built in: crypto + client state in node/crates/c0mpute-secure-chat, the +# network service in node/crates/c0mpute-core/src/dm.rs, the CLI in +# node/crates/c0mpute-cli/src/dm_cli.rs. # -# Brief for the implementer: -# E2E encrypted p2p messaging, no central database. -# Crypto stack: X25519 + Ed25519 (derived from a single seed), -# NaCl crypto_box for DMs, XChaCha20-Poly1305 for group keys, -# Argon2id-AES256-GCM for password-encrypted key backups. -# Transport: c0mpute gossip (message type 0x20) + WebSocket relay. -# Store-and-forward: nodes hold encrypted envelopes up to TTL (max 7d). -# Public key discovery: DHT keyed on SHA256("secure-chat-key:" + did). -# Identity: CoinPay DID (DIP-0007) is the canonical identity. -# First-party in-tree; compile into the c0mpute binary alongside transcode. -# Implementation target: node/crates/c0mpute-secure-chat +# What runs where: +# * Every worker/gateway serves the `mailbox` role by default: it holds +# sealed messages for offline recipients (bounded) and caches +# self-published key records. Mailboxes gossip on c0mpute/dm/v1 and +# answer over the /c0mpute/rpc/1.0.0 request/response protocol. +# * `c0mpute dm send` delivers straight to the recipient's home node when it +# is online, otherwise stores on 3 mailboxes picked by rendezvous hashing +# on the recipient tag. `c0mpute dm pull` polls the same nodes, opens, +# verifies, and acks (deletes). +# +# Not in this release (DIP-0018 later phases): group chats, attachments, +# web/TUI surfaces, CoinPay DID binding, Double Ratchet forward secrecy. [module] id = "secure-chat" @@ -21,43 +24,42 @@ name = "Secure Chat — E2E encrypted p2p messaging" version = "0.2.81" kind = "service" category = "applications" -status = "partial" -status_note = "Key generation works; sending and receiving messages is not live yet." -description = "End-to-end encrypted peer-to-peer messaging across the c0mpute network. No central server. Keys stay with the user. Identity and reputation via CoinPay DID. Group chats, sealed-sender mode, and offline store-and-forward included." +status = "available" +builtin = true +description = "End-to-end encrypted one-to-one messages between c0mpute users, peer to peer with no central server. Your address is your public key (hybrid X25519 + ML-KEM-768 encryption, Ed25519 + ML-DSA-65 signatures). Messages go straight to the recipient's node when it is online, or wait sealed on three mailbox nodes for up to 30 days; mailboxes never see who sent a message, who it is for, or what it says. Contacts, blocklist and inbox stay on your machine, encrypted under your key. Keys are found by fingerprint from self-signed records nodes gossip. One-to-one text only for now: no group chats, attachments or web UI yet." author = "Profullstack" license = "MIT" source = "https://github.com/profullstack/c0mpute/tree/master/plugins/secure-chat" -keywords = ["chat", "messaging", "e2e", "encryption", "p2p", "did", "privacy"] +keywords = ["chat", "messaging", "e2e", "encryption", "p2p", "post-quantum", "privacy", "mailbox"] [module.requirements] -c0mpute = ">=0.1.0" +c0mpute = ">=NEXT" os = ["linux", "darwin"] arch = ["x86_64", "aarch64"] capabilities = ["network:ingress", "network:egress", "storage:ephemeral"] [module.workloads] -# No one-shot workloads. secure-chat is a continuous service role. -# Node-side operations (relay, DHT key lookup) are triggered by the -# gossip layer, not the workload dispatcher. +# No one-shot workloads. secure-chat is a continuous service role +# (`mailbox`, on by default) driven by gossip and p2p request/response, +# not by the job dispatcher. [module.dispatch] mode = "in-process" -# First-party in-tree, compiled into the c0mpute binary. [module.surfaces] -cli = "c0mpute chat" -# web = "c0mpute.com/chat" # planned (Next.js, key material in sessionStorage only) -# tui = "chat panel" # planned (real-time WebSocket panel in apps/tui) +cli = "c0mpute dm" +# web = "c0mpute.com/chat" # planned +# tui = "chat panel" # planned [module.config.defaults] -# Maximum time a relay node holds an envelope for an offline recipient. -offline_ttl_secs = 86400 # 24 hours default, max 604800 (7 days) - -# Maximum encrypted storage per recipient DID on a relay node. -max_queue_bytes = 10_485_760 # 10 MB - -# DHT key announcement refresh interval. -announce_interval_secs = 43200 # 12 hours - -# Rate limit: max messages per sender DID per minute (node-enforced). -rate_limit_per_min = 60 +# Longest a mailbox holds a message for an offline recipient (30 days). +offline_ttl_secs = 2592000 +# Encrypted bytes a mailbox holds per recipient tag (10 MiB) and in total (256 MiB). +max_queue_bytes = 10485760 +max_total_bytes = 268435456 +# Largest sealed message (64 KiB; message text up to 16 KiB). +max_message_bytes = 65536 +# Messages a mailbox accepts per sending peer per minute. +rate_limit_per_min = 60 +# Mailboxes each message is stored on. +store_replicas = 3 From 5abb0a8cad796b9ec83a454e2d857ea9406ad864 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger <anthony@profullstack.com> Date: Mon, 9 Nov 2026 05:10:23 +0000 Subject: [PATCH 7/7] bump v0.2.82 Ships train, infernet, video, gameserver (available), ads on the shared payments rail, hosting release race fix. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- Cargo.lock | 32 +++++++++---------- Cargo.toml | 2 +- apps/tui/package.json | 2 +- apps/web/package.json | 2 +- .../web/src/app/releases/latest.json/route.ts | 2 +- package.json | 2 +- packages/shared/package.json | 2 +- plugins/ad-generator-agent/module.toml | 2 +- plugins/ads-manager/module.toml | 4 +-- plugins/coinpay/module.toml | 2 +- plugins/crawl/module.toml | 2 +- plugins/ctv-ads/module.toml | 4 +-- plugins/diffusion/module.toml | 2 +- plugins/embed/module.toml | 2 +- plugins/gameserver/module.toml | 4 +-- plugins/hosting/module.toml | 4 +-- plugins/infernet/module.toml | 4 +-- plugins/live-stream/module.toml | 2 +- plugins/music/module.toml | 2 +- plugins/ocr/module.toml | 2 +- plugins/podcast-ads/module.toml | 4 +-- plugins/podcasting/module.toml | 2 +- plugins/podcasts/module.toml | 2 +- plugins/render/module.toml | 2 +- plugins/secure-chat/module.toml | 2 +- plugins/storage/module.toml | 2 +- plugins/train/module.toml | 4 +-- plugins/transcode/module.toml | 2 +- plugins/tts/module.toml | 2 +- plugins/video/module.toml | 4 +-- plugins/webhook/module.toml | 2 +- plugins/whisper/module.toml | 2 +- plugins/workflow-step/module.toml | 2 +- 33 files changed, 56 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b9b7de0..a232bdfa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -478,7 +478,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "c0mpute-api" -version = "0.2.81" +version = "0.2.82" dependencies = [ "anyhow", "c0mpute-proto", @@ -493,7 +493,7 @@ dependencies = [ [[package]] name = "c0mpute-bench" -version = "0.2.81" +version = "0.2.82" dependencies = [ "anyhow", "blake3", @@ -504,7 +504,7 @@ dependencies = [ [[package]] name = "c0mpute-cli" -version = "0.2.81" +version = "0.2.82" dependencies = [ "anyhow", "axum", @@ -534,7 +534,7 @@ dependencies = [ [[package]] name = "c0mpute-core" -version = "0.2.81" +version = "0.2.82" dependencies = [ "aes-gcm", "anyhow", @@ -579,7 +579,7 @@ dependencies = [ [[package]] name = "c0mpute-doctor" -version = "0.2.81" +version = "0.2.82" dependencies = [ "anyhow", "serde", @@ -591,7 +591,7 @@ dependencies = [ [[package]] name = "c0mpute-envelope" -version = "0.2.81" +version = "0.2.82" dependencies = [ "blake3", "bs58", @@ -604,7 +604,7 @@ dependencies = [ [[package]] name = "c0mpute-gateway" -version = "0.2.81" +version = "0.2.82" dependencies = [ "anyhow", "axum", @@ -626,7 +626,7 @@ dependencies = [ [[package]] name = "c0mpute-market" -version = "0.2.81" +version = "0.2.82" dependencies = [ "c0mpute-envelope", "serde", @@ -636,7 +636,7 @@ dependencies = [ [[package]] name = "c0mpute-net" -version = "0.2.81" +version = "0.2.82" dependencies = [ "anyhow", "async-trait", @@ -653,7 +653,7 @@ dependencies = [ [[package]] name = "c0mpute-placement" -version = "0.2.81" +version = "0.2.82" dependencies = [ "anyhow", "async-trait", @@ -674,7 +674,7 @@ dependencies = [ [[package]] name = "c0mpute-proto" -version = "0.2.81" +version = "0.2.82" dependencies = [ "blake3", "hex", @@ -686,7 +686,7 @@ dependencies = [ [[package]] name = "c0mpute-secure-chat" -version = "0.2.81" +version = "0.2.82" dependencies = [ "aes-gcm", "anyhow", @@ -709,7 +709,7 @@ dependencies = [ [[package]] name = "c0mpute-store" -version = "0.2.81" +version = "0.2.82" dependencies = [ "anyhow", "blake3", @@ -727,7 +727,7 @@ dependencies = [ [[package]] name = "c0mpute-transcode" -version = "0.2.81" +version = "0.2.82" dependencies = [ "anyhow", "c0mpute-proto", @@ -740,7 +740,7 @@ dependencies = [ [[package]] name = "c0mpute-update" -version = "0.2.81" +version = "0.2.82" dependencies = [ "anyhow", "hex", @@ -754,7 +754,7 @@ dependencies = [ [[package]] name = "c0mpute-verify" -version = "0.2.81" +version = "0.2.82" dependencies = [ "anyhow", "blake3", diff --git a/Cargo.toml b/Cargo.toml index af39839f..6af6290e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ ] [workspace.package] -version = "0.2.81" +version = "0.2.82" edition = "2024" license = "MIT" repository = "https://github.com/profullstack/c0mpute" diff --git a/apps/tui/package.json b/apps/tui/package.json index 0f787e5d..f8e9df37 100644 --- a/apps/tui/package.json +++ b/apps/tui/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/tui", - "version": "0.2.81", + "version": "0.2.82", "private": true, "type": "module", "bin": { diff --git a/apps/web/package.json b/apps/web/package.json index 66b4ec9e..9f23c7cb 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute", - "version": "0.2.81", + "version": "0.2.82", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/app/releases/latest.json/route.ts b/apps/web/src/app/releases/latest.json/route.ts index 100b3db6..baac1d39 100644 --- a/apps/web/src/app/releases/latest.json/route.ts +++ b/apps/web/src/app/releases/latest.json/route.ts @@ -28,7 +28,7 @@ export const dynamic = "force-dynamic"; * this in sync with the workspace version in `Cargo.toml` on each release so a * freshly built CLI reports "already latest" rather than a phantom upgrade. */ -const FALLBACK_VERSION = "0.2.81"; +const FALLBACK_VERSION = "0.2.82"; const FALLBACK_MIN_REQUIRED = "0.0.1"; type Channel = "stable" | "beta" | "nightly"; diff --git a/package.json b/package.json index 0c664975..ffb6bb59 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute-monorepo", - "version": "0.2.81", + "version": "0.2.82", "private": true, "workspaces": [ "apps/*", diff --git a/packages/shared/package.json b/packages/shared/package.json index 9e74d2c4..14d584bf 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/shared", - "version": "0.2.81", + "version": "0.2.82", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/plugins/ad-generator-agent/module.toml b/plugins/ad-generator-agent/module.toml index 968f6d48..6333c736 100644 --- a/plugins/ad-generator-agent/module.toml +++ b/plugins/ad-generator-agent/module.toml @@ -21,7 +21,7 @@ [module] id = "ad-generator-agent" name = "Ad Generator Agent — every ad type from one URL" -version = "0.2.81" +version = "0.2.82" kind = "service" category = "applications" status = "available" diff --git a/plugins/ads-manager/module.toml b/plugins/ads-manager/module.toml index ade7b096..3901f7eb 100644 --- a/plugins/ads-manager/module.toml +++ b/plugins/ads-manager/module.toml @@ -38,7 +38,7 @@ [module] id = "ads-manager" name = "Ads Manager — shared ad operations layer" -version = "0.2.81" +version = "0.2.82" kind = "service" category = "applications" status = "partial" @@ -51,7 +51,7 @@ source = "https://github.com/profullstack/c0mpute/tree/master/node/crates/c keywords = ["ads", "advertising", "campaigns", "p2p", "receipts", "publisher", "creative", "billing", "coinpay"] [module.requirements] -c0mpute = ">=NEXT" +c0mpute = ">=0.2.82" os = ["linux", "darwin"] arch = ["x86_64", "aarch64"] capabilities = ["network:ingress?", "network:egress", "disk"] diff --git a/plugins/coinpay/module.toml b/plugins/coinpay/module.toml index b3662868..b9e47b06 100644 --- a/plugins/coinpay/module.toml +++ b/plugins/coinpay/module.toml @@ -7,7 +7,7 @@ [module] id = "coinpay" name = "CoinPay — DID, payments, escrow, reputation" -version = "0.2.81" +version = "0.2.82" kind = "service" category = "applications" status = "available" diff --git a/plugins/crawl/module.toml b/plugins/crawl/module.toml index e45c7d10..a39bb91f 100644 --- a/plugins/crawl/module.toml +++ b/plugins/crawl/module.toml @@ -8,7 +8,7 @@ [module] id = "crawl" name = "Crawl — distributed web fetching" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "data" status = "available" diff --git a/plugins/ctv-ads/module.toml b/plugins/ctv-ads/module.toml index e338e423..e4147b61 100644 --- a/plugins/ctv-ads/module.toml +++ b/plugins/ctv-ads/module.toml @@ -28,7 +28,7 @@ [module] id = "ctv-ads" name = "CTV Ads — VAST and VMAP from every gateway" -version = "0.2.81" +version = "0.2.82" kind = "service" category = "applications" status = "partial" @@ -41,7 +41,7 @@ source = "https://github.com/profullstack/c0mpute/tree/master/node/crates/c keywords = ["ctv", "connected-tv", "advertising", "vast", "vmap", "video", "streaming", "p2p"] [module.requirements] -c0mpute = ">=NEXT" +c0mpute = ">=0.2.82" os = ["linux", "darwin"] arch = ["x86_64", "aarch64"] capabilities = ["network:ingress"] diff --git a/plugins/diffusion/module.toml b/plugins/diffusion/module.toml index 26d0c0ff..79c368a8 100644 --- a/plugins/diffusion/module.toml +++ b/plugins/diffusion/module.toml @@ -11,7 +11,7 @@ [module] id = "diffusion" name = "Diffusion — image generation" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "ai" status = "available" diff --git a/plugins/embed/module.toml b/plugins/embed/module.toml index a13249bf..818f6fa5 100644 --- a/plugins/embed/module.toml +++ b/plugins/embed/module.toml @@ -19,7 +19,7 @@ [module] id = "embed" name = "Embeddings — vector generation" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "ai" status = "available" diff --git a/plugins/gameserver/module.toml b/plugins/gameserver/module.toml index 57a363d3..673293bf 100644 --- a/plugins/gameserver/module.toml +++ b/plugins/gameserver/module.toml @@ -55,7 +55,7 @@ [module] id = "gameserver" name = "Game server — Minecraft / Factorio / Valheim" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "infrastructure" status = "available" @@ -66,7 +66,7 @@ license = "MIT" keywords = ["gameserver", "minecraft", "factorio", "valheim", "hosting", "docker"] [module.requirements] -c0mpute = ">=NEXT" +c0mpute = ">=0.2.82" os = ["linux"] arch = ["x86_64", "aarch64"] capabilities = ["container-runtime", "network:public-ip", "network:ingress"] diff --git a/plugins/hosting/module.toml b/plugins/hosting/module.toml index 4e5063f9..c104a0b0 100644 --- a/plugins/hosting/module.toml +++ b/plugins/hosting/module.toml @@ -30,7 +30,7 @@ [module] id = "hosting" name = "Hosting — p2p static sites" -version = "0.2.81" +version = "0.2.82" kind = "service" category = "data" status = "available" @@ -42,7 +42,7 @@ source = "https://github.com/profullstack/c0mpute/tree/master/node/crates/c keywords = ["hosting", "static", "p2p", "censorship-resistant", "self-healing", "dweb", "marketplace", "opendisk"] [module.requirements] -c0mpute = ">=NEXT" +c0mpute = ">=0.2.82" os = ["linux", "darwin"] arch = ["x86_64", "aarch64"] capabilities = ["disk", "network:ingress?"] diff --git a/plugins/infernet/module.toml b/plugins/infernet/module.toml index 91a0db19..941d54f6 100644 --- a/plugins/infernet/module.toml +++ b/plugins/infernet/module.toml @@ -22,7 +22,7 @@ [module] id = "infernet" name = "Infernet Protocol — AI inference" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "ai" status = "available" @@ -34,7 +34,7 @@ source = "https://github.com/infernetprotocol/infernet-protocol" keywords = ["ai", "llm", "inference", "gpu", "temporal-policy", "liquid-neural-network", "cfc", "lora", "fine-tuning"] [module.requirements] -c0mpute = ">=NEXT" +c0mpute = ">=0.2.82" os = ["linux", "darwin"] arch = ["x86_64", "aarch64"] capabilities = ["gpu:nvidia?", "model-runtime"] diff --git a/plugins/live-stream/module.toml b/plugins/live-stream/module.toml index 22b10f86..2ea55908 100644 --- a/plugins/live-stream/module.toml +++ b/plugins/live-stream/module.toml @@ -34,7 +34,7 @@ [module] id = "live-stream" name = "Live Stream — p2p HLS live streaming" -version = "0.2.81" +version = "0.2.82" kind = "service" category = "media" status = "available" diff --git a/plugins/music/module.toml b/plugins/music/module.toml index 479f01f1..246a5c1e 100644 --- a/plugins/music/module.toml +++ b/plugins/music/module.toml @@ -15,7 +15,7 @@ [module] id = "music" name = "Music — song generation (ACE-Step 1.5)" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "ai" status = "available" diff --git a/plugins/ocr/module.toml b/plugins/ocr/module.toml index d33b44c3..a5a8a0b2 100644 --- a/plugins/ocr/module.toml +++ b/plugins/ocr/module.toml @@ -19,7 +19,7 @@ [module] id = "ocr" name = "OCR — document text extraction" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "ai" status = "available" diff --git a/plugins/podcast-ads/module.toml b/plugins/podcast-ads/module.toml index 92787e6a..7298b423 100644 --- a/plugins/podcast-ads/module.toml +++ b/plugins/podcast-ads/module.toml @@ -31,7 +31,7 @@ [module] id = "podcast-ads" name = "Podcast Ads — dynamic ad insertion" -version = "0.2.81" +version = "0.2.82" kind = "service" category = "applications" status = "partial" @@ -44,7 +44,7 @@ source = "https://github.com/profullstack/c0mpute/tree/master/node/crates/c keywords = ["podcast", "audio", "ads", "advertising", "dai", "dynamic-ad-insertion", "p2p"] [module.requirements] -c0mpute = ">=NEXT" +c0mpute = ">=0.2.82" os = ["linux", "darwin"] arch = ["x86_64", "aarch64"] capabilities = ["network:ingress", "bin:ffmpeg", "bin:ffprobe"] diff --git a/plugins/podcasting/module.toml b/plugins/podcasting/module.toml index fb3b3822..08539a87 100644 --- a/plugins/podcasting/module.toml +++ b/plugins/podcasting/module.toml @@ -24,7 +24,7 @@ [module] id = "podcasting" name = "Podcasting — p2p podcast publishing" -version = "0.2.81" +version = "0.2.82" kind = "service" category = "media" status = "available" diff --git a/plugins/podcasts/module.toml b/plugins/podcasts/module.toml index 2c9c7882..5c4e67c6 100644 --- a/plugins/podcasts/module.toml +++ b/plugins/podcasts/module.toml @@ -24,7 +24,7 @@ [module] id = "podcasts" name = "Podcasts — p2p podcast listener" -version = "0.2.81" +version = "0.2.82" kind = "service" category = "media" status = "available" diff --git a/plugins/render/module.toml b/plugins/render/module.toml index 47ed9919..cafbc866 100644 --- a/plugins/render/module.toml +++ b/plugins/render/module.toml @@ -33,7 +33,7 @@ [module] id = "render" name = "Render — Blender 3D rendering" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "media" status = "available" diff --git a/plugins/secure-chat/module.toml b/plugins/secure-chat/module.toml index 68a78ca6..0cfcdbfd 100644 --- a/plugins/secure-chat/module.toml +++ b/plugins/secure-chat/module.toml @@ -18,7 +18,7 @@ [module] id = "secure-chat" name = "Secure Chat — E2E encrypted p2p messaging" -version = "0.2.81" +version = "0.2.82" kind = "service" category = "applications" status = "partial" diff --git a/plugins/storage/module.toml b/plugins/storage/module.toml index 9045feb9..f0a8f011 100644 --- a/plugins/storage/module.toml +++ b/plugins/storage/module.toml @@ -20,7 +20,7 @@ [module] id = "storage" name = "Storage — Reed-Solomon shard service" -version = "0.2.81" +version = "0.2.82" kind = "service" category = "data" status = "available" diff --git a/plugins/train/module.toml b/plugins/train/module.toml index 15c7e0ce..66743f1e 100644 --- a/plugins/train/module.toml +++ b/plugins/train/module.toml @@ -41,7 +41,7 @@ [module] id = "train" name = "Fine-tune — LoRA training" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "ai" status = "available" @@ -52,7 +52,7 @@ license = "MIT" keywords = ["train", "finetune", "lora", "peft", "adapter", "infernet"] [module.requirements] -c0mpute = ">=NEXT" +c0mpute = ">=0.2.82" os = ["linux", "darwin"] arch = ["x86_64", "aarch64"] capabilities = ["bin:infernet", "infernet:text:runtime:peft", "infernet:text:format:messages?", "gpu:cuda?"] diff --git a/plugins/transcode/module.toml b/plugins/transcode/module.toml index ce5868e1..cd2d38b4 100644 --- a/plugins/transcode/module.toml +++ b/plugins/transcode/module.toml @@ -7,7 +7,7 @@ [module] id = "transcode" name = "FFmpeg Transcoding" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "media" status = "available" diff --git a/plugins/tts/module.toml b/plugins/tts/module.toml index e694c910..7aff9f49 100644 --- a/plugins/tts/module.toml +++ b/plugins/tts/module.toml @@ -15,7 +15,7 @@ [module] id = "tts" name = "TTS — text-to-speech" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "ai" status = "available" diff --git a/plugins/video/module.toml b/plugins/video/module.toml index 636357b1..2890a030 100644 --- a/plugins/video/module.toml +++ b/plugins/video/module.toml @@ -50,7 +50,7 @@ [module] id = "video" name = "Video — generation (Wan 2.2, LTX-2.3 with audio)" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "ai" status = "available" @@ -61,7 +61,7 @@ license = "MIT" keywords = ["video", "wan", "wan2.2", "ltx-2", "ltx-2.3", "audio", "txt2vid", "img2vid", "generation", "diffusion"] [module.requirements] -c0mpute = ">=NEXT" +c0mpute = ">=0.2.82" os = ["linux"] arch = ["x86_64"] capabilities = ["gpu:nvidia", "model-runtime", "container-runtime"] diff --git a/plugins/webhook/module.toml b/plugins/webhook/module.toml index 945cb80b..6f831c22 100644 --- a/plugins/webhook/module.toml +++ b/plugins/webhook/module.toml @@ -7,7 +7,7 @@ [module] id = "webhook" name = "Webhooks — paid, end-to-end encrypted delivery" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/whisper/module.toml b/plugins/whisper/module.toml index af4c15fe..3498ed63 100644 --- a/plugins/whisper/module.toml +++ b/plugins/whisper/module.toml @@ -7,7 +7,7 @@ [module] id = "whisper" name = "Whisper — speech-to-text" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "ai" status = "available" diff --git a/plugins/workflow-step/module.toml b/plugins/workflow-step/module.toml index 98095cf6..7f842e83 100644 --- a/plugins/workflow-step/module.toml +++ b/plugins/workflow-step/module.toml @@ -14,7 +14,7 @@ [module] id = "workflow-step" name = "Workflow step — webhook workflow steps (meshhook)" -version = "0.2.81" +version = "0.2.82" kind = "workload" category = "infrastructure" status = "available"