From f2193f730f96206687ff1370b821f0acfe887010 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 9 Nov 2026 04:21:50 +0000 Subject: [PATCH 1/2] video + gameserver: finish both plugins, mark available video: pin the published runner by digest (RUNNER_PULL_REF), run it under a local name, tag :stable in the workflow, label the package with its source, and check anonymous pulls in CI. `video setup` (now in core) pulls the pinned digest or builds from source. C0MPUTE_DOCKER overrides the container CLI. Speech gateway exposes router() so POST /v1/videos is tested end to end on loopback with a fake runner that writes a real MP4. gameserver (spec v2): secrets never ride in the offer (sealed handoff to the winning worker); backups every 30 min and at stop, encrypted to the buyer's X25519 key; signed status records on c0mpute/gameserver/v1; `gameserver watch` probes the port (Minecraft status, Valheim A2S, Factorio UDP + status), mirrors backups and fails over to another worker from the latest one; hard expiry via timeout + --init + --rm plus a boot/sweep watchdog. Runtime trait lets the loopback e2e run without docker. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/video-runner.yml | 45 +- apps/web/src/app/plugins/page.tsx | 6 + apps/web/src/lib/plugins.ts | 2 + node/crates/c0mpute-cli/src/gameserver.rs | 80 +- node/crates/c0mpute-cli/src/main.rs | 85 +- node/crates/c0mpute-core/src/container.rs | 57 + node/crates/c0mpute-core/src/gameserver.rs | 1650 ++++++++++++++--- node/crates/c0mpute-core/src/handoff.rs | 251 +++ node/crates/c0mpute-core/src/lib.rs | 2 + .../crates/c0mpute-core/src/speech_gateway.rs | 43 +- node/crates/c0mpute-core/src/video.rs | 120 +- .../c0mpute-core/tests/fixtures/tiny.mp4 | Bin 0 -> 1542 bytes .../c0mpute-core/tests/gameserver_e2e.rs | 396 ++++ node/crates/c0mpute-core/tests/video_e2e.rs | 279 +++ plugins/gameserver/module.toml | 59 +- plugins/video/install.sh | 12 +- plugins/video/module.toml | 21 +- plugins/video/runner/Dockerfile | 2 +- plugins/video/runner/README.md | 8 +- 19 files changed, 2755 insertions(+), 363 deletions(-) create mode 100644 node/crates/c0mpute-core/src/container.rs create mode 100644 node/crates/c0mpute-core/src/handoff.rs create mode 100644 node/crates/c0mpute-core/tests/fixtures/tiny.mp4 create mode 100644 node/crates/c0mpute-core/tests/gameserver_e2e.rs create mode 100644 node/crates/c0mpute-core/tests/video_e2e.rs diff --git a/.github/workflows/video-runner.yml b/.github/workflows/video-runner.yml index 40e1aff4..76a8b244 100644 --- a/.github/workflows/video-runner.yml +++ b/.github/workflows/video-runner.yml @@ -6,8 +6,19 @@ # torch image is several GB and not worth building to learn that a validation # rule changed. # -# The image the node pulls is the tag in plugins/video/module.toml under -# [module.dispatch]; keep the two in step. +# Tags on every master build: :stable (what operators pull by hand), +# :candidate (kept for older nodes) and :. Nodes pull by DIGEST, the one +# pinned in plugins/video/module.toml [module.dispatch] and RUNNER_DIGEST in +# node/crates/c0mpute-core/src/video.rs (a unit test keeps those two equal). +# The job summary prints the new digest; bump both pins to it after a runner +# change. +# +# Visibility: the package must be PUBLIC for outside operators to pull it. +# GHCR creates org packages private and has no API to change that, so it is +# one manual click by an org admin: github.com/orgs/profullstack/packages -> +# c0mpute-video-runner -> Package settings -> Change visibility -> Public. +# The last step below checks an anonymous pull and warns until it works. +# Until then `c0mpute video setup` builds the same image from source. name: video runner on: @@ -81,15 +92,21 @@ jobs: EXTRA: ${{ inputs.tag }} run: | set -eu - tags="${IMAGE}:candidate,${IMAGE}:${GITHUB_SHA}" + tags="${IMAGE}:stable,${IMAGE}:candidate,${IMAGE}:${GITHUB_SHA}" if [ -n "${EXTRA:-}" ]; then tags="${tags},${IMAGE}:${EXTRA}" fi echo "tags=${tags}" >> "$GITHUB_OUTPUT" - name: Build and push + id: push uses: docker/build-push-action@v6 with: + # Links the package to this public repo (and its README/license). + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.description=c0mpute video runner (Wan 2.2, LTX-2.3) + org.opencontainers.image.licenses=MIT # The context is plugins/video so module.toml — the runner's model # registry — is part of the image. context: plugins/video @@ -103,3 +120,25 @@ jobs: # Proves the thing we just pushed can parse the manifest, encode and # probe. Needs no GPU, so a hosted runner can do it. run: docker run --rm "${IMAGE}:${GITHUB_SHA}" selftest + + - name: Digest to pin, and can a stranger pull it? + env: + DIGEST: ${{ steps.push.outputs.digest }} + run: | + set -u + { + echo "### video runner published" + echo "" + echo "Pin this in plugins/video/module.toml and RUNNER_DIGEST in video.rs:" + echo "" + echo " ${IMAGE}@${DIGEST}" + } >> "$GITHUB_STEP_SUMMARY" + # Anonymous pull: a fresh docker config with no credentials. + export DOCKER_CONFIG="$(mktemp -d)" + if docker manifest inspect "${IMAGE}@${DIGEST}" >/dev/null 2>&1; then + echo "Anonymous pull: OK (package is public)." >> "$GITHUB_STEP_SUMMARY" + else + msg="ghcr.io package c0mpute-video-runner is not publicly pullable. An org admin must set it Public once: github.com/orgs/profullstack/packages -> c0mpute-video-runner -> Package settings -> Change visibility. Until then nodes build the runner from source." + echo "::warning::${msg}" + echo "Anonymous pull: FAILED. ${msg}" >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/apps/web/src/app/plugins/page.tsx b/apps/web/src/app/plugins/page.tsx index 07369c32..1dc50997 100644 --- a/apps/web/src/app/plugins/page.tsx +++ b/apps/web/src/app/plugins/page.tsx @@ -186,6 +186,12 @@ function PluginCard({ p }: { p: PluginManifest }) { + {typeof p.requirements?.hardware === "string" && ( +

+ requires: {p.requirements.hardware} +

+ )} + {p.status === "partial" && p.statusNote && (

partial: {p.statusNote} diff --git a/apps/web/src/lib/plugins.ts b/apps/web/src/lib/plugins.ts index be09bf39..1fc20167 100644 --- a/apps/web/src/lib/plugins.ts +++ b/apps/web/src/lib/plugins.ts @@ -113,6 +113,8 @@ export interface PluginManifest { os?: string[]; arch?: string[]; capabilities?: string[]; + /** What a node needs to serve it, in words (GPU, docker, public IP). */ + hardware?: string; }; workloads?: Record; models?: Record; diff --git a/node/crates/c0mpute-cli/src/gameserver.rs b/node/crates/c0mpute-cli/src/gameserver.rs index 51d7deb0..0bdcc76f 100644 --- a/node/crates/c0mpute-cli/src/gameserver.rs +++ b/node/crates/c0mpute-cli/src/gameserver.rs @@ -34,12 +34,16 @@ enum Sub { /// Restore a world: the hash `gameserver stop` printed, or a .tar of the game's data dir. #[arg(long)] world: Option, - /// Valheim server password (5-32 letters/digits; peers on the job topic can see it). + /// Valheim server password (5-32 letters/digits). Sent sealed to the + /// winning worker after the auction, never in the public offer. #[arg(long)] password: Option, /// Server name / MOTD. #[arg(long)] name: Option, + /// Minutes between live world backups, sealed to your key (0 = only at stop/expiry). + #[arg(long, default_value_t = gameserver::DEFAULT_BACKUP_MINS)] + backup_mins: u32, /// Most you pay for the whole reservation, USD. #[arg(long, default_value_t = 1.0)] max_price: f64, @@ -56,6 +60,32 @@ enum Sub { #[arg(long)] json: bool, }, + /// Watch a server: probe its port, mirror its backups, and if it stays + /// down for --fail-after-secs, start the same world on another worker. + Watch { + id: String, + #[arg(long, default_value_t = 30)] + probe_every_secs: u64, + #[arg(long, default_value_t = 300)] + fail_after_secs: u64, + /// Only probe and mirror backups; never start a replacement. + #[arg(long)] + no_failover: bool, + /// Most you pay for a replacement, USD. + #[arg(long, default_value_t = 1.0)] + max_price: f64, + /// Only accept replacement bids from these worker peer ids. Repeatable. + #[arg(long)] + worker: Vec, + }, + /// Show a server's probe log (what `watch` recorded). + Health { + id: String, + #[arg(long, default_value_t = 20)] + last: usize, + #[arg(long)] + json: bool, + }, /// Stop a server early (or collect an expired one's backup) and download its world. Stop { id: String, @@ -85,7 +115,7 @@ fn left(expires_at: u64) -> String { pub async fn run(args: &[String]) -> Result<()> { let cli = GameserverCli::try_parse_from(std::iter::once("c0mpute gameserver".to_string()).chain(args.iter().cloned())).unwrap_or_else(|e| e.exit()); match cli.cmd { - Sub::Start { game, hours, ram, cpus, accept_eula, world, password, name, max_price, worker, timeout_secs, json } => { + Sub::Start { game, hours, ram, cpus, accept_eula, world, password, name, backup_mins, max_price, worker, timeout_secs, json } => { let game = Game::parse(&game).with_context(|| format!("unknown game {game:?}: minecraft, factorio or valheim"))?; let req = StartRequest { game, @@ -96,8 +126,10 @@ pub async fn run(args: &[String]) -> Result<()> { world, password, server_name: name, + backup_mins, max_price_usd: max_price, workers: worker, + exclude: Vec::new(), timeout: Duration::from_secs(timeout_secs.clamp(60, 3600)), }; eprintln!("looking for a worker that hosts {} ...", game.name()); @@ -107,6 +139,7 @@ pub async fn run(args: &[String]) -> Result<()> { } else { println!("{} {} is up at {}", r.game.name(), r.id, r.address); println!(" worker {} ${} stops in {}", r.worker, r.price_usd, left(r.expires_at)); + println!(" keep it healthy (probe, mirror backups, fail over): c0mpute gameserver watch {}", r.id); println!(" stop early (and keep the world): c0mpute gameserver stop {}", r.id); } Ok(()) @@ -121,15 +154,50 @@ pub async fn run(args: &[String]) -> Result<()> { println!("no servers rented from this machine"); } for r in all { - let status = match (r.stopped_at, &r.backup) { - (Some(_), Some(b)) => format!("stopped world {b}"), - (Some(_), None) => "stopped".into(), - (None, _) => left(r.expires_at), + let status = match (r.stopped_at, &r.backup, &r.superseded_by) { + (_, _, Some(next)) => format!("failed over to {next}"), + (Some(_), Some(b), _) => format!("stopped world {b}"), + (Some(_), None, _) => "stopped".into(), + (None, _, _) => match &r.latest_backup { + Some(b) => format!("{} backup {}", left(r.expires_at), &b[..12.min(b.len())]), + None => left(r.expires_at), + }, }; println!("{:20} {:10} {:28} {}", r.id, r.game.name(), r.address, status); } Ok(()) } + Sub::Watch { id, probe_every_secs, fail_after_secs, no_failover, max_price, worker } => { + let opts = gameserver::WatchOptions { + probe_every: Duration::from_secs(probe_every_secs.clamp(5, 3600)), + fail_after: Duration::from_secs(fail_after_secs.clamp(30, 86_400)), + failover: !no_failover, + max_price_usd: max_price, + workers: worker, + ..Default::default() + }; + eprintln!("watching {id} (probe every {}s, fail over after {}s down)", opts.probe_every.as_secs(), opts.fail_after.as_secs()); + match gameserver::watch(&id, opts).await? { + gameserver::WatchEnd::Ended(r) => println!("{} ended (reservation over or stopped)", r.id), + gameserver::WatchEnd::FailedOver { from, to } => println!("{} failed over to {} at {}", from.id, to.id, to.address), + } + Ok(()) + } + Sub::Health { id, last, json } => { + let log = gameserver::health_log(&gameserver::buyer_dir(), &id); + let tail = &log[log.len().saturating_sub(last)..]; + if json { + println!("{}", serde_json::to_string(tail)?); + return Ok(()); + } + if tail.is_empty() { + println!("no probes recorded for {id} (run `c0mpute gameserver watch {id}`)"); + } + for p in tail { + println!("{} {} {:16} {:>5}ms {}", p.ts, if p.ok { "up " } else { "DOWN" }, p.method, p.rtt_ms, p.detail); + } + Ok(()) + } Sub::Stop { id, timeout_secs, json } => { let r = gameserver::stop(&id, Duration::from_secs(timeout_secs.clamp(30, 3600))).await?; if json { diff --git a/node/crates/c0mpute-cli/src/main.rs b/node/crates/c0mpute-cli/src/main.rs index 0395e71f..7332ecce 100644 --- a/node/crates/c0mpute-cli/src/main.rs +++ b/node/crates/c0mpute-cli/src/main.rs @@ -2345,62 +2345,6 @@ async fn run_stt(cmd: SttCmd) -> Result<()> { } } -/// Build the runner image from a shallow clone of the public repo. -/// -/// The fallback for `c0mpute video setup` when the registry will not serve -/// the image. Slower than a pull (it downloads the Python stack rather than -/// finished layers) but it needs no registry account, which is the point. -async fn build_video_runner() -> Result<()> { - use c0mpute_core::video; - - for bin in ["docker", "git"] { - if tokio::process::Command::new(bin) - .arg("--version") - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .status() - .await - .is_err() - { - anyhow::bail!("building the runner needs both docker and git; {bin} is missing"); - } - } - - // A fresh directory per run: a half-finished clone from a previous - // attempt would make `git clone` fail rather than resume. - let checkout = std::env::temp_dir().join(format!("c0mpute-video-src-{}", std::process::id())); - let _ = tokio::fs::remove_dir_all(&checkout).await; - - println!("cloning {} (shallow)", video::RUNNER_SOURCE_REPO); - let status = tokio::process::Command::new("git") - .args(["clone", "--depth", "1", video::RUNNER_SOURCE_REPO]) - .arg(&checkout) - .status() - .await - .context("could not run git")?; - if !status.success() { - anyhow::bail!("git clone failed with {status}"); - } - - println!("building {} (several minutes)", video::RUNNER_IMAGE); - let args = video::build_argv(&checkout, video::RUNNER_IMAGE); - let status = tokio::process::Command::new("docker") - .args(&args) - .status() - .await - .context("could not run docker build")?; - - // Clean up whether or not the build worked: the checkout is scratch and - // a stale one in /tmp is worse than no cache at all. - let _ = tokio::fs::remove_dir_all(&checkout).await; - - if !status.success() { - anyhow::bail!("docker build failed with {status}"); - } - println!("built {}", video::RUNNER_IMAGE); - Ok(()) -} - /// `c0mpute video` — the surface `plugins/video/module.toml` declares. /// /// Everything here talks to the runner image rather than to the network: a @@ -2411,29 +2355,8 @@ async fn run_video(cmd: VideoCmd) -> Result<()> { match cmd { VideoCmd::Setup { build } => { - let mut have_image = false; - if !build { - println!("pulling {} (this is a large image)", video::RUNNER_IMAGE); - let status = tokio::process::Command::new("docker") - .args(["pull", video::RUNNER_IMAGE]) - .status() - .await - .context("could not run docker; install a container runtime first")?; - have_image = status.success(); - if !have_image { - // A registry is the kind of dependency that is private by - // default, rate-limited or down, and none of those are a - // reason a node with a card cannot serve video. The repo - // is public, so build the same image from source. - println!(); - println!("could not pull the runner image (private, rate-limited or offline)."); - println!("building it from source instead — same Dockerfile, same result."); - println!(); - } - } - if !have_image { - build_video_runner().await?; - } + let how = video::setup(build, &std::env::temp_dir(), &|line: &str| println!("{line}")).await?; + println!("runner image ready as {} ({how:?})", video::RUNNER_IMAGE); match video::readiness(video::RUNNER_IMAGE) { None => { println!("this node can serve video jobs"); @@ -2460,7 +2383,7 @@ async fn run_video(cmd: VideoCmd) -> Result<()> { } if selftest { println!("running the runner selftest in the container"); - let status = tokio::process::Command::new("docker") + let status = c0mpute_core::container::command() .args(["run", "--rm", video::RUNNER_IMAGE, "selftest"]) .status() .await @@ -2475,7 +2398,7 @@ async fn run_video(cmd: VideoCmd) -> Result<()> { // The image carries the manifest it was built from, so this is // the model set that node would actually serve — not what this // CLI happened to be compiled with. - let out = tokio::process::Command::new("docker") + let out = c0mpute_core::container::command() .args([ "run", "--rm", diff --git a/node/crates/c0mpute-core/src/container.rs b/node/crates/c0mpute-core/src/container.rs new file mode 100644 index 00000000..b1513ce2 --- /dev/null +++ b/node/crates/c0mpute-core/src/container.rs @@ -0,0 +1,57 @@ +//! The container CLI the container workloads (`video`, `gameserver`) drive. +//! +//! `docker` by default. `C0MPUTE_DOCKER` names another binary with the same +//! command line: `podman`, a wrapper that adds `--context`, or a test double +//! that stands in for the daemon in the loopback end-to-end tests. + +use std::ffi::OsString; +use std::path::PathBuf; + +/// Environment variable that overrides the container CLI. +pub const ENV: &str = "C0MPUTE_DOCKER"; + +/// The container CLI to run: `$C0MPUTE_DOCKER`, else `docker`. +pub fn cli() -> OsString { + match std::env::var_os(ENV) { + Some(v) if !v.is_empty() => v, + _ => OsString::from("docker"), + } +} + +/// Where [`cli`] resolves: the path itself when it names a file, else the +/// first match on `PATH`. `None` means there is no container runtime. +pub fn which() -> Option { + let bin = cli(); + let as_path = PathBuf::from(&bin); + if as_path.components().count() > 1 { + return as_path.is_file().then_some(as_path); + } + std::env::var_os("PATH").and_then(|paths| { + std::env::split_paths(&paths) + .map(|dir| dir.join(&bin)) + .find(|candidate| candidate.is_file()) + }) +} + +/// A `tokio` command for the container CLI. +pub fn command() -> tokio::process::Command { + tokio::process::Command::new(cli()) +} + +/// A blocking command for the container CLI (readiness probes only). +pub fn command_sync() -> std::process::Command { + std::process::Command::new(cli()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_to_docker() { + // The variable is not set in the unit-test process. + if std::env::var_os(ENV).is_none() { + assert_eq!(cli(), OsString::from("docker")); + } + } +} diff --git a/node/crates/c0mpute-core/src/gameserver.rs b/node/crates/c0mpute-core/src/gameserver.rs index 0213a842..ee396cb4 100644 --- a/node/crates/c0mpute-core/src/gameserver.rs +++ b/node/crates/c0mpute-core/src/gameserver.rs @@ -4,14 +4,34 @@ //! A start job is an ordinary auction whose job is short: the winner creates //! a container from a pinned, well-known image (`docker create`, argv only, //! never a shell) with memory, CPU and pid limits, a named volume for the -//! world and a published port, optionally restores a world backup the buyer -//! serves by hash, starts it, and returns `address:port` and the expiry. The -//! container then runs until the reservation ends: the worker's reaper stops -//! it at expiry. `gameserver.stop` (bid on only by the worker holding the -//! server, and only with the stop token whose sha256 the start job carried) -//! ends it early. Either way the worker tars the world out of the volume, -//! stores it as 1 MiB content-addressed chunks plus a manifest, and hands the -//! manifest hash back: `start --world ` restores it on any worker. +//! world and a published port, optionally restores a world the buyer serves +//! by hash, starts it, and returns `address:port` and the expiry. +//! +//! Secrets never ride in the offer. The offer carries the buyer's fresh +//! X25519 key; after the accept the winner announces its own and the buyer +//! seals the Valheim password and the restore key to it ([`crate::handoff`]). +//! +//! Worlds are encrypted. A world the buyer restores is published as AES-GCM +//! chunks under a fresh key that only the winning worker receives. Every +//! backup a worker takes is sealed to the buyer's long-term backup key +//! (X25519 → HKDF → AES-256-GCM per 1 MiB chunk): the manifest hash in a +//! receipt or a status record is useless to anyone else. +//! +//! While a server runs the worker backs the world up every `backup_mins` +//! (default 30) and gossips a status record (container running, latest +//! backup hash) on [`STATUS_TOPIC`], signed by its peer key. The buyer's +//! `gameserver watch` probes the game port (Minecraft status ping, Valheim +//! Steam A2S query, Factorio UDP reachability), logs every probe, mirrors +//! each new backup to its own machine, and when the server fails its checks +//! for `fail_after` it starts the same world on another worker from the +//! latest backup. No coordinator anywhere. +//! +//! Expiry does not depend on the worker being online: the container runs +//! under `timeout` (with `--init`) so it exits by itself when the +//! reservation ends, and `--rm` removes it; the named volume keeps the world +//! until the worker, when it next runs, backs it up and removes it. At boot +//! and on every sweep the worker also kills any labelled container whose +//! reservation has passed. //! //! Only opt-in nodes serve it (`--roles ...,gameserver`): it holds a port and //! RAM for hours. A node advertises it only when `docker info` works and it @@ -20,13 +40,15 @@ //! The operator must open the port range in the firewall. use std::collections::{HashMap, HashSet}; -use std::net::IpAddr; +use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use anyhow::{Context, Result, bail}; +use aes_gcm::aead::{Aead, KeyInit}; +use aes_gcm::{Aes256Gcm, Nonce}; +use anyhow::{Context, Result, anyhow, bail}; use c0mpute_net::topics::{JobOffer, JobStatus}; use c0mpute_net::{Libp2pNetwork, Network}; use c0mpute_proto::Hash; @@ -35,27 +57,43 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::sync::{Semaphore, mpsc}; +use tokio::sync::{Semaphore, broadcast, mpsc}; use tracing::{info, warn}; +use x25519_dalek::StaticSecret; use crate::runner::{AcceptedJob, WorkerLoad, publish_receipt}; +use crate::temporal::crypto; pub const WORKLOAD_START: &str = "gameserver.start"; pub const WORKLOAD_STOP: &str = "gameserver.stop"; pub const ROLE_TAG: &str = "c0mpute:role:gameserver"; +/// Where workers gossip [`StatusRecord`]s. +pub const STATUS_TOPIC: &str = "c0mpute/gameserver/v1"; +pub const SPEC_VERSION: u32 = 2; pub const MAX_HOURS: u32 = 72; pub const MAX_RAM_GB: u32 = 32; pub const MAX_CPUS: f64 = 8.0; -/// World backups are stored and moved as chunks of this size. +/// World backups are stored and moved as chunks of this size (plaintext). pub const CHUNK_BYTES: usize = 1024 * 1024; +/// AES-GCM tag on every chunk. +const TAG_BYTES: usize = 16; pub const MAX_WORLD_BYTES: u64 = 4 * 1024 * 1024 * 1024; +pub const DEFAULT_BACKUP_MINS: u32 = 30; +pub const MIN_BACKUP_MINS: u32 = 5; +pub const MAX_BACKUP_MINS: u32 = 720; +pub const ENC_SCHEME: &str = "x25519-hkdf-sha256-aes256gcm-chunked/v1"; const MANIFEST_KIND: &str = "c0mpute-gameserver-world"; +const MANIFEST_VERSION: u32 = 2; const MAX_MANIFEST_BYTES: usize = 1024 * 1024; /// Start/stop jobs handled at once (the servers themselves are not jobs). const JOB_SLOTS: usize = 2; /// Stopped-server records (and their backup hash) are kept this long. const KEEP_STOPPED_SECS: u64 = 7 * 86_400; -const REAP_EVERY: Duration = Duration::from_secs(30); +pub const REAP_EVERY: Duration = Duration::from_secs(30); +/// How long a worker waits for the buyer's sealed secrets after the accept. +const HANDOFF_WAIT: Duration = Duration::from_secs(120); +/// `timeout -k`: after the TERM at expiry, KILL this many seconds later. +const EXPIRY_KILL_AFTER_SECS: u64 = 90; // ── games ─────────────────────────────────────────────────────────────── @@ -104,6 +142,16 @@ impl Game { Game::Valheim => "lloesche/valheim-server:sha-0ab6cdcc16dd@sha256:fde7829b12d39203e155b6209689b70234f8a27d12c4d48baf890cc2ae2c2df3", } } + /// The pinned image's own ENTRYPOINT + CMD (read from each image config + /// at the digest above). The container runs it under `timeout`, so the + /// server exits at expiry whether or not the worker is online. + pub fn command(self) -> &'static [&'static str] { + match self { + Game::Minecraft => &["/image/scripts/start"], + Game::Factorio => &["/docker-entrypoint.sh"], + Game::Valheim => &["/usr/local/sbin/bootstrap"], + } + } /// Where the image keeps its world; the named volume mounts here. pub fn data_dir(self) -> &'static str { match self { @@ -140,7 +188,8 @@ impl Game { // ── specs ─────────────────────────────────────────────────────────────── -/// `spec_inline` of a `gameserver.start` offer. +/// `spec_inline` of a `gameserver.start` offer. Everything in it is public: +/// every subscriber of the job topic can read it. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct StartSpec { pub v: u32, @@ -153,16 +202,25 @@ pub struct StartSpec { pub accept_eula: bool, /// `sha256:` of the stop token only the buyer holds. pub stop_hash: String, - /// Manifest hash (blake3 hex) of a world backup to restore. + /// The buyer's fresh X25519 key for this job's [`crate::handoff`]. + pub client_pubkey: String, + /// The buyer's long-term X25519 key every backup is sealed to. + pub backup_pubkey: String, + /// Minutes between live backups; 0 = only at stop or expiry. + #[serde(default = "default_backup_mins")] + pub backup_mins: u32, + /// Manifest hash (blake3 hex) of an encrypted world to restore; its key + /// arrives in the sealed [`StartSecrets`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub world: Option, - /// Valheim only (it requires one). Visible to peers on the job topic. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub password: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub server_name: Option, } +fn default_backup_mins() -> u32 { + DEFAULT_BACKUP_MINS +} + fn is_hex(s: &str, len: usize) -> bool { s.len() == len && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) } @@ -173,8 +231,8 @@ fn valid_sha256_ref(s: &str) -> bool { impl StartSpec { pub fn validate(&self) -> std::result::Result<(), String> { - if self.v != 1 { - return Err(format!("unsupported gameserver spec version {}", self.v)); + if self.v != SPEC_VERSION { + return Err(format!("unsupported gameserver spec version {} (this node speaks {SPEC_VERSION})", self.v)); } if !(1..=MAX_HOURS).contains(&self.hours) { return Err(format!("hours must be 1..={MAX_HOURS}")); @@ -191,15 +249,15 @@ impl StartSpec { if !valid_sha256_ref(&self.stop_hash) { return Err("stop_hash must be sha256:<64 lowercase hex>".into()); } + if !is_hex(&self.client_pubkey, 64) || !is_hex(&self.backup_pubkey, 64) { + return Err("client_pubkey and backup_pubkey must be X25519 keys (64 hex)".into()); + } + if self.backup_mins != 0 && !(MIN_BACKUP_MINS..=MAX_BACKUP_MINS).contains(&self.backup_mins) { + return Err(format!("backup_mins must be 0 or {MIN_BACKUP_MINS}..={MAX_BACKUP_MINS}")); + } if self.world.as_deref().is_some_and(|w| !is_hex(w, 64)) { return Err("world must be a 64-hex manifest hash".into()); } - match (self.game, &self.password) { - (Game::Valheim, Some(p)) if (5..=32).contains(&p.len()) && p.chars().all(|c| c.is_ascii_alphanumeric()) => {} - (Game::Valheim, _) => return Err("valheim needs a password of 5-32 letters or digits".into()), - (_, Some(_)) => return Err("only valheim takes a password".into()), - _ => {} - } if let Some(n) = &self.server_name && (n.is_empty() || n.len() > 32 || !n.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, ' ' | '-' | '_' | '.'))) { @@ -209,6 +267,51 @@ impl StartSpec { } } +/// What the buyer seals to the winning worker after the accept. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct StartSecrets { + pub v: u32, + /// Valheim only (it requires one). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + /// Hex AES-256 key of the world named in the spec. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub world_key: Option, +} + +impl StartSecrets { + pub fn validate(&self, spec: &StartSpec) -> std::result::Result<(), String> { + if self.v != 1 { + return Err("unsupported secrets version".into()); + } + match (spec.game, &self.password) { + (Game::Valheim, Some(p)) if valid_password(p) => {} + (Game::Valheim, _) => return Err("valheim needs a password of 5-32 letters or digits".into()), + (_, Some(_)) => return Err("only valheim takes a password".into()), + _ => {} + } + match (&spec.world, &self.world_key) { + (Some(_), Some(k)) if is_hex(k, 64) => {} + (Some(_), _) => return Err("a world restore needs its 32-byte key".into()), + (None, Some(_)) => return Err("a world key without a world".into()), + (None, None) => {} + } + Ok(()) + } + + pub fn world_key(&self) -> Result> { + self.world_key.as_deref().map(key_from_hex).transpose() + } +} + +pub fn valid_password(p: &str) -> bool { + (5..=32).contains(&p.len()) && p.chars().all(|c| c.is_ascii_alphanumeric()) +} + +fn key_from_hex(h: &str) -> Result<[u8; 32]> { + hex::decode(h)?.try_into().map_err(|_| anyhow!("key must be 32 bytes")) +} + /// `spec_inline` of a `gameserver.stop` offer. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct StopSpec { @@ -264,12 +367,14 @@ pub struct ContainerPlan { pub cpus: f64, /// First host port; the game's ports map onto consecutive host ports. pub host_port: u16, + /// Seconds the server may run before it stops itself (the reservation). + pub run_secs: u64, pub env: Vec<(String, String)>, pub labels: Vec<(String, String)>, } impl ContainerPlan { - pub fn new(id: &str, spec: &StartSpec, host_port: u16, expires: u64) -> Self { + pub fn new(id: &str, spec: &StartSpec, secrets: &StartSecrets, host_port: u16, now: u64, expires: u64) -> Self { let memory_mb = u64::from(spec.ram_gb) * 1024; let name = format!("c0mpute-{id}"); let mut env: Vec<(String, String)> = Vec::new(); @@ -285,7 +390,7 @@ impl ContainerPlan { Game::Valheim => { env.push(("SERVER_NAME".into(), label)); env.push(("WORLD_NAME".into(), "c0mpute".into())); - env.push(("SERVER_PASS".into(), spec.password.clone().unwrap_or_default())); + env.push(("SERVER_PASS".into(), secrets.password.clone().unwrap_or_default())); env.push(("SERVER_PUBLIC".into(), "false".into())); env.push(("SERVER_PORT".into(), "2456".into())); } @@ -296,13 +401,28 @@ impl ContainerPlan { ("c0mpute.gameserver.game".into(), spec.game.name().into()), ("c0mpute.gameserver.expires".into(), expires.to_string()), ]; - Self { volume: name.clone(), name, game: spec.game, memory_mb, cpus: spec.cpus, host_port, env, labels } + Self { + volume: name.clone(), + name, + game: spec.game, + memory_mb, + cpus: spec.cpus, + host_port, + run_secs: expires.saturating_sub(now).max(1), + env, + labels, + } } /// `docker create` argv (without the leading `docker`). Every value is its /// own argument: nothing passes through a shell. + /// + /// The image's own command runs under `timeout ` behind + /// `--init`, and `--rm` removes the container when it exits: the server + /// ends at expiry even if this worker is offline then. The world lives + /// in the named volume, which `--rm` keeps. pub fn create_argv(&self) -> Vec { - let mut a: Vec = vec!["create".into(), "--name".into(), self.name.clone()]; + let mut a: Vec = vec!["create".into(), "--name".into(), self.name.clone(), "--rm".into(), "--init".into()]; for (k, v) in [ ("--restart", "no".to_string()), ("--memory", format!("{}m", self.memory_mb)), @@ -313,6 +433,7 @@ impl ContainerPlan { ("--log-opt", "max-size=10m".into()), ("--log-opt", "max-file=2".into()), ("--stop-timeout", "60".into()), + ("--entrypoint", "timeout".into()), ] { a.push(k.into()); a.push(v); @@ -332,12 +453,41 @@ impl ContainerPlan { a.push(format!("{k}={v}")); } a.push(self.game.image().into()); + // Arguments to `timeout`: TERM at expiry, KILL if still alive after. + for x in ["-s".to_string(), "TERM".into(), "-k".into(), format!("{EXPIRY_KILL_AFTER_SECS}s"), format!("{}s", self.run_secs)] { + a.push(x); + } + a.extend(self.game.command().iter().map(|s| s.to_string())); a } } +/// What a worker needs from a container runtime. [`Docker`] in production; +/// the loopback end-to-end tests supply a double. +#[async_trait::async_trait] +pub trait Runtime: Send + Sync + 'static { + /// Create (not start) the container and its volume. + async fn create(&self, plan: &ContainerPlan) -> Result<()>; + /// Extract a world tar (entries under the data dir's name) into the + /// created container's volume. + async fn restore(&self, plan: &ContainerPlan, tar: &Path) -> Result<()>; + async fn start(&self, name: &str) -> Result<()>; + async fn running(&self, name: &str) -> bool; + /// Ask the server to write its world to disk (best effort). + async fn flush(&self, name: &str, game: Game); + async fn stop(&self, name: &str) -> Result<()>; + /// Write a tar of the volume's world to `out`. Works whether or not the + /// server container still exists. + async fn tar_world(&self, volume: &str, game: Game, out: &Path) -> Result<()>; + /// Remove the container (if it is still there) and its volume. + async fn remove(&self, name: &str, volume: &str); + /// Containers this runtime labelled as game servers, with their expiry. + async fn labelled(&self) -> Result>; + async fn kill(&self, name: &str); +} + async fn docker(args: &[&str], timeout: Duration) -> Result { - let out = tokio::time::timeout(timeout, tokio::process::Command::new("docker").args(args).stdin(Stdio::null()).kill_on_drop(true).output()) + let out = tokio::time::timeout(timeout, crate::container::command().args(args).stdin(Stdio::null()).kill_on_drop(true).output()) .await .with_context(|| format!("docker {} timed out", args.first().unwrap_or(&"")))? .context("run docker")?; @@ -348,7 +498,124 @@ async fn docker(args: &[&str], timeout: Duration) -> Result { } fn docker_sync_ok(args: &[&str]) -> bool { - std::process::Command::new("docker").args(args).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success()) + crate::container::command_sync().args(args).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success()) +} + +/// `docker run` argv that tars a world volume to stdout with the game's own +/// (already pulled) image: no network, the volume read-only. +pub fn tar_argv(volume: &str, game: Game) -> Vec { + let dir = game.data_dir(); + vec![ + "run".into(), + "--rm".into(), + "--network".into(), + "none".into(), + "--entrypoint".into(), + "tar".into(), + "-v".into(), + format!("{volume}:{dir}:ro"), + game.image().into(), + "-C".into(), + "/".into(), + "-cf".into(), + "-".into(), + dir.trim_start_matches('/').into(), + ] +} + +/// The real container runtime: the `docker` CLI (or `$C0MPUTE_DOCKER`). +pub struct Docker; + +#[async_trait::async_trait] +impl Runtime for Docker { + async fn create(&self, plan: &ContainerPlan) -> Result<()> { + let argv = plan.create_argv(); + let args: Vec<&str> = argv.iter().map(String::as_str).collect(); + docker(&args, Duration::from_secs(1800)).await.map(drop) + } + + async fn restore(&self, plan: &ContainerPlan, tar: &Path) -> Result<()> { + // `docker cp - :/` extracts the tar (entries `data/...`) into the volume. + let f = std::fs::File::open(tar).with_context(|| format!("open {}", tar.display()))?; + let out = crate::container::command() + .args(["cp", "-", &format!("{}:/", plan.name)]) + .stdin(Stdio::from(f)) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .output() + .await + .context("run docker cp")?; + if !out.status.success() { + bail!("docker cp into {}: {}", plan.name, String::from_utf8_lossy(&out.stderr).trim()); + } + Ok(()) + } + + async fn start(&self, name: &str) -> Result<()> { + docker(&["start", name], Duration::from_secs(120)).await.map(drop) + } + + async fn running(&self, name: &str) -> bool { + docker(&["inspect", "-f", "{{.State.Running}}", name], Duration::from_secs(30)).await.is_ok_and(|s| s == "true") + } + + async fn flush(&self, name: &str, game: Game) { + // itzg's image ships rcon-cli with RCON on; the others autosave. + if game == Game::Minecraft { + let _ = docker(&["exec", name, "rcon-cli", "save-all", "flush"], Duration::from_secs(60)).await; + } + } + + async fn stop(&self, name: &str) -> Result<()> { + docker(&["stop", "-t", "60", name], Duration::from_secs(120)).await.map(drop) + } + + async fn tar_world(&self, volume: &str, game: Game, out: &Path) -> Result<()> { + let f = std::fs::File::create(out).with_context(|| format!("create {}", out.display()))?; + let argv = tar_argv(volume, game); + let res = tokio::time::timeout( + Duration::from_secs(1800), + crate::container::command().args(&argv).stdin(Stdio::null()).stdout(Stdio::from(f)).stderr(Stdio::piped()).kill_on_drop(true).output(), + ) + .await + .context("tar of the world timed out")? + .context("run docker")?; + if !res.status.success() { + bail!("tar of volume {volume}: {}", String::from_utf8_lossy(&res.stderr).trim()); + } + Ok(()) + } + + async fn remove(&self, name: &str, volume: &str) { + let _ = docker(&["rm", "-f", name], Duration::from_secs(60)).await; + let _ = docker(&["volume", "rm", volume], Duration::from_secs(60)).await; + } + + async fn labelled(&self) -> Result> { + let out = docker(&["ps", "-a", "--filter", "label=c0mpute.gameserver=1", "--format", "{{.Names}}\t{{.Label \"c0mpute.gameserver.expires\"}}"], Duration::from_secs(30)).await?; + Ok(parse_labelled(&out)) + } + + async fn kill(&self, name: &str) { + let _ = docker(&["kill", name], Duration::from_secs(30)).await; + let _ = docker(&["rm", "-f", name], Duration::from_secs(30)).await; + } +} + +/// `docker ps --format '{{.Names}}\t{{.Label ...expires}}'` → (name, expiry). +/// A container without a readable expiry counts as already expired. +pub fn parse_labelled(out: &str) -> Vec<(String, u64)> { + out.lines() + .filter_map(|l| { + let mut it = l.split('\t'); + let name = it.next()?.trim(); + if name.is_empty() { + return None; + } + Some((name.to_string(), it.next().and_then(|e| e.trim().parse().ok()).unwrap_or(0))) + }) + .collect() } // ── public address ────────────────────────────────────────────────────── @@ -483,10 +750,15 @@ pub async fn ensure_images(cfg: &crate::config::GameserverConfig) { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct Backup { - /// Blake3 hex of the world manifest. + /// Blake3 hex of the (encrypted) world manifest. pub manifest: String, + /// Plaintext tar size. pub bytes: u64, + /// sha256 of the plaintext tar. pub sha256: String, + /// When it was taken (unix seconds). + #[serde(default)] + pub at: u64, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] @@ -502,8 +774,15 @@ pub struct ServerState { pub expires_at: u64, pub stop_hash: String, pub buyer: String, + /// Every backup is sealed to this key. + #[serde(default)] + pub backup_pubkey: String, + /// Seconds between live backups (0 = none). + #[serde(default)] + pub backup_every_secs: u64, #[serde(default)] pub stopped_at: Option, + /// The latest backup (live, or the final one at stop/expiry). #[serde(default)] pub backup: Option, #[serde(default)] @@ -527,6 +806,26 @@ pub fn due_for_stop(servers: &HashMap, now: u64) -> Vec, now: u64) -> Vec { + let mut ids: Vec = servers + .values() + .filter(|s| { + let last = s.backup.as_ref().map(|b| b.at).unwrap_or(s.started_at).max(s.started_at); + s.running() && s.backup_every_secs > 0 && now < s.expires_at && now >= last + s.backup_every_secs + }) + .map(|s| s.id.clone()) + .collect(); + ids.sort(); + ids +} + +/// Labelled containers whose reservation has passed: killed on sight, at +/// boot and on every sweep, whatever the state file says. +pub fn expired_containers(labelled: &[(String, u64)], now: u64) -> Vec { + labelled.iter().filter(|(_, exp)| now >= *exp).map(|(n, _)| n.clone()).collect() +} + /// Can this node take `spec` next to what it already runs? pub fn has_capacity(cfg: &crate::config::GameserverConfig, total_ram_mb: u64, servers: &HashMap, spec: &StartSpec) -> std::result::Result<(), String> { if !cfg.games.contains(&spec.game) { @@ -567,6 +866,55 @@ fn port_bindable(p: u16) -> bool { std::net::TcpListener::bind(("0.0.0.0", p)).is_ok() && std::net::UdpSocket::bind(("0.0.0.0", p)).is_ok() } +// ── world encryption ──────────────────────────────────────────────────── + +/// How a world's chunks are encrypted. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct WorldEnc { + pub scheme: String, + /// Present on backups: the worker's one-time X25519 key; the buyer's + /// backup key and this give the chunk key. Absent on a world the buyer + /// publishes for a restore: its key travels in the sealed secrets. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub epk: Option, +} + +impl WorldEnc { + pub fn handed_off() -> Self { + Self { scheme: ENC_SCHEME.into(), epk: None } + } +} + +/// A fresh random world key. +pub fn random_key() -> [u8; 32] { + rand::random() +} + +/// The chunk key of a backup sealed to `recipient_pub`: derived from the +/// worker's one-time secret and the recipient's key (worker side), or the +/// recipient's secret and the one-time `epk` (buyer side). +pub fn backup_key(secret: &StaticSecret, peer_pub_hex: &str, epk_hex: &str, recipient_hex: &str) -> Result<[u8; 32]> { + crypto::session_key(secret, peer_pub_hex, &format!("gameserver-backup:{epk_hex}:{recipient_hex}")) +} + +fn chunk_nonce(i: u64) -> [u8; 12] { + let mut n = [0u8; 12]; + n[4..].copy_from_slice(&i.to_be_bytes()); + n +} + +/// Chunk `i` under `key`. The key is single-use (one per world), so the +/// index is a safe nonce, and it pins every chunk to its position. +pub fn seal_chunk(key: &[u8; 32], i: u64, plain: &[u8]) -> Result> { + let c = Aes256Gcm::new_from_slice(key).map_err(|_| anyhow!("key"))?; + c.encrypt(Nonce::from_slice(&chunk_nonce(i)), plain).map_err(|_| anyhow!("encrypt chunk")) +} + +pub fn open_chunk(key: &[u8; 32], i: u64, sealed: &[u8]) -> Result> { + let c = Aes256Gcm::new_from_slice(key).map_err(|_| anyhow!("key"))?; + c.decrypt(Nonce::from_slice(&chunk_nonce(i)), sealed).map_err(|_| anyhow!("world chunk {i} does not decrypt (wrong key or tampered)")) +} + // ── world backups ─────────────────────────────────────────────────────── #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] @@ -576,28 +924,34 @@ pub struct WorldManifest { pub game: Game, /// A tar of this directory (entries start with its name, e.g. `data/`). pub data_dir: String, + /// Plaintext tar size. pub bytes: u64, + /// sha256 of the plaintext tar, checked after decryption. pub sha256: String, - /// Blake3 hex of each chunk, in order. + pub enc: WorldEnc, + /// Blake3 hex of each encrypted chunk, in order. pub chunks: Vec, } impl WorldManifest { - pub fn new(game: Game) -> Self { - Self { v: 1, kind: MANIFEST_KIND.into(), game, data_dir: game.data_dir().into(), bytes: 0, sha256: String::new(), chunks: Vec::new() } + pub fn new(game: Game, enc: WorldEnc) -> Self { + Self { v: MANIFEST_VERSION, kind: MANIFEST_KIND.into(), game, data_dir: game.data_dir().into(), bytes: 0, sha256: String::new(), enc, chunks: Vec::new() } } pub fn to_bytes(&self) -> Vec { serde_json::to_vec(self).unwrap_or_default() } pub fn from_bytes(b: &[u8]) -> Result { - let m: Self = serde_json::from_slice(b).context("world manifest is not JSON")?; + let m: Self = serde_json::from_slice(b).context("world manifest is not a v2 JSON manifest")?; m.check()?; Ok(m) } pub fn check(&self) -> Result<()> { - if self.v != 1 || self.kind != MANIFEST_KIND { + if self.v != MANIFEST_VERSION || self.kind != MANIFEST_KIND { bail!("not a c0mpute game world manifest"); } + if self.enc.scheme != ENC_SCHEME || self.enc.epk.as_deref().is_some_and(|k| !is_hex(k, 64)) { + bail!("unknown world encryption"); + } if self.data_dir != self.game.data_dir() { bail!("manifest data_dir does not match {}", self.game.name()); } @@ -612,10 +966,11 @@ impl WorldManifest { } } -/// Read `r` to the end into 1 MiB chunks in the store; returns the manifest -/// (not yet stored). Chunks are announced so any peer can fetch them. -pub async fn chunk_into_store(store: &ChunkStore, net: Option<&Libp2pNetwork>, game: Game, mut r: R) -> Result { - let mut m = WorldManifest::new(game); +/// Read `r` to the end, encrypt it under `key` in 1 MiB chunks, and put the +/// chunks in the store; returns the manifest (not yet stored). Chunks are +/// announced so any peer can fetch them; only the key holder can read them. +pub async fn chunk_into_store(store: &ChunkStore, net: Option<&Libp2pNetwork>, game: Game, mut r: R, key: &[u8; 32], enc: WorldEnc) -> Result { + let mut m = WorldManifest::new(game, enc); let mut sha = Sha256::new(); let mut buf = vec![0u8; CHUNK_BYTES]; loop { @@ -635,7 +990,8 @@ pub async fn chunk_into_store(store: &ChunkStor bail!("world is over {} GiB", MAX_WORLD_BYTES >> 30); } sha.update(&buf[..filled]); - let h = store.put(&buf[..filled]).await?; + let sealed = seal_chunk(key, m.chunks.len() as u64, &buf[..filled])?; + let h = store.put(&sealed).await?; if let Some(net) = net { let _ = net.announce(&h).await; } @@ -663,18 +1019,32 @@ async fn fetch_bytes(net: &Libp2pNetwork, store: &ChunkStore, hash: &str, provid { return Ok(b); } - crate::gateway_util::fetch_verified(net, hash, provider, max, Duration::from_secs(60)).await.map_err(|e| anyhow::anyhow!("fetch {hash} from {provider}: {e:?}")) + let b = crate::gateway_util::fetch_verified(net, hash, provider, max, Duration::from_secs(60)).await.map_err(|e| anyhow!("fetch {hash} from {provider}: {e:?}"))?; + // Keep a copy: a buyer mirroring backups must be able to restore after + // the worker that took them is gone. + let _ = store.put(&b).await; + Ok(b) } -/// Fetch a world (manifest, then each chunk) from `provider` (or any peer -/// that announced it), verifying every chunk and the whole tar's sha256, -/// and write the tar to `out` as it arrives. -pub async fn fetch_world(net: &Libp2pNetwork, store: &ChunkStore, manifest_hash: &str, provider: &str, mut out: W) -> Result { +/// Fetch an encrypted world (manifest, then each chunk) from `provider` (or +/// a local copy), verify every chunk's hash, decrypt it with the key +/// `key_of` gives for the manifest, check the plaintext sha256, and write the +/// tar to `out` as it arrives. +pub async fn fetch_world( + net: &Libp2pNetwork, + store: &ChunkStore, + manifest_hash: &str, + provider: &str, + key_of: impl FnOnce(&WorldManifest) -> Result<[u8; 32]>, + mut out: W, +) -> Result { let mb = fetch_bytes(net, store, manifest_hash, provider, MAX_MANIFEST_BYTES).await.context("world manifest")?; let m = WorldManifest::from_bytes(&mb)?; + let key = key_of(&m)?; let mut sha = Sha256::new(); - for c in &m.chunks { - let b = fetch_bytes(net, store, c, provider, CHUNK_BYTES).await?; + for (i, c) in m.chunks.iter().enumerate() { + let sealed = fetch_bytes(net, store, c, provider, CHUNK_BYTES + TAG_BYTES).await?; + let b = open_chunk(&key, i as u64, &sealed)?; sha.update(&b); out.write_all(&b).await?; } @@ -685,6 +1055,32 @@ pub async fn fetch_world(net: &Libp2pNetwork, Ok(m) } +// ── status records ────────────────────────────────────────────────────── + +/// What a worker gossips about each server it holds, every sweep. The +/// gossipsub message is signed by the worker's peer key (strict validation), +/// so a watcher trusts a record only when its source is the rental's worker. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct StatusRecord { + pub t: String, + pub v: u32, + pub id: String, + pub worker: String, + pub game: Game, + pub address: String, + /// The container is up, per the worker's runtime. + pub running: bool, + pub expires_at: u64, + #[serde(default)] + pub stopped_at: Option, + /// Latest backup: sealed to the buyer, so the hash is safe to gossip. + #[serde(default)] + pub backup: Option, + pub ts: u64, +} + +pub const STATUS_KIND: &str = "gs_status"; + // ── worker ────────────────────────────────────────────────────────────── pub struct Worker { @@ -692,28 +1088,36 @@ pub struct Worker { store: ChunkStore, cfg: crate::config::GameserverConfig, host: String, + state_dir: PathBuf, state_path: PathBuf, + rt: Arc, servers: Mutex>, - /// Servers a start, stop or reap is working on right now. + /// Servers a start, stop, backup or reap is working on right now. busy: Mutex>, } impl Worker { pub fn new(net: Arc, store: ChunkStore, cfg: crate::config::GameserverConfig, host: String, state_dir: &Path) -> Arc { + Self::with_runtime(net, store, cfg, host, state_dir, Arc::new(Docker)) + } + + pub fn with_runtime(net: Arc, store: ChunkStore, cfg: crate::config::GameserverConfig, host: String, state_dir: &Path, rt: Arc) -> Arc { let state_path = state_dir.join("servers.json"); let servers: HashMap = std::fs::read(&state_path).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default(); - Arc::new(Self { net, store, cfg, host, state_path, servers: Mutex::new(servers), busy: Mutex::new(HashSet::new()) }) + Arc::new(Self { net, store, cfg, host, state_dir: state_dir.to_path_buf(), state_path, rt, servers: Mutex::new(servers), busy: Mutex::new(HashSet::new()) }) + } + + pub fn net(&self) -> &Libp2pNetwork { + &self.net } - fn snapshot(&self) -> HashMap { + pub fn snapshot(&self) -> HashMap { self.servers.lock().map(|g| g.clone()).unwrap_or_default() } fn save(&self) { let snap = self.snapshot(); - if let Some(dir) = self.state_path.parent() { - let _ = std::fs::create_dir_all(dir); - } + let _ = std::fs::create_dir_all(&self.state_dir); let tmp = self.state_path.with_extension("tmp"); if std::fs::write(&tmp, serde_json::to_vec_pretty(&snap).unwrap_or_default()).is_ok() { let _ = std::fs::rename(&tmp, &self.state_path); @@ -729,6 +1133,10 @@ impl Worker { } } + fn scratch(&self, what: &str, id: &str) -> PathBuf { + self.state_dir.join(format!("{what}-{id}.tar")) + } + pub fn accepts_start(&self, offer: &JobOffer) -> bool { let Some(spec) = offer.spec_inline.as_ref().and_then(|v| serde_json::from_value::(v.clone()).ok()) else { return false; @@ -757,20 +1165,25 @@ impl Worker { async fn start(&self, job: &AcceptedJob) -> Result { let spec: StartSpec = serde_json::from_value(job.accept.spec_inline.clone().context("accept lacks spec_inline")?)?; - spec.validate().map_err(|e| anyhow::anyhow!(e))?; + spec.validate().map_err(|e| anyhow!(e))?; + // The password and the restore key come sealed, after the accept. + let raw = crate::handoff::receive(&self.net, WORKLOAD_START, &job.accept, &spec.client_pubkey, HANDOFF_WAIT).await.context("receive the sealed secrets")?; + let secrets: StartSecrets = serde_json::from_slice(&raw).context("sealed secrets are not JSON")?; + drop(raw); + secrets.validate(&spec).map_err(|e| anyhow!(e))?; let id = server_id(&job.accept.job_id); // Reserve RAM and ports under the lock, so two starts cannot both take the last slot. let (state, plan) = { - let mut servers = self.servers.lock().map_err(|_| anyhow::anyhow!("state lock poisoned"))?; + let mut servers = self.servers.lock().map_err(|_| anyhow!("state lock poisoned"))?; if servers.contains_key(&id) { bail!("server {id} already exists"); } - has_capacity(&self.cfg, crate::text::local_ram_mb(), &servers, &spec).map_err(|e| anyhow::anyhow!(e))?; + has_capacity(&self.cfg, crate::text::local_ram_mb(), &servers, &spec).map_err(|e| anyhow!(e))?; let used: HashSet = servers.values().filter(|s| s.running()).flat_map(|s| s.ports()).collect(); let port = allocate_ports(self.cfg.port_min, self.cfg.port_max, spec.game.ports().len(), &used, port_bindable).context("no free port in the range")?; let started = now_secs(); let expires = expires_at(started, spec.hours); - let plan = ContainerPlan::new(&id, &spec, port, expires); + let plan = ContainerPlan::new(&id, &spec, &secrets, port, started, expires); let state = ServerState { id: id.clone(), game: spec.game, @@ -783,6 +1196,8 @@ impl Worker { expires_at: expires, stop_hash: spec.stop_hash.clone(), buyer: job.accept.buyer_peer_id.clone(), + backup_pubkey: spec.backup_pubkey.clone(), + backup_every_secs: u64::from(spec.backup_mins) * 60, stopped_at: None, backup: None, backup_error: None, @@ -791,11 +1206,10 @@ impl Worker { (state, plan) }; self.claim(&id); - let res = self.create_and_start(&plan, &spec, &job.accept.buyer_peer_id).await; + let res = self.create_and_start(&id, &plan, &spec, &secrets, &job.accept.buyer_peer_id).await; self.release(&id); if let Err(e) = res { - let _ = docker(&["rm", "-f", &plan.name], Duration::from_secs(60)).await; - let _ = docker(&["volume", "rm", &plan.volume], Duration::from_secs(60)).await; + self.rt.remove(&plan.name, &plan.volume).await; if let Ok(mut s) = self.servers.lock() { s.remove(&id); } @@ -807,35 +1221,60 @@ impl Worker { "v": 1, "ok": true, "id": id, "game": spec.game.name(), "address": state.address, "ports": spec.game.ports().iter().enumerate().map(|(i, (p, proto))| json!({"host": plan.host_port + i as u16, "container": p, "proto": proto.as_str()})).collect::>(), "expires_at": state.expires_at, "image": spec.game.image(), "restored": spec.world, + "backup_mins": spec.backup_mins, })) } - async fn create_and_start(&self, plan: &ContainerPlan, spec: &StartSpec, buyer: &str) -> Result<()> { - let argv = plan.create_argv(); - let args: Vec<&str> = argv.iter().map(String::as_str).collect(); - docker(&args, Duration::from_secs(1800)).await?; + async fn create_and_start(&self, id: &str, plan: &ContainerPlan, spec: &StartSpec, secrets: &StartSecrets, buyer: &str) -> Result<()> { + self.rt.create(plan).await?; if let Some(world) = &spec.world { - // `docker cp - :/` extracts the tar (entries `data/...`) into the volume. - let mut child = tokio::process::Command::new("docker") - .args(["cp", "-", &format!("{}:/", plan.name)]) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .context("spawn docker cp")?; - let stdin = child.stdin.take().context("docker cp stdin")?; - let m = fetch_world(&self.net, &self.store, world, buyer, stdin).await.context("restore world")?; - if m.game != spec.game { - bail!("world is a {} world, not {}", m.game.name(), spec.game.name()); + let key = secrets.world_key()?.context("world key missing")?; + let tmp = self.scratch("restore", id); + let _ = tokio::fs::create_dir_all(&self.state_dir).await; + let res = async { + let f = tokio::fs::File::create(&tmp).await?; + let m = fetch_world(&self.net, &self.store, world, buyer, |m| { + if m.enc.epk.is_some() { + bail!("a restore world must be encrypted under a handed-off key"); + } + Ok(key) + }, f) + .await + .context("restore world")?; + if m.game != spec.game { + bail!("world is a {} world, not {}", m.game.name(), spec.game.name()); + } + self.rt.restore(plan, &tmp).await } - let out = child.wait_with_output().await?; - if !out.status.success() { - bail!("docker cp into {}: {}", plan.name, String::from_utf8_lossy(&out.stderr).trim()); + .await; + let _ = tokio::fs::remove_file(&tmp).await; + res?; + } + self.rt.start(&plan.name).await + } + + /// Back the world up now (live if the server runs), seal it to the + /// buyer, and record it as the latest backup. + pub async fn backup_now(&self, id: &str) -> Result { + let s = self.snapshot().get(id).cloned().with_context(|| format!("no server {id}"))?; + if !self.claim(id) { + bail!("server {id} is busy"); + } + let res = self.backup(&s).await; + self.release(id); + if let Ok(mut servers) = self.servers.lock() + && let Some(e) = servers.get_mut(id) + { + match &res { + Ok(b) => { + e.backup = Some(b.clone()); + e.backup_error = None; + } + Err(err) => e.backup_error = Some(format!("{err:#}")), } } - docker(&["start", &plan.name], Duration::from_secs(120)).await?; - Ok(()) + self.save(); + res } /// Stop a server, back its world up into the store, remove the container @@ -849,11 +1288,11 @@ impl Worker { bail!("server {id} is busy"); } let res = async { - let _ = docker(&["stop", "-t", "60", &s.container], Duration::from_secs(120)).await; + // Already gone when it stopped itself at expiry (`--rm`). + let _ = self.rt.stop(&s.container).await; let backup = self.backup(&s).await; if backup.is_ok() { - let _ = docker(&["rm", "-f", &s.container], Duration::from_secs(60)).await; - let _ = docker(&["volume", "rm", &s.volume], Duration::from_secs(60)).await; + self.rt.remove(&s.container, &s.volume).await; } backup } @@ -873,28 +1312,33 @@ impl Worker { } async fn backup(&self, s: &ServerState) -> Result { - let mut child = tokio::process::Command::new("docker") - .args(["cp", &format!("{}:{}", s.container, s.game.data_dir()), "-"]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .context("spawn docker cp")?; - let stdout = child.stdout.take().context("docker cp stdout")?; - let m = chunk_into_store(&self.store, Some(&self.net), s.game, stdout).await?; - let out = child.wait_with_output().await?; - if !out.status.success() { - bail!("docker cp out of {}: {}", s.container, String::from_utf8_lossy(&out.stderr).trim()); + if !is_hex(&s.backup_pubkey, 64) { + bail!("server {} has no backup key", s.id); + } + if self.rt.running(&s.container).await { + self.rt.flush(&s.container, s.game).await; + } + let _ = tokio::fs::create_dir_all(&self.state_dir).await; + let tmp = self.scratch("backup", &s.id); + let res = async { + self.rt.tar_world(&s.volume, s.game, &tmp).await?; + let e = crypto::keypair(); + let key = backup_key(&e.secret, &s.backup_pubkey, &e.public_hex, &s.backup_pubkey)?; + let f = tokio::fs::File::open(&tmp).await?; + let m = chunk_into_store(&self.store, Some(&self.net), s.game, f, &key, WorldEnc { scheme: ENC_SCHEME.into(), epk: Some(e.public_hex.clone()) }).await?; + let manifest = put_manifest(&self.store, &self.net, &m).await?; + Ok::<_, anyhow::Error>(Backup { manifest, bytes: m.bytes, sha256: m.sha256, at: now_secs() }) } - let manifest = put_manifest(&self.store, &self.net, &m).await?; - info!(id = %s.id, %manifest, bytes = m.bytes, "gameserver: world backed up"); - Ok(Backup { manifest, bytes: m.bytes, sha256: m.sha256 }) + .await; + let _ = tokio::fs::remove_file(&tmp).await; + let b = res?; + info!(id = %s.id, manifest = %b.manifest, bytes = b.bytes, "gameserver: world backed up (sealed to the buyer)"); + Ok(b) } async fn stop(&self, job: &AcceptedJob) -> Result { let spec: StopSpec = serde_json::from_value(job.accept.spec_inline.clone().context("accept lacks spec_inline")?)?; - spec.validate().map_err(|e| anyhow::anyhow!(e))?; + spec.validate().map_err(|e| anyhow!(e))?; let s = self.snapshot().get(&spec.id).cloned().context("unknown server")?; if s.stop_hash != sha256_ref(spec.token.as_bytes()) { bail!("stop token does not match"); @@ -903,15 +1347,34 @@ impl Worker { Ok(json!({"v": 1, "ok": true, "id": spec.id, "game": s.game.name(), "backup": backup})) } - /// Stop servers whose reservation ended; forget old stopped ones. + /// Kill labelled containers whose reservation passed. Runs at boot (the + /// worker may have been offline at expiry) and on every sweep. + pub async fn sweep_expired(&self) -> Vec { + let Ok(labelled) = self.rt.labelled().await else { return Vec::new() }; + let dead = expired_containers(&labelled, now_secs()); + for name in &dead { + warn!(container = %name, "gameserver: reservation passed; killing"); + self.rt.kill(name).await; + } + dead + } + + /// Kill expired containers, stop + back up servers whose reservation + /// ended, take due live backups, forget old stopped ones. pub async fn reap_once(&self) { let now = now_secs(); + self.sweep_expired().await; for id in due_for_stop(&self.snapshot(), now) { info!(%id, "gameserver: reservation ended, stopping"); if let Err(e) = self.stop_and_backup(&id).await { warn!(%id, err = %format!("{e:#}"), "gameserver: stop at expiry failed"); } } + for id in due_for_backup(&self.snapshot(), now) { + if let Err(e) = self.backup_now(&id).await { + warn!(%id, err = %format!("{e:#}"), "gameserver: live backup failed"); + } + } let mut changed = false; if let Ok(mut s) = self.servers.lock() { let before = s.len(); @@ -923,6 +1386,38 @@ impl Worker { } } + /// The status record for one server. + pub async fn status(&self, s: &ServerState) -> StatusRecord { + StatusRecord { + t: STATUS_KIND.into(), + v: 1, + id: s.id.clone(), + worker: self.net.peer_id().to_base58(), + game: s.game, + address: s.address.clone(), + running: s.running() && self.rt.running(&s.container).await, + expires_at: s.expires_at, + stopped_at: s.stopped_at, + backup: s.backup.clone(), + ts: now_secs(), + } + } + + /// Gossip a status record for every server held (stopped ones for a day, + /// so a watcher learns the final backup). + pub async fn publish_status(&self) { + let now = now_secs(); + for s in self.snapshot().values() { + if s.stopped_at.is_some_and(|t| now.saturating_sub(t) > 86_400) { + continue; + } + let r = self.status(s).await; + if let Ok(b) = serde_json::to_vec(&r) { + let _ = self.net.publish(STATUS_TOPIC, b).await; + } + } + } + async fn publish_result(&self, v: &Value) -> Result { let h = self.store.put(&serde_json::to_vec(v)?).await?; self.net.announce(&h).await?; @@ -956,8 +1451,42 @@ fn spawn_runner(worker: Arc, slots: Arc, stop: bool) -> mpsc: tx } -/// Supervisor entry: pull images, then subscribe to start and stop offers and -/// run the expiry reaper. Returns without subscribing when the node cannot host. +/// Run the sweep (expiry, live backups) and gossip status every `every`. +pub fn spawn_reaper(worker: Arc, every: Duration) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let _ = worker.net.subscribe(STATUS_TOPIC).await; + loop { + worker.reap_once().await; + worker.publish_status().await; + tokio::time::sleep(every).await; + } + }) +} + +/// Subscribe a worker to start and stop offers. +pub fn subscribe(worker: Arc, tags: Vec, pricing: Option) { + let slots = Arc::new(Semaphore::new(JOB_SLOTS)); + let load = WorkerLoad::new(slots.clone(), JOB_SLOTS); + let w = worker.clone(); + let start_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_start(o)); + let w = worker.clone(); + let stop_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_stop(o)); + crate::dispatch::run_worker_subscriber_with_load( + worker.net.clone(), + WORKLOAD_START.into(), + tags.clone(), + Some(spawn_runner(worker.clone(), slots.clone(), false)), + Some(start_filter), + pricing, + Some(load.clone()), + ); + // Stopping is part of the rent: no price floor. + crate::dispatch::run_worker_subscriber_with_load(worker.net.clone(), WORKLOAD_STOP.into(), tags, Some(spawn_runner(worker.clone(), slots, true)), Some(stop_filter), None, Some(load)); +} + +/// Supervisor entry: kill expired leftovers, pull images, then subscribe to +/// start and stop offers and run the sweep. Returns without subscribing when +/// the node cannot host. pub async fn run_worker(net: Arc, store: ChunkStore, config: crate::Config, state_dir: PathBuf) { if !docker_sync_ok(&["info", "--format", "{{.ServerVersion}}"]) { info!("gameserver: docker is not usable here; not serving game servers"); @@ -968,14 +1497,10 @@ pub async fn run_worker(net: Arc, store: ChunkStore, config: crat return; }; let worker = Worker::new(net.clone(), store, config.gameserver.clone(), host, &state_dir); + // Boot watchdog: anything whose reservation passed while we were down dies now. + worker.sweep_expired().await; // Enforce expiries even before the images finish pulling. - let reaper = worker.clone(); - tokio::spawn(async move { - loop { - reaper.reap_once().await; - tokio::time::sleep(REAP_EVERY).await; - } - }); + spawn_reaper(worker.clone(), REAP_EVERY); ensure_images(&config.gameserver).await; let ready = probe(&config); if !ready.ready() { @@ -986,29 +1511,255 @@ pub async fn run_worker(net: Arc, store: ChunkStore, config: crat *g = Some((Instant::now(), ready.clone())); } let tags = crate::capabilities::tags_from_config(&config); - let slots = Arc::new(Semaphore::new(JOB_SLOTS)); - let load = WorkerLoad::new(slots.clone(), JOB_SLOTS); - let w = worker.clone(); - let start_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_start(o)); - let w = worker.clone(); - let stop_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_stop(o)); info!(games = ?ready.games, host = ?ready.host, "gameserver: serving"); - crate::dispatch::run_worker_subscriber_with_load( - net.clone(), - WORKLOAD_START.into(), - tags.clone(), - Some(spawn_runner(worker.clone(), slots.clone(), false)), - Some(start_filter), - Some(config.pricing.clone()), - Some(load.clone()), - ); - // Stopping is part of the rent: no price floor. - crate::dispatch::run_worker_subscriber_with_load(net, WORKLOAD_STOP.into(), tags, Some(spawn_runner(worker, slots, true)), Some(stop_filter), None, Some(load)); + subscribe(worker, tags, Some(config.pricing.clone())); +} + +// ── health probes ─────────────────────────────────────────────────────── + +/// One probe of a server's game port. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ProbeResult { + pub ts: u64, + pub ok: bool, + /// `minecraft-status`, `a2s-info` or `udp-reachable`. + pub method: String, + pub rtt_ms: u64, + pub detail: String, +} + +fn put_varint(out: &mut Vec, mut v: u32) { + loop { + let mut b = (v & 0x7f) as u8; + v >>= 7; + if v != 0 { + b |= 0x80; + } + out.push(b); + if v == 0 { + break; + } + } +} + +async fn read_varint(r: &mut R) -> Result { + let mut v = 0u32; + for i in 0..5 { + let b = r.read_u8().await?; + v |= u32::from(b & 0x7f) << (7 * i); + if b & 0x80 == 0 { + return Ok(v); + } + } + bail!("varint too long") +} + +fn read_varint_slice(b: &[u8], at: &mut usize) -> Result { + let mut v = 0u32; + for i in 0..5 { + let x = *b.get(*at).context("truncated varint")?; + *at += 1; + v |= u32::from(x & 0x7f) << (7 * i); + if x & 0x80 == 0 { + return Ok(v); + } + } + bail!("varint too long") +} + +/// Minecraft Java "server list ping": handshake (next state 1) + status request. +pub fn minecraft_status_request(host: &str, port: u16) -> Vec { + let mut body = Vec::new(); + put_varint(&mut body, 0x00); + put_varint(&mut body, 767); + put_varint(&mut body, host.len() as u32); + body.extend_from_slice(host.as_bytes()); + body.extend_from_slice(&port.to_be_bytes()); + put_varint(&mut body, 1); + let mut pkt = Vec::new(); + put_varint(&mut pkt, body.len() as u32); + pkt.extend(body); + pkt.extend([0x01, 0x00]); + pkt +} + +/// A status response packet carrying `json` (what a server answers). +pub fn minecraft_status_response(json: &str) -> Vec { + let mut body = Vec::new(); + put_varint(&mut body, 0x00); + put_varint(&mut body, json.len() as u32); + body.extend_from_slice(json.as_bytes()); + let mut pkt = Vec::new(); + put_varint(&mut pkt, body.len() as u32); + pkt.extend(body); + pkt +} + +/// Parse a status response body (after the length prefix) into its JSON. +pub fn parse_minecraft_status(body: &[u8]) -> Result { + let mut at = 0; + if read_varint_slice(body, &mut at)? != 0 { + bail!("not a status response"); + } + let n = read_varint_slice(body, &mut at)? as usize; + let s = body.get(at..at + n).context("truncated status")?; + let v: Value = serde_json::from_slice(s).context("status is not JSON")?; + if !v.is_object() { + bail!("status is not an object"); + } + Ok(v) +} + +async fn minecraft_status(addr: SocketAddr, host: &str, timeout: Duration) -> Result { + tokio::time::timeout(timeout, async { + let mut s = tokio::net::TcpStream::connect(addr).await?; + s.write_all(&minecraft_status_request(host, addr.port())).await?; + let len = read_varint(&mut s).await? as usize; + if len == 0 || len > 256 * 1024 { + bail!("status packet length {len}"); + } + let mut body = vec![0u8; len]; + s.read_exact(&mut body).await?; + let v = parse_minecraft_status(&body)?; + let online = v["players"]["online"].as_u64().unwrap_or(0); + let version = v["version"]["name"].as_str().unwrap_or("?").to_string(); + Ok(format!("{version}, {online} online")) + }) + .await + .context("timed out")? +} + +/// Steam A2S_INFO request (Valheim answers on its query port, game port + 1). +pub const A2S_INFO: &[u8] = b"\xFF\xFF\xFF\xFFTSource Engine Query\0"; + +async fn a2s_info(addr: SocketAddr, timeout: Duration) -> Result { + tokio::time::timeout(timeout, async { + let bind: SocketAddr = if addr.is_ipv4() { "0.0.0.0:0".parse()? } else { "[::]:0".parse()? }; + let sock = tokio::net::UdpSocket::bind(bind).await?; + sock.connect(addr).await?; + let mut q = A2S_INFO.to_vec(); + let mut buf = vec![0u8; 1400]; + for _ in 0..2 { + sock.send(&q).await?; + let n = sock.recv(&mut buf).await?; + let r = &buf[..n]; + if n < 5 || r[..4] != [0xFF; 4] { + bail!("not an A2S answer"); + } + match r[4] { + // Challenge: repeat the query with it appended. + 0x41 if n >= 9 => { + q = A2S_INFO.to_vec(); + q.extend_from_slice(&r[5..9]); + } + 0x49 => return Ok(format!("a2s info ({n} bytes)")), + other => bail!("unexpected A2S answer 0x{other:02x}"), + } + } + bail!("no A2S info after the challenge") + }) + .await + .context("timed out")? +} + +/// Factorio has no cheap unauthenticated query. A connected UDP socket learns +/// of a closed port from the ICMP "port unreachable" (ECONNREFUSED on recv); +/// silence means "not refused". Weak on its own, so a watcher also requires +/// a fresh worker status record saying the container runs. +async fn udp_reachable(addr: SocketAddr, timeout: Duration) -> Result { + let bind: SocketAddr = if addr.is_ipv4() { "0.0.0.0:0".parse()? } else { "[::]:0".parse()? }; + let sock = tokio::net::UdpSocket::bind(bind).await?; + sock.connect(addr).await?; + sock.send(&[0u8]).await?; + let mut buf = [0u8; 64]; + match tokio::time::timeout(timeout, sock.recv(&mut buf)).await { + Ok(Err(e)) if e.kind() == std::io::ErrorKind::ConnectionRefused => bail!("port unreachable"), + Ok(Err(e)) => bail!("{e}"), + Ok(Ok(_)) => Ok("answered".into()), + Err(_) => Ok("not refused".into()), + } +} + +fn split_address(address: &str) -> Option<(String, u16)> { + let (h, p) = address.rsplit_once(':')?; + Some((h.trim_start_matches('[').trim_end_matches(']').to_string(), p.parse().ok()?)) +} + +/// Probe a server's game port the way its players' clients would. +pub async fn probe_server(game: Game, address: &str, timeout: Duration) -> ProbeResult { + let started = Instant::now(); + let method = match game { + Game::Minecraft => "minecraft-status", + Game::Valheim => "a2s-info", + Game::Factorio => "udp-reachable", + }; + let res: Result = async { + let (host, port) = split_address(address).context("address is not host:port")?; + let mut addrs = tokio::net::lookup_host((host.as_str(), port)).await?; + let addr = addrs.next().context("address does not resolve")?; + match game { + Game::Minecraft => minecraft_status(addr, &host, timeout).await, + Game::Valheim => a2s_info(SocketAddr::new(addr.ip(), port.checked_add(1).context("port")?), timeout).await, + Game::Factorio => udp_reachable(addr, timeout).await, + } + } + .await; + let (ok, detail) = match res { + Ok(d) => (true, d), + Err(e) => (false, format!("{e:#}")), + }; + ProbeResult { ts: now_secs(), ok, method: method.into(), rtt_ms: started.elapsed().as_millis() as u64, detail } +} + +/// Decides when a server has been down long enough to fail over. +#[derive(Clone, Debug)] +pub struct HealthTracker { + fail_after_secs: u64, + first_fail: Option, + pub last_ok: Option, +} + +impl HealthTracker { + pub fn new(fail_after: Duration) -> Self { + Self { fail_after_secs: fail_after.as_secs().max(1), first_fail: None, last_ok: None } + } + /// Record one check; `true` once checks have failed without a pass for + /// `fail_after`. + pub fn observe(&mut self, now: u64, ok: bool) -> bool { + if ok { + self.first_fail = None; + self.last_ok = Some(now); + return false; + } + let since = *self.first_fail.get_or_insert(now); + now.saturating_sub(since) >= self.fail_after_secs + } + pub fn reset(&mut self) { + self.first_fail = None; + } +} + +/// Is the server healthy, given a probe and the worker's latest status +/// record (and when it arrived)? A record that says the container is down +/// always fails it; for Factorio, whose probe is weak, a missing or stale +/// record fails it too. +pub fn judge(game: Game, probe: &ProbeResult, status: Option<(&StatusRecord, u64)>, now: u64) -> bool { + if let Some((s, _)) = status + && (!s.running || s.stopped_at.is_some()) + { + return false; + } + if game == Game::Factorio { + let fresh = status.is_some_and(|(s, seen)| s.running && now.saturating_sub(seen) <= 180); + return probe.ok && fresh; + } + probe.ok } // ── buyer ─────────────────────────────────────────────────────────────── -/// What the buyer keeps about a server it rented (`gameserver ls`). +/// What the buyer keeps about a server it rented (`gameserver ls`). The file +/// is mode 0600 and never leaves this machine. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct Rental { pub id: String, @@ -1025,6 +1776,28 @@ pub struct Rental { pub stopped_at: Option, #[serde(default)] pub backup: Option, + // What a failover needs to start the same server elsewhere. + #[serde(default)] + pub ram_gb: u32, + #[serde(default)] + pub cpus: f64, + #[serde(default)] + pub accept_eula: bool, + #[serde(default)] + pub password: Option, + #[serde(default)] + pub server_name: Option, + #[serde(default)] + pub backup_mins: u32, + /// The newest backup mirrored to `worlds/.tar` on this machine. + #[serde(default)] + pub latest_backup: Option, + /// The rental this one replaced after a failover. + #[serde(default)] + pub replaces: Option, + /// The rental that replaced this one after a failover. + #[serde(default)] + pub superseded_by: Option, } pub fn buyer_dir() -> PathBuf { @@ -1035,47 +1808,95 @@ pub fn load_rentals(dir: &Path) -> Vec { std::fs::read(dir.join("rentals.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default() } -pub fn save_rentals(dir: &Path, rentals: &[Rental]) -> Result<()> { - std::fs::create_dir_all(dir)?; - let p = dir.join("rentals.json"); - std::fs::write(&p, serde_json::to_vec_pretty(rentals)?)?; +fn write_private(p: &Path, bytes: &[u8]) -> Result<()> { + if let Some(d) = p.parent() { + std::fs::create_dir_all(d)?; + } + let tmp = p.with_extension("tmp"); + std::fs::write(&tmp, bytes)?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let _ = std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o600)); + std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600))?; } + std::fs::rename(&tmp, p)?; Ok(()) } +pub fn save_rentals(dir: &Path, rentals: &[Rental]) -> Result<()> { + write_private(&dir.join("rentals.json"), &serde_json::to_vec_pretty(rentals)?) +} + +fn upsert_rental(dir: &Path, r: &Rental) -> Result<()> { + let mut all = load_rentals(dir); + match all.iter_mut().find(|x| x.id == r.id) { + Some(x) => *x = r.clone(), + None => all.push(r.clone()), + } + save_rentals(dir, &all) +} + +/// The buyer's long-term backup key (`

/backup.key`, 0600), created on +/// first use. Every worker seals its backups to the public half. +pub fn backup_keypair(dir: &Path) -> Result { + let p = dir.join("backup.key"); + if let Ok(h) = std::fs::read_to_string(&p) { + let secret = crypto::secret_from_hex(h.trim()).context("backup.key is not a 32-byte hex key")?; + let public_hex = hex::encode(x25519_dalek::PublicKey::from(&secret).as_bytes()); + return Ok(crypto::KeyPair { secret, public_hex }); + } + let kp = crypto::keypair(); + write_private(&p, hex::encode(kp.secret.to_bytes()).as_bytes())?; + Ok(kp) +} + pub struct StartRequest { pub game: Game, pub hours: u32, pub ram_gb: u32, pub cpus: f64, pub accept_eula: bool, - /// A manifest hash from an earlier stop, or a `.tar` laid out like a backup. + /// A manifest hash from an earlier backup, or a `.tar` laid out like a backup. pub world: Option, pub password: Option, pub server_name: Option, + /// Minutes between live backups (0 = only at stop/expiry). + pub backup_mins: u32, pub max_price_usd: f64, pub workers: Vec, + /// Never use these workers (a failover excludes the one that failed). + pub exclude: Vec, pub timeout: Duration, } -async fn auction(net: Arc, workload: &str, spec: Value, caps: Vec, workers: Vec, max_price: f64, timeout: Duration) -> Result<(crate::buyer::AuctionOutcome, Value)> { - let a = crate::JobAuction::new(workload, spec) +async fn auction( + net: Arc, + workload: &str, + spec: Value, + caps: Vec, + workers: Vec, + exclude: Vec, + max_price: f64, + timeout: Duration, + notify: Option>, +) -> Result<(crate::buyer::AuctionOutcome, Value)> { + let mut a = crate::JobAuction::new(workload, spec) .with_required_capabilities(caps) .with_allowed_bidders(workers) + .with_excluded_bidders(exclude) .with_max_price_usd(max_price) .with_bid_window(Duration::from_secs(4)) .with_deadline(timeout) .with_receipt_timeout(timeout); + if let Some(tx) = notify { + a = a.with_accept_notify(tx); + } let o = crate::run_auction(net.clone(), a).await?; if o.receipt.status != JobStatus::Completed { bail!("worker {} reported {:?}", o.accepted_bid.bidder_peer_id, o.receipt.status); } let hash = o.receipt.output_hash.clone().context("worker returned no result hash")?; - let bytes = crate::gateway_util::fetch_verified(&net, &hash, &o.accepted_bid.bidder_peer_id, 64 * 1024, Duration::from_secs(30)).await.map_err(|e| anyhow::anyhow!("fetch result: {e:?}"))?; + let bytes = crate::gateway_util::fetch_verified(&net, &hash, &o.accepted_bid.bidder_peer_id, 64 * 1024, Duration::from_secs(30)).await.map_err(|e| anyhow!("fetch result: {e:?}"))?; let v: Value = serde_json::from_slice(&bytes)?; if v["ok"] != json!(true) { bail!("worker result: {v}"); @@ -1083,54 +1904,76 @@ async fn auction(net: Arc, workload: &str, spec: Value, caps: Vec Ok((o, v)) } -/// Put a world into the local store for the worker to fetch; returns the manifest hash. -async fn publish_world(store: &ChunkStore, net: &Libp2pNetwork, dir: &Path, game: Game, world: &str) -> Result { +/// Encrypt a world under a fresh key and put it in the local store for the +/// worker to fetch; returns the manifest hash and the key. +async fn publish_world(store: &ChunkStore, net: &Libp2pNetwork, dir: &Path, game: Game, world: &str) -> Result<(String, [u8; 32])> { let (tar, saved_manifest) = if is_hex(world, 64) { (dir.join("worlds").join(format!("{world}.tar")), Some(dir.join("worlds").join(format!("{world}.json")))) } else { (PathBuf::from(world), None) }; let f = tokio::fs::File::open(&tar).await.with_context(|| format!("open world {}", tar.display()))?; - let m = chunk_into_store(store, Some(net), game, f).await?; - if let Some(p) = saved_manifest { - let want = WorldManifest::from_bytes(&std::fs::read(&p).with_context(|| format!("read {}", p.display()))?)?; - if want != m { + let key = random_key(); + let m = chunk_into_store(store, Some(net), game, f, &key, WorldEnc::handed_off()).await?; + if let Some(p) = saved_manifest + && let Ok(b) = std::fs::read(&p) + { + // The manifest saved beside the tar names its game and plaintext sha256. + let saved: Value = serde_json::from_slice(&b).with_context(|| format!("read {}", p.display()))?; + if saved["sha256"].as_str() != Some(m.sha256.as_str()) { bail!("{} does not match its manifest", tar.display()); } - if want.game != game { - bail!("that world is a {} world", want.game.name()); + if saved["game"].as_str().is_some_and(|g| g != game.name()) { + bail!("that world is a {} world", saved["game"].as_str().unwrap_or("?")); } } - put_manifest(store, net, &m).await + Ok((put_manifest(store, net, &m).await?, key)) } -/// Rent a server: auction it, wait for the address, remember the rental. -pub async fn start(req: StartRequest) -> Result { - let token = { - let b: [u8; 32] = rand::random(); - hex::encode(b) - }; - let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; - let dir = buyer_dir(); - let world = match &req.world { - Some(w) => Some(publish_world(&sup.store, &sup.libp2p, &dir, req.game, w).await?), - None => None, +/// Rent a server on `net` (a booted buyer node), remembering it under `dir`. +pub async fn start_with(net: Arc, store: &ChunkStore, dir: &Path, req: StartRequest) -> Result { + let token = hex::encode(rand::random::<[u8; 32]>()); + let backup = backup_keypair(dir)?; + let (world, world_key) = match &req.world { + Some(w) => { + let (h, k) = publish_world(store, &net, dir, req.game, w).await?; + (Some(h), Some(hex::encode(k))) + } + None => (None, None), }; + let kp = crate::handoff::keypair(); let spec = StartSpec { - v: 1, + v: SPEC_VERSION, game: req.game, hours: req.hours, ram_gb: req.ram_gb, cpus: req.cpus, accept_eula: req.accept_eula, stop_hash: sha256_ref(token.as_bytes()), + client_pubkey: kp.public_hex.clone(), + backup_pubkey: backup.public_hex.clone(), + backup_mins: req.backup_mins, world, - password: req.password.clone(), server_name: req.server_name.clone(), }; - spec.validate().map_err(|e| anyhow::anyhow!(e))?; - tokio::time::sleep(Duration::from_secs(5)).await; - let (o, v) = auction(sup.libp2p.clone(), WORKLOAD_START, serde_json::to_value(&spec)?, vec![ROLE_TAG.into(), req.game.tag()], req.workers.clone(), req.max_price_usd, req.timeout).await?; + spec.validate().map_err(|e| anyhow!(e))?; + let secrets = StartSecrets { v: 1, password: req.password.clone(), world_key }; + secrets.validate(&spec).map_err(|e| anyhow!(e))?; + let mut handoff = crate::handoff::BuyerHandoff::spawn(net.clone(), WORKLOAD_START, kp, serde_json::to_vec(&secrets)?)?; + let res = auction( + net.clone(), + WORKLOAD_START, + serde_json::to_value(&spec)?, + vec![ROLE_TAG.into(), req.game.tag()], + req.workers.clone(), + req.exclude.clone(), + req.max_price_usd, + req.timeout, + Some(handoff.notify()?), + ) + .await; + handoff.finish(); + let (o, v) = res?; let rental = Rental { id: v["id"].as_str().context("result has no id")?.to_string(), game: req.game, @@ -1143,65 +1986,278 @@ pub async fn start(req: StartRequest) -> Result { price_usd: o.accepted_bid.price_usd, stopped_at: None, backup: None, + ram_gb: req.ram_gb, + cpus: req.cpus, + accept_eula: req.accept_eula, + password: req.password, + server_name: req.server_name, + backup_mins: req.backup_mins, + latest_backup: None, + replaces: None, + superseded_by: None, }; - let mut all = load_rentals(&dir); - all.push(rental.clone()); - save_rentals(&dir, &all)?; + upsert_rental(dir, &rental)?; Ok(rental) } -/// Stop a rental early (or collect the backup of one that expired) and -/// download its world to `/worlds/.tar`. -pub async fn stop(id: &str, timeout: Duration) -> Result { - let dir = buyer_dir(); - let mut all = load_rentals(&dir); - let idx = all.iter().position(|r| r.id == id).with_context(|| format!("no rental {id} on this machine (see `c0mpute gameserver ls`)"))?; - let r = all[idx].clone(); +/// Rent a server: boot a buyer node, auction it, wait for the address, +/// remember the rental. +pub async fn start(req: StartRequest) -> Result { let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; tokio::time::sleep(Duration::from_secs(5)).await; + start_with(sup.libp2p.clone(), &sup.store, &buyer_dir(), req).await +} + +/// Download, decrypt and keep a backup as `/worlds/.tar` (+ its +/// manifest as `.json`). +pub async fn download_backup(net: &Libp2pNetwork, store: &ChunkStore, dir: &Path, manifest: &str, provider: &str) -> Result { + let key = backup_keypair(dir)?; + let worlds = dir.join("worlds"); + tokio::fs::create_dir_all(&worlds).await?; + let tmp = worlds.join(format!("{manifest}.partial")); + let f = tokio::fs::File::create(&tmp).await?; + let res = fetch_world(net, store, manifest, provider, |m| { + let epk = m.enc.epk.as_deref().context("backup has no one-time key; it was not sealed to us")?; + backup_key(&key.secret, epk, epk, &key.public_hex) + }, f) + .await; + let m = match res { + Ok(m) => m, + Err(e) => { + let _ = tokio::fs::remove_file(&tmp).await; + return Err(e.context("download world backup")); + } + }; + tokio::fs::rename(&tmp, worlds.join(format!("{manifest}.tar"))).await?; + tokio::fs::write(worlds.join(format!("{manifest}.json")), m.to_bytes()).await?; + Ok(m) +} + +/// Stop a rental early (or collect the backup of one that expired) and +/// download its world to `/worlds/.tar`. +pub async fn stop_with(net: Arc, store: &ChunkStore, dir: &Path, id: &str, timeout: Duration) -> Result { + let all = load_rentals(dir); + let r = all.iter().find(|r| r.id == id).cloned().with_context(|| format!("no rental {id} on this machine (see `c0mpute gameserver ls`)"))?; let spec = StopSpec { v: 1, id: r.id.clone(), token: r.token.clone() }; - let (_, v) = auction(sup.libp2p.clone(), WORKLOAD_STOP, serde_json::to_value(&spec)?, vec![ROLE_TAG.into()], vec![r.worker.clone()], 0.0, timeout).await?; + let (_, v) = auction(net.clone(), WORKLOAD_STOP, serde_json::to_value(&spec)?, vec![ROLE_TAG.into()], vec![r.worker.clone()], vec![], 0.0, timeout, None).await?; let backup: Option = serde_json::from_value(v["backup"].clone()).ok().flatten(); let mut updated = r.clone(); updated.stopped_at = Some(now_secs()); if let Some(b) = &backup { - let worlds = dir.join("worlds"); - tokio::fs::create_dir_all(&worlds).await?; - let tmp = worlds.join(format!("{}.partial", b.manifest)); - let f = tokio::fs::File::create(&tmp).await?; - let m = fetch_world(&sup.libp2p, &sup.store, &b.manifest, &r.worker, f).await.context("download world backup")?; - tokio::fs::rename(&tmp, worlds.join(format!("{}.tar", b.manifest))).await?; - tokio::fs::write(worlds.join(format!("{}.json", b.manifest)), m.to_bytes()).await?; + download_backup(&net, store, dir, &b.manifest, &r.worker).await?; updated.backup = Some(b.manifest.clone()); + updated.latest_backup = Some(b.manifest.clone()); } - all[idx] = updated.clone(); - save_rentals(&dir, &all)?; + upsert_rental(dir, &updated)?; Ok(updated) } +pub async fn stop(id: &str, timeout: Duration) -> Result { + let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; + tokio::time::sleep(Duration::from_secs(5)).await; + stop_with(sup.libp2p.clone(), &sup.store, &buyer_dir(), id, timeout).await +} + +/// `gameserver watch` knobs. +#[derive(Clone, Debug)] +pub struct WatchOptions { + pub probe_every: Duration, + pub probe_timeout: Duration, + /// Fail over after the server has failed every check for this long. + pub fail_after: Duration, + pub failover: bool, + pub max_price_usd: f64, + pub workers: Vec, + pub start_timeout: Duration, + /// Return after the first failover instead of watching the replacement. + pub stop_after_failover: bool, +} + +impl Default for WatchOptions { + fn default() -> Self { + Self { + probe_every: Duration::from_secs(30), + probe_timeout: Duration::from_secs(5), + fail_after: Duration::from_secs(5 * 60), + failover: true, + max_price_usd: 1.0, + workers: Vec::new(), + start_timeout: Duration::from_secs(900), + stop_after_failover: false, + } + } +} + +/// How a watch ended. +#[derive(Clone, Debug, PartialEq)] +pub enum WatchEnd { + /// The reservation ended or the server was stopped. + Ended(Rental), + /// It failed over and `stop_after_failover` was set. + FailedOver { from: Rental, to: Rental }, +} + +fn log_probe(dir: &Path, id: &str, p: &ProbeResult) { + let d = dir.join("health"); + let _ = std::fs::create_dir_all(&d); + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(d.join(format!("{id}.jsonl"))) { + use std::io::Write; + let _ = writeln!(f, "{}", serde_json::to_string(p).unwrap_or_default()); + } +} + +/// Read a server's probe log. +pub fn health_log(dir: &Path, id: &str) -> Vec { + std::fs::read_to_string(dir.join("health").join(format!("{id}.jsonl"))).unwrap_or_default().lines().filter_map(|l| serde_json::from_str(l).ok()).collect() +} + +/// Watch a rental: probe its port every `probe_every` (logged to +/// `/health/.jsonl`), mirror each backup its worker announces, and +/// when it has failed every check for `fail_after`, start the same game on +/// another worker from the newest mirrored backup. +pub async fn watch_with(net: Arc, store: &ChunkStore, dir: &Path, id: &str, opts: WatchOptions) -> Result { + net.subscribe(STATUS_TOPIC).await?; + let mut rx = net.messages(); + let mut rental = load_rentals(dir).into_iter().find(|r| r.id == id).with_context(|| format!("no rental {id} on this machine"))?; + let mut health = HealthTracker::new(opts.fail_after); + let mut status: Option<(StatusRecord, u64)> = None; + let mut tick = tokio::time::interval(opts.probe_every); + loop { + if now_secs() >= rental.expires_at || rental.stopped_at.is_some() { + return Ok(WatchEnd::Ended(rental)); + } + tokio::select! { + m = rx.recv() => { + let msg = match m { + Ok(m) => m, + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(_) => bail!("network closed"), + }; + if msg.topic != STATUS_TOPIC || msg.source.map(|p| p.to_base58()).as_deref() != Some(rental.worker.as_str()) { + continue; + } + let Ok(rec) = serde_json::from_slice::(&msg.data) else { continue }; + if rec.t != STATUS_KIND || rec.id != rental.id || rec.worker != rental.worker { + continue; + } + if let Some(b) = &rec.backup + && rental.latest_backup.as_deref() != Some(b.manifest.as_str()) + { + match download_backup(&net, store, dir, &b.manifest, &rental.worker).await { + Ok(_) => { + info!(id = %rental.id, manifest = %b.manifest, "gameserver watch: backup mirrored"); + rental.latest_backup = Some(b.manifest.clone()); + upsert_rental(dir, &rental)?; + } + Err(e) => warn!(id = %rental.id, err = %format!("{e:#}"), "gameserver watch: could not mirror backup"), + } + } + status = Some((rec, now_secs())); + } + _ = tick.tick() => { + let p = probe_server(rental.game, &rental.address, opts.probe_timeout).await; + let now = now_secs(); + let ok = judge(rental.game, &p, status.as_ref().map(|(s, at)| (s, *at)), now); + log_probe(dir, &rental.id, &ProbeResult { ok, ..p.clone() }); + if !health.observe(now, ok) || !opts.failover { + continue; + } + warn!(id = %rental.id, worker = %rental.worker, "gameserver watch: failing checks; failing over"); + match failover(net.clone(), store, dir, &rental, &opts).await { + Ok(next) => { + let from = rental.clone(); + if opts.stop_after_failover { + return Ok(WatchEnd::FailedOver { from, to: next }); + } + info!(from = %from.id, to = %next.id, address = %next.address, "gameserver watch: failed over"); + rental = next; + status = None; + health = HealthTracker::new(opts.fail_after); + } + Err(e) => { + warn!(id = %rental.id, err = %format!("{e:#}"), "gameserver watch: failover failed; retrying next check"); + health.reset(); + } + } + } + } + } +} + +pub async fn watch(id: &str, opts: WatchOptions) -> Result { + let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; + tokio::time::sleep(Duration::from_secs(5)).await; + watch_with(sup.libp2p.clone(), &sup.store, &buyer_dir(), id, opts).await +} + +/// Start `old`'s game on a different worker from its newest mirrored backup +/// for the rest of its reservation, and link the two rentals. +pub async fn failover(net: Arc, store: &ChunkStore, dir: &Path, old: &Rental, opts: &WatchOptions) -> Result { + let left = old.expires_at.saturating_sub(now_secs()); + if left == 0 { + bail!("the reservation already ended"); + } + let req = StartRequest { + game: old.game, + hours: left.div_ceil(3600).clamp(1, u64::from(MAX_HOURS)) as u32, + ram_gb: old.ram_gb.max(old.game.min_ram_gb()), + cpus: if old.cpus > 0.0 { old.cpus } else { 2.0 }, + accept_eula: old.accept_eula, + world: old.latest_backup.clone(), + password: old.password.clone(), + server_name: old.server_name.clone(), + backup_mins: if old.backup_mins == 0 { 0 } else { old.backup_mins.clamp(MIN_BACKUP_MINS, MAX_BACKUP_MINS) }, + max_price_usd: opts.max_price_usd, + workers: opts.workers.clone(), + exclude: vec![old.worker.clone()], + timeout: opts.start_timeout, + }; + if req.world.is_none() { + warn!(id = %old.id, "gameserver failover: no backup mirrored yet; the replacement starts a fresh world"); + } + let mut next = start_with(net, store, dir, req).await?; + next.replaces = Some(old.id.clone()); + upsert_rental(dir, &next)?; + let mut prev = old.clone(); + prev.superseded_by = Some(next.id.clone()); + upsert_rental(dir, &prev)?; + Ok(next) +} + #[cfg(test)] mod tests { use super::*; + fn key() -> String { + crypto::keypair().public_hex + } + fn spec(game: Game) -> StartSpec { StartSpec { - v: 1, + v: SPEC_VERSION, game, hours: 4, ram_gb: 4, cpus: 2.0, accept_eula: true, stop_hash: sha256_ref(b"t"), + client_pubkey: key(), + backup_pubkey: key(), + backup_mins: 30, world: None, - password: (game == Game::Valheim).then(|| "hunter22".into()), server_name: None, } } + fn secrets(game: Game) -> StartSecrets { + StartSecrets { v: 1, password: (game == Game::Valheim).then(|| "hunter22".into()), world_key: None } + } + #[test] fn spec_validation() { for g in Game::ALL { assert!(spec(g).validate().is_ok(), "{g:?}"); + assert!(secrets(g).validate(&spec(g)).is_ok(), "{g:?}"); } let bad = |f: &dyn Fn(&mut StartSpec)| { let mut s = spec(Game::Minecraft); @@ -1218,18 +2274,19 @@ mod tests { assert!(bad(&|s| s.cpus = 64.0)); assert!(bad(&|s| s.stop_hash = "sha256:XYZ".into())); assert!(bad(&|s| s.world = Some("../etc".into()))); - assert!(bad(&|s| s.password = Some("hunter22".into())), "only valheim takes a password"); assert!(bad(&|s| s.server_name = Some("$(rm -rf /)".into()))); - assert!(bad(&|s| s.v = 2)); - // Factorio does not need the EULA flag; Valheim needs a password. + assert!(bad(&|s| s.v = 1), "v1 offers put the password in public"); + assert!(bad(&|s| s.client_pubkey = "nope".into())); + assert!(bad(&|s| s.backup_pubkey = String::new())); + assert!(bad(&|s| s.backup_mins = 1)); + assert!(bad(&|s| s.backup_mins = MAX_BACKUP_MINS + 1)); + assert!(!bad(&|s| s.backup_mins = 0), "0 turns live backups off"); + // Factorio does not need the EULA flag. let mut f = spec(Game::Factorio); f.accept_eula = false; assert!(f.validate().is_ok()); let mut v = spec(Game::Valheim); - v.password = Some("abc".into()); - assert!(v.validate().is_err()); v.ram_gb = 2; - v.password = Some("hunter22".into()); assert!(v.validate().is_err(), "valheim needs 4 GB"); // Unknown games never parse. let mut raw = serde_json::to_value(spec(Game::Minecraft)).unwrap(); @@ -1239,6 +2296,23 @@ mod tests { assert_eq!(Game::parse("terraria"), None); } + #[test] + fn secrets_never_ride_in_the_offer() { + // There is no password field in a start spec at all. + let raw = serde_json::to_string(&spec(Game::Valheim)).unwrap(); + assert!(!raw.contains("password") && !raw.contains("hunter22"), "{raw}"); + // Secrets are checked against the spec. + let v = spec(Game::Valheim); + assert!(StartSecrets { v: 1, password: None, world_key: None }.validate(&v).is_err(), "valheim needs one"); + assert!(StartSecrets { v: 1, password: Some("abc".into()), world_key: None }.validate(&v).is_err()); + assert!(StartSecrets { v: 1, password: Some("hunter22".into()), world_key: None }.validate(&spec(Game::Factorio)).is_err()); + let mut w = spec(Game::Minecraft); + w.world = Some("a".repeat(64)); + assert!(StartSecrets { v: 1, password: None, world_key: None }.validate(&w).is_err(), "a restore needs its key"); + assert!(StartSecrets { v: 1, password: None, world_key: Some("b".repeat(64)) }.validate(&w).is_ok()); + assert!(StartSecrets { v: 1, password: None, world_key: Some("b".repeat(64)) }.validate(&spec(Game::Minecraft)).is_err()); + } + #[test] fn stop_spec_validation() { let ok = StopSpec { v: 1, id: server_id("job"), token: "a".repeat(64) }; @@ -1249,9 +2323,9 @@ mod tests { } #[test] - fn docker_argv_has_limits_ports_volume_and_no_shell() { + fn docker_argv_has_limits_ports_volume_expiry_and_no_shell() { let s = spec(Game::Minecraft); - let plan = ContainerPlan::new("gs-0123456789abcdef", &s, 27500, 1_000); + let plan = ContainerPlan::new("gs-0123456789abcdef", &s, &secrets(Game::Minecraft), 27500, 1_000, 1_000 + 4 * 3600); let a = plan.create_argv(); let pair = |k: &str, v: &str| a.windows(2).any(|w| w[0] == k && w[1] == v); assert_eq!(a[0], "create"); @@ -1262,25 +2336,48 @@ mod tests { assert!(pair("--pids-limit", "1024")); assert!(pair("--restart", "no")); assert!(pair("--security-opt", "no-new-privileges")); + assert!(pair("--stop-timeout", "60")); assert!(pair("-p", "27500:25565/tcp")); assert!(pair("-v", "c0mpute-gs-0123456789abcdef:/data")); assert!(pair("-e", "EULA=TRUE")); assert!(pair("-e", "MEMORY=3328M")); - assert!(pair("--label", "c0mpute.gameserver.expires=1000")); - assert_eq!(a.last().unwrap(), Game::Minecraft.image()); + assert!(pair("--label", &format!("c0mpute.gameserver.expires={}", 1_000 + 4 * 3600))); + // Hard expiry without the worker: --rm + --init + timeout around the image's own command. + assert!(a.contains(&"--rm".to_string()) && a.contains(&"--init".to_string())); + assert!(pair("--entrypoint", "timeout")); + let img = a.iter().position(|x| x == Game::Minecraft.image()).unwrap(); + assert_eq!(&a[img + 1..], &["-s", "TERM", "-k", "90s", "14400s", "/image/scripts/start"]); assert!(!a.iter().any(|x| x == "sh" || x == "-c" || x == "--privileged" || x.starts_with("--network=host"))); - // Valheim maps two consecutive UDP ports and passes its password as one argument. - let v = ContainerPlan::new("gs-0123456789abcdef", &spec(Game::Valheim), 27510, 1).create_argv(); + // Valheim maps two consecutive UDP ports; its password comes from the sealed secrets. + let v = ContainerPlan::new("gs-0123456789abcdef", &spec(Game::Valheim), &secrets(Game::Valheim), 27510, 1, 2).create_argv(); assert!(v.windows(2).any(|w| w[0] == "-p" && w[1] == "27510:2456/udp")); assert!(v.windows(2).any(|w| w[0] == "-p" && w[1] == "27511:2457/udp")); assert!(v.windows(2).any(|w| w[0] == "-e" && w[1] == "SERVER_PASS=hunter22")); assert!(v.windows(2).any(|w| w[0] == "-v" && w[1] == "c0mpute-gs-0123456789abcdef:/config")); + assert_eq!(v.last().unwrap(), "/usr/local/sbin/bootstrap"); // Images are pinned by digest. for g in Game::ALL { assert!(g.image().contains("@sha256:"), "{g:?}"); + assert!(!g.command().is_empty()); } } + #[test] + fn tar_argv_reads_the_volume_read_only_with_no_network() { + let a = tar_argv("c0mpute-gs-x", Game::Factorio); + assert!(a.windows(2).any(|w| w[0] == "--network" && w[1] == "none")); + assert!(a.windows(2).any(|w| w[0] == "-v" && w[1] == "c0mpute-gs-x:/factorio:ro")); + assert!(a.windows(2).any(|w| w[0] == "--entrypoint" && w[1] == "tar")); + assert_eq!(a.last().unwrap(), "factorio"); + } + + #[test] + fn labelled_listing_and_expired_containers() { + let l = parse_labelled("c0mpute-gs-a\t100\nc0mpute-gs-b\t9999999999\nc0mpute-gs-c\t\n\n"); + assert_eq!(l, vec![("c0mpute-gs-a".into(), 100), ("c0mpute-gs-b".into(), 9_999_999_999), ("c0mpute-gs-c".into(), 0)]); + assert_eq!(expired_containers(&l, 1_000), vec!["c0mpute-gs-a".to_string(), "c0mpute-gs-c".to_string()]); + } + fn state(id: &str, ram_mb: u64, port: u16, expires: u64, stopped: Option) -> ServerState { ServerState { id: id.into(), @@ -1294,6 +2391,8 @@ mod tests { expires_at: expires, stop_hash: sha256_ref(b"t"), buyer: "b".into(), + backup_pubkey: key(), + backup_every_secs: 1800, stopped_at: stopped, backup: None, backup_error: None, @@ -1301,7 +2400,7 @@ mod tests { } #[test] - fn expiry() { + fn expiry_and_backup_schedule() { assert_eq!(expires_at(100, 4), 100 + 4 * 3600); let mut m = HashMap::new(); m.insert("a".into(), state("a", 1024, 1, 1_000, None)); @@ -1310,6 +2409,14 @@ mod tests { assert!(due_for_stop(&m, 999).is_empty()); assert_eq!(due_for_stop(&m, 1_000), vec!["a".to_string()]); assert_eq!(due_for_stop(&m, 9_999), vec!["a".to_string(), "b".to_string()]); + // Live backups: every 30 min from the start, then from the last backup. + assert!(due_for_backup(&m, 1_799).is_empty()); + assert_eq!(due_for_backup(&m, 1_800), vec!["b".to_string()], "a is past expiry, c stopped"); + m.get_mut("b").unwrap().backup = Some(Backup { manifest: "x".into(), bytes: 1, sha256: "s".into(), at: 1_800 }); + assert!(due_for_backup(&m, 3_599).is_empty()); + assert_eq!(due_for_backup(&m, 3_600), vec!["b".to_string()]); + m.get_mut("b").unwrap().backup_every_secs = 0; + assert!(due_for_backup(&m, 4_000).is_empty(), "0 = no live backups"); } #[test] @@ -1327,7 +2434,6 @@ mod tests { assert!(has_capacity(&cfg, 0, &m, &spec(Game::Minecraft)).unwrap_err().contains("slots")); m.get_mut("b").unwrap().stopped_at = Some(1); assert!(has_capacity(&cfg, 0, &m, &spec(Game::Minecraft)).is_ok(), "stopped servers free their slot"); - // Default budget is half the box. let half = crate::config::GameserverConfig { max_total_ram_gb: 0, ..cfg.clone() }; assert!(has_capacity(&half, 6 * 1024, &HashMap::new(), &spec(Game::Minecraft)).is_err()); assert!(has_capacity(&half, 16 * 1024, &HashMap::new(), &spec(Game::Minecraft)).is_ok()); @@ -1347,11 +2453,14 @@ mod tests { assert_eq!(public_host(Some("localhost"), Some("http://box.local"), Some("/ip4/127.0.0.1/tcp/1")), None); assert_eq!(public_host(None, None, None), None); assert_eq!(host_port_str("2001:db8::1", 5), "[2001:db8::1]:5"); + assert_eq!(split_address("[2001:db8::1]:5"), Some(("2001:db8::1".into(), 5))); + assert_eq!(split_address("1.2.3.4:27500"), Some(("1.2.3.4".into(), 27500))); + assert_eq!(split_address("nope"), None); } #[test] fn manifest_checks() { - let mut m = WorldManifest::new(Game::Factorio); + let mut m = WorldManifest::new(Game::Factorio, WorldEnc::handed_off()); m.bytes = CHUNK_BYTES as u64 + 1; m.sha256 = sha256_ref(b"x"); m.chunks = vec!["a".repeat(64), "b".repeat(64)]; @@ -1365,24 +2474,147 @@ mod tests { let mut huge = m.clone(); huge.bytes = MAX_WORLD_BYTES + 1; assert!(huge.check().is_err()); + let mut plain = m.clone(); + plain.enc.scheme = "none".into(); + assert!(plain.check().is_err(), "unencrypted worlds are refused"); + let mut v1 = serde_json::to_value(&m).unwrap(); + v1["v"] = json!(1); + v1.as_object_mut().unwrap().remove("enc"); + assert!(WorldManifest::from_bytes(&serde_json::to_vec(&v1).unwrap()).is_err()); } #[tokio::test] - async fn chunking_round_trips_through_the_store() { + async fn encrypted_chunking_round_trips_and_only_the_key_reads_it() { let dir = tempfile::tempdir().unwrap(); let store = ChunkStore::open(dir.path()).await.unwrap(); let data: Vec = (0..(CHUNK_BYTES * 2 + 17)).map(|i| (i % 251) as u8).collect(); - let m = chunk_into_store(&store, None, Game::Minecraft, &data[..]).await.unwrap(); + let k = random_key(); + let m = chunk_into_store(&store, None, Game::Minecraft, &data[..], &k, WorldEnc::handed_off()).await.unwrap(); m.check().unwrap(); assert_eq!(m.chunks.len(), 3); assert_eq!(m.bytes, data.len() as u64); assert_eq!(m.sha256, sha256_ref(&data)); let mut back = Vec::new(); - for c in &m.chunks { - back.extend(store.get(&Hash::from_hex(c).unwrap()).await.unwrap()); + for (i, c) in m.chunks.iter().enumerate() { + let sealed = store.get(&Hash::from_hex(c).unwrap()).await.unwrap(); + assert_ne!(&sealed[..64], &data[i * CHUNK_BYTES..i * CHUNK_BYTES + 64], "chunks are stored encrypted"); + assert!(open_chunk(&random_key(), i as u64, &sealed).is_err(), "another key cannot read it"); + assert!(open_chunk(&k, i as u64 + 1, &sealed).is_err(), "a chunk cannot move position"); + back.extend(open_chunk(&k, i as u64, &sealed).unwrap()); } assert_eq!(back, data); - let empty = chunk_into_store(&store, None, Game::Minecraft, &b""[..]).await.unwrap(); + let empty = chunk_into_store(&store, None, Game::Minecraft, &b""[..], &k, WorldEnc::handed_off()).await.unwrap(); assert!(empty.chunks.is_empty() && empty.check().is_ok()); } + + #[test] + fn backups_are_sealed_to_the_buyer_key() { + let buyer = crypto::keypair(); + let e = crypto::keypair(); + let worker_side = backup_key(&e.secret, &buyer.public_hex, &e.public_hex, &buyer.public_hex).unwrap(); + let buyer_side = backup_key(&buyer.secret, &e.public_hex, &e.public_hex, &buyer.public_hex).unwrap(); + assert_eq!(worker_side, buyer_side); + let eve = crypto::keypair(); + assert_ne!(worker_side, backup_key(&eve.secret, &e.public_hex, &e.public_hex, &buyer.public_hex).unwrap()); + } + + #[test] + fn backup_key_file_is_created_once_and_private() { + let dir = tempfile::tempdir().unwrap(); + let a = backup_keypair(dir.path()).unwrap(); + let b = backup_keypair(dir.path()).unwrap(); + assert_eq!(a.public_hex, b.public_hex); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(dir.path().join("backup.key")).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + } + } + + #[test] + fn minecraft_status_codec() { + let req = minecraft_status_request("play.example.com", 25565); + // length-prefixed handshake, then the 1-byte status request. + assert_eq!(&req[req.len() - 2..], &[0x01, 0x00]); + assert_eq!(req[0] as usize, req.len() - 3); + let resp = minecraft_status_response(r#"{"version":{"name":"1.21"},"players":{"online":3,"max":10}}"#); + let mut at = 0; + let len = read_varint_slice(&resp, &mut at).unwrap() as usize; + let v = parse_minecraft_status(&resp[at..at + len]).unwrap(); + assert_eq!(v["players"]["online"], 3); + assert!(parse_minecraft_status(&[0x01, 0x00]).is_err()); + assert!(parse_minecraft_status(&[0x00, 0x05, b'{']).is_err()); + let mut big = Vec::new(); + put_varint(&mut big, 300); + assert_eq!(big, vec![0xAC, 0x02]); + } + + #[test] + fn health_tracker_fails_over_only_after_a_sustained_outage() { + let mut h = HealthTracker::new(Duration::from_secs(300)); + assert!(!h.observe(0, true)); + assert!(!h.observe(30, false)); + assert!(!h.observe(200, false)); + assert!(!h.observe(250, true), "one pass resets the outage"); + assert!(!h.observe(300, false)); + assert!(!h.observe(599, false)); + assert!(h.observe(600, false)); + assert_eq!(h.last_ok, Some(250)); + } + + #[test] + fn judging_uses_the_signed_status_record() { + let up = ProbeResult { ts: 0, ok: true, method: "x".into(), rtt_ms: 1, detail: String::new() }; + let down = ProbeResult { ok: false, ..up.clone() }; + let rec = StatusRecord { t: STATUS_KIND.into(), v: 1, id: "gs-x".into(), worker: "w".into(), game: Game::Minecraft, address: "a:1".into(), running: true, expires_at: 9, stopped_at: None, backup: None, ts: 0 }; + assert!(judge(Game::Minecraft, &up, None, 0)); + assert!(!judge(Game::Minecraft, &down, Some((&rec, 0)), 0)); + let stopped = StatusRecord { running: false, ..rec.clone() }; + assert!(!judge(Game::Minecraft, &up, Some((&stopped, 0)), 0), "the worker says it is down"); + // Factorio's probe is weak: it also needs a fresh "running" record. + assert!(!judge(Game::Factorio, &up, None, 0)); + assert!(judge(Game::Factorio, &up, Some((&rec, 100)), 200)); + assert!(!judge(Game::Factorio, &up, Some((&rec, 0)), 500), "stale record"); + } + + #[tokio::test] + async fn probes_answer_for_live_servers_and_fail_for_dead_ports() { + // A Minecraft-status responder on loopback. + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut s, _)) = l.accept().await { + tokio::spawn(async move { + let mut buf = [0u8; 512]; + let _ = s.read(&mut buf).await; + let _ = s.write_all(&minecraft_status_response(r#"{"version":{"name":"fake"},"players":{"online":0}}"#)).await; + }); + } + }); + let p = probe_server(Game::Minecraft, &format!("127.0.0.1:{port}"), Duration::from_secs(3)).await; + assert!(p.ok, "{p:?}"); + assert_eq!(p.method, "minecraft-status"); + // An A2S responder (with a challenge round) on game port + 1. + let game = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + let gport = game.local_addr().unwrap().port(); + let q = tokio::net::UdpSocket::bind(("127.0.0.1", gport + 1)).await; + if let Ok(q) = q { + tokio::spawn(async move { + let mut buf = [0u8; 256]; + while let Ok((n, from)) = q.recv_from(&mut buf).await { + let reply: Vec = if n == A2S_INFO.len() { b"\xFF\xFF\xFF\xFFA\x01\x02\x03\x04".to_vec() } else { b"\xFF\xFF\xFF\xFFI\x11fake".to_vec() }; + let _ = q.send_to(&reply, from).await; + } + }); + let p = probe_server(Game::Valheim, &format!("127.0.0.1:{gport}"), Duration::from_secs(3)).await; + assert!(p.ok, "{p:?}"); + } + drop(game); + // A closed port: refused (TCP) / unreachable (UDP). + let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); + assert!(!probe_server(Game::Minecraft, &format!("127.0.0.1:{closed}"), Duration::from_secs(2)).await.ok); + let uclosed = std::net::UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); + assert!(!probe_server(Game::Factorio, &format!("127.0.0.1:{uclosed}"), Duration::from_secs(2)).await.ok); + } } diff --git a/node/crates/c0mpute-core/src/handoff.rs b/node/crates/c0mpute-core/src/handoff.rs new file mode 100644 index 00000000..86f50c17 --- /dev/null +++ b/node/crates/c0mpute-core/src/handoff.rs @@ -0,0 +1,251 @@ +//! Hand a small secret to the one worker that won an auction. +//! +//! Gossip topics are readable by every subscriber, so a secret (a game +//! server password, the key a world backup is encrypted with) can never ride +//! in an offer. Instead the offer carries the buyer's fresh X25519 public key +//! and, after the accept: +//! +//! ```text +//! worker → handoff_ready its own fresh X25519 key (gossip-signed by the worker) +//! buyer → handoff_secret the secret, sealed to it (gossip-signed by the buyer) +//! ``` +//! +//! The key is X25519 → HKDF-SHA256 → AES-256-GCM (`temporal::crypto`), bound +//! to the workload and job id. The buyer seals only to the bidder it +//! accepted, and only on a `handoff_ready` whose gossip source is that +//! bidder; the worker opens only a secret whose gossip source is the offer's +//! buyer. Gossipsub runs with strict signing, so a source cannot be forged. +//! Every other node sees two public keys and ciphertext. +//! +//! The same exchange `webhook.deliver` does for its request, cut down to one +//! direction and a payload of at most [`MAX_SECRET`] bytes. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; +use c0mpute_net::topics::{JobAccept, JobBid, job_topic}; +use c0mpute_net::{GossipMessage, Libp2pNetwork}; +use serde::{Deserialize, Serialize}; +use tokio::sync::{broadcast, mpsc}; +use tokio::task::JoinHandle; +use tracing::debug; + +use crate::temporal::crypto; + +pub use crate::temporal::crypto::KeyPair; + +/// A sealed secret must fit one gossip message with room to spare. +pub const MAX_SECRET: usize = 16 * 1024; +const REPUBLISH: Duration = Duration::from_secs(2); + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Ready { + t: String, + job_id: String, + worker_peer_id: String, + client_pubkey: String, + worker_pubkey: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Sealed { + t: String, + job_id: String, + worker_peer_id: String, + sealed: String, +} + +/// A fresh key pair for one handoff. +pub fn keypair() -> KeyPair { + crypto::keypair() +} + +/// `true` for 64 lowercase hex characters (an X25519 public key). +pub fn valid_pubkey(s: &str) -> bool { + s.len() == 64 && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// The shared key, bound to the workload and the job. +pub fn key(secret: &x25519_dalek::StaticSecret, peer_pub_hex: &str, workload: &str, job_id: &str) -> Result<[u8; 32]> { + crypto::session_key(secret, peer_pub_hex, &format!("handoff:{workload}:{job_id}")) +} + +fn from(msg: &GossipMessage, peer: &str) -> bool { + msg.source.map(|p| p.to_base58()).as_deref() == Some(peer) +} + +/// Worker side: announce a fresh key until the buyer's sealed secret +/// arrives (or `wait` passes), and return the opened secret. +pub async fn receive(net: &Libp2pNetwork, workload: &str, accept: &JobAccept, client_pubkey: &str, wait: Duration) -> Result> { + if !valid_pubkey(client_pubkey) { + bail!("the offer's client key is not an X25519 public key"); + } + let topic = job_topic(workload); + let me = net.peer_id().to_base58(); + let mut rx = net.messages(); + let kp = crypto::keypair(); + let k = key(&kp.secret, client_pubkey, workload, &accept.job_id)?; + let ready = serde_json::to_vec(&Ready { + t: "handoff_ready".into(), + job_id: accept.job_id.clone(), + worker_peer_id: me.clone(), + client_pubkey: client_pubkey.to_string(), + worker_pubkey: kp.public_hex.clone(), + })?; + let deadline = Instant::now() + wait; + let mut next = Instant::now(); + loop { + if Instant::now() >= deadline { + bail!("the buyer never sent the sealed secret"); + } + if Instant::now() >= next { + let _ = net.publish(&topic, ready.clone()).await; + next = Instant::now() + REPUBLISH; + } + let left = next.min(deadline).saturating_duration_since(Instant::now()); + let msg = match tokio::time::timeout(left, rx.recv()).await { + Ok(Ok(m)) => m, + Ok(Err(broadcast::error::RecvError::Closed)) => bail!("network closed"), + _ => continue, + }; + if msg.topic != topic || msg.data.len() > MAX_SECRET * 2 { + continue; + } + let Ok(s) = serde_json::from_slice::(&msg.data) else { continue }; + if s.t != "handoff_secret" || s.job_id != accept.job_id || s.worker_peer_id != me || !from(&msg, &accept.buyer_peer_id) { + continue; + } + match crypto::open(&k, &s.sealed) { + Ok(plain) if plain.len() <= MAX_SECRET => return Ok(plain), + Ok(_) => bail!("sealed secret too large"), + Err(e) => debug!(job_id = %accept.job_id, err = %e, "handoff: unreadable secret ignored"), + } + } +} + +/// Buyer side of one auction's handoff. Pass [`BuyerHandoff::notify`] to +/// `JobAuction::with_accept_notify`; drop (or [`BuyerHandoff::finish`]) it +/// once the receipt is in. +pub struct BuyerHandoff { + notify: Option>, + task: JoinHandle<()>, +} + +impl BuyerHandoff { + /// Start listening for the winner's key. `kp` is the pair whose public + /// half went into the offer. + pub fn spawn(net: Arc, workload: &str, kp: KeyPair, secret: Vec) -> Result { + if secret.len() > MAX_SECRET { + bail!("secret is {} bytes; a handoff carries at most {MAX_SECRET}", secret.len()); + } + let topic = job_topic(workload); + let workload = workload.to_string(); + let mut rx = net.messages(); + let (tx, mut acc_rx) = mpsc::unbounded_channel::<(String, JobBid)>(); + let task = tokio::spawn(async move { + let mut winners: HashMap = HashMap::new(); + let mut readies: HashMap = HashMap::new(); + // job_id → sealed message, republished until the task ends. + let mut sent: HashMap> = HashMap::new(); + let mut tick = tokio::time::interval(REPUBLISH); + let seal = |r: &Ready| -> Result> { + let k = key(&kp.secret, &r.worker_pubkey, &workload, &r.job_id)?; + Ok(serde_json::to_vec(&Sealed { + t: "handoff_secret".into(), + job_id: r.job_id.clone(), + worker_peer_id: r.worker_peer_id.clone(), + sealed: crypto::seal(&k, &secret)?, + })?) + }; + loop { + tokio::select! { + Some((job_id, bid)) = acc_rx.recv() => { + if let Some(r) = readies.remove(&job_id) + && r.worker_peer_id == bid.bidder_peer_id + && let Ok(m) = seal(&r) + { + let _ = net.publish(&topic, m.clone()).await; + sent.insert(job_id.clone(), m); + } + winners.insert(job_id, bid.bidder_peer_id); + } + m = rx.recv() => { + let msg = match m { + Ok(m) => m, + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(_) => return, + }; + if msg.topic != topic { + continue; + } + let Ok(r) = serde_json::from_slice::(&msg.data) else { continue }; + // Seal only to the bidder we accepted, on a key it signed. + if r.t != "handoff_ready" || r.client_pubkey != kp.public_hex || !from(&msg, &r.worker_peer_id) || sent.contains_key(&r.job_id) || !valid_pubkey(&r.worker_pubkey) { + continue; + } + match winners.get(&r.job_id) { + Some(w) if *w == r.worker_peer_id => { + if let Ok(m) = seal(&r) { + let _ = net.publish(&topic, m.clone()).await; + sent.insert(r.job_id.clone(), m); + } + } + Some(_) => debug!(job_id = %r.job_id, "handoff: key from a worker we did not accept ignored"), + None if readies.len() < 64 => { readies.insert(r.job_id.clone(), r); } + None => {} + } + } + _ = tick.tick() => { + for m in sent.values() { + let _ = net.publish(&topic, m.clone()).await; + } + } + } + } + }); + Ok(Self { notify: Some(tx), task }) + } + + /// The accept notifier for `JobAuction::with_accept_notify`. + pub fn notify(&mut self) -> Result> { + self.notify.take().context("handoff notifier already taken") + } + + pub fn finish(self) {} +} + +impl Drop for BuyerHandoff { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn both_sides_derive_one_key_bound_to_the_job() { + let buyer = keypair(); + let worker = keypair(); + let b = key(&buyer.secret, &worker.public_hex, "gameserver.start", "job-1").unwrap(); + let w = key(&worker.secret, &buyer.public_hex, "gameserver.start", "job-1").unwrap(); + assert_eq!(b, w); + assert_ne!(b, key(&buyer.secret, &worker.public_hex, "gameserver.start", "job-2").unwrap()); + assert_ne!(b, key(&buyer.secret, &worker.public_hex, "webhook.deliver", "job-1").unwrap()); + let sealed = crypto::seal(&b, b"hunter22").unwrap(); + assert!(!sealed.contains("hunter22")); + assert_eq!(crypto::open(&w, &sealed).unwrap(), b"hunter22"); + let eve = keypair(); + assert!(crypto::open(&key(&eve.secret, &buyer.public_hex, "gameserver.start", "job-1").unwrap(), &sealed).is_err()); + } + + #[test] + fn pubkeys_are_checked() { + assert!(valid_pubkey(&keypair().public_hex)); + assert!(!valid_pubkey("zz")); + assert!(!valid_pubkey(&"A".repeat(64))); + } +} diff --git a/node/crates/c0mpute-core/src/lib.rs b/node/crates/c0mpute-core/src/lib.rs index 0bc26f3f..d1c9a35e 100644 --- a/node/crates/c0mpute-core/src/lib.rs +++ b/node/crates/c0mpute-core/src/lib.rs @@ -8,6 +8,7 @@ pub mod ads; pub mod buyer; pub mod capabilities; pub mod config; +pub mod container; pub mod crawl; pub mod diffusion; pub mod hosting; @@ -16,6 +17,7 @@ pub mod dispatch; pub mod embed; pub mod gameserver; pub(crate) mod gateway_util; +pub mod handoff; pub mod jobs_gateway; pub mod net_guard; pub mod ocr; diff --git a/node/crates/c0mpute-core/src/speech_gateway.rs b/node/crates/c0mpute-core/src/speech_gateway.rs index abb0bcb7..e9783b63 100644 --- a/node/crates/c0mpute-core/src/speech_gateway.rs +++ b/node/crates/c0mpute-core/src/speech_gateway.rs @@ -132,9 +132,32 @@ pub async fn serve(bind: std::net::SocketAddr, token: String, max_price_usd: f64 } let supervisor = Supervisor::boot_buyer(Config::default()).await?; + let penalties = crate::penalty_box::PenaltyBox::persistent( + crate::config::data_dir() + .unwrap_or_else(|| std::path::PathBuf::from("./c0mpute-data")) + .join("gateway-penalties.json"), + ); + let app = router(supervisor.libp2p, supervisor.store, token, max_price_usd, penalties); + let listener = tokio::net::TcpListener::bind(bind) + .await + .with_context(|| format!("bind c0mpute speech gateway at {bind}"))?; + tracing::info!(%bind, "c0mpute speech gateway listening"); + axum::serve(listener, app).await?; + Ok(()) +} + +/// The gateway's routes over an already booted buyer node. `serve` uses it; +/// so do the loopback end-to-end tests, with their own node and listener. +pub fn router( + net: Arc, + store: ChunkStore, + token: String, + max_price_usd: f64, + penalties: crate::penalty_box::PenaltyBox, +) -> Router { let state = GatewayState { - net: supervisor.libp2p, - store: supervisor.store, + net, + store, token: Arc::from(token), max_price_usd, // Let one multi-chunk readback fan out across up to six network @@ -145,13 +168,9 @@ pub async fn serve(bind: std::net::SocketAddr, token: String, max_price_usd: f64 ocr_slots: Arc::new(Semaphore::new(MAX_CONCURRENT_OCR_JOBS)), embed_slots: Arc::new(Semaphore::new(MAX_CONCURRENT_EMBED_JOBS)), sessions: crate::speech_session::SessionStore::new(), - penalties: crate::penalty_box::PenaltyBox::persistent( - crate::config::data_dir() - .unwrap_or_else(|| std::path::PathBuf::from("./c0mpute-data")) - .join("gateway-penalties.json"), - ), + penalties, }; - let app = Router::new() + Router::new() .route("/healthz", get(healthz)) .route("/v1/voices", get(voices)) .route("/v1/tts", post(synthesize)) @@ -171,13 +190,7 @@ pub async fn serve(bind: std::net::SocketAddr, token: String, max_price_usd: f64 .route("/v1/sessions/{session}", delete(close_session)) .layer(DefaultBodyLimit::max(MAX_AUDIO_BYTES)) .layer(TraceLayer::new_for_http()) - .with_state(state); - let listener = tokio::net::TcpListener::bind(bind) - .await - .with_context(|| format!("bind c0mpute speech gateway at {bind}"))?; - tracing::info!(%bind, "c0mpute speech gateway listening"); - axum::serve(listener, app).await?; - Ok(()) + .with_state(state) } async fn healthz() -> Json { diff --git a/node/crates/c0mpute-core/src/video.rs b/node/crates/c0mpute-core/src/video.rs index b78ce87c..0afefc5c 100644 --- a/node/crates/c0mpute-core/src/video.rs +++ b/node/crates/c0mpute-core/src/video.rs @@ -43,9 +43,25 @@ pub const WORKLOAD_IMG2VID: &str = "video.img2vid"; /// are all present — see [`readiness`]. pub const ROLE_TAG: &str = "c0mpute:role:video"; -/// The image the runner lives in. Kept in step with -/// `plugins/video/module.toml` [module.dispatch]. -pub const RUNNER_IMAGE: &str = "ghcr.io/profullstack/c0mpute-video-runner:candidate"; +/// Where `.github/workflows/video-runner.yml` publishes the runner. +pub const RUNNER_REPOSITORY: &str = "ghcr.io/profullstack/c0mpute-video-runner"; + +/// The stable tag the workflow moves on every master build. +pub const RUNNER_TAG: &str = "stable"; + +/// The published runner this release trusts, pinned by digest: a tag that +/// moves cannot change what a node runs. Bump it (and +/// `plugins/video/module.toml` [module.dispatch]) to the digest the workflow +/// prints in its job summary after a runner change. +pub const RUNNER_DIGEST: &str = "sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c"; + +/// What `c0mpute video setup` pulls: the repository at the pinned digest. +pub const RUNNER_PULL_REF: &str = "ghcr.io/profullstack/c0mpute-video-runner@sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c"; + +/// The local name the node runs. Setup tags the pinned pull as this, or +/// builds it from source under this name when the pull fails, so the worker +/// never depends on how the image arrived. +pub const RUNNER_IMAGE: &str = "c0mpute/video-runner:stable"; /// Where a node can build the runner itself when it cannot pull it. /// @@ -302,6 +318,71 @@ pub fn build_argv(checkout: &Path, image: &str) -> Vec { ] } +/// How `c0mpute video setup` got the runner image. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SetupSource { + /// Pulled at the pinned digest and tagged as [`RUNNER_IMAGE`]. + Pulled, + /// The pull failed (or `--build`): built from a clone of the source repo. + Built, +} + +/// The source repo to build from: `$C0MPUTE_VIDEO_SOURCE_REPO` (the same +/// override `install.sh` takes), else [`RUNNER_SOURCE_REPO`]. +pub fn source_repo() -> String { + std::env::var("C0MPUTE_VIDEO_SOURCE_REPO").ok().filter(|s| !s.is_empty()).unwrap_or_else(|| RUNNER_SOURCE_REPO.to_string()) +} + +async fn run_status(cmd: &mut tokio::process::Command, what: &str) -> Result { + Ok(cmd + .stdin(std::process::Stdio::null()) + .status() + .await + .with_context(|| format!("could not run {what}"))? + .success()) +} + +/// Put the runner image on this node as [`RUNNER_IMAGE`]: pull the pinned +/// digest and tag it, or, when the registry will not serve it (private, +/// rate-limited, offline) or `build_only`, shallow-clone the public repo +/// into `scratch` and build the same Dockerfile. `say` gets progress lines. +pub async fn setup(build_only: bool, scratch: &Path, say: &(dyn Fn(&str) + Sync)) -> Result { + if !build_only { + say(&format!("pulling {RUNNER_PULL_REF} (this is a large image)")); + let pulled = run_status(crate::container::command().args(["pull", RUNNER_PULL_REF]), "docker; install a container runtime first").await?; + if pulled && run_status(crate::container::command().args(["tag", RUNNER_PULL_REF, RUNNER_IMAGE]), "docker tag").await? { + return Ok(SetupSource::Pulled); + } + // A registry is the kind of dependency that is private by default, + // rate-limited or down, and none of those are a reason a node with a + // card cannot serve video. The repo is public: build it. + say("could not pull the runner image (private, rate-limited or offline)."); + say("building it from source instead: same Dockerfile, same runner."); + } + // A fresh directory per run: a half-finished clone from an earlier + // attempt would make `git clone` fail rather than resume. + let checkout = scratch.join(format!("c0mpute-video-src-{}", std::process::id())); + let _ = fs::remove_dir_all(&checkout).await; + let repo = source_repo(); + say(&format!("cloning {repo} (shallow)")); + let cloned = run_status(tokio::process::Command::new("git").args(["clone", "--depth", "1", &repo]).arg(&checkout), "git (building the runner needs git)").await; + let built = match cloned { + Ok(true) => { + say(&format!("building {RUNNER_IMAGE} (several minutes)")); + run_status(crate::container::command().args(build_argv(&checkout, RUNNER_IMAGE)), "docker build").await + } + Ok(false) => Err(anyhow::anyhow!("git clone of {repo} failed")), + Err(e) => Err(e), + }; + // The checkout is scratch whether or not the build worked. + let _ = fs::remove_dir_all(&checkout).await; + if !built? { + bail!("docker build of the runner failed"); + } + say(&format!("built {RUNNER_IMAGE}")); + Ok(SetupSource::Built) +} + /// The runner command for a workload type. pub fn command_for(workload: &str) -> Option<&'static str> { match workload { @@ -335,7 +416,7 @@ impl NotReady { /// Probe the host. Blocking, and only called at boot or from the CLI. pub fn readiness(image: &str) -> Option { - if which("docker").is_none() { + if crate::container::which().is_none() { return Some(NotReady::NoDocker); } if !nvidia_gpu_present() { @@ -369,14 +450,6 @@ pub fn readiness_cached(image: &str) -> Option { verdict } -fn which(bin: &str) -> Option { - std::env::var_os("PATH").and_then(|paths| { - std::env::split_paths(&paths) - .map(|dir| dir.join(bin)) - .find(|candidate| candidate.is_file()) - }) -} - fn nvidia_gpu_present() -> bool { std::process::Command::new("nvidia-smi") .args(["--query-gpu=name", "--format=csv,noheader"]) @@ -386,7 +459,7 @@ fn nvidia_gpu_present() -> bool { } fn image_present(image: &str) -> bool { - std::process::Command::new("docker") + crate::container::command_sync() .args(["image", "inspect", image]) .output() .map(|out| out.status.success()) @@ -539,7 +612,7 @@ async fn owner_of(_path: &Path) -> Result<(u32, u32)> { async fn run_container(run: &ContainerRun) -> Result<()> { let args = docker_argv(run); info!(image = %run.image, command = %run.command, "video runner: starting container"); - let mut child = tokio::process::Command::new("docker") + let mut child = crate::container::command() .args(&args) .stdin(std::process::Stdio::null()) .kill_on_drop(true) @@ -557,7 +630,7 @@ async fn run_container(run: &ContainerRun) -> Result<()> { Err(_) => { // kill_on_drop reaps `docker run`, but the container it started is // a child of the daemon and would keep the GPU busy. - let _ = tokio::process::Command::new("docker") + let _ = crate::container::command() .args(["kill", &run.container_name]) .output() .await; @@ -746,6 +819,23 @@ mod tests { assert_eq!(readiness_cached(RUNNER_IMAGE).is_none(), fresh); } + #[test] + fn the_runner_is_pinned_by_digest_and_matches_the_manifest() { + assert_eq!(RUNNER_PULL_REF, format!("{RUNNER_REPOSITORY}@{RUNNER_DIGEST}")); + let digest = RUNNER_DIGEST.strip_prefix("sha256:").unwrap(); + assert!(digest.len() == 64 && digest.bytes().all(|b| b.is_ascii_hexdigit())); + // The local name is not a registry reference: nothing can pull a + // different image under it. + assert!(!RUNNER_IMAGE.contains("ghcr.io") && !RUNNER_IMAGE.contains('@')); + let manifest: toml::Value = + toml::from_str(include_str!("../../../../plugins/video/module.toml")).unwrap(); + assert_eq!( + manifest["module"]["dispatch"]["image"].as_str(), + Some(RUNNER_PULL_REF), + "plugins/video/module.toml [module.dispatch] image must be the pinned digest" + ); + } + #[test] fn weights_live_under_the_cache_root() { let dir = weights_dir(Path::new("/var/c0mpute")); diff --git a/node/crates/c0mpute-core/tests/fixtures/tiny.mp4 b/node/crates/c0mpute-core/tests/fixtures/tiny.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b245e00c6506f65775a42915f154695e3c4f31c GIT binary patch literal 1542 zcmZux&ub(_6s}1$1o7Y@3sHiUZVn!jNq0{&u7So#2u4uwVn7g@>gt+wGu>U4s+weG zPYZg`s|PP0JTD01NiX6La#-;n*nkH`SQJ6gQ!YAwuRF8J1|R8q-+S+Sud3dw31f^8 z{KOPaSB%Xu()c2AEW^rdhK$YMsI)$2jFr`Koz$0q#Zi)+rQY>v>k%24OiaGBv}b4`AzkQD>zhPDfTnJpfobu9~9FRz!rX zW^HUGvx+z*6n}DLOWxwnP3O}R_-p6gY#_w>jD`kHaA2JH(m$7A9cg9 z8wLR{i*y|IH``#dE>EY>TVU;WGWNvuXSRF@{ps0jt&=cTYc1cRFk#|NWOA zp8w{b^K;65$_Qtm=@|4arQTljxTeODEIZ%ypYR25k?;zxB?&-g`7^zQ*zGnn_w7X- zKB-$D@#pt|#VGRv`t3?Jn!?Xroe}10g)!8|hq6|*4`KDL*HmyN6HPL0&d{s@n$@FT z!CV|zg(1&Bm3r{r1&^v;8kVTD#BBZvOEnHI%P&sjcE+e+zKG3)jXQ zZ|=RizR6!X*hMT;5`kT9CZ!s99tA;fJqjXX=iZy`jg6C&lP*0fU5dJ^?coO1?dHBJ z!O_MSx^~-Kid4ur7p2-2h2gcvXy5mQckI8HpA3p^C#km$}uINRS$GZ&g_q4(QhSRwg}wJgVXV zM-bU*TRqwg!74U!@>}wEpTKKJs3FU?G< zWZKcc(DLAMWRsBWXhIPSj$hx5wB zo4hcX5#9qvh+5(Z anyhow::Result> { + self.0.get(hash).await + } +} + +/// A world: 2.5 MiB of incompressible bytes, so it spans three chunks. +fn world(seed: u64) -> Vec { + let mut x = seed | 1; + let mut v = Vec::with_capacity(2_621_440); + while v.len() < 2_621_440 { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + v.extend_from_slice(&x.to_le_bytes()); + } + v +} + +#[derive(Default)] +struct Fake { + seed: Vec, + plans: Mutex>, + vols: Mutex>>, + live: Mutex>>, + labelled: Mutex>, + killed: Mutex>, + restored: Mutex>, +} + +impl Fake { + fn new(seed: Vec) -> Arc { + Arc::new(Self { seed, ..Default::default() }) + } + fn plan(&self, name: &str) -> Option { + self.plans.lock().unwrap().iter().rev().find(|p| p.name == name).cloned() + } + /// The server process dies (host trouble, OOM, the worker went away). + fn crash(&self, name: &str) { + if let Some(h) = self.live.lock().unwrap().remove(name) { + h.abort(); + } + } +} + +#[async_trait::async_trait] +impl Runtime for Fake { + async fn create(&self, plan: &ContainerPlan) -> anyhow::Result<()> { + self.plans.lock().unwrap().push(plan.clone()); + self.vols.lock().unwrap().entry(plan.volume.clone()).or_insert_with(|| self.seed.clone()); + let exp = plan.labels.iter().find(|(k, _)| k == "c0mpute.gameserver.expires").and_then(|(_, v)| v.parse().ok()).unwrap_or(0); + self.labelled.lock().unwrap().push((plan.name.clone(), exp)); + Ok(()) + } + async fn restore(&self, plan: &ContainerPlan, tar: &Path) -> anyhow::Result<()> { + self.vols.lock().unwrap().insert(plan.volume.clone(), std::fs::read(tar)?); + self.restored.lock().unwrap().push(plan.volume.clone()); + Ok(()) + } + async fn start(&self, name: &str) -> anyhow::Result<()> { + let plan = self.plan(name).ok_or_else(|| anyhow::anyhow!("no container {name}"))?; + let listener = tokio::net::TcpListener::bind(("127.0.0.1", plan.host_port)).await?; + let h = tokio::spawn(async move { + while let Ok((mut s, _)) = listener.accept().await { + tokio::spawn(async move { + let mut buf = [0u8; 512]; + let _ = s.read(&mut buf).await; + let _ = s.write_all(&gameserver::minecraft_status_response(r#"{"version":{"name":"fake 1.21"},"players":{"online":1,"max":8}}"#)).await; + }); + } + }); + self.live.lock().unwrap().insert(name.to_string(), h); + Ok(()) + } + async fn running(&self, name: &str) -> bool { + self.live.lock().unwrap().get(name).is_some_and(|h| !h.is_finished()) + } + async fn flush(&self, _name: &str, _game: Game) {} + async fn stop(&self, name: &str) -> anyhow::Result<()> { + self.crash(name); + Ok(()) + } + async fn tar_world(&self, volume: &str, _game: Game, out: &Path) -> anyhow::Result<()> { + let v = self.vols.lock().unwrap().get(volume).cloned().ok_or_else(|| anyhow::anyhow!("no volume {volume}"))?; + std::fs::write(out, v)?; + Ok(()) + } + async fn remove(&self, name: &str, volume: &str) { + self.crash(name); + self.vols.lock().unwrap().remove(volume); + self.labelled.lock().unwrap().retain(|(n, _)| n != name); + } + async fn labelled(&self) -> anyhow::Result> { + Ok(self.labelled.lock().unwrap().clone()) + } + async fn kill(&self, name: &str) { + self.crash(name); + self.killed.lock().unwrap().push(name.to_string()); + self.labelled.lock().unwrap().retain(|(n, _)| n != name); + } +} + +async fn node(dir: &Path, bootstrap: Vec) -> (Arc, ChunkStore) { + let store = ChunkStore::open(dir.join("store")).await.unwrap(); + let net = Libp2pNetwork::spawn( + NetworkConfig::for_dir(dir.join("id")) + .with_listen(vec!["/ip4/127.0.0.1/tcp/0".parse().unwrap()]) + .with_local_source(Arc::new(StoreSource(store.clone()))) + .with_bootstrap(bootstrap), + ) + .await + .unwrap(); + (Arc::new(net), store) +} + +async fn wait_for_listen(net: &Libp2pNetwork) -> libp2p::Multiaddr { + for _ in 0..50 { + if let Ok(addrs) = net.listen_addrs().await + && let Some(a) = addrs.into_iter().next() + { + return a; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("no listen addr after 2.5s"); +} + +fn cfg(port_min: u16) -> c0mpute_core::config::GameserverConfig { + c0mpute_core::config::GameserverConfig { port_min, port_max: port_min + 9, max_servers: 2, max_total_ram_gb: 16, max_hours: 24, ..Default::default() } +} + +/// A gameserver worker over `rt`, serving on loopback. +async fn worker(dir: &Path, bootstrap: Vec, port_min: u16, rt: Arc) -> Arc { + let (net, store) = node(dir, bootstrap).await; + let w = gameserver::Worker::with_runtime(net, store, cfg(port_min), "127.0.0.1".into(), &dir.join("state"), rt); + let tags = vec![gameserver::ROLE_TAG.to_string(), Game::Minecraft.tag(), Game::Valheim.tag()]; + gameserver::subscribe(w.clone(), tags, Some(PricingConfig::default())); + w +} + +/// Everything a bystander node sees on the job and status topics. +async fn bystander(dir: &Path, bootstrap: Vec) -> (Arc, Arc>>) { + let (net, _) = node(dir, bootstrap).await; + net.subscribe(&job_topic(gameserver::WORKLOAD_START)).await.unwrap(); + net.subscribe(gameserver::STATUS_TOPIC).await.unwrap(); + let seen = Arc::new(Mutex::new(Vec::new())); + let mut rx = net.messages(); + let s = seen.clone(); + tokio::spawn(async move { + while let Ok(m) = rx.recv().await { + let mut g = s.lock().unwrap(); + g.extend_from_slice(&m.data); + g.push(b'\n'); + } + }); + (net, seen) +} + +fn req(game: Game, password: Option<&str>, workers: Vec) -> StartRequest { + StartRequest { + game, + hours: 2, + ram_gb: game.min_ram_gb(), + cpus: 1.0, + accept_eula: true, + world: None, + password: password.map(str::to_string), + server_name: None, + backup_mins: 30, + max_price_usd: 100.0, + workers, + exclude: Vec::new(), + timeout: Duration::from_secs(120), + } +} + +fn contains(hay: &[u8], needle: &[u8]) -> bool { + hay.windows(needle.len()).any(|w| w == needle) +} + +/// Valheim: the password reaches only the winning worker (sealed), the +/// world comes back sealed to the buyer's key, and the stop downloads and +/// decrypts it. A bystander sees neither. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn secrets_are_sealed_and_backups_readable_only_by_the_buyer() { + let _ = tracing_subscriber::fmt().with_env_filter("c0mpute_core=info,c0mpute_net=warn").with_test_writer().try_init(); + let tmp = tempdir().unwrap(); + let rt = Fake::new(world(7)); + let a = worker(&tmp.path().join("a"), vec![], 41100, rt.clone()).await; + let a_addr = wait_for_listen(worker_net(&a)).await.with(Protocol::P2p(worker_net(&a).peer_id())); + let a_id = worker_net(&a).peer_id().to_base58(); + let (eve, seen) = bystander(&tmp.path().join("eve"), vec![a_addr.clone()]).await; + let (buyer, buyer_store) = node(&tmp.path().join("buyer"), vec![a_addr.clone()]).await; + let dir = tmp.path().join("buyer-data"); + tokio::time::sleep(Duration::from_secs(2)).await; + + let rental = gameserver::start_with(buyer.clone(), &buyer_store, &dir, req(Game::Valheim, Some("hunter22pw"), vec![a_id.clone()])).await.expect("valheim start"); + assert_eq!(rental.worker, a_id); + assert!(rental.address.starts_with("127.0.0.1:")); + let plan = rt.plan(&format!("c0mpute-{}", rental.id)).unwrap(); + assert!(plan.env.iter().any(|(k, v)| k == "SERVER_PASS" && v == "hunter22pw"), "the worker got the password through the handoff"); + assert!(plan.run_secs > 7000 && plan.run_secs <= 7200, "the container stops itself at expiry: {}", plan.run_secs); + + // A live backup, announced in a status record the bystander can see. + let live = a.backup_now(&rental.id).await.unwrap(); + a.publish_status().await; + + let stopped = gameserver::stop_with(buyer.clone(), &buyer_store, &dir, &rental.id, Duration::from_secs(60)).await.expect("stop"); + let manifest = stopped.backup.clone().expect("stop returns a backup"); + assert_ne!(manifest, live.manifest, "the final backup is a new one"); + let tar = std::fs::read(dir.join("worlds").join(format!("{manifest}.tar"))).unwrap(); + assert!(tar == world(7), "the buyer decrypted exactly the world"); + assert!(rt.vols.lock().unwrap().is_empty(), "the volume is removed after a good backup"); + + // Eve has every gossip message and can fetch every chunk, and learns nothing. + tokio::time::sleep(Duration::from_millis(500)).await; + let wire = seen.lock().unwrap().clone(); + assert!(contains(&wire, b"handoff_ready") && contains(&wire, b"handoff_secret"), "eve saw the sealed exchange"); + assert!(contains(&wire, live.manifest.as_bytes()), "eve saw the backup hash in a status record"); + assert!(!contains(&wire, b"hunter22pw"), "the password leaked onto gossip"); + let mb = eve.fetch_from_peer(worker_net(&a).peer_id(), &c0mpute_proto::ChunkRequest { chunk_hash: Hash::from_hex(&manifest).unwrap(), shard_index: None }).await.unwrap(); + let m = gameserver::WorldManifest::from_bytes(&mb).unwrap(); + assert!(m.enc.epk.is_some()); + let first = eve.fetch_from_peer(worker_net(&a).peer_id(), &c0mpute_proto::ChunkRequest { chunk_hash: Hash::from_hex(&m.chunks[0]).unwrap(), shard_index: None }).await.unwrap(); + assert!(!contains(&first, &world(7)[..64]), "chunks are ciphertext"); + assert!(gameserver::open_chunk(&gameserver::random_key(), 0, &first).is_err()); + + // The rentals file (token, password) is private. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!(std::fs::metadata(dir.join("rentals.json")).unwrap().permissions().mode() & 0o777, 0o600); + } +} + +/// Minecraft on worker A: the buyer's watch probes it, mirrors A's backup, +/// and when the server stops answering it starts the same world on worker B +/// from that backup (handed to B under a fresh key). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn watch_fails_over_to_another_worker_from_the_latest_backup() { + let tmp = tempdir().unwrap(); + let rt_a = Fake::new(world(11)); + let rt_b = Fake::new(world(99)); + let a = worker(&tmp.path().join("a"), vec![], 41200, rt_a.clone()).await; + let a_addr = wait_for_listen(worker_net(&a)).await.with(Protocol::P2p(worker_net(&a).peer_id())); + let a_id = worker_net(&a).peer_id().to_base58(); + let b = worker(&tmp.path().join("b"), vec![a_addr.clone()], 41210, rt_b.clone()).await; + let b_id = worker_net(&b).peer_id().to_base58(); + let (buyer, buyer_store) = node(&tmp.path().join("buyer"), vec![a_addr.clone()]).await; + let dir = tmp.path().join("buyer-data"); + tokio::time::sleep(Duration::from_secs(3)).await; + + let rental = gameserver::start_with(buyer.clone(), &buyer_store, &dir, req(Game::Minecraft, None, vec![a_id.clone()])).await.expect("minecraft start"); + assert_eq!(rental.worker, a_id); + let p = gameserver::probe_server(Game::Minecraft, &rental.address, Duration::from_secs(3)).await; + assert!(p.ok && p.detail.contains("fake 1.21"), "{p:?}"); + + let opts = WatchOptions { + probe_every: Duration::from_secs(1), + probe_timeout: Duration::from_secs(1), + fail_after: Duration::from_secs(4), + max_price_usd: 100.0, + start_timeout: Duration::from_secs(120), + stop_after_failover: true, + ..Default::default() + }; + let watch = { + let (buyer, store, dir, id) = (buyer.clone(), buyer_store.clone(), dir.clone(), rental.id.clone()); + tokio::spawn(async move { gameserver::watch_with(buyer, &store, &dir, &id, opts).await }) + }; + + // A's 30-minute backup, then its status records until the watch mirrors it. + let backup = a.backup_now(&rental.id).await.unwrap(); + let mut mirrored = false; + for _ in 0..40 { + a.publish_status().await; + tokio::time::sleep(Duration::from_millis(500)).await; + if gameserver::load_rentals(&dir).iter().any(|r| r.id == rental.id && r.latest_backup.as_deref() == Some(backup.manifest.as_str())) { + mirrored = true; + break; + } + } + assert!(mirrored, "the watch mirrored A's backup"); + assert!(std::fs::read(dir.join("worlds").join(format!("{}.tar", backup.manifest))).unwrap() == world(11)); + assert!(gameserver::health_log(&dir, &rental.id).iter().any(|p| p.ok), "healthy probes were logged"); + + // The server stops answering (and A says nothing more). + rt_a.crash(&format!("c0mpute-{}", rental.id)); + let end = tokio::time::timeout(Duration::from_secs(150), watch).await.expect("watch returns").unwrap().expect("watch ok"); + let WatchEnd::FailedOver { from, to } = end else { panic!("expected a failover, got {end:?}") }; + assert_eq!(from.id, rental.id); + assert_eq!(to.worker, b_id, "the replacement runs on the other worker"); + assert_eq!(to.replaces.as_deref(), Some(rental.id.as_str())); + let log = gameserver::health_log(&dir, &rental.id); + assert!(log.iter().any(|p| !p.ok), "failed probes were logged"); + + // B restored A's world (not its own seed) and is serving it. + let vol = format!("c0mpute-{}", to.id); + assert_eq!(rt_b.restored.lock().unwrap().as_slice(), &[vol.clone()]); + assert!(rt_b.vols.lock().unwrap().get(&vol).unwrap() == &world(11)); + assert!(gameserver::probe_server(Game::Minecraft, &to.address, Duration::from_secs(3)).await.ok); + let all = gameserver::load_rentals(&dir); + assert_eq!(all.iter().find(|r| r.id == rental.id).unwrap().superseded_by.as_deref(), Some(to.id.as_str())); + drop(b); +} + +/// The worker was offline when a reservation ended: at its next sweep it +/// kills the expired container, backs the world up from the volume (sealed +/// to the buyer) and removes the volume. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_expiry_missed_while_offline_is_enforced_at_the_next_sweep() { + let tmp = tempdir().unwrap(); + let state_dir = tmp.path().join("w").join("state"); + std::fs::create_dir_all(&state_dir).unwrap(); + let now = gameserver::now_secs(); + let id = gameserver::server_id("job-expired"); + let buyer_key = c0mpute_core::temporal::crypto::keypair(); + let st = ServerState { + id: id.clone(), + game: Game::Factorio, + container: format!("c0mpute-{id}"), + volume: format!("c0mpute-{id}"), + ram_mb: 2048, + host_port: 41300, + address: "127.0.0.1:41300".into(), + started_at: now - 7200, + expires_at: now - 60, + stop_hash: gameserver::sha256_ref(b"t"), + buyer: "buyer".into(), + backup_pubkey: buyer_key.public_hex.clone(), + backup_every_secs: 1800, + stopped_at: None, + backup: None, + backup_error: None, + }; + let servers: HashMap = [(id.clone(), st.clone())].into_iter().collect(); + std::fs::write(state_dir.join("servers.json"), serde_json::to_vec(&servers).unwrap()).unwrap(); + + let rt = Fake::new(world(5)); + rt.vols.lock().unwrap().insert(st.volume.clone(), world(5)); + rt.labelled.lock().unwrap().push((st.container.clone(), now - 60)); + rt.labelled.lock().unwrap().push(("c0mpute-gs-ffffffffffffffff".into(), now + 3600)); + let (net, store) = node(&tmp.path().join("w"), vec![]).await; + let w = gameserver::Worker::with_runtime(net, store.clone(), cfg(41300), "127.0.0.1".into(), &state_dir, rt.clone()); + + assert_eq!(w.sweep_expired().await, vec![st.container.clone()], "only the expired container dies"); + w.reap_once().await; + let after = w.snapshot().get(&id).cloned().unwrap(); + assert!(after.stopped_at.is_some()); + let b = after.backup.expect("backed up from the volume"); + assert!(rt.vols.lock().unwrap().get(&st.volume).is_none(), "volume removed"); + // The backup opens with the buyer's key and nobody else's. + let m = gameserver::WorldManifest::from_bytes(&store.get(&Hash::from_hex(&b.manifest).unwrap()).await.unwrap()).unwrap(); + let epk = m.enc.epk.clone().unwrap(); + let k = gameserver::backup_key(&buyer_key.secret, &epk, &epk, &buyer_key.public_hex).unwrap(); + let mut plain = Vec::new(); + for (i, c) in m.chunks.iter().enumerate() { + plain.extend(gameserver::open_chunk(&k, i as u64, &store.get(&Hash::from_hex(c).unwrap()).await.unwrap()).unwrap()); + } + assert!(plain == world(5)); + // Persisted: a restart does not stop it twice. + let saved: HashMap = serde_json::from_slice(&std::fs::read(state_dir.join("servers.json")).unwrap()).unwrap(); + assert!(saved[&id].stopped_at.is_some()); +} + +fn worker_net(w: &gameserver::Worker) -> &Libp2pNetwork { + w.net() +} diff --git a/node/crates/c0mpute-core/tests/video_e2e.rs b/node/crates/c0mpute-core/tests/video_e2e.rs new file mode 100644 index 00000000..90ad964f --- /dev/null +++ b/node/crates/c0mpute-core/tests/video_e2e.rs @@ -0,0 +1,279 @@ +//! The video buyer path end to end on loopback, with a fake runner. +//! +//! `C0MPUTE_DOCKER` points the node's container CLI at a shell script that +//! behaves like `docker run `: it finds the job directory the node +//! bind-mounted at /job, checks `job.json` is there, and writes a real (tiny) +//! H.264 MP4 as `out.mp4`. Everything else is the shipping code: the worker's +//! video runner and bid filter, the auction, the store and announce, and the +//! buyer gateway's `POST /v1/videos`, which answers `video/mp4`. +//! +//! The same fake stands in for `docker pull/tag/build` to prove +//! `c0mpute video setup` builds from source when the registry will not serve +//! the pinned image. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use c0mpute_core::pricing::PricingConfig; +use c0mpute_core::{dispatch, penalty_box::PenaltyBox, speech_gateway, video}; +use c0mpute_net::{ChunkSource, Libp2pNetwork, NetworkConfig}; +use c0mpute_proto::Hash; +use c0mpute_store::ChunkStore; +use libp2p::multiaddr::Protocol; +use tempfile::tempdir; + +const TOKEN: &str = "video-e2e-token-0123456789abcdef"; +const FAKE_DOCKER: &str = r#"#!/bin/sh +# Stand-in for the docker CLI (see video_e2e.rs). +echo "$*" >> "$FAKE_DIR/log" +case "$1" in + run) + jd="" + for a in "$@"; do + case "$a" in *:/job) jd="${a%:/job}" ;; esac + done + [ -n "$jd" ] && [ -f "$jd/job.json" ] || exit 3 + if grep -q BIGCLIP "$jd/job.json"; then cp "$FAKE_DIR/big.mp4" "$jd/out.mp4"; else cp "$FAKE_DIR/tiny.mp4" "$jd/out.mp4"; fi + exit 0 ;; + pull) [ -f "$FAKE_DIR/pull-ok" ] && exit 0; echo "denied" >&2; exit 1 ;; + build) [ -f "$5" ] && [ -d "$6" ] || exit 4; exit 0 ;; + *) exit 0 ;; +esac +"#; + +struct StoreSource(ChunkStore); + +#[async_trait::async_trait] +impl ChunkSource for StoreSource { + async fn read_chunk(&self, hash: &Hash) -> anyhow::Result> { + self.0.get(hash).await + } +} + +fn tiny_mp4() -> Vec { + std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/tiny.mp4")).unwrap() +} + +/// The tiny clip plus a top-level `free` box: still a valid MP4, `len` bytes. +fn big_mp4(len: usize) -> Vec { + let mut v = tiny_mp4(); + let pad = len - v.len(); + v.extend_from_slice(&(pad as u32).to_be_bytes()); + v.extend_from_slice(b"free"); + let mut x: u64 = 0x9E37_79B9_7F4A_7C15; + while v.len() < len { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + v.extend_from_slice(&x.to_le_bytes()); + } + v.truncate(len); + v +} + +/// Install the fake docker once per test binary. +fn fake_docker() -> &'static PathBuf { + static DIR: OnceLock = OnceLock::new(); + DIR.get_or_init(|| { + let dir = tempdir().unwrap().keep(); + std::fs::write(dir.join("tiny.mp4"), tiny_mp4()).unwrap(); + std::fs::write(dir.join("big.mp4"), big_mp4(6 * 1024 * 1024 + 123)).unwrap(); + let bin = dir.join("docker"); + std::fs::write(&bin, FAKE_DOCKER).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + // SAFETY: set once, before any thread of this binary reads them. + unsafe { + std::env::set_var(c0mpute_core::container::ENV, &bin); + std::env::set_var("FAKE_DIR", &dir); + } + dir + }) +} + +async fn node(dir: &Path, bootstrap: Vec) -> (Arc, ChunkStore) { + let store = ChunkStore::open(dir.join("store")).await.unwrap(); + let net = Libp2pNetwork::spawn( + NetworkConfig::for_dir(dir.join("id")) + .with_listen(vec!["/ip4/127.0.0.1/tcp/0".parse().unwrap()]) + .with_local_source(Arc::new(StoreSource(store.clone()))) + .with_bootstrap(bootstrap), + ) + .await + .unwrap(); + (Arc::new(net), store) +} + +async fn wait_for_listen(net: &Libp2pNetwork) -> libp2p::Multiaddr { + for _ in 0..50 { + if let Ok(addrs) = net.listen_addrs().await + && let Some(a) = addrs.into_iter().next() + { + return a; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("no listen addr after 2.5s"); +} + +/// A video worker (the real runner over the fake docker), a buyer node, and +/// the buyer gateway on a loopback port. Returns the gateway URL. +async fn network(tmp: &Path) -> (String, Arc) { + fake_docker(); + let (worker, worker_store) = node(&tmp.join("w"), vec![]).await; + let worker_addr = wait_for_listen(&worker).await.with(Protocol::P2p(worker.peer_id())); + let load = c0mpute_core::WorkerLoad::new(Arc::new(tokio::sync::Semaphore::new(1)), 1); + let runner = video::spawn_video_runner( + worker.clone(), + worker_store, + tmp.join("cache"), + video::WORKLOAD_TXT2VID.to_string(), + video::RUNNER_IMAGE.to_string(), + load.clone(), + ); + dispatch::run_worker_subscriber_with_load( + worker.clone(), + video::WORKLOAD_TXT2VID.to_string(), + vec![video::ROLE_TAG.to_string(), "c0mpute:gpu:nvidia".to_string()], + Some(runner), + None, + Some(PricingConfig::default()), + Some(load), + ); + + let (buyer, buyer_store) = node(&tmp.join("b"), vec![worker_addr]).await; + let app = speech_gateway::router(buyer, buyer_store, TOKEN.into(), 100.0, PenaltyBox::new()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + tokio::time::sleep(Duration::from_secs(2)).await; + (url, worker) +} + +async fn post_video(url: &str, prompt: &str) -> reqwest::Response { + reqwest::Client::new() + .post(format!("{url}/v1/videos")) + .bearer_auth(TOKEN) + .json(&serde_json::json!({ + "model": "wan2.2-ti2v-5b", + "prompt": prompt, + "frames": 9, + "steps": 2, + "width": 64, + "height": 64, + })) + .timeout(Duration::from_secs(180)) + .send() + .await + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn post_v1_videos_returns_the_runner_clip() { + let _ = tracing_subscriber::fmt().with_env_filter("c0mpute_core=info,c0mpute_net=warn").with_test_writer().try_init(); + let tmp = tempdir().unwrap(); + let (url, worker) = network(tmp.path()).await; + + // No token, no job. + let anon = reqwest::Client::new().post(format!("{url}/v1/videos")).json(&serde_json::json!({"model": "m", "prompt": "p"})).send().await.unwrap(); + assert_eq!(anon.status(), 401); + + let res = post_video(&url, "a neon alley in the rain").await; + assert_eq!(res.status(), 200, "{:?}", res.text().await); + assert_eq!(res.headers()["content-type"], "video/mp4"); + assert_eq!(res.headers()["x-c0mpute-worker-peer-id"], worker.peer_id().to_base58().as_str()); + let body = res.bytes().await.unwrap(); + assert!(body.as_ref() == tiny_mp4().as_slice(), "the gateway returns exactly the runner's out.mp4"); + + // The node ran the runner with the GPU, the job dir and the weights cache. + let log = std::fs::read_to_string(fake_docker().join("log")).unwrap(); + let run = log.lines().find(|l| l.starts_with("run ")).expect("docker run was called"); + assert!(run.contains("--gpus all") && run.contains(":/job") && run.contains(":/weights"), "{run}"); + assert!(run.contains(video::RUNNER_IMAGE) && run.contains("txt2vid"), "{run}"); + + // An invalid job is refused before any auction. + let bad = reqwest::Client::new() + .post(format!("{url}/v1/videos")) + .bearer_auth(TOKEN) + .json(&serde_json::json!({"model": "wan2.2-ti2v-5b", "prompt": "x", "frames": 9999})) + .send() + .await + .unwrap(); + assert_eq!(bad.status(), 400); +} + +/// A clip over one network message (6 MiB): the worker publishes it as +/// pieces and the gateway fetches and reassembles them (fix/large-results). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs PR #160 (fix/large-results): results over ~5 MiB travel as pieces"] +async fn a_clip_over_5_mib_reaches_the_buyer() { + let tmp = tempdir().unwrap(); + let (url, _) = network(tmp.path()).await; + let res = post_video(&url, "BIGCLIP a long take").await; + assert_eq!(res.status(), 200, "{:?}", res.text().await); + let body = res.bytes().await.unwrap(); + let want = std::fs::read(fake_docker().join("big.mp4")).unwrap(); + assert_eq!(body.len(), want.len()); + assert!(body.as_ref() == want.as_slice()); +} + +/// `c0mpute video setup`: the pinned pull is tried first and tagged as the +/// local name; when the registry refuses it, the runner is built from a +/// clone of the source repo instead. +#[tokio::test] +async fn setup_pulls_the_pinned_digest_or_builds_from_source() { + let dir = fake_docker().clone(); + let git_ok = std::process::Command::new("git").arg("--version").output().is_ok_and(|o| o.status.success()); + if !git_ok { + eprintln!("skipping: git is not installed"); + return; + } + // A local "source repo" laid out like this one. + let src = tempdir().unwrap(); + std::fs::create_dir_all(src.path().join("plugins/video/runner")).unwrap(); + std::fs::write(src.path().join("plugins/video/runner/Dockerfile"), "FROM scratch\n").unwrap(); + std::fs::write(src.path().join("plugins/video/module.toml"), "[module]\n").unwrap(); + let git = |args: &[&str]| { + let ok = std::process::Command::new("git") + .args(["-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "init.defaultBranch=master"]) + .args(args) + .current_dir(src.path()) + .output() + .unwrap(); + assert!(ok.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&ok.stderr)); + }; + git(&["init", "-q"]); + git(&["add", "."]); + git(&["commit", "-qm", "runner"]); + // SAFETY: only this test reads it. + unsafe { std::env::set_var("C0MPUTE_VIDEO_SOURCE_REPO", format!("file://{}", src.path().display())) }; + + let scratch = tempdir().unwrap(); + let said = std::sync::Mutex::new(Vec::::new()); + let say = |l: &str| said.lock().unwrap().push(l.to_string()); + + // The registry refuses the pull: build from source. + let _ = std::fs::remove_file(dir.join("pull-ok")); + let got = video::setup(false, scratch.path(), &say).await.expect("setup falls back to a build"); + assert_eq!(got, video::SetupSource::Built); + let log = std::fs::read_to_string(dir.join("log")).unwrap(); + assert!(log.contains(&format!("pull {}", video::RUNNER_PULL_REF)), "{log}"); + let build = log.lines().find(|l| l.starts_with("build ")).expect("docker build ran"); + assert!(build.contains(&format!("-t {}", video::RUNNER_IMAGE)) && build.contains("plugins/video/runner/Dockerfile"), "{build}"); + assert!(said.lock().unwrap().iter().any(|l| l.contains("building it from source"))); + assert_eq!(std::fs::read_dir(scratch.path()).unwrap().count(), 0, "the scratch checkout is removed"); + + // The pull works: tag the pinned digest as the local name, no build. + std::fs::write(dir.join("pull-ok"), "").unwrap(); + let got = video::setup(false, scratch.path(), &say).await.unwrap(); + assert_eq!(got, video::SetupSource::Pulled); + let log = std::fs::read_to_string(dir.join("log")).unwrap(); + assert!(log.contains(&format!("tag {} {}", video::RUNNER_PULL_REF, video::RUNNER_IMAGE)), "{log}"); + let _ = std::fs::remove_file(dir.join("pull-ok")); +} diff --git a/plugins/gameserver/module.toml b/plugins/gameserver/module.toml index 1df53102..31d6fcb1 100644 --- a/plugins/gameserver/module.toml +++ b/plugins/gameserver/module.toml @@ -1,26 +1,47 @@ # c0mpute plugin manifest. See dips/0006-module-model.md. # # Built in: workloads `gameserver.start` and `gameserver.stop` -# (node/crates/c0mpute-core/src/gameserver.rs, CLI c0mpute-cli/src/gameserver.rs). +# (node/crates/c0mpute-core/src/gameserver.rs and handoff.rs, CLI +# c0mpute-cli/src/gameserver.rs). # # Start is an ordinary auction with a short job: the winning worker runs # `docker create` (argv only, no shell) on a pinned image with --memory, -# --memory-swap, --cpus, --pids-limit 1024, no-new-privileges and --restart no, -# a named volume for the world and a published port block from its range, -# restores a world if asked, starts it and returns address:port plus the -# expiry. The server then runs for the reserved hours; the worker's reaper -# stops it at expiry. Images (multi-arch index digests): +# --memory-swap, --cpus, --pids-limit 1024, no-new-privileges, --restart no, +# --rm and --init, a named volume for the world and a published port block +# from its range, restores a world if asked, starts it and returns +# address:port plus the expiry. Images (multi-arch index digests): # minecraft itzg/minecraft-server:java21 (EULA=TRUE only with the buyer's --accept-eula) # factorio factoriotools/factorio:stable-2.1.24 # valheim lloesche/valheim-server (x86_64 only; buyer sets the password) # +# Secrets: nothing secret is in the offer (spec v2). The offer carries the +# buyer's fresh X25519 key; after the accept the winner announces its own and +# the buyer seals the Valheim password and any restore key to it +# (X25519 -> HKDF-SHA256 -> AES-256-GCM, gossip-signed both ways). +# # Stop: the start spec carries sha256(stop token); the token stays on the # buyer's machine until `gameserver stop`, and only the worker holding that -# server bids. On stop or expiry the worker tars the world out of the volume -# (`docker cp`), stores it as 1 MiB content-addressed chunks plus a manifest, -# announces them, and returns the manifest hash; the buyer downloads the tar -# (verified chunk by chunk and by sha256). `start --world ` serves it -# from the buyer's node and any worker restores it. No coordinator anywhere. +# server bids. +# +# Backups: every `backup_mins` (default 30) and at stop or expiry the worker +# tars the world out of the volume, encrypts it to the buyer's long-term +# X25519 backup key in 1 MiB AES-GCM chunks, and stores and announces the +# chunks plus a manifest. The manifest hash is in the stop receipt and in the +# worker's status record; only the buyer can read what it names. +# +# Health and failover: workers gossip a status record per server (container +# running, latest backup) on c0mpute/gameserver/v1, signed by their peer key. +# `c0mpute gameserver watch` probes the game port (Minecraft status ping, +# Valheim Steam A2S query, Factorio UDP reachability plus a fresh "running" +# record), logs every probe (`gameserver health`), mirrors each backup to the +# buyer's machine, and after `--fail-after-secs` (default 300) of failed checks +# starts the same world on another worker from the newest backup. No +# coordinator anywhere. +# +# Expiry: the server runs under `timeout ` behind --init and +# --rm, so it stops itself at expiry even if the worker is offline then. At +# boot and on every sweep the worker kills any labelled container past its +# expiry, then backs the world up from the volume and removes it. # # Worker: opt-in role (`c0mpute worker start --roles ...,gameserver`), never # default. Advertised only when `docker info` works, the node knows a public @@ -28,11 +49,8 @@ # C0MPUTE_P2P_EXTERNAL) and a pinned image has been pulled. The operator opens # [gameserver] port_min..port_max (default 27500-27599, TCP and UDP). # -# Not built: replicas / live failover, proof-of-serve (RCON or player -# heartbeat) and per-epoch settlement: a reservation is paid once, up front, -# at the price the auction cleared. Backups are readable by anyone who has the -# manifest hash (it is in the stop receipt), and a Valheim password rides in -# the public offer. +# Billing: one payment up front at the price the auction cleared. A failover +# is a new rental, paid the same way. [module] id = "gameserver" @@ -40,19 +58,19 @@ name = "Game server — Minecraft / Factorio / Valheim" version = "0.2.79" kind = "workload" category = "infrastructure" -status = "partial" -status_note = "Start, stop, expiry and world backup/restore work. Only opt-in workers with docker and a public IP whose firewall opens the port range can host, and few do yet. No replicas or proof-of-serve: one worker, paid once up front; if it goes offline the server is gone until it returns, and the world survives only in its last backup." +status = "available" builtin = true -description = "Rent a Minecraft, Factorio or Valheim server on a c0mpute worker for 1-72 hours: pinned docker images with RAM and CPU limits, address:port back in seconds, auto-stop at expiry, and the world backed up p2p by hash so any worker can restore it." +description = "Rent a Minecraft, Factorio or Valheim server on a c0mpute worker for 1-72 hours: pinned docker images with RAM and CPU limits, address:port back in seconds, a hard stop at expiry, world backups every 30 minutes encrypted to your key, and a watcher that probes the server and fails over to another worker from the latest backup." author = "Profullstack" license = "MIT" keywords = ["gameserver", "minecraft", "factorio", "valheim", "hosting", "docker"] [module.requirements] -c0mpute = ">=0.2.77" +c0mpute = ">=NEXT" os = ["linux"] arch = ["x86_64", "aarch64"] capabilities = ["container-runtime", "network:public-ip", "network:ingress"] +hardware = "opt-in worker role: docker + a public IP with the port range open (TCP and UDP)" [module.workloads] "gameserver.start" = { command = "start", validation = "result_schema" } @@ -72,3 +90,4 @@ max_total_ram_gb = 0 max_hours = 24 min_usd_per_gb_hour = 0.0 games = ["minecraft", "factorio", "valheim"] +backup_mins = 30 diff --git a/plugins/video/install.sh b/plugins/video/install.sh index 0ceb8e3d..744a8fe9 100755 --- a/plugins/video/install.sh +++ b/plugins/video/install.sh @@ -11,7 +11,11 @@ # Source: https://github.com/profullstack/c0mpute/tree/master/plugins/video set -eu -IMAGE="${C0MPUTE_VIDEO_IMAGE:-ghcr.io/profullstack/c0mpute-video-runner:candidate}" +# The published runner, pinned by digest (keep in step with module.toml +# [module.dispatch] and RUNNER_DIGEST in video.rs), and the local name the +# node runs, whichever way the image arrives. +PULL_REF="${C0MPUTE_VIDEO_PULL_REF:-ghcr.io/profullstack/c0mpute-video-runner@sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c}" +IMAGE="c0mpute/video-runner:stable" SOURCE_REPO="${C0MPUTE_VIDEO_SOURCE_REPO:-https://github.com/profullstack/c0mpute.git}" say() { printf '\033[1;36m→\033[0m %s\n' "$*"; } @@ -43,12 +47,12 @@ else fi if [ "$missing" -eq 0 ]; then - say "pulling $IMAGE (several GB, once)" - if ! docker pull "$IMAGE"; then + say "pulling $PULL_REF (several GB, once)" + if ! { docker pull "$PULL_REF" && docker tag "$PULL_REF" "$IMAGE"; }; then # The registry is a convenience, not a dependency: private, rate-limited # or offline is no reason a box with a card cannot serve video. This repo # is public, so build the same image from source. - warn "could not pull $IMAGE (private, rate-limited or offline)" + warn "could not pull $PULL_REF (private, rate-limited or offline)" if command -v git >/dev/null 2>&1; then say "building it from source instead (several minutes)" src=$(mktemp -d) diff --git a/plugins/video/module.toml b/plugins/video/module.toml index a8695933..45bce2b7 100644 --- a/plugins/video/module.toml +++ b/plugins/video/module.toml @@ -5,9 +5,15 @@ # published as the image below; the node's video runner invokes it per job and # reads files, never stdout. See that directory's README for the job contract. # -# The image below is a convenience: `c0mpute video setup` falls back to -# building it from this public repo when the registry will not serve it, so a -# node with a GPU never needs a registry account. +# The image below is pinned by digest (the same digest as RUNNER_DIGEST in +# node/crates/c0mpute-core/src/video.rs, checked by a unit test). +# `c0mpute video setup` pulls it and tags it locally as +# c0mpute/video-runner:stable, the name the worker runs. If the registry will +# not serve it, setup builds the same Dockerfile from this public repo, so a +# node with a GPU never needs a registry account. The workflow +# .github/workflows/video-runner.yml publishes :stable, :candidate and : +# on every master build and prints the new digest; bump both pins to it after +# a runner change. # # Backend: Diffusers, not ComfyUI. ComfyUI would mean shipping hand-authored # API graph JSON per model, which cannot be checked by reading it; the @@ -47,18 +53,19 @@ name = "Video — generation (Wan 2.2, LTX-2.3 with audio)" version = "0.2.79" kind = "workload" category = "ai" -status = "partial" -status_note = "Needs an NVIDIA GPU and the c0mpute video runner image." +status = "available" +builtin = true description = "Decentralized video generation on consumer/prosumer GPUs: Wan 2.2 text/image-to-video, and LTX-2.3 video with a synchronized soundtrack." author = "Profullstack" license = "MIT" keywords = ["video", "wan", "wan2.2", "ltx-2", "ltx-2.3", "audio", "txt2vid", "img2vid", "generation", "diffusion"] [module.requirements] -c0mpute = ">=0.1.0" +c0mpute = ">=NEXT" os = ["linux"] arch = ["x86_64"] capabilities = ["gpu:nvidia", "model-runtime", "container-runtime"] +hardware = "NVIDIA GPU (24 GiB VRAM for the default model, 48 GiB for the 14B and LTX models) + docker with the NVIDIA container runtime" [module.workloads] "video.txt2vid" = { command = "txt2vid", validation = "schema_and_ffprobe" } @@ -80,7 +87,7 @@ capabilities = ["gpu:nvidia", "model-runtime", "container-runtime"] [module.dispatch] mode = "container" -image = "ghcr.io/profullstack/c0mpute-video-runner:candidate" +image = "ghcr.io/profullstack/c0mpute-video-runner@sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c" [module.surfaces] cli = "c0mpute video" diff --git a/plugins/video/runner/Dockerfile b/plugins/video/runner/Dockerfile index c28a48cd..ce752640 100644 --- a/plugins/video/runner/Dockerfile +++ b/plugins/video/runner/Dockerfile @@ -11,7 +11,7 @@ # docker run --rm --gpus all \ # -v c0mpute-video-weights:/weights \ # -v "$PWD/job:/job" \ -# ghcr.io/profullstack/c0mpute-video-runner:candidate txt2vid +# c0mpute/video-runner:stable txt2vid # # No CUDA base image on purpose: the torch wheels from PyPI bring their own # CUDA runtime, so only the driver comes from the host via the NVIDIA diff --git a/plugins/video/runner/README.md b/plugins/video/runner/README.md index 056779dc..efe3267b 100644 --- a/plugins/video/runner/README.md +++ b/plugins/video/runner/README.md @@ -4,8 +4,12 @@ The container the `video` plugin dispatches to. One job per process: read `job.json`, generate a clip, encode an MP4, write a sidecar, exit non-zero on any failure. -Published as `ghcr.io/profullstack/c0mpute-video-runner`. The tag a node pulls -is the one in [`../module.toml`](../module.toml) under `[module.dispatch]`. +Published as `ghcr.io/profullstack/c0mpute-video-runner` with the tags +`:stable`, `:candidate` and `:` on every master build. A node +pulls by digest: the one in [`../module.toml`](../module.toml) under +`[module.dispatch]` (and `RUNNER_DIGEST` in `video.rs`), then tags it locally +as `c0mpute/video-runner:stable`, the name the worker runs. After a runner +change, copy the digest from the workflow's job summary into both pins. **The registry is a convenience, not a dependency.** `c0mpute video setup` pulls the image, and if the registry will not serve it — private package, From 873b6ea3c1c84046f35b4e1ab388c4670267a531 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 9 Nov 2026 04:45:20 +0000 Subject: [PATCH 2/2] handoff: exchange dialable addresses and connect the buyer and the winner CI showed a failover worker that shared no connection with the buyer could not fetch the restore world (Failed to dial the requested peer). Both sides now put their addresses in the handoff and dial each other, so either side can be the one behind NAT. Also fixes a unit test slicing a short last chunk. Co-Authored-By: Claude Opus 5.5 --- node/crates/c0mpute-core/src/handoff.rs | 35 ++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/node/crates/c0mpute-core/src/handoff.rs b/node/crates/c0mpute-core/src/handoff.rs index 86f50c17..cf17fbb1 100644 --- a/node/crates/c0mpute-core/src/handoff.rs +++ b/node/crates/c0mpute-core/src/handoff.rs @@ -47,6 +47,9 @@ struct Ready { worker_peer_id: String, client_pubkey: String, worker_pubkey: String, + /// The worker's dialable addresses, so the buyer can connect directly. + #[serde(default)] + addrs: Vec, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -55,6 +58,28 @@ struct Sealed { job_id: String, worker_peer_id: String, sealed: String, + /// The buyer's dialable addresses: a worker that must fetch from the + /// buyer (a world to restore) may share no connection with it yet. + #[serde(default)] + addrs: Vec, +} + +const MAX_ADDRS: usize = 8; + +async fn my_addrs(net: &Libp2pNetwork) -> Vec { + net.advertised_addrs().await.into_iter().take(MAX_ADDRS).map(|a| a.to_string()).collect() +} + +/// Learn `peer`'s addresses from a handoff message and open a direct +/// connection (either side may be the one that can dial out). +async fn link(net: &Libp2pNetwork, peer: &str, addrs: &[String]) { + let Ok(peer) = peer.parse::() else { return }; + for a in addrs.iter().take(MAX_ADDRS).filter(|a| a.len() <= 256) { + if let Ok(addr) = a.parse::() { + net.add_peer_address(peer, addr).await; + } + } + let _ = net.connect(peer).await; } /// A fresh key pair for one handoff. @@ -93,6 +118,7 @@ pub async fn receive(net: &Libp2pNetwork, workload: &str, accept: &JobAccept, cl worker_peer_id: me.clone(), client_pubkey: client_pubkey.to_string(), worker_pubkey: kp.public_hex.clone(), + addrs: my_addrs(net).await, })?; let deadline = Instant::now() + wait; let mut next = Instant::now(); @@ -118,7 +144,10 @@ pub async fn receive(net: &Libp2pNetwork, workload: &str, accept: &JobAccept, cl continue; } match crypto::open(&k, &s.sealed) { - Ok(plain) if plain.len() <= MAX_SECRET => return Ok(plain), + Ok(plain) if plain.len() <= MAX_SECRET => { + link(net, &accept.buyer_peer_id, &s.addrs).await; + return Ok(plain); + } Ok(_) => bail!("sealed secret too large"), Err(e) => debug!(job_id = %accept.job_id, err = %e, "handoff: unreadable secret ignored"), } @@ -150,6 +179,7 @@ impl BuyerHandoff { // job_id → sealed message, republished until the task ends. let mut sent: HashMap> = HashMap::new(); let mut tick = tokio::time::interval(REPUBLISH); + let addrs = my_addrs(&net).await; let seal = |r: &Ready| -> Result> { let k = key(&kp.secret, &r.worker_pubkey, &workload, &r.job_id)?; Ok(serde_json::to_vec(&Sealed { @@ -157,6 +187,7 @@ impl BuyerHandoff { job_id: r.job_id.clone(), worker_peer_id: r.worker_peer_id.clone(), sealed: crypto::seal(&k, &secret)?, + addrs: addrs.clone(), })?) }; loop { @@ -168,6 +199,7 @@ impl BuyerHandoff { { let _ = net.publish(&topic, m.clone()).await; sent.insert(job_id.clone(), m); + link(&net, &r.worker_peer_id, &r.addrs).await; } winners.insert(job_id, bid.bidder_peer_id); } @@ -191,6 +223,7 @@ impl BuyerHandoff { let _ = net.publish(&topic, m.clone()).await; sent.insert(r.job_id.clone(), m); } + link(&net, &r.worker_peer_id, &r.addrs).await; } Some(_) => debug!(job_id = %r.job_id, "handoff: key from a worker we did not accept ignored"), None if readies.len() < 64 => { readies.insert(r.job_id.clone(), r); }