diff --git a/.github/workflows/video-runner.yml b/.github/workflows/video-runner.yml index 40e1aff4..76a8b244 100644 --- a/.github/workflows/video-runner.yml +++ b/.github/workflows/video-runner.yml @@ -6,8 +6,19 @@ # torch image is several GB and not worth building to learn that a validation # rule changed. # -# The image the node pulls is the tag in plugins/video/module.toml under -# [module.dispatch]; keep the two in step. +# Tags on every master build: :stable (what operators pull by hand), +# :candidate (kept for older nodes) and :. Nodes pull by DIGEST, the one +# pinned in plugins/video/module.toml [module.dispatch] and RUNNER_DIGEST in +# node/crates/c0mpute-core/src/video.rs (a unit test keeps those two equal). +# The job summary prints the new digest; bump both pins to it after a runner +# change. +# +# Visibility: the package must be PUBLIC for outside operators to pull it. +# GHCR creates org packages private and has no API to change that, so it is +# one manual click by an org admin: github.com/orgs/profullstack/packages -> +# c0mpute-video-runner -> Package settings -> Change visibility -> Public. +# The last step below checks an anonymous pull and warns until it works. +# Until then `c0mpute video setup` builds the same image from source. name: video runner on: @@ -81,15 +92,21 @@ jobs: EXTRA: ${{ inputs.tag }} run: | set -eu - tags="${IMAGE}:candidate,${IMAGE}:${GITHUB_SHA}" + tags="${IMAGE}:stable,${IMAGE}:candidate,${IMAGE}:${GITHUB_SHA}" if [ -n "${EXTRA:-}" ]; then tags="${tags},${IMAGE}:${EXTRA}" fi echo "tags=${tags}" >> "$GITHUB_OUTPUT" - name: Build and push + id: push uses: docker/build-push-action@v6 with: + # Links the package to this public repo (and its README/license). + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.description=c0mpute video runner (Wan 2.2, LTX-2.3) + org.opencontainers.image.licenses=MIT # The context is plugins/video so module.toml — the runner's model # registry — is part of the image. context: plugins/video @@ -103,3 +120,25 @@ jobs: # Proves the thing we just pushed can parse the manifest, encode and # probe. Needs no GPU, so a hosted runner can do it. run: docker run --rm "${IMAGE}:${GITHUB_SHA}" selftest + + - name: Digest to pin, and can a stranger pull it? + env: + DIGEST: ${{ steps.push.outputs.digest }} + run: | + set -u + { + echo "### video runner published" + echo "" + echo "Pin this in plugins/video/module.toml and RUNNER_DIGEST in video.rs:" + echo "" + echo " ${IMAGE}@${DIGEST}" + } >> "$GITHUB_STEP_SUMMARY" + # Anonymous pull: a fresh docker config with no credentials. + export DOCKER_CONFIG="$(mktemp -d)" + if docker manifest inspect "${IMAGE}@${DIGEST}" >/dev/null 2>&1; then + echo "Anonymous pull: OK (package is public)." >> "$GITHUB_STEP_SUMMARY" + else + msg="ghcr.io package c0mpute-video-runner is not publicly pullable. An org admin must set it Public once: github.com/orgs/profullstack/packages -> c0mpute-video-runner -> Package settings -> Change visibility. Until then nodes build the runner from source." + echo "::warning::${msg}" + echo "Anonymous pull: FAILED. ${msg}" >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/apps/web/src/app/plugins/page.tsx b/apps/web/src/app/plugins/page.tsx index 07369c32..1dc50997 100644 --- a/apps/web/src/app/plugins/page.tsx +++ b/apps/web/src/app/plugins/page.tsx @@ -186,6 +186,12 @@ function PluginCard({ p }: { p: PluginManifest }) { + {typeof p.requirements?.hardware === "string" && ( +

+ requires: {p.requirements.hardware} +

+ )} + {p.status === "partial" && p.statusNote && (

partial: {p.statusNote} diff --git a/apps/web/src/lib/plugins.ts b/apps/web/src/lib/plugins.ts index be09bf39..1fc20167 100644 --- a/apps/web/src/lib/plugins.ts +++ b/apps/web/src/lib/plugins.ts @@ -113,6 +113,8 @@ export interface PluginManifest { os?: string[]; arch?: string[]; capabilities?: string[]; + /** What a node needs to serve it, in words (GPU, docker, public IP). */ + hardware?: string; }; workloads?: Record; models?: Record; diff --git a/node/crates/c0mpute-cli/src/gameserver.rs b/node/crates/c0mpute-cli/src/gameserver.rs index 51d7deb0..0bdcc76f 100644 --- a/node/crates/c0mpute-cli/src/gameserver.rs +++ b/node/crates/c0mpute-cli/src/gameserver.rs @@ -34,12 +34,16 @@ enum Sub { /// Restore a world: the hash `gameserver stop` printed, or a .tar of the game's data dir. #[arg(long)] world: Option, - /// Valheim server password (5-32 letters/digits; peers on the job topic can see it). + /// Valheim server password (5-32 letters/digits). Sent sealed to the + /// winning worker after the auction, never in the public offer. #[arg(long)] password: Option, /// Server name / MOTD. #[arg(long)] name: Option, + /// Minutes between live world backups, sealed to your key (0 = only at stop/expiry). + #[arg(long, default_value_t = gameserver::DEFAULT_BACKUP_MINS)] + backup_mins: u32, /// Most you pay for the whole reservation, USD. #[arg(long, default_value_t = 1.0)] max_price: f64, @@ -56,6 +60,32 @@ enum Sub { #[arg(long)] json: bool, }, + /// Watch a server: probe its port, mirror its backups, and if it stays + /// down for --fail-after-secs, start the same world on another worker. + Watch { + id: String, + #[arg(long, default_value_t = 30)] + probe_every_secs: u64, + #[arg(long, default_value_t = 300)] + fail_after_secs: u64, + /// Only probe and mirror backups; never start a replacement. + #[arg(long)] + no_failover: bool, + /// Most you pay for a replacement, USD. + #[arg(long, default_value_t = 1.0)] + max_price: f64, + /// Only accept replacement bids from these worker peer ids. Repeatable. + #[arg(long)] + worker: Vec, + }, + /// Show a server's probe log (what `watch` recorded). + Health { + id: String, + #[arg(long, default_value_t = 20)] + last: usize, + #[arg(long)] + json: bool, + }, /// Stop a server early (or collect an expired one's backup) and download its world. Stop { id: String, @@ -85,7 +115,7 @@ fn left(expires_at: u64) -> String { pub async fn run(args: &[String]) -> Result<()> { let cli = GameserverCli::try_parse_from(std::iter::once("c0mpute gameserver".to_string()).chain(args.iter().cloned())).unwrap_or_else(|e| e.exit()); match cli.cmd { - Sub::Start { game, hours, ram, cpus, accept_eula, world, password, name, max_price, worker, timeout_secs, json } => { + Sub::Start { game, hours, ram, cpus, accept_eula, world, password, name, backup_mins, max_price, worker, timeout_secs, json } => { let game = Game::parse(&game).with_context(|| format!("unknown game {game:?}: minecraft, factorio or valheim"))?; let req = StartRequest { game, @@ -96,8 +126,10 @@ pub async fn run(args: &[String]) -> Result<()> { world, password, server_name: name, + backup_mins, max_price_usd: max_price, workers: worker, + exclude: Vec::new(), timeout: Duration::from_secs(timeout_secs.clamp(60, 3600)), }; eprintln!("looking for a worker that hosts {} ...", game.name()); @@ -107,6 +139,7 @@ pub async fn run(args: &[String]) -> Result<()> { } else { println!("{} {} is up at {}", r.game.name(), r.id, r.address); println!(" worker {} ${} stops in {}", r.worker, r.price_usd, left(r.expires_at)); + println!(" keep it healthy (probe, mirror backups, fail over): c0mpute gameserver watch {}", r.id); println!(" stop early (and keep the world): c0mpute gameserver stop {}", r.id); } Ok(()) @@ -121,15 +154,50 @@ pub async fn run(args: &[String]) -> Result<()> { println!("no servers rented from this machine"); } for r in all { - let status = match (r.stopped_at, &r.backup) { - (Some(_), Some(b)) => format!("stopped world {b}"), - (Some(_), None) => "stopped".into(), - (None, _) => left(r.expires_at), + let status = match (r.stopped_at, &r.backup, &r.superseded_by) { + (_, _, Some(next)) => format!("failed over to {next}"), + (Some(_), Some(b), _) => format!("stopped world {b}"), + (Some(_), None, _) => "stopped".into(), + (None, _, _) => match &r.latest_backup { + Some(b) => format!("{} backup {}", left(r.expires_at), &b[..12.min(b.len())]), + None => left(r.expires_at), + }, }; println!("{:20} {:10} {:28} {}", r.id, r.game.name(), r.address, status); } Ok(()) } + Sub::Watch { id, probe_every_secs, fail_after_secs, no_failover, max_price, worker } => { + let opts = gameserver::WatchOptions { + probe_every: Duration::from_secs(probe_every_secs.clamp(5, 3600)), + fail_after: Duration::from_secs(fail_after_secs.clamp(30, 86_400)), + failover: !no_failover, + max_price_usd: max_price, + workers: worker, + ..Default::default() + }; + eprintln!("watching {id} (probe every {}s, fail over after {}s down)", opts.probe_every.as_secs(), opts.fail_after.as_secs()); + match gameserver::watch(&id, opts).await? { + gameserver::WatchEnd::Ended(r) => println!("{} ended (reservation over or stopped)", r.id), + gameserver::WatchEnd::FailedOver { from, to } => println!("{} failed over to {} at {}", from.id, to.id, to.address), + } + Ok(()) + } + Sub::Health { id, last, json } => { + let log = gameserver::health_log(&gameserver::buyer_dir(), &id); + let tail = &log[log.len().saturating_sub(last)..]; + if json { + println!("{}", serde_json::to_string(tail)?); + return Ok(()); + } + if tail.is_empty() { + println!("no probes recorded for {id} (run `c0mpute gameserver watch {id}`)"); + } + for p in tail { + println!("{} {} {:16} {:>5}ms {}", p.ts, if p.ok { "up " } else { "DOWN" }, p.method, p.rtt_ms, p.detail); + } + Ok(()) + } Sub::Stop { id, timeout_secs, json } => { let r = gameserver::stop(&id, Duration::from_secs(timeout_secs.clamp(30, 3600))).await?; if json { diff --git a/node/crates/c0mpute-cli/src/main.rs b/node/crates/c0mpute-cli/src/main.rs index ca4126db..867ca2fd 100644 --- a/node/crates/c0mpute-cli/src/main.rs +++ b/node/crates/c0mpute-cli/src/main.rs @@ -2367,62 +2367,6 @@ async fn run_stt(cmd: SttCmd) -> Result<()> { } } -/// Build the runner image from a shallow clone of the public repo. -/// -/// The fallback for `c0mpute video setup` when the registry will not serve -/// the image. Slower than a pull (it downloads the Python stack rather than -/// finished layers) but it needs no registry account, which is the point. -async fn build_video_runner() -> Result<()> { - use c0mpute_core::video; - - for bin in ["docker", "git"] { - if tokio::process::Command::new(bin) - .arg("--version") - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .status() - .await - .is_err() - { - anyhow::bail!("building the runner needs both docker and git; {bin} is missing"); - } - } - - // A fresh directory per run: a half-finished clone from a previous - // attempt would make `git clone` fail rather than resume. - let checkout = std::env::temp_dir().join(format!("c0mpute-video-src-{}", std::process::id())); - let _ = tokio::fs::remove_dir_all(&checkout).await; - - println!("cloning {} (shallow)", video::RUNNER_SOURCE_REPO); - let status = tokio::process::Command::new("git") - .args(["clone", "--depth", "1", video::RUNNER_SOURCE_REPO]) - .arg(&checkout) - .status() - .await - .context("could not run git")?; - if !status.success() { - anyhow::bail!("git clone failed with {status}"); - } - - println!("building {} (several minutes)", video::RUNNER_IMAGE); - let args = video::build_argv(&checkout, video::RUNNER_IMAGE); - let status = tokio::process::Command::new("docker") - .args(&args) - .status() - .await - .context("could not run docker build")?; - - // Clean up whether or not the build worked: the checkout is scratch and - // a stale one in /tmp is worse than no cache at all. - let _ = tokio::fs::remove_dir_all(&checkout).await; - - if !status.success() { - anyhow::bail!("docker build failed with {status}"); - } - println!("built {}", video::RUNNER_IMAGE); - Ok(()) -} - /// `c0mpute video` — the surface `plugins/video/module.toml` declares. /// /// Everything here talks to the runner image rather than to the network: a @@ -2433,29 +2377,8 @@ async fn run_video(cmd: VideoCmd) -> Result<()> { match cmd { VideoCmd::Setup { build } => { - let mut have_image = false; - if !build { - println!("pulling {} (this is a large image)", video::RUNNER_IMAGE); - let status = tokio::process::Command::new("docker") - .args(["pull", video::RUNNER_IMAGE]) - .status() - .await - .context("could not run docker; install a container runtime first")?; - have_image = status.success(); - if !have_image { - // A registry is the kind of dependency that is private by - // default, rate-limited or down, and none of those are a - // reason a node with a card cannot serve video. The repo - // is public, so build the same image from source. - println!(); - println!("could not pull the runner image (private, rate-limited or offline)."); - println!("building it from source instead — same Dockerfile, same result."); - println!(); - } - } - if !have_image { - build_video_runner().await?; - } + let how = video::setup(build, &std::env::temp_dir(), &|line: &str| println!("{line}")).await?; + println!("runner image ready as {} ({how:?})", video::RUNNER_IMAGE); match video::readiness(video::RUNNER_IMAGE) { None => { println!("this node can serve video jobs"); @@ -2482,7 +2405,7 @@ async fn run_video(cmd: VideoCmd) -> Result<()> { } if selftest { println!("running the runner selftest in the container"); - let status = tokio::process::Command::new("docker") + let status = c0mpute_core::container::command() .args(["run", "--rm", video::RUNNER_IMAGE, "selftest"]) .status() .await @@ -2497,7 +2420,7 @@ async fn run_video(cmd: VideoCmd) -> Result<()> { // The image carries the manifest it was built from, so this is // the model set that node would actually serve — not what this // CLI happened to be compiled with. - let out = tokio::process::Command::new("docker") + let out = c0mpute_core::container::command() .args([ "run", "--rm", diff --git a/node/crates/c0mpute-core/src/container.rs b/node/crates/c0mpute-core/src/container.rs new file mode 100644 index 00000000..b1513ce2 --- /dev/null +++ b/node/crates/c0mpute-core/src/container.rs @@ -0,0 +1,57 @@ +//! The container CLI the container workloads (`video`, `gameserver`) drive. +//! +//! `docker` by default. `C0MPUTE_DOCKER` names another binary with the same +//! command line: `podman`, a wrapper that adds `--context`, or a test double +//! that stands in for the daemon in the loopback end-to-end tests. + +use std::ffi::OsString; +use std::path::PathBuf; + +/// Environment variable that overrides the container CLI. +pub const ENV: &str = "C0MPUTE_DOCKER"; + +/// The container CLI to run: `$C0MPUTE_DOCKER`, else `docker`. +pub fn cli() -> OsString { + match std::env::var_os(ENV) { + Some(v) if !v.is_empty() => v, + _ => OsString::from("docker"), + } +} + +/// Where [`cli`] resolves: the path itself when it names a file, else the +/// first match on `PATH`. `None` means there is no container runtime. +pub fn which() -> Option { + let bin = cli(); + let as_path = PathBuf::from(&bin); + if as_path.components().count() > 1 { + return as_path.is_file().then_some(as_path); + } + std::env::var_os("PATH").and_then(|paths| { + std::env::split_paths(&paths) + .map(|dir| dir.join(&bin)) + .find(|candidate| candidate.is_file()) + }) +} + +/// A `tokio` command for the container CLI. +pub fn command() -> tokio::process::Command { + tokio::process::Command::new(cli()) +} + +/// A blocking command for the container CLI (readiness probes only). +pub fn command_sync() -> std::process::Command { + std::process::Command::new(cli()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_to_docker() { + // The variable is not set in the unit-test process. + if std::env::var_os(ENV).is_none() { + assert_eq!(cli(), OsString::from("docker")); + } + } +} diff --git a/node/crates/c0mpute-core/src/gameserver.rs b/node/crates/c0mpute-core/src/gameserver.rs index d5304f44..ed9dca53 100644 --- a/node/crates/c0mpute-core/src/gameserver.rs +++ b/node/crates/c0mpute-core/src/gameserver.rs @@ -4,14 +4,34 @@ //! A start job is an ordinary auction whose job is short: the winner creates //! a container from a pinned, well-known image (`docker create`, argv only, //! never a shell) with memory, CPU and pid limits, a named volume for the -//! world and a published port, optionally restores a world backup the buyer -//! serves by hash, starts it, and returns `address:port` and the expiry. The -//! container then runs until the reservation ends: the worker's reaper stops -//! it at expiry. `gameserver.stop` (bid on only by the worker holding the -//! server, and only with the stop token whose sha256 the start job carried) -//! ends it early. Either way the worker tars the world out of the volume, -//! stores it as 1 MiB content-addressed chunks plus a manifest, and hands the -//! manifest hash back: `start --world ` restores it on any worker. +//! world and a published port, optionally restores a world the buyer serves +//! by hash, starts it, and returns `address:port` and the expiry. +//! +//! Secrets never ride in the offer. The offer carries the buyer's fresh +//! X25519 key; after the accept the winner announces its own and the buyer +//! seals the Valheim password and the restore key to it ([`crate::handoff`]). +//! +//! Worlds are encrypted. A world the buyer restores is published as AES-GCM +//! chunks under a fresh key that only the winning worker receives. Every +//! backup a worker takes is sealed to the buyer's long-term backup key +//! (X25519 → HKDF → AES-256-GCM per 1 MiB chunk): the manifest hash in a +//! receipt or a status record is useless to anyone else. +//! +//! While a server runs the worker backs the world up every `backup_mins` +//! (default 30) and gossips a status record (container running, latest +//! backup hash) on [`STATUS_TOPIC`], signed by its peer key. The buyer's +//! `gameserver watch` probes the game port (Minecraft status ping, Valheim +//! Steam A2S query, Factorio UDP reachability), logs every probe, mirrors +//! each new backup to its own machine, and when the server fails its checks +//! for `fail_after` it starts the same world on another worker from the +//! latest backup. No coordinator anywhere. +//! +//! Expiry does not depend on the worker being online: the container runs +//! under `timeout` (with `--init`) so it exits by itself when the +//! reservation ends, and `--rm` removes it; the named volume keeps the world +//! until the worker, when it next runs, backs it up and removes it. At boot +//! and on every sweep the worker also kills any labelled container whose +//! reservation has passed. //! //! Only opt-in nodes serve it (`--roles ...,gameserver`): it holds a port and //! RAM for hours. A node advertises it only when `docker info` works and it @@ -20,13 +40,15 @@ //! The operator must open the port range in the firewall. use std::collections::{HashMap, HashSet}; -use std::net::IpAddr; +use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use anyhow::{Context, Result, bail}; +use aes_gcm::aead::{Aead, KeyInit}; +use aes_gcm::{Aes256Gcm, Nonce}; +use anyhow::{Context, Result, anyhow, bail}; use c0mpute_net::topics::{JobOffer, JobStatus}; use c0mpute_net::{Libp2pNetwork, Network}; use c0mpute_proto::Hash; @@ -35,27 +57,43 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::sync::{Semaphore, mpsc}; +use tokio::sync::{Semaphore, broadcast, mpsc}; use tracing::{info, warn}; +use x25519_dalek::StaticSecret; use crate::runner::{AcceptedJob, WorkerLoad, publish_receipt}; +use crate::temporal::crypto; pub const WORKLOAD_START: &str = "gameserver.start"; pub const WORKLOAD_STOP: &str = "gameserver.stop"; pub const ROLE_TAG: &str = "c0mpute:role:gameserver"; +/// Where workers gossip [`StatusRecord`]s. +pub const STATUS_TOPIC: &str = "c0mpute/gameserver/v1"; +pub const SPEC_VERSION: u32 = 2; pub const MAX_HOURS: u32 = 72; pub const MAX_RAM_GB: u32 = 32; pub const MAX_CPUS: f64 = 8.0; -/// World backups are stored and moved as chunks of this size. +/// World backups are stored and moved as chunks of this size (plaintext). pub const CHUNK_BYTES: usize = 1024 * 1024; +/// AES-GCM tag on every chunk. +const TAG_BYTES: usize = 16; pub const MAX_WORLD_BYTES: u64 = 4 * 1024 * 1024 * 1024; +pub const DEFAULT_BACKUP_MINS: u32 = 30; +pub const MIN_BACKUP_MINS: u32 = 5; +pub const MAX_BACKUP_MINS: u32 = 720; +pub const ENC_SCHEME: &str = "x25519-hkdf-sha256-aes256gcm-chunked/v1"; const MANIFEST_KIND: &str = "c0mpute-gameserver-world"; +const MANIFEST_VERSION: u32 = 2; const MAX_MANIFEST_BYTES: usize = 1024 * 1024; /// Start/stop jobs handled at once (the servers themselves are not jobs). const JOB_SLOTS: usize = 2; /// Stopped-server records (and their backup hash) are kept this long. const KEEP_STOPPED_SECS: u64 = 7 * 86_400; -const REAP_EVERY: Duration = Duration::from_secs(30); +pub const REAP_EVERY: Duration = Duration::from_secs(30); +/// How long a worker waits for the buyer's sealed secrets after the accept. +const HANDOFF_WAIT: Duration = Duration::from_secs(120); +/// `timeout -k`: after the TERM at expiry, KILL this many seconds later. +const EXPIRY_KILL_AFTER_SECS: u64 = 90; // ── games ─────────────────────────────────────────────────────────────── @@ -104,6 +142,16 @@ impl Game { Game::Valheim => "lloesche/valheim-server:sha-0ab6cdcc16dd@sha256:fde7829b12d39203e155b6209689b70234f8a27d12c4d48baf890cc2ae2c2df3", } } + /// The pinned image's own ENTRYPOINT + CMD (read from each image config + /// at the digest above). The container runs it under `timeout`, so the + /// server exits at expiry whether or not the worker is online. + pub fn command(self) -> &'static [&'static str] { + match self { + Game::Minecraft => &["/image/scripts/start"], + Game::Factorio => &["/docker-entrypoint.sh"], + Game::Valheim => &["/usr/local/sbin/bootstrap"], + } + } /// Where the image keeps its world; the named volume mounts here. pub fn data_dir(self) -> &'static str { match self { @@ -140,7 +188,8 @@ impl Game { // ── specs ─────────────────────────────────────────────────────────────── -/// `spec_inline` of a `gameserver.start` offer. +/// `spec_inline` of a `gameserver.start` offer. Everything in it is public: +/// every subscriber of the job topic can read it. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct StartSpec { pub v: u32, @@ -153,16 +202,25 @@ pub struct StartSpec { pub accept_eula: bool, /// `sha256:` of the stop token only the buyer holds. pub stop_hash: String, - /// Manifest hash (blake3 hex) of a world backup to restore. + /// The buyer's fresh X25519 key for this job's [`crate::handoff`]. + pub client_pubkey: String, + /// The buyer's long-term X25519 key every backup is sealed to. + pub backup_pubkey: String, + /// Minutes between live backups; 0 = only at stop or expiry. + #[serde(default = "default_backup_mins")] + pub backup_mins: u32, + /// Manifest hash (blake3 hex) of an encrypted world to restore; its key + /// arrives in the sealed [`StartSecrets`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub world: Option, - /// Valheim only (it requires one). Visible to peers on the job topic. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub password: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub server_name: Option, } +fn default_backup_mins() -> u32 { + DEFAULT_BACKUP_MINS +} + fn is_hex(s: &str, len: usize) -> bool { s.len() == len && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) } @@ -173,8 +231,8 @@ fn valid_sha256_ref(s: &str) -> bool { impl StartSpec { pub fn validate(&self) -> std::result::Result<(), String> { - if self.v != 1 { - return Err(format!("unsupported gameserver spec version {}", self.v)); + if self.v != SPEC_VERSION { + return Err(format!("unsupported gameserver spec version {} (this node speaks {SPEC_VERSION})", self.v)); } if !(1..=MAX_HOURS).contains(&self.hours) { return Err(format!("hours must be 1..={MAX_HOURS}")); @@ -191,15 +249,15 @@ impl StartSpec { if !valid_sha256_ref(&self.stop_hash) { return Err("stop_hash must be sha256:<64 lowercase hex>".into()); } + if !is_hex(&self.client_pubkey, 64) || !is_hex(&self.backup_pubkey, 64) { + return Err("client_pubkey and backup_pubkey must be X25519 keys (64 hex)".into()); + } + if self.backup_mins != 0 && !(MIN_BACKUP_MINS..=MAX_BACKUP_MINS).contains(&self.backup_mins) { + return Err(format!("backup_mins must be 0 or {MIN_BACKUP_MINS}..={MAX_BACKUP_MINS}")); + } if self.world.as_deref().is_some_and(|w| !is_hex(w, 64)) { return Err("world must be a 64-hex manifest hash".into()); } - match (self.game, &self.password) { - (Game::Valheim, Some(p)) if (5..=32).contains(&p.len()) && p.chars().all(|c| c.is_ascii_alphanumeric()) => {} - (Game::Valheim, _) => return Err("valheim needs a password of 5-32 letters or digits".into()), - (_, Some(_)) => return Err("only valheim takes a password".into()), - _ => {} - } if let Some(n) = &self.server_name && (n.is_empty() || n.len() > 32 || !n.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, ' ' | '-' | '_' | '.'))) { @@ -209,6 +267,51 @@ impl StartSpec { } } +/// What the buyer seals to the winning worker after the accept. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct StartSecrets { + pub v: u32, + /// Valheim only (it requires one). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + /// Hex AES-256 key of the world named in the spec. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub world_key: Option, +} + +impl StartSecrets { + pub fn validate(&self, spec: &StartSpec) -> std::result::Result<(), String> { + if self.v != 1 { + return Err("unsupported secrets version".into()); + } + match (spec.game, &self.password) { + (Game::Valheim, Some(p)) if valid_password(p) => {} + (Game::Valheim, _) => return Err("valheim needs a password of 5-32 letters or digits".into()), + (_, Some(_)) => return Err("only valheim takes a password".into()), + _ => {} + } + match (&spec.world, &self.world_key) { + (Some(_), Some(k)) if is_hex(k, 64) => {} + (Some(_), _) => return Err("a world restore needs its 32-byte key".into()), + (None, Some(_)) => return Err("a world key without a world".into()), + (None, None) => {} + } + Ok(()) + } + + pub fn world_key(&self) -> Result> { + self.world_key.as_deref().map(key_from_hex).transpose() + } +} + +pub fn valid_password(p: &str) -> bool { + (5..=32).contains(&p.len()) && p.chars().all(|c| c.is_ascii_alphanumeric()) +} + +fn key_from_hex(h: &str) -> Result<[u8; 32]> { + hex::decode(h)?.try_into().map_err(|_| anyhow!("key must be 32 bytes")) +} + /// `spec_inline` of a `gameserver.stop` offer. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct StopSpec { @@ -264,12 +367,14 @@ pub struct ContainerPlan { pub cpus: f64, /// First host port; the game's ports map onto consecutive host ports. pub host_port: u16, + /// Seconds the server may run before it stops itself (the reservation). + pub run_secs: u64, pub env: Vec<(String, String)>, pub labels: Vec<(String, String)>, } impl ContainerPlan { - pub fn new(id: &str, spec: &StartSpec, host_port: u16, expires: u64) -> Self { + pub fn new(id: &str, spec: &StartSpec, secrets: &StartSecrets, host_port: u16, now: u64, expires: u64) -> Self { let memory_mb = u64::from(spec.ram_gb) * 1024; let name = format!("c0mpute-{id}"); let mut env: Vec<(String, String)> = Vec::new(); @@ -285,7 +390,7 @@ impl ContainerPlan { Game::Valheim => { env.push(("SERVER_NAME".into(), label)); env.push(("WORLD_NAME".into(), "c0mpute".into())); - env.push(("SERVER_PASS".into(), spec.password.clone().unwrap_or_default())); + env.push(("SERVER_PASS".into(), secrets.password.clone().unwrap_or_default())); env.push(("SERVER_PUBLIC".into(), "false".into())); env.push(("SERVER_PORT".into(), "2456".into())); } @@ -296,13 +401,28 @@ impl ContainerPlan { ("c0mpute.gameserver.game".into(), spec.game.name().into()), ("c0mpute.gameserver.expires".into(), expires.to_string()), ]; - Self { volume: name.clone(), name, game: spec.game, memory_mb, cpus: spec.cpus, host_port, env, labels } + Self { + volume: name.clone(), + name, + game: spec.game, + memory_mb, + cpus: spec.cpus, + host_port, + run_secs: expires.saturating_sub(now).max(1), + env, + labels, + } } /// `docker create` argv (without the leading `docker`). Every value is its /// own argument: nothing passes through a shell. + /// + /// The image's own command runs under `timeout ` behind + /// `--init`, and `--rm` removes the container when it exits: the server + /// ends at expiry even if this worker is offline then. The world lives + /// in the named volume, which `--rm` keeps. pub fn create_argv(&self) -> Vec { - let mut a: Vec = vec!["create".into(), "--name".into(), self.name.clone()]; + let mut a: Vec = vec!["create".into(), "--name".into(), self.name.clone(), "--rm".into(), "--init".into()]; for (k, v) in [ ("--restart", "no".to_string()), ("--memory", format!("{}m", self.memory_mb)), @@ -313,6 +433,7 @@ impl ContainerPlan { ("--log-opt", "max-size=10m".into()), ("--log-opt", "max-file=2".into()), ("--stop-timeout", "60".into()), + ("--entrypoint", "timeout".into()), ] { a.push(k.into()); a.push(v); @@ -332,12 +453,41 @@ impl ContainerPlan { a.push(format!("{k}={v}")); } a.push(self.game.image().into()); + // Arguments to `timeout`: TERM at expiry, KILL if still alive after. + for x in ["-s".to_string(), "TERM".into(), "-k".into(), format!("{EXPIRY_KILL_AFTER_SECS}s"), format!("{}s", self.run_secs)] { + a.push(x); + } + a.extend(self.game.command().iter().map(|s| s.to_string())); a } } +/// What a worker needs from a container runtime. [`Docker`] in production; +/// the loopback end-to-end tests supply a double. +#[async_trait::async_trait] +pub trait Runtime: Send + Sync + 'static { + /// Create (not start) the container and its volume. + async fn create(&self, plan: &ContainerPlan) -> Result<()>; + /// Extract a world tar (entries under the data dir's name) into the + /// created container's volume. + async fn restore(&self, plan: &ContainerPlan, tar: &Path) -> Result<()>; + async fn start(&self, name: &str) -> Result<()>; + async fn running(&self, name: &str) -> bool; + /// Ask the server to write its world to disk (best effort). + async fn flush(&self, name: &str, game: Game); + async fn stop(&self, name: &str) -> Result<()>; + /// Write a tar of the volume's world to `out`. Works whether or not the + /// server container still exists. + async fn tar_world(&self, volume: &str, game: Game, out: &Path) -> Result<()>; + /// Remove the container (if it is still there) and its volume. + async fn remove(&self, name: &str, volume: &str); + /// Containers this runtime labelled as game servers, with their expiry. + async fn labelled(&self) -> Result>; + async fn kill(&self, name: &str); +} + async fn docker(args: &[&str], timeout: Duration) -> Result { - let out = tokio::time::timeout(timeout, tokio::process::Command::new("docker").args(args).stdin(Stdio::null()).kill_on_drop(true).output()) + let out = tokio::time::timeout(timeout, crate::container::command().args(args).stdin(Stdio::null()).kill_on_drop(true).output()) .await .with_context(|| format!("docker {} timed out", args.first().unwrap_or(&"")))? .context("run docker")?; @@ -348,7 +498,124 @@ async fn docker(args: &[&str], timeout: Duration) -> Result { } fn docker_sync_ok(args: &[&str]) -> bool { - std::process::Command::new("docker").args(args).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success()) + crate::container::command_sync().args(args).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success()) +} + +/// `docker run` argv that tars a world volume to stdout with the game's own +/// (already pulled) image: no network, the volume read-only. +pub fn tar_argv(volume: &str, game: Game) -> Vec { + let dir = game.data_dir(); + vec![ + "run".into(), + "--rm".into(), + "--network".into(), + "none".into(), + "--entrypoint".into(), + "tar".into(), + "-v".into(), + format!("{volume}:{dir}:ro"), + game.image().into(), + "-C".into(), + "/".into(), + "-cf".into(), + "-".into(), + dir.trim_start_matches('/').into(), + ] +} + +/// The real container runtime: the `docker` CLI (or `$C0MPUTE_DOCKER`). +pub struct Docker; + +#[async_trait::async_trait] +impl Runtime for Docker { + async fn create(&self, plan: &ContainerPlan) -> Result<()> { + let argv = plan.create_argv(); + let args: Vec<&str> = argv.iter().map(String::as_str).collect(); + docker(&args, Duration::from_secs(1800)).await.map(drop) + } + + async fn restore(&self, plan: &ContainerPlan, tar: &Path) -> Result<()> { + // `docker cp - :/` extracts the tar (entries `data/...`) into the volume. + let f = std::fs::File::open(tar).with_context(|| format!("open {}", tar.display()))?; + let out = crate::container::command() + .args(["cp", "-", &format!("{}:/", plan.name)]) + .stdin(Stdio::from(f)) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .output() + .await + .context("run docker cp")?; + if !out.status.success() { + bail!("docker cp into {}: {}", plan.name, String::from_utf8_lossy(&out.stderr).trim()); + } + Ok(()) + } + + async fn start(&self, name: &str) -> Result<()> { + docker(&["start", name], Duration::from_secs(120)).await.map(drop) + } + + async fn running(&self, name: &str) -> bool { + docker(&["inspect", "-f", "{{.State.Running}}", name], Duration::from_secs(30)).await.is_ok_and(|s| s == "true") + } + + async fn flush(&self, name: &str, game: Game) { + // itzg's image ships rcon-cli with RCON on; the others autosave. + if game == Game::Minecraft { + let _ = docker(&["exec", name, "rcon-cli", "save-all", "flush"], Duration::from_secs(60)).await; + } + } + + async fn stop(&self, name: &str) -> Result<()> { + docker(&["stop", "-t", "60", name], Duration::from_secs(120)).await.map(drop) + } + + async fn tar_world(&self, volume: &str, game: Game, out: &Path) -> Result<()> { + let f = std::fs::File::create(out).with_context(|| format!("create {}", out.display()))?; + let argv = tar_argv(volume, game); + let res = tokio::time::timeout( + Duration::from_secs(1800), + crate::container::command().args(&argv).stdin(Stdio::null()).stdout(Stdio::from(f)).stderr(Stdio::piped()).kill_on_drop(true).output(), + ) + .await + .context("tar of the world timed out")? + .context("run docker")?; + if !res.status.success() { + bail!("tar of volume {volume}: {}", String::from_utf8_lossy(&res.stderr).trim()); + } + Ok(()) + } + + async fn remove(&self, name: &str, volume: &str) { + let _ = docker(&["rm", "-f", name], Duration::from_secs(60)).await; + let _ = docker(&["volume", "rm", volume], Duration::from_secs(60)).await; + } + + async fn labelled(&self) -> Result> { + let out = docker(&["ps", "-a", "--filter", "label=c0mpute.gameserver=1", "--format", "{{.Names}}\t{{.Label \"c0mpute.gameserver.expires\"}}"], Duration::from_secs(30)).await?; + Ok(parse_labelled(&out)) + } + + async fn kill(&self, name: &str) { + let _ = docker(&["kill", name], Duration::from_secs(30)).await; + let _ = docker(&["rm", "-f", name], Duration::from_secs(30)).await; + } +} + +/// `docker ps --format '{{.Names}}\t{{.Label ...expires}}'` → (name, expiry). +/// A container without a readable expiry counts as already expired. +pub fn parse_labelled(out: &str) -> Vec<(String, u64)> { + out.lines() + .filter_map(|l| { + let mut it = l.split('\t'); + let name = it.next()?.trim(); + if name.is_empty() { + return None; + } + Some((name.to_string(), it.next().and_then(|e| e.trim().parse().ok()).unwrap_or(0))) + }) + .collect() } // ── public address ────────────────────────────────────────────────────── @@ -483,10 +750,15 @@ pub async fn ensure_images(cfg: &crate::config::GameserverConfig) { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct Backup { - /// Blake3 hex of the world manifest. + /// Blake3 hex of the (encrypted) world manifest. pub manifest: String, + /// Plaintext tar size. pub bytes: u64, + /// sha256 of the plaintext tar. pub sha256: String, + /// When it was taken (unix seconds). + #[serde(default)] + pub at: u64, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] @@ -502,8 +774,15 @@ pub struct ServerState { pub expires_at: u64, pub stop_hash: String, pub buyer: String, + /// Every backup is sealed to this key. + #[serde(default)] + pub backup_pubkey: String, + /// Seconds between live backups (0 = none). + #[serde(default)] + pub backup_every_secs: u64, #[serde(default)] pub stopped_at: Option, + /// The latest backup (live, or the final one at stop/expiry). #[serde(default)] pub backup: Option, #[serde(default)] @@ -527,6 +806,26 @@ pub fn due_for_stop(servers: &HashMap, now: u64) -> Vec, now: u64) -> Vec { + let mut ids: Vec = servers + .values() + .filter(|s| { + let last = s.backup.as_ref().map(|b| b.at).unwrap_or(s.started_at).max(s.started_at); + s.running() && s.backup_every_secs > 0 && now < s.expires_at && now >= last + s.backup_every_secs + }) + .map(|s| s.id.clone()) + .collect(); + ids.sort(); + ids +} + +/// Labelled containers whose reservation has passed: killed on sight, at +/// boot and on every sweep, whatever the state file says. +pub fn expired_containers(labelled: &[(String, u64)], now: u64) -> Vec { + labelled.iter().filter(|(_, exp)| now >= *exp).map(|(n, _)| n.clone()).collect() +} + /// Can this node take `spec` next to what it already runs? pub fn has_capacity(cfg: &crate::config::GameserverConfig, total_ram_mb: u64, servers: &HashMap, spec: &StartSpec) -> std::result::Result<(), String> { if !cfg.games.contains(&spec.game) { @@ -567,6 +866,55 @@ fn port_bindable(p: u16) -> bool { std::net::TcpListener::bind(("0.0.0.0", p)).is_ok() && std::net::UdpSocket::bind(("0.0.0.0", p)).is_ok() } +// ── world encryption ──────────────────────────────────────────────────── + +/// How a world's chunks are encrypted. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct WorldEnc { + pub scheme: String, + /// Present on backups: the worker's one-time X25519 key; the buyer's + /// backup key and this give the chunk key. Absent on a world the buyer + /// publishes for a restore: its key travels in the sealed secrets. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub epk: Option, +} + +impl WorldEnc { + pub fn handed_off() -> Self { + Self { scheme: ENC_SCHEME.into(), epk: None } + } +} + +/// A fresh random world key. +pub fn random_key() -> [u8; 32] { + rand::random() +} + +/// The chunk key of a backup sealed to `recipient_pub`: derived from the +/// worker's one-time secret and the recipient's key (worker side), or the +/// recipient's secret and the one-time `epk` (buyer side). +pub fn backup_key(secret: &StaticSecret, peer_pub_hex: &str, epk_hex: &str, recipient_hex: &str) -> Result<[u8; 32]> { + crypto::session_key(secret, peer_pub_hex, &format!("gameserver-backup:{epk_hex}:{recipient_hex}")) +} + +fn chunk_nonce(i: u64) -> [u8; 12] { + let mut n = [0u8; 12]; + n[4..].copy_from_slice(&i.to_be_bytes()); + n +} + +/// Chunk `i` under `key`. The key is single-use (one per world), so the +/// index is a safe nonce, and it pins every chunk to its position. +pub fn seal_chunk(key: &[u8; 32], i: u64, plain: &[u8]) -> Result> { + let c = Aes256Gcm::new_from_slice(key).map_err(|_| anyhow!("key"))?; + c.encrypt(Nonce::from_slice(&chunk_nonce(i)), plain).map_err(|_| anyhow!("encrypt chunk")) +} + +pub fn open_chunk(key: &[u8; 32], i: u64, sealed: &[u8]) -> Result> { + let c = Aes256Gcm::new_from_slice(key).map_err(|_| anyhow!("key"))?; + c.decrypt(Nonce::from_slice(&chunk_nonce(i)), sealed).map_err(|_| anyhow!("world chunk {i} does not decrypt (wrong key or tampered)")) +} + // ── world backups ─────────────────────────────────────────────────────── #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] @@ -576,28 +924,34 @@ pub struct WorldManifest { pub game: Game, /// A tar of this directory (entries start with its name, e.g. `data/`). pub data_dir: String, + /// Plaintext tar size. pub bytes: u64, + /// sha256 of the plaintext tar, checked after decryption. pub sha256: String, - /// Blake3 hex of each chunk, in order. + pub enc: WorldEnc, + /// Blake3 hex of each encrypted chunk, in order. pub chunks: Vec, } impl WorldManifest { - pub fn new(game: Game) -> Self { - Self { v: 1, kind: MANIFEST_KIND.into(), game, data_dir: game.data_dir().into(), bytes: 0, sha256: String::new(), chunks: Vec::new() } + pub fn new(game: Game, enc: WorldEnc) -> Self { + Self { v: MANIFEST_VERSION, kind: MANIFEST_KIND.into(), game, data_dir: game.data_dir().into(), bytes: 0, sha256: String::new(), enc, chunks: Vec::new() } } pub fn to_bytes(&self) -> Vec { serde_json::to_vec(self).unwrap_or_default() } pub fn from_bytes(b: &[u8]) -> Result { - let m: Self = serde_json::from_slice(b).context("world manifest is not JSON")?; + let m: Self = serde_json::from_slice(b).context("world manifest is not a v2 JSON manifest")?; m.check()?; Ok(m) } pub fn check(&self) -> Result<()> { - if self.v != 1 || self.kind != MANIFEST_KIND { + if self.v != MANIFEST_VERSION || self.kind != MANIFEST_KIND { bail!("not a c0mpute game world manifest"); } + if self.enc.scheme != ENC_SCHEME || self.enc.epk.as_deref().is_some_and(|k| !is_hex(k, 64)) { + bail!("unknown world encryption"); + } if self.data_dir != self.game.data_dir() { bail!("manifest data_dir does not match {}", self.game.name()); } @@ -612,10 +966,11 @@ impl WorldManifest { } } -/// Read `r` to the end into 1 MiB chunks in the store; returns the manifest -/// (not yet stored). Chunks are announced so any peer can fetch them. -pub async fn chunk_into_store(store: &ChunkStore, net: Option<&Libp2pNetwork>, game: Game, mut r: R) -> Result { - let mut m = WorldManifest::new(game); +/// Read `r` to the end, encrypt it under `key` in 1 MiB chunks, and put the +/// chunks in the store; returns the manifest (not yet stored). Chunks are +/// announced so any peer can fetch them; only the key holder can read them. +pub async fn chunk_into_store(store: &ChunkStore, net: Option<&Libp2pNetwork>, game: Game, mut r: R, key: &[u8; 32], enc: WorldEnc) -> Result { + let mut m = WorldManifest::new(game, enc); let mut sha = Sha256::new(); let mut buf = vec![0u8; CHUNK_BYTES]; loop { @@ -635,7 +990,8 @@ pub async fn chunk_into_store(store: &ChunkStor bail!("world is over {} GiB", MAX_WORLD_BYTES >> 30); } sha.update(&buf[..filled]); - let h = store.put(&buf[..filled]).await?; + let sealed = seal_chunk(key, m.chunks.len() as u64, &buf[..filled])?; + let h = store.put(&sealed).await?; if let Some(net) = net { let _ = net.announce(&h).await; } @@ -663,18 +1019,32 @@ async fn fetch_bytes(net: &Libp2pNetwork, store: &ChunkStore, hash: &str, provid { return Ok(b); } - crate::gateway_util::fetch_verified(net, hash, provider, max, Duration::from_secs(60)).await.map_err(|e| anyhow::anyhow!("fetch {hash} from {provider}: {e:?}")) + let b = crate::gateway_util::fetch_verified(net, hash, provider, max, Duration::from_secs(60)).await.map_err(|e| anyhow!("fetch {hash} from {provider}: {e:?}"))?; + // Keep a copy: a buyer mirroring backups must be able to restore after + // the worker that took them is gone. + let _ = store.put(&b).await; + Ok(b) } -/// Fetch a world (manifest, then each chunk) from `provider` (or any peer -/// that announced it), verifying every chunk and the whole tar's sha256, -/// and write the tar to `out` as it arrives. -pub async fn fetch_world(net: &Libp2pNetwork, store: &ChunkStore, manifest_hash: &str, provider: &str, mut out: W) -> Result { +/// Fetch an encrypted world (manifest, then each chunk) from `provider` (or +/// a local copy), verify every chunk's hash, decrypt it with the key +/// `key_of` gives for the manifest, check the plaintext sha256, and write the +/// tar to `out` as it arrives. +pub async fn fetch_world( + net: &Libp2pNetwork, + store: &ChunkStore, + manifest_hash: &str, + provider: &str, + key_of: impl FnOnce(&WorldManifest) -> Result<[u8; 32]>, + mut out: W, +) -> Result { let mb = fetch_bytes(net, store, manifest_hash, provider, MAX_MANIFEST_BYTES).await.context("world manifest")?; let m = WorldManifest::from_bytes(&mb)?; + let key = key_of(&m)?; let mut sha = Sha256::new(); - for c in &m.chunks { - let b = fetch_bytes(net, store, c, provider, CHUNK_BYTES).await?; + for (i, c) in m.chunks.iter().enumerate() { + let sealed = fetch_bytes(net, store, c, provider, CHUNK_BYTES + TAG_BYTES).await?; + let b = open_chunk(&key, i as u64, &sealed)?; sha.update(&b); out.write_all(&b).await?; } @@ -685,6 +1055,32 @@ pub async fn fetch_world(net: &Libp2pNetwork, Ok(m) } +// ── status records ────────────────────────────────────────────────────── + +/// What a worker gossips about each server it holds, every sweep. The +/// gossipsub message is signed by the worker's peer key (strict validation), +/// so a watcher trusts a record only when its source is the rental's worker. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct StatusRecord { + pub t: String, + pub v: u32, + pub id: String, + pub worker: String, + pub game: Game, + pub address: String, + /// The container is up, per the worker's runtime. + pub running: bool, + pub expires_at: u64, + #[serde(default)] + pub stopped_at: Option, + /// Latest backup: sealed to the buyer, so the hash is safe to gossip. + #[serde(default)] + pub backup: Option, + pub ts: u64, +} + +pub const STATUS_KIND: &str = "gs_status"; + // ── worker ────────────────────────────────────────────────────────────── pub struct Worker { @@ -692,28 +1088,36 @@ pub struct Worker { store: ChunkStore, cfg: crate::config::GameserverConfig, host: String, + state_dir: PathBuf, state_path: PathBuf, + rt: Arc, servers: Mutex>, - /// Servers a start, stop or reap is working on right now. + /// Servers a start, stop, backup or reap is working on right now. busy: Mutex>, } impl Worker { pub fn new(net: Arc, store: ChunkStore, cfg: crate::config::GameserverConfig, host: String, state_dir: &Path) -> Arc { + Self::with_runtime(net, store, cfg, host, state_dir, Arc::new(Docker)) + } + + pub fn with_runtime(net: Arc, store: ChunkStore, cfg: crate::config::GameserverConfig, host: String, state_dir: &Path, rt: Arc) -> Arc { let state_path = state_dir.join("servers.json"); let servers: HashMap = std::fs::read(&state_path).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default(); - Arc::new(Self { net, store, cfg, host, state_path, servers: Mutex::new(servers), busy: Mutex::new(HashSet::new()) }) + Arc::new(Self { net, store, cfg, host, state_dir: state_dir.to_path_buf(), state_path, rt, servers: Mutex::new(servers), busy: Mutex::new(HashSet::new()) }) + } + + pub fn net(&self) -> &Libp2pNetwork { + &self.net } - fn snapshot(&self) -> HashMap { + pub fn snapshot(&self) -> HashMap { self.servers.lock().map(|g| g.clone()).unwrap_or_default() } fn save(&self) { let snap = self.snapshot(); - if let Some(dir) = self.state_path.parent() { - let _ = std::fs::create_dir_all(dir); - } + let _ = std::fs::create_dir_all(&self.state_dir); let tmp = self.state_path.with_extension("tmp"); if std::fs::write(&tmp, serde_json::to_vec_pretty(&snap).unwrap_or_default()).is_ok() { let _ = std::fs::rename(&tmp, &self.state_path); @@ -729,6 +1133,10 @@ impl Worker { } } + fn scratch(&self, what: &str, id: &str) -> PathBuf { + self.state_dir.join(format!("{what}-{id}.tar")) + } + pub fn accepts_start(&self, offer: &JobOffer) -> bool { let Some(spec) = offer.spec_inline.as_ref().and_then(|v| serde_json::from_value::(v.clone()).ok()) else { return false; @@ -757,20 +1165,25 @@ impl Worker { async fn start(&self, job: &AcceptedJob) -> Result { let spec: StartSpec = serde_json::from_value(job.accept.spec_inline.clone().context("accept lacks spec_inline")?)?; - spec.validate().map_err(|e| anyhow::anyhow!(e))?; + spec.validate().map_err(|e| anyhow!(e))?; + // The password and the restore key come sealed, after the accept. + let raw = crate::handoff::receive(&self.net, WORKLOAD_START, &job.accept, &spec.client_pubkey, HANDOFF_WAIT).await.context("receive the sealed secrets")?; + let secrets: StartSecrets = serde_json::from_slice(&raw).context("sealed secrets are not JSON")?; + drop(raw); + secrets.validate(&spec).map_err(|e| anyhow!(e))?; let id = server_id(&job.accept.job_id); // Reserve RAM and ports under the lock, so two starts cannot both take the last slot. let (state, plan) = { - let mut servers = self.servers.lock().map_err(|_| anyhow::anyhow!("state lock poisoned"))?; + let mut servers = self.servers.lock().map_err(|_| anyhow!("state lock poisoned"))?; if servers.contains_key(&id) { bail!("server {id} already exists"); } - has_capacity(&self.cfg, crate::text::local_ram_mb(), &servers, &spec).map_err(|e| anyhow::anyhow!(e))?; + has_capacity(&self.cfg, crate::text::local_ram_mb(), &servers, &spec).map_err(|e| anyhow!(e))?; let used: HashSet = servers.values().filter(|s| s.running()).flat_map(|s| s.ports()).collect(); let port = allocate_ports(self.cfg.port_min, self.cfg.port_max, spec.game.ports().len(), &used, port_bindable).context("no free port in the range")?; let started = now_secs(); let expires = expires_at(started, spec.hours); - let plan = ContainerPlan::new(&id, &spec, port, expires); + let plan = ContainerPlan::new(&id, &spec, &secrets, port, started, expires); let state = ServerState { id: id.clone(), game: spec.game, @@ -783,6 +1196,8 @@ impl Worker { expires_at: expires, stop_hash: spec.stop_hash.clone(), buyer: job.accept.buyer_peer_id.clone(), + backup_pubkey: spec.backup_pubkey.clone(), + backup_every_secs: u64::from(spec.backup_mins) * 60, stopped_at: None, backup: None, backup_error: None, @@ -791,11 +1206,10 @@ impl Worker { (state, plan) }; self.claim(&id); - let res = self.create_and_start(&plan, &spec, &job.accept.buyer_peer_id).await; + let res = self.create_and_start(&id, &plan, &spec, &secrets, &job.accept.buyer_peer_id).await; self.release(&id); if let Err(e) = res { - let _ = docker(&["rm", "-f", &plan.name], Duration::from_secs(60)).await; - let _ = docker(&["volume", "rm", &plan.volume], Duration::from_secs(60)).await; + self.rt.remove(&plan.name, &plan.volume).await; if let Ok(mut s) = self.servers.lock() { s.remove(&id); } @@ -807,35 +1221,60 @@ impl Worker { "v": 1, "ok": true, "id": id, "game": spec.game.name(), "address": state.address, "ports": spec.game.ports().iter().enumerate().map(|(i, (p, proto))| json!({"host": plan.host_port + i as u16, "container": p, "proto": proto.as_str()})).collect::>(), "expires_at": state.expires_at, "image": spec.game.image(), "restored": spec.world, + "backup_mins": spec.backup_mins, })) } - async fn create_and_start(&self, plan: &ContainerPlan, spec: &StartSpec, buyer: &str) -> Result<()> { - let argv = plan.create_argv(); - let args: Vec<&str> = argv.iter().map(String::as_str).collect(); - docker(&args, Duration::from_secs(1800)).await?; + async fn create_and_start(&self, id: &str, plan: &ContainerPlan, spec: &StartSpec, secrets: &StartSecrets, buyer: &str) -> Result<()> { + self.rt.create(plan).await?; if let Some(world) = &spec.world { - // `docker cp - :/` extracts the tar (entries `data/...`) into the volume. - let mut child = tokio::process::Command::new("docker") - .args(["cp", "-", &format!("{}:/", plan.name)]) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .context("spawn docker cp")?; - let stdin = child.stdin.take().context("docker cp stdin")?; - let m = fetch_world(&self.net, &self.store, world, buyer, stdin).await.context("restore world")?; - if m.game != spec.game { - bail!("world is a {} world, not {}", m.game.name(), spec.game.name()); + let key = secrets.world_key()?.context("world key missing")?; + let tmp = self.scratch("restore", id); + let _ = tokio::fs::create_dir_all(&self.state_dir).await; + let res = async { + let f = tokio::fs::File::create(&tmp).await?; + let m = fetch_world(&self.net, &self.store, world, buyer, |m| { + if m.enc.epk.is_some() { + bail!("a restore world must be encrypted under a handed-off key"); + } + Ok(key) + }, f) + .await + .context("restore world")?; + if m.game != spec.game { + bail!("world is a {} world, not {}", m.game.name(), spec.game.name()); + } + self.rt.restore(plan, &tmp).await } - let out = child.wait_with_output().await?; - if !out.status.success() { - bail!("docker cp into {}: {}", plan.name, String::from_utf8_lossy(&out.stderr).trim()); + .await; + let _ = tokio::fs::remove_file(&tmp).await; + res?; + } + self.rt.start(&plan.name).await + } + + /// Back the world up now (live if the server runs), seal it to the + /// buyer, and record it as the latest backup. + pub async fn backup_now(&self, id: &str) -> Result { + let s = self.snapshot().get(id).cloned().with_context(|| format!("no server {id}"))?; + if !self.claim(id) { + bail!("server {id} is busy"); + } + let res = self.backup(&s).await; + self.release(id); + if let Ok(mut servers) = self.servers.lock() + && let Some(e) = servers.get_mut(id) + { + match &res { + Ok(b) => { + e.backup = Some(b.clone()); + e.backup_error = None; + } + Err(err) => e.backup_error = Some(format!("{err:#}")), } } - docker(&["start", &plan.name], Duration::from_secs(120)).await?; - Ok(()) + self.save(); + res } /// Stop a server, back its world up into the store, remove the container @@ -849,11 +1288,11 @@ impl Worker { bail!("server {id} is busy"); } let res = async { - let _ = docker(&["stop", "-t", "60", &s.container], Duration::from_secs(120)).await; + // Already gone when it stopped itself at expiry (`--rm`). + let _ = self.rt.stop(&s.container).await; let backup = self.backup(&s).await; if backup.is_ok() { - let _ = docker(&["rm", "-f", &s.container], Duration::from_secs(60)).await; - let _ = docker(&["volume", "rm", &s.volume], Duration::from_secs(60)).await; + self.rt.remove(&s.container, &s.volume).await; } backup } @@ -873,28 +1312,33 @@ impl Worker { } async fn backup(&self, s: &ServerState) -> Result { - let mut child = tokio::process::Command::new("docker") - .args(["cp", &format!("{}:{}", s.container, s.game.data_dir()), "-"]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .context("spawn docker cp")?; - let stdout = child.stdout.take().context("docker cp stdout")?; - let m = chunk_into_store(&self.store, Some(&self.net), s.game, stdout).await?; - let out = child.wait_with_output().await?; - if !out.status.success() { - bail!("docker cp out of {}: {}", s.container, String::from_utf8_lossy(&out.stderr).trim()); + if !is_hex(&s.backup_pubkey, 64) { + bail!("server {} has no backup key", s.id); + } + if self.rt.running(&s.container).await { + self.rt.flush(&s.container, s.game).await; + } + let _ = tokio::fs::create_dir_all(&self.state_dir).await; + let tmp = self.scratch("backup", &s.id); + let res = async { + self.rt.tar_world(&s.volume, s.game, &tmp).await?; + let e = crypto::keypair(); + let key = backup_key(&e.secret, &s.backup_pubkey, &e.public_hex, &s.backup_pubkey)?; + let f = tokio::fs::File::open(&tmp).await?; + let m = chunk_into_store(&self.store, Some(&self.net), s.game, f, &key, WorldEnc { scheme: ENC_SCHEME.into(), epk: Some(e.public_hex.clone()) }).await?; + let manifest = put_manifest(&self.store, &self.net, &m).await?; + Ok::<_, anyhow::Error>(Backup { manifest, bytes: m.bytes, sha256: m.sha256, at: now_secs() }) } - let manifest = put_manifest(&self.store, &self.net, &m).await?; - info!(id = %s.id, %manifest, bytes = m.bytes, "gameserver: world backed up"); - Ok(Backup { manifest, bytes: m.bytes, sha256: m.sha256 }) + .await; + let _ = tokio::fs::remove_file(&tmp).await; + let b = res?; + info!(id = %s.id, manifest = %b.manifest, bytes = b.bytes, "gameserver: world backed up (sealed to the buyer)"); + Ok(b) } async fn stop(&self, job: &AcceptedJob) -> Result { let spec: StopSpec = serde_json::from_value(job.accept.spec_inline.clone().context("accept lacks spec_inline")?)?; - spec.validate().map_err(|e| anyhow::anyhow!(e))?; + spec.validate().map_err(|e| anyhow!(e))?; let s = self.snapshot().get(&spec.id).cloned().context("unknown server")?; if s.stop_hash != sha256_ref(spec.token.as_bytes()) { bail!("stop token does not match"); @@ -903,15 +1347,34 @@ impl Worker { Ok(json!({"v": 1, "ok": true, "id": spec.id, "game": s.game.name(), "backup": backup})) } - /// Stop servers whose reservation ended; forget old stopped ones. + /// Kill labelled containers whose reservation passed. Runs at boot (the + /// worker may have been offline at expiry) and on every sweep. + pub async fn sweep_expired(&self) -> Vec { + let Ok(labelled) = self.rt.labelled().await else { return Vec::new() }; + let dead = expired_containers(&labelled, now_secs()); + for name in &dead { + warn!(container = %name, "gameserver: reservation passed; killing"); + self.rt.kill(name).await; + } + dead + } + + /// Kill expired containers, stop + back up servers whose reservation + /// ended, take due live backups, forget old stopped ones. pub async fn reap_once(&self) { let now = now_secs(); + self.sweep_expired().await; for id in due_for_stop(&self.snapshot(), now) { info!(%id, "gameserver: reservation ended, stopping"); if let Err(e) = self.stop_and_backup(&id).await { warn!(%id, err = %format!("{e:#}"), "gameserver: stop at expiry failed"); } } + for id in due_for_backup(&self.snapshot(), now) { + if let Err(e) = self.backup_now(&id).await { + warn!(%id, err = %format!("{e:#}"), "gameserver: live backup failed"); + } + } let mut changed = false; if let Ok(mut s) = self.servers.lock() { let before = s.len(); @@ -923,6 +1386,38 @@ impl Worker { } } + /// The status record for one server. + pub async fn status(&self, s: &ServerState) -> StatusRecord { + StatusRecord { + t: STATUS_KIND.into(), + v: 1, + id: s.id.clone(), + worker: self.net.peer_id().to_base58(), + game: s.game, + address: s.address.clone(), + running: s.running() && self.rt.running(&s.container).await, + expires_at: s.expires_at, + stopped_at: s.stopped_at, + backup: s.backup.clone(), + ts: now_secs(), + } + } + + /// Gossip a status record for every server held (stopped ones for a day, + /// so a watcher learns the final backup). + pub async fn publish_status(&self) { + let now = now_secs(); + for s in self.snapshot().values() { + if s.stopped_at.is_some_and(|t| now.saturating_sub(t) > 86_400) { + continue; + } + let r = self.status(s).await; + if let Ok(b) = serde_json::to_vec(&r) { + let _ = self.net.publish(STATUS_TOPIC, b).await; + } + } + } + async fn publish_result(&self, v: &Value) -> Result { let h = self.store.put(&serde_json::to_vec(v)?).await?; self.net.announce(&h).await?; @@ -956,8 +1451,42 @@ fn spawn_runner(worker: Arc, slots: Arc, stop: bool) -> mpsc: tx } -/// Supervisor entry: pull images, then subscribe to start and stop offers and -/// run the expiry reaper. Returns without subscribing when the node cannot host. +/// Run the sweep (expiry, live backups) and gossip status every `every`. +pub fn spawn_reaper(worker: Arc, every: Duration) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let _ = worker.net.subscribe(STATUS_TOPIC).await; + loop { + worker.reap_once().await; + worker.publish_status().await; + tokio::time::sleep(every).await; + } + }) +} + +/// Subscribe a worker to start and stop offers. +pub fn subscribe(worker: Arc, tags: Vec, pricing: Option) { + let slots = Arc::new(Semaphore::new(JOB_SLOTS)); + let load = WorkerLoad::new(slots.clone(), JOB_SLOTS); + let w = worker.clone(); + let start_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_start(o)); + let w = worker.clone(); + let stop_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_stop(o)); + crate::dispatch::run_worker_subscriber_with_load( + worker.net.clone(), + WORKLOAD_START.into(), + tags.clone(), + Some(spawn_runner(worker.clone(), slots.clone(), false)), + Some(start_filter), + pricing, + Some(load.clone()), + ); + // Stopping is part of the rent: no price floor. + crate::dispatch::run_worker_subscriber_with_load(worker.net.clone(), WORKLOAD_STOP.into(), tags, Some(spawn_runner(worker.clone(), slots, true)), Some(stop_filter), None, Some(load)); +} + +/// Supervisor entry: kill expired leftovers, pull images, then subscribe to +/// start and stop offers and run the sweep. Returns without subscribing when +/// the node cannot host. pub async fn run_worker(net: Arc, store: ChunkStore, config: crate::Config, state_dir: PathBuf) { if !docker_sync_ok(&["info", "--format", "{{.ServerVersion}}"]) { info!("gameserver: docker is not usable here; not serving game servers"); @@ -968,14 +1497,10 @@ pub async fn run_worker(net: Arc, store: ChunkStore, config: crat return; }; let worker = Worker::new(net.clone(), store, config.gameserver.clone(), host, &state_dir); + // Boot watchdog: anything whose reservation passed while we were down dies now. + worker.sweep_expired().await; // Enforce expiries even before the images finish pulling. - let reaper = worker.clone(); - tokio::spawn(async move { - loop { - reaper.reap_once().await; - tokio::time::sleep(REAP_EVERY).await; - } - }); + spawn_reaper(worker.clone(), REAP_EVERY); ensure_images(&config.gameserver).await; let ready = probe(&config); if !ready.ready() { @@ -986,29 +1511,255 @@ pub async fn run_worker(net: Arc, store: ChunkStore, config: crat *g = Some((Instant::now(), ready.clone())); } let tags = crate::capabilities::tags_from_config(&config); - let slots = Arc::new(Semaphore::new(JOB_SLOTS)); - let load = WorkerLoad::new(slots.clone(), JOB_SLOTS); - let w = worker.clone(); - let start_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_start(o)); - let w = worker.clone(); - let stop_filter: crate::dispatch::OfferFilter = Arc::new(move |o| w.accepts_stop(o)); info!(games = ?ready.games, host = ?ready.host, "gameserver: serving"); - crate::dispatch::run_worker_subscriber_with_load( - net.clone(), - WORKLOAD_START.into(), - tags.clone(), - Some(spawn_runner(worker.clone(), slots.clone(), false)), - Some(start_filter), - Some(config.pricing.clone()), - Some(load.clone()), - ); - // Stopping is part of the rent: no price floor. - crate::dispatch::run_worker_subscriber_with_load(net, WORKLOAD_STOP.into(), tags, Some(spawn_runner(worker, slots, true)), Some(stop_filter), None, Some(load)); + subscribe(worker, tags, Some(config.pricing.clone())); +} + +// ── health probes ─────────────────────────────────────────────────────── + +/// One probe of a server's game port. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ProbeResult { + pub ts: u64, + pub ok: bool, + /// `minecraft-status`, `a2s-info` or `udp-reachable`. + pub method: String, + pub rtt_ms: u64, + pub detail: String, +} + +fn put_varint(out: &mut Vec, mut v: u32) { + loop { + let mut b = (v & 0x7f) as u8; + v >>= 7; + if v != 0 { + b |= 0x80; + } + out.push(b); + if v == 0 { + break; + } + } +} + +async fn read_varint(r: &mut R) -> Result { + let mut v = 0u32; + for i in 0..5 { + let b = r.read_u8().await?; + v |= u32::from(b & 0x7f) << (7 * i); + if b & 0x80 == 0 { + return Ok(v); + } + } + bail!("varint too long") +} + +fn read_varint_slice(b: &[u8], at: &mut usize) -> Result { + let mut v = 0u32; + for i in 0..5 { + let x = *b.get(*at).context("truncated varint")?; + *at += 1; + v |= u32::from(x & 0x7f) << (7 * i); + if x & 0x80 == 0 { + return Ok(v); + } + } + bail!("varint too long") +} + +/// Minecraft Java "server list ping": handshake (next state 1) + status request. +pub fn minecraft_status_request(host: &str, port: u16) -> Vec { + let mut body = Vec::new(); + put_varint(&mut body, 0x00); + put_varint(&mut body, 767); + put_varint(&mut body, host.len() as u32); + body.extend_from_slice(host.as_bytes()); + body.extend_from_slice(&port.to_be_bytes()); + put_varint(&mut body, 1); + let mut pkt = Vec::new(); + put_varint(&mut pkt, body.len() as u32); + pkt.extend(body); + pkt.extend([0x01, 0x00]); + pkt +} + +/// A status response packet carrying `json` (what a server answers). +pub fn minecraft_status_response(json: &str) -> Vec { + let mut body = Vec::new(); + put_varint(&mut body, 0x00); + put_varint(&mut body, json.len() as u32); + body.extend_from_slice(json.as_bytes()); + let mut pkt = Vec::new(); + put_varint(&mut pkt, body.len() as u32); + pkt.extend(body); + pkt +} + +/// Parse a status response body (after the length prefix) into its JSON. +pub fn parse_minecraft_status(body: &[u8]) -> Result { + let mut at = 0; + if read_varint_slice(body, &mut at)? != 0 { + bail!("not a status response"); + } + let n = read_varint_slice(body, &mut at)? as usize; + let s = body.get(at..at + n).context("truncated status")?; + let v: Value = serde_json::from_slice(s).context("status is not JSON")?; + if !v.is_object() { + bail!("status is not an object"); + } + Ok(v) +} + +async fn minecraft_status(addr: SocketAddr, host: &str, timeout: Duration) -> Result { + tokio::time::timeout(timeout, async { + let mut s = tokio::net::TcpStream::connect(addr).await?; + s.write_all(&minecraft_status_request(host, addr.port())).await?; + let len = read_varint(&mut s).await? as usize; + if len == 0 || len > 256 * 1024 { + bail!("status packet length {len}"); + } + let mut body = vec![0u8; len]; + s.read_exact(&mut body).await?; + let v = parse_minecraft_status(&body)?; + let online = v["players"]["online"].as_u64().unwrap_or(0); + let version = v["version"]["name"].as_str().unwrap_or("?").to_string(); + Ok(format!("{version}, {online} online")) + }) + .await + .context("timed out")? +} + +/// Steam A2S_INFO request (Valheim answers on its query port, game port + 1). +pub const A2S_INFO: &[u8] = b"\xFF\xFF\xFF\xFFTSource Engine Query\0"; + +async fn a2s_info(addr: SocketAddr, timeout: Duration) -> Result { + tokio::time::timeout(timeout, async { + let bind: SocketAddr = if addr.is_ipv4() { "0.0.0.0:0".parse()? } else { "[::]:0".parse()? }; + let sock = tokio::net::UdpSocket::bind(bind).await?; + sock.connect(addr).await?; + let mut q = A2S_INFO.to_vec(); + let mut buf = vec![0u8; 1400]; + for _ in 0..2 { + sock.send(&q).await?; + let n = sock.recv(&mut buf).await?; + let r = &buf[..n]; + if n < 5 || r[..4] != [0xFF; 4] { + bail!("not an A2S answer"); + } + match r[4] { + // Challenge: repeat the query with it appended. + 0x41 if n >= 9 => { + q = A2S_INFO.to_vec(); + q.extend_from_slice(&r[5..9]); + } + 0x49 => return Ok(format!("a2s info ({n} bytes)")), + other => bail!("unexpected A2S answer 0x{other:02x}"), + } + } + bail!("no A2S info after the challenge") + }) + .await + .context("timed out")? +} + +/// Factorio has no cheap unauthenticated query. A connected UDP socket learns +/// of a closed port from the ICMP "port unreachable" (ECONNREFUSED on recv); +/// silence means "not refused". Weak on its own, so a watcher also requires +/// a fresh worker status record saying the container runs. +async fn udp_reachable(addr: SocketAddr, timeout: Duration) -> Result { + let bind: SocketAddr = if addr.is_ipv4() { "0.0.0.0:0".parse()? } else { "[::]:0".parse()? }; + let sock = tokio::net::UdpSocket::bind(bind).await?; + sock.connect(addr).await?; + sock.send(&[0u8]).await?; + let mut buf = [0u8; 64]; + match tokio::time::timeout(timeout, sock.recv(&mut buf)).await { + Ok(Err(e)) if e.kind() == std::io::ErrorKind::ConnectionRefused => bail!("port unreachable"), + Ok(Err(e)) => bail!("{e}"), + Ok(Ok(_)) => Ok("answered".into()), + Err(_) => Ok("not refused".into()), + } +} + +fn split_address(address: &str) -> Option<(String, u16)> { + let (h, p) = address.rsplit_once(':')?; + Some((h.trim_start_matches('[').trim_end_matches(']').to_string(), p.parse().ok()?)) +} + +/// Probe a server's game port the way its players' clients would. +pub async fn probe_server(game: Game, address: &str, timeout: Duration) -> ProbeResult { + let started = Instant::now(); + let method = match game { + Game::Minecraft => "minecraft-status", + Game::Valheim => "a2s-info", + Game::Factorio => "udp-reachable", + }; + let res: Result = async { + let (host, port) = split_address(address).context("address is not host:port")?; + let mut addrs = tokio::net::lookup_host((host.as_str(), port)).await?; + let addr = addrs.next().context("address does not resolve")?; + match game { + Game::Minecraft => minecraft_status(addr, &host, timeout).await, + Game::Valheim => a2s_info(SocketAddr::new(addr.ip(), port.checked_add(1).context("port")?), timeout).await, + Game::Factorio => udp_reachable(addr, timeout).await, + } + } + .await; + let (ok, detail) = match res { + Ok(d) => (true, d), + Err(e) => (false, format!("{e:#}")), + }; + ProbeResult { ts: now_secs(), ok, method: method.into(), rtt_ms: started.elapsed().as_millis() as u64, detail } +} + +/// Decides when a server has been down long enough to fail over. +#[derive(Clone, Debug)] +pub struct HealthTracker { + fail_after_secs: u64, + first_fail: Option, + pub last_ok: Option, +} + +impl HealthTracker { + pub fn new(fail_after: Duration) -> Self { + Self { fail_after_secs: fail_after.as_secs().max(1), first_fail: None, last_ok: None } + } + /// Record one check; `true` once checks have failed without a pass for + /// `fail_after`. + pub fn observe(&mut self, now: u64, ok: bool) -> bool { + if ok { + self.first_fail = None; + self.last_ok = Some(now); + return false; + } + let since = *self.first_fail.get_or_insert(now); + now.saturating_sub(since) >= self.fail_after_secs + } + pub fn reset(&mut self) { + self.first_fail = None; + } +} + +/// Is the server healthy, given a probe and the worker's latest status +/// record (and when it arrived)? A record that says the container is down +/// always fails it; for Factorio, whose probe is weak, a missing or stale +/// record fails it too. +pub fn judge(game: Game, probe: &ProbeResult, status: Option<(&StatusRecord, u64)>, now: u64) -> bool { + if let Some((s, _)) = status + && (!s.running || s.stopped_at.is_some()) + { + return false; + } + if game == Game::Factorio { + let fresh = status.is_some_and(|(s, seen)| s.running && now.saturating_sub(seen) <= 180); + return probe.ok && fresh; + } + probe.ok } // ── buyer ─────────────────────────────────────────────────────────────── -/// What the buyer keeps about a server it rented (`gameserver ls`). +/// What the buyer keeps about a server it rented (`gameserver ls`). The file +/// is mode 0600 and never leaves this machine. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct Rental { pub id: String, @@ -1025,6 +1776,28 @@ pub struct Rental { pub stopped_at: Option, #[serde(default)] pub backup: Option, + // What a failover needs to start the same server elsewhere. + #[serde(default)] + pub ram_gb: u32, + #[serde(default)] + pub cpus: f64, + #[serde(default)] + pub accept_eula: bool, + #[serde(default)] + pub password: Option, + #[serde(default)] + pub server_name: Option, + #[serde(default)] + pub backup_mins: u32, + /// The newest backup mirrored to `worlds/.tar` on this machine. + #[serde(default)] + pub latest_backup: Option, + /// The rental this one replaced after a failover. + #[serde(default)] + pub replaces: Option, + /// The rental that replaced this one after a failover. + #[serde(default)] + pub superseded_by: Option, } pub fn buyer_dir() -> PathBuf { @@ -1035,41 +1808,89 @@ pub fn load_rentals(dir: &Path) -> Vec { std::fs::read(dir.join("rentals.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default() } -pub fn save_rentals(dir: &Path, rentals: &[Rental]) -> Result<()> { - std::fs::create_dir_all(dir)?; - let p = dir.join("rentals.json"); - std::fs::write(&p, serde_json::to_vec_pretty(rentals)?)?; +fn write_private(p: &Path, bytes: &[u8]) -> Result<()> { + if let Some(d) = p.parent() { + std::fs::create_dir_all(d)?; + } + let tmp = p.with_extension("tmp"); + std::fs::write(&tmp, bytes)?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let _ = std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o600)); + std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600))?; } + std::fs::rename(&tmp, p)?; Ok(()) } +pub fn save_rentals(dir: &Path, rentals: &[Rental]) -> Result<()> { + write_private(&dir.join("rentals.json"), &serde_json::to_vec_pretty(rentals)?) +} + +fn upsert_rental(dir: &Path, r: &Rental) -> Result<()> { + let mut all = load_rentals(dir); + match all.iter_mut().find(|x| x.id == r.id) { + Some(x) => *x = r.clone(), + None => all.push(r.clone()), + } + save_rentals(dir, &all) +} + +/// The buyer's long-term backup key (`

/backup.key`, 0600), created on +/// first use. Every worker seals its backups to the public half. +pub fn backup_keypair(dir: &Path) -> Result { + let p = dir.join("backup.key"); + if let Ok(h) = std::fs::read_to_string(&p) { + let secret = crypto::secret_from_hex(h.trim()).context("backup.key is not a 32-byte hex key")?; + let public_hex = hex::encode(x25519_dalek::PublicKey::from(&secret).as_bytes()); + return Ok(crypto::KeyPair { secret, public_hex }); + } + let kp = crypto::keypair(); + write_private(&p, hex::encode(kp.secret.to_bytes()).as_bytes())?; + Ok(kp) +} + pub struct StartRequest { pub game: Game, pub hours: u32, pub ram_gb: u32, pub cpus: f64, pub accept_eula: bool, - /// A manifest hash from an earlier stop, or a `.tar` laid out like a backup. + /// A manifest hash from an earlier backup, or a `.tar` laid out like a backup. pub world: Option, pub password: Option, pub server_name: Option, + /// Minutes between live backups (0 = only at stop/expiry). + pub backup_mins: u32, pub max_price_usd: f64, pub workers: Vec, + /// Never use these workers (a failover excludes the one that failed). + pub exclude: Vec, pub timeout: Duration, } -async fn auction(net: Arc, workload: &str, spec: Value, caps: Vec, workers: Vec, max_price: f64, timeout: Duration) -> Result<(crate::buyer::AuctionOutcome, Value)> { - let a = crate::JobAuction::new(workload, spec) +async fn auction( + net: Arc, + workload: &str, + spec: Value, + caps: Vec, + workers: Vec, + exclude: Vec, + max_price: f64, + timeout: Duration, + notify: Option>, +) -> Result<(crate::buyer::AuctionOutcome, Value)> { + let mut a = crate::JobAuction::new(workload, spec) .with_required_capabilities(caps) .with_allowed_bidders(workers) + .with_excluded_bidders(exclude) .with_max_price_usd(max_price) .with_bid_window(Duration::from_secs(4)) .with_deadline(timeout) .with_receipt_timeout(timeout); + if let Some(tx) = notify { + a = a.with_accept_notify(tx); + } let o = crate::run_auction(net.clone(), a).await?; if o.receipt.status != JobStatus::Completed { bail!("worker {} reported {:?}", o.accepted_bid.bidder_peer_id, o.receipt.status); @@ -1083,54 +1904,76 @@ async fn auction(net: Arc, workload: &str, spec: Value, caps: Vec Ok((o, v)) } -/// Put a world into the local store for the worker to fetch; returns the manifest hash. -async fn publish_world(store: &ChunkStore, net: &Libp2pNetwork, dir: &Path, game: Game, world: &str) -> Result { +/// Encrypt a world under a fresh key and put it in the local store for the +/// worker to fetch; returns the manifest hash and the key. +async fn publish_world(store: &ChunkStore, net: &Libp2pNetwork, dir: &Path, game: Game, world: &str) -> Result<(String, [u8; 32])> { let (tar, saved_manifest) = if is_hex(world, 64) { (dir.join("worlds").join(format!("{world}.tar")), Some(dir.join("worlds").join(format!("{world}.json")))) } else { (PathBuf::from(world), None) }; let f = tokio::fs::File::open(&tar).await.with_context(|| format!("open world {}", tar.display()))?; - let m = chunk_into_store(store, Some(net), game, f).await?; - if let Some(p) = saved_manifest { - let want = WorldManifest::from_bytes(&std::fs::read(&p).with_context(|| format!("read {}", p.display()))?)?; - if want != m { + let key = random_key(); + let m = chunk_into_store(store, Some(net), game, f, &key, WorldEnc::handed_off()).await?; + if let Some(p) = saved_manifest + && let Ok(b) = std::fs::read(&p) + { + // The manifest saved beside the tar names its game and plaintext sha256. + let saved: Value = serde_json::from_slice(&b).with_context(|| format!("read {}", p.display()))?; + if saved["sha256"].as_str() != Some(m.sha256.as_str()) { bail!("{} does not match its manifest", tar.display()); } - if want.game != game { - bail!("that world is a {} world", want.game.name()); + if saved["game"].as_str().is_some_and(|g| g != game.name()) { + bail!("that world is a {} world", saved["game"].as_str().unwrap_or("?")); } } - put_manifest(store, net, &m).await + Ok((put_manifest(store, net, &m).await?, key)) } -/// Rent a server: auction it, wait for the address, remember the rental. -pub async fn start(req: StartRequest) -> Result { - let token = { - let b: [u8; 32] = rand::random(); - hex::encode(b) - }; - let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; - let dir = buyer_dir(); - let world = match &req.world { - Some(w) => Some(publish_world(&sup.store, &sup.libp2p, &dir, req.game, w).await?), - None => None, +/// Rent a server on `net` (a booted buyer node), remembering it under `dir`. +pub async fn start_with(net: Arc, store: &ChunkStore, dir: &Path, req: StartRequest) -> Result { + let token = hex::encode(rand::random::<[u8; 32]>()); + let backup = backup_keypair(dir)?; + let (world, world_key) = match &req.world { + Some(w) => { + let (h, k) = publish_world(store, &net, dir, req.game, w).await?; + (Some(h), Some(hex::encode(k))) + } + None => (None, None), }; + let kp = crate::handoff::keypair(); let spec = StartSpec { - v: 1, + v: SPEC_VERSION, game: req.game, hours: req.hours, ram_gb: req.ram_gb, cpus: req.cpus, accept_eula: req.accept_eula, stop_hash: sha256_ref(token.as_bytes()), + client_pubkey: kp.public_hex.clone(), + backup_pubkey: backup.public_hex.clone(), + backup_mins: req.backup_mins, world, - password: req.password.clone(), server_name: req.server_name.clone(), }; - spec.validate().map_err(|e| anyhow::anyhow!(e))?; - tokio::time::sleep(Duration::from_secs(5)).await; - let (o, v) = auction(sup.libp2p.clone(), WORKLOAD_START, serde_json::to_value(&spec)?, vec![ROLE_TAG.into(), req.game.tag()], req.workers.clone(), req.max_price_usd, req.timeout).await?; + spec.validate().map_err(|e| anyhow!(e))?; + let secrets = StartSecrets { v: 1, password: req.password.clone(), world_key }; + secrets.validate(&spec).map_err(|e| anyhow!(e))?; + let mut handoff = crate::handoff::BuyerHandoff::spawn(net.clone(), WORKLOAD_START, kp, serde_json::to_vec(&secrets)?)?; + let res = auction( + net.clone(), + WORKLOAD_START, + serde_json::to_value(&spec)?, + vec![ROLE_TAG.into(), req.game.tag()], + req.workers.clone(), + req.exclude.clone(), + req.max_price_usd, + req.timeout, + Some(handoff.notify()?), + ) + .await; + handoff.finish(); + let (o, v) = res?; let rental = Rental { id: v["id"].as_str().context("result has no id")?.to_string(), game: req.game, @@ -1143,65 +1986,278 @@ pub async fn start(req: StartRequest) -> Result { price_usd: o.accepted_bid.price_usd, stopped_at: None, backup: None, + ram_gb: req.ram_gb, + cpus: req.cpus, + accept_eula: req.accept_eula, + password: req.password, + server_name: req.server_name, + backup_mins: req.backup_mins, + latest_backup: None, + replaces: None, + superseded_by: None, }; - let mut all = load_rentals(&dir); - all.push(rental.clone()); - save_rentals(&dir, &all)?; + upsert_rental(dir, &rental)?; Ok(rental) } -/// Stop a rental early (or collect the backup of one that expired) and -/// download its world to `/worlds/.tar`. -pub async fn stop(id: &str, timeout: Duration) -> Result { - let dir = buyer_dir(); - let mut all = load_rentals(&dir); - let idx = all.iter().position(|r| r.id == id).with_context(|| format!("no rental {id} on this machine (see `c0mpute gameserver ls`)"))?; - let r = all[idx].clone(); +/// Rent a server: boot a buyer node, auction it, wait for the address, +/// remember the rental. +pub async fn start(req: StartRequest) -> Result { let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; tokio::time::sleep(Duration::from_secs(5)).await; + start_with(sup.libp2p.clone(), &sup.store, &buyer_dir(), req).await +} + +/// Download, decrypt and keep a backup as `/worlds/.tar` (+ its +/// manifest as `.json`). +pub async fn download_backup(net: &Libp2pNetwork, store: &ChunkStore, dir: &Path, manifest: &str, provider: &str) -> Result { + let key = backup_keypair(dir)?; + let worlds = dir.join("worlds"); + tokio::fs::create_dir_all(&worlds).await?; + let tmp = worlds.join(format!("{manifest}.partial")); + let f = tokio::fs::File::create(&tmp).await?; + let res = fetch_world(net, store, manifest, provider, |m| { + let epk = m.enc.epk.as_deref().context("backup has no one-time key; it was not sealed to us")?; + backup_key(&key.secret, epk, epk, &key.public_hex) + }, f) + .await; + let m = match res { + Ok(m) => m, + Err(e) => { + let _ = tokio::fs::remove_file(&tmp).await; + return Err(e.context("download world backup")); + } + }; + tokio::fs::rename(&tmp, worlds.join(format!("{manifest}.tar"))).await?; + tokio::fs::write(worlds.join(format!("{manifest}.json")), m.to_bytes()).await?; + Ok(m) +} + +/// Stop a rental early (or collect the backup of one that expired) and +/// download its world to `/worlds/.tar`. +pub async fn stop_with(net: Arc, store: &ChunkStore, dir: &Path, id: &str, timeout: Duration) -> Result { + let all = load_rentals(dir); + let r = all.iter().find(|r| r.id == id).cloned().with_context(|| format!("no rental {id} on this machine (see `c0mpute gameserver ls`)"))?; let spec = StopSpec { v: 1, id: r.id.clone(), token: r.token.clone() }; - let (_, v) = auction(sup.libp2p.clone(), WORKLOAD_STOP, serde_json::to_value(&spec)?, vec![ROLE_TAG.into()], vec![r.worker.clone()], 0.0, timeout).await?; + let (_, v) = auction(net.clone(), WORKLOAD_STOP, serde_json::to_value(&spec)?, vec![ROLE_TAG.into()], vec![r.worker.clone()], vec![], 0.0, timeout, None).await?; let backup: Option = serde_json::from_value(v["backup"].clone()).ok().flatten(); let mut updated = r.clone(); updated.stopped_at = Some(now_secs()); if let Some(b) = &backup { - let worlds = dir.join("worlds"); - tokio::fs::create_dir_all(&worlds).await?; - let tmp = worlds.join(format!("{}.partial", b.manifest)); - let f = tokio::fs::File::create(&tmp).await?; - let m = fetch_world(&sup.libp2p, &sup.store, &b.manifest, &r.worker, f).await.context("download world backup")?; - tokio::fs::rename(&tmp, worlds.join(format!("{}.tar", b.manifest))).await?; - tokio::fs::write(worlds.join(format!("{}.json", b.manifest)), m.to_bytes()).await?; + download_backup(&net, store, dir, &b.manifest, &r.worker).await?; updated.backup = Some(b.manifest.clone()); + updated.latest_backup = Some(b.manifest.clone()); } - all[idx] = updated.clone(); - save_rentals(&dir, &all)?; + upsert_rental(dir, &updated)?; Ok(updated) } +pub async fn stop(id: &str, timeout: Duration) -> Result { + let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; + tokio::time::sleep(Duration::from_secs(5)).await; + stop_with(sup.libp2p.clone(), &sup.store, &buyer_dir(), id, timeout).await +} + +/// `gameserver watch` knobs. +#[derive(Clone, Debug)] +pub struct WatchOptions { + pub probe_every: Duration, + pub probe_timeout: Duration, + /// Fail over after the server has failed every check for this long. + pub fail_after: Duration, + pub failover: bool, + pub max_price_usd: f64, + pub workers: Vec, + pub start_timeout: Duration, + /// Return after the first failover instead of watching the replacement. + pub stop_after_failover: bool, +} + +impl Default for WatchOptions { + fn default() -> Self { + Self { + probe_every: Duration::from_secs(30), + probe_timeout: Duration::from_secs(5), + fail_after: Duration::from_secs(5 * 60), + failover: true, + max_price_usd: 1.0, + workers: Vec::new(), + start_timeout: Duration::from_secs(900), + stop_after_failover: false, + } + } +} + +/// How a watch ended. +#[derive(Clone, Debug, PartialEq)] +pub enum WatchEnd { + /// The reservation ended or the server was stopped. + Ended(Rental), + /// It failed over and `stop_after_failover` was set. + FailedOver { from: Rental, to: Rental }, +} + +fn log_probe(dir: &Path, id: &str, p: &ProbeResult) { + let d = dir.join("health"); + let _ = std::fs::create_dir_all(&d); + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(d.join(format!("{id}.jsonl"))) { + use std::io::Write; + let _ = writeln!(f, "{}", serde_json::to_string(p).unwrap_or_default()); + } +} + +/// Read a server's probe log. +pub fn health_log(dir: &Path, id: &str) -> Vec { + std::fs::read_to_string(dir.join("health").join(format!("{id}.jsonl"))).unwrap_or_default().lines().filter_map(|l| serde_json::from_str(l).ok()).collect() +} + +/// Watch a rental: probe its port every `probe_every` (logged to +/// `/health/.jsonl`), mirror each backup its worker announces, and +/// when it has failed every check for `fail_after`, start the same game on +/// another worker from the newest mirrored backup. +pub async fn watch_with(net: Arc, store: &ChunkStore, dir: &Path, id: &str, opts: WatchOptions) -> Result { + net.subscribe(STATUS_TOPIC).await?; + let mut rx = net.messages(); + let mut rental = load_rentals(dir).into_iter().find(|r| r.id == id).with_context(|| format!("no rental {id} on this machine"))?; + let mut health = HealthTracker::new(opts.fail_after); + let mut status: Option<(StatusRecord, u64)> = None; + let mut tick = tokio::time::interval(opts.probe_every); + loop { + if now_secs() >= rental.expires_at || rental.stopped_at.is_some() { + return Ok(WatchEnd::Ended(rental)); + } + tokio::select! { + m = rx.recv() => { + let msg = match m { + Ok(m) => m, + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(_) => bail!("network closed"), + }; + if msg.topic != STATUS_TOPIC || msg.source.map(|p| p.to_base58()).as_deref() != Some(rental.worker.as_str()) { + continue; + } + let Ok(rec) = serde_json::from_slice::(&msg.data) else { continue }; + if rec.t != STATUS_KIND || rec.id != rental.id || rec.worker != rental.worker { + continue; + } + if let Some(b) = &rec.backup + && rental.latest_backup.as_deref() != Some(b.manifest.as_str()) + { + match download_backup(&net, store, dir, &b.manifest, &rental.worker).await { + Ok(_) => { + info!(id = %rental.id, manifest = %b.manifest, "gameserver watch: backup mirrored"); + rental.latest_backup = Some(b.manifest.clone()); + upsert_rental(dir, &rental)?; + } + Err(e) => warn!(id = %rental.id, err = %format!("{e:#}"), "gameserver watch: could not mirror backup"), + } + } + status = Some((rec, now_secs())); + } + _ = tick.tick() => { + let p = probe_server(rental.game, &rental.address, opts.probe_timeout).await; + let now = now_secs(); + let ok = judge(rental.game, &p, status.as_ref().map(|(s, at)| (s, *at)), now); + log_probe(dir, &rental.id, &ProbeResult { ok, ..p.clone() }); + if !health.observe(now, ok) || !opts.failover { + continue; + } + warn!(id = %rental.id, worker = %rental.worker, "gameserver watch: failing checks; failing over"); + match failover(net.clone(), store, dir, &rental, &opts).await { + Ok(next) => { + let from = rental.clone(); + if opts.stop_after_failover { + return Ok(WatchEnd::FailedOver { from, to: next }); + } + info!(from = %from.id, to = %next.id, address = %next.address, "gameserver watch: failed over"); + rental = next; + status = None; + health = HealthTracker::new(opts.fail_after); + } + Err(e) => { + warn!(id = %rental.id, err = %format!("{e:#}"), "gameserver watch: failover failed; retrying next check"); + health.reset(); + } + } + } + } + } +} + +pub async fn watch(id: &str, opts: WatchOptions) -> Result { + let sup = crate::Supervisor::boot_buyer(crate::Config::default()).await?; + tokio::time::sleep(Duration::from_secs(5)).await; + watch_with(sup.libp2p.clone(), &sup.store, &buyer_dir(), id, opts).await +} + +/// Start `old`'s game on a different worker from its newest mirrored backup +/// for the rest of its reservation, and link the two rentals. +pub async fn failover(net: Arc, store: &ChunkStore, dir: &Path, old: &Rental, opts: &WatchOptions) -> Result { + let left = old.expires_at.saturating_sub(now_secs()); + if left == 0 { + bail!("the reservation already ended"); + } + let req = StartRequest { + game: old.game, + hours: left.div_ceil(3600).clamp(1, u64::from(MAX_HOURS)) as u32, + ram_gb: old.ram_gb.max(old.game.min_ram_gb()), + cpus: if old.cpus > 0.0 { old.cpus } else { 2.0 }, + accept_eula: old.accept_eula, + world: old.latest_backup.clone(), + password: old.password.clone(), + server_name: old.server_name.clone(), + backup_mins: if old.backup_mins == 0 { 0 } else { old.backup_mins.clamp(MIN_BACKUP_MINS, MAX_BACKUP_MINS) }, + max_price_usd: opts.max_price_usd, + workers: opts.workers.clone(), + exclude: vec![old.worker.clone()], + timeout: opts.start_timeout, + }; + if req.world.is_none() { + warn!(id = %old.id, "gameserver failover: no backup mirrored yet; the replacement starts a fresh world"); + } + let mut next = start_with(net, store, dir, req).await?; + next.replaces = Some(old.id.clone()); + upsert_rental(dir, &next)?; + let mut prev = old.clone(); + prev.superseded_by = Some(next.id.clone()); + upsert_rental(dir, &prev)?; + Ok(next) +} + #[cfg(test)] mod tests { use super::*; + fn key() -> String { + crypto::keypair().public_hex + } + fn spec(game: Game) -> StartSpec { StartSpec { - v: 1, + v: SPEC_VERSION, game, hours: 4, ram_gb: 4, cpus: 2.0, accept_eula: true, stop_hash: sha256_ref(b"t"), + client_pubkey: key(), + backup_pubkey: key(), + backup_mins: 30, world: None, - password: (game == Game::Valheim).then(|| "hunter22".into()), server_name: None, } } + fn secrets(game: Game) -> StartSecrets { + StartSecrets { v: 1, password: (game == Game::Valheim).then(|| "hunter22".into()), world_key: None } + } + #[test] fn spec_validation() { for g in Game::ALL { assert!(spec(g).validate().is_ok(), "{g:?}"); + assert!(secrets(g).validate(&spec(g)).is_ok(), "{g:?}"); } let bad = |f: &dyn Fn(&mut StartSpec)| { let mut s = spec(Game::Minecraft); @@ -1218,18 +2274,19 @@ mod tests { assert!(bad(&|s| s.cpus = 64.0)); assert!(bad(&|s| s.stop_hash = "sha256:XYZ".into())); assert!(bad(&|s| s.world = Some("../etc".into()))); - assert!(bad(&|s| s.password = Some("hunter22".into())), "only valheim takes a password"); assert!(bad(&|s| s.server_name = Some("$(rm -rf /)".into()))); - assert!(bad(&|s| s.v = 2)); - // Factorio does not need the EULA flag; Valheim needs a password. + assert!(bad(&|s| s.v = 1), "v1 offers put the password in public"); + assert!(bad(&|s| s.client_pubkey = "nope".into())); + assert!(bad(&|s| s.backup_pubkey = String::new())); + assert!(bad(&|s| s.backup_mins = 1)); + assert!(bad(&|s| s.backup_mins = MAX_BACKUP_MINS + 1)); + assert!(!bad(&|s| s.backup_mins = 0), "0 turns live backups off"); + // Factorio does not need the EULA flag. let mut f = spec(Game::Factorio); f.accept_eula = false; assert!(f.validate().is_ok()); let mut v = spec(Game::Valheim); - v.password = Some("abc".into()); - assert!(v.validate().is_err()); v.ram_gb = 2; - v.password = Some("hunter22".into()); assert!(v.validate().is_err(), "valheim needs 4 GB"); // Unknown games never parse. let mut raw = serde_json::to_value(spec(Game::Minecraft)).unwrap(); @@ -1239,6 +2296,23 @@ mod tests { assert_eq!(Game::parse("terraria"), None); } + #[test] + fn secrets_never_ride_in_the_offer() { + // There is no password field in a start spec at all. + let raw = serde_json::to_string(&spec(Game::Valheim)).unwrap(); + assert!(!raw.contains("password") && !raw.contains("hunter22"), "{raw}"); + // Secrets are checked against the spec. + let v = spec(Game::Valheim); + assert!(StartSecrets { v: 1, password: None, world_key: None }.validate(&v).is_err(), "valheim needs one"); + assert!(StartSecrets { v: 1, password: Some("abc".into()), world_key: None }.validate(&v).is_err()); + assert!(StartSecrets { v: 1, password: Some("hunter22".into()), world_key: None }.validate(&spec(Game::Factorio)).is_err()); + let mut w = spec(Game::Minecraft); + w.world = Some("a".repeat(64)); + assert!(StartSecrets { v: 1, password: None, world_key: None }.validate(&w).is_err(), "a restore needs its key"); + assert!(StartSecrets { v: 1, password: None, world_key: Some("b".repeat(64)) }.validate(&w).is_ok()); + assert!(StartSecrets { v: 1, password: None, world_key: Some("b".repeat(64)) }.validate(&spec(Game::Minecraft)).is_err()); + } + #[test] fn stop_spec_validation() { let ok = StopSpec { v: 1, id: server_id("job"), token: "a".repeat(64) }; @@ -1249,9 +2323,9 @@ mod tests { } #[test] - fn docker_argv_has_limits_ports_volume_and_no_shell() { + fn docker_argv_has_limits_ports_volume_expiry_and_no_shell() { let s = spec(Game::Minecraft); - let plan = ContainerPlan::new("gs-0123456789abcdef", &s, 27500, 1_000); + let plan = ContainerPlan::new("gs-0123456789abcdef", &s, &secrets(Game::Minecraft), 27500, 1_000, 1_000 + 4 * 3600); let a = plan.create_argv(); let pair = |k: &str, v: &str| a.windows(2).any(|w| w[0] == k && w[1] == v); assert_eq!(a[0], "create"); @@ -1262,25 +2336,48 @@ mod tests { assert!(pair("--pids-limit", "1024")); assert!(pair("--restart", "no")); assert!(pair("--security-opt", "no-new-privileges")); + assert!(pair("--stop-timeout", "60")); assert!(pair("-p", "27500:25565/tcp")); assert!(pair("-v", "c0mpute-gs-0123456789abcdef:/data")); assert!(pair("-e", "EULA=TRUE")); assert!(pair("-e", "MEMORY=3328M")); - assert!(pair("--label", "c0mpute.gameserver.expires=1000")); - assert_eq!(a.last().unwrap(), Game::Minecraft.image()); + assert!(pair("--label", &format!("c0mpute.gameserver.expires={}", 1_000 + 4 * 3600))); + // Hard expiry without the worker: --rm + --init + timeout around the image's own command. + assert!(a.contains(&"--rm".to_string()) && a.contains(&"--init".to_string())); + assert!(pair("--entrypoint", "timeout")); + let img = a.iter().position(|x| x == Game::Minecraft.image()).unwrap(); + assert_eq!(&a[img + 1..], &["-s", "TERM", "-k", "90s", "14400s", "/image/scripts/start"]); assert!(!a.iter().any(|x| x == "sh" || x == "-c" || x == "--privileged" || x.starts_with("--network=host"))); - // Valheim maps two consecutive UDP ports and passes its password as one argument. - let v = ContainerPlan::new("gs-0123456789abcdef", &spec(Game::Valheim), 27510, 1).create_argv(); + // Valheim maps two consecutive UDP ports; its password comes from the sealed secrets. + let v = ContainerPlan::new("gs-0123456789abcdef", &spec(Game::Valheim), &secrets(Game::Valheim), 27510, 1, 2).create_argv(); assert!(v.windows(2).any(|w| w[0] == "-p" && w[1] == "27510:2456/udp")); assert!(v.windows(2).any(|w| w[0] == "-p" && w[1] == "27511:2457/udp")); assert!(v.windows(2).any(|w| w[0] == "-e" && w[1] == "SERVER_PASS=hunter22")); assert!(v.windows(2).any(|w| w[0] == "-v" && w[1] == "c0mpute-gs-0123456789abcdef:/config")); + assert_eq!(v.last().unwrap(), "/usr/local/sbin/bootstrap"); // Images are pinned by digest. for g in Game::ALL { assert!(g.image().contains("@sha256:"), "{g:?}"); + assert!(!g.command().is_empty()); } } + #[test] + fn tar_argv_reads_the_volume_read_only_with_no_network() { + let a = tar_argv("c0mpute-gs-x", Game::Factorio); + assert!(a.windows(2).any(|w| w[0] == "--network" && w[1] == "none")); + assert!(a.windows(2).any(|w| w[0] == "-v" && w[1] == "c0mpute-gs-x:/factorio:ro")); + assert!(a.windows(2).any(|w| w[0] == "--entrypoint" && w[1] == "tar")); + assert_eq!(a.last().unwrap(), "factorio"); + } + + #[test] + fn labelled_listing_and_expired_containers() { + let l = parse_labelled("c0mpute-gs-a\t100\nc0mpute-gs-b\t9999999999\nc0mpute-gs-c\t\n\n"); + assert_eq!(l, vec![("c0mpute-gs-a".into(), 100), ("c0mpute-gs-b".into(), 9_999_999_999), ("c0mpute-gs-c".into(), 0)]); + assert_eq!(expired_containers(&l, 1_000), vec!["c0mpute-gs-a".to_string(), "c0mpute-gs-c".to_string()]); + } + fn state(id: &str, ram_mb: u64, port: u16, expires: u64, stopped: Option) -> ServerState { ServerState { id: id.into(), @@ -1294,6 +2391,8 @@ mod tests { expires_at: expires, stop_hash: sha256_ref(b"t"), buyer: "b".into(), + backup_pubkey: key(), + backup_every_secs: 1800, stopped_at: stopped, backup: None, backup_error: None, @@ -1301,7 +2400,7 @@ mod tests { } #[test] - fn expiry() { + fn expiry_and_backup_schedule() { assert_eq!(expires_at(100, 4), 100 + 4 * 3600); let mut m = HashMap::new(); m.insert("a".into(), state("a", 1024, 1, 1_000, None)); @@ -1310,6 +2409,14 @@ mod tests { assert!(due_for_stop(&m, 999).is_empty()); assert_eq!(due_for_stop(&m, 1_000), vec!["a".to_string()]); assert_eq!(due_for_stop(&m, 9_999), vec!["a".to_string(), "b".to_string()]); + // Live backups: every 30 min from the start, then from the last backup. + assert!(due_for_backup(&m, 1_799).is_empty()); + assert_eq!(due_for_backup(&m, 1_800), vec!["b".to_string()], "a is past expiry, c stopped"); + m.get_mut("b").unwrap().backup = Some(Backup { manifest: "x".into(), bytes: 1, sha256: "s".into(), at: 1_800 }); + assert!(due_for_backup(&m, 3_599).is_empty()); + assert_eq!(due_for_backup(&m, 3_600), vec!["b".to_string()]); + m.get_mut("b").unwrap().backup_every_secs = 0; + assert!(due_for_backup(&m, 4_000).is_empty(), "0 = no live backups"); } #[test] @@ -1327,7 +2434,6 @@ mod tests { assert!(has_capacity(&cfg, 0, &m, &spec(Game::Minecraft)).unwrap_err().contains("slots")); m.get_mut("b").unwrap().stopped_at = Some(1); assert!(has_capacity(&cfg, 0, &m, &spec(Game::Minecraft)).is_ok(), "stopped servers free their slot"); - // Default budget is half the box. let half = crate::config::GameserverConfig { max_total_ram_gb: 0, ..cfg.clone() }; assert!(has_capacity(&half, 6 * 1024, &HashMap::new(), &spec(Game::Minecraft)).is_err()); assert!(has_capacity(&half, 16 * 1024, &HashMap::new(), &spec(Game::Minecraft)).is_ok()); @@ -1347,11 +2453,14 @@ mod tests { assert_eq!(public_host(Some("localhost"), Some("http://box.local"), Some("/ip4/127.0.0.1/tcp/1")), None); assert_eq!(public_host(None, None, None), None); assert_eq!(host_port_str("2001:db8::1", 5), "[2001:db8::1]:5"); + assert_eq!(split_address("[2001:db8::1]:5"), Some(("2001:db8::1".into(), 5))); + assert_eq!(split_address("1.2.3.4:27500"), Some(("1.2.3.4".into(), 27500))); + assert_eq!(split_address("nope"), None); } #[test] fn manifest_checks() { - let mut m = WorldManifest::new(Game::Factorio); + let mut m = WorldManifest::new(Game::Factorio, WorldEnc::handed_off()); m.bytes = CHUNK_BYTES as u64 + 1; m.sha256 = sha256_ref(b"x"); m.chunks = vec!["a".repeat(64), "b".repeat(64)]; @@ -1365,24 +2474,148 @@ mod tests { let mut huge = m.clone(); huge.bytes = MAX_WORLD_BYTES + 1; assert!(huge.check().is_err()); + let mut plain = m.clone(); + plain.enc.scheme = "none".into(); + assert!(plain.check().is_err(), "unencrypted worlds are refused"); + let mut v1 = serde_json::to_value(&m).unwrap(); + v1["v"] = json!(1); + v1.as_object_mut().unwrap().remove("enc"); + assert!(WorldManifest::from_bytes(&serde_json::to_vec(&v1).unwrap()).is_err()); } #[tokio::test] - async fn chunking_round_trips_through_the_store() { + async fn encrypted_chunking_round_trips_and_only_the_key_reads_it() { let dir = tempfile::tempdir().unwrap(); let store = ChunkStore::open(dir.path()).await.unwrap(); let data: Vec = (0..(CHUNK_BYTES * 2 + 17)).map(|i| (i % 251) as u8).collect(); - let m = chunk_into_store(&store, None, Game::Minecraft, &data[..]).await.unwrap(); + let k = random_key(); + let m = chunk_into_store(&store, None, Game::Minecraft, &data[..], &k, WorldEnc::handed_off()).await.unwrap(); m.check().unwrap(); assert_eq!(m.chunks.len(), 3); assert_eq!(m.bytes, data.len() as u64); assert_eq!(m.sha256, sha256_ref(&data)); let mut back = Vec::new(); - for c in &m.chunks { - back.extend(store.get(&Hash::from_hex(c).unwrap()).await.unwrap()); + for (i, c) in m.chunks.iter().enumerate() { + let sealed = store.get(&Hash::from_hex(c).unwrap()).await.unwrap(); + let n = (sealed.len() - TAG_BYTES).min(64); + assert_ne!(&sealed[..n], &data[i * CHUNK_BYTES..i * CHUNK_BYTES + n], "chunks are stored encrypted"); + assert!(open_chunk(&random_key(), i as u64, &sealed).is_err(), "another key cannot read it"); + assert!(open_chunk(&k, i as u64 + 1, &sealed).is_err(), "a chunk cannot move position"); + back.extend(open_chunk(&k, i as u64, &sealed).unwrap()); } assert_eq!(back, data); - let empty = chunk_into_store(&store, None, Game::Minecraft, &b""[..]).await.unwrap(); + let empty = chunk_into_store(&store, None, Game::Minecraft, &b""[..], &k, WorldEnc::handed_off()).await.unwrap(); assert!(empty.chunks.is_empty() && empty.check().is_ok()); } + + #[test] + fn backups_are_sealed_to_the_buyer_key() { + let buyer = crypto::keypair(); + let e = crypto::keypair(); + let worker_side = backup_key(&e.secret, &buyer.public_hex, &e.public_hex, &buyer.public_hex).unwrap(); + let buyer_side = backup_key(&buyer.secret, &e.public_hex, &e.public_hex, &buyer.public_hex).unwrap(); + assert_eq!(worker_side, buyer_side); + let eve = crypto::keypair(); + assert_ne!(worker_side, backup_key(&eve.secret, &e.public_hex, &e.public_hex, &buyer.public_hex).unwrap()); + } + + #[test] + fn backup_key_file_is_created_once_and_private() { + let dir = tempfile::tempdir().unwrap(); + let a = backup_keypair(dir.path()).unwrap(); + let b = backup_keypair(dir.path()).unwrap(); + assert_eq!(a.public_hex, b.public_hex); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(dir.path().join("backup.key")).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + } + } + + #[test] + fn minecraft_status_codec() { + let req = minecraft_status_request("play.example.com", 25565); + // length-prefixed handshake, then the 1-byte status request. + assert_eq!(&req[req.len() - 2..], &[0x01, 0x00]); + assert_eq!(req[0] as usize, req.len() - 3); + let resp = minecraft_status_response(r#"{"version":{"name":"1.21"},"players":{"online":3,"max":10}}"#); + let mut at = 0; + let len = read_varint_slice(&resp, &mut at).unwrap() as usize; + let v = parse_minecraft_status(&resp[at..at + len]).unwrap(); + assert_eq!(v["players"]["online"], 3); + assert!(parse_minecraft_status(&[0x01, 0x00]).is_err()); + assert!(parse_minecraft_status(&[0x00, 0x05, b'{']).is_err()); + let mut big = Vec::new(); + put_varint(&mut big, 300); + assert_eq!(big, vec![0xAC, 0x02]); + } + + #[test] + fn health_tracker_fails_over_only_after_a_sustained_outage() { + let mut h = HealthTracker::new(Duration::from_secs(300)); + assert!(!h.observe(0, true)); + assert!(!h.observe(30, false)); + assert!(!h.observe(200, false)); + assert!(!h.observe(250, true), "one pass resets the outage"); + assert!(!h.observe(300, false)); + assert!(!h.observe(599, false)); + assert!(h.observe(600, false)); + assert_eq!(h.last_ok, Some(250)); + } + + #[test] + fn judging_uses_the_signed_status_record() { + let up = ProbeResult { ts: 0, ok: true, method: "x".into(), rtt_ms: 1, detail: String::new() }; + let down = ProbeResult { ok: false, ..up.clone() }; + let rec = StatusRecord { t: STATUS_KIND.into(), v: 1, id: "gs-x".into(), worker: "w".into(), game: Game::Minecraft, address: "a:1".into(), running: true, expires_at: 9, stopped_at: None, backup: None, ts: 0 }; + assert!(judge(Game::Minecraft, &up, None, 0)); + assert!(!judge(Game::Minecraft, &down, Some((&rec, 0)), 0)); + let stopped = StatusRecord { running: false, ..rec.clone() }; + assert!(!judge(Game::Minecraft, &up, Some((&stopped, 0)), 0), "the worker says it is down"); + // Factorio's probe is weak: it also needs a fresh "running" record. + assert!(!judge(Game::Factorio, &up, None, 0)); + assert!(judge(Game::Factorio, &up, Some((&rec, 100)), 200)); + assert!(!judge(Game::Factorio, &up, Some((&rec, 0)), 500), "stale record"); + } + + #[tokio::test] + async fn probes_answer_for_live_servers_and_fail_for_dead_ports() { + // A Minecraft-status responder on loopback. + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut s, _)) = l.accept().await { + tokio::spawn(async move { + let mut buf = [0u8; 512]; + let _ = s.read(&mut buf).await; + let _ = s.write_all(&minecraft_status_response(r#"{"version":{"name":"fake"},"players":{"online":0}}"#)).await; + }); + } + }); + let p = probe_server(Game::Minecraft, &format!("127.0.0.1:{port}"), Duration::from_secs(3)).await; + assert!(p.ok, "{p:?}"); + assert_eq!(p.method, "minecraft-status"); + // An A2S responder (with a challenge round) on game port + 1. + let game = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + let gport = game.local_addr().unwrap().port(); + let q = tokio::net::UdpSocket::bind(("127.0.0.1", gport + 1)).await; + if let Ok(q) = q { + tokio::spawn(async move { + let mut buf = [0u8; 256]; + while let Ok((n, from)) = q.recv_from(&mut buf).await { + let reply: Vec = if n == A2S_INFO.len() { b"\xFF\xFF\xFF\xFFA\x01\x02\x03\x04".to_vec() } else { b"\xFF\xFF\xFF\xFFI\x11fake".to_vec() }; + let _ = q.send_to(&reply, from).await; + } + }); + let p = probe_server(Game::Valheim, &format!("127.0.0.1:{gport}"), Duration::from_secs(3)).await; + assert!(p.ok, "{p:?}"); + } + drop(game); + // A closed port: refused (TCP) / unreachable (UDP). + let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); + assert!(!probe_server(Game::Minecraft, &format!("127.0.0.1:{closed}"), Duration::from_secs(2)).await.ok); + let uclosed = std::net::UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); + assert!(!probe_server(Game::Factorio, &format!("127.0.0.1:{uclosed}"), Duration::from_secs(2)).await.ok); + } } diff --git a/node/crates/c0mpute-core/src/handoff.rs b/node/crates/c0mpute-core/src/handoff.rs new file mode 100644 index 00000000..cf17fbb1 --- /dev/null +++ b/node/crates/c0mpute-core/src/handoff.rs @@ -0,0 +1,284 @@ +//! Hand a small secret to the one worker that won an auction. +//! +//! Gossip topics are readable by every subscriber, so a secret (a game +//! server password, the key a world backup is encrypted with) can never ride +//! in an offer. Instead the offer carries the buyer's fresh X25519 public key +//! and, after the accept: +//! +//! ```text +//! worker → handoff_ready its own fresh X25519 key (gossip-signed by the worker) +//! buyer → handoff_secret the secret, sealed to it (gossip-signed by the buyer) +//! ``` +//! +//! The key is X25519 → HKDF-SHA256 → AES-256-GCM (`temporal::crypto`), bound +//! to the workload and job id. The buyer seals only to the bidder it +//! accepted, and only on a `handoff_ready` whose gossip source is that +//! bidder; the worker opens only a secret whose gossip source is the offer's +//! buyer. Gossipsub runs with strict signing, so a source cannot be forged. +//! Every other node sees two public keys and ciphertext. +//! +//! The same exchange `webhook.deliver` does for its request, cut down to one +//! direction and a payload of at most [`MAX_SECRET`] bytes. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; +use c0mpute_net::topics::{JobAccept, JobBid, job_topic}; +use c0mpute_net::{GossipMessage, Libp2pNetwork}; +use serde::{Deserialize, Serialize}; +use tokio::sync::{broadcast, mpsc}; +use tokio::task::JoinHandle; +use tracing::debug; + +use crate::temporal::crypto; + +pub use crate::temporal::crypto::KeyPair; + +/// A sealed secret must fit one gossip message with room to spare. +pub const MAX_SECRET: usize = 16 * 1024; +const REPUBLISH: Duration = Duration::from_secs(2); + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Ready { + t: String, + job_id: String, + worker_peer_id: String, + client_pubkey: String, + worker_pubkey: String, + /// The worker's dialable addresses, so the buyer can connect directly. + #[serde(default)] + addrs: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Sealed { + t: String, + job_id: String, + worker_peer_id: String, + sealed: String, + /// The buyer's dialable addresses: a worker that must fetch from the + /// buyer (a world to restore) may share no connection with it yet. + #[serde(default)] + addrs: Vec, +} + +const MAX_ADDRS: usize = 8; + +async fn my_addrs(net: &Libp2pNetwork) -> Vec { + net.advertised_addrs().await.into_iter().take(MAX_ADDRS).map(|a| a.to_string()).collect() +} + +/// Learn `peer`'s addresses from a handoff message and open a direct +/// connection (either side may be the one that can dial out). +async fn link(net: &Libp2pNetwork, peer: &str, addrs: &[String]) { + let Ok(peer) = peer.parse::() else { return }; + for a in addrs.iter().take(MAX_ADDRS).filter(|a| a.len() <= 256) { + if let Ok(addr) = a.parse::() { + net.add_peer_address(peer, addr).await; + } + } + let _ = net.connect(peer).await; +} + +/// A fresh key pair for one handoff. +pub fn keypair() -> KeyPair { + crypto::keypair() +} + +/// `true` for 64 lowercase hex characters (an X25519 public key). +pub fn valid_pubkey(s: &str) -> bool { + s.len() == 64 && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// The shared key, bound to the workload and the job. +pub fn key(secret: &x25519_dalek::StaticSecret, peer_pub_hex: &str, workload: &str, job_id: &str) -> Result<[u8; 32]> { + crypto::session_key(secret, peer_pub_hex, &format!("handoff:{workload}:{job_id}")) +} + +fn from(msg: &GossipMessage, peer: &str) -> bool { + msg.source.map(|p| p.to_base58()).as_deref() == Some(peer) +} + +/// Worker side: announce a fresh key until the buyer's sealed secret +/// arrives (or `wait` passes), and return the opened secret. +pub async fn receive(net: &Libp2pNetwork, workload: &str, accept: &JobAccept, client_pubkey: &str, wait: Duration) -> Result> { + if !valid_pubkey(client_pubkey) { + bail!("the offer's client key is not an X25519 public key"); + } + let topic = job_topic(workload); + let me = net.peer_id().to_base58(); + let mut rx = net.messages(); + let kp = crypto::keypair(); + let k = key(&kp.secret, client_pubkey, workload, &accept.job_id)?; + let ready = serde_json::to_vec(&Ready { + t: "handoff_ready".into(), + job_id: accept.job_id.clone(), + worker_peer_id: me.clone(), + client_pubkey: client_pubkey.to_string(), + worker_pubkey: kp.public_hex.clone(), + addrs: my_addrs(net).await, + })?; + let deadline = Instant::now() + wait; + let mut next = Instant::now(); + loop { + if Instant::now() >= deadline { + bail!("the buyer never sent the sealed secret"); + } + if Instant::now() >= next { + let _ = net.publish(&topic, ready.clone()).await; + next = Instant::now() + REPUBLISH; + } + let left = next.min(deadline).saturating_duration_since(Instant::now()); + let msg = match tokio::time::timeout(left, rx.recv()).await { + Ok(Ok(m)) => m, + Ok(Err(broadcast::error::RecvError::Closed)) => bail!("network closed"), + _ => continue, + }; + if msg.topic != topic || msg.data.len() > MAX_SECRET * 2 { + continue; + } + let Ok(s) = serde_json::from_slice::(&msg.data) else { continue }; + if s.t != "handoff_secret" || s.job_id != accept.job_id || s.worker_peer_id != me || !from(&msg, &accept.buyer_peer_id) { + continue; + } + match crypto::open(&k, &s.sealed) { + Ok(plain) if plain.len() <= MAX_SECRET => { + link(net, &accept.buyer_peer_id, &s.addrs).await; + return Ok(plain); + } + Ok(_) => bail!("sealed secret too large"), + Err(e) => debug!(job_id = %accept.job_id, err = %e, "handoff: unreadable secret ignored"), + } + } +} + +/// Buyer side of one auction's handoff. Pass [`BuyerHandoff::notify`] to +/// `JobAuction::with_accept_notify`; drop (or [`BuyerHandoff::finish`]) it +/// once the receipt is in. +pub struct BuyerHandoff { + notify: Option>, + task: JoinHandle<()>, +} + +impl BuyerHandoff { + /// Start listening for the winner's key. `kp` is the pair whose public + /// half went into the offer. + pub fn spawn(net: Arc, workload: &str, kp: KeyPair, secret: Vec) -> Result { + if secret.len() > MAX_SECRET { + bail!("secret is {} bytes; a handoff carries at most {MAX_SECRET}", secret.len()); + } + let topic = job_topic(workload); + let workload = workload.to_string(); + let mut rx = net.messages(); + let (tx, mut acc_rx) = mpsc::unbounded_channel::<(String, JobBid)>(); + let task = tokio::spawn(async move { + let mut winners: HashMap = HashMap::new(); + let mut readies: HashMap = HashMap::new(); + // job_id → sealed message, republished until the task ends. + let mut sent: HashMap> = HashMap::new(); + let mut tick = tokio::time::interval(REPUBLISH); + let addrs = my_addrs(&net).await; + let seal = |r: &Ready| -> Result> { + let k = key(&kp.secret, &r.worker_pubkey, &workload, &r.job_id)?; + Ok(serde_json::to_vec(&Sealed { + t: "handoff_secret".into(), + job_id: r.job_id.clone(), + worker_peer_id: r.worker_peer_id.clone(), + sealed: crypto::seal(&k, &secret)?, + addrs: addrs.clone(), + })?) + }; + loop { + tokio::select! { + Some((job_id, bid)) = acc_rx.recv() => { + if let Some(r) = readies.remove(&job_id) + && r.worker_peer_id == bid.bidder_peer_id + && let Ok(m) = seal(&r) + { + let _ = net.publish(&topic, m.clone()).await; + sent.insert(job_id.clone(), m); + link(&net, &r.worker_peer_id, &r.addrs).await; + } + winners.insert(job_id, bid.bidder_peer_id); + } + m = rx.recv() => { + let msg = match m { + Ok(m) => m, + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(_) => return, + }; + if msg.topic != topic { + continue; + } + let Ok(r) = serde_json::from_slice::(&msg.data) else { continue }; + // Seal only to the bidder we accepted, on a key it signed. + if r.t != "handoff_ready" || r.client_pubkey != kp.public_hex || !from(&msg, &r.worker_peer_id) || sent.contains_key(&r.job_id) || !valid_pubkey(&r.worker_pubkey) { + continue; + } + match winners.get(&r.job_id) { + Some(w) if *w == r.worker_peer_id => { + if let Ok(m) = seal(&r) { + let _ = net.publish(&topic, m.clone()).await; + sent.insert(r.job_id.clone(), m); + } + link(&net, &r.worker_peer_id, &r.addrs).await; + } + Some(_) => debug!(job_id = %r.job_id, "handoff: key from a worker we did not accept ignored"), + None if readies.len() < 64 => { readies.insert(r.job_id.clone(), r); } + None => {} + } + } + _ = tick.tick() => { + for m in sent.values() { + let _ = net.publish(&topic, m.clone()).await; + } + } + } + } + }); + Ok(Self { notify: Some(tx), task }) + } + + /// The accept notifier for `JobAuction::with_accept_notify`. + pub fn notify(&mut self) -> Result> { + self.notify.take().context("handoff notifier already taken") + } + + pub fn finish(self) {} +} + +impl Drop for BuyerHandoff { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn both_sides_derive_one_key_bound_to_the_job() { + let buyer = keypair(); + let worker = keypair(); + let b = key(&buyer.secret, &worker.public_hex, "gameserver.start", "job-1").unwrap(); + let w = key(&worker.secret, &buyer.public_hex, "gameserver.start", "job-1").unwrap(); + assert_eq!(b, w); + assert_ne!(b, key(&buyer.secret, &worker.public_hex, "gameserver.start", "job-2").unwrap()); + assert_ne!(b, key(&buyer.secret, &worker.public_hex, "webhook.deliver", "job-1").unwrap()); + let sealed = crypto::seal(&b, b"hunter22").unwrap(); + assert!(!sealed.contains("hunter22")); + assert_eq!(crypto::open(&w, &sealed).unwrap(), b"hunter22"); + let eve = keypair(); + assert!(crypto::open(&key(&eve.secret, &buyer.public_hex, "gameserver.start", "job-1").unwrap(), &sealed).is_err()); + } + + #[test] + fn pubkeys_are_checked() { + assert!(valid_pubkey(&keypair().public_hex)); + assert!(!valid_pubkey("zz")); + assert!(!valid_pubkey(&"A".repeat(64))); + } +} diff --git a/node/crates/c0mpute-core/src/lib.rs b/node/crates/c0mpute-core/src/lib.rs index 5ed89964..c924f48d 100644 --- a/node/crates/c0mpute-core/src/lib.rs +++ b/node/crates/c0mpute-core/src/lib.rs @@ -8,6 +8,7 @@ pub mod ads; pub mod buyer; pub mod capabilities; pub mod config; +pub mod container; pub mod crawl; pub mod diffusion; pub mod music; @@ -17,6 +18,7 @@ pub mod dispatch; pub mod embed; pub mod gameserver; pub mod gateway_util; +pub mod handoff; pub mod jobs_gateway; pub mod net_guard; pub mod ocr; diff --git a/node/crates/c0mpute-core/src/speech_gateway.rs b/node/crates/c0mpute-core/src/speech_gateway.rs index cf52e5a0..6095f688 100644 --- a/node/crates/c0mpute-core/src/speech_gateway.rs +++ b/node/crates/c0mpute-core/src/speech_gateway.rs @@ -137,9 +137,32 @@ pub async fn serve(bind: std::net::SocketAddr, token: String, max_price_usd: f64 } let supervisor = Supervisor::boot_buyer(Config::default()).await?; + let penalties = crate::penalty_box::PenaltyBox::persistent( + crate::config::data_dir() + .unwrap_or_else(|| std::path::PathBuf::from("./c0mpute-data")) + .join("gateway-penalties.json"), + ); + let app = router(supervisor.libp2p, supervisor.store, token, max_price_usd, penalties); + let listener = tokio::net::TcpListener::bind(bind) + .await + .with_context(|| format!("bind c0mpute speech gateway at {bind}"))?; + tracing::info!(%bind, "c0mpute speech gateway listening"); + axum::serve(listener, app).await?; + Ok(()) +} + +/// The gateway's routes over an already booted buyer node. `serve` uses it; +/// so do the loopback end-to-end tests, with their own node and listener. +pub fn router( + net: Arc, + store: ChunkStore, + token: String, + max_price_usd: f64, + penalties: crate::penalty_box::PenaltyBox, +) -> Router { let state = GatewayState { - net: supervisor.libp2p, - store: supervisor.store, + net, + store, token: Arc::from(token), max_price_usd, // Let one multi-chunk readback fan out across up to six network @@ -151,13 +174,9 @@ pub async fn serve(bind: std::net::SocketAddr, token: String, max_price_usd: f64 ocr_slots: Arc::new(Semaphore::new(MAX_CONCURRENT_OCR_JOBS)), embed_slots: Arc::new(Semaphore::new(MAX_CONCURRENT_EMBED_JOBS)), sessions: crate::speech_session::SessionStore::new(), - penalties: crate::penalty_box::PenaltyBox::persistent( - crate::config::data_dir() - .unwrap_or_else(|| std::path::PathBuf::from("./c0mpute-data")) - .join("gateway-penalties.json"), - ), + penalties, }; - let app = Router::new() + Router::new() .route("/healthz", get(healthz)) .route("/v1/voices", get(voices)) .route("/v1/tts", post(synthesize)) @@ -178,13 +197,7 @@ pub async fn serve(bind: std::net::SocketAddr, token: String, max_price_usd: f64 .route("/v1/sessions/{session}", delete(close_session)) .layer(DefaultBodyLimit::max(MAX_AUDIO_BYTES)) .layer(TraceLayer::new_for_http()) - .with_state(state); - let listener = tokio::net::TcpListener::bind(bind) - .await - .with_context(|| format!("bind c0mpute speech gateway at {bind}"))?; - tracing::info!(%bind, "c0mpute speech gateway listening"); - axum::serve(listener, app).await?; - Ok(()) + .with_state(state) } async fn healthz() -> Json { diff --git a/node/crates/c0mpute-core/src/video.rs b/node/crates/c0mpute-core/src/video.rs index 480a52bb..26f55bd6 100644 --- a/node/crates/c0mpute-core/src/video.rs +++ b/node/crates/c0mpute-core/src/video.rs @@ -43,9 +43,25 @@ pub const WORKLOAD_IMG2VID: &str = "video.img2vid"; /// are all present — see [`readiness`]. pub const ROLE_TAG: &str = "c0mpute:role:video"; -/// The image the runner lives in. Kept in step with -/// `plugins/video/module.toml` [module.dispatch]. -pub const RUNNER_IMAGE: &str = "ghcr.io/profullstack/c0mpute-video-runner:candidate"; +/// Where `.github/workflows/video-runner.yml` publishes the runner. +pub const RUNNER_REPOSITORY: &str = "ghcr.io/profullstack/c0mpute-video-runner"; + +/// The stable tag the workflow moves on every master build. +pub const RUNNER_TAG: &str = "stable"; + +/// The published runner this release trusts, pinned by digest: a tag that +/// moves cannot change what a node runs. Bump it (and +/// `plugins/video/module.toml` [module.dispatch]) to the digest the workflow +/// prints in its job summary after a runner change. +pub const RUNNER_DIGEST: &str = "sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c"; + +/// What `c0mpute video setup` pulls: the repository at the pinned digest. +pub const RUNNER_PULL_REF: &str = "ghcr.io/profullstack/c0mpute-video-runner@sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c"; + +/// The local name the node runs. Setup tags the pinned pull as this, or +/// builds it from source under this name when the pull fails, so the worker +/// never depends on how the image arrived. +pub const RUNNER_IMAGE: &str = "c0mpute/video-runner:stable"; /// Where a node can build the runner itself when it cannot pull it. /// @@ -302,6 +318,71 @@ pub fn build_argv(checkout: &Path, image: &str) -> Vec { ] } +/// How `c0mpute video setup` got the runner image. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SetupSource { + /// Pulled at the pinned digest and tagged as [`RUNNER_IMAGE`]. + Pulled, + /// The pull failed (or `--build`): built from a clone of the source repo. + Built, +} + +/// The source repo to build from: `$C0MPUTE_VIDEO_SOURCE_REPO` (the same +/// override `install.sh` takes), else [`RUNNER_SOURCE_REPO`]. +pub fn source_repo() -> String { + std::env::var("C0MPUTE_VIDEO_SOURCE_REPO").ok().filter(|s| !s.is_empty()).unwrap_or_else(|| RUNNER_SOURCE_REPO.to_string()) +} + +async fn run_status(cmd: &mut tokio::process::Command, what: &str) -> Result { + Ok(cmd + .stdin(std::process::Stdio::null()) + .status() + .await + .with_context(|| format!("could not run {what}"))? + .success()) +} + +/// Put the runner image on this node as [`RUNNER_IMAGE`]: pull the pinned +/// digest and tag it, or, when the registry will not serve it (private, +/// rate-limited, offline) or `build_only`, shallow-clone the public repo +/// into `scratch` and build the same Dockerfile. `say` gets progress lines. +pub async fn setup(build_only: bool, scratch: &Path, say: &(dyn Fn(&str) + Sync)) -> Result { + if !build_only { + say(&format!("pulling {RUNNER_PULL_REF} (this is a large image)")); + let pulled = run_status(crate::container::command().args(["pull", RUNNER_PULL_REF]), "docker; install a container runtime first").await?; + if pulled && run_status(crate::container::command().args(["tag", RUNNER_PULL_REF, RUNNER_IMAGE]), "docker tag").await? { + return Ok(SetupSource::Pulled); + } + // A registry is the kind of dependency that is private by default, + // rate-limited or down, and none of those are a reason a node with a + // card cannot serve video. The repo is public: build it. + say("could not pull the runner image (private, rate-limited or offline)."); + say("building it from source instead: same Dockerfile, same runner."); + } + // A fresh directory per run: a half-finished clone from an earlier + // attempt would make `git clone` fail rather than resume. + let checkout = scratch.join(format!("c0mpute-video-src-{}", std::process::id())); + let _ = fs::remove_dir_all(&checkout).await; + let repo = source_repo(); + say(&format!("cloning {repo} (shallow)")); + let cloned = run_status(tokio::process::Command::new("git").args(["clone", "--depth", "1", &repo]).arg(&checkout), "git (building the runner needs git)").await; + let built = match cloned { + Ok(true) => { + say(&format!("building {RUNNER_IMAGE} (several minutes)")); + run_status(crate::container::command().args(build_argv(&checkout, RUNNER_IMAGE)), "docker build").await + } + Ok(false) => Err(anyhow::anyhow!("git clone of {repo} failed")), + Err(e) => Err(e), + }; + // The checkout is scratch whether or not the build worked. + let _ = fs::remove_dir_all(&checkout).await; + if !built? { + bail!("docker build of the runner failed"); + } + say(&format!("built {RUNNER_IMAGE}")); + Ok(SetupSource::Built) +} + /// The runner command for a workload type. pub fn command_for(workload: &str) -> Option<&'static str> { match workload { @@ -335,7 +416,7 @@ impl NotReady { /// Probe the host. Blocking, and only called at boot or from the CLI. pub fn readiness(image: &str) -> Option { - if which("docker").is_none() { + if crate::container::which().is_none() { return Some(NotReady::NoDocker); } if !nvidia_gpu_present() { @@ -369,14 +450,6 @@ pub fn readiness_cached(image: &str) -> Option { verdict } -fn which(bin: &str) -> Option { - std::env::var_os("PATH").and_then(|paths| { - std::env::split_paths(&paths) - .map(|dir| dir.join(bin)) - .find(|candidate| candidate.is_file()) - }) -} - fn nvidia_gpu_present() -> bool { std::process::Command::new("nvidia-smi") .args(["--query-gpu=name", "--format=csv,noheader"]) @@ -386,7 +459,7 @@ fn nvidia_gpu_present() -> bool { } fn image_present(image: &str) -> bool { - std::process::Command::new("docker") + crate::container::command_sync() .args(["image", "inspect", image]) .output() .map(|out| out.status.success()) @@ -546,7 +619,7 @@ async fn owner_of(_path: &Path) -> Result<(u32, u32)> { async fn run_container(run: &ContainerRun) -> Result<()> { let args = docker_argv(run); info!(image = %run.image, command = %run.command, "video runner: starting container"); - let mut child = tokio::process::Command::new("docker") + let mut child = crate::container::command() .args(&args) .stdin(std::process::Stdio::null()) .kill_on_drop(true) @@ -564,7 +637,7 @@ async fn run_container(run: &ContainerRun) -> Result<()> { Err(_) => { // kill_on_drop reaps `docker run`, but the container it started is // a child of the daemon and would keep the GPU busy. - let _ = tokio::process::Command::new("docker") + let _ = crate::container::command() .args(["kill", &run.container_name]) .output() .await; @@ -753,6 +826,23 @@ mod tests { assert_eq!(readiness_cached(RUNNER_IMAGE).is_none(), fresh); } + #[test] + fn the_runner_is_pinned_by_digest_and_matches_the_manifest() { + assert_eq!(RUNNER_PULL_REF, format!("{RUNNER_REPOSITORY}@{RUNNER_DIGEST}")); + let digest = RUNNER_DIGEST.strip_prefix("sha256:").unwrap(); + assert!(digest.len() == 64 && digest.bytes().all(|b| b.is_ascii_hexdigit())); + // The local name is not a registry reference: nothing can pull a + // different image under it. + assert!(!RUNNER_IMAGE.contains("ghcr.io") && !RUNNER_IMAGE.contains('@')); + let manifest: toml::Value = + toml::from_str(include_str!("../../../../plugins/video/module.toml")).unwrap(); + assert_eq!( + manifest["module"]["dispatch"]["image"].as_str(), + Some(RUNNER_PULL_REF), + "plugins/video/module.toml [module.dispatch] image must be the pinned digest" + ); + } + #[test] fn weights_live_under_the_cache_root() { let dir = weights_dir(Path::new("/var/c0mpute")); diff --git a/node/crates/c0mpute-core/tests/fixtures/tiny.mp4 b/node/crates/c0mpute-core/tests/fixtures/tiny.mp4 new file mode 100644 index 00000000..7b245e00 Binary files /dev/null and b/node/crates/c0mpute-core/tests/fixtures/tiny.mp4 differ diff --git a/node/crates/c0mpute-core/tests/gameserver_e2e.rs b/node/crates/c0mpute-core/tests/gameserver_e2e.rs new file mode 100644 index 00000000..bcb46d82 --- /dev/null +++ b/node/crates/c0mpute-core/tests/gameserver_e2e.rs @@ -0,0 +1,396 @@ +//! Game servers end to end on loopback libp2p, with a fake container runtime. +//! +//! The workers, buyer, auction, sealed handoff, encrypted backups, status +//! gossip, health probes and failover are the shipping code. Only +//! [`gameserver::Runtime`] is a double: a "volume" is a byte blob, and a +//! started Minecraft "container" is a TCP listener on the allocated port +//! that answers the real Minecraft status ping, so the buyer's probe has +//! something true to test. + +use std::collections::HashMap; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use c0mpute_core::gameserver::{self, ContainerPlan, Game, Runtime, ServerState, StartRequest, WatchEnd, WatchOptions}; +use c0mpute_core::pricing::PricingConfig; +use c0mpute_net::topics::job_topic; +use c0mpute_net::{ChunkSource, Libp2pNetwork, NetworkConfig}; +use c0mpute_proto::Hash; +use c0mpute_store::ChunkStore; +use libp2p::multiaddr::Protocol; +use tempfile::tempdir; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::task::JoinHandle; + +struct StoreSource(ChunkStore); + +#[async_trait::async_trait] +impl ChunkSource for StoreSource { + async fn read_chunk(&self, hash: &Hash) -> anyhow::Result> { + self.0.get(hash).await + } +} + +/// A world: 2.5 MiB of incompressible bytes, so it spans three chunks. +fn world(seed: u64) -> Vec { + let mut x = seed | 1; + let mut v = Vec::with_capacity(2_621_440); + while v.len() < 2_621_440 { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + v.extend_from_slice(&x.to_le_bytes()); + } + v +} + +#[derive(Default)] +struct Fake { + seed: Vec, + plans: Mutex>, + vols: Mutex>>, + live: Mutex>>, + labelled: Mutex>, + killed: Mutex>, + restored: Mutex>, +} + +impl Fake { + fn new(seed: Vec) -> Arc { + Arc::new(Self { seed, ..Default::default() }) + } + fn plan(&self, name: &str) -> Option { + self.plans.lock().unwrap().iter().rev().find(|p| p.name == name).cloned() + } + /// The server process dies (host trouble, OOM, the worker went away). + fn crash(&self, name: &str) { + if let Some(h) = self.live.lock().unwrap().remove(name) { + h.abort(); + } + } +} + +#[async_trait::async_trait] +impl Runtime for Fake { + async fn create(&self, plan: &ContainerPlan) -> anyhow::Result<()> { + self.plans.lock().unwrap().push(plan.clone()); + self.vols.lock().unwrap().entry(plan.volume.clone()).or_insert_with(|| self.seed.clone()); + let exp = plan.labels.iter().find(|(k, _)| k == "c0mpute.gameserver.expires").and_then(|(_, v)| v.parse().ok()).unwrap_or(0); + self.labelled.lock().unwrap().push((plan.name.clone(), exp)); + Ok(()) + } + async fn restore(&self, plan: &ContainerPlan, tar: &Path) -> anyhow::Result<()> { + self.vols.lock().unwrap().insert(plan.volume.clone(), std::fs::read(tar)?); + self.restored.lock().unwrap().push(plan.volume.clone()); + Ok(()) + } + async fn start(&self, name: &str) -> anyhow::Result<()> { + let plan = self.plan(name).ok_or_else(|| anyhow::anyhow!("no container {name}"))?; + let listener = tokio::net::TcpListener::bind(("127.0.0.1", plan.host_port)).await?; + let h = tokio::spawn(async move { + while let Ok((mut s, _)) = listener.accept().await { + tokio::spawn(async move { + let mut buf = [0u8; 512]; + let _ = s.read(&mut buf).await; + let _ = s.write_all(&gameserver::minecraft_status_response(r#"{"version":{"name":"fake 1.21"},"players":{"online":1,"max":8}}"#)).await; + }); + } + }); + self.live.lock().unwrap().insert(name.to_string(), h); + Ok(()) + } + async fn running(&self, name: &str) -> bool { + self.live.lock().unwrap().get(name).is_some_and(|h| !h.is_finished()) + } + async fn flush(&self, _name: &str, _game: Game) {} + async fn stop(&self, name: &str) -> anyhow::Result<()> { + self.crash(name); + Ok(()) + } + async fn tar_world(&self, volume: &str, _game: Game, out: &Path) -> anyhow::Result<()> { + let v = self.vols.lock().unwrap().get(volume).cloned().ok_or_else(|| anyhow::anyhow!("no volume {volume}"))?; + std::fs::write(out, v)?; + Ok(()) + } + async fn remove(&self, name: &str, volume: &str) { + self.crash(name); + self.vols.lock().unwrap().remove(volume); + self.labelled.lock().unwrap().retain(|(n, _)| n != name); + } + async fn labelled(&self) -> anyhow::Result> { + Ok(self.labelled.lock().unwrap().clone()) + } + async fn kill(&self, name: &str) { + self.crash(name); + self.killed.lock().unwrap().push(name.to_string()); + self.labelled.lock().unwrap().retain(|(n, _)| n != name); + } +} + +async fn node(dir: &Path, bootstrap: Vec) -> (Arc, ChunkStore) { + let store = ChunkStore::open(dir.join("store")).await.unwrap(); + let net = Libp2pNetwork::spawn( + NetworkConfig::for_dir(dir.join("id")) + .with_listen(vec!["/ip4/127.0.0.1/tcp/0".parse().unwrap()]) + .with_local_source(Arc::new(StoreSource(store.clone()))) + .with_bootstrap(bootstrap), + ) + .await + .unwrap(); + (Arc::new(net), store) +} + +async fn wait_for_listen(net: &Libp2pNetwork) -> libp2p::Multiaddr { + for _ in 0..50 { + if let Ok(addrs) = net.listen_addrs().await + && let Some(a) = addrs.into_iter().next() + { + return a; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("no listen addr after 2.5s"); +} + +fn cfg(port_min: u16) -> c0mpute_core::config::GameserverConfig { + c0mpute_core::config::GameserverConfig { port_min, port_max: port_min + 9, max_servers: 2, max_total_ram_gb: 16, max_hours: 24, ..Default::default() } +} + +/// A gameserver worker over `rt`, serving on loopback. +async fn worker(dir: &Path, bootstrap: Vec, port_min: u16, rt: Arc) -> Arc { + let (net, store) = node(dir, bootstrap).await; + let w = gameserver::Worker::with_runtime(net, store, cfg(port_min), "127.0.0.1".into(), &dir.join("state"), rt); + let tags = vec![gameserver::ROLE_TAG.to_string(), Game::Minecraft.tag(), Game::Valheim.tag()]; + gameserver::subscribe(w.clone(), tags, Some(PricingConfig::default())); + w +} + +/// Everything a bystander node sees on the job and status topics. +async fn bystander(dir: &Path, bootstrap: Vec) -> (Arc, Arc>>) { + let (net, _) = node(dir, bootstrap).await; + net.subscribe(&job_topic(gameserver::WORKLOAD_START)).await.unwrap(); + net.subscribe(gameserver::STATUS_TOPIC).await.unwrap(); + let seen = Arc::new(Mutex::new(Vec::new())); + let mut rx = net.messages(); + let s = seen.clone(); + tokio::spawn(async move { + while let Ok(m) = rx.recv().await { + let mut g = s.lock().unwrap(); + g.extend_from_slice(&m.data); + g.push(b'\n'); + } + }); + (net, seen) +} + +fn req(game: Game, password: Option<&str>, workers: Vec) -> StartRequest { + StartRequest { + game, + hours: 2, + ram_gb: game.min_ram_gb(), + cpus: 1.0, + accept_eula: true, + world: None, + password: password.map(str::to_string), + server_name: None, + backup_mins: 30, + max_price_usd: 100.0, + workers, + exclude: Vec::new(), + timeout: Duration::from_secs(120), + } +} + +fn contains(hay: &[u8], needle: &[u8]) -> bool { + hay.windows(needle.len()).any(|w| w == needle) +} + +/// Valheim: the password reaches only the winning worker (sealed), the +/// world comes back sealed to the buyer's key, and the stop downloads and +/// decrypts it. A bystander sees neither. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn secrets_are_sealed_and_backups_readable_only_by_the_buyer() { + let _ = tracing_subscriber::fmt().with_env_filter("c0mpute_core=info,c0mpute_net=warn").with_test_writer().try_init(); + let tmp = tempdir().unwrap(); + let rt = Fake::new(world(7)); + let a = worker(&tmp.path().join("a"), vec![], 41100, rt.clone()).await; + let a_addr = wait_for_listen(worker_net(&a)).await.with(Protocol::P2p(worker_net(&a).peer_id())); + let a_id = worker_net(&a).peer_id().to_base58(); + let (eve, seen) = bystander(&tmp.path().join("eve"), vec![a_addr.clone()]).await; + let (buyer, buyer_store) = node(&tmp.path().join("buyer"), vec![a_addr.clone()]).await; + let dir = tmp.path().join("buyer-data"); + tokio::time::sleep(Duration::from_secs(2)).await; + + let rental = gameserver::start_with(buyer.clone(), &buyer_store, &dir, req(Game::Valheim, Some("hunter22pw"), vec![a_id.clone()])).await.expect("valheim start"); + assert_eq!(rental.worker, a_id); + assert!(rental.address.starts_with("127.0.0.1:")); + let plan = rt.plan(&format!("c0mpute-{}", rental.id)).unwrap(); + assert!(plan.env.iter().any(|(k, v)| k == "SERVER_PASS" && v == "hunter22pw"), "the worker got the password through the handoff"); + assert!(plan.run_secs > 7000 && plan.run_secs <= 7200, "the container stops itself at expiry: {}", plan.run_secs); + + // A live backup, announced in a status record the bystander can see. + let live = a.backup_now(&rental.id).await.unwrap(); + a.publish_status().await; + + let stopped = gameserver::stop_with(buyer.clone(), &buyer_store, &dir, &rental.id, Duration::from_secs(60)).await.expect("stop"); + let manifest = stopped.backup.clone().expect("stop returns a backup"); + assert_ne!(manifest, live.manifest, "the final backup is a new one"); + let tar = std::fs::read(dir.join("worlds").join(format!("{manifest}.tar"))).unwrap(); + assert!(tar == world(7), "the buyer decrypted exactly the world"); + assert!(rt.vols.lock().unwrap().is_empty(), "the volume is removed after a good backup"); + + // Eve has every gossip message and can fetch every chunk, and learns nothing. + tokio::time::sleep(Duration::from_millis(500)).await; + let wire = seen.lock().unwrap().clone(); + assert!(contains(&wire, b"handoff_ready") && contains(&wire, b"handoff_secret"), "eve saw the sealed exchange"); + assert!(contains(&wire, live.manifest.as_bytes()), "eve saw the backup hash in a status record"); + assert!(!contains(&wire, b"hunter22pw"), "the password leaked onto gossip"); + let mb = eve.fetch_from_peer(worker_net(&a).peer_id(), &c0mpute_proto::ChunkRequest { chunk_hash: Hash::from_hex(&manifest).unwrap(), shard_index: None }).await.unwrap(); + let m = gameserver::WorldManifest::from_bytes(&mb).unwrap(); + assert!(m.enc.epk.is_some()); + let first = eve.fetch_from_peer(worker_net(&a).peer_id(), &c0mpute_proto::ChunkRequest { chunk_hash: Hash::from_hex(&m.chunks[0]).unwrap(), shard_index: None }).await.unwrap(); + assert!(!contains(&first, &world(7)[..64]), "chunks are ciphertext"); + assert!(gameserver::open_chunk(&gameserver::random_key(), 0, &first).is_err()); + + // The rentals file (token, password) is private. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!(std::fs::metadata(dir.join("rentals.json")).unwrap().permissions().mode() & 0o777, 0o600); + } +} + +/// Minecraft on worker A: the buyer's watch probes it, mirrors A's backup, +/// and when the server stops answering it starts the same world on worker B +/// from that backup (handed to B under a fresh key). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn watch_fails_over_to_another_worker_from_the_latest_backup() { + let tmp = tempdir().unwrap(); + let rt_a = Fake::new(world(11)); + let rt_b = Fake::new(world(99)); + let a = worker(&tmp.path().join("a"), vec![], 41200, rt_a.clone()).await; + let a_addr = wait_for_listen(worker_net(&a)).await.with(Protocol::P2p(worker_net(&a).peer_id())); + let a_id = worker_net(&a).peer_id().to_base58(); + let b = worker(&tmp.path().join("b"), vec![a_addr.clone()], 41210, rt_b.clone()).await; + let b_id = worker_net(&b).peer_id().to_base58(); + let (buyer, buyer_store) = node(&tmp.path().join("buyer"), vec![a_addr.clone()]).await; + let dir = tmp.path().join("buyer-data"); + tokio::time::sleep(Duration::from_secs(3)).await; + + let rental = gameserver::start_with(buyer.clone(), &buyer_store, &dir, req(Game::Minecraft, None, vec![a_id.clone()])).await.expect("minecraft start"); + assert_eq!(rental.worker, a_id); + let p = gameserver::probe_server(Game::Minecraft, &rental.address, Duration::from_secs(3)).await; + assert!(p.ok && p.detail.contains("fake 1.21"), "{p:?}"); + + let opts = WatchOptions { + probe_every: Duration::from_secs(1), + probe_timeout: Duration::from_secs(1), + fail_after: Duration::from_secs(4), + max_price_usd: 100.0, + start_timeout: Duration::from_secs(120), + stop_after_failover: true, + ..Default::default() + }; + let watch = { + let (buyer, store, dir, id) = (buyer.clone(), buyer_store.clone(), dir.clone(), rental.id.clone()); + tokio::spawn(async move { gameserver::watch_with(buyer, &store, &dir, &id, opts).await }) + }; + + // A's 30-minute backup, then its status records until the watch mirrors it. + let backup = a.backup_now(&rental.id).await.unwrap(); + let mut mirrored = false; + for _ in 0..40 { + a.publish_status().await; + tokio::time::sleep(Duration::from_millis(500)).await; + if gameserver::load_rentals(&dir).iter().any(|r| r.id == rental.id && r.latest_backup.as_deref() == Some(backup.manifest.as_str())) { + mirrored = true; + break; + } + } + assert!(mirrored, "the watch mirrored A's backup"); + assert!(std::fs::read(dir.join("worlds").join(format!("{}.tar", backup.manifest))).unwrap() == world(11)); + assert!(gameserver::health_log(&dir, &rental.id).iter().any(|p| p.ok), "healthy probes were logged"); + + // The server stops answering (and A says nothing more). + rt_a.crash(&format!("c0mpute-{}", rental.id)); + let end = tokio::time::timeout(Duration::from_secs(150), watch).await.expect("watch returns").unwrap().expect("watch ok"); + let WatchEnd::FailedOver { from, to } = end else { panic!("expected a failover, got {end:?}") }; + assert_eq!(from.id, rental.id); + assert_eq!(to.worker, b_id, "the replacement runs on the other worker"); + assert_eq!(to.replaces.as_deref(), Some(rental.id.as_str())); + let log = gameserver::health_log(&dir, &rental.id); + assert!(log.iter().any(|p| !p.ok), "failed probes were logged"); + + // B restored A's world (not its own seed) and is serving it. + let vol = format!("c0mpute-{}", to.id); + assert_eq!(rt_b.restored.lock().unwrap().as_slice(), &[vol.clone()]); + assert!(rt_b.vols.lock().unwrap().get(&vol).unwrap() == &world(11)); + assert!(gameserver::probe_server(Game::Minecraft, &to.address, Duration::from_secs(3)).await.ok); + let all = gameserver::load_rentals(&dir); + assert_eq!(all.iter().find(|r| r.id == rental.id).unwrap().superseded_by.as_deref(), Some(to.id.as_str())); + drop(b); +} + +/// The worker was offline when a reservation ended: at its next sweep it +/// kills the expired container, backs the world up from the volume (sealed +/// to the buyer) and removes the volume. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_expiry_missed_while_offline_is_enforced_at_the_next_sweep() { + let tmp = tempdir().unwrap(); + let state_dir = tmp.path().join("w").join("state"); + std::fs::create_dir_all(&state_dir).unwrap(); + let now = gameserver::now_secs(); + let id = gameserver::server_id("job-expired"); + let buyer_key = c0mpute_core::temporal::crypto::keypair(); + let st = ServerState { + id: id.clone(), + game: Game::Factorio, + container: format!("c0mpute-{id}"), + volume: format!("c0mpute-{id}"), + ram_mb: 2048, + host_port: 41300, + address: "127.0.0.1:41300".into(), + started_at: now - 7200, + expires_at: now - 60, + stop_hash: gameserver::sha256_ref(b"t"), + buyer: "buyer".into(), + backup_pubkey: buyer_key.public_hex.clone(), + backup_every_secs: 1800, + stopped_at: None, + backup: None, + backup_error: None, + }; + let servers: HashMap = [(id.clone(), st.clone())].into_iter().collect(); + std::fs::write(state_dir.join("servers.json"), serde_json::to_vec(&servers).unwrap()).unwrap(); + + let rt = Fake::new(world(5)); + rt.vols.lock().unwrap().insert(st.volume.clone(), world(5)); + rt.labelled.lock().unwrap().push((st.container.clone(), now - 60)); + rt.labelled.lock().unwrap().push(("c0mpute-gs-ffffffffffffffff".into(), now + 3600)); + let (net, store) = node(&tmp.path().join("w"), vec![]).await; + let w = gameserver::Worker::with_runtime(net, store.clone(), cfg(41300), "127.0.0.1".into(), &state_dir, rt.clone()); + + assert_eq!(w.sweep_expired().await, vec![st.container.clone()], "only the expired container dies"); + w.reap_once().await; + let after = w.snapshot().get(&id).cloned().unwrap(); + assert!(after.stopped_at.is_some()); + let b = after.backup.expect("backed up from the volume"); + assert!(rt.vols.lock().unwrap().get(&st.volume).is_none(), "volume removed"); + // The backup opens with the buyer's key and nobody else's. + let m = gameserver::WorldManifest::from_bytes(&store.get(&Hash::from_hex(&b.manifest).unwrap()).await.unwrap()).unwrap(); + let epk = m.enc.epk.clone().unwrap(); + let k = gameserver::backup_key(&buyer_key.secret, &epk, &epk, &buyer_key.public_hex).unwrap(); + let mut plain = Vec::new(); + for (i, c) in m.chunks.iter().enumerate() { + plain.extend(gameserver::open_chunk(&k, i as u64, &store.get(&Hash::from_hex(c).unwrap()).await.unwrap()).unwrap()); + } + assert!(plain == world(5)); + // Persisted: a restart does not stop it twice. + let saved: HashMap = serde_json::from_slice(&std::fs::read(state_dir.join("servers.json")).unwrap()).unwrap(); + assert!(saved[&id].stopped_at.is_some()); +} + +fn worker_net(w: &gameserver::Worker) -> &Libp2pNetwork { + w.net() +} diff --git a/node/crates/c0mpute-core/tests/video_e2e.rs b/node/crates/c0mpute-core/tests/video_e2e.rs new file mode 100644 index 00000000..092f6008 --- /dev/null +++ b/node/crates/c0mpute-core/tests/video_e2e.rs @@ -0,0 +1,278 @@ +//! The video buyer path end to end on loopback, with a fake runner. +//! +//! `C0MPUTE_DOCKER` points the node's container CLI at a shell script that +//! behaves like `docker run `: it finds the job directory the node +//! bind-mounted at /job, checks `job.json` is there, and writes a real (tiny) +//! H.264 MP4 as `out.mp4`. Everything else is the shipping code: the worker's +//! video runner and bid filter, the auction, the store and announce, and the +//! buyer gateway's `POST /v1/videos`, which answers `video/mp4`. +//! +//! The same fake stands in for `docker pull/tag/build` to prove +//! `c0mpute video setup` builds from source when the registry will not serve +//! the pinned image. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use c0mpute_core::pricing::PricingConfig; +use c0mpute_core::{dispatch, penalty_box::PenaltyBox, speech_gateway, video}; +use c0mpute_net::{ChunkSource, Libp2pNetwork, NetworkConfig}; +use c0mpute_proto::Hash; +use c0mpute_store::ChunkStore; +use libp2p::multiaddr::Protocol; +use tempfile::tempdir; + +const TOKEN: &str = "video-e2e-token-0123456789abcdef"; +const FAKE_DOCKER: &str = r#"#!/bin/sh +# Stand-in for the docker CLI (see video_e2e.rs). +echo "$*" >> "$FAKE_DIR/log" +case "$1" in + run) + jd="" + for a in "$@"; do + case "$a" in *:/job) jd="${a%:/job}" ;; esac + done + [ -n "$jd" ] && [ -f "$jd/job.json" ] || exit 3 + if grep -q BIGCLIP "$jd/job.json"; then cp "$FAKE_DIR/big.mp4" "$jd/out.mp4"; else cp "$FAKE_DIR/tiny.mp4" "$jd/out.mp4"; fi + exit 0 ;; + pull) [ -f "$FAKE_DIR/pull-ok" ] && exit 0; echo "denied" >&2; exit 1 ;; + build) [ -f "$5" ] && [ -d "$6" ] || exit 4; exit 0 ;; + *) exit 0 ;; +esac +"#; + +struct StoreSource(ChunkStore); + +#[async_trait::async_trait] +impl ChunkSource for StoreSource { + async fn read_chunk(&self, hash: &Hash) -> anyhow::Result> { + self.0.get(hash).await + } +} + +fn tiny_mp4() -> Vec { + std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/tiny.mp4")).unwrap() +} + +/// The tiny clip plus a top-level `free` box: still a valid MP4, `len` bytes. +fn big_mp4(len: usize) -> Vec { + let mut v = tiny_mp4(); + let pad = len - v.len(); + v.extend_from_slice(&(pad as u32).to_be_bytes()); + v.extend_from_slice(b"free"); + let mut x: u64 = 0x9E37_79B9_7F4A_7C15; + while v.len() < len { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + v.extend_from_slice(&x.to_le_bytes()); + } + v.truncate(len); + v +} + +/// Install the fake docker once per test binary. +fn fake_docker() -> &'static PathBuf { + static DIR: OnceLock = OnceLock::new(); + DIR.get_or_init(|| { + let dir = tempdir().unwrap().keep(); + std::fs::write(dir.join("tiny.mp4"), tiny_mp4()).unwrap(); + std::fs::write(dir.join("big.mp4"), big_mp4(6 * 1024 * 1024 + 123)).unwrap(); + let bin = dir.join("docker"); + std::fs::write(&bin, FAKE_DOCKER).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + // SAFETY: set once, before any thread of this binary reads them. + unsafe { + std::env::set_var(c0mpute_core::container::ENV, &bin); + std::env::set_var("FAKE_DIR", &dir); + } + dir + }) +} + +async fn node(dir: &Path, bootstrap: Vec) -> (Arc, ChunkStore) { + let store = ChunkStore::open(dir.join("store")).await.unwrap(); + let net = Libp2pNetwork::spawn( + NetworkConfig::for_dir(dir.join("id")) + .with_listen(vec!["/ip4/127.0.0.1/tcp/0".parse().unwrap()]) + .with_local_source(Arc::new(StoreSource(store.clone()))) + .with_bootstrap(bootstrap), + ) + .await + .unwrap(); + (Arc::new(net), store) +} + +async fn wait_for_listen(net: &Libp2pNetwork) -> libp2p::Multiaddr { + for _ in 0..50 { + if let Ok(addrs) = net.listen_addrs().await + && let Some(a) = addrs.into_iter().next() + { + return a; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("no listen addr after 2.5s"); +} + +/// A video worker (the real runner over the fake docker), a buyer node, and +/// the buyer gateway on a loopback port. Returns the gateway URL. +async fn network(tmp: &Path) -> (String, Arc) { + fake_docker(); + let (worker, worker_store) = node(&tmp.join("w"), vec![]).await; + let worker_addr = wait_for_listen(&worker).await.with(Protocol::P2p(worker.peer_id())); + let load = c0mpute_core::WorkerLoad::new(Arc::new(tokio::sync::Semaphore::new(1)), 1); + let runner = video::spawn_video_runner( + worker.clone(), + worker_store, + tmp.join("cache"), + video::WORKLOAD_TXT2VID.to_string(), + video::RUNNER_IMAGE.to_string(), + load.clone(), + ); + dispatch::run_worker_subscriber_with_load( + worker.clone(), + video::WORKLOAD_TXT2VID.to_string(), + vec![video::ROLE_TAG.to_string(), "c0mpute:gpu:nvidia".to_string()], + Some(runner), + None, + Some(PricingConfig::default()), + Some(load), + ); + + let (buyer, buyer_store) = node(&tmp.join("b"), vec![worker_addr]).await; + let app = speech_gateway::router(buyer, buyer_store, TOKEN.into(), 100.0, PenaltyBox::new()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + tokio::time::sleep(Duration::from_secs(2)).await; + (url, worker) +} + +async fn post_video(url: &str, prompt: &str) -> reqwest::Response { + reqwest::Client::new() + .post(format!("{url}/v1/videos")) + .bearer_auth(TOKEN) + .json(&serde_json::json!({ + "model": "wan2.2-ti2v-5b", + "prompt": prompt, + "frames": 9, + "steps": 2, + "width": 64, + "height": 64, + })) + .timeout(Duration::from_secs(180)) + .send() + .await + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn post_v1_videos_returns_the_runner_clip() { + let _ = tracing_subscriber::fmt().with_env_filter("c0mpute_core=info,c0mpute_net=warn").with_test_writer().try_init(); + let tmp = tempdir().unwrap(); + let (url, worker) = network(tmp.path()).await; + + // No token, no job. + let anon = reqwest::Client::new().post(format!("{url}/v1/videos")).json(&serde_json::json!({"model": "m", "prompt": "p"})).send().await.unwrap(); + assert_eq!(anon.status(), 401); + + let res = post_video(&url, "a neon alley in the rain").await; + assert_eq!(res.status(), 200, "{:?}", res.text().await); + assert_eq!(res.headers()["content-type"], "video/mp4"); + assert_eq!(res.headers()["x-c0mpute-worker-peer-id"], worker.peer_id().to_base58().as_str()); + let body = res.bytes().await.unwrap(); + assert!(body.as_ref() == tiny_mp4().as_slice(), "the gateway returns exactly the runner's out.mp4"); + + // The node ran the runner with the GPU, the job dir and the weights cache. + let log = std::fs::read_to_string(fake_docker().join("log")).unwrap(); + let run = log.lines().find(|l| l.starts_with("run ")).expect("docker run was called"); + assert!(run.contains("--gpus all") && run.contains(":/job") && run.contains(":/weights"), "{run}"); + assert!(run.contains(video::RUNNER_IMAGE) && run.contains("txt2vid"), "{run}"); + + // An invalid job is refused before any auction. + let bad = reqwest::Client::new() + .post(format!("{url}/v1/videos")) + .bearer_auth(TOKEN) + .json(&serde_json::json!({"model": "wan2.2-ti2v-5b", "prompt": "x", "frames": 9999})) + .send() + .await + .unwrap(); + assert_eq!(bad.status(), 400); +} + +/// A clip over one network message (6 MiB): the worker publishes it as +/// pieces and the gateway fetches and reassembles them (fix/large-results). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_clip_over_5_mib_reaches_the_buyer() { + let tmp = tempdir().unwrap(); + let (url, _) = network(tmp.path()).await; + let res = post_video(&url, "BIGCLIP a long take").await; + assert_eq!(res.status(), 200, "{:?}", res.text().await); + let body = res.bytes().await.unwrap(); + let want = std::fs::read(fake_docker().join("big.mp4")).unwrap(); + assert_eq!(body.len(), want.len()); + assert!(body.as_ref() == want.as_slice()); +} + +/// `c0mpute video setup`: the pinned pull is tried first and tagged as the +/// local name; when the registry refuses it, the runner is built from a +/// clone of the source repo instead. +#[tokio::test] +async fn setup_pulls_the_pinned_digest_or_builds_from_source() { + let dir = fake_docker().clone(); + let git_ok = std::process::Command::new("git").arg("--version").output().is_ok_and(|o| o.status.success()); + if !git_ok { + eprintln!("skipping: git is not installed"); + return; + } + // A local "source repo" laid out like this one. + let src = tempdir().unwrap(); + std::fs::create_dir_all(src.path().join("plugins/video/runner")).unwrap(); + std::fs::write(src.path().join("plugins/video/runner/Dockerfile"), "FROM scratch\n").unwrap(); + std::fs::write(src.path().join("plugins/video/module.toml"), "[module]\n").unwrap(); + let git = |args: &[&str]| { + let ok = std::process::Command::new("git") + .args(["-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "init.defaultBranch=master"]) + .args(args) + .current_dir(src.path()) + .output() + .unwrap(); + assert!(ok.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&ok.stderr)); + }; + git(&["init", "-q"]); + git(&["add", "."]); + git(&["commit", "-qm", "runner"]); + // SAFETY: only this test reads it. + unsafe { std::env::set_var("C0MPUTE_VIDEO_SOURCE_REPO", format!("file://{}", src.path().display())) }; + + let scratch = tempdir().unwrap(); + let said = std::sync::Mutex::new(Vec::::new()); + let say = |l: &str| said.lock().unwrap().push(l.to_string()); + + // The registry refuses the pull: build from source. + let _ = std::fs::remove_file(dir.join("pull-ok")); + let got = video::setup(false, scratch.path(), &say).await.expect("setup falls back to a build"); + assert_eq!(got, video::SetupSource::Built); + let log = std::fs::read_to_string(dir.join("log")).unwrap(); + assert!(log.contains(&format!("pull {}", video::RUNNER_PULL_REF)), "{log}"); + let build = log.lines().find(|l| l.starts_with("build ")).expect("docker build ran"); + assert!(build.contains(&format!("-t {}", video::RUNNER_IMAGE)) && build.contains("plugins/video/runner/Dockerfile"), "{build}"); + assert!(said.lock().unwrap().iter().any(|l| l.contains("building it from source"))); + assert_eq!(std::fs::read_dir(scratch.path()).unwrap().count(), 0, "the scratch checkout is removed"); + + // The pull works: tag the pinned digest as the local name, no build. + std::fs::write(dir.join("pull-ok"), "").unwrap(); + let got = video::setup(false, scratch.path(), &say).await.unwrap(); + assert_eq!(got, video::SetupSource::Pulled); + let log = std::fs::read_to_string(dir.join("log")).unwrap(); + assert!(log.contains(&format!("tag {} {}", video::RUNNER_PULL_REF, video::RUNNER_IMAGE)), "{log}"); + let _ = std::fs::remove_file(dir.join("pull-ok")); +} diff --git a/plugins/gameserver/module.toml b/plugins/gameserver/module.toml index f023666e..57a363d3 100644 --- a/plugins/gameserver/module.toml +++ b/plugins/gameserver/module.toml @@ -1,26 +1,47 @@ # c0mpute plugin manifest. See dips/0006-module-model.md. # # Built in: workloads `gameserver.start` and `gameserver.stop` -# (node/crates/c0mpute-core/src/gameserver.rs, CLI c0mpute-cli/src/gameserver.rs). +# (node/crates/c0mpute-core/src/gameserver.rs and handoff.rs, CLI +# c0mpute-cli/src/gameserver.rs). # # Start is an ordinary auction with a short job: the winning worker runs # `docker create` (argv only, no shell) on a pinned image with --memory, -# --memory-swap, --cpus, --pids-limit 1024, no-new-privileges and --restart no, -# a named volume for the world and a published port block from its range, -# restores a world if asked, starts it and returns address:port plus the -# expiry. The server then runs for the reserved hours; the worker's reaper -# stops it at expiry. Images (multi-arch index digests): +# --memory-swap, --cpus, --pids-limit 1024, no-new-privileges, --restart no, +# --rm and --init, a named volume for the world and a published port block +# from its range, restores a world if asked, starts it and returns +# address:port plus the expiry. Images (multi-arch index digests): # minecraft itzg/minecraft-server:java21 (EULA=TRUE only with the buyer's --accept-eula) # factorio factoriotools/factorio:stable-2.1.24 # valheim lloesche/valheim-server (x86_64 only; buyer sets the password) # +# Secrets: nothing secret is in the offer (spec v2). The offer carries the +# buyer's fresh X25519 key; after the accept the winner announces its own and +# the buyer seals the Valheim password and any restore key to it +# (X25519 -> HKDF-SHA256 -> AES-256-GCM, gossip-signed both ways). +# # Stop: the start spec carries sha256(stop token); the token stays on the # buyer's machine until `gameserver stop`, and only the worker holding that -# server bids. On stop or expiry the worker tars the world out of the volume -# (`docker cp`), stores it as 1 MiB content-addressed chunks plus a manifest, -# announces them, and returns the manifest hash; the buyer downloads the tar -# (verified chunk by chunk and by sha256). `start --world ` serves it -# from the buyer's node and any worker restores it. No coordinator anywhere. +# server bids. +# +# Backups: every `backup_mins` (default 30) and at stop or expiry the worker +# tars the world out of the volume, encrypts it to the buyer's long-term +# X25519 backup key in 1 MiB AES-GCM chunks, and stores and announces the +# chunks plus a manifest. The manifest hash is in the stop receipt and in the +# worker's status record; only the buyer can read what it names. +# +# Health and failover: workers gossip a status record per server (container +# running, latest backup) on c0mpute/gameserver/v1, signed by their peer key. +# `c0mpute gameserver watch` probes the game port (Minecraft status ping, +# Valheim Steam A2S query, Factorio UDP reachability plus a fresh "running" +# record), logs every probe (`gameserver health`), mirrors each backup to the +# buyer's machine, and after `--fail-after-secs` (default 300) of failed checks +# starts the same world on another worker from the newest backup. No +# coordinator anywhere. +# +# Expiry: the server runs under `timeout ` behind --init and +# --rm, so it stops itself at expiry even if the worker is offline then. At +# boot and on every sweep the worker kills any labelled container past its +# expiry, then backs the world up from the volume and removes it. # # Worker: opt-in role (`c0mpute worker start --roles ...,gameserver`), never # default. Advertised only when `docker info` works, the node knows a public @@ -28,11 +49,8 @@ # C0MPUTE_P2P_EXTERNAL) and a pinned image has been pulled. The operator opens # [gameserver] port_min..port_max (default 27500-27599, TCP and UDP). # -# Not built: replicas / live failover, proof-of-serve (RCON or player -# heartbeat) and per-epoch settlement: a reservation is paid once, up front, -# at the price the auction cleared. Backups are readable by anyone who has the -# manifest hash (it is in the stop receipt), and a Valheim password rides in -# the public offer. +# Billing: one payment up front at the price the auction cleared. A failover +# is a new rental, paid the same way. [module] id = "gameserver" @@ -40,19 +58,19 @@ name = "Game server — Minecraft / Factorio / Valheim" version = "0.2.81" kind = "workload" category = "infrastructure" -status = "partial" -status_note = "Start, stop, expiry and world backup/restore work. Only opt-in workers with docker and a public IP whose firewall opens the port range can host, and few do yet. No replicas or proof-of-serve: one worker, paid once up front; if it goes offline the server is gone until it returns, and the world survives only in its last backup." +status = "available" builtin = true -description = "Rent a Minecraft, Factorio or Valheim server on a c0mpute worker for 1-72 hours: pinned docker images with RAM and CPU limits, address:port back in seconds, auto-stop at expiry, and the world backed up p2p by hash so any worker can restore it." +description = "Rent a Minecraft, Factorio or Valheim server on a c0mpute worker for 1-72 hours: pinned docker images with RAM and CPU limits, address:port back in seconds, a hard stop at expiry, world backups every 30 minutes encrypted to your key, and a watcher that probes the server and fails over to another worker from the latest backup." author = "Profullstack" license = "MIT" keywords = ["gameserver", "minecraft", "factorio", "valheim", "hosting", "docker"] [module.requirements] -c0mpute = ">=0.2.77" +c0mpute = ">=NEXT" os = ["linux"] arch = ["x86_64", "aarch64"] capabilities = ["container-runtime", "network:public-ip", "network:ingress"] +hardware = "opt-in worker role: docker + a public IP with the port range open (TCP and UDP)" [module.workloads] "gameserver.start" = { command = "start", validation = "result_schema" } @@ -72,3 +90,4 @@ max_total_ram_gb = 0 max_hours = 24 min_usd_per_gb_hour = 0.0 games = ["minecraft", "factorio", "valheim"] +backup_mins = 30 diff --git a/plugins/video/install.sh b/plugins/video/install.sh index 0ceb8e3d..744a8fe9 100755 --- a/plugins/video/install.sh +++ b/plugins/video/install.sh @@ -11,7 +11,11 @@ # Source: https://github.com/profullstack/c0mpute/tree/master/plugins/video set -eu -IMAGE="${C0MPUTE_VIDEO_IMAGE:-ghcr.io/profullstack/c0mpute-video-runner:candidate}" +# The published runner, pinned by digest (keep in step with module.toml +# [module.dispatch] and RUNNER_DIGEST in video.rs), and the local name the +# node runs, whichever way the image arrives. +PULL_REF="${C0MPUTE_VIDEO_PULL_REF:-ghcr.io/profullstack/c0mpute-video-runner@sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c}" +IMAGE="c0mpute/video-runner:stable" SOURCE_REPO="${C0MPUTE_VIDEO_SOURCE_REPO:-https://github.com/profullstack/c0mpute.git}" say() { printf '\033[1;36m→\033[0m %s\n' "$*"; } @@ -43,12 +47,12 @@ else fi if [ "$missing" -eq 0 ]; then - say "pulling $IMAGE (several GB, once)" - if ! docker pull "$IMAGE"; then + say "pulling $PULL_REF (several GB, once)" + if ! { docker pull "$PULL_REF" && docker tag "$PULL_REF" "$IMAGE"; }; then # The registry is a convenience, not a dependency: private, rate-limited # or offline is no reason a box with a card cannot serve video. This repo # is public, so build the same image from source. - warn "could not pull $IMAGE (private, rate-limited or offline)" + warn "could not pull $PULL_REF (private, rate-limited or offline)" if command -v git >/dev/null 2>&1; then say "building it from source instead (several minutes)" src=$(mktemp -d) diff --git a/plugins/video/module.toml b/plugins/video/module.toml index c239ee4f..636357b1 100644 --- a/plugins/video/module.toml +++ b/plugins/video/module.toml @@ -5,9 +5,15 @@ # published as the image below; the node's video runner invokes it per job and # reads files, never stdout. See that directory's README for the job contract. # -# The image below is a convenience: `c0mpute video setup` falls back to -# building it from this public repo when the registry will not serve it, so a -# node with a GPU never needs a registry account. +# The image below is pinned by digest (the same digest as RUNNER_DIGEST in +# node/crates/c0mpute-core/src/video.rs, checked by a unit test). +# `c0mpute video setup` pulls it and tags it locally as +# c0mpute/video-runner:stable, the name the worker runs. If the registry will +# not serve it, setup builds the same Dockerfile from this public repo, so a +# node with a GPU never needs a registry account. The workflow +# .github/workflows/video-runner.yml publishes :stable, :candidate and : +# on every master build and prints the new digest; bump both pins to it after +# a runner change. # # Backend: Diffusers, not ComfyUI. ComfyUI would mean shipping hand-authored # API graph JSON per model, which cannot be checked by reading it; the @@ -47,18 +53,19 @@ name = "Video — generation (Wan 2.2, LTX-2.3 with audio)" version = "0.2.81" kind = "workload" category = "ai" -status = "partial" -status_note = "Needs an NVIDIA GPU and the c0mpute video runner image." +status = "available" +builtin = true description = "Decentralized video generation on consumer/prosumer GPUs: Wan 2.2 text/image-to-video, and LTX-2.3 video with a synchronized soundtrack." author = "Profullstack" license = "MIT" keywords = ["video", "wan", "wan2.2", "ltx-2", "ltx-2.3", "audio", "txt2vid", "img2vid", "generation", "diffusion"] [module.requirements] -c0mpute = ">=0.1.0" +c0mpute = ">=NEXT" os = ["linux"] arch = ["x86_64"] capabilities = ["gpu:nvidia", "model-runtime", "container-runtime"] +hardware = "NVIDIA GPU (24 GiB VRAM for the default model, 48 GiB for the 14B and LTX models) + docker with the NVIDIA container runtime" [module.workloads] "video.txt2vid" = { command = "txt2vid", validation = "schema_and_ffprobe" } @@ -80,7 +87,7 @@ capabilities = ["gpu:nvidia", "model-runtime", "container-runtime"] [module.dispatch] mode = "container" -image = "ghcr.io/profullstack/c0mpute-video-runner:candidate" +image = "ghcr.io/profullstack/c0mpute-video-runner@sha256:2ba52bedfaa2f8b0bb76a2778a3184c6baf591b9879c7fa090abb56bbddf141c" [module.surfaces] cli = "c0mpute video" diff --git a/plugins/video/runner/Dockerfile b/plugins/video/runner/Dockerfile index c28a48cd..ce752640 100644 --- a/plugins/video/runner/Dockerfile +++ b/plugins/video/runner/Dockerfile @@ -11,7 +11,7 @@ # docker run --rm --gpus all \ # -v c0mpute-video-weights:/weights \ # -v "$PWD/job:/job" \ -# ghcr.io/profullstack/c0mpute-video-runner:candidate txt2vid +# c0mpute/video-runner:stable txt2vid # # No CUDA base image on purpose: the torch wheels from PyPI bring their own # CUDA runtime, so only the driver comes from the host via the NVIDIA diff --git a/plugins/video/runner/README.md b/plugins/video/runner/README.md index 056779dc..efe3267b 100644 --- a/plugins/video/runner/README.md +++ b/plugins/video/runner/README.md @@ -4,8 +4,12 @@ The container the `video` plugin dispatches to. One job per process: read `job.json`, generate a clip, encode an MP4, write a sidecar, exit non-zero on any failure. -Published as `ghcr.io/profullstack/c0mpute-video-runner`. The tag a node pulls -is the one in [`../module.toml`](../module.toml) under `[module.dispatch]`. +Published as `ghcr.io/profullstack/c0mpute-video-runner` with the tags +`:stable`, `:candidate` and `:` on every master build. A node +pulls by digest: the one in [`../module.toml`](../module.toml) under +`[module.dispatch]` (and `RUNNER_DIGEST` in `video.rs`), then tags it locally +as `c0mpute/video-runner:stable`, the name the worker runs. After a runner +change, copy the digest from the workflow's job summary into both pins. **The registry is a convenience, not a dependency.** `c0mpute video setup` pulls the image, and if the registry will not serve it — private package,