From 7fa5dc360d0791f1b5c57d8b78264a2182506db1 Mon Sep 17 00:00:00 2001 From: Vasili Pascal Date: Thu, 15 Oct 2026 10:41:19 +0300 Subject: [PATCH 1/2] Build bake in CI and ship it in the Docker image bake turns a deployed build box into a Quick Deploy snapshot. It was not in the image, so every bake meant compiling it by hand and `docker cp` into the running container. CI now compiles it with the other release binaries, archives and unpacks it, and both Dockerfile paths (prebuilt from CI and the local builder stage) copy it to /app/bake. tests/image_binaries_contract.rs checks every image binary on both paths (failed for bake before this change). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012TKZy6f5shPzM9WLdswH6f --- .github/workflows/docker.yml | 8 +-- Dockerfile | 5 +- tests/image_binaries_contract.rs | 87 ++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 tests/image_binaries_contract.rs diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8b97853b..19b07e10 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -163,7 +163,8 @@ jobs: cargo build --release \ --bin server \ --bin cleanup-notify \ - --bin backfill_field_policy + --bin backfill_field_policy \ + --bin bake cargo build --release --bin console --features explain - name: Set up Node.js @@ -202,6 +203,7 @@ jobs: cp target/release/console app/stacker/console cp target/release/cleanup-notify app/stacker/cleanup-notify cp target/release/backfill_field_policy app/stacker/backfill_field_policy + cp target/release/bake app/stacker/bake if [ -d web/dist ]; then cp -a web/dist/. app/stacker; fi cp Dockerfile app/Dockerfile cp access_control.conf.dist app/access_control.conf.dist @@ -248,8 +250,8 @@ jobs: tar -xzf app.tar.gz -C "${{ runner.temp }}/prebuilt" cd "${{ runner.temp }}/prebuilt" mv stacker/server stacker/console stacker/cleanup-notify \ - stacker/backfill_field_policy . - chmod +x server console cleanup-notify backfill_field_policy + stacker/backfill_field_policy stacker/bake . + chmod +x server console cleanup-notify backfill_field_policy bake rm -f "${GITHUB_WORKSPACE}/app.tar.gz" - diff --git a/Dockerfile b/Dockerfile index 50d6a87f..7dd0e04f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,8 @@ RUN apt-get update && apt-get install --no-install-recommends -y libssl-dev; \ cargo build --release --bin server; \ cargo build --release --bin console --features explain; \ cargo build --release --bin cleanup-notify; \ - cargo build --release --bin backfill_field_policy + cargo build --release --bin backfill_field_policy; \ + cargo build --release --bin bake #RUN ls -la /app/target/release/ >&2 @@ -78,6 +79,7 @@ COPY --from=builder /app/target/release/server /server COPY --from=builder /app/target/release/console /console COPY --from=builder /app/target/release/cleanup-notify /cleanup-notify COPY --from=builder /app/target/release/backfill_field_policy /backfill_field_policy +COPY --from=builder /app/target/release/bake /bake FROM ${BINARIES}-source AS binaries @@ -94,6 +96,7 @@ COPY --from=binaries /server . COPY --from=binaries /console . COPY --from=binaries /cleanup-notify . COPY --from=binaries /backfill_field_policy . +COPY --from=binaries /bake . COPY --from=config /app/.env . COPY --from=config /app/configuration.yaml . COPY --from=config /usr/local/cargo/bin/sqlx /usr/local/bin/sqlx diff --git a/tests/image_binaries_contract.rs b/tests/image_binaries_contract.rs new file mode 100644 index 00000000..0b90f4d4 --- /dev/null +++ b/tests/image_binaries_contract.rs @@ -0,0 +1,87 @@ +//! Every binary the stacker image ships is built and copied on both paths into +//! the image: the CI job (`.github/workflows/docker.yml` compiles, archives, +//! unpacks, and the Dockerfile copies it in as `prebuilt`) and a local +//! `docker build` (the Dockerfile's own `builder` stage). +//! +//! `bake` turns a deployed build box into a Quick Deploy snapshot. It was left +//! out of the image, so every bake meant compiling it by hand and `docker cp` +//! into the running production container. + +use std::fs; +use std::path::PathBuf; + +/// The binaries in /app of the runtime image. +const IMAGE_BINARIES: &[&str] = &[ + "server", + "console", + "cleanup-notify", + "backfill_field_policy", + "bake", +]; + +fn read(file: &str) -> String { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(file); + fs::read_to_string(&path).unwrap_or_else(|err| panic!("read {file}: {err}")) +} + +/// Lines that are not comments, so commented-out history does not count. +fn active(text: &str) -> Vec { + text.lines() + .filter(|line| !line.trim_start().starts_with('#')) + .map(str::to_string) + .collect() +} + +#[test] +fn the_dockerfile_builds_and_ships_every_image_binary() { + let dockerfile = active(&read("Dockerfile")); + for bin in IMAGE_BINARIES { + assert!( + dockerfile + .iter() + .any(|l| l.contains("cargo build --release") && l.contains(&format!("--bin {bin}"))), + "builder stage must compile `{bin}`" + ); + assert!( + dockerfile.iter().any( + |l| l.trim() == format!("COPY --from=builder /app/target/release/{bin} /{bin}") + ), + "builder-source must take `{bin}` from the builder stage" + ); + assert!( + dockerfile + .iter() + .any(|l| l.trim() == format!("COPY --from=binaries /{bin} .")), + "the runtime image must contain `{bin}`" + ); + } +} + +#[test] +fn ci_builds_archives_and_unpacks_every_image_binary() { + let workflow = active(&read(".github/workflows/docker.yml")).join("\n"); + for bin in IMAGE_BINARIES { + assert!( + workflow.contains(&format!("--bin {bin}")), + "CI must compile `{bin}`" + ); + assert!( + workflow.contains(&format!("cp target/release/{bin} app/stacker/{bin}")), + "CI must put `{bin}` in the artifact" + ); + assert!( + workflow.contains(&format!("stacker/{bin}")), + "the docker job must unpack `{bin}` from the artifact" + ); + } + let chmod = workflow + .lines() + .find(|l| l.trim_start().starts_with("chmod +x")) + .expect("the docker job makes the binaries executable"); + for bin in IMAGE_BINARIES { + assert!( + chmod.split_whitespace().any(|word| word == *bin), + "`{bin}` must be made executable: {chmod}" + ); + } +} From 79b51278aee67864d2d15414bfb1783bf485df62 Mon Sep 17 00:00:00 2001 From: Vasili Pascal Date: Thu, 15 Oct 2026 11:04:00 +0300 Subject: [PATCH 2/2] Start a bake from the API: POST /api/admin/bake POST /api/admin/bake {deployment_id, stack, health_url} bakes a deployed build box into a Quick Deploy snapshot; GET /api/admin/bake/{id} reports the job. Admins only (group_admin, admin_service), like template approval. What the bake binary needs typed by hand comes from stacker itself: the server row of the deployment's project (IP, SSH user and port), its SSH key in Vault, and the latest approved version of the template with its config_contract. Refused at once, with no job recorded and the box untouched: unknown deployment, not exactly one server, a template that is not approved, no or unusable contract, a health URL that is not on the box, a box Hetzner does not know in the platform's project, a bake of the same box already running. The job (202 + id, status in the new bake_job table) then checks the health URL, finds the stack's directory with `docker compose ls` and requires every service up (`docker compose ps`), and runs the existing chain unchanged: finalize_build_box, run_bake, baked_snapshot::record. The build box is never removed. Tests: helpers::bake_job (7, written failing first); tests/admin_bake.rs (refusals, accepted job, status, non-admin refused; a mutation that drops the Hetzner lookup fails it). lib + route matrix 2270 passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012TKZy6f5shPzM9WLdswH6f --- migrations/20261015100000_bake_job.down.sql | 2 + migrations/20261015100000_bake_job.up.sql | 23 ++ src/db/bake_job.rs | 80 +++++ src/db/mod.rs | 1 + src/helpers/bake_job.rs | 250 ++++++++++++++ src/helpers/mod.rs | 1 + src/models/bake_job.rs | 22 ++ src/models/mod.rs | 2 + src/project_app/mod.rs | 2 +- src/routes/admin_bake.rs | 347 ++++++++++++++++++++ src/routes/mod.rs | 1 + src/startup.rs | 5 + tests/admin_bake.rs | 215 ++++++++++++ 13 files changed, 950 insertions(+), 1 deletion(-) create mode 100644 migrations/20261015100000_bake_job.down.sql create mode 100644 migrations/20261015100000_bake_job.up.sql create mode 100644 src/db/bake_job.rs create mode 100644 src/helpers/bake_job.rs create mode 100644 src/models/bake_job.rs create mode 100644 src/routes/admin_bake.rs create mode 100644 tests/admin_bake.rs diff --git a/migrations/20261015100000_bake_job.down.sql b/migrations/20261015100000_bake_job.down.sql new file mode 100644 index 00000000..2d6d1e24 --- /dev/null +++ b/migrations/20261015100000_bake_job.down.sql @@ -0,0 +1,2 @@ +DELETE FROM public.casbin_rule WHERE ptype = 'p' AND v1 IN ('/api/admin/bake', '/api/admin/bake/:id'); +DROP TABLE IF EXISTS bake_job; diff --git a/migrations/20261015100000_bake_job.up.sql b/migrations/20261015100000_bake_job.up.sql new file mode 100644 index 00000000..8d0382b9 --- /dev/null +++ b/migrations/20261015100000_bake_job.up.sql @@ -0,0 +1,23 @@ +-- Bake started from the API: one row per POST /api/admin/bake. +CREATE TABLE IF NOT EXISTS bake_job ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + stack TEXT NOT NULL, + version TEXT NOT NULL, + deployment_id INTEGER NOT NULL, + server_id INTEGER NOT NULL, + server_ip TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'running' CHECK (status IN ('running', 'done', 'failed')), + image_id BIGINT, + snapshot_id INTEGER, + error TEXT, + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS bake_job_created_at_idx ON bake_job (created_at DESC); + +INSERT INTO public.casbin_rule (ptype, v0, v1, v2, v3, v4, v5) VALUES ('p', 'group_admin', '/api/admin/bake', 'POST', '', '', '') ON CONFLICT ON CONSTRAINT unique_key_sqlx_adapter DO NOTHING; +INSERT INTO public.casbin_rule (ptype, v0, v1, v2, v3, v4, v5) VALUES ('p', 'group_admin', '/api/admin/bake/:id', 'GET', '', '', '') ON CONFLICT ON CONSTRAINT unique_key_sqlx_adapter DO NOTHING; +INSERT INTO public.casbin_rule (ptype, v0, v1, v2, v3, v4, v5) VALUES ('p', 'admin_service', '/api/admin/bake', 'POST', '', '', '') ON CONFLICT ON CONSTRAINT unique_key_sqlx_adapter DO NOTHING; +INSERT INTO public.casbin_rule (ptype, v0, v1, v2, v3, v4, v5) VALUES ('p', 'admin_service', '/api/admin/bake/:id', 'GET', '', '', '') ON CONFLICT ON CONSTRAINT unique_key_sqlx_adapter DO NOTHING; diff --git a/src/db/bake_job.rs b/src/db/bake_job.rs new file mode 100644 index 00000000..04ce2e2d --- /dev/null +++ b/src/db/bake_job.rs @@ -0,0 +1,80 @@ +use crate::models::BakeJob; +use sqlx::PgPool; + +pub struct NewBakeJob<'a> { + pub stack: &'a str, + pub version: &'a str, + pub deployment_id: i32, + pub server_id: i32, + pub server_ip: &'a str, + pub created_by: &'a str, +} + +pub async fn insert(pool: &PgPool, job: NewBakeJob<'_>) -> Result { + sqlx::query_as::<_, BakeJob>( + r#" + INSERT INTO bake_job (stack, version, deployment_id, server_id, server_ip, created_by) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING * + "#, + ) + .bind(job.stack) + .bind(job.version) + .bind(job.deployment_id) + .bind(job.server_id) + .bind(job.server_ip) + .bind(job.created_by) + .fetch_one(pool) + .await + .map_err(|e| format!("Failed to record bake job: {e}")) +} + +pub async fn fetch(pool: &PgPool, id: uuid::Uuid) -> Result, String> { + sqlx::query_as::<_, BakeJob>("SELECT * FROM bake_job WHERE id = $1") + .bind(id) + .fetch_optional(pool) + .await + .map_err(|e| format!("Failed to read bake job: {e}")) +} + +/// A bake already running for this build box. One at a time: two would +/// sanitize and snapshot the same box concurrently. +pub async fn running_for_server(pool: &PgPool, server_id: i32) -> Result, String> { + sqlx::query_as::<_, BakeJob>( + "SELECT * FROM bake_job WHERE server_id = $1 AND status = 'running' LIMIT 1", + ) + .bind(server_id) + .fetch_optional(pool) + .await + .map_err(|e| format!("Failed to read bake jobs: {e}")) +} + +pub async fn mark_done( + pool: &PgPool, + id: uuid::Uuid, + image_id: i64, + snapshot_id: i32, +) -> Result<(), String> { + sqlx::query( + "UPDATE bake_job SET status = 'done', image_id = $2, snapshot_id = $3, updated_at = NOW() WHERE id = $1", + ) + .bind(id) + .bind(image_id) + .bind(snapshot_id) + .execute(pool) + .await + .map(|_| ()) + .map_err(|e| format!("Failed to finish bake job: {e}")) +} + +pub async fn mark_failed(pool: &PgPool, id: uuid::Uuid, error: &str) -> Result<(), String> { + sqlx::query( + "UPDATE bake_job SET status = 'failed', error = $2, updated_at = NOW() WHERE id = $1", + ) + .bind(id) + .bind(error) + .execute(pool) + .await + .map(|_| ()) + .map_err(|e| format!("Failed to mark bake job failed: {e}")) +} diff --git a/src/db/mod.rs b/src/db/mod.rs index 831b4172..ba5d8f72 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,6 +1,7 @@ pub mod agent; pub mod agent_audit_log; pub(crate) mod agreement; +pub mod bake_job; pub mod baked_snapshot; pub mod chat; pub mod chat_admin; diff --git a/src/helpers/bake_job.rs b/src/helpers/bake_job.rs new file mode 100644 index 00000000..3e73d10d --- /dev/null +++ b/src/helpers/bake_job.rs @@ -0,0 +1,250 @@ +//! Bake started from the API (`POST /api/admin/bake`) instead of the `bake` +//! binary. +//! +//! The binary needs the operator to type the box's IP, its SSH key, the +//! version and the health URL. Here they come from what stacker already +//! holds: the deployment's server row (IP, SSH user and port), the server's +//! SSH key in Vault, and the template's latest approved version. These pure +//! helpers check what the box reports before anything on it is changed. + +/// The stack's project directory on the build box, from +/// `docker compose ls --all --format json`. +/// +/// The deploy puts each stack in its own directory under the deploy root; +/// platform services (status panel, proxy manager) have their own. Anything +/// but exactly one project directory is refused: sanitizing the wrong one +/// would publish the author's secrets. +pub fn project_dir_from_compose_ls(stdout: &str, platform_dirs: &[&str]) -> Result { + let projects: Vec = serde_json::from_str(stdout.trim()) + .map_err(|e| format!("could not read `docker compose ls` output: {e}"))?; + + let mut dirs: Vec = projects + .iter() + .filter_map(|p| p.get("ConfigFiles").and_then(|v| v.as_str())) + .filter_map(|files| files.split(',').next()) + .filter_map(|file| file.trim().rsplit_once('/').map(|(dir, _)| dir.to_string())) + .filter(|dir| !platform_dirs.contains(&dir.as_str())) + .collect(); + dirs.sort(); + dirs.dedup(); + + match dirs.as_slice() { + [dir] if is_safe_dir(dir) => Ok(dir.clone()), + [dir] => Err(format!("refusing the stack directory `{dir}`: not a plain absolute path")), + [] => Err("no stack is deployed on the build box (only platform services)".to_string()), + many => Err(format!( + "more than one stack is deployed on the build box: {}. Bake needs a box with one stack.", + many.join(", ") + )), + } +} + +/// An absolute path of letters, digits and `/ _ . -` only: it goes into a +/// shell command on the box. +fn is_safe_dir(dir: &str) -> bool { + dir.starts_with('/') + && !dir.contains("..") + && dir + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '/' | '_' | '.' | '-')) +} + +/// Every service of the stack is up, from `docker compose ps --all --format +/// json` (one JSON object per line in compose v2.21+, a JSON array before). +/// +/// Running without a healthcheck or healthy counts as up; a one-shot job that +/// exited 0 (migrations, init) is done, not down. +pub fn check_compose_services(stdout: &str) -> Result<(), String> { + let text = stdout.trim(); + let services: Vec = if text.starts_with('[') { + serde_json::from_str(text) + .map_err(|e| format!("could not read `docker compose ps`: {e}"))? + } else { + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(serde_json::from_str) + .collect::>() + .map_err(|e| format!("could not read `docker compose ps`: {e}"))? + }; + if services.is_empty() { + return Err("the stack has no services on the build box".to_string()); + } + + let mut down = Vec::new(); + for service in &services { + let field = |name: &str| service.get(name).and_then(|v| v.as_str()).unwrap_or(""); + let name = field("Service"); + let state = field("State"); + let health = field("Health"); + let exit_code = service + .get("ExitCode") + .and_then(|v| v.as_i64()) + .unwrap_or(-1); + let up = match state { + "running" => health.is_empty() || health == "healthy", + "exited" => exit_code == 0, + _ => false, + }; + if !up { + let detail = if health.is_empty() { + state.to_string() + } else { + format!("{state}, {health}") + }; + down.push(format!("{name} ({detail})")); + } + } + if down.is_empty() { + Ok(()) + } else { + Err(format!("services not up: {}", down.join(", "))) + } +} + +/// The health URL must point at the build box itself: the bake runs on the +/// stacker backend, so any other host would make it a request the caller +/// chose from inside the platform's network. +pub fn check_health_url(url: &str, server_ip: &str) -> Result<(), String> { + let parsed = reqwest::Url::parse(url).map_err(|e| format!("health_url is not a URL: {e}"))?; + if !matches!(parsed.scheme(), "http" | "https") { + return Err("health_url must be http or https".to_string()); + } + if !parsed.username().is_empty() || parsed.password().is_some() { + return Err("health_url must not carry credentials".to_string()); + } + match parsed.host_str() { + Some(host) if host == server_ip => Ok(()), + _ => Err(format!( + "health_url must point at the build box ({server_ip})" + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const PLATFORM: &[&str] = &[ + "/home/trydirect/statuspanel", + "/home/trydirect/nginx_proxy_manager", + ]; + + #[test] + fn the_stack_directory_is_found_next_to_platform_services() { + let ls = r#"[ + {"Name":"statuspanel","Status":"running(1)","ConfigFiles":"/home/trydirect/statuspanel/docker-compose.yml"}, + {"Name":"ai-workflows-v2-12","Status":"running(4)","ConfigFiles":"/home/trydirect/ai-workflows-v2-12/docker-compose.yml"} + ]"#; + assert_eq!( + project_dir_from_compose_ls(ls, PLATFORM).unwrap(), + "/home/trydirect/ai-workflows-v2-12" + ); + } + + #[test] + fn several_config_files_of_one_project_point_at_its_directory() { + let ls = r#"[{"Name":"app","Status":"running(2)","ConfigFiles":"/home/trydirect/app/docker-compose.yml,/home/trydirect/app/docker-compose.override.yml"}]"#; + assert_eq!( + project_dir_from_compose_ls(ls, PLATFORM).unwrap(), + "/home/trydirect/app" + ); + } + + #[test] + fn no_stack_or_two_stacks_are_refused() { + let only_platform = r#"[{"Name":"statuspanel","Status":"running(1)","ConfigFiles":"/home/trydirect/statuspanel/docker-compose.yml"}]"#; + assert!(project_dir_from_compose_ls(only_platform, PLATFORM).is_err()); + + let two = r#"[ + {"Name":"a","Status":"running(1)","ConfigFiles":"/home/trydirect/a/docker-compose.yml"}, + {"Name":"b","Status":"running(1)","ConfigFiles":"/home/trydirect/b/docker-compose.yml"} + ]"#; + let err = project_dir_from_compose_ls(two, PLATFORM).unwrap_err(); + assert!( + err.contains("/home/trydirect/a") && err.contains("/home/trydirect/b"), + "{err}" + ); + + assert!(project_dir_from_compose_ls("not json", PLATFORM).is_err()); + } + + #[test] + fn a_directory_unsafe_for_a_shell_command_is_refused() { + // The directory is put into `cd && ...` on the box. + for dir in [ + "/home/trydirect/a b", + "/home/trydirect/a;rm -rf /", + "/home/trydirect/$(id)", + "relative/dir", + ] { + let ls = format!( + r#"[{{"Name":"x","Status":"running(1)","ConfigFiles":"{dir}/docker-compose.yml"}}]"# + ); + assert!(project_dir_from_compose_ls(&ls, PLATFORM).is_err(), "{dir}"); + } + } + + #[test] + fn services_running_healthy_or_finished_are_up() { + let ps = "\ +{\"Service\":\"n8n\",\"State\":\"running\",\"Health\":\"healthy\",\"ExitCode\":0} +{\"Service\":\"redis\",\"State\":\"running\",\"Health\":\"\",\"ExitCode\":0} +{\"Service\":\"migrate\",\"State\":\"exited\",\"Health\":\"\",\"ExitCode\":0}"; + assert_eq!(check_compose_services(ps), Ok(())); + + let array = r#"[{"Service":"n8n","State":"running","Health":"healthy","ExitCode":0}]"#; + assert_eq!(check_compose_services(array), Ok(())); + } + + #[test] + fn a_starting_unhealthy_crashed_or_missing_service_fails_the_gate() { + for (ps, culprit) in [ + ( + r#"{"Service":"db","State":"running","Health":"starting","ExitCode":0}"#, + "db", + ), + ( + r#"{"Service":"db","State":"running","Health":"unhealthy","ExitCode":0}"#, + "db", + ), + ( + r#"{"Service":"api","State":"exited","Health":"","ExitCode":1}"#, + "api", + ), + ( + r#"{"Service":"api","State":"restarting","Health":"","ExitCode":1}"#, + "api", + ), + ] { + let err = check_compose_services(ps).unwrap_err(); + assert!(err.contains(culprit), "{err}"); + } + assert!( + check_compose_services("").is_err(), + "no services is not healthy" + ); + assert!(check_compose_services("garbage").is_err()); + } + + #[test] + fn the_health_url_must_point_at_the_build_box() { + assert_eq!( + check_health_url("http://203.0.113.7/health", "203.0.113.7"), + Ok(()) + ); + assert_eq!( + check_health_url("https://203.0.113.7:8443/api/ping", "203.0.113.7"), + Ok(()) + ); + for url in [ + "http://198.51.100.1/health", + "http://203.0.113.7.evil.example/health", + "http://user@198.51.100.1/", + "file:///etc/passwd", + "http://localhost/health", + "not a url", + ] { + assert!(check_health_url(url, "203.0.113.7").is_err(), "{url}"); + } + } +} diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs index 95919c6d..3a6876aa 100644 --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -36,6 +36,7 @@ pub use dockerhub::*; pub use cloud::*; pub mod audit_cache; pub mod bake; +pub mod bake_job; pub mod bake_finalize; pub mod bake_registry; pub mod cloud_init; diff --git a/src/models/bake_job.rs b/src/models/bake_job.rs new file mode 100644 index 00000000..ea745c05 --- /dev/null +++ b/src/models/bake_job.rs @@ -0,0 +1,22 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// A bake started through `POST /api/admin/bake`. `running` until the job ends +/// as `done` (with the snapshot) or `failed` (with the reason). The build box +/// (`server_id`, `server_ip`) is never removed by the job. +#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow)] +pub struct BakeJob { + pub id: uuid::Uuid, + pub stack: String, + pub version: String, + pub deployment_id: i32, + pub server_id: i32, + pub server_ip: String, + pub status: String, + pub image_id: Option, + pub snapshot_id: Option, + pub error: Option, + pub created_by: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} diff --git a/src/models/mod.rs b/src/models/mod.rs index b99d3855..cac5b82c 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -2,6 +2,7 @@ mod agent; pub mod agent_audit_log; pub mod agent_protocol; mod agreement; +pub mod bake_job; pub mod baked_snapshot; pub mod cdc; mod chat; @@ -31,6 +32,7 @@ pub mod user; pub use agent::*; pub use agent_audit_log::AgentAuditLog; pub use agreement::*; +pub use bake_job::*; pub use baked_snapshot::*; pub use chat::*; pub use client::*; diff --git a/src/project_app/mod.rs b/src/project_app/mod.rs index 49d31a0d..38a4f5f1 100644 --- a/src/project_app/mod.rs +++ b/src/project_app/mod.rs @@ -50,7 +50,7 @@ const PLATFORM_DISPLAY_CODES: &[&str] = &[ /// Directories platform-managed services are deployed into, by the convention /// in `stacker/docs/APP_DEPLOYMENT.md`: project services live under the /// project directory, platform ones get their own. -const PLATFORM_INSTALL_DIRS: &[&str] = &[ +pub(crate) const PLATFORM_INSTALL_DIRS: &[&str] = &[ "/home/trydirect/statuspanel", "/home/trydirect/nginx_proxy_manager", ]; diff --git a/src/routes/admin_bake.rs b/src/routes/admin_bake.rs new file mode 100644 index 00000000..762b8b9c --- /dev/null +++ b/src/routes/admin_bake.rs @@ -0,0 +1,347 @@ +//! `POST /api/admin/bake` and `GET /api/admin/bake/{id}`: bake a deployed +//! build box into a Quick Deploy snapshot without the operator typing its IP, +//! SSH key or version. +//! +//! Everything that can be checked without touching the box is checked before +//! the job starts, and answered at once: the deployment and its server, the +//! approved template with a usable contract, a health URL on the box, and the +//! box being one Hetzner knows. Only then does the job connect, check that +//! every service is up, sanitize the box, snapshot it and record the snapshot. +//! The box is never removed: the result lists it for the operator. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use actix_web::{get, post, web, HttpResponse, Responder, Result}; +use serde::Deserialize; +use sqlx::PgPool; + +use crate::connectors::config::HetznerConfig; +use crate::connectors::hetzner::{ + HetznerCloudClient, HetznerCloudConnector, HetznerSnapshotTarget, +}; +use crate::db; +use crate::helpers::bake_job::{ + check_compose_services, check_health_url, project_dir_from_compose_ls, +}; +use crate::helpers::{JsonResponse, VaultClient}; +use crate::models; + +#[derive(Debug, Deserialize)] +pub struct BakeRequest { + /// The build box's deployment (from the CLI deployment lock). + pub deployment_id: i32, + /// Approved marketplace template slug; its latest version is baked. + pub stack: String, + /// The stack's health endpoint on the build box. + pub health_url: String, +} + +/// What the job needs, all resolved and checked before it starts. +struct Prepared { + server: models::Server, + server_ip: String, + version: String, + contract: serde_json::Value, + parsed_contract: crate::cli::config_parser::ConfigContract, + protected_keys: BTreeSet, + token: String, + connector: Arc, + target: HetznerSnapshotTarget, +} + +async fn prepare(pool: &PgPool, form: &BakeRequest) -> Result { + let deployment = db::deployment::fetch(pool, form.deployment_id) + .await + .map_err(JsonResponse::internal_server_error)? + .ok_or_else(|| JsonResponse::not_found("deployment not found"))?; + + let servers: Vec = db::server::fetch_by_project(pool, deployment.project_id) + .await + .map_err(JsonResponse::internal_server_error)? + .into_iter() + .filter(|s| { + s.srv_ip + .as_deref() + .map_or(false, |ip| !ip.trim().is_empty()) + }) + .collect(); + let server = match servers.as_slice() { + [server] => server.clone(), + [] => { + return Err(JsonResponse::<()>::build() + .conflict("the deployment's project has no server with an IP")) + } + _ => { + return Err(JsonResponse::<()>::build().conflict( + "the deployment's project has more than one server; bake needs one build box", + )) + } + }; + let server_ip = server.srv_ip.clone().unwrap_or_default(); + + check_health_url(&form.health_url, &server_ip).map_err(JsonResponse::bad_request)?; + + // The latest approved version: the contract describes it, so version and + // contract cannot disagree. + let (template, version) = db::marketplace::get_by_slug_with_latest(pool, &form.stack) + .await + .map_err(|_| JsonResponse::not_found("no approved template with this slug"))?; + let version = version + .map(|v| v.version) + .ok_or_else(|| JsonResponse::<()>::build().conflict("the template has no version"))?; + let contract = db::marketplace::get_config_contract(pool, template.id) + .await + .map_err(JsonResponse::internal_server_error)?; + if contract.is_null() { + return Err(JsonResponse::<()>::build() + .conflict("the template declares no config_contract; nothing would be sanitized")); + } + let protected_keys = crate::helpers::bake_finalize::protected_keys_from_contract(&contract); + let parsed_contract = serde_json::from_value(contract.clone()).map_err(|e| { + JsonResponse::<()>::build().conflict(format!( + "the stored config_contract could not be parsed: {e}" + )) + })?; + crate::helpers::bake_finalize::check_contract_usable(&protected_keys, false) + .map_err(|e| JsonResponse::<()>::build().conflict(e.to_string()))?; + + let htz = HetznerConfig::from_env(); + let token = htz.token.filter(|t| !t.trim().is_empty()).ok_or_else(|| { + JsonResponse::internal_server_error("HETZNER_TOKEN is not set on the stacker backend") + })?; + let client = HetznerCloudClient::new(htz.base_url) + .map_err(|e| JsonResponse::internal_server_error(e.to_string()))?; + let connector: Arc = Arc::new(client); + let target = HetznerSnapshotTarget { + provider_server_id: None, + server_name: None, + public_ip: Some(server_ip.clone()), + }; + // Sanitizing removes SSH access, so a box Hetzner cannot match must be + // refused before anything on it changes. + let provider_server_id = connector + .resolve_snapshot_target(&token, &target) + .await + .map_err(|e| { + JsonResponse::<()>::build().conflict(format!( + "Hetzner does not know a server at {server_ip} in the platform's project: {e}" + )) + })?; + let target = HetznerSnapshotTarget { + provider_server_id: Some(provider_server_id), + ..target + }; + + Ok(Prepared { + server, + server_ip, + version, + contract, + parsed_contract, + protected_keys, + token, + connector, + target, + }) +} + +#[tracing::instrument(name = "Start a bake (admin)", skip_all)] +#[post("")] +pub async fn start_handler( + admin: web::ReqData>, // role enforced by Casbin + form: web::Json, + pg_pool: web::Data, + vault_client: web::Data, +) -> Result { + let form = form.into_inner(); + let prepared = prepare(pg_pool.get_ref(), &form).await?; + + if db::bake_job::running_for_server(pg_pool.get_ref(), prepared.server.id) + .await + .map_err(JsonResponse::internal_server_error)? + .is_some() + { + return Err( + JsonResponse::<()>::build().conflict("a bake of this build box is already running") + ); + } + + let job = db::bake_job::insert( + pg_pool.get_ref(), + db::bake_job::NewBakeJob { + stack: &form.stack, + version: &prepared.version, + deployment_id: form.deployment_id, + server_id: prepared.server.id, + server_ip: &prepared.server_ip, + created_by: &admin.id, + }, + ) + .await + .map_err(JsonResponse::internal_server_error)?; + + let pool = pg_pool.get_ref().clone(); + let vault = vault_client.clone(); // web::Data: an Arc, moved into the job + let job_id = job.id; + let stack = form.stack.clone(); + let health_url = form.health_url.clone(); + tokio::spawn(async move { + let outcome = run_job(&pool, &vault, &stack, &health_url, prepared).await; + let saved = match outcome { + Ok((image_id, snapshot_id)) => { + tracing::info!(%job_id, image_id, "bake finished"); + db::bake_job::mark_done(&pool, job_id, image_id, snapshot_id).await + } + Err(err) => { + tracing::warn!(%job_id, error = %err, "bake failed"); + db::bake_job::mark_failed(&pool, job_id, &err).await + } + }; + if let Err(err) = saved { + tracing::error!(%job_id, error = %err, "could not save the bake result"); + } + }); + + Ok(HttpResponse::Accepted().json(serde_json::json!({ "message": "accepted", "item": job }))) +} + +/// Connect, check, sanitize, snapshot, record. Returns (image_id, snapshot row id). +async fn run_job( + pool: &PgPool, + vault: &VaultClient, + stack: &str, + health_url: &str, + p: Prepared, +) -> Result<(i64, i32), String> { + use crate::helpers::ssh_client::{disconnect_ssh, exec_remote, open_ssh, HostKeyPolicy}; + + let response = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .map_err(|e| e.to_string())? + .get(health_url) + .send() + .await + .map_err(|e| format!("health check: {health_url} unreachable: {e}"))?; + if !response.status().is_success() { + return Err(format!( + "health check: {health_url} returned {}", + response.status() + )); + } + + let private_key_pem = vault + .fetch_ssh_key(&p.server.user_id, p.server.id) + .await + .map_err(|e| format!("SSH key of the build box: {e}"))?; + let user = p + .server + .ssh_user + .clone() + .unwrap_or_else(|| "root".to_string()); + let port = p + .server + .ssh_port + .and_then(|v| u16::try_from(v).ok()) + .unwrap_or(22); + + let session = open_ssh( + &p.server_ip, + port, + &user, + &private_key_pem, + std::time::Duration::from_secs(30), + &HostKeyPolicy::Pin(p.server.host_key_fingerprint.clone()), + ) + .await + .map_err(|e| format!("ssh connect: {e}"))?; + let inspect = async { + let (ls, stderr, code) = exec_remote(&session, "docker compose ls --all --format json", 60) + .await + .map_err(|e| e.to_string())?; + if code != 0 { + return Err(format!("docker compose ls exited {code}: {stderr}")); + } + let dir = project_dir_from_compose_ls(&ls, crate::project_app::PLATFORM_INSTALL_DIRS)?; + let (ps, stderr, code) = exec_remote( + &session, + &format!("cd {dir} && docker compose ps --all --format json"), + 60, + ) + .await + .map_err(|e| e.to_string())?; + if code != 0 { + return Err(format!("docker compose ps exited {code}: {stderr}")); + } + check_compose_services(&ps)?; + Ok::(dir) + } + .await; + disconnect_ssh(session).await; + let project_dir = inspect?; + + let outcome = crate::helpers::bake_finalize::finalize_build_box( + &crate::helpers::bake_finalize::FinalizeContext { + host: p.server_ip.clone(), + port, + user, + private_key_pem, + project_dir, + stack: stack.to_string(), + contract: p.parsed_contract, + protected_keys: p.protected_keys, + }, + ) + .await + .map_err(|e| e.to_string())?; + + let record = crate::helpers::bake::run_bake( + p.connector.as_ref(), + &p.token, + p.target, + stack, + &p.version, + true, + "", + ) + .await + .map_err(|e| e.to_string())?; + + let required_env_keys = serde_json::Value::Array( + outcome + .required_env_keys + .into_iter() + .map(serde_json::Value::String) + .collect(), + ); + let row = db::baked_snapshot::record( + pool, + &record.stack, + &record.version, + &record.provider, + record.image_id, + record.healthy, + None, + Some(p.contract), + Some(required_env_keys), + ) + .await?; + Ok((record.image_id, row.id)) +} + +#[tracing::instrument(name = "Bake status (admin)", skip_all)] +#[get("/{id}")] +pub async fn status_handler( + _admin: web::ReqData>, // role enforced by Casbin + path: web::Path<(String,)>, + pg_pool: web::Data, +) -> Result { + let id = uuid::Uuid::parse_str(&path.into_inner().0) + .map_err(|_| JsonResponse::bad_request("invalid bake id"))?; + let job = db::bake_job::fetch(pg_pool.get_ref(), id) + .await + .map_err(JsonResponse::internal_server_error)? + .ok_or_else(|| JsonResponse::not_found("bake not found"))?; + Ok(JsonResponse::build().set_item(job).ok("OK")) +} diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 89dee705..2dc2a92c 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod admin_bake; pub(crate) mod admin_notifications; pub(crate) mod agent; pub mod audit; diff --git a/src/startup.rs b/src/startup.rs index b1fc1513..56c19028 100644 --- a/src/startup.rs +++ b/src/startup.rs @@ -458,6 +458,11 @@ pub async fn run( ) .service( web::scope("/admin") + .service( + web::scope("/bake") + .service(crate::routes::admin_bake::start_handler) + .service(crate::routes::admin_bake::status_handler), + ) .service( web::scope("/templates") .service( diff --git a/tests/admin_bake.rs b/tests/admin_bake.rs new file mode 100644 index 00000000..d79d80fc --- /dev/null +++ b/tests/admin_bake.rs @@ -0,0 +1,215 @@ +//! `POST /api/admin/bake`: everything that can be checked without touching the +//! build box is refused at once, with no job recorded and nothing done to the +//! box. Hetzner is a wiremock; `HETZNER_TOKEN` / `HETZNER_API_BASE_URL` are +//! process-wide, so every case runs inside one test. + +mod common; + +use chrono::{Duration, Utc}; +use serde_json::{json, Value}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const BOX_IP: &str = "203.0.113.7"; +const STACK: &str = "bake-endpoint-stack"; + +fn admin_jwt() -> String { + use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; + let header = URL_SAFE_NO_PAD.encode(json!({"alg": "HS256", "typ": "JWT"}).to_string()); + let payload = URL_SAFE_NO_PAD.encode( + json!({ + "role": "admin_service", + "email": "ops@test.com", + "exp": (Utc::now() + Duration::minutes(30)).timestamp(), + }) + .to_string(), + ); + let signature = common::sign_test_jwt(&header, &payload); + format!("{header}.{payload}.{signature}") +} + +fn contract() -> Value { + json!({ "services": { "app": { "fields": { + "APP_SECRET": { "mutability": "generated", "type": "hex", "length": 32 } + } } } }) +} + +async fn seed_template(pool: &sqlx::PgPool, slug: &str, status: &str, contract: Option) { + let id: uuid::Uuid = sqlx::query_scalar( + r#"INSERT INTO stack_template (creator_user_id, creator_name, name, slug, status, approved_at) + VALUES ('vendor', 'Vendor', $1, $1, $2, CASE WHEN $2 = 'approved' THEN NOW() END) + RETURNING id"#, + ) + .bind(slug) + .bind(status) + .fetch_one(pool) + .await + .expect("seed template"); + sqlx::query( + r#"INSERT INTO stack_template_version (template_id, version, stack_definition, definition_format, is_latest, config_contract) + VALUES ($1, '1.2.0', '{}'::jsonb, 'json', true, $2)"#, + ) + .bind(id) + .bind(contract) + .execute(pool) + .await + .expect("seed version"); +} + +/// A deployment whose project has one server at `ip`. +async fn seed_build_box(pool: &sqlx::PgPool, ip: &str) -> i32 { + let project = common::create_test_project(pool, common::USER_A_ID).await; + let hash = format!("bake-{}", uuid::Uuid::new_v4()); + let deployment = common::create_test_deployment(pool, common::USER_A_ID, project, &hash).await; + let server = common::create_test_server(pool, common::USER_A_ID, project, "active", None).await; + sqlx::query("UPDATE server SET srv_ip = $2 WHERE id = $1") + .bind(server) + .bind(ip) + .execute(pool) + .await + .unwrap(); + deployment +} + +async fn job_count(pool: &sqlx::PgPool) -> i64 { + sqlx::query_scalar("SELECT COUNT(*) FROM bake_job") + .fetch_one(pool) + .await + .unwrap() +} + +async fn post(app: &common::TestApp, body: Value) -> (u16, Value) { + let resp = reqwest::Client::new() + .post(format!("{}/api/admin/bake", app.address)) + .bearer_auth(admin_jwt()) + .json(&body) + .send() + .await + .expect("request"); + let status = resp.status().as_u16(); + (status, resp.json().await.unwrap_or(Value::Null)) +} + +#[tokio::test] +async fn a_bake_is_refused_before_the_box_is_touched_and_accepted_when_everything_checks_out() { + let Some(app) = common::spawn_app().await else { + return; + }; + let hetzner = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/servers")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({ "servers": [ + { "id": 4242, "name": "build-box", "public_net": { "ipv4": { "ip": BOX_IP } } } + ] })), + ) + .mount(&hetzner) + .await; + std::env::set_var("HETZNER_API_BASE_URL", hetzner.uri()); + std::env::set_var("HETZNER_TOKEN", "test-hetzner-token"); + + seed_template(&app.db_pool, STACK, "approved", Some(contract())).await; + seed_template( + &app.db_pool, + "bake-submitted-stack", + "submitted", + Some(contract()), + ) + .await; + seed_template(&app.db_pool, "bake-no-contract-stack", "approved", None).await; + let deployment = seed_build_box(&app.db_pool, BOX_IP).await; + let unknown_box = seed_build_box(&app.db_pool, "198.51.100.9").await; + let health = format!("http://{BOX_IP}/health"); + + for (body, expected, why) in [ + ( + json!({"deployment_id": 999999, "stack": STACK, "health_url": health}), + 404, + "unknown deployment", + ), + ( + json!({"deployment_id": deployment, "stack": "bake-submitted-stack", "health_url": health}), + 404, + "template not approved", + ), + ( + json!({"deployment_id": deployment, "stack": "bake-no-contract-stack", "health_url": health}), + 409, + "no contract to sanitize against", + ), + ( + json!({"deployment_id": deployment, "stack": STACK, "health_url": "http://198.51.100.1/health"}), + 400, + "health URL off the box", + ), + ( + json!({"deployment_id": unknown_box, "stack": STACK, "health_url": "http://198.51.100.9/health"}), + 409, + "box Hetzner does not know", + ), + ] { + let (status, resp) = post(&app, body).await; + assert_eq!(status, expected, "{why}: {resp}"); + assert_eq!( + job_count(&app.db_pool).await, + 0, + "{why}: no job may be recorded" + ); + } + + let (status, resp) = post( + &app, + json!({"deployment_id": deployment, "stack": STACK, "health_url": health}), + ) + .await; + assert_eq!(status, 202, "{resp}"); + let job = &resp["item"]; + assert_eq!(job["stack"], STACK); + assert_eq!( + job["version"], "1.2.0", + "the latest approved version is baked" + ); + assert_eq!(job["server_ip"], BOX_IP); + + let (status, resp) = post( + &app, + json!({"deployment_id": deployment, "stack": STACK, "health_url": health}), + ) + .await; + // Either still running (409) or already failed (no SSH in tests): never two at once. + if status == 202 { + assert_eq!(job_count(&app.db_pool).await, 2); + } else { + assert_eq!(status, 409, "{resp}"); + } + + let id = job["id"].as_str().unwrap(); + let status = reqwest::Client::new() + .get(format!("{}/api/admin/bake/{id}", app.address)) + .bearer_auth(admin_jwt()) + .send() + .await + .unwrap(); + assert_eq!(status.status().as_u16(), 200); + let body: Value = status.json().await.unwrap(); + assert_eq!(body["item"]["id"], id); +} + +#[tokio::test] +async fn a_user_who_is_not_an_admin_cannot_bake() { + let Some(app) = common::spawn_app().await else { + return; + }; + let resp = reqwest::Client::new() + .post(format!("{}/api/admin/bake", app.address)) + .bearer_auth(common::USER_A_TOKEN) + .json(&json!({"deployment_id": 1, "stack": STACK, "health_url": "http://203.0.113.7/"})) + .send() + .await + .unwrap(); + assert!( + matches!(resp.status().as_u16(), 401 | 403), + "got {}", + resp.status() + ); +}