diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8b97853b..19b07e10 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -163,7 +163,8 @@ jobs: cargo build --release \ --bin server \ --bin cleanup-notify \ - --bin backfill_field_policy + --bin backfill_field_policy \ + --bin bake cargo build --release --bin console --features explain - name: Set up Node.js @@ -202,6 +203,7 @@ jobs: cp target/release/console app/stacker/console cp target/release/cleanup-notify app/stacker/cleanup-notify cp target/release/backfill_field_policy app/stacker/backfill_field_policy + cp target/release/bake app/stacker/bake if [ -d web/dist ]; then cp -a web/dist/. app/stacker; fi cp Dockerfile app/Dockerfile cp access_control.conf.dist app/access_control.conf.dist @@ -248,8 +250,8 @@ jobs: tar -xzf app.tar.gz -C "${{ runner.temp }}/prebuilt" cd "${{ runner.temp }}/prebuilt" mv stacker/server stacker/console stacker/cleanup-notify \ - stacker/backfill_field_policy . - chmod +x server console cleanup-notify backfill_field_policy + stacker/backfill_field_policy stacker/bake . + chmod +x server console cleanup-notify backfill_field_policy bake rm -f "${GITHUB_WORKSPACE}/app.tar.gz" - diff --git a/Dockerfile b/Dockerfile index 50d6a87f..7dd0e04f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,8 @@ RUN apt-get update && apt-get install --no-install-recommends -y libssl-dev; \ cargo build --release --bin server; \ cargo build --release --bin console --features explain; \ cargo build --release --bin cleanup-notify; \ - cargo build --release --bin backfill_field_policy + cargo build --release --bin backfill_field_policy; \ + cargo build --release --bin bake #RUN ls -la /app/target/release/ >&2 @@ -78,6 +79,7 @@ COPY --from=builder /app/target/release/server /server COPY --from=builder /app/target/release/console /console COPY --from=builder /app/target/release/cleanup-notify /cleanup-notify COPY --from=builder /app/target/release/backfill_field_policy /backfill_field_policy +COPY --from=builder /app/target/release/bake /bake FROM ${BINARIES}-source AS binaries @@ -94,6 +96,7 @@ COPY --from=binaries /server . COPY --from=binaries /console . COPY --from=binaries /cleanup-notify . COPY --from=binaries /backfill_field_policy . +COPY --from=binaries /bake . COPY --from=config /app/.env . COPY --from=config /app/configuration.yaml . COPY --from=config /usr/local/cargo/bin/sqlx /usr/local/bin/sqlx diff --git a/migrations/20261015100000_bake_job.down.sql b/migrations/20261015100000_bake_job.down.sql new file mode 100644 index 00000000..2d6d1e24 --- /dev/null +++ b/migrations/20261015100000_bake_job.down.sql @@ -0,0 +1,2 @@ +DELETE FROM public.casbin_rule WHERE ptype = 'p' AND v1 IN ('/api/admin/bake', '/api/admin/bake/:id'); +DROP TABLE IF EXISTS bake_job; diff --git a/migrations/20261015100000_bake_job.up.sql b/migrations/20261015100000_bake_job.up.sql new file mode 100644 index 00000000..8d0382b9 --- /dev/null +++ b/migrations/20261015100000_bake_job.up.sql @@ -0,0 +1,23 @@ +-- Bake started from the API: one row per POST /api/admin/bake. +CREATE TABLE IF NOT EXISTS bake_job ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + stack TEXT NOT NULL, + version TEXT NOT NULL, + deployment_id INTEGER NOT NULL, + server_id INTEGER NOT NULL, + server_ip TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'running' CHECK (status IN ('running', 'done', 'failed')), + image_id BIGINT, + snapshot_id INTEGER, + error TEXT, + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS bake_job_created_at_idx ON bake_job (created_at DESC); + +INSERT INTO public.casbin_rule (ptype, v0, v1, v2, v3, v4, v5) VALUES ('p', 'group_admin', '/api/admin/bake', 'POST', '', '', '') ON CONFLICT ON CONSTRAINT unique_key_sqlx_adapter DO NOTHING; +INSERT INTO public.casbin_rule (ptype, v0, v1, v2, v3, v4, v5) VALUES ('p', 'group_admin', '/api/admin/bake/:id', 'GET', '', '', '') ON CONFLICT ON CONSTRAINT unique_key_sqlx_adapter DO NOTHING; +INSERT INTO public.casbin_rule (ptype, v0, v1, v2, v3, v4, v5) VALUES ('p', 'admin_service', '/api/admin/bake', 'POST', '', '', '') ON CONFLICT ON CONSTRAINT unique_key_sqlx_adapter DO NOTHING; +INSERT INTO public.casbin_rule (ptype, v0, v1, v2, v3, v4, v5) VALUES ('p', 'admin_service', '/api/admin/bake/:id', 'GET', '', '', '') ON CONFLICT ON CONSTRAINT unique_key_sqlx_adapter DO NOTHING; diff --git a/src/db/bake_job.rs b/src/db/bake_job.rs new file mode 100644 index 00000000..04ce2e2d --- /dev/null +++ b/src/db/bake_job.rs @@ -0,0 +1,80 @@ +use crate::models::BakeJob; +use sqlx::PgPool; + +pub struct NewBakeJob<'a> { + pub stack: &'a str, + pub version: &'a str, + pub deployment_id: i32, + pub server_id: i32, + pub server_ip: &'a str, + pub created_by: &'a str, +} + +pub async fn insert(pool: &PgPool, job: NewBakeJob<'_>) -> Result { + sqlx::query_as::<_, BakeJob>( + r#" + INSERT INTO bake_job (stack, version, deployment_id, server_id, server_ip, created_by) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING * + "#, + ) + .bind(job.stack) + .bind(job.version) + .bind(job.deployment_id) + .bind(job.server_id) + .bind(job.server_ip) + .bind(job.created_by) + .fetch_one(pool) + .await + .map_err(|e| format!("Failed to record bake job: {e}")) +} + +pub async fn fetch(pool: &PgPool, id: uuid::Uuid) -> Result, String> { + sqlx::query_as::<_, BakeJob>("SELECT * FROM bake_job WHERE id = $1") + .bind(id) + .fetch_optional(pool) + .await + .map_err(|e| format!("Failed to read bake job: {e}")) +} + +/// A bake already running for this build box. One at a time: two would +/// sanitize and snapshot the same box concurrently. +pub async fn running_for_server(pool: &PgPool, server_id: i32) -> Result, String> { + sqlx::query_as::<_, BakeJob>( + "SELECT * FROM bake_job WHERE server_id = $1 AND status = 'running' LIMIT 1", + ) + .bind(server_id) + .fetch_optional(pool) + .await + .map_err(|e| format!("Failed to read bake jobs: {e}")) +} + +pub async fn mark_done( + pool: &PgPool, + id: uuid::Uuid, + image_id: i64, + snapshot_id: i32, +) -> Result<(), String> { + sqlx::query( + "UPDATE bake_job SET status = 'done', image_id = $2, snapshot_id = $3, updated_at = NOW() WHERE id = $1", + ) + .bind(id) + .bind(image_id) + .bind(snapshot_id) + .execute(pool) + .await + .map(|_| ()) + .map_err(|e| format!("Failed to finish bake job: {e}")) +} + +pub async fn mark_failed(pool: &PgPool, id: uuid::Uuid, error: &str) -> Result<(), String> { + sqlx::query( + "UPDATE bake_job SET status = 'failed', error = $2, updated_at = NOW() WHERE id = $1", + ) + .bind(id) + .bind(error) + .execute(pool) + .await + .map(|_| ()) + .map_err(|e| format!("Failed to mark bake job failed: {e}")) +} diff --git a/src/db/mod.rs b/src/db/mod.rs index 831b4172..ba5d8f72 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,6 +1,7 @@ pub mod agent; pub mod agent_audit_log; pub(crate) mod agreement; +pub mod bake_job; pub mod baked_snapshot; pub mod chat; pub mod chat_admin; diff --git a/src/helpers/bake_job.rs b/src/helpers/bake_job.rs new file mode 100644 index 00000000..3e73d10d --- /dev/null +++ b/src/helpers/bake_job.rs @@ -0,0 +1,250 @@ +//! Bake started from the API (`POST /api/admin/bake`) instead of the `bake` +//! binary. +//! +//! The binary needs the operator to type the box's IP, its SSH key, the +//! version and the health URL. Here they come from what stacker already +//! holds: the deployment's server row (IP, SSH user and port), the server's +//! SSH key in Vault, and the template's latest approved version. These pure +//! helpers check what the box reports before anything on it is changed. + +/// The stack's project directory on the build box, from +/// `docker compose ls --all --format json`. +/// +/// The deploy puts each stack in its own directory under the deploy root; +/// platform services (status panel, proxy manager) have their own. Anything +/// but exactly one project directory is refused: sanitizing the wrong one +/// would publish the author's secrets. +pub fn project_dir_from_compose_ls(stdout: &str, platform_dirs: &[&str]) -> Result { + let projects: Vec = serde_json::from_str(stdout.trim()) + .map_err(|e| format!("could not read `docker compose ls` output: {e}"))?; + + let mut dirs: Vec = projects + .iter() + .filter_map(|p| p.get("ConfigFiles").and_then(|v| v.as_str())) + .filter_map(|files| files.split(',').next()) + .filter_map(|file| file.trim().rsplit_once('/').map(|(dir, _)| dir.to_string())) + .filter(|dir| !platform_dirs.contains(&dir.as_str())) + .collect(); + dirs.sort(); + dirs.dedup(); + + match dirs.as_slice() { + [dir] if is_safe_dir(dir) => Ok(dir.clone()), + [dir] => Err(format!("refusing the stack directory `{dir}`: not a plain absolute path")), + [] => Err("no stack is deployed on the build box (only platform services)".to_string()), + many => Err(format!( + "more than one stack is deployed on the build box: {}. Bake needs a box with one stack.", + many.join(", ") + )), + } +} + +/// An absolute path of letters, digits and `/ _ . -` only: it goes into a +/// shell command on the box. +fn is_safe_dir(dir: &str) -> bool { + dir.starts_with('/') + && !dir.contains("..") + && dir + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '/' | '_' | '.' | '-')) +} + +/// Every service of the stack is up, from `docker compose ps --all --format +/// json` (one JSON object per line in compose v2.21+, a JSON array before). +/// +/// Running without a healthcheck or healthy counts as up; a one-shot job that +/// exited 0 (migrations, init) is done, not down. +pub fn check_compose_services(stdout: &str) -> Result<(), String> { + let text = stdout.trim(); + let services: Vec = if text.starts_with('[') { + serde_json::from_str(text) + .map_err(|e| format!("could not read `docker compose ps`: {e}"))? + } else { + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(serde_json::from_str) + .collect::>() + .map_err(|e| format!("could not read `docker compose ps`: {e}"))? + }; + if services.is_empty() { + return Err("the stack has no services on the build box".to_string()); + } + + let mut down = Vec::new(); + for service in &services { + let field = |name: &str| service.get(name).and_then(|v| v.as_str()).unwrap_or(""); + let name = field("Service"); + let state = field("State"); + let health = field("Health"); + let exit_code = service + .get("ExitCode") + .and_then(|v| v.as_i64()) + .unwrap_or(-1); + let up = match state { + "running" => health.is_empty() || health == "healthy", + "exited" => exit_code == 0, + _ => false, + }; + if !up { + let detail = if health.is_empty() { + state.to_string() + } else { + format!("{state}, {health}") + }; + down.push(format!("{name} ({detail})")); + } + } + if down.is_empty() { + Ok(()) + } else { + Err(format!("services not up: {}", down.join(", "))) + } +} + +/// The health URL must point at the build box itself: the bake runs on the +/// stacker backend, so any other host would make it a request the caller +/// chose from inside the platform's network. +pub fn check_health_url(url: &str, server_ip: &str) -> Result<(), String> { + let parsed = reqwest::Url::parse(url).map_err(|e| format!("health_url is not a URL: {e}"))?; + if !matches!(parsed.scheme(), "http" | "https") { + return Err("health_url must be http or https".to_string()); + } + if !parsed.username().is_empty() || parsed.password().is_some() { + return Err("health_url must not carry credentials".to_string()); + } + match parsed.host_str() { + Some(host) if host == server_ip => Ok(()), + _ => Err(format!( + "health_url must point at the build box ({server_ip})" + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const PLATFORM: &[&str] = &[ + "/home/trydirect/statuspanel", + "/home/trydirect/nginx_proxy_manager", + ]; + + #[test] + fn the_stack_directory_is_found_next_to_platform_services() { + let ls = r#"[ + {"Name":"statuspanel","Status":"running(1)","ConfigFiles":"/home/trydirect/statuspanel/docker-compose.yml"}, + {"Name":"ai-workflows-v2-12","Status":"running(4)","ConfigFiles":"/home/trydirect/ai-workflows-v2-12/docker-compose.yml"} + ]"#; + assert_eq!( + project_dir_from_compose_ls(ls, PLATFORM).unwrap(), + "/home/trydirect/ai-workflows-v2-12" + ); + } + + #[test] + fn several_config_files_of_one_project_point_at_its_directory() { + let ls = r#"[{"Name":"app","Status":"running(2)","ConfigFiles":"/home/trydirect/app/docker-compose.yml,/home/trydirect/app/docker-compose.override.yml"}]"#; + assert_eq!( + project_dir_from_compose_ls(ls, PLATFORM).unwrap(), + "/home/trydirect/app" + ); + } + + #[test] + fn no_stack_or_two_stacks_are_refused() { + let only_platform = r#"[{"Name":"statuspanel","Status":"running(1)","ConfigFiles":"/home/trydirect/statuspanel/docker-compose.yml"}]"#; + assert!(project_dir_from_compose_ls(only_platform, PLATFORM).is_err()); + + let two = r#"[ + {"Name":"a","Status":"running(1)","ConfigFiles":"/home/trydirect/a/docker-compose.yml"}, + {"Name":"b","Status":"running(1)","ConfigFiles":"/home/trydirect/b/docker-compose.yml"} + ]"#; + let err = project_dir_from_compose_ls(two, PLATFORM).unwrap_err(); + assert!( + err.contains("/home/trydirect/a") && err.contains("/home/trydirect/b"), + "{err}" + ); + + assert!(project_dir_from_compose_ls("not json", PLATFORM).is_err()); + } + + #[test] + fn a_directory_unsafe_for_a_shell_command_is_refused() { + // The directory is put into `cd && ...` on the box. + for dir in [ + "/home/trydirect/a b", + "/home/trydirect/a;rm -rf /", + "/home/trydirect/$(id)", + "relative/dir", + ] { + let ls = format!( + r#"[{{"Name":"x","Status":"running(1)","ConfigFiles":"{dir}/docker-compose.yml"}}]"# + ); + assert!(project_dir_from_compose_ls(&ls, PLATFORM).is_err(), "{dir}"); + } + } + + #[test] + fn services_running_healthy_or_finished_are_up() { + let ps = "\ +{\"Service\":\"n8n\",\"State\":\"running\",\"Health\":\"healthy\",\"ExitCode\":0} +{\"Service\":\"redis\",\"State\":\"running\",\"Health\":\"\",\"ExitCode\":0} +{\"Service\":\"migrate\",\"State\":\"exited\",\"Health\":\"\",\"ExitCode\":0}"; + assert_eq!(check_compose_services(ps), Ok(())); + + let array = r#"[{"Service":"n8n","State":"running","Health":"healthy","ExitCode":0}]"#; + assert_eq!(check_compose_services(array), Ok(())); + } + + #[test] + fn a_starting_unhealthy_crashed_or_missing_service_fails_the_gate() { + for (ps, culprit) in [ + ( + r#"{"Service":"db","State":"running","Health":"starting","ExitCode":0}"#, + "db", + ), + ( + r#"{"Service":"db","State":"running","Health":"unhealthy","ExitCode":0}"#, + "db", + ), + ( + r#"{"Service":"api","State":"exited","Health":"","ExitCode":1}"#, + "api", + ), + ( + r#"{"Service":"api","State":"restarting","Health":"","ExitCode":1}"#, + "api", + ), + ] { + let err = check_compose_services(ps).unwrap_err(); + assert!(err.contains(culprit), "{err}"); + } + assert!( + check_compose_services("").is_err(), + "no services is not healthy" + ); + assert!(check_compose_services("garbage").is_err()); + } + + #[test] + fn the_health_url_must_point_at_the_build_box() { + assert_eq!( + check_health_url("http://203.0.113.7/health", "203.0.113.7"), + Ok(()) + ); + assert_eq!( + check_health_url("https://203.0.113.7:8443/api/ping", "203.0.113.7"), + Ok(()) + ); + for url in [ + "http://198.51.100.1/health", + "http://203.0.113.7.evil.example/health", + "http://user@198.51.100.1/", + "file:///etc/passwd", + "http://localhost/health", + "not a url", + ] { + assert!(check_health_url(url, "203.0.113.7").is_err(), "{url}"); + } + } +} diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs index 95919c6d..3a6876aa 100644 --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -36,6 +36,7 @@ pub use dockerhub::*; pub use cloud::*; pub mod audit_cache; pub mod bake; +pub mod bake_job; pub mod bake_finalize; pub mod bake_registry; pub mod cloud_init; diff --git a/src/models/bake_job.rs b/src/models/bake_job.rs new file mode 100644 index 00000000..ea745c05 --- /dev/null +++ b/src/models/bake_job.rs @@ -0,0 +1,22 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// A bake started through `POST /api/admin/bake`. `running` until the job ends +/// as `done` (with the snapshot) or `failed` (with the reason). The build box +/// (`server_id`, `server_ip`) is never removed by the job. +#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow)] +pub struct BakeJob { + pub id: uuid::Uuid, + pub stack: String, + pub version: String, + pub deployment_id: i32, + pub server_id: i32, + pub server_ip: String, + pub status: String, + pub image_id: Option, + pub snapshot_id: Option, + pub error: Option, + pub created_by: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} diff --git a/src/models/mod.rs b/src/models/mod.rs index b99d3855..cac5b82c 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -2,6 +2,7 @@ mod agent; pub mod agent_audit_log; pub mod agent_protocol; mod agreement; +pub mod bake_job; pub mod baked_snapshot; pub mod cdc; mod chat; @@ -31,6 +32,7 @@ pub mod user; pub use agent::*; pub use agent_audit_log::AgentAuditLog; pub use agreement::*; +pub use bake_job::*; pub use baked_snapshot::*; pub use chat::*; pub use client::*; diff --git a/src/project_app/mod.rs b/src/project_app/mod.rs index 49d31a0d..38a4f5f1 100644 --- a/src/project_app/mod.rs +++ b/src/project_app/mod.rs @@ -50,7 +50,7 @@ const PLATFORM_DISPLAY_CODES: &[&str] = &[ /// Directories platform-managed services are deployed into, by the convention /// in `stacker/docs/APP_DEPLOYMENT.md`: project services live under the /// project directory, platform ones get their own. -const PLATFORM_INSTALL_DIRS: &[&str] = &[ +pub(crate) const PLATFORM_INSTALL_DIRS: &[&str] = &[ "/home/trydirect/statuspanel", "/home/trydirect/nginx_proxy_manager", ]; diff --git a/src/routes/admin_bake.rs b/src/routes/admin_bake.rs new file mode 100644 index 00000000..762b8b9c --- /dev/null +++ b/src/routes/admin_bake.rs @@ -0,0 +1,347 @@ +//! `POST /api/admin/bake` and `GET /api/admin/bake/{id}`: bake a deployed +//! build box into a Quick Deploy snapshot without the operator typing its IP, +//! SSH key or version. +//! +//! Everything that can be checked without touching the box is checked before +//! the job starts, and answered at once: the deployment and its server, the +//! approved template with a usable contract, a health URL on the box, and the +//! box being one Hetzner knows. Only then does the job connect, check that +//! every service is up, sanitize the box, snapshot it and record the snapshot. +//! The box is never removed: the result lists it for the operator. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use actix_web::{get, post, web, HttpResponse, Responder, Result}; +use serde::Deserialize; +use sqlx::PgPool; + +use crate::connectors::config::HetznerConfig; +use crate::connectors::hetzner::{ + HetznerCloudClient, HetznerCloudConnector, HetznerSnapshotTarget, +}; +use crate::db; +use crate::helpers::bake_job::{ + check_compose_services, check_health_url, project_dir_from_compose_ls, +}; +use crate::helpers::{JsonResponse, VaultClient}; +use crate::models; + +#[derive(Debug, Deserialize)] +pub struct BakeRequest { + /// The build box's deployment (from the CLI deployment lock). + pub deployment_id: i32, + /// Approved marketplace template slug; its latest version is baked. + pub stack: String, + /// The stack's health endpoint on the build box. + pub health_url: String, +} + +/// What the job needs, all resolved and checked before it starts. +struct Prepared { + server: models::Server, + server_ip: String, + version: String, + contract: serde_json::Value, + parsed_contract: crate::cli::config_parser::ConfigContract, + protected_keys: BTreeSet, + token: String, + connector: Arc, + target: HetznerSnapshotTarget, +} + +async fn prepare(pool: &PgPool, form: &BakeRequest) -> Result { + let deployment = db::deployment::fetch(pool, form.deployment_id) + .await + .map_err(JsonResponse::internal_server_error)? + .ok_or_else(|| JsonResponse::not_found("deployment not found"))?; + + let servers: Vec = db::server::fetch_by_project(pool, deployment.project_id) + .await + .map_err(JsonResponse::internal_server_error)? + .into_iter() + .filter(|s| { + s.srv_ip + .as_deref() + .map_or(false, |ip| !ip.trim().is_empty()) + }) + .collect(); + let server = match servers.as_slice() { + [server] => server.clone(), + [] => { + return Err(JsonResponse::<()>::build() + .conflict("the deployment's project has no server with an IP")) + } + _ => { + return Err(JsonResponse::<()>::build().conflict( + "the deployment's project has more than one server; bake needs one build box", + )) + } + }; + let server_ip = server.srv_ip.clone().unwrap_or_default(); + + check_health_url(&form.health_url, &server_ip).map_err(JsonResponse::bad_request)?; + + // The latest approved version: the contract describes it, so version and + // contract cannot disagree. + let (template, version) = db::marketplace::get_by_slug_with_latest(pool, &form.stack) + .await + .map_err(|_| JsonResponse::not_found("no approved template with this slug"))?; + let version = version + .map(|v| v.version) + .ok_or_else(|| JsonResponse::<()>::build().conflict("the template has no version"))?; + let contract = db::marketplace::get_config_contract(pool, template.id) + .await + .map_err(JsonResponse::internal_server_error)?; + if contract.is_null() { + return Err(JsonResponse::<()>::build() + .conflict("the template declares no config_contract; nothing would be sanitized")); + } + let protected_keys = crate::helpers::bake_finalize::protected_keys_from_contract(&contract); + let parsed_contract = serde_json::from_value(contract.clone()).map_err(|e| { + JsonResponse::<()>::build().conflict(format!( + "the stored config_contract could not be parsed: {e}" + )) + })?; + crate::helpers::bake_finalize::check_contract_usable(&protected_keys, false) + .map_err(|e| JsonResponse::<()>::build().conflict(e.to_string()))?; + + let htz = HetznerConfig::from_env(); + let token = htz.token.filter(|t| !t.trim().is_empty()).ok_or_else(|| { + JsonResponse::internal_server_error("HETZNER_TOKEN is not set on the stacker backend") + })?; + let client = HetznerCloudClient::new(htz.base_url) + .map_err(|e| JsonResponse::internal_server_error(e.to_string()))?; + let connector: Arc = Arc::new(client); + let target = HetznerSnapshotTarget { + provider_server_id: None, + server_name: None, + public_ip: Some(server_ip.clone()), + }; + // Sanitizing removes SSH access, so a box Hetzner cannot match must be + // refused before anything on it changes. + let provider_server_id = connector + .resolve_snapshot_target(&token, &target) + .await + .map_err(|e| { + JsonResponse::<()>::build().conflict(format!( + "Hetzner does not know a server at {server_ip} in the platform's project: {e}" + )) + })?; + let target = HetznerSnapshotTarget { + provider_server_id: Some(provider_server_id), + ..target + }; + + Ok(Prepared { + server, + server_ip, + version, + contract, + parsed_contract, + protected_keys, + token, + connector, + target, + }) +} + +#[tracing::instrument(name = "Start a bake (admin)", skip_all)] +#[post("")] +pub async fn start_handler( + admin: web::ReqData>, // role enforced by Casbin + form: web::Json, + pg_pool: web::Data, + vault_client: web::Data, +) -> Result { + let form = form.into_inner(); + let prepared = prepare(pg_pool.get_ref(), &form).await?; + + if db::bake_job::running_for_server(pg_pool.get_ref(), prepared.server.id) + .await + .map_err(JsonResponse::internal_server_error)? + .is_some() + { + return Err( + JsonResponse::<()>::build().conflict("a bake of this build box is already running") + ); + } + + let job = db::bake_job::insert( + pg_pool.get_ref(), + db::bake_job::NewBakeJob { + stack: &form.stack, + version: &prepared.version, + deployment_id: form.deployment_id, + server_id: prepared.server.id, + server_ip: &prepared.server_ip, + created_by: &admin.id, + }, + ) + .await + .map_err(JsonResponse::internal_server_error)?; + + let pool = pg_pool.get_ref().clone(); + let vault = vault_client.clone(); // web::Data: an Arc, moved into the job + let job_id = job.id; + let stack = form.stack.clone(); + let health_url = form.health_url.clone(); + tokio::spawn(async move { + let outcome = run_job(&pool, &vault, &stack, &health_url, prepared).await; + let saved = match outcome { + Ok((image_id, snapshot_id)) => { + tracing::info!(%job_id, image_id, "bake finished"); + db::bake_job::mark_done(&pool, job_id, image_id, snapshot_id).await + } + Err(err) => { + tracing::warn!(%job_id, error = %err, "bake failed"); + db::bake_job::mark_failed(&pool, job_id, &err).await + } + }; + if let Err(err) = saved { + tracing::error!(%job_id, error = %err, "could not save the bake result"); + } + }); + + Ok(HttpResponse::Accepted().json(serde_json::json!({ "message": "accepted", "item": job }))) +} + +/// Connect, check, sanitize, snapshot, record. Returns (image_id, snapshot row id). +async fn run_job( + pool: &PgPool, + vault: &VaultClient, + stack: &str, + health_url: &str, + p: Prepared, +) -> Result<(i64, i32), String> { + use crate::helpers::ssh_client::{disconnect_ssh, exec_remote, open_ssh, HostKeyPolicy}; + + let response = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .map_err(|e| e.to_string())? + .get(health_url) + .send() + .await + .map_err(|e| format!("health check: {health_url} unreachable: {e}"))?; + if !response.status().is_success() { + return Err(format!( + "health check: {health_url} returned {}", + response.status() + )); + } + + let private_key_pem = vault + .fetch_ssh_key(&p.server.user_id, p.server.id) + .await + .map_err(|e| format!("SSH key of the build box: {e}"))?; + let user = p + .server + .ssh_user + .clone() + .unwrap_or_else(|| "root".to_string()); + let port = p + .server + .ssh_port + .and_then(|v| u16::try_from(v).ok()) + .unwrap_or(22); + + let session = open_ssh( + &p.server_ip, + port, + &user, + &private_key_pem, + std::time::Duration::from_secs(30), + &HostKeyPolicy::Pin(p.server.host_key_fingerprint.clone()), + ) + .await + .map_err(|e| format!("ssh connect: {e}"))?; + let inspect = async { + let (ls, stderr, code) = exec_remote(&session, "docker compose ls --all --format json", 60) + .await + .map_err(|e| e.to_string())?; + if code != 0 { + return Err(format!("docker compose ls exited {code}: {stderr}")); + } + let dir = project_dir_from_compose_ls(&ls, crate::project_app::PLATFORM_INSTALL_DIRS)?; + let (ps, stderr, code) = exec_remote( + &session, + &format!("cd {dir} && docker compose ps --all --format json"), + 60, + ) + .await + .map_err(|e| e.to_string())?; + if code != 0 { + return Err(format!("docker compose ps exited {code}: {stderr}")); + } + check_compose_services(&ps)?; + Ok::(dir) + } + .await; + disconnect_ssh(session).await; + let project_dir = inspect?; + + let outcome = crate::helpers::bake_finalize::finalize_build_box( + &crate::helpers::bake_finalize::FinalizeContext { + host: p.server_ip.clone(), + port, + user, + private_key_pem, + project_dir, + stack: stack.to_string(), + contract: p.parsed_contract, + protected_keys: p.protected_keys, + }, + ) + .await + .map_err(|e| e.to_string())?; + + let record = crate::helpers::bake::run_bake( + p.connector.as_ref(), + &p.token, + p.target, + stack, + &p.version, + true, + "", + ) + .await + .map_err(|e| e.to_string())?; + + let required_env_keys = serde_json::Value::Array( + outcome + .required_env_keys + .into_iter() + .map(serde_json::Value::String) + .collect(), + ); + let row = db::baked_snapshot::record( + pool, + &record.stack, + &record.version, + &record.provider, + record.image_id, + record.healthy, + None, + Some(p.contract), + Some(required_env_keys), + ) + .await?; + Ok((record.image_id, row.id)) +} + +#[tracing::instrument(name = "Bake status (admin)", skip_all)] +#[get("/{id}")] +pub async fn status_handler( + _admin: web::ReqData>, // role enforced by Casbin + path: web::Path<(String,)>, + pg_pool: web::Data, +) -> Result { + let id = uuid::Uuid::parse_str(&path.into_inner().0) + .map_err(|_| JsonResponse::bad_request("invalid bake id"))?; + let job = db::bake_job::fetch(pg_pool.get_ref(), id) + .await + .map_err(JsonResponse::internal_server_error)? + .ok_or_else(|| JsonResponse::not_found("bake not found"))?; + Ok(JsonResponse::build().set_item(job).ok("OK")) +} diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 89dee705..2dc2a92c 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod admin_bake; pub(crate) mod admin_notifications; pub(crate) mod agent; pub mod audit; diff --git a/src/startup.rs b/src/startup.rs index b1fc1513..56c19028 100644 --- a/src/startup.rs +++ b/src/startup.rs @@ -458,6 +458,11 @@ pub async fn run( ) .service( web::scope("/admin") + .service( + web::scope("/bake") + .service(crate::routes::admin_bake::start_handler) + .service(crate::routes::admin_bake::status_handler), + ) .service( web::scope("/templates") .service( diff --git a/tests/admin_bake.rs b/tests/admin_bake.rs new file mode 100644 index 00000000..d79d80fc --- /dev/null +++ b/tests/admin_bake.rs @@ -0,0 +1,215 @@ +//! `POST /api/admin/bake`: everything that can be checked without touching the +//! build box is refused at once, with no job recorded and nothing done to the +//! box. Hetzner is a wiremock; `HETZNER_TOKEN` / `HETZNER_API_BASE_URL` are +//! process-wide, so every case runs inside one test. + +mod common; + +use chrono::{Duration, Utc}; +use serde_json::{json, Value}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const BOX_IP: &str = "203.0.113.7"; +const STACK: &str = "bake-endpoint-stack"; + +fn admin_jwt() -> String { + use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; + let header = URL_SAFE_NO_PAD.encode(json!({"alg": "HS256", "typ": "JWT"}).to_string()); + let payload = URL_SAFE_NO_PAD.encode( + json!({ + "role": "admin_service", + "email": "ops@test.com", + "exp": (Utc::now() + Duration::minutes(30)).timestamp(), + }) + .to_string(), + ); + let signature = common::sign_test_jwt(&header, &payload); + format!("{header}.{payload}.{signature}") +} + +fn contract() -> Value { + json!({ "services": { "app": { "fields": { + "APP_SECRET": { "mutability": "generated", "type": "hex", "length": 32 } + } } } }) +} + +async fn seed_template(pool: &sqlx::PgPool, slug: &str, status: &str, contract: Option) { + let id: uuid::Uuid = sqlx::query_scalar( + r#"INSERT INTO stack_template (creator_user_id, creator_name, name, slug, status, approved_at) + VALUES ('vendor', 'Vendor', $1, $1, $2, CASE WHEN $2 = 'approved' THEN NOW() END) + RETURNING id"#, + ) + .bind(slug) + .bind(status) + .fetch_one(pool) + .await + .expect("seed template"); + sqlx::query( + r#"INSERT INTO stack_template_version (template_id, version, stack_definition, definition_format, is_latest, config_contract) + VALUES ($1, '1.2.0', '{}'::jsonb, 'json', true, $2)"#, + ) + .bind(id) + .bind(contract) + .execute(pool) + .await + .expect("seed version"); +} + +/// A deployment whose project has one server at `ip`. +async fn seed_build_box(pool: &sqlx::PgPool, ip: &str) -> i32 { + let project = common::create_test_project(pool, common::USER_A_ID).await; + let hash = format!("bake-{}", uuid::Uuid::new_v4()); + let deployment = common::create_test_deployment(pool, common::USER_A_ID, project, &hash).await; + let server = common::create_test_server(pool, common::USER_A_ID, project, "active", None).await; + sqlx::query("UPDATE server SET srv_ip = $2 WHERE id = $1") + .bind(server) + .bind(ip) + .execute(pool) + .await + .unwrap(); + deployment +} + +async fn job_count(pool: &sqlx::PgPool) -> i64 { + sqlx::query_scalar("SELECT COUNT(*) FROM bake_job") + .fetch_one(pool) + .await + .unwrap() +} + +async fn post(app: &common::TestApp, body: Value) -> (u16, Value) { + let resp = reqwest::Client::new() + .post(format!("{}/api/admin/bake", app.address)) + .bearer_auth(admin_jwt()) + .json(&body) + .send() + .await + .expect("request"); + let status = resp.status().as_u16(); + (status, resp.json().await.unwrap_or(Value::Null)) +} + +#[tokio::test] +async fn a_bake_is_refused_before_the_box_is_touched_and_accepted_when_everything_checks_out() { + let Some(app) = common::spawn_app().await else { + return; + }; + let hetzner = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/servers")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({ "servers": [ + { "id": 4242, "name": "build-box", "public_net": { "ipv4": { "ip": BOX_IP } } } + ] })), + ) + .mount(&hetzner) + .await; + std::env::set_var("HETZNER_API_BASE_URL", hetzner.uri()); + std::env::set_var("HETZNER_TOKEN", "test-hetzner-token"); + + seed_template(&app.db_pool, STACK, "approved", Some(contract())).await; + seed_template( + &app.db_pool, + "bake-submitted-stack", + "submitted", + Some(contract()), + ) + .await; + seed_template(&app.db_pool, "bake-no-contract-stack", "approved", None).await; + let deployment = seed_build_box(&app.db_pool, BOX_IP).await; + let unknown_box = seed_build_box(&app.db_pool, "198.51.100.9").await; + let health = format!("http://{BOX_IP}/health"); + + for (body, expected, why) in [ + ( + json!({"deployment_id": 999999, "stack": STACK, "health_url": health}), + 404, + "unknown deployment", + ), + ( + json!({"deployment_id": deployment, "stack": "bake-submitted-stack", "health_url": health}), + 404, + "template not approved", + ), + ( + json!({"deployment_id": deployment, "stack": "bake-no-contract-stack", "health_url": health}), + 409, + "no contract to sanitize against", + ), + ( + json!({"deployment_id": deployment, "stack": STACK, "health_url": "http://198.51.100.1/health"}), + 400, + "health URL off the box", + ), + ( + json!({"deployment_id": unknown_box, "stack": STACK, "health_url": "http://198.51.100.9/health"}), + 409, + "box Hetzner does not know", + ), + ] { + let (status, resp) = post(&app, body).await; + assert_eq!(status, expected, "{why}: {resp}"); + assert_eq!( + job_count(&app.db_pool).await, + 0, + "{why}: no job may be recorded" + ); + } + + let (status, resp) = post( + &app, + json!({"deployment_id": deployment, "stack": STACK, "health_url": health}), + ) + .await; + assert_eq!(status, 202, "{resp}"); + let job = &resp["item"]; + assert_eq!(job["stack"], STACK); + assert_eq!( + job["version"], "1.2.0", + "the latest approved version is baked" + ); + assert_eq!(job["server_ip"], BOX_IP); + + let (status, resp) = post( + &app, + json!({"deployment_id": deployment, "stack": STACK, "health_url": health}), + ) + .await; + // Either still running (409) or already failed (no SSH in tests): never two at once. + if status == 202 { + assert_eq!(job_count(&app.db_pool).await, 2); + } else { + assert_eq!(status, 409, "{resp}"); + } + + let id = job["id"].as_str().unwrap(); + let status = reqwest::Client::new() + .get(format!("{}/api/admin/bake/{id}", app.address)) + .bearer_auth(admin_jwt()) + .send() + .await + .unwrap(); + assert_eq!(status.status().as_u16(), 200); + let body: Value = status.json().await.unwrap(); + assert_eq!(body["item"]["id"], id); +} + +#[tokio::test] +async fn a_user_who_is_not_an_admin_cannot_bake() { + let Some(app) = common::spawn_app().await else { + return; + }; + let resp = reqwest::Client::new() + .post(format!("{}/api/admin/bake", app.address)) + .bearer_auth(common::USER_A_TOKEN) + .json(&json!({"deployment_id": 1, "stack": STACK, "health_url": "http://203.0.113.7/"})) + .send() + .await + .unwrap(); + assert!( + matches!(resp.status().as_u16(), 401 | 403), + "got {}", + resp.status() + ); +} diff --git a/tests/image_binaries_contract.rs b/tests/image_binaries_contract.rs new file mode 100644 index 00000000..0b90f4d4 --- /dev/null +++ b/tests/image_binaries_contract.rs @@ -0,0 +1,87 @@ +//! Every binary the stacker image ships is built and copied on both paths into +//! the image: the CI job (`.github/workflows/docker.yml` compiles, archives, +//! unpacks, and the Dockerfile copies it in as `prebuilt`) and a local +//! `docker build` (the Dockerfile's own `builder` stage). +//! +//! `bake` turns a deployed build box into a Quick Deploy snapshot. It was left +//! out of the image, so every bake meant compiling it by hand and `docker cp` +//! into the running production container. + +use std::fs; +use std::path::PathBuf; + +/// The binaries in /app of the runtime image. +const IMAGE_BINARIES: &[&str] = &[ + "server", + "console", + "cleanup-notify", + "backfill_field_policy", + "bake", +]; + +fn read(file: &str) -> String { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(file); + fs::read_to_string(&path).unwrap_or_else(|err| panic!("read {file}: {err}")) +} + +/// Lines that are not comments, so commented-out history does not count. +fn active(text: &str) -> Vec { + text.lines() + .filter(|line| !line.trim_start().starts_with('#')) + .map(str::to_string) + .collect() +} + +#[test] +fn the_dockerfile_builds_and_ships_every_image_binary() { + let dockerfile = active(&read("Dockerfile")); + for bin in IMAGE_BINARIES { + assert!( + dockerfile + .iter() + .any(|l| l.contains("cargo build --release") && l.contains(&format!("--bin {bin}"))), + "builder stage must compile `{bin}`" + ); + assert!( + dockerfile.iter().any( + |l| l.trim() == format!("COPY --from=builder /app/target/release/{bin} /{bin}") + ), + "builder-source must take `{bin}` from the builder stage" + ); + assert!( + dockerfile + .iter() + .any(|l| l.trim() == format!("COPY --from=binaries /{bin} .")), + "the runtime image must contain `{bin}`" + ); + } +} + +#[test] +fn ci_builds_archives_and_unpacks_every_image_binary() { + let workflow = active(&read(".github/workflows/docker.yml")).join("\n"); + for bin in IMAGE_BINARIES { + assert!( + workflow.contains(&format!("--bin {bin}")), + "CI must compile `{bin}`" + ); + assert!( + workflow.contains(&format!("cp target/release/{bin} app/stacker/{bin}")), + "CI must put `{bin}` in the artifact" + ); + assert!( + workflow.contains(&format!("stacker/{bin}")), + "the docker job must unpack `{bin}` from the artifact" + ); + } + let chmod = workflow + .lines() + .find(|l| l.trim_start().starts_with("chmod +x")) + .expect("the docker job makes the binaries executable"); + for bin in IMAGE_BINARIES { + assert!( + chmod.split_whitespace().any(|word| word == *bin), + "`{bin}` must be made executable: {chmod}" + ); + } +}