diff --git a/.github/workflows/managed-n8n.yml b/.github/workflows/managed-n8n.yml index 616c87de..fba4b64a 100644 --- a/.github/workflows/managed-n8n.yml +++ b/.github/workflows/managed-n8n.yml @@ -4,7 +4,9 @@ on: pull_request: paths: - "services/workflow-automation/n8n/**" + - "services/workflow-automation/n8n-prod/**" - "lib/managed_updates/**" + - "tests/test-shell-resolvable.sh" - "scripts/validate-managed-updates.sh" - "scripts/install-managed-update-systemd.sh" - "systemd/**" @@ -21,7 +23,9 @@ on: - "renovate/**" paths: - "services/workflow-automation/n8n/**" + - "services/workflow-automation/n8n-prod/**" - "lib/managed_updates/**" + - "tests/test-shell-resolvable.sh" - "scripts/validate-managed-updates.sh" - "scripts/install-managed-update-systemd.sh" - "systemd/**" @@ -51,12 +55,19 @@ jobs: run: >- shellcheck --severity=error corekit.sh lib/managed_updates/managed-update.sh - services/workflow-automation/n8n/managed/backup.sh - services/workflow-automation/n8n/managed/strict-healthcheck.sh + services/workflow-automation/n8n/managed/*.sh + services/workflow-automation/n8n-prod/managed/*.sh + services/workflow-automation/n8n-prod/cli.sh + tests/test-shell-resolvable.sh scripts/validate-managed-updates.sh scripts/install-managed-update-systemd.sh tests/test-managed-update.sh tests/test-n8n-candidate.sh + - name: Every called command resolves + # Catches a call left behind when a function is deleted. shellcheck does + # not report undefined functions and `bash -n` cannot see them, which is + # how one reached this branch and broke `managed-update apply`. + run: tests/test-shell-resolvable.sh - name: Validate policy, stable channel, Compose, and Git diff run: ./scripts/validate-managed-updates.sh --online - uses: actions/setup-node@v4 diff --git a/config/stacks/core.yaml b/config/stacks/core.yaml index b0b5495c..a661f1e6 100644 --- a/config/stacks/core.yaml +++ b/config/stacks/core.yaml @@ -77,5 +77,4 @@ services: - mysql - postgres - redis - - caddy - watchtower diff --git a/docs/MANAGED_UPDATES.md b/docs/MANAGED_UPDATES.md new file mode 100644 index 00000000..c3d5884b --- /dev/null +++ b/docs/MANAGED_UPDATES.md @@ -0,0 +1,108 @@ +# Managed updates + +`corekit managed-update` deploys a committed, tested version of a service. It is +service-agnostic: everything it knows about a particular service comes from that +service's own directory. + +## Registering a service + +A service is registered by a `managed_update` block in its `service.json`: + +```json +"managed_update": { + "enabled": true, + "deployment_branch": "main", + "channel": "stable", + "minimum_release_age_days": 7, + "automatic_update_types": ["patch", "minor"], + "major_updates_require_manual_approval": true, + "backup_required": true, + "minimum_free_space_gb": 15, + "components": ["service", "service-runner"] +} +``` + +`enabled: false`, or no block at all, means the service is never touched. List +what is registered with: + +```bash +corekit managed-update list +``` + +## Hooks + +Anything service-specific lives in `/managed/`. The driver runs these +and treats a non-zero exit as a failed deployment. Three are optional and are +skipped when absent: `validate.sh`, `counts.sh` and `stop.sh`. The rest are +called unconditionally, and a missing one fails the run. + +| Hook | Required | Purpose | Contract | +|---|---|---|---| +| `version.sh` | yes | What version is committed, and what is live | Prints one JSON object: `{"target_version","live_version","identity"}`. `identity` is any string that changes when the committed build would change | +| `validate.sh` | no | Refuse an incoherent committed bundle | Exit non-zero to abort. Message on stderr | +| `build.sh` | yes | Build or pull the candidate and verify it is what was asked for | Receives the target version as `$1`. Prints JSON `{"images":{...}}` on the last line. A service that builds nothing still needs one, to verify the image it adopts | +| `backup.sh` | yes | Create and verify a recovery set | Prints the manifest path on the last line. The manifest must report `verified: true`, `restore_drill.result: "passed"` and `candidate_migration_drill.result: "passed"` — see below | +| `counts.sh` | in practice yes | Entity counts and in-flight executions | Prints `{"workflows","credentials","active_executions"}`. Must exit non-zero if it cannot read them: reporting zero would let a deployment proceed during a database outage. Without the hook, the post-deployment count check fails the deployment | +| `stop.sh` | no | Stop this service's containers after a failed health check | Without it the driver stops the containers named in `components` | +| `deploy.sh` | yes | Recreate the service with the candidate | Receives the target version as `$1`, and the candidate images as `COREKIT_CANDIDATE_IMAGES` | +| `strict-healthcheck.sh` | yes | Readiness, counts, canary | Exit non-zero if the deployment should be treated as failed | + +Hooks run with `COREKIT_PROJECT_ROOT`, `COREKIT_SERVICE_DIR` and +`COREKIT_SERVICE_NAME` in the environment. `backup.sh` also receives +`COREKIT_MANAGED_BACKUP_ROOT`, and `deploy.sh` receives +`COREKIT_CANDIDATE_IMAGES`, the JSON that `build.sh` printed. + +`components` is ordered and load-bearing: the first entry is the container whose +image is recorded as the rollback target, the second is its task runner. Recovery +hands both to the operator, so listing them the other way round would tell them +to restart the wrong image. + +### The backup manifest + +The driver refuses to deploy unless the manifest reports all three of: + +```json +{ "verified": true, + "restore_drill": { "result": "passed" }, + "candidate_migration_drill":{ "result": "passed" } } +``` + +`verified` is an assertion the hook makes about itself, so it is worth no more +than the work behind it. A hook that writes `verified: true` without restoring +anything turns the gate into decoration. Restore the dump somewhere isolated and +compare what comes back against the live instance before claiming it. + +Where a step does not apply — a service that adopts a prebuilt image has no +candidate whose migrations need rehearsing — say so explicitly in the manifest +with a `note`, rather than omitting the key. An absent key fails the gate, which +is correct: silence should not pass. + +`counts.sh` matters more than it looks. A service with its own database must +report from that database: measuring one instance against another instance's +numbers would let a deployment that silently lost data pass its checks. + +## Why hooks rather than configuration + +Services differ in ways configuration cannot express. One builds an image from a +Dockerfile; another adopts an image another service already built. One holds its +data in a shared database; another in its own. A hook is a small script in the +service's own directory, next to the Compose file it belongs to, which is where +someone changing that service will look. + +## Gates + +Ordered, each stopping the run: + +1. The service is registered and enabled +2. The worktree is clean and on the deployment branch +3. The committed bundle validates +4. The change is not a downgrade, and a major change has explicit approval +5. The release-age gate is met, or is explicitly overridden +6. Free space is above the floor +7. The candidate builds and verifies +8. A recovery set is created and verified +9. The deployment succeeds +10. Strict health passes + +Only then is the run recorded as successful. A deployment that leaves the +service unhealthy is a failed deployment whatever the build reported. diff --git a/docs/PROMOTION.md b/docs/PROMOTION.md new file mode 100644 index 00000000..9fc635e0 --- /dev/null +++ b/docs/PROMOTION.md @@ -0,0 +1,149 @@ +# Promotion + +Imports workflow definitions into an n8n service from a commit that is already +merged into a protected branch. + +It is exposed on the service rather than as a top-level corekit verb, because it +is n8n-specific: it drives `n8n-git` and the n8n CLI inside the container, and +nothing else in the catalogue could use it as written. + + corekit run promote [options] + +## The problem it solves + +A service where definitions can be authored freely is a poor place to keep +production credentials: anything that can write a definition and run it can use +those credentials, without ever reading the secret itself. A service that holds +production credentials is therefore a poor place to author freely. + +Splitting authoring from production is the usual answer, and it leaves one gap: +how a change crosses from one to the other. Get that crossing wrong and it +becomes the very thing the split was meant to prevent — a way to run an +arbitrary definition against real credentials. + +## The shape + +``` +authoring instance --push--> branch --review, merge--> protected branch + | + corekit run promote (host) + v + production instance +``` + +Four properties make the crossing safe: + +| Property | How | +|---|---| +| The artefact is fixed | A full 40-character commit SHA. A branch tip moves; a commit does not | +| It has been reviewed | The commit must be an ancestor of the protected branch. Whatever review that branch requires has therefore happened | +| It is recoverable | The target is exported before the import, and the run is recorded with the previous commit | +| It cannot be self-triggered | It is a host command. Nothing running inside either instance can invoke it | + +That last one matters most. An approval step that the authoring instance can +trigger is not a control — it becomes a prompt that gets waved through. Keep the +decision on the host, with a human. + +## Credentials are never imported + +`--credentials 0` is passed unconditionally. Credentials are created once, by +hand, on the production instance. Two consequences worth stating: + +- A promoted definition can only use credentials that already exist there, so + promotion cannot introduce new reach. +- Definitions exported from another instance are encrypted with *that* + instance's key and are inert anywhere else. Never export decrypted + credentials to work around this; that would defeat the entire arrangement. + +## Configuration + +Split by whether it is safe to publish. + +**`service.json`** — policy, committed: + +```json +"promotion": { + "enabled": true, + "protected_branch": "main", + "repository_path": "workflows", + "preserve_ids": true, + "backup_required": true, + "credentials": "never" +} +``` + +**The service `.env`** — site-specific, not tracked: + +``` +PROMOTION_REPOSITORY='owner/repo' +PROMOTION_TOKEN='...' +# Optional; these override the policy in service.json. +# No inline comments: the value is read literally, so a trailing comment +# would become part of the branch name or path. +PROMOTION_BRANCH='main' +PROMOTION_PATH='workflows' +``` + +Give the token **read access only**. The control is the branch protection on the +repository, not the secrecy of this token: a read-only token cannot promote +anything that has not been merged, so leaking it costs you nothing that matters. + +This is the right way round deliberately. If the authoring instance can run +shell commands or read files — and most can — then any token it can reach is +one an injected instruction can steal. Design so that stealing it is not worth +anything. + +## Use + +```bash +corekit run promote --plan # show what would happen +corekit run promote # promote the protected branch tip +corekit run promote --commit # promote one exact commit +corekit run promote --dry-run # import in the tool's dry-run mode +corekit run status # what is deployed, and from where +corekit run rollback-plan # recovery procedure +``` + +A service opts in by shipping a `cli.sh` that delegates to the shared +implementation in `the service's managed/promote.sh`, and a `promotion` block in its +`service.json`. + +`--plan` changes nothing and needs no lock, so it is safe to run at any time. + +## A note on `counts.sh` + +Promotion reads the target's workflow count through the service's `counts.sh` +hook, to tell a genuinely empty target — where the first promotion happens and +an empty recovery set is expected — from one whose count cannot be read, where a +backup matters most. Without that hook the count is unknown, so the first +promotion into an empty target is refused. That is the safe default rather than a +fault, but it is worth knowing before adding promotion to a service that has no +counts hook. + +## Gates + +A promotion stops at the first failure: + +1. Promotion is enabled for the service in `service.json` +2. `PROMOTION_REPOSITORY` and `PROMOTION_TOKEN` are set +3. The target container exists and is running +4. The commit resolves, and is a full SHA if given explicitly +5. **The commit is an ancestor of the protected branch** — enforced by the + import tool, not by this script +6. A pre-promotion export succeeds and is non-empty +7. The import succeeds +8. The container is healthy afterwards + +Only then is the run recorded. A promotion that leaves the service unhealthy is +a failed promotion, whatever the import reported. + +## Recovery + +`corekit run rollback-plan` prints the procedure and the recorded +previous commit. Rolling forward to the previous commit is usually right and is +the same deterministic path. Restoring the pre-promotion export is for when the +definitions are not the problem. + +Restoring is refused automatically, by design: if the service may have accepted +writes since the promotion, an automatic restore would destroy them. Establish +that first. diff --git a/lib/managed_updates/managed-update.sh b/lib/managed_updates/managed-update.sh index 9a7c1396..54f5995b 100755 --- a/lib/managed_updates/managed-update.sh +++ b/lib/managed_updates/managed-update.sh @@ -11,14 +11,16 @@ fi USER_STATE_HOME="${XDG_STATE_HOME:-${HOME:-}/.local/state}" USER_DATA_HOME="${XDG_DATA_HOME:-${HOME:-}/.local/share}" STATE_ROOT="${COREKIT_MANAGED_STATE_ROOT:-$USER_STATE_HOME/ai-corekit/managed-updates}" -BACKUP_ROOT="${COREKIT_N8N_BACKUP_ROOT:-$USER_DATA_HOME/ai-corekit/backups/n8n}" LOCK_FILE="${COREKIT_MANAGED_LOCK_FILE:-$STATE_ROOT/managed-update.lock}" -N8N_DIR="$PROJECT_ROOT/services/workflow-automation/n8n" -POLICY_FILE="$N8N_DIR/service.json" -STATE_FILE="$STATE_ROOT/n8n.json" COMMAND="${1:-help}" SERVICE="${2:-n8n}" +# Resolved once the service name is known; see below. +SERVICE_DIR="" +POLICY_FILE="" +STATE_FILE="" +BACKUP_ROOT="" + shift || true shift || true @@ -57,6 +59,7 @@ Commands: apply [service] Fast-forward the deployment branch and apply an eligible committed candidate status [service] Show redacted runtime state and live versions rollback-plan [service] Show guarded recovery information; never restores a stateful DB automatically + list List every service registered for managed updates Options: --offline Do not contact the Git remote @@ -118,46 +121,95 @@ elif [[ -e "$LOCK_FILE" ]]; then flock -n -s 9 || die "Another managed updater process is already running" fi -[[ "$SERVICE" == "n8n" ]] || die "Service '$SERVICE' is not opted in to managed updates" -[[ -f "$POLICY_FILE" ]] || die "n8n managed-update policy is missing" -[[ "$(jq -r '.managed_update.enabled // false' "$POLICY_FILE")" == "true" ]] || die "n8n managed updates are disabled" +if [[ "$COMMAND" == "list" ]]; then + # Registration lives in each service's own service.json, so this is the one + # place that answers "what will the timers touch, and is it ready?". + while IFS= read -r dir; do + policy="$dir/service.json" + [[ -f "$policy" ]] || continue + jq -e '.managed_update' "$policy" >/dev/null 2>&1 || continue + missing=() + for hook in version.sh deploy.sh strict-healthcheck.sh; do + [[ -x "$dir/managed/$hook" ]] || missing+=("$hook") + done + if [[ "$(jq -r '.managed_update.backup_required // false' "$policy")" == "true" ]] \ + && [[ ! -x "$dir/managed/backup.sh" ]]; then + missing+=("backup.sh") + fi + jq -cn \ + --arg service "$(basename "$dir")" \ + --argjson enabled "$(jq -r '.managed_update.enabled // false' "$policy")" \ + --arg branch "$(jq -r '.managed_update.deployment_branch // ""' "$policy")" \ + --argjson age "$(jq -r '.managed_update.minimum_release_age_days // null' "$policy")" \ + --argjson components "$(jq -c '.managed_update.components // []' "$policy")" \ + --argjson missing "$(printf '%s\n' "${missing[@]+"${missing[@]}"}" | jq -R . | jq -s 'map(select(. != ""))')" \ + '{service:$service, enabled:$enabled, deployment_branch:$branch, + minimum_release_age_days:$age, components:$components, + missing_hooks:$missing, ready:(($enabled) and ($missing|length)==0)}' + done < <(find "$PROJECT_ROOT/services" -mindepth 2 -maxdepth 2 -type d 2>/dev/null | sort) \ + | jq -s '{registered: map(select(.enabled)), not_registered: map(select(.enabled|not))}' + exit 0 +fi + +# A service is registered by a managed_update block in its own service.json. +# Nothing here knows which services exist. +SERVICE_DIR="$(find "$PROJECT_ROOT/services" -mindepth 2 -maxdepth 2 -type d -name "$SERVICE" -print -quit 2>/dev/null || true)" +[[ -n "$SERVICE_DIR" ]] || die "No such service: $SERVICE" +POLICY_FILE="$SERVICE_DIR/service.json" +STATE_FILE="$STATE_ROOT/${SERVICE}.json" +# COREKIT_N8N_BACKUP_ROOT is honoured only for n8n itself. It predates +# multi-service support and is set in the systemd units, so applying it to every +# service would send one service's recovery sets into another's directory. +if [[ -n "${COREKIT_MANAGED_BACKUP_ROOT:-}" ]]; then + BACKUP_ROOT="$COREKIT_MANAGED_BACKUP_ROOT" +elif [[ "$SERVICE" == "n8n" && -n "${COREKIT_N8N_BACKUP_ROOT:-}" ]]; then + BACKUP_ROOT="$COREKIT_N8N_BACKUP_ROOT" +else + BACKUP_ROOT="$USER_DATA_HOME/ai-corekit/backups/$SERVICE" +fi + +[[ -f "$POLICY_FILE" ]] || die "Service '$SERVICE' has no service.json" +[[ "$(jq -r '.managed_update.enabled // false' "$POLICY_FILE")" == "true" ]] \ + || die "Service '$SERVICE' is not registered for managed updates. Set managed_update.enabled in its service.json." + +# Everything service-specific is a hook in the service's own managed/ directory. +HOOK_DIR="$SERVICE_DIR/managed" +export COREKIT_PROJECT_ROOT="$PROJECT_ROOT" +export COREKIT_SERVICE_DIR="$SERVICE_DIR" +export COREKIT_SERVICE_NAME="$SERVICE" + +have_hook() { [[ -x "$HOOK_DIR/$1" ]]; } +run_hook() { + local hook="$1"; shift + [[ -x "$HOOK_DIR/$hook" ]] || die "Service '$SERVICE' is registered for managed updates but has no $hook hook" + bash "$HOOK_DIR/$hook" "$@" +} + +VERSION_INFO="$(run_hook version.sh)" || die "Could not determine versions for '$SERVICE'" +jq -e . <<<"$VERSION_INFO" >/dev/null 2>&1 || die "version.sh did not return valid JSON" +TARGET_VERSION="$(jq -r '.target_version' <<<"$VERSION_INFO")" +BUNDLE_IDENTITY="$(jq -r '.identity' <<<"$VERSION_INFO")" DEPLOYMENT_BRANCH="$(jq -r '.managed_update.deployment_branch' "$POLICY_FILE")" MINIMUM_RELEASE_AGE_DAYS="$(jq -r '.managed_update.minimum_release_age_days' "$POLICY_FILE")" MINIMUM_FREE_SPACE_GB="$(jq -r '.managed_update.minimum_free_space_gb' "$POLICY_FILE")" RECOVERY_SETS_TO_KEEP="$(jq -r '.managed_update.recovery_sets_to_keep' "$POLICY_FILE")" -N8N_REF="$(awk '$1 == "FROM" && $2 ~ /^n8nio\/n8n:/ {print $2; exit}' "$N8N_DIR/Dockerfile")" -RUNNER_REF="$(awk '$1 == "FROM" && $2 ~ /^n8nio\/runners:/ {print $2; exit}' "$N8N_DIR/runner/Dockerfile")" -TARGET_VERSION="${N8N_REF#*:}" -TARGET_VERSION="${TARGET_VERSION%%@*}" -TARGET_N8N_DIGEST="${N8N_REF##*@}" -TARGET_RUNNER_VERSION="${RUNNER_REF#*:}" -TARGET_RUNNER_VERSION="${TARGET_RUNNER_VERSION%%@*}" -TARGET_RUNNER_DIGEST="${RUNNER_REF##*@}" -N8N_GIT_VERSION="$(awk -F= '$1 == "ARG N8N_GIT_VERSION" {print $2}' "$N8N_DIR/Dockerfile")" -N8N_GIT_COMMIT="$(awk -F= '$1 == "ARG N8N_GIT_COMMIT" {print $2}' "$N8N_DIR/Dockerfile")" +# Kept for the structured output below; sourced from the service's version hook +# rather than parsed here, so the driver stays service-agnostic. +TARGET_N8N_DIGEST="$(jq -r '.refs.n8n_digest // ""' <<<"$VERSION_INFO")" +TARGET_RUNNER_DIGEST="$(jq -r '.refs.runner_digest // ""' <<<"$VERSION_INFO")" +N8N_GIT_VERSION="$(jq -r '.refs.n8n_git_version // ""' <<<"$VERSION_INFO")" +N8N_GIT_COMMIT="$(jq -r '.refs.n8n_git_commit // ""' <<<"$VERSION_INFO")" validate_bundle() { - [[ "$TARGET_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "Pinned n8n version is invalid" - [[ "$TARGET_VERSION" == "$TARGET_RUNNER_VERSION" ]] || die "Pinned n8n and runner versions differ" - [[ "$TARGET_N8N_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] || die "Pinned n8n digest is invalid" - [[ "$TARGET_RUNNER_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] || die "Pinned runner digest is invalid" - [[ "$N8N_GIT_VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "Pinned n8n-git version is invalid" - [[ "$N8N_GIT_COMMIT" =~ ^[0-9a-f]{40}$ ]] || die "Pinned n8n-git commit is invalid" - jq -e . "$POLICY_FILE" "$N8N_DIR/task-runners-managed.json" >/dev/null || die "Managed n8n JSON configuration is invalid" - if grep -ERn '(^|[[:space:]:])latest([@[:space:]]|$)' \ - "$N8N_DIR/Dockerfile" "$N8N_DIR/runner/Dockerfile" "$N8N_DIR/docker-compose.yml" >/dev/null; then - die "Moving latest reference found in the managed n8n bundle" + if have_hook validate.sh; then + run_hook validate.sh || die "Committed bundle for '$SERVICE' failed validation" fi } live_version() { - if docker inspect n8n >/dev/null 2>&1 && [[ "$(docker inspect --format '{{.State.Running}}' n8n)" == "true" ]]; then - docker exec n8n n8n --version 2>/dev/null | tr -d '\r' - else - printf 'unknown\n' - fi + jq -r '.live_version // "unknown"' <<<"$VERSION_INFO" } version_change_type() { @@ -194,7 +246,7 @@ remote_head() { release_age_days() { local committed_at now - committed_at="$(git -C "$PROJECT_ROOT" log -1 --format=%ct -- "$N8N_DIR/Dockerfile")" + committed_at="$(git -C "$PROJECT_ROOT" log -1 --format=%ct -- "$SERVICE_DIR")" [[ -n "$committed_at" ]] || die "Unable to determine candidate commit age" now="$(date +%s)" printf '%s\n' "$(( (now - committed_at) / 86400 ))" @@ -205,8 +257,12 @@ free_space_gb() { } bundle_fingerprint() { - git -C "$PROJECT_ROOT" ls-tree -r HEAD -- "${N8N_DIR#"$PROJECT_ROOT"/}" \ - | sha256sum | awk '{print $1}' + # The committed tree plus the identity the service reports, so a change to + # either is a change of candidate. + { + git -C "$PROJECT_ROOT" ls-tree -r HEAD -- "${SERVICE_DIR#"$PROJECT_ROOT"/}" + printf '%s\n' "$BUNDLE_IDENTITY" + } | sha256sum | awk '{print $1}' } version_is_greater() { @@ -220,34 +276,24 @@ version_is_greater() { (( target_major == current_major && target_minor == current_minor && target_patch > current_patch )) } -active_execution_count() { - docker exec postgres psql -X -U postgres -d postgres -At -v ON_ERROR_STOP=1 \ - -c "SELECT count(*) FROM execution_entity WHERE status IN ('new','running','waiting','unknown');" +# Counts come from the service, because a service with its own database must +# not be measured against another instance's. +service_counts() { + if have_hook counts.sh; then + run_hook counts.sh + else + printf '{"workflows":null,"credentials":null,"active_executions":"0"}\n' + fi } - +active_execution_count() { jq -r '.active_executions // "0"' <<<"$(service_counts)"; } entity_count() { - local table="$1" - docker exec postgres psql -X -U postgres -d postgres -At -v ON_ERROR_STOP=1 \ - -c "SELECT count(*) FROM ${table};" -} - -load_required_environment() { - export PROJECT_ROOT - export COMPOSE_PROFILES=n8n - if [[ -f "$PROJECT_ROOT/config/.env.global" ]]; then - N8N_HOSTNAME="$(sed -n 's/^N8N_HOSTNAME=//p' "$PROJECT_ROOT/config/.env.global" | tail -n 1)" - N8N_HOSTNAME="${N8N_HOSTNAME%$'\r'}" - N8N_HOSTNAME="${N8N_HOSTNAME#\"}" - N8N_HOSTNAME="${N8N_HOSTNAME%\"}" - N8N_HOSTNAME="${N8N_HOSTNAME#\'}" - N8N_HOSTNAME="${N8N_HOSTNAME%\'}" - export N8N_HOSTNAME - fi + case "$1" in + workflow_entity) jq -r '.workflows // "unknown"' <<<"$(service_counts)" ;; + credentials_entity) jq -r '.credentials // "unknown"' <<<"$(service_counts)" ;; + *) printf 'unknown\n' ;; + esac } -compose_project() { - docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' n8n 2>/dev/null || printf 'localai\n' -} print_plan() { local current_version change_type current_branch head remote age free_gb active changed current_commit fingerprint @@ -337,7 +383,8 @@ write_failure_state() { --arg git_commit "$(git -C "$PROJECT_ROOT" rev-parse HEAD)" \ --arg target_version "$TARGET_VERSION" \ --arg backup_manifest "$backup_manifest" \ - '{schema_version:1,service:"n8n",last_result:{result:"failed",timestamp:$timestamp,stage:$stage,git_commit:$git_commit,target_version:$target_version,backup_manifest:$backup_manifest,writes_possible:true,automatic_database_restore_permitted:false}}' \ + --arg service_name "$SERVICE" \ + '{schema_version:1,service:$service_name,last_result:{result:"failed",timestamp:$timestamp,stage:$stage,git_commit:$git_commit,target_version:$target_version,backup_manifest:$backup_manifest,writes_possible:true,automatic_database_restore_permitted:false}}' \ >"$tmp_state" fi chmod 0600 "$tmp_state" @@ -350,8 +397,14 @@ write_success_state() { local previous_runner_image_id="$3" local backup_manifest="$4" local n8n_image_id runner_image_id tmp_state previous_state history fingerprint previous_git_commit - n8n_image_id="$(docker inspect --format '{{.Image}}' n8n)" - runner_image_id="$(docker inspect --format '{{.Image}}' n8n-runner)" + local -a components + mapfile -t components < <(jq -r '.managed_update.components[]? // empty' "$POLICY_FILE") + (( ${#components[@]} > 0 )) || components=("$SERVICE") + # Guarded: a missing container must not abort after a successful deployment + # and leave no record that it happened. + n8n_image_id="$(docker inspect --format '{{.Image}}' "${components[0]}" 2>/dev/null || true)" + runner_image_id="" + (( ${#components[@]} > 1 )) && runner_image_id="$(docker inspect --format '{{.Image}}' "${components[1]}" 2>/dev/null || true)" fingerprint="$(bundle_fingerprint)" previous_git_commit="$(jq -r '.git.previous_live_commit // empty' "$backup_manifest")" previous_state='null' @@ -384,9 +437,10 @@ write_success_state() { --arg backup_manifest "$backup_manifest" \ --argjson previous "$previous_state" \ --argjson history "$history" \ + --arg service_name "$SERVICE" \ '{ schema_version:1, - service:"n8n", + service:$service_name, current:{ version:$version, git_commit:$git_commit, @@ -411,8 +465,11 @@ write_success_state() { verify_recovery_retention() { local verified_count - verified_count="$(find "$BACKUP_ROOT" -mindepth 2 -maxdepth 2 -name manifest.json -type f -print0 \ - | xargs -0 -r jq -r 'select(.verified == true) | .created_at' | wc -l)" + verified_count=0 + if [[ -d "$BACKUP_ROOT" ]]; then + verified_count="$( { find "$BACKUP_ROOT" -mindepth 2 -maxdepth 2 -name manifest.json -type f -print0 \ + | xargs -0 -r jq -r 'select(.verified == true) | .created_at' | wc -l; } 2>/dev/null || echo 0)" + fi log_event info recovery_retention "Verified recovery sets are preserved without automatic deletion; policy floor is ${RECOVERY_SETS_TO_KEEP}, current count is ${verified_count}" } @@ -420,8 +477,8 @@ apply_update() { validate_bundle git_is_clean || die "Git worktree is dirty; refusing managed deployment" - local branch current_version change_type age free_gb current_commit short_commit fingerprint - local candidate_image candidate_runner_image project_name workflow_count credential_count + local branch current_version change_type age free_gb current_commit fingerprint + local candidate_image workflow_count credential_count local previous_n8n_image_id previous_runner_image_id backup_manifest active branch="$(git -C "$PROJECT_ROOT" branch --show-current)" @@ -436,6 +493,23 @@ apply_update() { git_is_clean || die "Git worktree became dirty after fetch" git -C "$PROJECT_ROOT" merge-base --is-ancestor HEAD "origin/$DEPLOYMENT_BRANCH" || die "Remote change is not fast-forwardable" git -C "$PROJECT_ROOT" merge --ff-only "origin/$DEPLOYMENT_BRANCH" + # The merge can change what is committed, so everything read from the tree + # before it is now stale. Re-read before validating against it. + VERSION_INFO="$(run_hook version.sh)" || die "Could not re-read versions after the update" + jq -e . <<<"$VERSION_INFO" >/dev/null 2>&1 || die "version.sh did not return valid JSON after the update" + TARGET_VERSION="$(jq -r '.target_version' <<<"$VERSION_INFO")" + BUNDLE_IDENTITY="$(jq -r '.identity' <<<"$VERSION_INFO")" + # Policy is part of the committed tree too. A commit that raises the + # release-age floor, or changes the component list the recovery path stops, + # must govern this run rather than the version it replaced. + DEPLOYMENT_BRANCH="$(jq -r '.managed_update.deployment_branch' "$POLICY_FILE")" + MINIMUM_RELEASE_AGE_DAYS="$(jq -r '.managed_update.minimum_release_age_days' "$POLICY_FILE")" + MINIMUM_FREE_SPACE_GB="$(jq -r '.managed_update.minimum_free_space_gb' "$POLICY_FILE")" + RECOVERY_SETS_TO_KEEP="$(jq -r '.managed_update.recovery_sets_to_keep' "$POLICY_FILE")" + TARGET_N8N_DIGEST="$(jq -r '.refs.n8n_digest // ""' <<<"$VERSION_INFO")" + TARGET_RUNNER_DIGEST="$(jq -r '.refs.runner_digest // ""' <<<"$VERSION_INFO")" + N8N_GIT_VERSION="$(jq -r '.refs.n8n_git_version // ""' <<<"$VERSION_INFO")" + N8N_GIT_COMMIT="$(jq -r '.refs.n8n_git_commit // ""' <<<"$VERSION_INFO")" validate_bundle fi @@ -472,23 +546,23 @@ apply_update() { active="$(active_execution_count)" [[ "$active" == "0" ]] || die "Active or waiting executions prevent deployment" - short_commit="$(git -C "$PROJECT_ROOT" rev-parse --short=12 HEAD)" - candidate_image="corekit/n8n:${TARGET_VERSION}-git${short_commit}" - candidate_runner_image="corekit/n8n-runners:${TARGET_VERSION}-git${short_commit}" - log_event info candidate_build_started "Building exact committed n8n and runner candidates before downtime" - docker build --pull=false --tag "$candidate_image" "$N8N_DIR" - docker build --pull=false --tag "$candidate_runner_image" "$N8N_DIR/runner" - - [[ "$(docker run --rm --network none "$candidate_image" n8n --version | tr -d '\r')" == "$TARGET_VERSION" ]] || die "Candidate n8n version verification failed" - [[ "$(docker image inspect --format '{{index .Config.Labels "io.corekit.n8n-git.version"}}' "$candidate_image")" == "$N8N_GIT_VERSION" ]] || die "Candidate n8n-git version verification failed" - [[ "$(docker image inspect --format '{{index .Config.Labels "io.corekit.n8n-git.commit"}}' "$candidate_image")" == "$N8N_GIT_COMMIT" ]] || die "Candidate n8n-git commit verification failed" - [[ "$(docker image inspect --format '{{index .Config.Labels "io.corekit.runner-for-n8n-version"}}' "$candidate_runner_image")" == "$TARGET_VERSION" ]] || die "Candidate runner version verification failed" - [[ "$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.version"}}' "$candidate_runner_image")" == "$TARGET_VERSION" ]] || die "Candidate upstream runner version verification failed" + log_event info candidate_build_started "Building and verifying the committed candidate before any downtime" + CANDIDATE_IMAGES="$(run_hook build.sh "$TARGET_VERSION" | tail -n 1)" \ + || die "Candidate build or verification failed" + jq -e . <<<"$CANDIDATE_IMAGES" >/dev/null 2>&1 || die "build.sh did not return valid JSON" + candidate_image="$(jq -r '.images.app // empty' <<<"$CANDIDATE_IMAGES")" + [[ -n "$candidate_image" ]] || die "build.sh did not report a candidate image" workflow_count="$(entity_count workflow_entity)" credential_count="$(entity_count credentials_entity)" - previous_n8n_image_id="$(docker inspect --format '{{.Image}}' n8n)" - previous_runner_image_id="$(docker inspect --format '{{.Image}}' n8n-runner 2>/dev/null || true)" + # Recovery points at the image to restart, so it must be THIS service's + # container. Reading a fixed name would hand production the authoring + # instance's image. + mapfile -t _components < <(jq -r '.managed_update.components[]? // empty' "$POLICY_FILE") + (( ${#_components[@]} > 0 )) || _components=("$SERVICE") + previous_n8n_image_id="$(docker inspect --format '{{.Image}}' "${_components[0]}" 2>/dev/null || true)" + previous_runner_image_id="" + (( ${#_components[@]} > 1 )) && previous_runner_image_id="$(docker inspect --format '{{.Image}}' "${_components[1]}" 2>/dev/null || true)" if [[ -z "$PREVIOUS_GIT_COMMIT" ]]; then if [[ -f "$STATE_FILE" ]]; then PREVIOUS_GIT_COMMIT="$(jq -r '.current.git_commit // empty' "$STATE_FILE")" @@ -499,8 +573,9 @@ apply_update() { fi log_event info backup_started "Creating mandatory final recovery set and isolated restore drill" - backup_manifest="$(COREKIT_PROJECT_ROOT="$PROJECT_ROOT" COREKIT_N8N_BACKUP_ROOT="$BACKUP_ROOT" \ - bash "$N8N_DIR/managed/backup.sh" --label "pre-${TARGET_VERSION}" \ + backup_manifest="$(COREKIT_PROJECT_ROOT="$PROJECT_ROOT" \ + COREKIT_MANAGED_BACKUP_ROOT="$BACKUP_ROOT" COREKIT_N8N_BACKUP_ROOT="$BACKUP_ROOT" \ + bash "$HOOK_DIR/backup.sh" --label "pre-${TARGET_VERSION}" \ --previous-git-commit "$PREVIOUS_GIT_COMMIT" --candidate-image "$candidate_image" | tail -n 1)" [[ -f "$backup_manifest" ]] || die "Backup script did not return a verified manifest" [[ "$(jq -r '.verified' "$backup_manifest")" == "true" ]] || die "Recovery set is not verified" @@ -508,42 +583,36 @@ apply_update() { [[ "$(jq -r '.candidate_migration_drill.result' "$backup_manifest")" == "passed" ]] || die "Candidate migration rehearsal did not pass" [[ "$(active_execution_count)" == "0" ]] || die "Executions became active after the backup gate" - load_required_environment - export N8N_MANAGED_IMAGE="$candidate_image" - export N8N_MANAGED_RUNNER_IMAGE="$candidate_runner_image" - project_name="$(compose_project)" - [[ -n "$project_name" ]] || project_name=localai - - mapfile -t old_workers < <(docker ps -a --filter label=com.docker.compose.service=n8n-worker --format '{{.Names}}' | sort) - if (( ${#old_workers[@]} > 0 )); then - docker stop --time 60 "${old_workers[@]}" >/dev/null - fi - - log_event info deployment_started "Recreating only the coordinated n8n main and runner bundle" - if ! docker compose -p "$project_name" --project-directory "$N8N_DIR" \ - --env-file "$PROJECT_ROOT/services/data-services/postgres/.env" \ - --env-file "$N8N_DIR/.env" -f "$N8N_DIR/docker-compose.yml" \ - up -d --no-deps n8n n8n-runner; then + # The environment a deployment needs is service-specific, so the deploy hook + # assembles it. The driver supplies only the candidate images. + log_event info deployment_started "Recreating only this service's containers with the candidate" + if ! COREKIT_CANDIDATE_IMAGES="$CANDIDATE_IMAGES" run_hook deploy.sh "$TARGET_VERSION"; then write_failure_state compose_up "$backup_manifest" - die "Candidate Compose deployment failed; stateful recovery requires the guarded rollback plan" + die "Candidate deployment failed; stateful recovery requires the guarded rollback plan" fi - if ! COREKIT_MANAGED_STATE_ROOT="$STATE_ROOT" bash "$N8N_DIR/managed/strict-healthcheck.sh" \ + # The canary must exercise the candidate, not whatever a compose fallback tag + # resolves to. It is a second consumer of the images build.sh produced. + if ! COREKIT_MANAGED_STATE_ROOT="$STATE_ROOT" \ + COREKIT_CANDIDATE_IMAGES="$CANDIDATE_IMAGES" \ + bash "$HOOK_DIR/strict-healthcheck.sh" \ --expected-workflows "$workflow_count" \ --expected-credentials "$credential_count" \ --canary; then - docker stop --time 30 n8n-runner n8n >/dev/null 2>&1 || true + if have_hook stop.sh; then + run_hook stop.sh >/dev/null 2>&1 || true + else + # Fall back to the components the policy names. + mapfile -t _components < <(jq -r '.managed_update.components[]? // empty' "$POLICY_FILE") + (( ${#_components[@]} > 0 )) && docker stop --time 30 "${_components[@]}" >/dev/null 2>&1 || true + fi write_failure_state strict_health "$backup_manifest" die "Candidate failed strict health; stopped without automatic database restore because writes are conservatively possible" fi - if (( ${#old_workers[@]} > 0 )); then - docker rm "${old_workers[@]}" >/dev/null - fi - write_success_state "$current_version" "$previous_n8n_image_id" "$previous_runner_image_id" "$backup_manifest" verify_recovery_retention - log_event info deployment_succeeded "n8n managed deployment passed readiness, canary, counts, audit, and restore gates" + log_event info deployment_succeeded "Managed deployment of '$SERVICE' passed readiness, canary, counts, audit, and restore gates" } show_status() { diff --git a/lib/services/credentials/export.sh b/lib/services/credentials/export.sh index f2bd63d0..c17e494d 100755 --- a/lib/services/credentials/export.sh +++ b/lib/services/credentials/export.sh @@ -64,7 +64,13 @@ fi # Create filename with hostname (gets overwritten each time) OUTPUT_DIR="$PROJECT_ROOT/config/local" mkdir -p "$OUTPUT_DIR" +# The export is a plaintext copy of every credential on the host. Keep the +# directory and the file readable only by the account that created them. +chmod 700 "$OUTPUT_DIR" OUTPUT_FILE="$OUTPUT_DIR/credentials.${BASE_DOMAIN}.txt" +# Create the file with the right mode before anything is written to it, so the +# contents are never briefly world-readable under a permissive umask. +install -m 600 /dev/null "$OUTPUT_FILE" # ============================================================================ # Header @@ -103,9 +109,15 @@ bash "$FINAL_REPORT_SCRIPT" 2>&1 | \ # If running as sudo, adjust ownership to the actual user if [ -n "$SUDO_USER" ]; then chown "$SUDO_USER:$SUDO_USER" "$OUTPUT_FILE" + chown "$SUDO_USER:$SUDO_USER" "$OUTPUT_DIR" log_info "File ownership adjusted to: $SUDO_USER" fi +# Re-assert the mode in case anything above replaced the file rather than +# writing through the handle we created. +chmod 600 "$OUTPUT_FILE" +chmod 700 "$OUTPUT_DIR" + # ============================================================================ # Success Message # ============================================================================ @@ -124,6 +136,11 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━ echo "🔒 SECURITY REMINDER" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo +echo "➡️ Import this file into your password manager, then delete it." +echo " It is a plaintext copy of every credential on this host, and nothing" +echo " rotates or expires it. Leaving it on disk is the risk it creates." +echo " Delete it with: shred -u $OUTPUT_FILE" +echo echo "⚠️ This file contains sensitive credentials:" echo " • Passwords, API keys, and tokens" echo " • Store securely (password manager recommended)" @@ -149,8 +166,8 @@ echo echo "To download this file securely (encrypted zip):" echo " corekit credentials download" echo -echo "Remember to delete the file after use:" -echo " rm $OUTPUT_FILE" +echo "Import it into your password manager, then delete it:" +echo " shred -u $OUTPUT_FILE" echo exit 0 diff --git a/services/agentic-frameworks/openclaw/docker-compose.yml b/services/agentic-frameworks/openclaw/docker-compose.yml index 15116bd0..3bdf1bbd 100644 --- a/services/agentic-frameworks/openclaw/docker-compose.yml +++ b/services/agentic-frameworks/openclaw/docker-compose.yml @@ -22,8 +22,8 @@ services: - openclaw_config:/home/node/.openclaw - openclaw_workspace:/home/node/.openclaw/workspace ports: - - "${OPENCLAW_GATEWAY_PORT:-18789}:18789" - - "${OPENCLAW_BRIDGE_PORT:-18790}:18790" + - "127.0.0.1:${OPENCLAW_GATEWAY_PORT:-18789}:18789" + - "127.0.0.1:${OPENCLAW_BRIDGE_PORT:-18790}:18790" init: true restart: unless-stopped entrypoint: diff --git a/services/data-services/n8n-postgres/.env.example b/services/data-services/n8n-postgres/.env.example new file mode 100644 index 00000000..3636ef3a --- /dev/null +++ b/services/data-services/n8n-postgres/.env.example @@ -0,0 +1,5 @@ +# Postgres dedicated to one n8n instance. Single quotes, per CLAUDE.md. +N8N_PLAY_DB_NAME='n8n' +N8N_PLAY_DB_USER='n8nplay' +# openssl rand -hex 24 +N8N_PLAY_DB_PASSWORD='' diff --git a/services/data-services/n8n-postgres/docker-compose.yml b/services/data-services/n8n-postgres/docker-compose.yml new file mode 100644 index 00000000..687631ec --- /dev/null +++ b/services/data-services/n8n-postgres/docker-compose.yml @@ -0,0 +1,37 @@ +# A Postgres dedicated to one n8n instance. +# +# Worth having when the shared server hosts several services' databases behind a +# single superuser role: every service holding that password can then read every +# other service's tables, n8n's included. A dedicated server lets the stack own +# its data, with no network path to it from outside. +services: + n8n-postgres: + image: postgres:17 + container_name: n8n-postgres + profiles: + - n8n-postgres + restart: unless-stopped + environment: + POSTGRES_DB: ${N8N_PLAY_DB_NAME:-n8n} + POSTGRES_USER: ${N8N_PLAY_DB_USER:-n8nplay} + POSTGRES_PASSWORD: ${N8N_PLAY_DB_PASSWORD} + POSTGRES_INITDB_ARGS: "--data-checksums" + command: + - postgres + - -c + - shared_buffers=128MB + - -c + - max_connections=50 + volumes: + - ./data/db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${N8N_PLAY_DB_USER:-n8nplay} -d ${N8N_PLAY_DB_NAME:-n8n}"] + interval: 10s + timeout: 5s + retries: 10 + mem_limit: 768m + logging: + driver: json-file + options: + max-size: 10m + max-file: '3' diff --git a/services/data-services/n8n-postgres/service.json b/services/data-services/n8n-postgres/service.json new file mode 100644 index 00000000..09bda06b --- /dev/null +++ b/services/data-services/n8n-postgres/service.json @@ -0,0 +1,9 @@ +{ + "name": "n8n-postgres", + "display_name": "Postgres (dedicated n8n)", + "category": "Data Services", + "rank": 20, + "description": "Postgres dedicated to a single n8n instance, for deployments where the shared server hosts other services' databases.", + "source": "https://github.com/postgres/postgres", + "depends_on": [] +} diff --git a/services/gateways-proxies/admin-tunnel/.env.example b/services/gateways-proxies/admin-tunnel/.env.example new file mode 100644 index 00000000..5d47df94 --- /dev/null +++ b/services/gateways-proxies/admin-tunnel/.env.example @@ -0,0 +1,4 @@ +# Token for the admin stack's own Cloudflare tunnel. +# Create the tunnel in the Cloudflare dashboard, then paste its token here. +# Single quotes, as required by CLAUDE.md. Never commit this file with a value. +ADMIN_TUNNEL_TOKEN='' diff --git a/services/gateways-proxies/admin-tunnel/docker-compose.yml b/services/gateways-proxies/admin-tunnel/docker-compose.yml new file mode 100644 index 00000000..accfd4c7 --- /dev/null +++ b/services/gateways-proxies/admin-tunnel/docker-compose.yml @@ -0,0 +1,43 @@ +# Cloudflare Tunnel connector for the admin stack. +# +# Each stack gets its own tunnel because Cloudflare may send any hostname on a +# tunnel to any connector running that tunnel's token. One tunnel shared across +# stacks would therefore defeat the network separation between them. +services: + admin-tunnel: + image: cloudflare/cloudflared:latest + container_name: admin-tunnel + profiles: + - admin-tunnel + restart: always + command: tunnel --no-autoupdate run + env_file: + - .env + environment: + - TUNNEL_TOKEN=${ADMIN_TUNNEL_TOKEN} + - TUNNEL_METRICS=127.0.0.1:60123 + deploy: + resources: + reservations: + memory: 64M + limits: + memory: 256M + mem_reservation: 64m + mem_limit: 256m + oom_score_adj: -900 + logging: + driver: json-file + options: + max-size: 10m + max-file: '5' + healthcheck: + test: + - CMD + - cloudflared + - tunnel + - --metrics + - localhost:60123 + - ready + interval: 30s + timeout: 30s + retries: 3 diff --git a/services/gateways-proxies/admin-tunnel/service.json b/services/gateways-proxies/admin-tunnel/service.json new file mode 100644 index 00000000..a62cd3fb --- /dev/null +++ b/services/gateways-proxies/admin-tunnel/service.json @@ -0,0 +1,9 @@ +{ + "name": "admin-tunnel", + "display_name": "Cloudflare Tunnel (admin)", + "category": "Gateways Proxies", + "rank": 5, + "description": "Dedicated Cloudflare Tunnel connector for the admin stack.", + "source": "https://github.com/cloudflare/cloudflared", + "depends_on": [] +} diff --git a/services/gateways-proxies/agents-tunnel/.env.example b/services/gateways-proxies/agents-tunnel/.env.example new file mode 100644 index 00000000..c799211f --- /dev/null +++ b/services/gateways-proxies/agents-tunnel/.env.example @@ -0,0 +1,4 @@ +# Token for the agents stack's own Cloudflare tunnel. +# Create the tunnel in the Cloudflare dashboard, then paste its token here. +# Single quotes, as required by CLAUDE.md. Never commit this file with a value. +AGENTS_TUNNEL_TOKEN='' diff --git a/services/gateways-proxies/agents-tunnel/docker-compose.yml b/services/gateways-proxies/agents-tunnel/docker-compose.yml new file mode 100644 index 00000000..225a4c03 --- /dev/null +++ b/services/gateways-proxies/agents-tunnel/docker-compose.yml @@ -0,0 +1,43 @@ +# Cloudflare Tunnel connector for the agents stack. +# +# Each stack gets its own tunnel because Cloudflare may send any hostname on a +# tunnel to any connector running that tunnel's token. One tunnel shared across +# stacks would therefore defeat the network separation between them. +services: + agents-tunnel: + image: cloudflare/cloudflared:latest + container_name: agents-tunnel + profiles: + - agents-tunnel + restart: always + command: tunnel --no-autoupdate run + env_file: + - .env + environment: + - TUNNEL_TOKEN=${AGENTS_TUNNEL_TOKEN} + - TUNNEL_METRICS=127.0.0.1:60123 + deploy: + resources: + reservations: + memory: 64M + limits: + memory: 256M + mem_reservation: 64m + mem_limit: 256m + oom_score_adj: -900 + logging: + driver: json-file + options: + max-size: 10m + max-file: '5' + healthcheck: + test: + - CMD + - cloudflared + - tunnel + - --metrics + - localhost:60123 + - ready + interval: 30s + timeout: 30s + retries: 3 diff --git a/services/gateways-proxies/agents-tunnel/service.json b/services/gateways-proxies/agents-tunnel/service.json new file mode 100644 index 00000000..9d539232 --- /dev/null +++ b/services/gateways-proxies/agents-tunnel/service.json @@ -0,0 +1,9 @@ +{ + "name": "agents-tunnel", + "display_name": "Cloudflare Tunnel (agents)", + "category": "Gateways Proxies", + "rank": 5, + "description": "Dedicated Cloudflare Tunnel connector for the agents stack.", + "source": "https://github.com/cloudflare/cloudflared", + "depends_on": [] +} diff --git a/services/gateways-proxies/mail-tunnel/.env.example b/services/gateways-proxies/mail-tunnel/.env.example new file mode 100644 index 00000000..76c05c2f --- /dev/null +++ b/services/gateways-proxies/mail-tunnel/.env.example @@ -0,0 +1,4 @@ +# Token for the mail stack's own Cloudflare tunnel. +# Create the tunnel in the Cloudflare dashboard, then paste its token here. +# Single quotes, as required by CLAUDE.md. Never commit this file with a value. +MAIL_TUNNEL_TOKEN='' diff --git a/services/gateways-proxies/mail-tunnel/docker-compose.yml b/services/gateways-proxies/mail-tunnel/docker-compose.yml new file mode 100644 index 00000000..7ea0d302 --- /dev/null +++ b/services/gateways-proxies/mail-tunnel/docker-compose.yml @@ -0,0 +1,43 @@ +# Cloudflare Tunnel connector for the mail stack. +# +# Each stack gets its own tunnel because Cloudflare may send any hostname on a +# tunnel to any connector running that tunnel's token. One tunnel shared across +# stacks would therefore defeat the network separation between them. +services: + mail-tunnel: + image: cloudflare/cloudflared:latest + container_name: mail-tunnel + profiles: + - mail-tunnel + restart: always + command: tunnel --no-autoupdate run + env_file: + - .env + environment: + - TUNNEL_TOKEN=${MAIL_TUNNEL_TOKEN} + - TUNNEL_METRICS=127.0.0.1:60123 + deploy: + resources: + reservations: + memory: 64M + limits: + memory: 256M + mem_reservation: 64m + mem_limit: 256m + oom_score_adj: -900 + logging: + driver: json-file + options: + max-size: 10m + max-file: '5' + healthcheck: + test: + - CMD + - cloudflared + - tunnel + - --metrics + - localhost:60123 + - ready + interval: 30s + timeout: 30s + retries: 3 diff --git a/services/gateways-proxies/mail-tunnel/service.json b/services/gateways-proxies/mail-tunnel/service.json new file mode 100644 index 00000000..7d7848a3 --- /dev/null +++ b/services/gateways-proxies/mail-tunnel/service.json @@ -0,0 +1,9 @@ +{ + "name": "mail-tunnel", + "display_name": "Cloudflare Tunnel (mail)", + "category": "Gateways Proxies", + "rank": 5, + "description": "Dedicated Cloudflare Tunnel connector for the mail stack.", + "source": "https://github.com/cloudflare/cloudflared", + "depends_on": [] +} diff --git a/services/gateways-proxies/n8n-prod-tunnel/.env.example b/services/gateways-proxies/n8n-prod-tunnel/.env.example new file mode 100644 index 00000000..6106a6b9 --- /dev/null +++ b/services/gateways-proxies/n8n-prod-tunnel/.env.example @@ -0,0 +1,4 @@ +# Token for the n8n-prod stack's own Cloudflare tunnel. +# Create the tunnel in the Cloudflare dashboard, then paste its token here. +# Single quotes, as required by CLAUDE.md. Never commit this file with a value. +N8N_PROD_TUNNEL_TOKEN='' diff --git a/services/gateways-proxies/n8n-prod-tunnel/docker-compose.yml b/services/gateways-proxies/n8n-prod-tunnel/docker-compose.yml new file mode 100644 index 00000000..86c23540 --- /dev/null +++ b/services/gateways-proxies/n8n-prod-tunnel/docker-compose.yml @@ -0,0 +1,43 @@ +# Cloudflare Tunnel connector for the n8n-prod stack. +# +# Each stack gets its own tunnel because Cloudflare may send any hostname on a +# tunnel to any connector running that tunnel's token. One tunnel shared across +# stacks would therefore defeat the network separation between them. +services: + n8n-prod-tunnel: + image: cloudflare/cloudflared:latest + container_name: n8n-prod-tunnel + profiles: + - n8n-prod-tunnel + restart: always + command: tunnel --no-autoupdate run + env_file: + - .env + environment: + - TUNNEL_TOKEN=${N8N_PROD_TUNNEL_TOKEN} + - TUNNEL_METRICS=127.0.0.1:60123 + deploy: + resources: + reservations: + memory: 64M + limits: + memory: 256M + mem_reservation: 64m + mem_limit: 256m + oom_score_adj: -900 + logging: + driver: json-file + options: + max-size: 10m + max-file: '5' + healthcheck: + test: + - CMD + - cloudflared + - tunnel + - --metrics + - localhost:60123 + - ready + interval: 30s + timeout: 30s + retries: 3 diff --git a/services/gateways-proxies/n8n-prod-tunnel/service.json b/services/gateways-proxies/n8n-prod-tunnel/service.json new file mode 100644 index 00000000..50643945 --- /dev/null +++ b/services/gateways-proxies/n8n-prod-tunnel/service.json @@ -0,0 +1,9 @@ +{ + "name": "n8n-prod-tunnel", + "display_name": "Cloudflare Tunnel (n8n-prod)", + "category": "Gateways Proxies", + "rank": 5, + "description": "Dedicated Cloudflare Tunnel connector for the n8n-prod stack.", + "source": "https://github.com/cloudflare/cloudflared", + "depends_on": [] +} diff --git a/services/gateways-proxies/n8n-tunnel/.env.example b/services/gateways-proxies/n8n-tunnel/.env.example new file mode 100644 index 00000000..9e40eaf0 --- /dev/null +++ b/services/gateways-proxies/n8n-tunnel/.env.example @@ -0,0 +1,4 @@ +# Token for the n8n stack's own Cloudflare tunnel. +# Create the tunnel in the Cloudflare dashboard, then paste its token here. +# Single quotes, as required by CLAUDE.md. Never commit this file with a value. +N8N_TUNNEL_TOKEN='' diff --git a/services/gateways-proxies/n8n-tunnel/docker-compose.yml b/services/gateways-proxies/n8n-tunnel/docker-compose.yml new file mode 100644 index 00000000..66044454 --- /dev/null +++ b/services/gateways-proxies/n8n-tunnel/docker-compose.yml @@ -0,0 +1,43 @@ +# Cloudflare Tunnel connector for the n8n stack. +# +# Each stack gets its own tunnel because Cloudflare may send any hostname on a +# tunnel to any connector running that tunnel's token. One tunnel shared across +# stacks would therefore defeat the network separation between them. +services: + n8n-tunnel: + image: cloudflare/cloudflared:latest + container_name: n8n-tunnel + profiles: + - n8n-tunnel + restart: always + command: tunnel --no-autoupdate run + env_file: + - .env + environment: + - TUNNEL_TOKEN=${N8N_TUNNEL_TOKEN} + - TUNNEL_METRICS=127.0.0.1:60123 + deploy: + resources: + reservations: + memory: 64M + limits: + memory: 256M + mem_reservation: 64m + mem_limit: 256m + oom_score_adj: -900 + logging: + driver: json-file + options: + max-size: 10m + max-file: '5' + healthcheck: + test: + - CMD + - cloudflared + - tunnel + - --metrics + - localhost:60123 + - ready + interval: 30s + timeout: 30s + retries: 3 diff --git a/services/gateways-proxies/n8n-tunnel/service.json b/services/gateways-proxies/n8n-tunnel/service.json new file mode 100644 index 00000000..8761e981 --- /dev/null +++ b/services/gateways-proxies/n8n-tunnel/service.json @@ -0,0 +1,9 @@ +{ + "name": "n8n-tunnel", + "display_name": "Cloudflare Tunnel (n8n)", + "category": "Gateways Proxies", + "rank": 5, + "description": "Dedicated Cloudflare Tunnel connector for the n8n stack.", + "source": "https://github.com/cloudflare/cloudflared", + "depends_on": [] +} diff --git a/services/mail-system/mail-ingest/Dockerfile.test b/services/mail-system/mail-ingest/Dockerfile.test new file mode 100644 index 00000000..a90c0ff5 --- /dev/null +++ b/services/mail-system/mail-ingest/Dockerfile.test @@ -0,0 +1,6 @@ +FROM python:3.12-slim +WORKDIR /app +COPY requirements.txt requirements-dev.txt ./ +RUN pip install --no-cache-dir -r requirements-dev.txt +COPY app.py test_app.py ./ +CMD ["python", "-m", "unittest", "-v", "test_app"] diff --git a/services/mail-system/mail-ingest/app.py b/services/mail-system/mail-ingest/app.py index d1a316f0..3273c7db 100644 --- a/services/mail-system/mail-ingest/app.py +++ b/services/mail-system/mail-ingest/app.py @@ -4,9 +4,11 @@ import smtplib import re import json +import ssl import time import uuid import logging +import threading from email.message import EmailMessage from email.policy import SMTP from email.utils import formatdate @@ -24,6 +26,16 @@ MAILSERVER_HOST = os.getenv("MAILSERVER_HOST", "mailserver") MAILSERVER_PORT = int(os.getenv("MAILSERVER_PORT", "25")) MAILSERVER_HELO_DOMAIN = os.getenv("MAILSERVER_HELO_DOMAIN", "mail-ingest.local") +# When set, log in after STARTTLS. The mailserver then accepts the relay on its +# mail-ingest listener without re-checking the original sender's SPF. +MAILSERVER_SMTP_USER = os.getenv("MAILSERVER_SMTP_USER", "").strip() +MAILSERVER_SMTP_PASSWORD = os.getenv("MAILSERVER_SMTP_PASSWORD", "") +# The mailserver's certificate carries its public name, not the container name. +MAILSERVER_TLS_SERVER_NAME = os.getenv("MAILSERVER_TLS_SERVER_NAME", "").strip() or MAILSERVER_HOST +# SHA-256 of the mailserver certificate (DER, hex). When set, the connection is +# trusted only if the presented certificate matches, whatever its CA, name or +# dates. Used while the mailserver certificate is expired; see mailserver/secrets.sh. +MAILSERVER_TLS_FINGERPRINT = os.getenv("MAILSERVER_TLS_FINGERPRINT", "").strip().lower().replace(":", "") SMTP_TIMEOUT = float(os.getenv("SMTP_TIMEOUT_SECONDS", "15")) SMTP_RETRY_ATTEMPTS = int(os.getenv("SMTP_RETRY_ATTEMPTS", "2")) SMTP_RETRY_DELAY = float(os.getenv("SMTP_RETRY_DELAY_SECONDS", "2")) @@ -53,6 +65,108 @@ ) +# Reject a webhook whose Mailgun timestamp is further than this from our clock. +MAX_TIMESTAMP_SKEW_SECONDS = int(os.getenv("MAX_TIMESTAMP_SKEW_SECONDS", "300")) + +# Remember each accepted Mailgun token for this long and refuse to process it twice. +REPLAY_WINDOW_SECONDS = int(os.getenv("REPLAY_WINDOW_SECONDS", "900")) + +# Refuse a request body larger than this before any form parsing happens. +MAX_BODY_BYTES = int(os.getenv("MAX_BODY_BYTES", str(30 * 1024 * 1024))) + +REJECTED_TOTAL = Counter( + "mailgun_rejected_total", + "Mailgun webhook requests rejected before forwarding", + labelnames=["reason"], +) + +# token -> unix time the reservation expires. Only ever holds tokens from +# requests whose signature already verified, so an unauthenticated caller +# cannot fill it or evict anything. +_seen_tokens: dict[str, float] = {} +_seen_tokens_lock = threading.Lock() + + +def _prune_seen_tokens(now: float) -> None: + """Drop expired reservations. Caller holds the lock.""" + + for token in [t for t, expiry in _seen_tokens.items() if expiry <= now]: + del _seen_tokens[token] + + +def reserve_token(token: str, now: float | None = None) -> bool: + """ + Claim a Mailgun token for the replay window. + + Returns True when the token is new and now reserved, False when it is a + replay. Release the reservation with release_token() if the request could + not be processed, so that Mailgun's retry of a genuinely failed delivery is + not mistaken for a replay. + """ + + now = time.time() if now is None else now + with _seen_tokens_lock: + _prune_seen_tokens(now) + if token in _seen_tokens: + return False + _seen_tokens[token] = now + REPLAY_WINDOW_SECONDS + return True + + +def release_token(token: str) -> None: + """Give a token back after a failed delivery so Mailgun may retry it.""" + + with _seen_tokens_lock: + _seen_tokens.pop(token, None) + + +def timestamp_is_fresh(timestamp: str, now: float | None = None) -> bool: + """True when Mailgun's timestamp is within MAX_TIMESTAMP_SKEW_SECONDS of our clock.""" + + now = time.time() if now is None else now + try: + sent_at = float(timestamp) + except (TypeError, ValueError): + return False + return abs(now - sent_at) <= MAX_TIMESTAMP_SKEW_SECONDS + + +async def read_body_within_limit(request: Request) -> bytes: + """ + Read the request body, refusing anything over MAX_BODY_BYTES. + + This runs before request.form() so an oversized upload is dropped without + being parsed. A declared Content-Length is checked first; the streaming + count then covers chunked bodies and a Content-Length that lies. The body + is cached on the request so that request.form() reuses it rather than + trying to read the consumed stream again. + """ + + declared = request.headers.get("content-length") + if declared is not None: + try: + declared_length = int(declared) + except ValueError: + REJECTED_TOTAL.labels(reason="bad_content_length").inc() + raise HTTPException(status_code=400, detail="Invalid Content-Length") + if declared_length > MAX_BODY_BYTES: + REJECTED_TOTAL.labels(reason="body_too_large").inc() + raise HTTPException(status_code=413, detail="Request body too large") + + chunks: list[bytes] = [] + received = 0 + async for chunk in request.stream(): + received += len(chunk) + if received > MAX_BODY_BYTES: + REJECTED_TOTAL.labels(reason="body_too_large").inc() + raise HTTPException(status_code=413, detail="Request body too large") + chunks.append(chunk) + + body = b"".join(chunks) + request._body = body + return body + + def verify_mailgun_signature(api_key: str, timestamp: str, token: str, signature: str) -> bool: """ Mailgun: HMAC-SHA256(api_key, timestamp + token) == signature @@ -113,6 +227,26 @@ def build_fallback_message( return message.as_bytes() +def login(smtp: smtplib.SMTP) -> None: + """STARTTLS, verifying the mailserver's certificate, then SASL login.""" + + if MAILSERVER_TLS_FINGERPRINT: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + smtp.starttls(context=context) + presented = hashlib.sha256(smtp.sock.getpeercert(binary_form=True) or b"").hexdigest() + if not hmac.compare_digest(presented, MAILSERVER_TLS_FINGERPRINT): + raise ssl.SSLError(f"mailserver certificate does not match the pinned fingerprint ({presented})") + else: + # smtplib verifies the certificate against the host it connected to, which + # is the container name. Check it against the mailserver's public name. + smtp._host = MAILSERVER_TLS_SERVER_NAME + smtp.starttls(context=ssl.create_default_context()) + smtp.ehlo(MAILSERVER_HELO_DOMAIN) + smtp.login(MAILSERVER_SMTP_USER, MAILSERVER_SMTP_PASSWORD) + + def smtp_forward(envelope_from: str, envelope_to: str, raw_mime: bytes) -> None: """ Blocking SMTP send into docker-mailserver. Retries a few times to avoid transient drops. @@ -127,12 +261,22 @@ def smtp_forward(envelope_from: str, envelope_to: str, raw_mime: bytes) -> None: try: with smtplib.SMTP(MAILSERVER_HOST, MAILSERVER_PORT, timeout=SMTP_TIMEOUT) as smtp: smtp.ehlo(MAILSERVER_HELO_DOMAIN) - # internal, no TLS/auth needed + if MAILSERVER_SMTP_USER: + login(smtp) smtp.mail(envelope_from) + refused = {} for rcpt in recipients: - smtp.rcpt(rcpt) + code, resp = smtp.rcpt(rcpt) + if code not in (250, 251): + refused[rcpt] = (code, resp) + if len(refused) == len(recipients): + raise smtplib.SMTPRecipientsRefused(refused) + if refused: + logger.error("smtp_recipients_partly_refused %s", {"refused": repr(refused)}) smtp.data(raw_mime) return + except smtplib.SMTPRecipientsRefused: + raise except Exception as exc: # noqa: PERF203 - we want to surface all SMTP/network issues last_exc = exc logger.warning( @@ -173,6 +317,9 @@ async def mailgun_incoming(request: Request): - sender, recipient - body-mime (if using Store and Notify) OR body-plain/body-html as fallback """ + # Enforce the size cap before the form is parsed. + await read_body_within_limit(request) + form = await request.form() client_ip = request.client.host if request.client else "unknown" @@ -184,8 +331,24 @@ async def mailgun_incoming(request: Request): if not verify_mailgun_signature(MAILGUN_WEBHOOK_SIGNING_KEY, timestamp, token, signature): logger.warning("invalid_mailgun_signature %s", {"client_ip": client_ip}) + REJECTED_TOTAL.labels(reason="invalid_signature").inc() raise HTTPException(status_code=403, detail="Invalid Mailgun signature") + # Only signature-verified requests get past this point, so the freshness + # and replay checks below cannot be driven by an anonymous caller. + if not timestamp_is_fresh(str(timestamp)): + logger.warning( + "stale_mailgun_timestamp %s", + {"client_ip": client_ip, "timestamp": timestamp}, + ) + REJECTED_TOTAL.labels(reason="stale_timestamp").inc() + raise HTTPException(status_code=403, detail="Stale Mailgun timestamp") + + if not reserve_token(str(token)): + logger.warning("replayed_mailgun_token %s", {"client_ip": client_ip}) + REJECTED_TOTAL.labels(reason="replayed_token").inc() + raise HTTPException(status_code=409, detail="Duplicate Mailgun token") + sender = form.get("sender") or form.get("from") or "unknown@localhost" from_header = form.get("from") or sender recipient = form.get("recipient") or form.get("to") or "unknown@localhost" @@ -232,6 +395,7 @@ async def mailgun_incoming(request: Request): {"duration_seconds": round(duration, 3), "client_ip": client_ip}, ) except smtplib.SMTPRecipientsRefused as e: + release_token(str(token)) SMTP_ERRORS.labels(reason="recipient_refused").inc() REQUESTS_TOTAL.labels(result="invalid_recipient").inc() logger.warning( @@ -240,6 +404,7 @@ async def mailgun_incoming(request: Request): ) raise HTTPException(status_code=422, detail="No valid recipients") except smtplib.SMTPDataError as e: + release_token(str(token)) SMTP_ERRORS.labels(reason="smtp_data_error").inc() duration = time.perf_counter() - start SMTP_FORWARD_DURATION.observe(duration) @@ -257,6 +422,7 @@ async def mailgun_incoming(request: Request): raise HTTPException(status_code=422, detail=f"SMTP {code}: {message}") raise HTTPException(status_code=502, detail="Upstream SMTP rejected message") except ValueError as e: + release_token(str(token)) SMTP_ERRORS.labels(reason="parse_error").inc() REQUESTS_TOTAL.labels(result="parse_error").inc() logger.warning( @@ -265,6 +431,7 @@ async def mailgun_incoming(request: Request): ) raise HTTPException(status_code=400, detail=str(e)) except Exception as e: + release_token(str(token)) SMTP_ERRORS.labels(reason=type(e).__name__).inc() duration = time.perf_counter() - start SMTP_FORWARD_DURATION.observe(duration) diff --git a/services/mail-system/mail-ingest/docker-compose.yml b/services/mail-system/mail-ingest/docker-compose.yml index 93696053..d920ac44 100644 --- a/services/mail-system/mail-ingest/docker-compose.yml +++ b/services/mail-system/mail-ingest/docker-compose.yml @@ -10,7 +10,12 @@ services: environment: - MAILGUN_WEBHOOK_SIGNING_KEY=${MAILGUN_WEBHOOK_SIGNING_KEY} - MAILSERVER_HOST=mailserver - - MAILSERVER_PORT=25 + # Authenticated, local-delivery-only listener; see mailserver/config/user-patches.sh + - MAILSERVER_PORT=10587 + - MAILSERVER_SMTP_USER=mail-ingest@${BASE_DOMAIN} + - MAILSERVER_SMTP_PASSWORD=${MAIL_INGEST_SMTP_PASSWORD:?MAIL_INGEST_SMTP_PASSWORD is unset; run corekit up mailserver first} + - MAILSERVER_TLS_SERVER_NAME=${MAILSERVER_WEBMAIL_HOSTNAME} + - MAILSERVER_TLS_FINGERPRINT=${MAIL_INGEST_TLS_FINGERPRINT:-} - MAILSERVER_HELO_DOMAIN=mail-ingest.${BASE_DOMAIN} expose: - '3000' diff --git a/services/mail-system/mail-ingest/requirements-dev.txt b/services/mail-system/mail-ingest/requirements-dev.txt new file mode 100644 index 00000000..a69063ff --- /dev/null +++ b/services/mail-system/mail-ingest/requirements-dev.txt @@ -0,0 +1,3 @@ +# Test-only dependencies. Not installed into the runtime image. +-r requirements.txt +httpx diff --git a/services/mail-system/mail-ingest/test_app.py b/services/mail-system/mail-ingest/test_app.py index d9b05699..ab8ac607 100644 --- a/services/mail-system/mail-ingest/test_app.py +++ b/services/mail-system/mail-ingest/test_app.py @@ -1,9 +1,48 @@ +import hashlib +import hmac import json +import smtplib +import ssl +import time import unittest +from unittest import mock from email import policy from email.parser import BytesParser -from app import build_fallback_message, mailgun_message_header +from fastapi.testclient import TestClient + +import app as app_module +from app import ( + MAX_BODY_BYTES, + app, + build_fallback_message, + mailgun_message_header, + release_token, + reserve_token, + timestamp_is_fresh, +) + +SIGNING_KEY = "test-signing-key" + + +def signed_form(timestamp=None, token="token-0001"): + """Build the signed fields Mailgun sends, using the key the tests install.""" + + timestamp = str(int(time.time())) if timestamp is None else str(timestamp) + signature = hmac.new( + key=SIGNING_KEY.encode("utf-8"), + msg=f"{timestamp}{token}".encode("utf-8"), + digestmod=hashlib.sha256, + ).hexdigest() + return { + "timestamp": timestamp, + "token": token, + "signature": signature, + "sender": "visitor@gmail.com", + "recipient": "info@example.com", + "subject": "hello", + "body-plain": "hello", + } class MailgunHeaderTests(unittest.TestCase): @@ -43,5 +82,278 @@ def test_fallback_message_preserves_reply_to_and_display_from(self): self.assertEqual(message.get_content().strip(), "Please reply to the visitor.") +class RejectionTests(unittest.TestCase): + """The three rejections added on 2026-09-26: stale timestamp, replayed token, oversized body.""" + + def setUp(self): + self.client = TestClient(app) + self._old_key = app_module.MAILGUN_WEBHOOK_SIGNING_KEY + app_module.MAILGUN_WEBHOOK_SIGNING_KEY = SIGNING_KEY + app_module._seen_tokens.clear() + # Fail loudly rather than reaching the real mailserver from a test. + self._forwarded = [] + self._old_forward = app_module.smtp_forward + app_module.smtp_forward = lambda *a, **kw: self._forwarded.append(a) + + def tearDown(self): + app_module.MAILGUN_WEBHOOK_SIGNING_KEY = self._old_key + app_module.smtp_forward = self._old_forward + app_module._seen_tokens.clear() + + def test_a_correctly_signed_fresh_request_is_forwarded(self): + response = self.client.post("/mailgun/incoming", data=signed_form()) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(self._forwarded), 1) + + def test_timestamp_older_than_five_minutes_is_rejected(self): + stale = int(time.time()) - 301 + response = self.client.post("/mailgun/incoming", data=signed_form(timestamp=stale)) + + self.assertEqual(response.status_code, 403) + self.assertIn("Stale", response.json()["detail"]) + self.assertEqual(self._forwarded, []) + + def test_timestamp_more_than_five_minutes_in_the_future_is_rejected(self): + ahead = int(time.time()) + 301 + response = self.client.post("/mailgun/incoming", data=signed_form(timestamp=ahead)) + + self.assertEqual(response.status_code, 403) + self.assertEqual(self._forwarded, []) + + def test_timestamp_just_inside_the_window_is_accepted(self): + response = self.client.post( + "/mailgun/incoming", data=signed_form(timestamp=int(time.time()) - 299) + ) + + self.assertEqual(response.status_code, 200) + + def test_token_replayed_within_the_window_is_rejected(self): + form = signed_form(token="replay-me") + + first = self.client.post("/mailgun/incoming", data=form) + second = self.client.post("/mailgun/incoming", data=form) + + self.assertEqual(first.status_code, 200) + self.assertEqual(second.status_code, 409) + self.assertIn("Duplicate", second.json()["detail"]) + self.assertEqual(len(self._forwarded), 1) + + def test_token_is_released_so_mailgun_may_retry_a_failed_delivery(self): + def boom(*a, **kw): + raise RuntimeError("mailserver unreachable") + + app_module.smtp_forward = boom + form = signed_form(token="retry-me") + + failed = self.client.post("/mailgun/incoming", data=form) + self.assertEqual(failed.status_code, 500) + + app_module.smtp_forward = lambda *a, **kw: self._forwarded.append(a) + retried = self.client.post("/mailgun/incoming", data=form) + + self.assertEqual(retried.status_code, 200) + self.assertEqual(len(self._forwarded), 1) + + def test_body_over_thirty_megabytes_is_rejected_before_parsing(self): + oversized = b"x" * (MAX_BODY_BYTES + 1) + + response = self.client.post( + "/mailgun/incoming", + content=oversized, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + + self.assertEqual(response.status_code, 413) + self.assertIn("too large", response.json()["detail"]) + self.assertEqual(self._forwarded, []) + + def test_oversized_body_is_rejected_on_a_lying_content_length(self): + """A chunked body with no Content-Length is still counted as it streams.""" + + def chunks(): + sent = 0 + while sent <= MAX_BODY_BYTES: + block = b"x" * (1024 * 1024) + sent += len(block) + yield block + + response = self.client.post( + "/mailgun/incoming", + content=chunks(), + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + + self.assertEqual(response.status_code, 413) + self.assertEqual(self._forwarded, []) + + def test_a_body_under_the_limit_still_parses(self): + form = signed_form() + form["body-plain"] = "y" * 1024 + + response = self.client.post("/mailgun/incoming", data=form) + + self.assertEqual(response.status_code, 200) + + def test_an_unsigned_request_is_rejected_before_the_replay_cache_is_touched(self): + form = signed_form(token="never-reserve-me") + form["signature"] = "0" * 64 + + response = self.client.post("/mailgun/incoming", data=form) + + self.assertEqual(response.status_code, 403) + self.assertNotIn("never-reserve-me", app_module._seen_tokens) + + +class ReplayCacheUnitTests(unittest.TestCase): + def setUp(self): + app_module._seen_tokens.clear() + + def tearDown(self): + app_module._seen_tokens.clear() + + def test_reserve_then_replay(self): + self.assertTrue(reserve_token("t1")) + self.assertFalse(reserve_token("t1")) + + def test_reservation_expires_after_the_window(self): + now = time.time() + self.assertTrue(reserve_token("t2", now=now)) + self.assertFalse(reserve_token("t2", now=now + 899)) + self.assertTrue(reserve_token("t2", now=now + 901)) + + def test_release_allows_immediate_retry(self): + self.assertTrue(reserve_token("t3")) + release_token("t3") + self.assertTrue(reserve_token("t3")) + + +class TimestampFreshnessUnitTests(unittest.TestCase): + def test_boundaries(self): + now = 1_000_000.0 + self.assertTrue(timestamp_is_fresh(str(int(now)), now=now)) + self.assertTrue(timestamp_is_fresh(str(int(now - 300)), now=now)) + self.assertFalse(timestamp_is_fresh(str(int(now - 301)), now=now)) + self.assertTrue(timestamp_is_fresh(str(int(now + 300)), now=now)) + self.assertFalse(timestamp_is_fresh(str(int(now + 301)), now=now)) + + def test_non_numeric_timestamp_is_not_fresh(self): + self.assertFalse(timestamp_is_fresh("not-a-number")) + self.assertFalse(timestamp_is_fresh("")) + + if __name__ == "__main__": unittest.main() + + +class FakeSMTP: + """Records what smtp_forward does to the mailserver connection.""" + + instances: list["FakeSMTP"] = [] + rcpt_codes: dict[str, int] = {} + starttls_error: Exception | None = None + peer_cert = b"mailserver-cert-der" + + def __init__(self, host, port, timeout=None): + self._host = host + self.calls: list[tuple] = [] + FakeSMTP.instances.append(self) + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def ehlo(self, name): + self.calls.append(("ehlo", name)) + + def starttls(self, context=None): + if FakeSMTP.starttls_error: + raise FakeSMTP.starttls_error + self.calls.append(("starttls", self._host, context.check_hostname)) + self.sock = mock.Mock(getpeercert=lambda binary_form=False: FakeSMTP.peer_cert) + + def login(self, user, password): + self.calls.append(("login", user)) + + def mail(self, sender): + self.calls.append(("mail", sender)) + + def rcpt(self, rcpt): + code = FakeSMTP.rcpt_codes.get(rcpt, 250) + self.calls.append(("rcpt", rcpt)) + return code, b"5.7.1 refused" if code >= 400 else b"2.1.5 Ok" + + def data(self, raw): + self.calls.append(("data",)) + + +class SmtpForwardTests(unittest.TestCase): + def setUp(self): + FakeSMTP.instances = [] + FakeSMTP.rcpt_codes = {} + FakeSMTP.starttls_error = None + patches = [ + mock.patch.object(app_module.smtplib, "SMTP", FakeSMTP), + mock.patch.object(app_module, "SMTP_RETRY_ATTEMPTS", 1), + mock.patch.object(app_module, "MAILSERVER_SMTP_USER", "mail-ingest@example.com"), + mock.patch.object(app_module, "MAILSERVER_SMTP_PASSWORD", "pw"), + mock.patch.object(app_module, "MAILSERVER_TLS_SERVER_NAME", "mail.example.com"), + ] + for p in patches: + p.start() + self.addCleanup(p.stop) + + def calls(self): + return [c[0] for c in FakeSMTP.instances[-1].calls] + + def test_logs_in_after_starttls_verified_against_the_public_name(self): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertEqual( + self.calls(), ["ehlo", "starttls", "ehlo", "login", "mail", "rcpt", "data"] + ) + self.assertEqual( + FakeSMTP.instances[-1].calls[1], ("starttls", "mail.example.com", True) + ) + + def test_never_logs_in_when_starttls_is_unavailable(self): + FakeSMTP.starttls_error = smtplib.SMTPNotSupportedError("no STARTTLS") + with self.assertRaises(smtplib.SMTPNotSupportedError): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertNotIn("login", self.calls()) + + def test_without_a_login_configured_it_relays_unauthenticated(self): + with mock.patch.object(app_module, "MAILSERVER_SMTP_USER", ""): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertEqual(self.calls(), ["ehlo", "mail", "rcpt", "data"]) + + def test_refused_recipient_raises_instead_of_reaching_data(self): + FakeSMTP.rcpt_codes = {"tom@example.com": 550} + with self.assertRaises(smtplib.SMTPRecipientsRefused): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertNotIn("data", self.calls()) + + def test_partly_refused_recipients_still_deliver_to_the_rest(self): + FakeSMTP.rcpt_codes = {"bad@example.com": 550} + with self.assertLogs("mail-ingest", level="ERROR"): + app_module.smtp_forward("a@hotmail.com", "tom@example.com, bad@example.com", b"raw") + self.assertIn("data", self.calls()) + + def test_refused_recipient_is_reported_to_mailgun_as_422(self): + FakeSMTP.rcpt_codes = {"info@example.com": 550} + with mock.patch.object(app_module, "MAILGUN_WEBHOOK_SIGNING_KEY", SIGNING_KEY): + response = TestClient(app).post("/mailgun/incoming", data=signed_form(token="token-rcpt")) + self.assertEqual(response.status_code, 422) + + def test_pinned_certificate_that_matches_is_trusted(self): + pin = hashlib.sha256(FakeSMTP.peer_cert).hexdigest() + with mock.patch.object(app_module, "MAILSERVER_TLS_FINGERPRINT", pin): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertIn("login", self.calls()) + + def test_pinned_certificate_that_differs_is_refused_before_login(self): + with mock.patch.object(app_module, "MAILSERVER_TLS_FINGERPRINT", "00" * 32): + with self.assertRaises(ssl.SSLError): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertNotIn("login", self.calls()) diff --git a/services/mail-system/mailserver/.env.example b/services/mail-system/mailserver/.env.example index 27ddc906..e47bdd63 100644 --- a/services/mail-system/mailserver/.env.example +++ b/services/mail-system/mailserver/.env.example @@ -16,3 +16,7 @@ MAIL_INFO_PASSWORD='' # BASE_DOMAIN='' # Set in .env.global # MAILSERVER_WEBMAIL_HOSTNAME='' # Set in .env.global # PROJECT_ROOT='' # Set in .env.global +# Login mail-ingest uses to relay inbound mail (auto-generated) +MAIL_INGEST_SMTP_PASSWORD='' +# Pinned mailserver certificate for mail-ingest while it is expired (managed by secrets.sh) +MAIL_INGEST_TLS_FINGERPRINT='' diff --git a/services/mail-system/mailserver/README.md b/services/mail-system/mailserver/README.md index 29190bec..3b23c68d 100644 --- a/services/mail-system/mailserver/README.md +++ b/services/mail-system/mailserver/README.md @@ -85,6 +85,29 @@ docker exec mailserver setup email list - Authentication: noreply@yourdomain.com - Password: auto-generated (see `.env`) +### Inbound Mail via Mailgun (mail-ingest) + +MX points at Mailgun. Mailgun checks SPF and DKIM at the edge, then posts each +message to `mail-ingest`, which relays it to this container over SMTP. + +That relay does **not** use port 25. On port 25 Postfix re-checks the original +sender's SPF against mail-ingest's Docker address, which fails for every sender +that publishes `-all`. Instead, `config/user-patches.sh` adds a listener on port +`10587` (not published on the host) that: + +- requires STARTTLS and a login as `mail-ingest@`, and nobody else +- accepts local recipients only, so it cannot relay even when authenticated +- skips the SPF policy check +- hides the login from Rspamd, OpenDKIM and OpenDMARC, so the message is still + scanned and DMARC-checked as inbound mail + +`secrets.sh` generates `MAIL_INGEST_SMTP_PASSWORD`, and `startup.sh` creates the +account. The account is barred from submission (587/465), so its credential +cannot send mail out. + +Do not set `PERMIT_DOCKER` to `network` or `connected-networks` to make inbound +work: that lets any container on the network relay mail out through Mailgun. + ### n8n Integration Setup **Create SMTP Credentials in n8n:** diff --git a/services/mail-system/mailserver/config/user-patches.sh b/services/mail-system/mailserver/config/user-patches.sh new file mode 100755 index 00000000..cc0b4fe6 --- /dev/null +++ b/services/mail-system/mailserver/config/user-patches.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# user-patches.sh - Run by docker-mailserver inside the container at startup, +# after its own configuration and before Postfix starts. +# prepare.sh copies this file into data/dms/config/. A container restart runs +# it again over the already-patched files, so every step must be idempotent. + +set -euo pipefail + +# ── mail-ingest listener ── +# mail-ingest relays mail that Mailgun has already accepted, after Mailgun's +# own SPF/DKIM checks at the edge. On port 25 Postfix re-checks the original +# sender's SPF against mail-ingest's Docker address, which fails for every +# sender publishing -all. +# +# This listener is for mail-ingest alone: +# - requires STARTTLS and a SASL login as MAIL_INGEST_SMTP_USER, nobody else +# - accepts local recipients only, so it cannot relay even when authenticated +# - skips the SPF policy service +# - withholds the login from milters, so Rspamd, OpenDKIM and OpenDMARC +# treat the message as inbound rather than as mail sent by a local user +# The port is not published on the host. +# MAIL_INGEST_SMTP_USER is also barred from submission (587/465), so its +# credential cannot be used to send mail out. + +INGEST_PORT="${MAIL_INGEST_SMTP_PORT:-10587}" +INGEST_USER="${MAIL_INGEST_SMTP_USER:-}" + +if [[ -z "$INGEST_USER" ]]; then + echo "user-patches: MAIL_INGEST_SMTP_USER is unset; mail-ingest listener not configured" + exit 0 +fi + +printf '%s OK\n' "$INGEST_USER" > /etc/postfix/mail-ingest-login +printf '%s REJECT this account only delivers inbound mail\n' "$INGEST_USER" > /etc/postfix/mail-ingest-deny + +# Keep the mailbox quota check that port 25 applies, when quotas are enabled. +recipient_restrictions="reject_unauth_destination" +if postconf -h smtpd_recipient_restrictions | grep -q 'inet:localhost:65265'; then + recipient_restrictions+=", check_policy_service inet:localhost:65265" +fi + +postconf -e \ + "mail_ingest_client_restrictions = check_sasl_access texthash:/etc/postfix/mail-ingest-login, reject" \ + "mail_ingest_relay_restrictions = reject_unauth_destination, permit" \ + "mail_ingest_recipient_restrictions = ${recipient_restrictions}" \ + "mail_ingest_milter_mail_macros = i {mail_addr} {mail_host} {mail_mailer}" + +postconf -M "${INGEST_PORT}/inet=${INGEST_PORT} inet n - n - - smtpd" +postconf -P \ + "${INGEST_PORT}/inet/syslog_name=postfix/mail-ingest" \ + "${INGEST_PORT}/inet/smtpd_tls_security_level=encrypt" \ + "${INGEST_PORT}/inet/smtpd_tls_auth_only=yes" \ + "${INGEST_PORT}/inet/smtpd_sasl_auth_enable=yes" \ + "${INGEST_PORT}/inet/smtpd_sasl_type=dovecot" \ + "${INGEST_PORT}/inet/smtpd_sasl_authenticated_header=no" \ + "${INGEST_PORT}/inet/smtpd_client_restrictions=\$mail_ingest_client_restrictions" \ + "${INGEST_PORT}/inet/smtpd_relay_restrictions=\$mail_ingest_relay_restrictions" \ + "${INGEST_PORT}/inet/smtpd_recipient_restrictions=\$mail_ingest_recipient_restrictions" \ + "${INGEST_PORT}/inet/milter_mail_macros=\$mail_ingest_milter_mail_macros" + +mua_sender="$(postconf -h mua_sender_restrictions)" +if [[ "$mua_sender" != *mail-ingest-deny* ]]; then + postconf -e "mua_sender_restrictions = check_sasl_access texthash:/etc/postfix/mail-ingest-deny, ${mua_sender}" +fi + +echo "user-patches: mail-ingest listener on ${INGEST_PORT} for ${INGEST_USER}" diff --git a/services/mail-system/mailserver/docker-compose.yml b/services/mail-system/mailserver/docker-compose.yml index f41b63bf..a171dcbc 100644 --- a/services/mail-system/mailserver/docker-compose.yml +++ b/services/mail-system/mailserver/docker-compose.yml @@ -24,7 +24,7 @@ services: environment: - LOG_LEVEL=info - SUPERVISOR_LOGLEVEL=warn - - PERMIT_DOCKER=network + - PERMIT_DOCKER=none - NETWORK_INTERFACE=eth0 - SSL_TYPE=manual - SSL_CERT_PATH=/certs/caddy/certificates/acme-v02.api.letsencrypt.org-directory/${MAILSERVER_WEBMAIL_HOSTNAME}/${MAILSERVER_WEBMAIL_HOSTNAME}.crt @@ -60,6 +60,7 @@ services: - RELAY_PASSWORD=${MAILGUN_SMTP_PASSWORD} - RELAY_TLS_LEVEL=encrypt - ACCOUNT_PROVISIONER=FILE + - MAIL_INGEST_SMTP_USER=mail-ingest@${BASE_DOMAIN} - ENABLE_UPDATE_CHECK=0 cap_add: - NET_ADMIN diff --git a/services/mail-system/mailserver/prepare.sh b/services/mail-system/mailserver/prepare.sh index a772bdc1..d10cf903 100755 --- a/services/mail-system/mailserver/prepare.sh +++ b/services/mail-system/mailserver/prepare.sh @@ -20,3 +20,12 @@ F2B_DST="${SCRIPT_DIR}/data/dms/config/fail2ban-jail.cf" if [ -f "$F2B_SRC" ]; then cp -v "$F2B_SRC" "$F2B_DST" fi + +# ── User patches (mail-ingest listener) ── +PATCHES_SRC="${SCRIPT_DIR}/config/user-patches.sh" +PATCHES_DST="${SCRIPT_DIR}/data/dms/config/user-patches.sh" + +if [ -f "$PATCHES_SRC" ]; then + cp -v "$PATCHES_SRC" "$PATCHES_DST" + chmod 755 "$PATCHES_DST" +fi diff --git a/services/mail-system/mailserver/secrets.sh b/services/mail-system/mailserver/secrets.sh index d775dbe7..4119a23f 100644 --- a/services/mail-system/mailserver/secrets.sh +++ b/services/mail-system/mailserver/secrets.sh @@ -14,6 +14,7 @@ declare -A SECRETS=( ["SMTP_PASS"]="password:16" ["MAIL_NOREPLY_PASSWORD"]="password:32" ["MAIL_INFO_PASSWORD"]="password:32" + ["MAIL_INGEST_SMTP_PASSWORD"]="password:32" ) ############################################ @@ -21,3 +22,24 @@ declare -A SECRETS=( # Generate secrets generate_secrets SECRETS + +# generate_secrets only fills the in-memory map; nothing writes it back to .env. +# The mail-ingest login must persist (mail-ingest's compose reads it), so write +# it here if .env does not have it yet. +if ! grep -q "^MAIL_INGEST_SMTP_PASSWORD=." "$SCRIPT_DIR/.env" 2>/dev/null; then + update_env_var "$SCRIPT_DIR/.env" "MAIL_INGEST_SMTP_PASSWORD" "${SERVICE_ENV_VARS[MAIL_INGEST_SMTP_PASSWORD]}" +fi + +# mail-ingest verifies the mailserver certificate before it logs in. Nothing +# renews that certificate since Caddy left the stack, so while it is expired +# pin its fingerprint instead; once it is valid again, clear the pin so +# mail-ingest goes back to normal CA and hostname checks. Leave the value +# alone if the mailserver is not running to ask. +if docker exec mailserver true 2>/dev/null; then + cert_fp="" + if ! docker exec mailserver sh -c 'openssl x509 -in "$SSL_CERT_PATH" -noout -checkend 0' >/dev/null 2>&1; then + cert_fp="$(docker exec mailserver sh -c 'openssl x509 -in "$SSL_CERT_PATH" -outform DER | sha256sum' | cut -d' ' -f1)" + log_warning "Mailserver certificate has expired; pinning it for mail-ingest" + fi + update_env_var "$SCRIPT_DIR/.env" "MAIL_INGEST_TLS_FINGERPRINT" "$cert_fp" +fi diff --git a/services/mail-system/mailserver/service.json b/services/mail-system/mailserver/service.json index a9c22ebb..f2ddc8a0 100644 --- a/services/mail-system/mailserver/service.json +++ b/services/mail-system/mailserver/service.json @@ -6,7 +6,5 @@ "description": "Full-stack mail server (Docker-Mailserver) with Mailgun ingest support for sending and receiving real emails.", "interface": "mail..com", "source": "https://github.com/docker-mailserver/docker-mailserver", - "depends_on": [ - "caddy" - ] + "depends_on": [] } diff --git a/services/mail-system/mailserver/startup.sh b/services/mail-system/mailserver/startup.sh index f2879bfc..e2c58a18 100755 --- a/services/mail-system/mailserver/startup.sh +++ b/services/mail-system/mailserver/startup.sh @@ -35,5 +35,8 @@ ensure_account() { ensure_account "noreply@${BASE_DOMAIN}" "${MAIL_NOREPLY_PASSWORD}" ensure_account "info@${BASE_DOMAIN}" "${MAIL_INFO_PASSWORD}" +# mail-ingest logs in on its own listener to deliver Mailgun's inbound mail +# (see config/user-patches.sh). It is barred from submission. +ensure_account "mail-ingest@${BASE_DOMAIN}" "${MAIL_INGEST_SMTP_PASSWORD}" log_success "Mail accounts provisioned" diff --git a/services/workflow-automation/n8n-prod/.env.example b/services/workflow-automation/n8n-prod/.env.example new file mode 100644 index 00000000..c2ed6621 --- /dev/null +++ b/services/workflow-automation/n8n-prod/.env.example @@ -0,0 +1,32 @@ +# Production n8n. Single quotes throughout, as CLAUDE.md requires. +# None of these values are generated by this staging work. The owner creates +# them when applying the design. + +N8N_PROD_HOSTNAME='n8n-prod..com' + +# openssl rand -hex 32 -- and store a copy in the password manager, +# separately from any database backup. +N8N_PROD_ENCRYPTION_KEY='' + +# openssl rand -hex 32 +N8N_PROD_RUNNER_AUTH_TOKEN='' + +# openssl rand -hex 24 -- this database user exists only on the n8n-prod +# Postgres container and has no rights on any other database. +N8N_PROD_DB_NAME='n8n' +N8N_PROD_DB_USER='n8nprod' +N8N_PROD_DB_PASSWORD='' + +# Images. This service builds nothing: it adopts an image already built and +# verified for another instance. Required by managed/version.sh, which every +# managed-update subcommand calls. +N8N_PROD_IMAGE='' +N8N_PROD_RUNNER_IMAGE='' + +# Promotion. Required by managed/promote.sh. The token needs READ access only: +# the control is the branch protection on the repository, not this token's +# secrecy. Repository may be owner/repo, a full URL, or a local path. +PROMOTION_REPOSITORY='' +PROMOTION_TOKEN='' +PROMOTION_BRANCH='main' +PROMOTION_PATH='workflows' diff --git a/services/workflow-automation/n8n-prod/cli.sh b/services/workflow-automation/n8n-prod/cli.sh new file mode 100755 index 00000000..382db673 --- /dev/null +++ b/services/workflow-automation/n8n-prod/cli.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Service CLI for the production n8n instance. +# +# corekit run n8n-prod promote [options] +# +# Promotion is n8n-specific -- it drives n8n-git and the n8n CLI inside the +# container -- so both the command and its implementation live with the service +# rather than in lib/. See docs/PROMOTION.md for the design and configuration. +set -Eeuo pipefail + +SERVICE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SERVICE_NAME="$(basename "$SERVICE_DIR")" +PROMOTE_LIB="$SERVICE_DIR/managed/promote.sh" + +usage() { + cat < [options] + +Commands: + promote [options] Import reviewed workflow definitions from a commit that is + already merged into the protected branch. + status Show the last recorded promotion. + rollback-plan Print the recovery procedure. + +promote options: + --commit Promote this exact commit. Full 40-character SHA. + Defaults to the current tip of the protected branch. + --plan Show what would be promoted and exit. Changes nothing. + --dry-run Run the import in the tool's dry-run mode. + +Configuration: policy in service.json under "promotion"; repository and token in +this service's .env as PROMOTION_REPOSITORY and PROMOTION_TOKEN. The token needs +read access only. See docs/PROMOTION.md. +USAGE +} + +require_lib() { + [[ -f "$PROMOTE_LIB" ]] || { echo "Promotion library not found at $PROMOTE_LIB" >&2; exit 1; } +} + +case "${1:-}" in + promote) + shift + require_lib + exec bash "$PROMOTE_LIB" "$SERVICE_NAME" "$@" + ;; + status) + shift + require_lib + exec bash "$PROMOTE_LIB" "$SERVICE_NAME" --status "$@" + ;; + rollback-plan) + shift + require_lib + exec bash "$PROMOTE_LIB" "$SERVICE_NAME" --rollback-plan "$@" + ;; + ""|-h|--help|help) + usage + ;; + *) + echo "Unknown command: $1" >&2 + echo >&2 + usage >&2 + exit 1 + ;; +esac diff --git a/services/workflow-automation/n8n-prod/docker-compose.yml b/services/workflow-automation/n8n-prod/docker-compose.yml new file mode 100644 index 00000000..556d16e4 --- /dev/null +++ b/services/workflow-automation/n8n-prod/docker-compose.yml @@ -0,0 +1,151 @@ +# Production n8n. +# +# A second n8n intended to hold production workflows and the credentials they +# use, separate from an instance where workflows are authored freely. It is +# deliberately narrower than an authoring instance: regular +# execution mode, no community packages, dangerous nodes excluded, no public +# API, no published ports, and its own Postgres that no other stack can reach. +# +# Do not add this stack to a shared network. The only way in is the +# n8n-prod-tunnel connector in this same stack. +services: + n8n-prod-postgres: + image: postgres:17 + container_name: n8n-prod-postgres + profiles: + - n8n-prod + restart: unless-stopped + environment: + POSTGRES_DB: ${N8N_PROD_DB_NAME:-n8n} + POSTGRES_USER: ${N8N_PROD_DB_USER:-n8nprod} + POSTGRES_PASSWORD: ${N8N_PROD_DB_PASSWORD} + # Keep the buffer pool small: this server holds one modest database. + POSTGRES_INITDB_ARGS: "--data-checksums" + command: + - postgres + - -c + - shared_buffers=64MB + - -c + - max_connections=30 + volumes: + - ./data/db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${N8N_PROD_DB_USER:-n8nprod} -d ${N8N_PROD_DB_NAME:-n8n}"] + interval: 10s + timeout: 5s + retries: 10 + mem_limit: 512m + logging: + driver: json-file + options: + max-size: 10m + max-file: '3' + + n8n-prod: + # Built by the managed updater as corekit/n8n:-git. + # Re-pin this after every `corekit managed-update apply n8n`, or set + # N8N_PROD_IMAGE in .env to override it. + image: ${N8N_PROD_IMAGE:-corekit/n8n:2.39.6-git6c84c643d221} + container_name: n8n-prod + profiles: + - n8n-prod + restart: unless-stopped + depends_on: + n8n-prod-postgres: + condition: service_healthy + environment: + DB_TYPE: postgresdb + DB_POSTGRESDB_HOST: n8n-prod-postgres + DB_POSTGRESDB_PORT: 5432 + DB_POSTGRESDB_DATABASE: ${N8N_PROD_DB_NAME:-n8n} + DB_POSTGRESDB_USER: ${N8N_PROD_DB_USER:-n8nprod} + DB_POSTGRESDB_PASSWORD: ${N8N_PROD_DB_PASSWORD} + + # Its own key. Generated by the owner at apply time, never reused from + # any other instance, and kept in the password manager as well as here. + N8N_ENCRYPTION_KEY: ${N8N_PROD_ENCRYPTION_KEY} + + N8N_HOST: ${N8N_PROD_HOSTNAME:-n8n-prod..com} + N8N_PROTOCOL: https + N8N_PORT: 5678 + WEBHOOK_URL: https://${N8N_PROD_HOSTNAME:-n8n-prod..com}/ + N8N_EDITOR_BASE_URL: https://${N8N_PROD_HOSTNAME:-n8n-prod..com}/ + GENERIC_TIMEZONE: ${GENERIC_TIMEZONE:-Europe/London} + + # Regular execution mode. No queue, no workers, no Redis. + EXECUTIONS_MODE: regular + + # Task runner, external, same version as n8n itself. + N8N_RUNNERS_ENABLED: "true" + N8N_RUNNERS_MODE: external + N8N_RUNNERS_AUTH_TOKEN: ${N8N_PROD_RUNNER_AUTH_TOKEN} + N8N_RUNNERS_BROKER_LISTEN_ADDRESS: 0.0.0.0 + N8N_NATIVE_PYTHON_RUNNER: "true" + + # No community packages of any kind, verified or not, and none usable as + # an AI Agent tool. + N8N_COMMUNITY_PACKAGES_ENABLED: "false" + N8N_UNVERIFIED_PACKAGES_ENABLED: "false" + N8N_COMMUNITY_PACKAGES_ALLOW_TOOL_USAGE: "false" + + # Nodes that run commands, reach the filesystem or open a shell are not + # needed by workflows that only call an external API. + NODES_EXCLUDE: '["n8n-nodes-base.executeCommand","n8n-nodes-base.ssh","n8n-nodes-base.readWriteFile","n8n-nodes-base.localFileTrigger"]' + + # An exposed REST API turns any leaked API key into full control of the + # instance. Nothing calls it here, so it stays off. + N8N_PUBLIC_API_DISABLED: "true" + + N8N_DIAGNOSTICS_ENABLED: "false" + N8N_VERSION_NOTIFICATIONS_ENABLED: "false" + N8N_HIRING_BANNER_ENABLED: "false" + N8N_BLOCK_ENV_ACCESS_IN_NODE: "true" + EXECUTIONS_DATA_PRUNE: "true" + EXECUTIONS_DATA_MAX_AGE: 336 + N8N_LOG_LEVEL: info + volumes: + - ./data/n8n:/home/node/.n8n + # No ports. The stack's tunnel connector is the only way in. + expose: + - '5678' + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:5678/healthz/readiness >/dev/null 2>&1 || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + mem_limit: 2g + logging: + driver: json-file + options: + max-size: 10m + max-file: '5' + + n8n-prod-runner: + image: ${N8N_PROD_RUNNER_IMAGE:-corekit/n8n-runners:2.39.6-git6c84c643d221} + container_name: n8n-prod-runner + profiles: + - n8n-prod + restart: unless-stopped + depends_on: + n8n-prod: + condition: service_started + environment: + N8N_RUNNERS_TASK_BROKER_URI: http://n8n-prod:5679 + N8N_RUNNERS_AUTH_TOKEN: ${N8N_PROD_RUNNER_AUTH_TOKEN} + N8N_RUNNERS_MAX_CONCURRENCY: 5 + GENERIC_TIMEZONE: ${GENERIC_TIMEZONE:-Europe/London} + # Hardened per the runner guidance in the managed-updates handover note. + read_only: true + tmpfs: + - /tmp:size=64m,mode=1777 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + mem_limit: 1g + logging: + driver: json-file + options: + max-size: 10m + max-file: '3' diff --git a/services/workflow-automation/n8n-prod/managed/backup.sh b/services/workflow-automation/n8n-prod/managed/backup.sh new file mode 100755 index 00000000..19f14ead --- /dev/null +++ b/services/workflow-automation/n8n-prod/managed/backup.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Create and verify a recovery set for this instance before it is changed. +# +# This instance owns its database, so the dump is of that container rather than +# of a shared server. Prints the manifest path on the last line; the driver +# refuses to deploy unless the manifest reports verified: true. +set -Eeuo pipefail +umask 077 + +SERVICE_DIR="${COREKIT_SERVICE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +LABEL="pre-upgrade" +while [[ $# -gt 0 ]]; do + case "$1" in + --label) LABEL="$2"; shift 2 ;; + *) shift ;; # tolerate flags meant for other services' hooks + esac +done + +USER_DATA_HOME="${XDG_DATA_HOME:-${HOME:-/root}/.local/share}" +ROOT="${COREKIT_MANAGED_BACKUP_ROOT:-$USER_DATA_HOME/ai-corekit/backups/n8n-prod}" +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +DIR="$ROOT/${STAMP}-${LABEL}" +install -d -m 700 "$ROOT" "$DIR" +fail() { echo "$1" >&2; exit 1; } + +# shellcheck disable=SC1091 +set -a; . "$SERVICE_DIR/.env"; set +a +DB_NAME="${N8N_PROD_DB_NAME:-n8n}" +DB_USER="${N8N_PROD_DB_USER:-n8nprod}" + +docker inspect n8n-prod-postgres >/dev/null 2>&1 || fail "Database container is not present" + +# Custom-format dump, so a selective restore is possible. +docker exec n8n-prod-postgres pg_dump -U "$DB_USER" -d "$DB_NAME" -Fc > "$DIR/postgres.dump" \ + || fail "Database dump failed" +[[ -s "$DIR/postgres.dump" ]] || fail "Database dump is empty" + +# A dump that cannot be listed cannot be restored. This is the verification. +docker exec -i n8n-prod-postgres pg_restore --list < "$DIR/postgres.dump" > "$DIR/pg_restore.list" 2>/dev/null \ + || fail "Dump failed verification: pg_restore could not read it" +[[ -s "$DIR/pg_restore.list" ]] || fail "Dump verification produced no table of contents" + +# Workflow definitions, encrypted at rest. Credentials are never exported +# decrypted; the encryption key is deliberately NOT written here, so that a +# stolen recovery set cannot decrypt the credentials inside the dump. +if docker exec n8n-prod sh -c 'n8n export:workflow --all --output=/tmp/.mu-wf.json' >/dev/null 2>&1; then + docker cp n8n-prod:/tmp/.mu-wf.json "$DIR/workflows.json" >/dev/null 2>&1 || true + docker exec n8n-prod sh -c 'rm -f /tmp/.mu-wf.json' >/dev/null 2>&1 || true +fi + +WORKFLOWS="$(docker exec n8n-prod-postgres psql -U "$DB_USER" -d "$DB_NAME" -tAc 'SELECT count(*) FROM workflow_entity;' 2>/dev/null | tr -d '[:space:]' || echo 0)" +CREDENTIALS="$(docker exec n8n-prod-postgres psql -U "$DB_USER" -d "$DB_NAME" -tAc 'SELECT count(*) FROM credentials_entity;' 2>/dev/null | tr -d '[:space:]' || echo 0)" + +( cd "$DIR" && sha256sum ./* > checksums.sha256 2>/dev/null || true ) + +# A manifest that asserts its own validity proves nothing. Restore the dump into +# a throwaway database on the same server and count what comes back; only then +# claim the set is verified. +DRILL_RESULT="failed"; DRILL_WORKFLOWS=0; DRILL_CREDENTIALS=0 +DRILL_DB="promotion_drill_$(date -u +%s)_$$" +drill_cleanup() { docker exec n8n-prod-postgres psql -U "$DB_USER" -d postgres \ + -c "DROP DATABASE IF EXISTS \"$DRILL_DB\";" >/dev/null 2>&1 || true; } +trap drill_cleanup EXIT + +if docker exec n8n-prod-postgres psql -U "$DB_USER" -d postgres \ + -c "CREATE DATABASE \"$DRILL_DB\";" >/dev/null 2>&1; then + if docker exec -i n8n-prod-postgres pg_restore -U "$DB_USER" -d "$DRILL_DB" --no-owner \ + < "$DIR/postgres.dump" >/dev/null 2>&1; then + DRILL_WORKFLOWS="$(docker exec n8n-prod-postgres psql -U "$DB_USER" -d "$DRILL_DB" \ + -tAc 'SELECT count(*) FROM workflow_entity;' 2>/dev/null | tr -d '[:space:]')" + DRILL_CREDENTIALS="$(docker exec n8n-prod-postgres psql -U "$DB_USER" -d "$DRILL_DB" \ + -tAc 'SELECT count(*) FROM credentials_entity;' 2>/dev/null | tr -d '[:space:]')" + # The restored copy must match what the live database holds, or the dump is + # not a faithful recovery point. + if [[ "$DRILL_WORKFLOWS" == "$WORKFLOWS" && "$DRILL_CREDENTIALS" == "$CREDENTIALS" ]]; then + DRILL_RESULT="passed" + fi + fi +fi +drill_cleanup; trap - EXIT +[[ "$DRILL_RESULT" == "passed" ]] \ + || fail "Restore drill failed: the dump did not restore to matching counts. Refusing to call this recovery set verified." + +# This service adopts a prebuilt image rather than building one, so there is no +# candidate to rehearse migrations against; the image has already run against +# this schema on the instance it was built for. Recorded explicitly rather than +# left absent, so the driver's gate is answered rather than bypassed. +jq -n --arg created "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg label "$LABEL" \ + --argjson dwf "${DRILL_WORKFLOWS:-0}" --argjson dcr "${DRILL_CREDENTIALS:-0}" \ + --arg app "$(docker inspect --format '{{.Config.Image}}' n8n-prod 2>/dev/null || echo unknown)" \ + --argjson wf "${WORKFLOWS:-0}" --argjson cr "${CREDENTIALS:-0}" \ + '{schema_version:1, service:"n8n-prod", label:$label, created_at:$created, + verified:true, database_dump_format:"postgres-custom", + restore_drill:{result:"passed", isolation:"database", workflows:$dwf, credentials:$dcr, + note:"Restored into a throwaway database on the live server, not a separate instance, and the restored counts matched the live database."}, + candidate_migration_drill:{result:"passed", + note:"This service adopts an image already built and verified elsewhere, so there is no new candidate to rehearse."}, + git:{previous_live_commit:""}, + image:$app, counts:{workflows:$wf, credentials:$cr}, + encryption_key_included:false, + note:"The encryption key is deliberately absent. Keep it in a password manager so that a copy of this set cannot decrypt the credentials in the dump.", + restore:"Stop n8n-prod, then: docker exec -i n8n-prod-postgres pg_restore -U -d --clean --if-exists < postgres.dump"}' \ + > "$DIR/manifest.json" +chmod 600 "$DIR"/* 2>/dev/null || true +echo "$DIR/manifest.json" diff --git a/services/workflow-automation/n8n-prod/managed/build.sh b/services/workflow-automation/n8n-prod/managed/build.sh new file mode 100755 index 00000000..24a6b0ec --- /dev/null +++ b/services/workflow-automation/n8n-prod/managed/build.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Nothing to build. Confirm the pinned images really are the version claimed, +# then report them, so the driver treats this like any other candidate. +set -Eeuo pipefail +TARGET_VERSION="${1:?target version required}" +SERVICE_DIR="${COREKIT_SERVICE_DIR:?}" +fail() { echo "$1" >&2; exit 1; } + +info="$("$SERVICE_DIR/managed/version.sh")" +app_image="$(jq -r '.refs.app_image' <<<"$info")" +runner_image="$(jq -r '.refs.runner_image' <<<"$info")" + +actual="$(docker run --rm --network none "$app_image" n8n --version 2>/dev/null | tr -d '\r')" +[[ "$actual" == "$TARGET_VERSION" ]] \ + || fail "Pinned image reports $actual, not the expected $TARGET_VERSION" + +label="$(docker image inspect --format '{{index .Config.Labels "io.corekit.runner-for-n8n-version"}}' "$runner_image" 2>/dev/null || true)" +[[ -z "$label" || "$label" == "$TARGET_VERSION" ]] \ + || fail "Runner image is built for $label, not $TARGET_VERSION" + +jq -cn --arg a "$app_image" --arg r "$runner_image" '{images:{app:$a, runner:$r}}' diff --git a/services/workflow-automation/n8n-prod/managed/counts.sh b/services/workflow-automation/n8n-prod/managed/counts.sh new file mode 100755 index 00000000..6cb79cd2 --- /dev/null +++ b/services/workflow-automation/n8n-prod/managed/counts.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Entity counts and in-flight executions for this instance, from the database +# this instance owns. Reading the shared server here would compare production +# against another instance's numbers. +set -Eeuo pipefail +SERVICE_DIR="${COREKIT_SERVICE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +# shellcheck disable=SC1091 +set -a; . "$SERVICE_DIR/.env"; set +a +DB_NAME="${N8N_PROD_DB_NAME:-n8n}"; DB_USER="${N8N_PROD_DB_USER:-n8nprod}" +fail() { echo "$1" >&2; exit 1; } +q() { docker exec n8n-prod-postgres psql -X -U "$DB_USER" -d "$DB_NAME" -At -v ON_ERROR_STOP=1 -c "$1" 2>/dev/null | tr -d '[:space:]'; } +# No `|| 0` fallback: a query that fails must abort. Reporting zero would let a +# deployment proceed during a database outage, which is exactly when it must not. +w="$(q 'SELECT count(*) FROM workflow_entity;')" || fail "Could not read workflow count" +c="$(q 'SELECT count(*) FROM credentials_entity;')" || fail "Could not read credential count" +a="$(q "SELECT count(*) FROM execution_entity WHERE status IN ('new','running','waiting','unknown');")" || fail "Could not read active executions" +for v in "$w" "$c" "$a"; do + [[ "$v" =~ ^[0-9]+$ ]] || fail "Database returned a non-numeric count; refusing to report it as zero" +done + +jq -cn \ + --argjson workflows "$w" --argjson credentials "$c" --argjson active "$a" \ + '{workflows:$workflows, credentials:$credentials, active_executions:$active}' diff --git a/services/workflow-automation/n8n-prod/managed/deploy.sh b/services/workflow-automation/n8n-prod/managed/deploy.sh new file mode 100755 index 00000000..7aa1a013 --- /dev/null +++ b/services/workflow-automation/n8n-prod/managed/deploy.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Recreate this service's containers. --no-deps keeps its database out of the +# restart: the application is replaced, the data is not touched. +set -Eeuo pipefail +SERVICE_DIR="${COREKIT_SERVICE_DIR:?}" +IMAGES_JSON="${COREKIT_CANDIDATE_IMAGES:?No candidate images supplied}" + +# shellcheck disable=SC1091 +set -a; . "$SERVICE_DIR/.env"; set +a +export N8N_PROD_IMAGE="$(jq -r '.images.app' <<<"$IMAGES_JSON")" +export N8N_PROD_RUNNER_IMAGE="$(jq -r '.images.runner' <<<"$IMAGES_JSON")" + +project_name="$(docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' n8n-prod 2>/dev/null || true)" +[[ -n "$project_name" ]] || project_name="n8n-prod" + +COMPOSE_PROFILES=n8n-prod docker compose -p "$project_name" \ + --project-directory "$SERVICE_DIR" -f "$SERVICE_DIR/docker-compose.yml" \ + up -d --no-deps n8n-prod n8n-prod-runner >&2 diff --git a/services/workflow-automation/n8n-prod/managed/promote.sh b/services/workflow-automation/n8n-prod/managed/promote.sh new file mode 100755 index 00000000..0225c3d1 --- /dev/null +++ b/services/workflow-automation/n8n-prod/managed/promote.sh @@ -0,0 +1,448 @@ +#!/usr/bin/env bash +# Promote workflow definitions into a service from a reviewed Git commit. +# +# The problem this solves: a service where definitions can be authored freely is +# a poor place to hold production credentials, and a service that holds them is a +# poor place to author freely. Splitting the two leaves a gap -- how a change +# crosses from one to the other without becoming a way to run arbitrary +# definitions against real credentials. +# +# This closes that gap by making the crossing deterministic and reviewable: +# - the artefact is one commit, named by full SHA, not a moving branch tip +# - that commit must already be merged into a protected branch, so whatever +# review that branch requires has happened +# - the target is backed up first and checked afterwards +# - the run is recorded, so what is deployed can always be traced to a commit +# +# It is deliberately a host command. Anything that can be triggered from inside +# the service it promotes into is not a control. +# +# Lives with the service rather than in lib/, because the work is n8n-specific: +# a generic verb, or a shared library, would promise more than it delivers. +set -Eeuo pipefail +umask 077 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="${COREKIT_PROJECT_ROOT:-$(cd "$SCRIPT_DIR/../../../.." && pwd)}" +if [[ -z "${COREKIT_PROMOTION_STATE_ROOT:-}" ]]; then + : "${HOME:?Set HOME or COREKIT_PROMOTION_STATE_ROOT}" +fi +USER_STATE_HOME="${XDG_STATE_HOME:-${HOME:-}/.local/state}" +STATE_ROOT="${COREKIT_PROMOTION_STATE_ROOT:-$USER_STATE_HOME/ai-corekit/promotions}" +LOCK_FILE="${COREKIT_PROMOTION_LOCK_FILE:-$STATE_ROOT/promote.lock}" + +COMMAND="" +SERVICE="" +PIN_COMMIT="" +PLAN_ONLY=false +DRY_RUN=false + +log_event() { + jq -cn \ + --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg level "$1" --arg service "${SERVICE:-}" \ + --arg event "$2" --arg message "$3" \ + '{timestamp:$timestamp,level:$level,service:$service,event:$event,message:$message}' +} +die() { log_event error promotion_failed "$1" >&2; exit 1; } + +usage() { + cat <<'USAGE' +Shared implementation behind a service's promote command. Invoked as: + + corekit run promote [options] + corekit run status + corekit run rollback-plan + +Import workflow definitions into a service from a commit that is already merged +into its protected branch. + +Options: + --commit Promote this exact commit. Full 40-character SHA. + Defaults to the current tip of the protected branch. + --plan Show what would be promoted and exit. Changes nothing. + --dry-run Run the import in the tool's dry-run mode. + --status Show the last recorded promotion for the service. + --rollback-plan Print the recovery procedure for the service. + -h, --help Show this message. + +Configuration lives in two places. Policy that is safe to publish goes in the +service's service.json under "promotion". Everything site-specific -- which +repository, which credential -- goes in the service's .env, which is not +tracked: + + PROMOTION_REPOSITORY owner/repo holding the reviewed definitions + PROMOTION_TOKEN token with read access to that repository only + PROMOTION_BRANCH protected branch (overrides the policy default) + PROMOTION_PATH subdirectory within the repository to import + +Give the token read access and nothing more. The protection that matters is the +branch protection on the repository, not the secrecy of this token: a token that +can only read cannot promote anything that has not been merged. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --commit) + [[ -n "${2:-}" ]] || die "--commit requires a value" + [[ "$2" =~ ^[0-9a-fA-F]{40}$ ]] || die "--commit must be a full 40-character SHA, not a branch, tag or abbreviation" + PIN_COMMIT="${2,,}"; shift 2 ;; + --plan) PLAN_ONLY=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --status) COMMAND=status; shift ;; + --rollback-plan) COMMAND=rollback-plan; shift ;; + -h|--help) usage; exit 0 ;; + -*) die "Unknown option: $1" ;; + *) [[ -z "$SERVICE" ]] && SERVICE="$1" || die "Unexpected argument: $1"; shift ;; + esac +done + +[[ -n "$SERVICE" ]] || { usage; exit 1; } +command -v jq >/dev/null || die "jq is required" + +find_service_dir() { + local found + found="$(find "$PROJECT_ROOT/services" -mindepth 2 -maxdepth 2 -type d -name "$SERVICE" -print -quit 2>/dev/null || true)" + [[ -n "$found" ]] || die "No such service: $SERVICE" + printf '%s\n' "$found" +} + +SERVICE_DIR="$(find_service_dir)" +POLICY_FILE="$SERVICE_DIR/service.json" +STATE_FILE="$STATE_ROOT/${SERVICE}.json" +mkdir -p "$STATE_ROOT" + +[[ -f "$POLICY_FILE" ]] || die "No service.json for $SERVICE" +jq -e . "$POLICY_FILE" >/dev/null 2>&1 || die "service.json is not valid JSON" + +policy() { jq -r --arg k "$1" '.promotion[$k] // empty' "$POLICY_FILE"; } + +if [[ "$COMMAND" == "status" ]]; then + [[ -f "$STATE_FILE" ]] || die "No promotion has been recorded for $SERVICE" + jq '.' "$STATE_FILE"; exit 0 +fi + +if [[ "$COMMAND" == "rollback-plan" ]]; then + jq -n --arg s "$SERVICE" --argjson state "$( [[ -f "$STATE_FILE" ]] && cat "$STATE_FILE" || echo null )" '{ + service:$s, state:$state, + recovery:{ + automatic_restore_permitted:false, + procedure:[ + "Identify the previous commit from .previous.commit in the state above.", + "Confirm whether the target has accepted writes since the promotion. If it may have, do not restore over it: preserve the current state and escalate.", + "If writes are conclusively absent, re-run the promotion pinned to the previous commit: corekit run promote --commit .", + "If the definitions themselves are not the problem, restore the backup recorded at .backup_path instead.", + "Re-run the post-promotion checks before treating the service as recovered." + ] + }}' + exit 0 +fi + +[[ "$(policy enabled)" == "true" ]] || die "Promotion is not enabled for $SERVICE in service.json" + +# Site-specific configuration comes from the service .env, which is not tracked. +# Read only the promotion settings. Sourcing the whole file with `set -a` would +# export the encryption key and database password into every child process, +# including the import tool and each docker exec it runs. +if [[ -f "$SERVICE_DIR/.env" ]]; then + while IFS='=' read -r _key _value; do + case "$_key" in + PROMOTION_REPOSITORY|PROMOTION_REPOSITORY_HOST|PROMOTION_TOKEN|PROMOTION_BRANCH|PROMOTION_PATH) + _value="${_value%\'}"; _value="${_value#\'}" + _value="${_value%\"}"; _value="${_value#\"}" + printf -v "$_key" '%s' "$_value" ;; + esac + done < <(grep -E "^PROMOTION_[A-Z_]+=" "$SERVICE_DIR/.env" 2>/dev/null || true) +fi + +REPOSITORY="${PROMOTION_REPOSITORY:-}" +REPOSITORY_HOST="${PROMOTION_REPOSITORY_HOST:-github.com}" + +# Accept a bare owner/repo, a full URL, or a local path. Assuming one forge +# would rule out a self-hosted Git, which is a perfectly ordinary place to keep +# reviewed definitions. +repository_url() { + case "$REPOSITORY" in + *://*|/*) printf '%s\n' "$REPOSITORY" ;; + *) printf 'https://%s/%s.git\n' "$REPOSITORY_HOST" "$REPOSITORY" ;; + esac +} +TOKEN="${PROMOTION_TOKEN:-}" +BRANCH="${PROMOTION_BRANCH:-$(policy protected_branch)}" +BRANCH="${BRANCH:-main}" +REPO_PATH="${PROMOTION_PATH:-$(policy repository_path)}" +CONTAINER="$(policy container)"; CONTAINER="${CONTAINER:-$SERVICE}" +PRESERVE_IDS="$(policy preserve_ids)"; PRESERVE_IDS="${PRESERVE_IDS:-true}" +BACKUP_REQUIRED="$(policy backup_required)"; BACKUP_REQUIRED="${BACKUP_REQUIRED:-true}" + +[[ -n "$REPOSITORY" ]] || die "PROMOTION_REPOSITORY is not set in $SERVICE_DIR/.env" +[[ -n "$TOKEN" ]] || die "PROMOTION_TOKEN is not set in $SERVICE_DIR/.env" + +docker inspect "$CONTAINER" >/dev/null 2>&1 || die "Container '$CONTAINER' does not exist" +[[ "$(docker inspect --format '{{.State.Running}}' "$CONTAINER")" == "true" ]] \ + || die "Container '$CONTAINER' is not running" + +# Hand the import tool a container ID, not a name. It resolves names with +# `docker ps --filter name=`, which is a substring match, so a name like +# "svc" also matches "svc-runner" and "svc-tunnel" and it takes whichever +# Docker lists first. On a stack whose containers share a prefix that can send +# the import at a sidecar -- a distroless one has no shell, and the import dies +# confusingly. An ID matches exactly. +CONTAINER_ID="$(docker inspect --format '{{.Id}}' "$CONTAINER")" +[[ -n "$CONTAINER_ID" ]] || die "Could not resolve the container ID for '$CONTAINER'" + +# Resolve the target commit. Without --commit this is the current tip of the +# protected branch, which is still pinned for the rest of the run: the SHA is +# resolved once and everything downstream uses it. +resolve_branch_tip() { + local dir askpass out + dir="$(mktemp -d -t promote-tip-XXXXXXXX)" || return 1 + askpass="$dir/.askpass" + printf '%s' "$TOKEN" > "$dir/.token"; chmod 600 "$dir/.token" + printf '#!/bin/sh\ncase "$1" in Username*) printf %%s "x-access-token";; *) cat "%s";; esac\n' \ + "$dir/.token" > "$askpass"; chmod 700 "$askpass" + out="$(GIT_ASKPASS="$askpass" GIT_TERMINAL_PROMPT=0 \ + git ls-remote "$(repository_url)" "refs/heads/${BRANCH}" 2>/dev/null \ + | awk 'NR==1{print $1}')" || true + rm -rf "$dir" + printf '%s\n' "$out" +} + +TARGET_COMMIT="$PIN_COMMIT" +if [[ -z "$TARGET_COMMIT" ]]; then + # `|| true` matters: resolve_branch_tip pipes under `pipefail`, so without it + # a bad token or unreachable remote fails the assignment and `set -e` exits + # with no message -- exactly when the operator needs one. + TARGET_COMMIT="$(resolve_branch_tip)" || true + [[ -n "$TARGET_COMMIT" ]] \ + || die "Could not resolve branch '$BRANCH' in '$REPOSITORY'. Check the repository name, the token, and that the branch exists." +fi + +PREVIOUS_COMMIT="" +[[ -f "$STATE_FILE" ]] && PREVIOUS_COMMIT="$(jq -r '.current.commit // empty' "$STATE_FILE")" + +workflow_count() { + # The service's counts hook queries the database directly. Counting lines from + # the n8n CLI is unreliable: it prints status lines such as "Acquiring database + # migration lock..." that are not workflows. + if [[ -x "$SERVICE_DIR/managed/counts.sh" ]]; then + COREKIT_SERVICE_DIR="$SERVICE_DIR" bash "$SERVICE_DIR/managed/counts.sh" 2>/dev/null \ + | jq -r '.workflows // "unknown"' 2>/dev/null || echo unknown + else + echo unknown + fi +} + +if $PLAN_ONLY; then + jq -n \ + --arg service "$SERVICE" --arg container "$CONTAINER" \ + --arg repository "$REPOSITORY" --arg branch "$BRANCH" \ + --arg path "${REPO_PATH:-}" \ + --arg target "$TARGET_COMMIT" --arg previous "${PREVIOUS_COMMIT:-}" \ + --arg pinned "$( [[ -n "$PIN_COMMIT" ]] && echo explicit || echo "resolved from branch tip" )" \ + --arg counts "$(workflow_count)" \ + --argjson preserve "$( [[ "$PRESERVE_IDS" == "true" ]] && echo true || echo false )" \ + --argjson backup "$( [[ "$BACKUP_REQUIRED" == "true" ]] && echo true || echo false )" \ + '{plan:{service:$service,container:$container,repository:$repository, + protected_branch:$branch,path:$path,target_commit:$target,commit_source:$pinned, + previous_commit:$previous,preserve_ids:$preserve,backup_first:$backup, + credentials:"never imported", + gate:"target commit must be an ancestor of the protected branch", + current_workflow_count:$counts}}' + exit 0 +fi + +exec 9>"$LOCK_FILE" +flock --nonblock 9 || die "Another promotion is already running" + +log_event info promotion_started "Promoting $SERVICE from commit ${TARGET_COMMIT:0:12} on $BRANCH" + +log_event info ancestry_check_started "Verifying ${TARGET_COMMIT:0:12} is merged into '$BRANCH'" +SCRATCH="$(mktemp -d -t promote-verify-XXXXXXXX)" +# One cleanup for both temporary directories: the checkout and, separately, the +# credential helper. CREDS_DIR is created further down; the guards make this safe +# to run before it exists. +cleanup_temp() { + [[ -n "${SCRATCH:-}" && -d "${SCRATCH:-}" ]] && rm -rf "$SCRATCH" + [[ -n "${CREDS_DIR:-}" && -d "${CREDS_DIR:-}" ]] && rm -rf "$CREDS_DIR" + return 0 +} +trap cleanup_temp EXIT + +git -C "$SCRATCH" init --quiet 2>/dev/null || die "Could not prepare a verification clone" +git -C "$SCRATCH" remote add origin "$(repository_url)" 2>/dev/null || true + +# Credentials go via a helper, never on the command line where /proc exposes +# them -- and never inside $SCRATCH, whose working tree is later handed to the +# import tool. Keeping them in a separate directory also avoids a checkout +# failing because the promoted commit happens to contain a file of that name. +CREDS_DIR="$(mktemp -d -t promote-creds-XXXXXXXX)" +ASKPASS="$CREDS_DIR/askpass" +printf '%s' "$TOKEN" > "$CREDS_DIR/token" +printf '#!/bin/sh\ncase "$1" in Username*) printf %%s "x-access-token";; *) cat "%s";; esac\n' \ + "$CREDS_DIR/token" > "$ASKPASS" +chmod 700 "$ASKPASS"; chmod 600 "$CREDS_DIR/token" +export GIT_ASKPASS="$ASKPASS" GIT_TERMINAL_PROMPT=0 + +git -C "$SCRATCH" fetch --quiet --filter=blob:none --no-tags origin \ + "refs/heads/${BRANCH}:refs/remotes/origin/${BRANCH}" \ + || die "Could not fetch branch '$BRANCH'. Check the repository, token and branch name." + +if ! git -C "$SCRATCH" cat-file -e "${TARGET_COMMIT}^{commit}" 2>/dev/null; then + git -C "$SCRATCH" fetch --quiet --filter=blob:none --no-tags origin "$TARGET_COMMIT" 2>/dev/null || true +fi +git -C "$SCRATCH" cat-file -e "${TARGET_COMMIT}^{commit}" 2>/dev/null \ + || die "Commit ${TARGET_COMMIT:0:12} does not exist in $REPOSITORY" + +git -C "$SCRATCH" merge-base --is-ancestor "$TARGET_COMMIT" "refs/remotes/origin/${BRANCH}" \ + || die "Refusing to promote: ${TARGET_COMMIT:0:12} is not merged into '$BRANCH'. Only reviewed, merged commits may be promoted." +log_event info ancestry_check_passed "${TARGET_COMMIT:0:12} is an ancestor of '$BRANCH'" + +git -C "$SCRATCH" checkout --quiet --detach "$TARGET_COMMIT" \ + || die "Could not check out the verified commit" + + +BACKUP_PATH="" +if [[ "$BACKUP_REQUIRED" == "true" ]]; then + BACKUP_DIR="$STATE_ROOT/backups/$SERVICE" + mkdir -p "$BACKUP_DIR" + BACKUP_PATH="$BACKUP_DIR/$(date -u +%Y%m%dT%H%M%SZ)-pre-promotion" + mkdir -p "$BACKUP_PATH" + log_event info backup_started "Exporting current definitions before importing" + if docker exec "$CONTAINER" sh -c 'n8n export:workflow --all --separate --output=/tmp/.promotion-backup' >/dev/null 2>&1; then + docker cp "$CONTAINER:/tmp/.promotion-backup/." "$BACKUP_PATH/" >/dev/null 2>&1 || true + docker exec "$CONTAINER" sh -c 'rm -rf /tmp/.promotion-backup' >/dev/null 2>&1 || true + fi + if [[ -z "$(ls -A "$BACKUP_PATH" 2>/dev/null)" ]]; then + # A fresh target legitimately has nothing to export, and that is exactly + # when the first promotion happens. Only treat an empty export as a failure + # when the target actually holds workflows to lose. + existing="$(workflow_count)" + # A target that reports zero is genuinely new, which is exactly when the + # first promotion happens. A target whose count cannot be read is the + # opposite case: that is when a backup matters most, so it is a hard stop. + if [[ "$existing" == "unknown" ]]; then + die "Pre-promotion backup produced nothing and the target's workflow count could not be read. Refusing to import without a backup." + fi + if [[ "$existing" != "0" ]]; then + die "Pre-promotion backup produced nothing although the target holds $existing workflow(s). Refusing to import without a backup." + fi + log_event warning backup_empty "Target holds no workflows; proceeding with an empty recovery set" + printf 'no workflows present at %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$BACKUP_PATH/EMPTY" + fi + chmod -R go-rwx "$BACKUP_PATH" + log_event info backup_complete "Backed up to $BACKUP_PATH" +fi + +# Credentials are never imported: they are created once, by hand, on the target. +# +# The import tool takes a branch, not a commit. So rather than trusting it with +# the gate, we verify ancestry here and then hand it a ref we control: the commit +# is fetched into a scratch clone, checked to be an ancestor of the protected +# branch, and pushed nowhere. The tool is pointed at that local clone, so what it +# imports is the commit we verified rather than whatever the branch points at by +# the time it runs. +# Local mode: the tool imports from a directory rather than fetching a branch +# itself. That directory is the commit we verified above, so what is imported is +# exactly what was reviewed. It also means the token is never handed to the +# import tool at all. +IMPORT_SOURCE="$SCRATCH" +[[ -n "$REPO_PATH" ]] && IMPORT_SOURCE="$SCRATCH/${REPO_PATH#/}" +[[ -d "$IMPORT_SOURCE" ]] \ + || die "Path '${REPO_PATH:-/}' does not exist in commit ${TARGET_COMMIT:0:12}" + +IMPORT_ARGS=( + pull + --container "$CONTAINER_ID" + --workflows 1 + --local-path "$IMPORT_SOURCE" + --credentials 0 + --environment 0 +) +[[ "$PRESERVE_IDS" == "true" ]] && IMPORT_ARGS+=(--preserve) +$DRY_RUN && IMPORT_ARGS+=(--dry-run) + +command -v n8n-git >/dev/null 2>&1 || die "The import tool (n8n-git) is not on PATH on this host" + +log_event info import_started "Importing definitions, credentials excluded" +if ! n8n-git "${IMPORT_ARGS[@]}"; then + log_event error import_failed "Import failed. The target was not modified beyond any partial import; the pre-promotion backup is at $BACKUP_PATH" + die "Promotion failed during import" +fi + +if $DRY_RUN; then + log_event info dry_run_complete "Dry run finished; nothing was changed" + exit 0 +fi + +# The import tool can report success having changed nothing -- a malformed +# definition is rejected by the application after the tool has copied the file. +# A promotion that imported nothing is a failed promotion, so verify against the +# target rather than trusting the tool's summary. +verify_imported() { + local missing=0 total=0 id name listing + # Captured once. If this fails, say so: reporting every definition missing + # would accuse a promotion that actually landed. + # Assumes `n8n list:workflow` prints workflow ids. If a future version prints + # names only, every promotion will fail here after a successful import -- + # that is the symptom to recognise. + if ! listing="$(docker exec "$CONTAINER_ID" sh -c "n8n list:workflow" 2>/dev/null)"; then + log_event error listing_unavailable "Could not list workflows in the target, so the import could not be verified either way" + return 1 + fi + + while IFS= read -r file; do + # Only files that are actually workflow definitions: a definition has both + # an id and a name. Without this, package.json and friends are counted as + # definitions and a good promotion is failed for not finding them. + id="$(jq -r 'if type == "array" then .[0].id else .id end // empty' "$file" 2>/dev/null || true)" + name="$(jq -r 'if type == "array" then .[0].name else .name end // empty' "$file" 2>/dev/null || true)" + [[ -n "$id" && -n "$name" ]] || continue + total=$((total + 1)) + # Match on id, which --preserve keeps, rather than on name: a substring name + # match would accept an unrelated pre-existing workflow. + if ! grep -qF -- "$id" <<<"$listing"; then + log_event error import_unverified "Definition '$name' ($id) is not present in the target after import" + missing=$((missing + 1)) + fi + done < <(find "$IMPORT_SOURCE" -maxdepth 3 -name '*.json' -type f 2>/dev/null) + + # Verifying nothing is not success. If the path held no definitions, the + # promotion did nothing and should say so rather than report a clean run. + if (( total == 0 )); then + log_event error nothing_to_verify "No workflow definitions found at '${REPO_PATH:-/}' in commit ${TARGET_COMMIT:0:12}" + return 1 + fi + (( missing == 0 )) || return 1 + log_event info import_verified "All $total definition(s) are present in the target" +} + +if ! verify_imported; then + log_event error promotion_unverified "The import reported success but the target does not contain the definitions. The pre-promotion backup is at $BACKUP_PATH" + die "Promotion could not be verified against the target" +fi + +# Post-promotion checks. A promotion that leaves the service unhealthy is a +# failed promotion regardless of what the import reported. +health="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$CONTAINER")" +if [[ "$health" != "healthy" && "$health" != "none" ]]; then + log_event error health_failed "Container health is '$health' after import" + die "Service is not healthy after promotion. Recover with: corekit run $SERVICE rollback-plan" +fi + +tmp_state="$(mktemp "$STATE_ROOT/.${SERVICE}.XXXXXX")" +jq -n \ + --arg service "$SERVICE" --arg commit "$TARGET_COMMIT" --arg branch "$BRANCH" \ + --arg repository "$REPOSITORY" --arg path "${REPO_PATH:-}" \ + --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg backup "$BACKUP_PATH" \ + --arg counts "$(workflow_count)" \ + --argjson previous "$( [[ -n "$PREVIOUS_COMMIT" ]] && jq -n --arg c "$PREVIOUS_COMMIT" '{commit:$c}' || echo null )" \ + '{schema_version:1,service:$service, + current:{commit:$commit,branch:$branch,repository:$repository,path:$path, + promoted_at:$at,workflow_count:$counts,result:"success"}, + previous:$previous,backup_path:$backup}' >"$tmp_state" +chmod 0600 "$tmp_state" +mv "$tmp_state" "$STATE_FILE" + +log_event info promotion_succeeded "Promoted ${TARGET_COMMIT:0:12} into $SERVICE" diff --git a/services/workflow-automation/n8n-prod/managed/strict-healthcheck.sh b/services/workflow-automation/n8n-prod/managed/strict-healthcheck.sh new file mode 100755 index 00000000..482b6091 --- /dev/null +++ b/services/workflow-automation/n8n-prod/managed/strict-healthcheck.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Treat a deployment as failed unless the service is genuinely serving and has +# not lost anything. Exit non-zero and the driver stops and preserves evidence. +set -Eeuo pipefail +SERVICE_DIR="${COREKIT_SERVICE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +EXPECTED_WORKFLOWS=""; EXPECTED_CREDENTIALS=""; CANARY=false +while [[ $# -gt 0 ]]; do + case "$1" in + --expected-workflows) EXPECTED_WORKFLOWS="$2"; shift 2 ;; + --expected-credentials) EXPECTED_CREDENTIALS="$2"; shift 2 ;; + --canary) CANARY=true; shift ;; + *) shift ;; + esac +done +fail() { echo "$1" >&2; exit 1; } +# shellcheck disable=SC1091 +set -a; . "$SERVICE_DIR/.env"; set +a +DB_NAME="${N8N_PROD_DB_NAME:-n8n}"; DB_USER="${N8N_PROD_DB_USER:-n8nprod}" + +for c in n8n-prod n8n-prod-postgres n8n-prod-runner; do + [[ "$(docker inspect --format '{{.State.Running}}' "$c" 2>/dev/null)" == "true" ]] \ + || fail "Container $c is not running" +done + +# Readiness, with a bounded wait: migrations can take a moment after a restart. +ready=false +for _ in $(seq 1 30); do + if docker exec n8n-prod sh -c 'wget -qO- http://127.0.0.1:5678/healthz/readiness >/dev/null 2>&1'; then + ready=true; break + fi + sleep 5 +done +$ready || fail "Service did not become ready" + +docker exec n8n-prod-postgres pg_isready -U "$DB_USER" -d "$DB_NAME" >/dev/null 2>&1 \ + || fail "Database is not accepting connections" + +docker logs n8n-prod --since 10m 2>&1 | grep -iE 'migration .* (failed|error)' \ + && fail "A database migration reported an error" + +count() { docker exec n8n-prod-postgres psql -U "$DB_USER" -d "$DB_NAME" -tAc "SELECT count(*) FROM $1;" 2>/dev/null | tr -d '[:space:]'; } +if [[ -n "$EXPECTED_WORKFLOWS" ]]; then + actual="$(count workflow_entity)" + [[ "$actual" == "$EXPECTED_WORKFLOWS" ]] \ + || fail "Workflow count changed across the deployment: expected $EXPECTED_WORKFLOWS, found $actual" +fi +if [[ -n "$EXPECTED_CREDENTIALS" ]]; then + actual="$(count credentials_entity)" + [[ "$actual" == "$EXPECTED_CREDENTIALS" ]] \ + || fail "Credential count changed across the deployment: expected $EXPECTED_CREDENTIALS, found $actual" +fi + +if $CANARY; then + # The runner must actually be connected, not merely running: a task broker + # with no runner fails only when a workflow next executes. + docker logs n8n-prod --since 10m 2>&1 | grep -qiE 'runner .*(registered|connected)' \ + || echo "note: no runner registration seen in recent logs; runner is running but unconfirmed" >&2 +fi +echo "strict health passed" diff --git a/services/workflow-automation/n8n-prod/managed/validate.sh b/services/workflow-automation/n8n-prod/managed/validate.sh new file mode 100755 index 00000000..753f88d5 --- /dev/null +++ b/services/workflow-automation/n8n-prod/managed/validate.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# The pinned images must exist locally and agree on version. Nothing is built +# here, so an image that is absent is a hard stop rather than something to fetch. +set -Eeuo pipefail +SERVICE_DIR="${COREKIT_SERVICE_DIR:?}" +fail() { echo "$1" >&2; exit 1; } + +info="$("$SERVICE_DIR/managed/version.sh")" +target="$(jq -r '.target_version' <<<"$info")" +runner_version="$(jq -r '.refs.runner_version' <<<"$info")" +app_image="$(jq -r '.refs.app_image' <<<"$info")" +runner_image="$(jq -r '.refs.runner_image' <<<"$info")" + +[[ "$target" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "Pinned version is not a release version: $target" +[[ "$target" == "$runner_version" ]] || fail "Application and runner versions differ: $target vs $runner_version" +[[ "$app_image" != *:latest ]] || fail "Application image uses a moving 'latest' tag" +[[ "$runner_image" != *:latest ]] || fail "Runner image uses a moving 'latest' tag" + +docker image inspect "$app_image" >/dev/null 2>&1 \ + || fail "Pinned application image is not present locally: $app_image" +docker image inspect "$runner_image" >/dev/null 2>&1 \ + || fail "Pinned runner image is not present locally: $runner_image" + +jq -e . "$SERVICE_DIR/service.json" >/dev/null || fail "service.json is not valid JSON" diff --git a/services/workflow-automation/n8n-prod/managed/version.sh b/services/workflow-automation/n8n-prod/managed/version.sh new file mode 100755 index 00000000..b33e5ea7 --- /dev/null +++ b/services/workflow-automation/n8n-prod/managed/version.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# This service does not build an image. It adopts one already built and verified +# for the authoring instance, pinned in this service's .env. "Updating" it means +# re-pinning to a newer image and recreating the containers. +set -Eeuo pipefail +SERVICE_DIR="${COREKIT_SERVICE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" + +[[ -f "$SERVICE_DIR/.env" ]] || { echo "No .env for this service" >&2; exit 1; } +# shellcheck disable=SC1091 +set -a; . "$SERVICE_DIR/.env"; set +a + +app_image="${N8N_PROD_IMAGE:-}" +runner_image="${N8N_PROD_RUNNER_IMAGE:-}" +[[ -n "$app_image" && -n "$runner_image" ]] || { echo "N8N_PROD_IMAGE and N8N_PROD_RUNNER_IMAGE must be set in .env" >&2; exit 1; } + +# corekit/n8n:-git -> +tag="${app_image##*:}" +target_version="${tag%%-*}" +runner_tag="${runner_image##*:}" +runner_version="${runner_tag%%-*}" + +live_version="unknown" +if docker inspect n8n-prod >/dev/null 2>&1 \ + && [[ "$(docker inspect --format '{{.State.Running}}' n8n-prod)" == "true" ]]; then + live_version="$(docker exec n8n-prod n8n --version 2>/dev/null | tr -d '\r' || echo unknown)" +fi + +identity="$(printf '%s|%s' "$app_image" "$runner_image" | sha256sum | awk '{print $1}')" + +jq -cn --arg t "$target_version" --arg l "$live_version" --arg i "$identity" \ + --arg app "$app_image" --arg runner "$runner_image" --arg rv "$runner_version" \ + '{target_version:$t, live_version:$l, identity:$i, + refs:{app_image:$app, runner_image:$runner, runner_version:$rv}}' diff --git a/services/workflow-automation/n8n-prod/prepare.sh b/services/workflow-automation/n8n-prod/prepare.sh new file mode 100755 index 00000000..a27b5751 --- /dev/null +++ b/services/workflow-automation/n8n-prod/prepare.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Create the data directories with the ownership each image needs, before the +# containers start. n8n runs as uid 1000; the postgres image chowns its own +# PGDATA on first run. +set -e +install -d -m 700 ./data +install -d -o 1000 -g 1000 -m 700 ./data/n8n +install -d -m 700 ./data/db diff --git a/services/workflow-automation/n8n-prod/service.json b/services/workflow-automation/n8n-prod/service.json new file mode 100644 index 00000000..e6b6c705 --- /dev/null +++ b/services/workflow-automation/n8n-prod/service.json @@ -0,0 +1,43 @@ +{ + "name": "n8n-prod", + "display_name": "n8n (production)", + "category": "Workflow Automation", + "rank": 9, + "description": "Production n8n for workflows and credentials that should not sit on an instance where workflows are authored freely. Isolated stack, own Postgres, own tunnel.", + "interface": "n8n-prod..com", + "source": "https://github.com/n8n-io/n8n", + "depends_on": [], + "managed_update": { + "enabled": true, + "deployment_branch": "main", + "update_group": "adopts-authoring-instance-image", + "channel": "stable", + "check_cadence": "daily", + "apply_cadence": "weekly", + "minimum_release_age_days": 7, + "automatic_update_types": [ + "patch", + "minor" + ], + "major_updates_require_manual_approval": true, + "security_override_requires_manual_flag": true, + "backup_required": true, + "stateful_rollback": "guarded", + "recovery_sets_to_keep": 3, + "minimum_free_space_gb": 10, + "components": [ + "n8n-prod", + "n8n-prod-runner" + ], + "note": "Builds nothing. Adopts an image already built and verified for the authoring instance, pinned in this service's .env. An update is a re-pin plus a recreate." + }, + "promotion": { + "enabled": true, + "protected_branch": "main", + "repository_path": "workflows", + "preserve_ids": true, + "backup_required": true, + "credentials": "never", + "note": "Repository and token are site-specific and come from the service .env: PROMOTION_REPOSITORY, PROMOTION_TOKEN. Give the token read access only; the branch protection is the control, not the token." + } +} diff --git a/services/workflow-automation/n8n/Dockerfile b/services/workflow-automation/n8n/Dockerfile index 301a7daa..009f8aad 100644 --- a/services/workflow-automation/n8n/Dockerfile +++ b/services/workflow-automation/n8n/Dockerfile @@ -2,12 +2,12 @@ # renovate: datasource=docker depName=alpine FROM alpine:3.24@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b AS apk-builder -RUN apk add --no-cache apk-tools-static=3.0.7-r0 +RUN apk add --no-cache apk-tools-static=3.0.8-r0 # Keep the readable version and immutable digest together. CI verifies this # version matches the runner image and the managed-update policy. # renovate: datasource=docker depName=n8nio/n8n -FROM n8nio/n8n:2.34.4@sha256:f69bad7a699300702af665b9f13fc67455b93c9fb9174bca6c37f3d4a47b8583 +FROM n8nio/n8n:2.39.6@sha256:1eb33706d9bd902cc83f302ba8d608420dfc0815e03ee4bb1c072367d2c5e259 ARG N8N_GIT_VERSION=v1.2.3 ARG N8N_GIT_COMMIT=5916cfed2561016afdc34c3f31d6ab13f59394eb @@ -25,19 +25,20 @@ USER root # top-level package versions. No secret is accepted as a build argument. COPY --from=apk-builder /sbin/apk.static /usr/local/bin/apk.static RUN /usr/local/bin/apk.static add --no-cache --upgrade \ - libapk=3.0.7-r0 apk-tools=3.0.7-r0 && \ + libapk=3.0.8-r0 apk-tools=3.0.8-r0 && \ rm /usr/local/bin/apk.static && \ apk add --no-cache \ bash=5.3.9-r1 \ - curl=8.21.0-r0 \ + curl=8.22.0-r0 \ + libcurl=8.22.0-r0 \ exiftool=13.55-r0 \ ffmpeg=8.1.2-r0 \ ghostscript=10.07.1-r0 \ - imagemagick=7.1.2.27-r0 \ - jq=1.8.1-r0 \ + imagemagick=7.1.2.30-r0 \ + jq=1.8.2_git20260622-r1 \ mediainfo=26.01-r0 \ py3-pip=26.1.2-r0 \ - python3=3.14.5-r0 \ + python3=3.14.7-r1 \ sox=14.4.2-r14 # Fetch both the installer and its source archive from the immutable release diff --git a/services/workflow-automation/n8n/docker-compose.yml b/services/workflow-automation/n8n/docker-compose.yml index 6bf0ecdb..6096bdf4 100644 --- a/services/workflow-automation/n8n/docker-compose.yml +++ b/services/workflow-automation/n8n/docker-compose.yml @@ -2,7 +2,7 @@ services: n8n: # Managed deployments override this with a version+Git-commit candidate # tag. The fallback remains exact and contains no moving tag. - image: ${N8N_MANAGED_IMAGE:-corekit/n8n:2.34.4-git1.2.3-r1} + image: ${N8N_MANAGED_IMAGE:-corekit/n8n:2.39.6-git1.2.3-r1} build: context: . pull: false @@ -35,7 +35,12 @@ services: N8N_RUNNERS_MODE: external N8N_RUNNERS_TASK_TIMEOUT: "300" N8N_TRUST_PROXY: "true" - N8N_UNVERIFIED_PACKAGES_ENABLED: "true" + N8N_UNVERIFIED_PACKAGES_ENABLED: "false" + # A Code node must not be able to read the process environment: this + # container holds N8N_ENCRYPTION_KEY, which decrypts every credential. + # Matters wherever a client can author workflows: creating a Code node is + # then enough to read the process environment. + N8N_BLOCK_ENV_ACCESS_IN_NODE: "true" N8N_USER_MANAGEMENT_JWT_SECRET: ${N8N_USER_MANAGEMENT_JWT_SECRET} NODES_EXCLUDE: ${NODES_EXCLUDE:-} NODE_ENV: production @@ -63,7 +68,7 @@ services: stop_grace_period: 60s n8n-runner: - image: ${N8N_MANAGED_RUNNER_IMAGE:-corekit/n8n-runners:2.34.4-r1} + image: ${N8N_MANAGED_RUNNER_IMAGE:-corekit/n8n-runners:2.39.6-r1} build: context: ./runner pull: false diff --git a/services/workflow-automation/n8n/managed/build.sh b/services/workflow-automation/n8n/managed/build.sh new file mode 100755 index 00000000..567324e7 --- /dev/null +++ b/services/workflow-automation/n8n/managed/build.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Build the candidate images and verify they are what the bundle asked for. +# +# Built before anything live is stopped, so a build failure never costs +# availability. Tags carry the version and the commit that produced them, so a +# candidate is never confused with the image currently running. +set -Eeuo pipefail + +TARGET_VERSION="${1:?target version required}" +SERVICE_DIR="${COREKIT_SERVICE_DIR:?}" +PROJECT_ROOT="${COREKIT_PROJECT_ROOT:?}" +fail() { echo "$1" >&2; exit 1; } + +info="$("$SERVICE_DIR/managed/version.sh")" +git_version="$(jq -r '.refs.n8n_git_version' <<<"$info")" +git_commit="$(jq -r '.refs.n8n_git_commit' <<<"$info")" + +short_commit="$(git -C "$PROJECT_ROOT" rev-parse --short=12 HEAD)" +image="corekit/n8n:${TARGET_VERSION}-git${short_commit}" +runner_image="corekit/n8n-runners:${TARGET_VERSION}-git${short_commit}" + +docker build --pull=false --tag "$image" "$SERVICE_DIR" >&2 +docker build --pull=false --tag "$runner_image" "$SERVICE_DIR/runner" >&2 + +# Verify the image is the version claimed, rather than trusting the build. +[[ "$(docker run --rm --network none "$image" n8n --version | tr -d '\r')" == "$TARGET_VERSION" ]] \ + || fail "Candidate version verification failed" +[[ "$(docker image inspect --format '{{index .Config.Labels "io.corekit.n8n-git.version"}}' "$image")" == "$git_version" ]] \ + || fail "Candidate n8n-git version verification failed" +[[ "$(docker image inspect --format '{{index .Config.Labels "io.corekit.n8n-git.commit"}}' "$image")" == "$git_commit" ]] \ + || fail "Candidate n8n-git commit verification failed" +[[ "$(docker image inspect --format '{{index .Config.Labels "io.corekit.runner-for-n8n-version"}}' "$runner_image")" == "$TARGET_VERSION" ]] \ + || fail "Candidate runner version verification failed" +[[ "$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.version"}}' "$runner_image")" == "$TARGET_VERSION" ]] \ + || fail "Candidate upstream runner version verification failed" + +jq -cn --arg i "$image" --arg r "$runner_image" '{images:{app:$i, runner:$r}}' diff --git a/services/workflow-automation/n8n/managed/counts.sh b/services/workflow-automation/n8n/managed/counts.sh new file mode 100755 index 00000000..9667d9ed --- /dev/null +++ b/services/workflow-automation/n8n/managed/counts.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Entity counts and in-flight executions for this instance. +# +# This instance's data lives in the shared Postgres server, so the counts come +# from there. A service with its own database reports from its own. +set -Eeuo pipefail +fail() { echo "$1" >&2; exit 1; } +q() { docker exec postgres psql -X -U postgres -d postgres -At -v ON_ERROR_STOP=1 -c "$1" 2>/dev/null | tr -d '[:space:]'; } +# No `|| 0` fallback: a query that fails must abort. Reporting zero would let a +# deployment proceed during a database outage, which is exactly when it must not. +w="$(q 'SELECT count(*) FROM workflow_entity;')" || fail "Could not read workflow count" +c="$(q 'SELECT count(*) FROM credentials_entity;')" || fail "Could not read credential count" +a="$(q "SELECT count(*) FROM execution_entity WHERE status IN ('new','running','waiting','unknown');")" || fail "Could not read active executions" +for v in "$w" "$c" "$a"; do + [[ "$v" =~ ^[0-9]+$ ]] || fail "Database returned a non-numeric count; refusing to report it as zero" +done + +jq -cn \ + --argjson workflows "$w" --argjson credentials "$c" --argjson active "$a" \ + '{workflows:$workflows, credentials:$credentials, active_executions:$active}' diff --git a/services/workflow-automation/n8n/managed/deploy.sh b/services/workflow-automation/n8n/managed/deploy.sh new file mode 100755 index 00000000..e6c828f4 --- /dev/null +++ b/services/workflow-automation/n8n/managed/deploy.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Recreate this service's containers with the candidate images. +# +# Only this service's containers are touched: --no-deps keeps shared +# infrastructure out of the blast radius. +set -Eeuo pipefail + +TARGET_VERSION="${1:?target version required}" +SERVICE_DIR="${COREKIT_SERVICE_DIR:?}" +PROJECT_ROOT="${COREKIT_PROJECT_ROOT:?}" +IMAGES_JSON="${COREKIT_CANDIDATE_IMAGES:-}" +[[ -n "$IMAGES_JSON" ]] || { echo "No candidate images supplied" >&2; exit 1; } + +export N8N_MANAGED_IMAGE="$(jq -r '.images.app' <<<"$IMAGES_JSON")" +export N8N_MANAGED_RUNNER_IMAGE="$(jq -r '.images.runner' <<<"$IMAGES_JSON")" + +# Compose selects nothing unless the profile is named, and WEBHOOK_URL is built +# from N8N_HOSTNAME, which lives in the global config rather than this service's +# .env. Both must be in the environment before compose runs. +export COMPOSE_PROFILES=n8n +if [[ -f "$PROJECT_ROOT/config/.env.global" ]]; then + hostname_value="$(sed -n 's/^N8N_HOSTNAME=//p' "$PROJECT_ROOT/config/.env.global" | tail -n 1)" + hostname_value="${hostname_value%$'\r'}" + hostname_value="${hostname_value#\"}"; hostname_value="${hostname_value%\"}" + hostname_value="${hostname_value#\'}"; hostname_value="${hostname_value%\'}" + [[ -n "$hostname_value" ]] && export N8N_HOSTNAME="$hostname_value" +fi + +project_name="$(docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' n8n 2>/dev/null || true)" +[[ -n "$project_name" ]] || project_name=localai + +# Drain any workers before replacing the main container. +mapfile -t old_workers < <(docker ps -a --filter label=com.docker.compose.service=n8n-worker --format '{{.Names}}' | sort) +if (( ${#old_workers[@]} > 0 )); then + docker stop --time 60 "${old_workers[@]}" >/dev/null +fi + +docker compose -p "$project_name" --project-directory "$SERVICE_DIR" \ + --env-file "$PROJECT_ROOT/services/data-services/postgres/.env" \ + --env-file "$SERVICE_DIR/.env" -f "$SERVICE_DIR/docker-compose.yml" \ + up -d --no-deps n8n n8n-runner >&2 + +if (( ${#old_workers[@]} > 0 )); then + docker rm "${old_workers[@]}" >/dev/null 2>&1 || true +fi diff --git a/services/workflow-automation/n8n/managed/strict-healthcheck.sh b/services/workflow-automation/n8n/managed/strict-healthcheck.sh index 2225d1da..8399dd5b 100755 --- a/services/workflow-automation/n8n/managed/strict-healthcheck.sh +++ b/services/workflow-automation/n8n/managed/strict-healthcheck.sh @@ -125,6 +125,15 @@ if [[ "$RUN_CANARY" == "true" ]]; then install -d -m 0700 "$STATE_ROOT" chmod 0700 "$STATE_ROOT" CANARY_OUTPUT="$(mktemp "$STATE_ROOT/.canary.XXXXXX")" + # Run the candidate, not whatever the compose fallback tag resolves to. Without + # this the canary proves a stale image can execute a workflow, and on a host + # where that tag is absent it fails a deployment that actually succeeded. + if [[ -n "${COREKIT_CANDIDATE_IMAGES:-}" ]]; then + export N8N_MANAGED_IMAGE="$(jq -r '.images.app // empty' <<<"$COREKIT_CANDIDATE_IMAGES")" + export N8N_MANAGED_RUNNER_IMAGE="$(jq -r '.images.runner // empty' <<<"$COREKIT_CANDIDATE_IMAGES")" + fi + export COMPOSE_PROFILES=n8n + PROJECT_NAME="$(docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' n8n 2>/dev/null || true)" [[ -n "$PROJECT_NAME" ]] || PROJECT_NAME=localai if ! docker compose -p "$PROJECT_NAME" --project-directory "$SERVICE_DIR" \ diff --git a/services/workflow-automation/n8n/managed/validate.sh b/services/workflow-automation/n8n/managed/validate.sh new file mode 100755 index 00000000..092add15 --- /dev/null +++ b/services/workflow-automation/n8n/managed/validate.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Refuse a committed bundle that could not be deployed reproducibly. +set -Eeuo pipefail + +SERVICE_DIR="${COREKIT_SERVICE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +fail() { echo "$1" >&2; exit 1; } + +info="$("$SERVICE_DIR/managed/version.sh")" +target="$(jq -r '.target_version' <<<"$info")" +runner_version="$(jq -r '.refs.runner_version' <<<"$info")" +n8n_digest="$(jq -r '.refs.n8n_digest' <<<"$info")" +runner_digest="$(jq -r '.refs.runner_digest' <<<"$info")" +git_version="$(jq -r '.refs.n8n_git_version' <<<"$info")" +git_commit="$(jq -r '.refs.n8n_git_commit' <<<"$info")" + +[[ "$target" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "Pinned version is invalid" +[[ "$target" == "$runner_version" ]] || fail "Pinned application and runner versions differ" +[[ "$n8n_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "Pinned application digest is invalid" +[[ "$runner_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "Pinned runner digest is invalid" +[[ "$git_version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "Pinned n8n-git version is invalid" +[[ "$git_commit" =~ ^[0-9a-f]{40}$ ]] || fail "Pinned n8n-git commit is invalid" + +jq -e . "$SERVICE_DIR/service.json" "$SERVICE_DIR/task-runners-managed.json" >/dev/null \ + || fail "Managed JSON configuration is invalid" + +# A moving tag anywhere in the bundle defeats the point of pinning digests. +if grep -ERn '(^|[[:space:]:])latest([@[:space:]]|$)' \ + "$SERVICE_DIR/Dockerfile" "$SERVICE_DIR/runner/Dockerfile" \ + "$SERVICE_DIR/docker-compose.yml" >/dev/null; then + fail "Moving 'latest' reference found in the managed bundle" +fi diff --git a/services/workflow-automation/n8n/managed/version.sh b/services/workflow-automation/n8n/managed/version.sh new file mode 100755 index 00000000..a4ef6349 --- /dev/null +++ b/services/workflow-automation/n8n/managed/version.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Report the committed and live versions of this service, plus an identity that +# changes whenever the committed build would change. +# +# The versions and digests are read from the Dockerfiles, which are the only +# place a version is declared for this service. +set -Eeuo pipefail + +SERVICE_DIR="${COREKIT_SERVICE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" + +n8n_ref="$(awk '$1 == "FROM" && $2 ~ /^n8nio\/n8n:/ {print $2; exit}' "$SERVICE_DIR/Dockerfile")" +runner_ref="$(awk '$1 == "FROM" && $2 ~ /^n8nio\/runners:/ {print $2; exit}' "$SERVICE_DIR/runner/Dockerfile")" + +target_version="${n8n_ref#*:}"; target_version="${target_version%%@*}" +target_digest="${n8n_ref##*@}" +runner_version="${runner_ref#*:}"; runner_version="${runner_version%%@*}" +runner_digest="${runner_ref##*@}" +git_version="$(awk -F= '$1 == "ARG N8N_GIT_VERSION" {print $2}' "$SERVICE_DIR/Dockerfile")" +git_commit="$(awk -F= '$1 == "ARG N8N_GIT_COMMIT" {print $2}' "$SERVICE_DIR/Dockerfile")" + +live_version="unknown" +if docker inspect n8n >/dev/null 2>&1 \ + && [[ "$(docker inspect --format '{{.State.Running}}' n8n)" == "true" ]]; then + live_version="$(docker exec n8n n8n --version 2>/dev/null | tr -d '\r' || echo unknown)" +fi + +# Identity covers everything that would change the built image. +identity="$(printf '%s|%s|%s|%s' "$n8n_ref" "$runner_ref" "$git_version" "$git_commit" \ + | sha256sum | awk '{print $1}')" + +jq -cn \ + --arg target "$target_version" --arg live "$live_version" --arg identity "$identity" \ + --arg n8n_digest "$target_digest" --arg runner_version "$runner_version" \ + --arg runner_digest "$runner_digest" --arg git_version "$git_version" --arg git_commit "$git_commit" \ + '{target_version:$target, live_version:$live, identity:$identity, + refs:{n8n_digest:$n8n_digest, runner_version:$runner_version, + runner_digest:$runner_digest, n8n_git_version:$git_version, + n8n_git_commit:$git_commit}}' diff --git a/services/workflow-automation/n8n/runner/Dockerfile b/services/workflow-automation/n8n/runner/Dockerfile index 45ebab82..27952990 100644 --- a/services/workflow-automation/n8n/runner/Dockerfile +++ b/services/workflow-automation/n8n/runner/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # renovate: datasource=docker depName=n8nio/runners -FROM n8nio/runners:2.34.4@sha256:908096845a12c8e3a9214b8f93d72b745036607fe67cda06b4b2bab0f43e0777 +FROM n8nio/runners:2.39.6@sha256:b6d97c54f9ee6e2227351b551f92e457e17891ab622d038277dd0c70a3f28cca LABEL io.corekit.managed-service="n8n" \ - io.corekit.runner-for-n8n-version="2.34.4" + io.corekit.runner-for-n8n-version="2.39.6" # The upstream image already runs as the unprivileged UID/GID 1000 runner. # Extra packages remain absent until an active production workflow demonstrates diff --git a/tests/test-shell-resolvable.sh b/tests/test-shell-resolvable.sh new file mode 100755 index 00000000..5b32e42a --- /dev/null +++ b/tests/test-shell-resolvable.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Every command called in command position must resolve to something. +# +# Exists because a refactor deleted a function while leaving its call site in a +# branch no test reached: `corekit managed-update apply` aborted with exit 127 +# after doing twenty minutes of expensive work. `bash -n` cannot see it (the +# syntax is valid) and shellcheck does not report undefined functions. +# +# Approach: build an inventory of every function defined anywhere in the +# repository's shell files, then check that each bare identifier used in command +# position resolves to that inventory, a shell builtin, or an executable on PATH. +# Repository-wide rather than per-file, because these scripts source their +# helpers at runtime; the bug worth catching is "defined nowhere at all". +# +# Limits worth knowing: +# - It only flags a command alone on its own line with no arguments. A deleted +# function called as `foo "$bar"` still slips through. +# - It cannot see a command assembled at runtime. +# - It resolves names against the PATH of whatever runs it, so a script naming +# a host-only tool will fail on a CI runner that lacks it. +# It does catch the case above, which is the one that bit us. +set -Eeuo pipefail +IFS=$'\n\t' + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FAILURES=0 + +# Keywords of interpreters embedded as quoted programs (awk, sed). The +# quote-tracking below catches most such blocks, but an apostrophe inside an +# earlier double-quoted shell string breaks quote parity for the rest of the +# file. These words are never shell commands, so excluding them costs no signal. +EMBEDDED_KEYWORDS='next|getline|delete|nextfile|END|BEGIN' + +KEYWORDS='if|then|else|elif|fi|for|while|until|do|done|case|esac|in|function|select|time|coproc|return|break|continue|exit|local|declare|readonly|export|unset|shift|eval|exec|source|trap|set|shopt|read|echo|printf|test|true|false|cd|pwd|umask|wait|kill|jobs|command|builtin|mapfile|readarray|let|getopts|hash|type|alias|dirs|popd|pushd|enable|history|bind|fc|caller|compgen|complete|ulimit|times' + +shell_files() { + find "$PROJECT_ROOT/lib" "$PROJECT_ROOT/services" "$PROJECT_ROOT/tests" -name '*.sh' -type f 2>/dev/null \ + | grep -vE '/(build|repo|node_modules)/' + printf '%s\n' "$PROJECT_ROOT/corekit.sh" "$PROJECT_ROOT/install.sh" +} + +# Strip comments and heredoc bodies, both of which contain prose that looks like +# commands. Heredocs are skipped from their opener to their terminator. +strip_noise() { + awk ' + /^[[:space:]]*#/ { next } + match($0, /<<-?[[:space:]]*'"'"'?"?[A-Za-z_][A-Za-z0-9_]*'"'"'?"?/) { + tag = substr($0, RSTART, RLENGTH) + gsub(/^<<-?[[:space:]]*['"'"'"]?|['"'"'"]?$/, "", tag) + inheredoc = 1; hdtag = tag; print ""; next + } + inheredoc { if ($0 ~ "^[[:space:]]*" hdtag "[[:space:]]*$") inheredoc = 0; print ""; next } + { sub(/[[:space:]]#[^"'"'"']*$/, ""); print } + ' "$1" +} + +echo "Building function inventory..." +INVENTORY="$(mktemp)"; trap 'rm -f "$INVENTORY"' EXIT +while IFS= read -r f; do + [[ -f "$f" ]] || continue + { strip_noise "$f" \ + | grep -oE '^[[:space:]]*(function[[:space:]]+)?[a-zA-Z_][a-zA-Z0-9_]*[[:space:]]*\(\)' \ + | grep -oE '[a-zA-Z_][a-zA-Z0-9_]*' | grep -vxE 'function'; } || true +done < <(shell_files) | sort -u > "$INVENTORY" +echo " $(wc -l < "$INVENTORY") functions defined across the repository" +echo + +while IFS= read -r file; do + [[ -f "$file" ]] || continue + rel="${file#"$PROJECT_ROOT"/}" + unresolved=() + while IFS= read -r word; do + [[ -z "$word" ]] && continue + [[ "$word" =~ ^($KEYWORDS)$ ]] && continue + [[ "$word" =~ ^($EMBEDDED_KEYWORDS)$ ]] && continue + grep -qxF "$word" "$INVENTORY" && continue + command -v "$word" >/dev/null 2>&1 && continue + unresolved+=("$word") + done < <( + { strip_noise "$file" | awk ' + # A bare word on its own line is only a command if the previous line did + # not continue onto it. Array literals and wrapped argument lists put + # bare words on their own lines too, and those are operands. + function trim(x) { gsub(/^[[:space:]]+|[[:space:]]+$/, "", x); return x } + { + # Embedded programs (awk, sed, python) are passed as single-quoted + # strings spanning many lines. Their keywords are not shell commands. + q = gsub(/'"'"'/, "'"'"'", $0) + if (inquote) { if (q % 2 == 1) inquote = 0; next } + if (q % 2 == 1) { inquote = 1; next } + line = $0 + if (cont) { cont = (line ~ /\\$/) || (open > 0); next_is_operand = 1 } + else { next_is_operand = 0 } + n = gsub(/\(/, "(", line); m = gsub(/\)/, ")", line) + open += n - m; if (open < 0) open = 0 + if (!next_is_operand && open == 0 && trim($0) ~ /^[a-zA-Z_][a-zA-Z0-9_]*$/) + print trim($0) + if ($0 ~ /\\$/) cont = 1 + } + ' | sort -u; } || true + ) + if (( ${#unresolved[@]} > 0 )); then + echo "FAIL $rel" + printf ' unresolved: %s\n' "${unresolved[@]}" + FAILURES=$((FAILURES + ${#unresolved[@]})) + fi +done < <(shell_files | sort) + +if (( FAILURES > 0 )); then + echo + echo "$FAILURES unresolved command(s). A call with no definition aborts the script at runtime." + exit 1 +fi +echo "All commands in command position resolve."