diff --git a/.gitignore b/.gitignore index 41099ea7..81176990 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,10 @@ Thumbs.db .env.local .env.*.local +# Local Compose customisation (host-specific, not part of the project) +docker-compose.override.yml +docker-compose.override.yaml + # State files (local to each machine) .claude/ralph-loop.local.md diff --git a/CLAUDE.md b/CLAUDE.md index a2fdb8de..935ee3b0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -211,7 +211,7 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph **Docker cases**: a case can point at a **container**, with any of the CLI run modes running inside it. Like remote-SSH this is a **LOCATION OVERLAY on cases, never a `SessionMode` of its own**. Exactly one long-lived container **per case**, shared by all its sessions, so killing a session kills only that session's in-container tmux and **never** `docker stop` while siblings remain. The workspace is a real host dir bind-mounted at the **same absolute path**, which is what keeps file-routes/watchers on real host bytes and makes the in-container transcript projHash match the host. Credentials are **seeded** (RO mount, copied into the container once) rather than shared RW, so in-container CLIs never write refreshed tokens back to the host, and bind mounts are excluded from `docker commit` so exports stay secret-free. **NEVER a create-time `-e` for secrets, NEVER `--privileged`, NEVER the docker socket.** Config drift is detected via a label hash and a drifted launch is REFUSED rather than silently launched with stale config. ⚠️ On the loopback-only prod bind a container cannot reach 127.0.0.1, so in-container hooks need `CODEMAN_DOCKER_BRIDGE_HOOKS=1`; otherwise idle detection falls back to output-based. → [architecture-invariants#docker-cases](docs/architecture-invariants.md#docker-cases), `docs/docker-cases.md` (user guide), `docs/docker-cases-plan.md` (design) -**Docker Compose deployment** (`docker/`, contributed): Codeman itself runs in a container and spawns Docker cases as **SIBLING** containers through the mounted host socket (Docker-outside-of-Docker), never nested. That inverts one assumption the bare-host path takes for granted: the daemon no longer shares Codeman's filesystem, so a bind source valid *inside* Codeman means nothing to it. `resolveDockerDaemonMountSource()` translates sources under HOME into the daemon's namespace via `CODEMAN_DOCKER_HOST_HOME`, and `CODEMAN_CASES_PATH` points the cases dir at a host-absolute bind mount so a workspace resolves to the SAME absolute path on both sides (which is what keeps the transcript projHash matching, per Docker cases above). ⚠️ **`CODEMAN_CASES_PATH` must move every consumer or none**: it is resolved once in `config/cases-dir.ts` because `src/cli.ts` resolves case paths too, and when only the server's `CASES_DIR` learned the override, `codeman skill install --case ` reported "Case not found" on exactly the deployment the override exists for. ⚠️ **`.dockerignore` patterns match the WHOLE context-relative path**, so a bare `.env` line excludes only the ROOT file: `docker/.env` (which holds `CODEMAN_PASSWORD` and any provider keys) rode `COPY . .` into the image until `**/.env` was added — verified in both directions with a real build context. ⚠️ A Compose LONG-form bind (`type: bind`) **creates a missing host source directory ROOT-OWNED** rather than refusing, so any bind source the runtime user must write to has to be pre-created and chowned. `CODEMAN_DOCKER_DISABLE_SWAP_LIMIT=1` drops `--memory-swap` (and filters only that one kernel warning) for hosts without swap accounting; `--memory` still applies. ⚠️ The deployment ALSO self-updates in place (the repo bind mount at `/opt/codeman` + a restart-by-exiting supervisor) — see Self-update below and `docs/docker-self-update.md` before touching `server.Dockerfile`, the compose file or `.env.example`, since each is an input to the updater's environment gate. `docs/docker-compose.md` + `docker/README.md` (user guides) +**Docker Compose deployment** (`docker/`, contributed): Codeman itself runs in a container and spawns Docker cases as **SIBLING** containers through the mounted host socket (Docker-outside-of-Docker), never nested. That inverts one assumption the bare-host path takes for granted: the daemon no longer shares Codeman's filesystem, so a bind source valid *inside* Codeman means nothing to it. `resolveDockerDaemonMountSource()` translates sources under HOME into the daemon's namespace via `CODEMAN_DOCKER_HOST_HOME`, and `CODEMAN_CASES_PATH` points the cases dir at a host-absolute bind mount so a workspace resolves to the SAME absolute path on both sides (which is what keeps the transcript projHash matching, per Docker cases above). ⚠️ **`CODEMAN_CASES_PATH` must move every consumer or none**: it is resolved once in `config/cases-dir.ts` because `src/cli.ts` resolves case paths too, and when only the server's `CASES_DIR` learned the override, `codeman skill install --case ` reported "Case not found" on exactly the deployment the override exists for. ⚠️ **`.dockerignore` patterns match the WHOLE context-relative path**, so a bare `.env` line excludes only the ROOT file: `docker/.env` (which holds `CODEMAN_PASSWORD` and any provider keys) rode `COPY . .` into the image until `**/.env` was added — verified in both directions with a real build context. ⚠️ A Compose LONG-form bind (`type: bind`) **creates a missing host source directory ROOT-OWNED** rather than refusing. `Start-Codeman.sh` pre-creates both `CODEMAN_APPDATA_PATH` and `CODEMAN_CASES_PATH` on the host before `up`, which is what keeps the daemon from ever having to materialise either as root in the first place; the container ALSO starts as root (`cap_add: [CHOWN, DAC_OVERRIDE, SETGID, SETUID]` against the base `cap_drop: ALL`) so `docker/entrypoint.sh` can correct a bind source that turns up root-owned anyway (a restored backup, a cleared directory, plain `docker compose up` run without the script) before dropping to `PUID:PGID` via `setpriv` — it refuses instead of chowning a directory owned by neither root nor `PUID:PGID`, since that ownership is not this container's to reassign. `CODEMAN_DOCKER_DISABLE_SWAP_LIMIT=1` drops `--memory-swap` (and filters only that one kernel warning) for hosts without swap accounting; `--memory` still applies. ⚠️ The deployment ALSO self-updates in place (the repo bind mount at `/opt/codeman` + a restart-by-exiting supervisor) — see Self-update below and `docs/docker-self-update.md` before touching `server.Dockerfile`, the compose file or `.env.example`, since each is an input to the updater's environment gate. `docs/docker-compose.md` + `docker/README.md` (user guides) **CLI registry** (`src/config/cli-registry/`): every run mode is a `CliEntry` — discovery (search dirs, version + identity probes), the launch argv template, env handling, the `capabilities` flags that replace per-CLI branching, and the `overlays` that back the remote/docker pane commands. **No code outside `stock.ts` may branch on a CLI id**; behaviour that genuinely differs is either a capability field or a NAMED PROFILE selected by one (`profiles.ts`), and `test/cli-registry-no-id-branching.test.ts` fails the build if an id check reappears — it matches `===`, `!==`, `case '':` and `[...].includes(mode)`, because an earlier `===`-only version let 36 negated branches survive the conversion (including a seven-mode Ralph chain whose own comment asked the next person to keep it in step with `isExternalCliMode()` by hand). ⚠️ Config contains no shell text: an entry declares typed argv tokens, literals are validated against a safe-word pattern at LOAD time (a bad literal rejects the whole entry — a silently dropped `--no-approve` is not cosmetic), and values resolve through patterns NAMED in code, so a user `clis.json` cannot widen its own validation. ⚠️ `external`, `hooks` and `altScreen` are three INDEPENDENT capabilities on purpose; deriving one from another shipped the `until=stop`-hangs-on-shell bug. ⚠️ **`param` is TWO namespaces.** `launch.params` keys, `env.configSetenv[].fromParam` and `capabilities.privilegedParams[].param` all name a LAUNCH PARAM; the legacy `Config` wire field is a separate namespace, bridged only by `launch.legacyConfigAliases`. Getting `privilegedParams[].param` wrong is SILENT — it is the multi-user bypass clamp's only handle on a CLI's privilege switch, and a wrong name clamps nothing with no load error and no failing test — so `schema.ts` rejects an entry naming a param it never declared. Codex is the entry where the two names differ (`bypassApprovals` vs `dangerouslyBypassApprovals`) and therefore the one that catches a regression. ⚠️ Six fields are DECLARED-FOR-LATER and read by nothing (`shortBadge`, `accent`, `capabilities.echo`/`wheelForward`/`keyboardAccessory`/`maxFrameBytes`): all frontend behaviour, transcribed rather than measured, so re-measure before wiring one up; the list is pinned so it cannot quietly grow. Spawn commands are pinned as literal strings in `test/cli-registry-spawn-golden.test.ts`, remote/docker pane commands in `test/location-overlay-commands.test.ts`. ⚠️ Anything reading the registry resolves it AT CALL TIME (`sessionModeSchema()`, `allowedEnvPrefixes()`, `dependencyRegistry()`, the resolvers' `searchDirs` thunks) — a module-level const freezes at first import, so a CLI enabled while the server ran moved the run menu but not that surface. `~/.codeman/clis.json` overrides any entry (read-only in this release; nothing writes it, so importing the registry has no filesystem side effects). → `docs/cli-registry.md` @@ -376,7 +376,7 @@ Frontend JS modules have `@fileoverview` with `@dependency`/`@loadorder` tags. L ## State Files -All in `~/.codeman/`: `state.json` (sessions, settings, respawn, orchestrator, cron jobs/runs, owner tab layouts), `mux-sessions.json` (tmux recovery), `settings.json` (user prefs), `push-keys.json` + `push-subscriptions.json`, `session-lifecycle.jsonl` (audit log), `update-status.json` (self-updater progress, polled across the service restart), `docker-env-applied.json` (Compose deployment only: sha256 of the Dockerfile + compose file the running container was built from, written by `Start-Codeman.sh`, read by the self-updater's environment gate), `linked-cases.json`, `webviews.json` (saved web-tab dashboard URLs), `remote-hosts.json` + `remote-cases.json`, `docker-hosts.json` + `docker-cases.json` + `docker-exports/`, `subagent-window-states.json` + `subagent-parents.json` (subagent window layout, GET/PUT `/api/subagent-window-states`/`-parents`), `hook-secret` (per-instance), `users.json` (multi-user, mode 0600) + `admin-audit.jsonl`, `intents.json` (Read My Mind intent profiles, mode 0600), `certs/` (self-signed TLS for `--https`), `.env` (CODEMAN_USERNAME/PASSWORD fallback for the `codeman attach` CLI). Transient: `self-update-runner.sh`. Multi-user case spaces live OUTSIDE the data dir at `~/codeman-users//cases` (shared across instances like `~/codeman-cases`, override `CODEMAN_USER_SPACES_DIR`). +All in `~/.codeman/`: `state.json` (sessions, settings, respawn, orchestrator, cron jobs/runs, owner tab layouts), `mux-sessions.json` (tmux recovery), `settings.json` (user prefs), `push-keys.json` + `push-subscriptions.json`, `session-lifecycle.jsonl` (audit log), `update-status.json` (self-updater progress, polled across the service restart), `docker-env-applied.json` (Compose deployment only: sha256 of the Dockerfile + compose file the running container was built from, written by `Start-Codeman.sh`, read by the self-updater's environment gate), `docker-build-source.json` (Compose deployment only: the checkout's HEAD commit and `package-lock.json` hash the `codeman-node-modules`/`codeman-dist` volumes currently reflect, written by both `Start-Codeman.sh` and a successful in-place self-update, compared to detect and refresh a volume left stale by an externally-triggered rebuild), `linked-cases.json`, `webviews.json` (saved web-tab dashboard URLs), `remote-hosts.json` + `remote-cases.json`, `docker-hosts.json` + `docker-cases.json` + `docker-exports/`, `subagent-window-states.json` + `subagent-parents.json` (subagent window layout, GET/PUT `/api/subagent-window-states`/`-parents`), `hook-secret` (per-instance), `users.json` (multi-user, mode 0600) + `admin-audit.jsonl`, `intents.json` (Read My Mind intent profiles, mode 0600), `certs/` (self-signed TLS for `--https`), `.env` (CODEMAN_USERNAME/PASSWORD fallback for the `codeman attach` CLI). Transient: `self-update-runner.sh`. Multi-user case spaces live OUTSIDE the data dir at `~/codeman-users//cases` (shared across instances like `~/codeman-cases`, override `CODEMAN_USER_SPACES_DIR`). **Generated top-level dirs** (all gitignored — don't edit or commit): `dist/` (esbuild output), `out/`, `coverage/`, `test-results/`, `tmp/`, `screenshots-echo-diag/`. The committed gesture bundle (`src/web/public/gesture/gesture-codeman.js`) IS tracked, but its runtime wasm/model assets (`src/web/public/gesture/wasm/`, `*.task`) are fetched and gitignored. diff --git a/docker/.env.example b/docker/.env.example index 70b128f7..1027b955 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -13,24 +13,24 @@ TZ=Australia/Perth # Name of the account that runs Codeman and all local CLI sessions. Changing # this value rebuilds the image with a matching account. -CODEMAN_RUNTIME_USER=opencode +CODEMAN_RUNTIME_USER=codeman # Required. Persistent Codeman application data, CLI credentials, and session # state are stored here on the host and mounted at the runtime account's home # directory in the container. -CODEMAN_APPDATA_PATH=/mnt/user/appdata/Coding/codeman +CODEMAN_APPDATA_PATH=/mnt/user/appdata/codeman # Optional. Absolute host path of this Codeman checkout, mounted at # /opt/codeman so App Settings -> Updates can update Codeman in place. The Bash # start script detects it from the compose file's own location, so it only needs # setting for direct `docker compose` use or a checkout kept elsewhere. Point it # at a directory that is not a git checkout and in-app updates are unavailable. -# CODEMAN_REPO_PATH=/mnt/user/appdata/Coding/codeman/app +# CODEMAN_REPO_PATH=/mnt/user/appdata/codeman/app # Required for Docker cases. This must be an absolute path on the Docker host. # Codeman and each isolated case use this same path, so it cannot be a -# container-only path such as /home/opencode/codeman-cases. -CODEMAN_CASES_PATH=/mnt/user/appdata/Coding/codeman/codeman-cases +# container-only path such as /home/codeman/codeman-cases. +CODEMAN_CASES_PATH=/mnt/user/appdata/codeman/codeman-cases # Required. Network bind address, host port, and local image tag. CODEMAN_HOST=0.0.0.0 diff --git a/docker/README.md b/docker/README.md index e03115e8..7d3a1e84 100644 --- a/docker/README.md +++ b/docker/README.md @@ -22,7 +22,7 @@ Every required value is defined and explained in `.env.example`. `GEMINI_API_KEY On Linux, `Start-Codeman.sh` stops with an error when required paths are missing. It creates the application-data directory when safe, detects its numeric owner as `PUID:PGID`, and detects `DOCKER_SOCKET_GID` from the configured Docker socket. It rejects a root-owned application-data directory because Codeman and its local CLI sessions must remain unprivileged. -Codeman, Claude, OpenCode, and other local sessions run as the unprivileged account named by `CODEMAN_RUNTIME_USER`, which defaults to `opencode`. When Compose is run directly, `PUID` and `PGID` default to `1000:1000`; set them in `.env` when the application-data directory has a different owner. The Bash start script determines them automatically instead. +Codeman, Claude, OpenCode, and other local sessions run as the unprivileged account named by `CODEMAN_RUNTIME_USER`, which defaults to `codeman`. When Compose is run directly, `PUID` and `PGID` default to `1000:1000`; set them in `.env` when the application-data directory has a different owner. The Bash start script determines them automatically instead. To retain Docker-case support without root when running Compose directly, set `DOCKER_SOCKET_GID` to the numeric group ID of the host socket. On a standard Linux Docker host, obtain it with `stat -c '%g' /var/run/docker.sock`. The Bash start script detects it automatically. @@ -38,6 +38,50 @@ Releases that change `server.Dockerfile`, `docker-compose.yaml`, or add a key to changed, and asks you to run `Start-Codeman.sh` here on the host instead. Details: [`../docs/docker-self-update.md`](../docs/docker-self-update.md). +## Local customisation + +Compose merges `docker-compose.override.yml` on top of `docker-compose.yaml`. Keep host-specific changes there rather than editing `docker-compose.yaml`, so this repository can be updated without losing them. Both `docker-compose.override.yml` and `docker-compose.override.yaml` are ignored by Git. + +`Start-Codeman.sh` names the Compose file explicitly, which disables Compose's automatic discovery of the override file, so the script adds it back when one is present and prints the file it used. Running `docker compose` from this folder without any `-f` option finds it automatically. When passing `-f docker/docker-compose.yaml` from the repository root, add `-f docker/docker-compose.override.yml` as well, or the override is silently ignored. + +An override file adds to and replaces individual settings. It cannot delete a key from `docker-compose.yaml`, and Compose concatenates rather than replaces `ports`, so removing a published port still requires editing `docker-compose.yaml`. The example below replaces the restart policy and adds a mount, leaving every other setting in place: + +```yaml +services: + codeman: + restart: always + volumes: + - /srv/projects:/srv/projects +``` + +### Reverse-proxy host allowlist + +Codeman rejects any request whose `Host` header is not on its own allowlist - a +DNS-rebinding guard, not a Compose or Docker concern. Loopback, any IP literal, +the configured `--host`, and a few tunnel-provider suffixes are allowed by +default; a reverse-proxied domain is not, and is rejected with +`403 Forbidden: host not allowed` before the request reaches any handler. + +Add the domain with `CODEMAN_ALLOWED_HOSTS` in `.env`: + +```sh +CODEMAN_ALLOWED_HOSTS='codeman.example.com,.internal.example.com' +``` + +`docker-compose.yaml` does not forward this variable into the container - it +only passes through the environment keys it explicitly lists, and this is not +one of them. Forward it yourself in `docker-compose.override.yml`: + +```yaml +services: + codeman: + environment: + CODEMAN_ALLOWED_HOSTS: ${CODEMAN_ALLOWED_HOSTS} +``` + +See the application's own `docs/wiki/Remote-Access.md` for the full allowlist +format and the tunnel providers it accepts by default. + ## Application data storage The default configuration uses a host-folder bind mount: @@ -49,7 +93,7 @@ volumes: target: /home/${CODEMAN_RUNTIME_USER} ``` -Set `CODEMAN_APPDATA_PATH` in `.env` to a directory that the Docker daemon can access. The example value is `/mnt/user/appdata/Coding/codeman`. +Set `CODEMAN_APPDATA_PATH` in `.env` to a directory that the Docker daemon can access. The example value is `/mnt/user/appdata/codeman`. `CODEMAN_CASES_PATH` is the separate host directory for managed case workspaces. It is mounted into Codeman at the same absolute path, allowing the host Docker daemon to bind it into an isolated case container. Set it to a child directory of `CODEMAN_APPDATA_PATH` unless you deliberately store workspaces elsewhere. @@ -60,7 +104,7 @@ Set `CODEMAN_DOCKER_DISABLE_SWAP_LIMIT=1` when `docker info` reports `SwapLimit= For an existing installation created by a root-running image, change ownership of the application-data directory before upgrading so the configured `PUID` and `PGID` can read the saved credentials and state: ```sh -chown -R 99:100 /mnt/user/appdata/Coding/codeman +chown -R 99:100 /mnt/user/appdata/codeman ``` Replace `99:100` and the path with the values from your `.env` file. @@ -69,7 +113,7 @@ Do not replace this bind mount with a Docker-managed named volume when Docker ca ## Static macvlan networking -The default configuration publishes a host port. It does not use `network_mode: host`. To attach Codeman directly to an existing external macvlan network with a static IP address and MAC address, remove the `ports:` section and add the following to the `codeman` service: +The default configuration publishes a host port. It does not use `network_mode: host`. To attach Codeman directly to an existing external macvlan network with a static IP address and MAC address, remove the `ports:` section from `docker-compose.yaml` and add the following to the `codeman` service. The service and network additions can instead be placed in `docker-compose.override.yml`, but the `ports:` removal cannot, as described under [Local customisation](#local-customisation): ```yaml mac_address: ${CODEMAN_MAC_ADDRESS} diff --git a/docker/Start-Codeman.sh b/docker/Start-Codeman.sh index c0294fe4..84c6b211 100644 --- a/docker/Start-Codeman.sh +++ b/docker/Start-Codeman.sh @@ -12,11 +12,34 @@ if [[ ! -f "$env_file" ]]; then exit 1 fi -compose_command=(docker compose --env-file "$env_file" -f "$compose_file") +# Naming a Compose file explicitly disables Compose's automatic discovery of +# the override file, so it has to be added back by hand. Without this, local +# customisation in docker-compose.override.yml is silently ignored. The +# candidates are checked in Compose's own precedence order - measured on +# Compose v5.5.0 with both present: it uses `.yml` and ignores `.yaml`. +override_yml="$script_dir/docker-compose.override.yml" +override_yaml="$script_dir/docker-compose.override.yaml" +if [[ -f "$override_yml" && -f "$override_yaml" ]]; then + printf 'Warning: both %s and %s exist; Compose uses .yml and ignores .yaml.\n' \ + "$override_yml" "$override_yaml" >&2 +fi +compose_files=(-f "$compose_file") +for override_file in "$override_yml" "$override_yaml"; do + if [[ -f "$override_file" ]]; then + compose_files+=(-f "$override_file") + printf 'Using Compose override file: %s\n' "$override_file" + break + fi +done +compose_command=(docker compose --env-file "$env_file" "${compose_files[@]}") appdata_path=$( "${compose_command[@]}" config --environment | awk -F= '$1 == "CODEMAN_APPDATA_PATH" { sub(/^[^=]*=/, ""); print; exit }' ) +cases_path=$( + "${compose_command[@]}" config --environment | + awk -F= '$1 == "CODEMAN_CASES_PATH" { sub(/^[^=]*=/, ""); print; exit }' +) docker_socket=$( "${compose_command[@]}" config --environment | awk -F= '$1 == "DOCKER_SOCKET" { sub(/^[^=]*=/, ""); print; exit }' @@ -36,6 +59,26 @@ if [[ ! -d "$appdata_path" ]]; then mkdir -p -- "$appdata_path" fi +if [[ -z "$cases_path" ]]; then + printf 'Error: CODEMAN_CASES_PATH is not set in %s\n' "$env_file" >&2 + exit 1 +fi + +# Pre-creating this here, exactly like CODEMAN_APPDATA_PATH above, means Compose +# never has to materialise a missing bind source itself - which it does as +# root:root - so the in-container entrypoint's chown never has to run for this +# path at all. Unlike appdata, an EXISTING cases directory is left exactly as +# it is: the README explicitly allows pointing this at a normal projects +# directory the host account already owns, so no ownership check happens here. +if [[ ! -d "$cases_path" ]]; then + if [[ "$EUID" == '0' ]]; then + printf 'Error: Refusing to create CODEMAN_CASES_PATH as root: %s\n' "$cases_path" >&2 + printf 'Create it as the unprivileged account that should run Codeman, then retry.\n' >&2 + exit 1 + fi + mkdir -p -- "$cases_path" +fi + if owner_ids=$(stat -c '%u:%g' -- "$appdata_path" 2>/dev/null); then : elif owner_ids=$(stat -f '%u:%g' "$appdata_path" 2>/dev/null); then @@ -92,6 +135,31 @@ if [[ ! -d "$repo_path/.git" ]]; then printf 'Note: %s is not a git checkout, so in-app updates are unavailable.\n' "$repo_path" >&2 fi +# Reads HEAD without requiring a `git` binary on the host — this script +# otherwise checks the checkout only by testing for `.git` as a directory, and +# resolving refs by hand keeps that the same "no host git needed" guarantee. +# ⚠️ A worktree checkout has `.git` as a FILE (`gitdir: `), not a +# directory, so this returns nothing there and the volume-refresh check below +# silently no-ops — consistent with the `-d .git` test used everywhere else in +# this script, not a special case, but worth knowing if a worktree checkout +# stops picking up a stale-volume refresh it should have caught. +git_head_commit() { + local git_dir="$1/.git" head_ref ref_path + [[ -d "$git_dir" ]] || return 1 + head_ref=$(cat -- "$git_dir/HEAD" 2>/dev/null) || return 1 + if [[ "$head_ref" == ref:* ]]; then + ref_path="${head_ref#ref: }" + if [[ -f "$git_dir/$ref_path" ]]; then + cat -- "$git_dir/$ref_path" + else + # Packed after a `git gc`; the loose ref file above is gone. + awk -v ref="$ref_path" '$2 == ref { print $1; exit }' "$git_dir/packed-refs" 2>/dev/null + fi + else + printf '%s' "$head_ref" + fi +} + # Record what the container is about to be built and created FROM. The in-app # updater compares these against the release it wants to apply: a release that # changes either file cannot be applied by the container restarting itself (a @@ -126,4 +194,65 @@ else printf 'Warning: no sha256 tool found; in-app updates will not detect environment changes.\n' >&2 fi -exec docker compose --env-file "$env_file" -f "$compose_file" up --build -d +# codeman-node-modules and codeman-dist (docker-compose.yaml) are seeded from +# the image only while EMPTY, so a rebuilt image's fresh output sits unused +# behind old volume content until something clears it. The in-app self-updater +# never hits this — it rebuilds INSIDE the running container, into the very +# volume already in use — but a `docker compose build` triggered from outside +# it (this script, after a `git pull`) does: the container comes back up +# looking unchanged. Detect that here and clear just the affected volume(s) so +# `--build` below actually takes effect. Best-effort: with no sha256 tool this +# quietly does nothing, same as the environment-gate block above. +if [[ -n "$dockerfile_sha" ]]; then + repo_head=$(git_head_commit "$repo_path" || true) + lockfile_sha=$(sha256_of "$repo_path/package-lock.json" 2>/dev/null || true) + source_state_file="$state_dir/docker-build-source.json" + prev_head='' + prev_lockfile_sha='' + if [[ -f "$source_state_file" ]]; then + prev_head=$(sed -n 's/.*"headCommit": *"\([^"]*\)".*/\1/p' "$source_state_file") + prev_lockfile_sha=$(sed -n 's/.*"lockfileSha256": *"\([^"]*\)".*/\1/p' "$source_state_file") + fi + + volumes_to_refresh=() + [[ -n "$repo_head" && "$repo_head" != "$prev_head" ]] && volumes_to_refresh+=('codeman-dist') + [[ -n "$lockfile_sha" && "$lockfile_sha" != "$prev_lockfile_sha" ]] && volumes_to_refresh+=('codeman-node-modules') + + if [[ ${#volumes_to_refresh[@]} -gt 0 ]]; then + # Runs even on this script's very first invocation against an EXISTING + # deployment, deliberately: that deployment's volumes may already be + # stale (there was no earlier version of this check to have caught it), + # and clearing an already-empty or nonexistent volume is a harmless + # no-op, so there is no fresh-install case this needs to avoid. + printf 'Source changed since the last start; refreshing: %s\n' "${volumes_to_refresh[*]}" + "${compose_command[@]}" down + # `com.docker.compose.volume` is the volume KEY, not a project-qualified + # name - a second stack on the same host (a beta instance started with a + # different COMPOSE_PROJECT_NAME, say) that also declares a volume keyed + # `codeman-dist` shares that label, and `head -n1` would pick whichever + # the daemon happens to list first. Scope the lookup to THIS stack's own + # resolved project name so it can only ever match this stack's volume. + project_name=$( + "${compose_command[@]}" config --format json 2>/dev/null | + sed -n 's/^ "name": "\(.*\)",\{0,1\}$/\1/p' | head -n1 + ) + for key in "${volumes_to_refresh[@]}"; do + volume_name=$( + docker volume ls -q \ + --filter "label=com.docker.compose.volume=$key" \ + --filter "label=com.docker.compose.project=$project_name" | + head -n1 + ) + [[ -n "$volume_name" ]] && docker volume rm -- "$volume_name" + done + fi + + printf '{\n "headCommit": "%s",\n "lockfileSha256": "%s"\n}\n' \ + "$repo_head" "$lockfile_sha" >"$source_state_file.tmp" + mv -- "$source_state_file.tmp" "$source_state_file" + if [[ "$EUID" == '0' ]]; then + chown -- "$PUID:$PGID" "$source_state_file" + fi +fi + +exec "${compose_command[@]}" up --build -d diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index ca61796a..1e888676 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -91,6 +91,13 @@ services: - no-new-privileges:true cap_drop: - ALL + cap_add: + # The entrypoint corrects bind-mount ownership as root before dropping to + # PUID:PGID. Everything not listed here remains dropped by cap_drop above. + - CHOWN + - DAC_OVERRIDE + - SETGID + - SETUID healthcheck: test: - CMD-SHELL diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000..be4912e4 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,96 @@ +#!/bin/sh +# Corrects ownership - host bind mounts, and the image-baked CLI prefix - +# then drops to PUID:PGID. +# +# Compose binds CODEMAN_APPDATA_PATH and CODEMAN_CASES_PATH from the host. When +# either path does not exist yet - a first run, a cleared application-data +# directory, a restored backup - the Docker daemon creates it owned by root, +# and an unprivileged server cannot then create its own state directory. The +# result is a container that restarts forever on: +# +# Failed to start web server: EACCES: permission denied, mkdir '/home//.codeman' +# +# Running this as root and dropping afterwards removes that failure mode without +# leaving the server privileged. The same root start also lets it re-assert +# /opt/codeman-cli's ownership on every start, not just at image build time - +# see the comment at that chown below for why that matters for anyone who +# runs the compose file directly rather than through Start-Codeman.sh. + +set -eu + +# Honour an explicit `user:` in Compose: when the container was not started as +# root there is nothing to correct and no privilege to drop. +if [ "$(id -u)" -ne 0 ]; then + exec "$@" +fi + +: "${PUID:=1000}" +: "${PGID:=1000}" + +for target in "${HOME:-}" "${CODEMAN_CASES_PATH:-}"; do + [ -n "$target" ] && [ -d "$target" ] || continue + owner=$(stat -c '%u:%g' "$target") + [ "$owner" = "${PUID}:${PGID}" ] && continue + + # Only ever correct a directory the DAEMON created (root-owned, because + # neither PUID nor PGID existed yet when it materialised the missing bind + # source). Anything else - a host tree that legitimately belongs to some + # OTHER account, such as an existing CODEMAN_CASES_PATH the README already + # allows pointing at a normal project directory - is not this container's + # to reassign; recursively chowning it on every mismatch silently rewrote + # a credentials tree or a projects directory to PUID:PGID with one log + # line to explain it. Refuse instead, the same way Start-Codeman.sh already + # refuses to touch a root-owned appdata directory it did not expect. + if [ "${owner%%:*}" != '0' ]; then + printf 'entrypoint: %s is owned by %s, which is neither root nor PUID:PGID (%s:%s).\n' \ + "$target" "$owner" "$PUID" "$PGID" >&2 + printf 'entrypoint: refusing to change ownership of a directory this container did not create.\n' >&2 + printf 'entrypoint: either chown it on the host, or set PUID/PGID to match its current owner.\n' >&2 + exit 1 + fi + + # Deliberately not fatal for a root-owned directory. A bind mount backed by + # NFS, CIFS or a rootless daemon can refuse chown while still being + # perfectly writable, and those deployments must keep working. A warning is + # more useful than a container that will not start. + if chown -R "${PUID}:${PGID}" "$target" 2>/dev/null; then + printf 'entrypoint: corrected ownership of %s to %s:%s\n' "$target" "$PUID" "$PGID" + else + printf 'entrypoint: warning: cannot change ownership of %s to %s:%s\n' \ + "$target" "$PUID" "$PGID" >&2 + printf 'entrypoint: warning: continuing; set the ownership on the host if startup fails\n' >&2 + fi +done + +# /opt/codeman-cli (the four agent CLIs) is chowned to PUID:PGID once, at +# image BUILD time, from the PUID/PGID build args - server.Dockerfile's own +# comment on that RUN step explains why it lives in its own prefix rather than +# /usr/local. Unlike HOME/CODEMAN_CASES_PATH above, that bake happens only +# when the image is actually rebuilt (`docker compose up --build`, which +# Start-Codeman.sh always does) - a deployment that instead runs the compose +# file directly (Unraid's Compose Manager, a native Debian systemd unit, any +# `docker compose up`/`restart` with no --build) can change PUID/PGID in .env +# and restart without ever rebuilding, at which point the container runs as +# the NEW uid while the CLI directory is still owned by the OLD one baked into +# the image layer - silently breaking the very "self-update a CLI in place" +# fix this directory exists for. Re-assert it here, every start, unconditionally: +# unlike the host bind mounts above, this is pure image content Codeman itself +# populated, never host data that might legitimately belong to someone else, +# so there is no ownership to be careful about - it is always correct for it +# to be owned by whoever this container is about to run as. +if [ -d /opt/codeman-cli ] && [ "$(stat -c '%u:%g' /opt/codeman-cli)" != "${PUID}:${PGID}" ]; then + chown -R "${PUID}:${PGID}" /opt/codeman-cli +fi + +# Preserve the supplementary groups Compose granted through group_add - that is +# how the Docker socket stays reachable - while discarding root's own group. +supplementary=$(id -G | tr ' ' '\n' | grep -vx 0 | paste -sd, -) +[ -n "$supplementary" ] || supplementary="$PGID" + +# --bounding-set -all: with the reuid/regid drop above, CapPrm/CapEff are +# already empty, but the bounding set otherwise still lists everything +# cap_add granted (visible as a nonzero CapBnd even post-drop). no-new-privileges +# already makes that moot - nothing can regain a capability outside the +# bounding set - but clearing it too is free and matches what "drops to +# PUID:PGID" actually promises. +exec setpriv --reuid "$PUID" --regid "$PGID" --groups "$supplementary" --bounding-set -all "$@" diff --git a/docker/server.Dockerfile b/docker/server.Dockerfile index ecb00d14..27ec4a43 100644 --- a/docker/server.Dockerfile +++ b/docker/server.Dockerfile @@ -24,7 +24,7 @@ RUN npm ci \ # docker/docker-compose.yaml. It does not run a Docker daemon in this container. FROM node:22-bookworm-slim -ARG CODEMAN_RUNTIME_USER=opencode +ARG CODEMAN_RUNTIME_USER=codeman ARG PUID=1000 ARG PGID=1000 @@ -71,6 +71,24 @@ COPY --from=docker:29-cli \ # Keep credentials out of the image. Users authenticate these CLIs at runtime # through Codeman sessions, and the configured host bind mount retains state. # +# Installed into a DEDICATED prefix, /opt/codeman-cli, not the base image's +# default /usr/local. A session needs write access to wherever these CLIs live +# so it can self-update one in place (observed via Codex's own +# `npm install -g @openai/codex`, which renames the old package directory +# aside before installing the new one — a rename needs write access to the +# PARENT directory, not just the target, so the runtime account needs that +# access at the directory level). Chowning /usr/local/bin and +# /usr/local/lib/node_modules directly to get it would ALSO hand away +# entrypoint.sh (COPY'd to /usr/local/bin below, root-owned, executed as root +# on every container start with CHOWN/DAC_OVERRIDE/SETUID/SETGID) and the node +# binary: owning the DIRECTORY is enough to rename it aside and drop a +# replacement, even though the file itself stays root-owned, which would let a +# compromised session arrange for its own script to run as root at the next +# restart — undoing the "the server itself never runs privileged" guarantee +# the entrypoint exists to provide. /opt/codeman-cli holds nothing else to +# escalate through, so owning it is exactly the CLI-update access it needs and +# no more. +# # ⚠️ PINNED ON PURPOSE. Unpinned, the agent CLI versions a user ends up with are # a function of WHEN their image was built, not of any commit — so a Codeman # release that depends on newer CLI behaviour (the trust-dialog handling is @@ -82,6 +100,8 @@ COPY --from=docker:29-cli \ # # Bump these deliberately, in a release. `--no-cache` is still needed to rebuild # this layer when only the pins change upstream. +ENV NPM_CONFIG_PREFIX=/opt/codeman-cli +ENV PATH=/opt/codeman-cli/bin:$PATH RUN npm install --global \ @anthropic-ai/claude-code@2.1.258 \ @google/gemini-cli@0.58.0 \ @@ -93,6 +113,11 @@ RUN npm install --global \ # PGID match the host-owned application-data directory mounted by Compose. The # requested GID may not exist in the base image, and a host UID such as 1000 may # already belong to the baked `node` account, so handle both cases explicitly. +# +# The trailing chown hands the CLI prefix (/opt/codeman-cli, populated above) +# to that same account, so a session can self-update one of the CLIs in place. +# /usr/local stays root-owned throughout — see the comment on the npm install +# above for why that boundary matters. RUN set -eux; \ case "${PUID}" in ''|*[!0-9]*) echo "PUID must be numeric" >&2; exit 1;; esac; \ case "${PGID}" in ''|*[!0-9]*) echo "PGID must be numeric" >&2; exit 1;; esac; \ @@ -120,7 +145,8 @@ RUN set -eux; \ --home-dir "/home/${CODEMAN_RUNTIME_USER}" \ --shell /bin/bash \ "${CODEMAN_RUNTIME_USER}"; \ - fi + fi; \ + chown -R "${PUID}:${PGID}" /opt/codeman-cli WORKDIR /opt/codeman @@ -135,8 +161,19 @@ ENV CODEMAN_IN_CONTAINER=1 \ HOME=/home/${CODEMAN_RUNTIME_USER} \ NODE_ENV=production +# Runtime defaults for the entrypoint, matching the account created above. +ENV PGID=${PGID} PUID=${PUID} + EXPOSE 3000 -USER ${CODEMAN_RUNTIME_USER} +# The container starts as root so the entrypoint can correct the ownership of +# the host bind mounts, which the daemon creates as root whenever they do not +# already exist. The entrypoint then drops to PUID:PGID with setpriv, so the +# server itself never runs privileged. Setting `user:` in Compose bypasses both +# steps, leaving the caller in full control. +COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod 0755 /usr/local/bin/entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] CMD ["node", "dist/index.js", "web"] diff --git a/docs/docker-compose.md b/docs/docker-compose.md index f2add195..da85e203 100644 --- a/docs/docker-compose.md +++ b/docs/docker-compose.md @@ -15,7 +15,7 @@ The application container mounts the Docker daemon socket so Codeman can create ## Start -Copy the environment template, set a strong password, and confirm `CODEMAN_APPDATA_PATH`. The example maps `/mnt/user/appdata/Coding/codeman` on the host to `/home/${CODEMAN_RUNTIME_USER}` in the container, preserving Codeman state and CLI credentials outside Docker-managed volumes. +Copy the environment template, set a strong password, and confirm `CODEMAN_APPDATA_PATH`. The example maps `/mnt/user/appdata/codeman` on the host to `/home/${CODEMAN_RUNTIME_USER}` in the container, preserving Codeman state and CLI credentials outside Docker-managed volumes. ```sh cp docker/.env.example docker/.env diff --git a/docs/docker-self-update.md b/docs/docker-self-update.md index 6521abd0..6f41505b 100644 --- a/docs/docker-self-update.md +++ b/docs/docker-self-update.md @@ -59,6 +59,7 @@ unchanged. The container path is a new `SupervisorKind`, not a new updater. | `CODEMAN_RESTART_BY_EXIT=1` | The Compose file's declaration of that policy, so the updater may exit even with no Docker socket. | | Toolchain + devDependencies in the image | Lets `npm install` and `npm run build` run inside the container. | | `docker-env-applied.json` | Fingerprint baseline, written by `Start-Codeman.sh` on every start. | +| `docker-build-source.json` | What HEAD/`package-lock.json` the build artefact volumes currently reflect. Written by both `Start-Codeman.sh` and this in-place update, so the two agree on whether those volumes are stale. | ### Why build artefacts are in named volumes @@ -72,6 +73,20 @@ Docker seeds an empty named volume from the image, so the first start inherits t image's already-built `node_modules` and `dist` and pays no bootstrap cost. `docker compose down -v` is the supported reset: the next start re-seeds them. +That seeding-only-while-empty behaviour has a second, less obvious edge: it also +means a plain `docker compose build` triggered from OUTSIDE the container (for +example `Start-Codeman.sh`, after a `git pull` done by hand rather than through +this in-app updater) produces a fresh image whose freshly-built `dist`/ +`node_modules` then sit unused behind the volumes' OLD content — the container +comes back up looking unchanged. `Start-Codeman.sh` detects this by comparing the +checkout's current HEAD and `package-lock.json` hash against `docker-build-source.json`, +and clears just the affected volume(s) before its own `--build` if they moved. +This in-place update writes that same file after a successful build precisely so +that comparison does not fire on stale information: without it, the next plain +`Start-Codeman.sh` run would see the HEAD this update just checked out, not +recognise it as already accounted for, and wipe the volumes this update just +correctly rebuilt right back to the OLDER image. + ### Why the runtime image carries a build toolchain `npm run build` is `tsc` plus `esbuild`, both devDependencies, so the image no diff --git a/scripts/self-update.sh b/scripts/self-update.sh index 6ac3f80d..fb3e7874 100755 --- a/scripts/self-update.sh +++ b/scripts/self-update.sh @@ -193,6 +193,29 @@ run_step "installing" "Installing dependencies" npm install --no-fund --no-audit # 5) Build (gate the restart on success — never restart into a torn dist/). run_step "building" "Building" npm run build || rollback_and_fail "Build failed" +# Docker Compose only: record what HEAD/package-lock.json the freshly-built +# codeman-dist/codeman-node-modules volumes now reflect. `Start-Codeman.sh` +# reads this same file (`$appdata_path/.codeman/…`, i.e. this container's own +# $HOME/.codeman since that path IS the appdata bind mount) to detect source +# changes an EXTERNAL `docker compose build` made and refresh those volumes — +# without this, the next plain `Start-Codeman.sh` run would see the HEAD this +# update just checked out, not recognise it as already accounted for, and wipe +# the volumes this update just correctly rebuilt right back to the OLDER image. +if [[ "$SUPERVISOR" == "docker-compose" ]]; then + build_source_file="$HOME/.codeman/docker-build-source.json" + mkdir -p -- "$HOME/.codeman" + build_head=$(git rev-parse HEAD 2>/dev/null || true) + build_lockfile_sha='' + if command -v sha256sum >/dev/null 2>&1; then + build_lockfile_sha=$(sha256sum -- package-lock.json 2>/dev/null | cut -d' ' -f1) + elif command -v shasum >/dev/null 2>&1; then + build_lockfile_sha=$(shasum -a 256 package-lock.json 2>/dev/null | cut -d' ' -f1) + fi + printf '{\n "headCommit": "%s",\n "lockfileSha256": "%s"\n}\n' \ + "$build_head" "$build_lockfile_sha" >"$build_source_file.tmp" \ + && mv -- "$build_source_file.tmp" "$build_source_file" +fi + # 6) Restart the service so the new code loads. Write the terminal pre-restart # marker FIRST so the freshly-booted server can reconcile it deterministically. write_status "restarting" "Restarting Codeman…"