From 37a325c440ecaaeca57cc990c43ceff742894595 Mon Sep 17 00:00:00 2001 From: Daniel Vaughn Date: Tue, 29 Sep 2026 11:06:46 -0400 Subject: [PATCH 1/5] fix: make web security tools easier to find Reuse Go when it is already installed and use the right permissions when installing it. Add tests for both cases. --- .../web-security/scripts/install_tools.sh | 5 +- .../tests/test_install_tools_offline.py | 53 +++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/capabilities/web-security/scripts/install_tools.sh b/capabilities/web-security/scripts/install_tools.sh index 119ce1b..2608c39 100755 --- a/capabilities/web-security/scripts/install_tools.sh +++ b/capabilities/web-security/scripts/install_tools.sh @@ -14,7 +14,7 @@ case "$OS" in ;; esac -export PATH="$HOME/.pdtm/go/bin:$HOME/go/bin:$PATH" +export PATH="$HOME/.pdtm/go/bin:$HOME/go/bin:$HOME/.local/bin:/usr/local/go/bin:$PATH" # `have ` — is this already on PATH, or in one of the two directories the # tools below install into? @@ -106,8 +106,7 @@ if [ "$need_go" = true ] && ! command -v go &>/dev/null; then aarch64|arm64) GOARCH="arm64" ;; *) GOARCH="amd64" ;; esac - curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${GOARCH}.tar.gz" | tar -xz -C /usr/local - export PATH="/usr/local/go/bin:$PATH" + curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${GOARCH}.tar.gz" | as_root tar -xz -C /usr/local fi # -- ProjectDiscovery tools ------------------------------------------------ diff --git a/capabilities/web-security/tests/test_install_tools_offline.py b/capabilities/web-security/tests/test_install_tools_offline.py index 74ad28b..28e1dbf 100644 --- a/capabilities/web-security/tests/test_install_tools_offline.py +++ b/capabilities/web-security/tests/test_install_tools_offline.py @@ -59,6 +59,59 @@ def _have_pd_httpx(tmp_path: Path) -> bool: return subprocess.run(["bash", "-c", script], env=env, check=False).returncode == 0 +def test_existing_go_outside_initial_path_is_reused(tmp_path: Path) -> None: + local = tmp_path / "usr-local" + _stub(local / "go/bin/go", exit_code=0) + result = _run_go_setup(tmp_path, local) + assert result.returncode == 0, result.stderr + assert not (tmp_path / "sudo.log").exists() + assert result.stdout.strip() == str(local / "go/bin/go") + + +def test_missing_go_extracts_with_noninteractive_sudo(tmp_path: Path) -> None: + local = tmp_path / "usr-local" + result = _run_go_setup(tmp_path, local) + assert result.returncode == 0, result.stderr + assert (tmp_path / "sudo.log").read_text().strip() == f"-n tar -xz -C {local}" + + +def _run_go_setup(tmp_path: Path, local: Path) -> subprocess.CompletedProcess[str]: + """Run Go setup with temporary paths and stubbed system commands.""" + bindir = tmp_path / "bin" + bindir.mkdir() + commands = { + "id": "printf '1000\\n'", + "curl": "printf 'archive\\n'", + "sudo": 'printf "%s\\n" "$*" > "$HOME/sudo.log"; shift; exec "$@"', + "tar": 'read -r archive; test "$archive" = archive', + } + for name, body in commands.items(): + executable = bindir / name + executable.write_text(f"#!/bin/bash\n{body}\n") + executable.chmod(0o755) + path_line = next(line for line in LINES if line.startswith("export PATH=")) + go_setup = INSTALL_SCRIPT.split("need_go=false", 1)[1].split( + "# -- ProjectDiscovery tools", 1 + )[0] + script = ( + "set -euo pipefail\n" + + path_line + + "\n" + + _shell_function("as_root") + + "missing_go_tools=protoscope; missing_pd_tools=''; ARCH=x86_64; GO_VERSION=test\n" + + "need_go=false" + + go_setup + + "\ncommand -v go || true\n" + ).replace("/usr/local", str(local)) + return subprocess.run( + ["/bin/bash", "-c", script], + env={"HOME": str(tmp_path), "PATH": str(bindir)}, + capture_output=True, + text=True, + check=False, + ) + + class TestVersionsArePinned: def test_no_unpinned_go_installs(self) -> None: # `go install ...@latest` re-resolves against the module proxy every From 9b117848b58eb775278ee1a3a0b02e3ddb308bfb Mon Sep 17 00:00:00 2001 From: Daniel Vaughn Date: Tue, 29 Sep 2026 11:40:18 -0400 Subject: [PATCH 2/5] fix: recover from incomplete tool installs --- .../web-security/scripts/install_tools.sh | 433 +++++++++++------- .../web-security/tests/test_caido_go_mcp.py | 11 +- .../tests/test_caido_mode_skill.py | 7 +- .../tests/test_install_tools_offline.py | 314 +++++++++++-- 4 files changed, 566 insertions(+), 199 deletions(-) diff --git a/capabilities/web-security/scripts/install_tools.sh b/capabilities/web-security/scripts/install_tools.sh index 2608c39..94530cf 100755 --- a/capabilities/web-security/scripts/install_tools.sh +++ b/capabilities/web-security/scripts/install_tools.sh @@ -16,6 +16,50 @@ esac export PATH="$HOME/.pdtm/go/bin:$HOME/go/bin:$HOME/.local/bin:/usr/local/go/bin:$PATH" +# Keep errexit inside each stage, but let independent stages finish. Do not +# invoke the subshell in an `if` or `||`: bash would disable errexit inside it. +failed_stages=() +run_stage() { + local name="$1" status + shift + echo "web-security: starting $name" >&2 + set +e + ( set -e; "$@" ) + status=$? + set -e + if [ "$status" -ne 0 ]; then + failed_stages+=("$name") + echo "web-security: $name failed (exit $status)" >&2 + fi +} + +# Retry commands that fetch dependencies, at most three times. The SDK's +# aggregate install deadline still bounds the entire pass. +retry() { + local attempt status + for attempt in 1 2 3; do + if "$@"; then + return 0 + else + status=$? + fi + [ "$attempt" -eq 3 ] && return "$status" + echo "web-security: retrying $1 after attempt $attempt" >&2 + sleep "$attempt" + done +} + +# Stage downloads and clones in an invocation-local temporary directory. +INSTALL_TMP="$(mktemp -d)" +trap 'rm -rf "$INSTALL_TMP"' EXIT + +clone_repo() { + local url="$1" target="$2" staging + shift 2 + staging="$(mktemp -d "$INSTALL_TMP/clone.XXXXXX")" || return + git clone --depth 1 "$@" "$url" "$staging" && mv "$staging" "$target" +} + # `have ` — is this already on PATH, or in one of the two directories the # tools below install into? # @@ -44,10 +88,7 @@ as_root() { fi } -# Install a Python package into whatever interpreter this runtime uses. -# `pip` is not always on PATH — a uv-managed virtualenv has no pip binary at -# all, which made the bare `pip install` calls below abort the run with -# "command not found" on exactly the images the SDK ships. +# Install Python packages using uv when available, then pip or python3 -m pip. py_install() { if command -v uv >/dev/null 2>&1; then uv pip install --python "$(command -v python3)" "$@" @@ -81,7 +122,8 @@ install_pd_tool() { local tool="$1" package="$2" version="$3" have_pd_tool "$tool" && return mkdir -p "$HOME/.pdtm/go/bin" - GOBIN="$HOME/.pdtm/go/bin" go install "${package}@${version}" + GOBIN="$HOME/.pdtm/go/bin" retry go install "${package}@${version}" + have_pd_tool "$tool" } # What is actually missing, before anything is fetched. @@ -101,63 +143,85 @@ done need_go=false [ -n "$missing_go_tools" ] && need_go=true [ -n "$missing_pd_tools" ] && need_go=true -if [ "$need_go" = true ] && ! command -v go &>/dev/null; then - case "$ARCH" in - aarch64|arm64) GOARCH="arm64" ;; - *) GOARCH="amd64" ;; - esac - curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${GOARCH}.tar.gz" | as_root tar -xz -C /usr/local -fi +have kr || need_go=true +install_go() { + if [ "$need_go" = true ] && ! command -v go &>/dev/null; then + case "$ARCH" in + aarch64|arm64) GOARCH="arm64" ;; + *) GOARCH="amd64" ;; + esac + retry curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${GOARCH}.tar.gz" -o "$INSTALL_TMP/go.tar.gz" + as_root tar -xzf "$INSTALL_TMP/go.tar.gz" -C /usr/local + go version + fi +} +run_stage go install_go # -- ProjectDiscovery tools ------------------------------------------------ if [ -n "$missing_pd_tools" ]; then - install_pd_tool nuclei github.com/projectdiscovery/nuclei/v3/cmd/nuclei v3.11.1 - install_pd_tool httpx github.com/projectdiscovery/httpx/cmd/httpx v1.12.0 - install_pd_tool subfinder github.com/projectdiscovery/subfinder/v2/cmd/subfinder v2.16.0 - install_pd_tool naabu github.com/projectdiscovery/naabu/v2/cmd/naabu v2.6.1 - install_pd_tool dnsx github.com/projectdiscovery/dnsx/cmd/dnsx v1.3.1 - install_pd_tool uncover github.com/projectdiscovery/uncover/cmd/uncover v1.2.1 - install_pd_tool alterx github.com/projectdiscovery/alterx/cmd/alterx v0.1.0 - install_pd_tool tlsx github.com/projectdiscovery/tlsx/cmd/tlsx v1.4.0 - install_pd_tool asnmap github.com/projectdiscovery/asnmap/cmd/asnmap v1.1.1 + run_stage nuclei install_pd_tool nuclei github.com/projectdiscovery/nuclei/v3/cmd/nuclei v3.11.1 + run_stage httpx install_pd_tool httpx github.com/projectdiscovery/httpx/cmd/httpx v1.12.0 + run_stage subfinder install_pd_tool subfinder github.com/projectdiscovery/subfinder/v2/cmd/subfinder v2.16.0 + run_stage naabu install_pd_tool naabu github.com/projectdiscovery/naabu/v2/cmd/naabu v2.6.1 + run_stage dnsx install_pd_tool dnsx github.com/projectdiscovery/dnsx/cmd/dnsx v1.3.1 + run_stage uncover install_pd_tool uncover github.com/projectdiscovery/uncover/cmd/uncover v1.2.1 + run_stage alterx install_pd_tool alterx github.com/projectdiscovery/alterx/cmd/alterx v0.1.0 + run_stage tlsx install_pd_tool tlsx github.com/projectdiscovery/tlsx/cmd/tlsx v1.4.0 + run_stage asnmap install_pd_tool asnmap github.com/projectdiscovery/asnmap/cmd/asnmap v1.1.1 fi -# -- katana (pre-built binary, go-tree-sitter build issue) ----------------- -if ! have katana; then - DEB_ARCH="$(dpkg --print-architecture 2>/dev/null || echo amd64)" - mkdir -p "$HOME/.pdtm/go/bin" - curl -fsSL "https://github.com/projectdiscovery/katana/releases/download/v${KATANA_VERSION}/katana_${KATANA_VERSION}_linux_${DEB_ARCH}.zip" \ - -o /tmp/katana.zip - unzip -o /tmp/katana.zip -d /tmp/katana_extract - mv /tmp/katana_extract/katana "$HOME/.pdtm/go/bin/katana" - chmod +x "$HOME/.pdtm/go/bin/katana" - rm -rf /tmp/katana.zip /tmp/katana_extract -fi +# -- katana (pre-built binary) -------------------------------------------- +install_katana() { + if ! have katana; then + DEB_ARCH="$(dpkg --print-architecture 2>/dev/null || echo amd64)" + mkdir -p "$HOME/.pdtm/go/bin" + retry curl -fsSL "https://github.com/projectdiscovery/katana/releases/download/v${KATANA_VERSION}/katana_${KATANA_VERSION}_linux_${DEB_ARCH}.zip" \ + -o "${INSTALL_TMP}/katana.zip" + unzip -o "${INSTALL_TMP}/katana.zip" -d "${INSTALL_TMP}/katana_extract" + mv "${INSTALL_TMP}/katana_extract/katana" "$HOME/.pdtm/go/bin/katana" + chmod +x "$HOME/.pdtm/go/bin/katana" + rm -rf "${INSTALL_TMP}/katana.zip" "${INSTALL_TMP}/katana_extract" + fi +} +run_stage katana install_katana # -- protoscope ------------------------------------------------------------ -have protoscope || \ - go install "github.com/protocolbuffers/protoscope/cmd/protoscope@${GO_TOOL_VERSIONS_protoscope}" +install_protoscope() { + have protoscope || \ + retry go install "github.com/protocolbuffers/protoscope/cmd/protoscope@${GO_TOOL_VERSIONS_protoscope}" +} +run_stage protoscope install_protoscope # -- interactsh-client ----------------------------------------------------- -have interactsh-client || \ - go install "github.com/projectdiscovery/interactsh/cmd/interactsh-client@${GO_TOOL_VERSIONS_interactsh}" +install_interactsh() { + have interactsh-client || \ + retry go install "github.com/projectdiscovery/interactsh/cmd/interactsh-client@${GO_TOOL_VERSIONS_interactsh}" +} +run_stage interactsh install_interactsh # -- 2fa (TOTP generator) -------------------------------------------------- -have 2fa || go install "rsc.io/2fa@${GO_TOOL_VERSIONS_2fa}" +install_twofa() { + have 2fa || retry go install "rsc.io/2fa@${GO_TOOL_VERSIONS_2fa}" +} +run_stage twofa install_twofa # surf is not installed: upstream grants no licence, so we have no right to use # or redistribute it (ADM-447). # -- kiterunner (API content discovery) ------------------------------------ -if ! have kr; then - if git clone --depth 1 --branch "$KITERUNNER_VERSION" https://github.com/assetnote/kiterunner /tmp/kiterunner; then - ( cd /tmp/kiterunner && make build ) \ - && as_root mv /tmp/kiterunner/dist/kr /usr/local/bin/kr - rm -rf /tmp/kiterunner - else - echo "WARN: kiterunner clone failed, skipping" +install_kiterunner() { + if ! have kr; then + if retry clone_repo https://github.com/assetnote/kiterunner "${INSTALL_TMP}/kiterunner" --branch "$KITERUNNER_VERSION"; then + ( cd "${INSTALL_TMP}/kiterunner" && retry make build ) + as_root mv "${INSTALL_TMP}/kiterunner/dist/kr" /usr/local/bin/kr + rm -rf "${INSTALL_TMP}/kiterunner" + else + echo "WARN: kiterunner clone failed, skipping" >&2 + return 1 + fi fi -fi +} +run_stage kiterunner install_kiterunner # -- Caido CLI ------------------------------------------------------------- # Pinned Caido CLI (headless server) release. Auth is handled at runtime via @@ -169,117 +233,137 @@ fi # task-based sending via startReplayTask). Pinning an older server here puts # the client and server on opposite sides of that schema break. # tests/test_caido_mode_skill.py enforces the floor. -if ! command -v caido-cli &>/dev/null; then - CAIDO_VERSION="0.57.1" - case "$ARCH" in - aarch64|arm64) CAIDO_ARCH="aarch64" ;; - *) CAIDO_ARCH="x86_64" ;; - esac - curl -fsSL "https://caido.download/releases/v${CAIDO_VERSION}/caido-cli-v${CAIDO_VERSION}-linux-${CAIDO_ARCH}.tar.gz" \ - -o /tmp/caido-cli.tar.gz \ - && as_root tar -xzf /tmp/caido-cli.tar.gz -C /usr/local/bin/ \ - && rm /tmp/caido-cli.tar.gz \ - || echo "WARN: Caido CLI install failed (check version), skipping" -fi +install_caido_cli() { + if ! command -v caido-cli &>/dev/null; then + CAIDO_VERSION="0.57.1" + case "$ARCH" in + aarch64|arm64) CAIDO_ARCH="aarch64" ;; + *) CAIDO_ARCH="x86_64" ;; + esac + retry curl -fsSL "https://caido.download/releases/v${CAIDO_VERSION}/caido-cli-v${CAIDO_VERSION}-linux-${CAIDO_ARCH}.tar.gz" \ + -o "${INSTALL_TMP}/caido-cli.tar.gz" \ + && as_root tar -xzf "${INSTALL_TMP}/caido-cli.tar.gz" -C /usr/local/bin/ \ + && rm "${INSTALL_TMP}/caido-cli.tar.gz" \ + || { echo "WARN: Caido CLI install failed (check version), skipping" >&2; return 1; } + fi +} +run_stage caido_cli install_caido_cli # -- Caido MCP server (Go, c0tton-fluff/caido-mcp-server) ------------------- # Full-surface Caido MCP server wired into capability.yaml as `caido-go`. # Pinned to a release with SHA-256 verification. Installed to /usr/local/bin # so it resolves on PATH for the MCP `command: caido-mcp-server`. -if ! command -v caido-mcp-server &>/dev/null; then - CAIDO_MCP_VERSION="4.3.0" - case "$ARCH" in - aarch64|arm64) - CAIDO_MCP_ARCH="arm64" - CAIDO_MCP_SHA256="7b8d6a89f6b404345715a25d8201a0fbe37db9a0f23b8b1868d01c68b110071b" - ;; - *) - CAIDO_MCP_ARCH="amd64" - CAIDO_MCP_SHA256="5236620c693f973d5725133c660ca0ac852796dd75e02ce1993bd66202d0b04c" - ;; - esac - CAIDO_MCP_URL="https://github.com/c0tton-fluff/caido-mcp-server/releases/download/v${CAIDO_MCP_VERSION}/caido-mcp-server-linux-${CAIDO_MCP_ARCH}" - if curl -fsSL "$CAIDO_MCP_URL" -o /tmp/caido-mcp-server; then - if echo "${CAIDO_MCP_SHA256} /tmp/caido-mcp-server" | sha256sum -c - >/dev/null 2>&1; then - as_root install -m 0755 /tmp/caido-mcp-server /usr/local/bin/caido-mcp-server - echo "caido-mcp-server v${CAIDO_MCP_VERSION} installed" +install_caido_mcp() { + if ! command -v caido-mcp-server &>/dev/null; then + CAIDO_MCP_VERSION="4.3.0" + case "$ARCH" in + aarch64|arm64) + CAIDO_MCP_ARCH="arm64" + CAIDO_MCP_SHA256="7b8d6a89f6b404345715a25d8201a0fbe37db9a0f23b8b1868d01c68b110071b" + ;; + *) + CAIDO_MCP_ARCH="amd64" + CAIDO_MCP_SHA256="5236620c693f973d5725133c660ca0ac852796dd75e02ce1993bd66202d0b04c" + ;; + esac + CAIDO_MCP_URL="https://github.com/c0tton-fluff/caido-mcp-server/releases/download/v${CAIDO_MCP_VERSION}/caido-mcp-server-linux-${CAIDO_MCP_ARCH}" + if retry curl -fsSL "$CAIDO_MCP_URL" -o "${INSTALL_TMP}/caido-mcp-server"; then + if echo "${CAIDO_MCP_SHA256} ${INSTALL_TMP}/caido-mcp-server" | sha256sum -c - >/dev/null 2>&1; then + as_root install -m 0755 "${INSTALL_TMP}/caido-mcp-server" /usr/local/bin/caido-mcp-server + echo "caido-mcp-server v${CAIDO_MCP_VERSION} installed" + else + echo "WARN: caido-mcp-server checksum mismatch, skipping install" >&2 + return 1 + fi + rm -f "${INSTALL_TMP}/caido-mcp-server" else - echo "WARN: caido-mcp-server checksum mismatch, skipping install" >&2 + echo "WARN: caido-mcp-server download failed (check version), skipping" >&2 + return 1 fi - rm -f /tmp/caido-mcp-server - else - echo "WARN: caido-mcp-server download failed (check version), skipping" >&2 fi -fi +} +run_stage caido_mcp install_caido_mcp # -- Burp Suite Community (headless) ---------------------------------------- -# Downloads the Burp Suite Community JAR for headless scanning. -# Pro features require BURP_LICENSE_KEY at runtime. -if [ ! -f /opt/burp/burpsuite.jar ]; then - BURP_VERSION="2025.5" - # /opt and /usr/local/bin are root-owned, and this script does not always run - # as root. Previously the unguarded `mkdir` aborted the entire provision under - # `set -e` on a non-root runtime, taking every tool below it down with it. - if as_root mkdir -p /opt/burp; then - as_root curl -fsSL "https://portswigger-cdn.net/burp/releases/download?product=community&version=${BURP_VERSION}&type=Jar" \ - -o /opt/burp/burpsuite.jar \ - || echo "WARN: Burp Suite download failed (check version), skipping" - # Only wrap a jar that actually arrived — a `burp` on PATH pointing at - # nothing is worse than no `burp` at all. - if [ -f /opt/burp/burpsuite.jar ]; then - as_root tee /usr/local/bin/burp >/dev/null <<'BURPEOF' +# Install the Burp Suite Community JAR and command-line launcher. +install_burp() { + if [ ! -s /opt/burp/burpsuite.jar ]; then + BURP_VERSION="2025.5" + # Use root privileges for the installation directory and published artifacts. + if as_root mkdir -p /opt/burp; then + retry curl -fsSL "https://portswigger-cdn.net/burp/releases/download?product=community&version=${BURP_VERSION}&type=Jar" \ + -o "$INSTALL_TMP/burpsuite.jar" \ + || { echo "WARN: Burp Suite download failed (check version), skipping" >&2; return 1; } + as_root install -m 0644 "$INSTALL_TMP/burpsuite.jar" /opt/burp/burpsuite.jar.tmp + as_root mv /opt/burp/burpsuite.jar.tmp /opt/burp/burpsuite.jar + else + echo "WARN: cannot create /opt/burp (requires root); skipping Burp Suite" >&2 + return 1 + fi + fi + # Retry wrapper creation even when the JAR arrived on an earlier pass. + if ! command -v burp >/dev/null 2>&1; then + as_root tee /usr/local/bin/burp >/dev/null <<'BURPEOF' #!/usr/bin/env bash exec java -jar /opt/burp/burpsuite.jar "$@" BURPEOF - as_root chmod +x /usr/local/bin/burp - fi - else - echo "WARN: cannot create /opt/burp (requires root); skipping Burp Suite" + as_root chmod +x /usr/local/bin/burp fi -fi +} +run_stage burp install_burp # -- exiftool (EXIF metadata manipulation) --------------------------------- -if ! command -v exiftool &>/dev/null; then - as_root apt-get install -y --no-install-recommends libimage-exiftool-perl \ - || echo "WARN: exiftool install failed, skipping" -fi +install_exiftool() { + if ! command -v exiftool &>/dev/null; then + retry as_root apt-get install -y --no-install-recommends libimage-exiftool-perl \ + || { echo "WARN: exiftool install failed, skipping" >&2; return 1; } + fi +} +run_stage exiftool install_exiftool # -- Node.js + agent-browser ----------------------------------------------- -NODE_MAJOR="$(node -p 'process.versions.node.split(".")[0]' 2>/dev/null || echo 0)" -if [ "$NODE_MAJOR" -lt 24 ]; then - curl -fsSL https://deb.nodesource.com/setup_24.x | as_root bash - \ - && as_root apt-get install -y --no-install-recommends nodejs \ - || echo "WARN: Node.js install failed, skipping" -fi -# agent-browser pinned to the current latest — an unpinned install resolves -# to a different tool on different days, which no SBOM can describe. -AGENT_BROWSER_VERSION="0.35.1" -if ! have agent-browser; then - as_root npm install -g "agent-browser@${AGENT_BROWSER_VERSION}" \ - || echo "WARN: agent-browser install failed, skipping" -fi -# `agent-browser install` downloads the browser binaries themselves. Guarded on -# its cache so a runtime that already has them makes no request, and left -# non-fatal because a disconnected deployment that cannot fetch a browser -# should still get the rest of this capability's tooling. -AGENT_BROWSER_CACHE="${AGENT_BROWSER_CACHE_DIR:-$HOME/.cache/agent-browser}" -if [ "${DREADNODE_CAPABILITY_INSTALL:-}" != "sealed" ] && [ ! -d "$AGENT_BROWSER_CACHE" ]; then - agent-browser install || echo "WARN: agent-browser browser download failed, skipping" -fi +install_browser() { + NODE_MAJOR="$(node -p 'process.versions.node.split(".")[0]' 2>/dev/null || echo 0)" + if [ "$NODE_MAJOR" -lt 24 ]; then + retry curl -fsSL https://deb.nodesource.com/setup_24.x -o "$INSTALL_TMP/node-setup.sh" + as_root bash "$INSTALL_TMP/node-setup.sh" + retry as_root apt-get install -y --no-install-recommends nodejs \ + || { echo "WARN: Node.js install failed, skipping" >&2; return 1; } + fi + # Pin the agent-browser package version for repeatable installs. + AGENT_BROWSER_VERSION="0.35.1" + if ! have agent-browser; then + retry as_root npm install -g "agent-browser@${AGENT_BROWSER_VERSION}" \ + || { echo "WARN: agent-browser install failed, skipping" >&2; return 1; } + fi + # `agent-browser install` downloads the browser binaries themselves. Guarded on + # a completion marker so a failed download that creates the cache directory + # is retried on the next pass. Sealed deployments still skip browser downloads. + AGENT_BROWSER_CACHE="${AGENT_BROWSER_CACHE_DIR:-$HOME/.cache/agent-browser}" + if [ "${DREADNODE_CAPABILITY_INSTALL:-}" != "sealed" ] && [ ! -f "$AGENT_BROWSER_CACHE/.dreadnode-installed" ]; then + retry agent-browser install || { echo "WARN: agent-browser browser download failed, skipping" >&2; return 1; } + mkdir -p "$AGENT_BROWSER_CACHE" + touch "$AGENT_BROWSER_CACHE/.dreadnode-installed" + fi +} +run_stage browser install_browser # -- caido-mode skill deps (Caido TypeScript SDK CLI) ----------------------- # The caido-mode skill bundles a tsx CLI built on @caido/sdk-client (caido-ts). # Pre-install its node_modules so `npx tsx caido-client.ts` resolves offline at # runtime. Path is relative to the capability root (CAPABILITY_ROOT if exported, # else the script's own location, which is /scripts). -CAIDO_MODE_DIR="${CAPABILITY_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}/skills/caido-mode" -# Guarded on node_modules: without it this reaches the npm registry on every -# boot even when the dependencies are already installed. -if [ -f "$CAIDO_MODE_DIR/package.json" ] && [ ! -d "$CAIDO_MODE_DIR/node_modules" ]; then - ( cd "$CAIDO_MODE_DIR" && npm install --no-audit --no-fund ) \ - && echo "caido-mode skill deps installed (@caido/sdk-client / caido-ts)" \ - || echo "WARN: caido-mode npm install failed, skipping" -fi +install_caido_mode() { + CAIDO_MODE_DIR="${CAPABILITY_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}/skills/caido-mode" + # npm ls checks installed dependencies locally, including incomplete node_modules + # left by a failed install, without contacting the registry. + if [ -f "$CAIDO_MODE_DIR/package.json" ] && ! ( cd "$CAIDO_MODE_DIR" && npm ls --depth=0 >/dev/null 2>&1 ); then + ( cd "$CAIDO_MODE_DIR" && retry npm install --no-audit --no-fund ) \ + && echo "caido-mode skill deps installed (@caido/sdk-client / caido-ts)" \ + || { echo "WARN: caido-mode npm install failed, skipping" >&2; return 1; } + fi +} +run_stage caido_mode install_caido_mode # -- wrangler (Cloudflare Workers CLI for OAST endpoints) ------------------ # Deploys Cloudflare Workers as custom OAST endpoints (blind XSS payload @@ -288,48 +372,85 @@ fi # CLOUDFLARE_API_TOKEN / CLOUDFLARE_ACCOUNT_ID (CF_* aliases accepted). # Pinned: an unpinned npm install re-resolves against the registry even when # the binary is already present, which a sealed deployment must never do. -WRANGLER_VERSION="4.127.0" -have wrangler || \ - as_root npm install -g "wrangler@${WRANGLER_VERSION}" \ - || echo "WARN: wrangler install failed, skipping" +install_wrangler() { + WRANGLER_VERSION="4.127.0" + have wrangler || \ + retry as_root npm install -g "wrangler@${WRANGLER_VERSION}" \ + || { echo "WARN: wrangler install failed, skipping" >&2; return 1; } +} +run_stage wrangler install_wrangler # -- ast-grep (AST-based code pattern search) --------------------------------- # Tree-sitter based structural code matching for JS/TS/HTML. Lightweight # alternative to semgrep for pattern matching (no taint analysis). -have ast-grep || py_install ast-grep-cli || echo "WARN: ast-grep install failed, skipping" +install_ast_grep() { + have ast-grep || retry py_install ast-grep-cli || { echo "WARN: ast-grep install failed, skipping" >&2; return 1; } +} +run_stage ast_grep install_ast_grep # -- waymore (Wayback Machine recon) ----------------------------------------- -have waymore || py_install waymore || echo "WARN: waymore install failed, skipping" +install_waymore() { + have waymore || retry py_install waymore || { echo "WARN: waymore install failed, skipping" >&2; return 1; } +} +run_stage waymore install_waymore # -- Pacu (AWS exploitation framework) ---------------------------------------- -have pacu || py_install pacu || echo "WARN: pacu install failed, skipping" +install_pacu() { + have pacu || retry py_install pacu || { echo "WARN: pacu install failed, skipping" >&2; return 1; } +} +run_stage pacu install_pacu # -- fireprox (AWS API Gateway IP rotation) --------------------------------- # Requires AWS credentials at runtime. Cloned to a predictable path so the # ip-rotation skill can reference it directly. -FIREPROX_DIR="$HOME/git/fireprox" -if [ ! -d "$FIREPROX_DIR" ]; then - # Requirements are installed only alongside a fresh clone. Re-running them on - # every boot re-resolves against PyPI for an environment that already - # satisfies them. - if git clone --depth 1 https://github.com/ustayready/fireprox "$FIREPROX_DIR"; then - py_install -r "$FIREPROX_DIR/requirements.txt" \ - || echo "WARN: fireprox requirements install failed, skipping" - else - echo "WARN: fireprox clone failed, skipping" +install_fireprox() { + FIREPROX_DIR="$HOME/git/fireprox" + if [ ! -d "$FIREPROX_DIR" ]; then + mkdir -p "$HOME/git" + retry clone_repo https://github.com/ustayready/fireprox "$FIREPROX_DIR" fi -fi + if [ ! -f "$FIREPROX_DIR/.dreadnode-deps-installed" ]; then + retry py_install -r "$FIREPROX_DIR/requirements.txt" + touch "$FIREPROX_DIR/.dreadnode-deps-installed" + fi +} +run_stage fireprox install_fireprox # -- archivealchemist (malicious archive crafter) --------------------------- # Pure Python CLI for crafting Zip Slip, symlink, polyglot, and Unicode path # confusion archives. Cloned to a predictable path for the agent prompt. -ARCHIVEALCHEMIST_DIR="$HOME/git/archivealchemist" -if [ ! -d "$ARCHIVEALCHEMIST_DIR" ]; then - git clone --depth 1 https://github.com/avlidienbrunn/archivealchemist "$ARCHIVEALCHEMIST_DIR" \ - || echo "WARN: archivealchemist clone failed, skipping" -fi +install_archivealchemist() { + ARCHIVEALCHEMIST_DIR="$HOME/git/archivealchemist" + if [ ! -d "$ARCHIVEALCHEMIST_DIR" ]; then + mkdir -p "$HOME/git" + retry clone_repo https://github.com/avlidienbrunn/archivealchemist "$ARCHIVEALCHEMIST_DIR" \ + || { echo "WARN: archivealchemist clone failed, skipping" >&2; return 1; } + fi +} +run_stage archivealchemist install_archivealchemist + +validate_tools() { + local tool missing=0 + for tool in nuclei httpx subfinder naabu dnsx uncover alterx tlsx asnmap; do + have_pd_tool "$tool" || { echo "Missing required tool: $tool" >&2; missing=1; } + done + for tool in katana protoscope interactsh-client 2fa kr caido-cli caido-mcp-server \ + burp exiftool agent-browser wrangler ast-grep waymore pacu; do + have "$tool" || { echo "Missing required tool: $tool" >&2; missing=1; } + done + for artifact in /opt/burp/burpsuite.jar "$HOME/git/fireprox/fire.py" \ + "$HOME/git/archivealchemist/archive-alchemist.py"; do + [ -s "$artifact" ] || { echo "Missing required artifact: $artifact" >&2; missing=1; } + done + return "$missing" +} +run_stage validation validate_tools -# -- Clean up Go build cache ----------------------------------------------- +if [ "${#failed_stages[@]}" -gt 0 ]; then + echo "web-security installation incomplete; failed stages: ${failed_stages[*]}" >&2 + exit 1 +fi +# Retain downloaded modules after a failed pass so the next pass can reuse them. if [ "$need_go" = true ]; then go clean -cache -modcache 2>/dev/null || true fi diff --git a/capabilities/web-security/tests/test_caido_go_mcp.py b/capabilities/web-security/tests/test_caido_go_mcp.py index 8bbc8bf..e53f151 100644 --- a/capabilities/web-security/tests/test_caido_go_mcp.py +++ b/capabilities/web-security/tests/test_caido_go_mcp.py @@ -60,8 +60,7 @@ def test_auth_and_redaction_env_are_optional(self) -> None: # raises at connect time (the server falls back to the token file). assert env["CAIDO_ACCESS_TOKEN"] == "${CAIDO_ACCESS_TOKEN:-}" assert ( - env["CAIDO_ALLOW_SENSITIVE_HEADERS"] - == "${CAIDO_ALLOW_SENSITIVE_HEADERS:-}" + env["CAIDO_ALLOW_SENSITIVE_HEADERS"] == "${CAIDO_ALLOW_SENSITIVE_HEADERS:-}" ) def test_has_init_timeout(self) -> None: @@ -118,7 +117,9 @@ def test_verifies_checksum_before_install(self) -> None: # placed on PATH — never install an unverified binary. assert "sha256sum -c -" in INSTALL_SCRIPT verify = INSTALL_SCRIPT.index("sha256sum -c -") - install = INSTALL_SCRIPT.index("install -m 0755 /tmp/caido-mcp-server") + install = INSTALL_SCRIPT.index( + 'install -m 0755 "${INSTALL_TMP}/caido-mcp-server"' + ) assert verify < install, "checksum must be verified before install" def test_checksum_mismatch_skips_install(self) -> None: @@ -126,7 +127,7 @@ def test_checksum_mismatch_skips_install(self) -> None: def test_installs_onto_path(self) -> None: assert ( - "install -m 0755 /tmp/caido-mcp-server /usr/local/bin/caido-mcp-server" + 'install -m 0755 "${INSTALL_TMP}/caido-mcp-server" /usr/local/bin/caido-mcp-server' in INSTALL_SCRIPT ) @@ -135,4 +136,4 @@ def test_idempotent_guard(self) -> None: assert "if ! command -v caido-mcp-server &>/dev/null; then" in INSTALL_SCRIPT def test_cleans_up_temp_download(self) -> None: - assert "rm -f /tmp/caido-mcp-server" in INSTALL_SCRIPT + assert 'rm -f "${INSTALL_TMP}/caido-mcp-server"' in INSTALL_SCRIPT diff --git a/capabilities/web-security/tests/test_caido_mode_skill.py b/capabilities/web-security/tests/test_caido_mode_skill.py index 849171b..1771f97 100644 --- a/capabilities/web-security/tests/test_caido_mode_skill.py +++ b/capabilities/web-security/tests/test_caido_mode_skill.py @@ -226,13 +226,10 @@ def test_resolves_skill_dir_from_capability_root(self) -> None: ) def test_install_is_guarded_on_package_json_and_installed_deps(self) -> None: - # Guarded on both: package.json alone re-runs `npm install` on every - # boot, which reaches the registry even when the dependencies are - # already present — an outbound attempt a disconnected deployment - # cannot satisfy and does not need. + # Check declared dependencies locally before invoking npm install. assert ( 'if [ -f "$CAIDO_MODE_DIR/package.json" ] ' - '&& [ ! -d "$CAIDO_MODE_DIR/node_modules" ]; then' + '&& ! ( cd "$CAIDO_MODE_DIR" && npm ls --depth=0 >/dev/null 2>&1 ); then' ) in INSTALL_SCRIPT def test_install_failure_is_non_fatal(self) -> None: diff --git a/capabilities/web-security/tests/test_install_tools_offline.py b/capabilities/web-security/tests/test_install_tools_offline.py index 28e1dbf..1a56825 100644 --- a/capabilities/web-security/tests/test_install_tools_offline.py +++ b/capabilities/web-security/tests/test_install_tools_offline.py @@ -1,17 +1,9 @@ -"""The install script must complete without reaching the network when its tools are present. +"""Guarded installs, pinned versions, and recovery from partial provisioning. -Self-hosted deployments run with no route to the internet, and this script -executes on every sandbox boot where the capability changed — not just the -first. An unguarded download is therefore a repeated outbound attempt that -cannot succeed, and on a disconnected install one failed fetch aborts the whole -script under ``set -e``, taking the rest of the tooling with it. - -These pin the two properties that keep that from happening: every fetch is -guarded on the artefact it produces, and every version is pinned. +Full-script tests stub external commands and redirect all writes into tmp_path. +No system packages or network access are needed. """ -from __future__ import annotations - import os import re import subprocess @@ -72,7 +64,9 @@ def test_missing_go_extracts_with_noninteractive_sudo(tmp_path: Path) -> None: local = tmp_path / "usr-local" result = _run_go_setup(tmp_path, local) assert result.returncode == 0, result.stderr - assert (tmp_path / "sudo.log").read_text().strip() == f"-n tar -xz -C {local}" + assert ( + tmp_path / "sudo.log" + ).read_text().strip() == f"-n tar -xzf {tmp_path}/go.tar.gz -C {local}" def _run_go_setup(tmp_path: Path, local: Path) -> subprocess.CompletedProcess[str]: @@ -81,9 +75,9 @@ def _run_go_setup(tmp_path: Path, local: Path) -> subprocess.CompletedProcess[st bindir.mkdir() commands = { "id": "printf '1000\\n'", - "curl": "printf 'archive\\n'", + "curl": "exit 0", "sudo": 'printf "%s\\n" "$*" > "$HOME/sudo.log"; shift; exec "$@"', - "tar": 'read -r archive; test "$archive" = archive', + "tar": '/bin/mkdir -p "$4/go/bin"; printf "#!/bin/bash\\nexit 0\\n" > "$4/go/bin/go"; /bin/chmod +x "$4/go/bin/go"', } for name, body in commands.items(): executable = bindir / name @@ -98,10 +92,14 @@ def _run_go_setup(tmp_path: Path, local: Path) -> subprocess.CompletedProcess[st + path_line + "\n" + _shell_function("as_root") + + _shell_function("have") + + _shell_function("retry") + + _shell_function("run_stage") + + 'failed_stages=(); INSTALL_TMP="$HOME"\n' + "missing_go_tools=protoscope; missing_pd_tools=''; ARCH=x86_64; GO_VERSION=test\n" + "need_go=false" + go_setup - + "\ncommand -v go || true\n" + + '\ntest "${#failed_stages[@]}" -eq 0; command -v go\n' ).replace("/usr/local", str(local)) return subprocess.run( ["/bin/bash", "-c", script], @@ -149,14 +147,14 @@ def test_projectdiscovery_tools_use_explicit_versions(self) -> None: def test_toolchain_and_kiterunner_versions_are_pinned(self) -> None: assert 'GO_VERSION="1.26.6"' in INSTALL_SCRIPT assert 'KITERUNNER_VERSION="v1.0.2"' in INSTALL_SCRIPT - assert 'git clone --depth 1 --branch "$KITERUNNER_VERSION"' in INSTALL_SCRIPT + assert '--branch "$KITERUNNER_VERSION"' in INSTALL_SCRIPT class TestFetchesAreGuarded: def test_every_go_install_is_guarded(self) -> None: unguarded = [] for i, line in enumerate(LINES): - if not line.strip().startswith(("go install", " go install")): + if not line.strip().startswith(("go install", "retry go install")): continue if "have " not in _preceding_context(i): unguarded.append(line.strip()) @@ -164,7 +162,7 @@ def test_every_go_install_is_guarded(self) -> None: def test_global_npm_install_is_guarded(self) -> None: for i, line in enumerate(LINES): - if re.search(r"^\s*(as_root\s+)?npm install -g", line): + if re.search(r"^\s*(retry\s+)?(as_root\s+)?npm install -g", line): assert "have " in _preceding_context( i ), f"unguarded global npm install at line {i + 1}: {line.strip()}" @@ -176,7 +174,7 @@ def test_npm_installs_are_version_pinned(self) -> None: unpinned = [ line.strip() for line in LINES - if re.search(r"^\s*(as_root\s+)?npm install -g", line) + if re.search(r"^\s*(retry\s+)?(as_root\s+)?npm install -g", line) and not re.search(r"@\$?\{?[A-Za-z0-9_.-]+\}?", line.split("-g", 1)[-1]) and not line.strip().startswith("#") ] @@ -254,7 +252,9 @@ def test_wrangler_install_is_guarded_and_pinned(self) -> None: ) assert "have wrangler" in _preceding_context(idx, span=6) pin = next( - i for i, line in enumerate(LINES) if line.startswith("WRANGLER_VERSION=") + i + for i, line in enumerate(LINES) + if line.strip().startswith("WRANGLER_VERSION=") ) assert re.fullmatch( r"WRANGLER_VERSION=\"[0-9]+\.[0-9]+\.[0-9]+\"", @@ -266,17 +266,14 @@ def test_git_clones_are_guarded_on_target_dir(self) -> None: # on the target directory existing. Clones to /tmp (kiterunner) are # guarded on the binary they produce. for i, line in enumerate(LINES): - if "git clone" not in line or line.strip().startswith("#"): + if "retry clone_repo" not in line or line.strip().startswith("#"): continue ctx = _preceding_context(i, span=6) has_dir_guard = "! -d " in ctx or "have " in ctx assert has_dir_guard, f"unguarded git clone at line {i + 1}: {line.strip()}" def test_go_toolchain_is_only_fetched_when_something_needs_building(self) -> None: - # The toolchain is a ~150 MB download whose only purpose is building - # the tools above. A runtime that already carries them must never ask - # for it — which is what made the whole script abort at its first line - # on a disconnected install. + # Fetch the Go toolchain only when a missing tool requires compilation. idx = next(i for i, line in enumerate(LINES) if "go.dev/dl/go" in line) context = _preceding_context(idx, span=10) assert "need_go" in context @@ -338,11 +335,9 @@ def test_nodejs_apt_uses_as_root(self) -> None: assert "as_root" in LINES[idx] -class TestFailuresStayNonFatal: - def test_optional_vendor_downloads_do_not_abort_the_run(self) -> None: - # Vendor downloads, system packages, and optional tooling all degrade - # gracefully — a disconnected deployment must not have its entire - # provision aborted because one optional tool could not be fetched. +class TestStageFailureDiagnostics: + def test_failed_stages_keep_actionable_diagnostics(self) -> None: + # Stages retain a specific error in addition to the final failed-stage list. for marker in ( "WARN: Caido CLI install failed", "WARN: Burp Suite download failed", @@ -350,8 +345,6 @@ def test_optional_vendor_downloads_do_not_abort_the_run(self) -> None: "WARN: exiftool install failed", "WARN: Node.js install failed", "WARN: kiterunner clone failed", - "WARN: fireprox clone failed", - "WARN: fireprox requirements install failed", "WARN: archivealchemist clone failed", "WARN: ast-grep install failed", "WARN: waymore install failed", @@ -360,4 +353,259 @@ def test_optional_vendor_downloads_do_not_abort_the_run(self) -> None: "WARN: wrangler install failed", "WARN: caido-mode npm install failed", ): - assert marker in INSTALL_SCRIPT, f"missing non-fatal fallback: {marker}" + assert marker in INSTALL_SCRIPT, f"missing stage diagnostic: {marker}" + + +def _command(path: Path, body: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"#!/bin/bash\nset -eu\n{body}\n") + path.chmod(0o755) + + +def _installer_fixture(tmp_path: Path) -> Path: + """Run the complete installer with no host tools, network, or privileged writes.""" + import shutil + + bindir = tmp_path / "bin" + bindir.mkdir() + for name in ( + "bash", + "mkdir", + "mktemp", + "rm", + "mv", + "chmod", + "touch", + "dirname", + "install", + ): + executable = shutil.which(name) + assert executable + (bindir / name).symlink_to(executable) + for name in ( + "httpx", + "subfinder", + "naabu", + "dnsx", + "uncover", + "alterx", + "tlsx", + "asnmap", + "katana", + "protoscope", + "interactsh-client", + "2fa", + "kr", + "caido-cli", + "caido-mcp-server", + "burp", + "exiftool", + "ast-grep", + "waymore", + "pacu", + "python3", + ): + _command(bindir / name, "exit 0") + _command( + bindir / "uname", 'if [ "$1" = -s ]; then echo Linux; else echo x86_64; fi' + ) + _command(bindir / "node", "echo 24") + _command(bindir / "sleep", "exit 0") + # Any unexpected fetch is a hard failure, never a real network request. + for name in ("curl", "git", "sudo", "apt-get", "uv", "pip"): + _command( + bindir / name, f'echo "unexpected {name}" >> "$HOME/commands"; exit 97' + ) + _command( + bindir / "go", + """ +if [ "$1" != install ]; then exit 0; fi +count=0 +if [ -f "$HOME/go-attempts" ]; then read -r count < "$HOME/go-attempts"; fi +count=$((count + 1)) +echo "$count" > "$HOME/go-attempts" +if [ "$count" -le "${GO_FAILURES:-0}" ]; then echo 'transient checksum fetch error' >&2; exit 1; fi +if [ "${GO_NO_ARTIFACT:-0}" = 1 ]; then exit 0; fi +mkdir -p "$GOBIN" +printf '#!/bin/bash\\nexit 0\\n' > "$GOBIN/nuclei" +chmod +x "$GOBIN/nuclei" +""", + ) + _command( + bindir / "npm", + """ +if [ "$1" = ls ]; then test -f node_modules/complete; exit; fi +if [ "${2:-}" = -g ]; then + echo wrangler-install >> "$HOME/commands" + printf '#!/bin/bash\\nexit 0\\n' > "$HOME/bin/wrangler" + chmod +x "$HOME/bin/wrangler" + exit 0 +fi +echo npm-install >> "$HOME/commands" +mkdir -p node_modules +if [ "${NPM_FAIL:-0}" = 1 ]; then exit 1; fi +touch node_modules/complete +""", + ) + _command( + bindir / "agent-browser", + """ +echo browser-install >> "$HOME/commands" +mkdir -p "$HOME/.cache/agent-browser" +if [ "${BROWSER_FAIL:-0}" = 1 ]; then exit 1; fi +""", + ) + # Wrangler is deliberately missing: prove a later installation runs after Go fails. + _command(bindir / "id", "echo 0") + for relative in ( + "opt/burp/burpsuite.jar", + "git/fireprox/fire.py", + "git/fireprox/requirements.txt", + "git/fireprox/.dreadnode-deps-installed", + "git/archivealchemist/archive-alchemist.py", + "skills/caido-mode/package.json", + "skills/caido-mode/node_modules/complete", + ".cache/agent-browser/.dreadnode-installed", + ): + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("installed\n") + script = tmp_path / "install.sh" + script.write_text( + INSTALL_SCRIPT.replace("/usr/local", str(tmp_path / "usr-local")).replace( + "/opt/burp", str(tmp_path / "opt/burp") + ) + ) + return script + + +def _run_installer(script: Path, **extra: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["/bin/bash", str(script)], + env={ + "HOME": str(script.parent), + "PATH": str(script.parent / "bin"), + "CAPABILITY_ROOT": str(script.parent), + "TMPDIR": str(script.parent), + **extra, + }, + cwd=script.parent, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + + +def test_transient_go_failure_recovers_with_bounded_retries(tmp_path: Path) -> None: + script = _installer_fixture(tmp_path) + result = _run_installer(script, GO_FAILURES="2") + assert result.returncode == 0, result.stderr + assert (tmp_path / "go-attempts").read_text().strip() == "3" + assert "installed successfully" in result.stdout + + +def test_failed_stage_continues_and_rerun_repairs_only_missing_tools( + tmp_path: Path, +) -> None: + script = _installer_fixture(tmp_path) + first = _run_installer(script, GO_FAILURES="3") + assert first.returncode == 1, first.stderr + assert "failed stages: nuclei validation" in first.stderr + assert "installed successfully" not in first.stdout + assert (tmp_path / "bin/wrangler").is_file() + assert (tmp_path / "go-attempts").read_text().strip() == "3" + second = _run_installer(script) + assert second.returncode == 0, second.stderr + third = _run_installer(script) + assert third.returncode == 0, third.stderr + assert (tmp_path / "go-attempts").read_text().strip() == "4" + assert (tmp_path / "commands").read_text().splitlines() == ["wrangler-install"] + + +def test_zero_exit_without_required_binary_is_not_success(tmp_path: Path) -> None: + script = _installer_fixture(tmp_path) + result = _run_installer(script, GO_NO_ARTIFACT="1") + assert result.returncode == 1 + assert "Missing required tool: nuclei" in result.stderr + + +def test_partial_browser_and_npm_directories_do_not_prevent_repair( + tmp_path: Path, +) -> None: + script = _installer_fixture(tmp_path) + (tmp_path / ".cache/agent-browser/.dreadnode-installed").unlink() + (tmp_path / "skills/caido-mode/node_modules/complete").unlink() + first = _run_installer(script, BROWSER_FAIL="1", NPM_FAIL="1") + assert first.returncode == 1 + assert "failed stages: browser caido_mode" in first.stderr + second = _run_installer(script) + assert second.returncode == 0, second.stderr + assert (tmp_path / "skills/caido-mode/node_modules/complete").is_file() + assert (tmp_path / ".cache/agent-browser/.dreadnode-installed").is_file() + + +def test_fireprox_requirements_retry_after_successful_clone(tmp_path: Path) -> None: + script = _installer_fixture(tmp_path) + (tmp_path / "git/fireprox/.dreadnode-deps-installed").unlink() + first = _run_installer(script) + assert first.returncode == 1 + assert "failed stages: fireprox" in first.stderr + assert not (tmp_path / "git/fireprox/.dreadnode-deps-installed").exists() + _command(tmp_path / "bin/uv", "exit 0") + second = _run_installer(script) + assert second.returncode == 0, second.stderr + assert (tmp_path / "git/fireprox/.dreadnode-deps-installed").is_file() + + +def test_failed_burp_download_is_not_published_and_recovers(tmp_path: Path) -> None: + script = _installer_fixture(tmp_path) + jar = tmp_path / "opt/burp/burpsuite.jar" + jar.unlink() + _command( + tmp_path / "bin/curl", + """ +while [ "$1" != -o ]; do shift; done +printf partial > "$2" +echo download >> "$HOME/downloads" +exit "${DOWNLOAD_FAIL:-0}" +""", + ) + first = _run_installer(script, DOWNLOAD_FAIL="1") + assert first.returncode == 1 + assert not jar.exists() + assert len((tmp_path / "downloads").read_text().splitlines()) == 3 + second = _run_installer(script) + assert second.returncode == 0, second.stderr + assert jar.read_text() == "partial" + + +def test_sealed_install_does_not_download_missing_browser(tmp_path: Path) -> None: + script = _installer_fixture(tmp_path) + (tmp_path / ".cache/agent-browser/.dreadnode-installed").unlink() + result = _run_installer(script, DREADNODE_CAPABILITY_INSTALL="sealed") + assert result.returncode == 0, result.stderr + assert "browser-install" not in (tmp_path / "commands").read_text() + + +def test_clone_retry_uses_fresh_staging_after_partial_failure(tmp_path: Path) -> None: + script = _installer_fixture(tmp_path) + # Model a new Fireprox install while retaining all other pre-baked tools. + (tmp_path / "git/fireprox").rename(tmp_path / "saved-fireprox") + _command( + tmp_path / "bin/git", + """ +while [ "$#" -gt 1 ]; do shift; done +test ! -f "$1/partial" +echo partial > "$1/partial" +echo clone >> "$HOME/clones" +if [ ! -f "$HOME/clone-retried" ]; then touch "$HOME/clone-retried"; exit 1; fi +echo requirements > "$1/requirements.txt" +echo source > "$1/fire.py" +""", + ) + _command(tmp_path / "bin/uv", "exit 0") + result = _run_installer(script) + assert result.returncode == 0, result.stderr + assert (tmp_path / "clones").read_text().splitlines() == ["clone", "clone"] + assert (tmp_path / "git/fireprox/.dreadnode-deps-installed").exists() From 1a1e68f7d378f1acaed4fbde9c822ed0ab6bfeac Mon Sep 17 00:00:00 2001 From: Daniel Vaughn Date: Tue, 29 Sep 2026 11:52:00 -0400 Subject: [PATCH 3/5] fix: use the right permissions for Python tools --- .../web-security/scripts/install_tools.sh | 22 ++++-- .../tests/test_install_tools_offline.py | 78 +++++++++++++++++++ 2 files changed, 94 insertions(+), 6 deletions(-) diff --git a/capabilities/web-security/scripts/install_tools.sh b/capabilities/web-security/scripts/install_tools.sh index 94530cf..8301003 100755 --- a/capabilities/web-security/scripts/install_tools.sh +++ b/capabilities/web-security/scripts/install_tools.sh @@ -88,14 +88,24 @@ as_root() { fi } -# Install Python packages using uv when available, then pip or python3 -m pip. +# Target the selected Python interpreter. Virtualenv installs run as the +# current user; system installs use root privileges and system-package flags. py_install() { - if command -v uv >/dev/null 2>&1; then - uv pip install --python "$(command -v python3)" "$@" - elif command -v pip >/dev/null 2>&1; then - pip install --break-system-packages "$@" + local python in_venv uv + local -a command system_flags + python="$(command -v python3)" || return + in_venv="$("$python" -c 'import sys; print(int(sys.prefix != sys.base_prefix))')" || return + if uv="$(command -v uv)"; then + command=("$uv" pip install --python "$python") + system_flags=(--system --break-system-packages) else - python3 -m pip install --break-system-packages "$@" + command=("$python" -m pip install) + system_flags=(--break-system-packages) + fi + if [ "$in_venv" = 1 ]; then + "${command[@]}" "$@" + else + as_root "${command[@]}" "${system_flags[@]}" "$@" fi } diff --git a/capabilities/web-security/tests/test_install_tools_offline.py b/capabilities/web-security/tests/test_install_tools_offline.py index 1a56825..c411e61 100644 --- a/capabilities/web-security/tests/test_install_tools_offline.py +++ b/capabilities/web-security/tests/test_install_tools_offline.py @@ -7,6 +7,8 @@ import os import re import subprocess + +import pytest from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -609,3 +611,79 @@ def test_clone_retry_uses_fresh_staging_after_partial_failure(tmp_path: Path) -> assert result.returncode == 0, result.stderr assert (tmp_path / "clones").read_text().splitlines() == ["clone", "clone"] assert (tmp_path / "git/fireprox/.dreadnode-deps-installed").exists() + + +@pytest.mark.parametrize("with_uv", [True, False]) +@pytest.mark.parametrize("in_venv", [True, False]) +@pytest.mark.parametrize("uid", [0, 1000]) +def test_python_install_targets_interpreter_with_required_privileges( + tmp_path: Path, with_uv: bool, in_venv: bool, uid: int +) -> None: + bindir = tmp_path / "bin" + python = bindir / "python3" + _command( + python, + """ +if [ "$1" = -c ]; then echo "$TEST_IN_VENV"; exit 0; fi +printf '%s\\n' "$0" "$@" > "$HOME/install-args" +""", + ) + _command(bindir / "id", f"echo {uid}") + _command( + bindir / "sudo", + """ +printf '%s\\n' "$@" > "$HOME/sudo-args" +test "$1" = -n +shift +# Model sudo's restricted PATH: the selected executable must be absolute. +PATH=/unavailable exec "$@" +""", + ) + # A separate pip executable must not override the selected interpreter. + _command(bindir / "pip", "exit 99") + if with_uv: + _command(bindir / "uv", 'printf \'%s\\n\' "$0" "$@" > "$HOME/install-args"') + script = _shell_function("as_root") + _shell_function("py_install") + result = subprocess.run( + ["/bin/bash", "-c", script + "\npy_install 'example>=1'\n"], + env={ + "HOME": str(tmp_path), + "PATH": str(bindir), + "TEST_IN_VENV": str(int(in_venv)), + }, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + expected = ( + [str(bindir / "uv"), "pip", "install", "--python", str(python)] + if with_uv + else [str(python), "-m", "pip", "install"] + ) + if not in_venv: + if with_uv: + expected.append("--system") + expected.append("--break-system-packages") + expected.append("example>=1") + assert (tmp_path / "install-args").read_text().splitlines() == expected + sudo_args = tmp_path / "sudo-args" + if uid != 0 and not in_venv: + assert sudo_args.read_text().splitlines() == ["-n", *expected] + else: + assert not sudo_args.exists() + + +def test_python_probe_failure_does_not_attempt_install(tmp_path: Path) -> None: + bindir = tmp_path / "bin" + _command(bindir / "python3", "exit 42") + _command(bindir / "uv", ': > "$HOME/attempted"') + _command(bindir / "sudo", ': > "$HOME/attempted"') + script = _shell_function("as_root") + _shell_function("py_install") + result = subprocess.run( + ["/bin/bash", "-c", script + "\npy_install example\n"], + env={"HOME": str(tmp_path), "PATH": str(bindir)}, + check=False, + ) + assert result.returncode == 42 + assert not (tmp_path / "attempted").exists() From c4569ee710395431631ae784972b622bd1658b93 Mon Sep 17 00:00:00 2001 From: Daniel Vaughn Date: Tue, 29 Sep 2026 12:14:41 -0400 Subject: [PATCH 4/5] fix: reuse browsers and respect Python permissions --- .../web-security/scripts/install_tools.sh | 42 +++++-- .../tests/test_install_tools_offline.py | 105 ++++++++++++++++-- 2 files changed, 125 insertions(+), 22 deletions(-) diff --git a/capabilities/web-security/scripts/install_tools.sh b/capabilities/web-security/scripts/install_tools.sh index 8301003..e456242 100755 --- a/capabilities/web-security/scripts/install_tools.sh +++ b/capabilities/web-security/scripts/install_tools.sh @@ -88,13 +88,32 @@ as_root() { fi } -# Target the selected Python interpreter. Virtualenv installs run as the -# current user; system installs use root privileges and system-package flags. +# Target the selected interpreter. Elevate only for non-virtualenv installs +# whose package or script directories are not writable by the current user. py_install() { - local python in_venv uv + local python install_access uv local -a command system_flags python="$(command -v python3)" || return - in_venv="$("$python" -c 'import sys; print(int(sys.prefix != sys.base_prefix))')" || return + install_access="$("$python" -c ' +import os +from pathlib import Path +import sys +import sysconfig + +if sys.prefix != sys.base_prefix: + print("venv") +else: + paths = sysconfig.get_paths() + writable = True + for key in ("purelib", "platlib", "scripts", "data"): + target = Path(paths[key]) + while not target.exists() and target != target.parent: + target = target.parent + if not target.is_dir() or not os.access(target, os.W_OK | os.X_OK): + writable = False + break + print("writable" if writable else "privileged") +')" || return if uv="$(command -v uv)"; then command=("$uv" pip install --python "$python") system_flags=(--system --break-system-packages) @@ -102,8 +121,10 @@ py_install() { command=("$python" -m pip install) system_flags=(--break-system-packages) fi - if [ "$in_venv" = 1 ]; then + if [ "$install_access" = venv ]; then "${command[@]}" "$@" + elif [ "$install_access" = writable ]; then + "${command[@]}" "${system_flags[@]}" "$@" else as_root "${command[@]}" "${system_flags[@]}" "$@" fi @@ -346,14 +367,11 @@ install_browser() { retry as_root npm install -g "agent-browser@${AGENT_BROWSER_VERSION}" \ || { echo "WARN: agent-browser install failed, skipping" >&2; return 1; } fi - # `agent-browser install` downloads the browser binaries themselves. Guarded on - # a completion marker so a failed download that creates the cache directory - # is retried on the next pass. Sealed deployments still skip browser downloads. + # Reuse browser caches supplied by the image. Browser downloads are optional + # and are skipped entirely in sealed deployments. AGENT_BROWSER_CACHE="${AGENT_BROWSER_CACHE_DIR:-$HOME/.cache/agent-browser}" - if [ "${DREADNODE_CAPABILITY_INSTALL:-}" != "sealed" ] && [ ! -f "$AGENT_BROWSER_CACHE/.dreadnode-installed" ]; then - retry agent-browser install || { echo "WARN: agent-browser browser download failed, skipping" >&2; return 1; } - mkdir -p "$AGENT_BROWSER_CACHE" - touch "$AGENT_BROWSER_CACHE/.dreadnode-installed" + if [ "${DREADNODE_CAPABILITY_INSTALL:-}" != "sealed" ] && [ ! -d "$AGENT_BROWSER_CACHE" ]; then + retry agent-browser install || echo "WARN: agent-browser browser download failed, skipping" >&2 fi } run_stage browser install_browser diff --git a/capabilities/web-security/tests/test_install_tools_offline.py b/capabilities/web-security/tests/test_install_tools_offline.py index c411e61..5a506ad 100644 --- a/capabilities/web-security/tests/test_install_tools_offline.py +++ b/capabilities/web-security/tests/test_install_tools_offline.py @@ -7,6 +7,8 @@ import os import re import subprocess +import sys +import shlex import pytest from pathlib import Path @@ -532,19 +534,17 @@ def test_zero_exit_without_required_binary_is_not_success(tmp_path: Path) -> Non assert "Missing required tool: nuclei" in result.stderr -def test_partial_browser_and_npm_directories_do_not_prevent_repair( +def test_partial_npm_directory_does_not_prevent_repair( tmp_path: Path, ) -> None: script = _installer_fixture(tmp_path) - (tmp_path / ".cache/agent-browser/.dreadnode-installed").unlink() (tmp_path / "skills/caido-mode/node_modules/complete").unlink() - first = _run_installer(script, BROWSER_FAIL="1", NPM_FAIL="1") + first = _run_installer(script, NPM_FAIL="1") assert first.returncode == 1 - assert "failed stages: browser caido_mode" in first.stderr + assert "failed stages: caido_mode" in first.stderr second = _run_installer(script) assert second.returncode == 0, second.stderr assert (tmp_path / "skills/caido-mode/node_modules/complete").is_file() - assert (tmp_path / ".cache/agent-browser/.dreadnode-installed").is_file() def test_fireprox_requirements_retry_after_successful_clone(tmp_path: Path) -> None: @@ -584,7 +584,7 @@ def test_failed_burp_download_is_not_published_and_recovers(tmp_path: Path) -> N def test_sealed_install_does_not_download_missing_browser(tmp_path: Path) -> None: script = _installer_fixture(tmp_path) - (tmp_path / ".cache/agent-browser/.dreadnode-installed").unlink() + (tmp_path / ".cache/agent-browser").rename(tmp_path / "saved-browser") result = _run_installer(script, DREADNODE_CAPABILITY_INSTALL="sealed") assert result.returncode == 0, result.stderr assert "browser-install" not in (tmp_path / "commands").read_text() @@ -615,16 +615,17 @@ def test_clone_retry_uses_fresh_staging_after_partial_failure(tmp_path: Path) -> @pytest.mark.parametrize("with_uv", [True, False]) @pytest.mark.parametrize("in_venv", [True, False]) +@pytest.mark.parametrize("writable", [True, False]) @pytest.mark.parametrize("uid", [0, 1000]) def test_python_install_targets_interpreter_with_required_privileges( - tmp_path: Path, with_uv: bool, in_venv: bool, uid: int + tmp_path: Path, with_uv: bool, in_venv: bool, writable: bool, uid: int ) -> None: bindir = tmp_path / "bin" python = bindir / "python3" _command( python, """ -if [ "$1" = -c ]; then echo "$TEST_IN_VENV"; exit 0; fi +if [ "$1" = -c ]; then echo "$TEST_INSTALL_ACCESS"; exit 0; fi printf '%s\\n' "$0" "$@" > "$HOME/install-args" """, ) @@ -649,7 +650,11 @@ def test_python_install_targets_interpreter_with_required_privileges( env={ "HOME": str(tmp_path), "PATH": str(bindir), - "TEST_IN_VENV": str(int(in_venv)), + "TEST_INSTALL_ACCESS": "venv" + if in_venv + else "writable" + if writable + else "privileged", }, capture_output=True, text=True, @@ -668,7 +673,7 @@ def test_python_install_targets_interpreter_with_required_privileges( expected.append("example>=1") assert (tmp_path / "install-args").read_text().splitlines() == expected sudo_args = tmp_path / "sudo-args" - if uid != 0 and not in_venv: + if uid != 0 and not in_venv and not writable: assert sudo_args.read_text().splitlines() == ["-n", *expected] else: assert not sudo_args.exists() @@ -687,3 +692,83 @@ def test_python_probe_failure_does_not_attempt_install(tmp_path: Path) -> None: ) assert result.returncode == 42 assert not (tmp_path / "attempted").exists() + + +@pytest.mark.parametrize("sealed", [True, False]) +def test_existing_browser_cache_needs_no_marker(tmp_path: Path, sealed: bool) -> None: + script = _installer_fixture(tmp_path) + (tmp_path / ".cache/agent-browser/.dreadnode-installed").unlink() + result = _run_installer( + script, + BROWSER_FAIL="1", + DREADNODE_CAPABILITY_INSTALL="sealed" if sealed else "", + ) + assert result.returncode == 0, result.stderr + assert "browser-install" not in (tmp_path / "commands").read_text() + + +def test_optional_browser_download_failure_does_not_fail_install( + tmp_path: Path, +) -> None: + script = _installer_fixture(tmp_path) + (tmp_path / ".cache/agent-browser").rename(tmp_path / "saved-browser") + result = _run_installer(script, BROWSER_FAIL="1") + assert result.returncode == 0, result.stderr + assert "WARN: agent-browser browser download failed" in result.stderr + assert (tmp_path / "commands").read_text().splitlines().count( + "browser-install" + ) == 3 + + +@pytest.mark.parametrize("blocked", [None, "purelib", "platlib", "scripts", "data"]) +def test_python_probe_checks_install_destinations( + tmp_path: Path, blocked: str | None +) -> None: + # Execute the actual interpreter probe with isolated installation paths. + # Package directories may not exist yet; their parent must be writable. + paths = { + key: str(tmp_path / key / "new") + for key in ("purelib", "platlib", "scripts", "data") + } + for key in paths: + (tmp_path / key).mkdir() + probe_setup = ( + "import os, sys, sysconfig; " + "sys.prefix = sys.base_prefix; " + f"sysconfig.get_paths = lambda: {paths!r}; " + ) + if blocked is not None: + probe_setup += ( + "original_access = os.access; " + f"os.access = lambda p, mode: str(p) != {str(tmp_path / blocked)!r} and original_access(p, mode); " + ) + probe_setup += "exec(sys.argv[1])" + bindir = tmp_path / "bin" + _command( + bindir / "python3", + "\n".join( + [ + 'if [ "$1" = -c ]; then', + f' exec {shlex.quote(sys.executable)} -c {shlex.quote(probe_setup)} "$2"', + "fi", + "exit 98", + ] + ), + ) + _command(bindir / "uv", "echo installed") + _command(bindir / "id", "echo 1000") + # No sudo is available in this environment. + script = _shell_function("as_root") + _shell_function("py_install") + result = subprocess.run( + ["/bin/bash", "-c", script + "\npy_install example\n"], + env={"HOME": str(tmp_path), "PATH": str(bindir)}, + text=True, + capture_output=True, + check=False, + ) + if blocked is None: + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "installed" + else: + assert result.returncode != 0 + assert "installed" not in result.stdout From fdcec68c9347359d7ef5dd3dab8fad6743cc31cb Mon Sep 17 00:00:00 2001 From: Daniel Vaughn Date: Tue, 29 Sep 2026 12:20:55 -0400 Subject: [PATCH 5/5] test: drop unused browser install marker Co-Authored-By: Claude Opus 5.5 --- capabilities/web-security/tests/test_install_tools_offline.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/capabilities/web-security/tests/test_install_tools_offline.py b/capabilities/web-security/tests/test_install_tools_offline.py index 5a506ad..4266be4 100644 --- a/capabilities/web-security/tests/test_install_tools_offline.py +++ b/capabilities/web-security/tests/test_install_tools_offline.py @@ -469,7 +469,7 @@ def _installer_fixture(tmp_path: Path) -> Path: "git/archivealchemist/archive-alchemist.py", "skills/caido-mode/package.json", "skills/caido-mode/node_modules/complete", - ".cache/agent-browser/.dreadnode-installed", + ".cache/agent-browser/chromium", ): path = tmp_path / relative path.parent.mkdir(parents=True, exist_ok=True) @@ -697,7 +697,6 @@ def test_python_probe_failure_does_not_attempt_install(tmp_path: Path) -> None: @pytest.mark.parametrize("sealed", [True, False]) def test_existing_browser_cache_needs_no_marker(tmp_path: Path, sealed: bool) -> None: script = _installer_fixture(tmp_path) - (tmp_path / ".cache/agent-browser/.dreadnode-installed").unlink() result = _run_installer( script, BROWSER_FAIL="1",