diff --git a/.gitignore b/.gitignore index 1b3727c..2be3db8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,211 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python,visualstudiocode,virtualenv +# Edit at https://www.toptal.com/developers/gitignore?templates=python,visualstudiocode,virtualenv + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments .env -.env.bak* +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +### VirtualEnv ### +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +pip-selfcheck.json + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode,virtualenv + data htpasswd *.log @@ -7,5 +213,3 @@ htpasswd .rendered/ # Rendered stack files (generated by stackctl/tools) *.rendered.yml -.venv/ -__pycache__/ \ No newline at end of file diff --git a/.sops.yaml b/.sops.yaml deleted file mode 100644 index 938ab94..0000000 --- a/.sops.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# SOPS configuration for encrypting secrets committed to the repo -creation_rules: - - path_regex: secrets/.*\.(env|yaml|yml)$ - # Replace with your age public key(s) - age: - - "age1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # Replace with your real age public key - encrypted_regex: '^(?!#)' diff --git a/.sops.yaml.example b/.sops.yaml.example new file mode 100644 index 0000000..9829064 --- /dev/null +++ b/.sops.yaml.example @@ -0,0 +1,12 @@ +# Example SOPS configuration +# Copy to .sops.yaml and adjust keys/providers for your environment. +# Docs: https://github.com/getsops/sops + +creation_rules: + # Example: age key (recommended for simplicity) + - path_regex: + - ".*\\.enc\\.yaml$" + - ".*secret.*\\.yaml$" + age: ["age1exampleexampleexampleexampleexampleexampleexampleexamplex"] + encrypted_regex: ".*" + # or use kms/gcp_kms/pgp depending on your setup diff --git a/README.md b/README.md index 3d54f95..69c0db3 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,22 @@ See `stacks/README.md` for the full runbook or use `stackctl.sh` helpers: ./stackctl.sh up ``` +### Bootstrap (developer setup) + +To quickly prepare a development machine for working with this repository, there's a helper built into the main wrapper script. + +```bash +./stackctl.sh --bootstrap +``` + +What this does: +- Copies missing `.env` files from `.env.example` where present. +- Installs Python requirements from `tools/requirements.txt` when a Python virtualenv is active. +- Checks that Docker is installed and whether Docker Swarm is active (prints next steps if not). + +This is a safe convenience for local setup; it does not commit or overwrite existing `.env` files. See `tools/README.md` for more details about renderers and SOPS usage. + + ### 3. Set up core infrastructure (Compose - legacy local-only) ```bash diff --git a/apisix/api-gateway/.env.example.enc.yaml b/apisix/api-gateway/.env.example.enc.yaml new file mode 100644 index 0000000..d3141f0 --- /dev/null +++ b/apisix/api-gateway/.env.example.enc.yaml @@ -0,0 +1,5 @@ +# Example SOPS-encrypted file (placeholder values). Use 'sops' to encrypt/decrypt. +# This is a sample and contains no real secrets. +DATABASE_URL: ENC[AES256_GCM,data:PLACEHOLDER,iv:PLACEHOLDER,key:PLACEHOLDER,tags:PLACEHOLDER] +ADMIN_KEY: ENC[AES256_GCM,data:PLACEHOLDER,iv:PLACEHOLDER,key:PLACEHOLDER,tags:PLACEHOLDER] +VIEWER_KEY: ENC[AES256_GCM,data:PLACEHOLDER,iv:PLACEHOLDER,key:PLACEHOLDER,tags:PLACEHOLDER] diff --git a/docs/SOPS-usage.md b/docs/SOPS-usage.md new file mode 100644 index 0000000..a07d45a --- /dev/null +++ b/docs/SOPS-usage.md @@ -0,0 +1,34 @@ +# SOPS Usage Guide (local development) + +This repository supports using SOPS to keep secrets encrypted in the repository while allowing local +developers to decrypt them during setup. + +High level +- Keep only encrypted files in the repository (e.g. `.env.enc.yaml`). +- Use `.sops.yaml` (copy from `.sops.yaml.example`) to configure encryption backends (age/KMS/PGP). +- Locally, decrypt with the `sops` CLI and write to `.env` when needed. + +Examples + +1) Decrypt for local use + +```bash +# requires sops installed and the proper keys available in your keystore +./stackctl.sh secrets decrypt --in apisix/api-gateway/.env.example.enc.yaml --out apisix/api-gateway/.env --force +``` + +2) Encrypt a plaintext .env into an encrypted file + +```bash +# Use sops to encrypt; this will respect .sops.yaml +./stackctl.sh secrets encrypt --in apisix/api-gateway/.env --out apisix/api-gateway/.env.enc.yaml +``` + +3) Best practices +- Never commit plaintext `.env` files. Add `.env` to your global or repo `.gitignore`. +- Add `.sops.yaml` to your local environment (do not commit a production `.sops.yaml` with real keys). +- Rotate keys and update `.sops.yaml` as needed. + +Security notes +- The `stackctl_cli` helpers never print secret values; decrypt writes to a file. +- For CI, configure a decrypt step using appropriate secret storage (KMS/PGP) and avoid storing keys in the repo. diff --git a/stackctl.sh b/stackctl.sh index 6074018..1762afc 100755 --- a/stackctl.sh +++ b/stackctl.sh @@ -1,761 +1,121 @@ #!/usr/bin/env bash -# Safe/strict shell settings +# Compatibility wrapper for the Python CLI (tools/stackctl_cli.py) +# Keeps existing entrypoint while delegating to the new implementation. + set -euo pipefail IFS=$'\n\t' -SCRIPT_NAME="$(basename "$0")" - -print_usage() { - cat < [options] - -Manage the Docker Swarm stacks for this repo. - -Commands: - up Deploy the stacks (default if no command is provided) - down Remove the stacks - status List services for each stack - logs Follow logs for key services or specified services - doctor Run preflight checks and optional fixes - env List or recreate .env files from .env.example (safe-guarded) - help Show this help message and exit - -Examples: - $SCRIPT_NAME up --no-logs -s infrastructure,observability - $SCRIPT_NAME down -y --remove-network -s platform - $SCRIPT_NAME status -s infrastructure - $SCRIPT_NAME logs infrastructure_traefik observability_prometheus - -Common options: - -h, --help Show help (also works per-command) -USAGE -} - -# Helpers -log() { printf '%s\n' "$*" >&2; } -err() { log "ERROR: $*"; } - -# Determine repository root (directory containing this script) SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" -# Swarm stack definitions live under stacks/ (preferred); root-level docker-compose.* files are legacy fallbacks -STACKS_DIR="$SCRIPT_DIR/stacks" - -check_command() { - command -v "$1" >/dev/null 2>&1 || { err "'$1' is required but not installed or not on PATH"; exit 2; } -} - -# Prefer docker compose plugin, fall back to docker-compose if available -compose_config() { - local file="$1" - if docker compose version >/dev/null 2>&1; then - docker compose -f "$file" config - elif command -v docker-compose >/dev/null 2>&1; then - docker-compose -f "$file" config - else - err "Neither 'docker compose' nor 'docker-compose' is available to validate $file" - return 1 - fi -} - -# Render a compose file with per-service env interpolation using tools/render_compose.py -render_compose_file() { - local in_file="$1" - local out_dir out_file base base_no_ext - # Allow override via RENDER_DIR, default to repo-local hidden folder - out_dir="${RENDER_DIR:-$SCRIPT_DIR/.rendered}" - mkdir -p "$out_dir" - base="$(basename "$in_file")" - base_no_ext="${base%.yml}" - base_no_ext="${base_no_ext%.yaml}" - - # Keep rendered filenames prefixed with docker-compose.* for consistency - local out_base - case "$base" in - docker-compose.*.yml|docker-compose.*.yaml) - out_base="$base_no_ext" # already prefixed - ;; - *.yml|*.yaml) - # If coming from stacks/.yml, prefix with docker-compose. - local name_no_ext="$base_no_ext" - out_base="docker-compose.${name_no_ext}" - ;; - *) - out_base="$base_no_ext" - ;; - esac - out_file="$out_dir/${out_base}.rendered.yml" - - if command -v python3 >/dev/null 2>&1; then - if python3 "$SCRIPT_DIR/tools/render_compose.py" -i "$in_file" -o "$out_file" --repo-root "$SCRIPT_DIR" >/dev/null 2>&1; then - printf '%s\n' "$out_file" - return 0 - else - log "Warning: compose render failed for $in_file; using original file" - fi - else - log "Warning: python3 not found; skipping compose render for $in_file" - fi - printf '%s\n' "$in_file" -} - -# Find a stack file by name with common fallbacks (.yml/.yaml in repo root) -find_stack_file() { - local name="$1" - local candidates=( - "$SCRIPT_DIR/stacks/${name}.yml" - "$SCRIPT_DIR/stacks/${name}.yaml" - "$SCRIPT_DIR/docker-compose.${name}.yml" - "$SCRIPT_DIR/docker-compose.${name}.yaml" - ) - for f in "${candidates[@]}"; do - if [[ -f "$f" ]]; then - printf '%s\n' "$f" - return 0 - fi - done - return 1 -} - -ensure_swarm_info() { - SWARM_STATE="$(docker info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null || true)" - if [[ "$SWARM_STATE" != "active" ]]; then - log "Docker Swarm does not appear to be active on this host (Swarm state: ${SWARM_STATE:-unknown})." - log "If you intend to deploy stacks to a Swarm, run: docker swarm init" - fi -} - -ensure_traefik_network() { - local dry_run=${1:-false} - if docker network ls --format '{{.Name}}' | grep -qx "traefik-public"; then - log "Found existing 'traefik-public' network." - else - if [[ "${SWARM_STATE:-}" = "active" ]]; then - log "Creating 'traefik-public' overlay network (attachable)" - if [[ "$dry_run" = false ]]; then - docker network create --driver=overlay --attachable traefik-public || log "Warning: failed to create traefik-public network (may already exist)." - else - log "DRY-RUN: would create 'traefik-public' network" - fi - else - log "Skipping network creation because Swarm is not active." - fi - fi -} - -STACK_FILES=(infrastructure observability platform) -DEFAULT_LOG_SERVICES=(observability_prometheus observability_loki infrastructure_traefik) - -# Selected stacks (defaults to all unless overridden via -s/--stacks) -TARGET_STACKS=() - -# Parse a comma-separated stacks list and validate -set_target_stacks() { - local arg="${1:-}" - local tokens=() - local IFS=',' - read -r -a tokens <<< "$arg" - local parsed=() - for t in "${tokens[@]}"; do - t="${t//[[:space:]]/}" - [[ -z "$t" ]] && continue - local valid=false - for a in "${STACK_FILES[@]}"; do - if [[ "$t" == "$a" ]]; then - valid=true - break - fi - done - if [[ "$valid" == false ]]; then - err "Unknown stack '$t'. Allowed: ${STACK_FILES[*]}" - exit 2 - fi - parsed+=("$t") - done - if [[ ${#parsed[@]} -eq 0 ]]; then - err "No valid stacks specified with --stacks" - exit 2 - fi - TARGET_STACKS=("${parsed[@]}") -} - -# Discover directories containing a .env.example (portable across macOS/Linux) -discover_env_example_dirs() { - local found=() - while IFS= read -r path; do - [[ -z "$path" ]] && continue - local dir - dir="$(dirname "$path")" - found+=("$dir") - done < <( (find "$SCRIPT_DIR" -type f -name '.env.example' 2>/dev/null) || true ) - - if [[ ${#found[@]} -gt 0 ]]; then - printf '%s\n' "${found[@]}" | awk '!x[$0]++' | sort - fi -} - -# Parse comma-separated user-provided paths and normalize to absolute directories -parse_env_paths() { - local input="$1" - local IFS=',' - read -r -a toks <<< "$input" - local out=() - for p in "${toks[@]}"; do - p="${p//[[:space:]]/}" - [[ -z "$p" ]] && continue - local abs - if [[ -d "$SCRIPT_DIR/$p" ]]; then - abs="$(cd "$SCRIPT_DIR/$p" && pwd)" - elif [[ -f "$SCRIPT_DIR/$p" ]]; then - abs="$(cd "$(dirname "$SCRIPT_DIR/$p")" && pwd)" - else - if [[ -d "$p" ]]; then - abs="$(cd "$p" && pwd)" - elif [[ -f "$p" ]]; then - abs="$(cd "$(dirname "$p")" && pwd)" - else - err "Path not found: $p" - continue - fi - fi - out+=("$abs") - done - if [[ ${#out[@]} -gt 0 ]]; then - printf '%s\n' "${out[@]}" | awk '!x[$0]++' - fi -} - -backup_file() { - local file="$1" - local ts - ts="$(date +%Y%m%d%H%M%S)" - local backup="${file}.bak.${ts}" - cp -p "$file" "$backup" - printf '%s' "$backup" -} - -cmd_env() { - local DO_LIST=false - local DO_RECREATE=false - local FORCE=false - local ASSUME_YES=false - local DRY_RUN=false - local PATHS="" - - # Summary accumulators - local total=0 - local have_env=0 - local missing_env=0 - local skipped_no_example=0 - local created_count=0 - local overwritten_count=0 - local skipped_exists_count=0 - local missing_example_count=0 - local -a missing_env_list=() - local -a skipped_list=() - local -a created_list=() - local -a overwritten_list=() - local -a backup_list=() - local -a skipped_exists_list=() - local -a missing_example_list=() - - while [[ $# -gt 0 ]]; do - case "${1:-}" in - --list) - DO_LIST=true; shift ;; - --recreate) - DO_RECREATE=true; shift ;; - --paths) - [[ $# -lt 2 ]] && { err "--paths requires a comma-separated list of dirs/files"; exit 2; } - PATHS="${2:-}"; shift 2 ;; - -f|--force) - FORCE=true; shift ;; - -y|--yes) - ASSUME_YES=true; shift ;; - --dry-run) - DRY_RUN=true; shift ;; - -h|--help) - log "Manage .env files from .env.example. Options: --list, --recreate, --paths , -f/--force, -y/--yes, --dry-run"; exit 0 ;; - --) - shift; break ;; - -*) - printf '%s: unknown option for env: %s\n' "$SCRIPT_NAME" "$1" >&2; exit 2 ;; - *) - break ;; - esac - done - - if [[ "$DO_LIST" = false && "$DO_RECREATE" = false ]]; then - DO_LIST=true - fi - - local targets=() - if [[ -n "$PATHS" ]]; then - while IFS= read -r dir; do - [[ -z "$dir" ]] || targets+=("$dir") - done < <(parse_env_paths "$PATHS") - else - while IFS= read -r dir; do - [[ -z "$dir" ]] || targets+=("$dir") - done < <(discover_env_example_dirs) - fi - - if [[ ${#targets[@]} -eq 0 ]]; then - log "No .env.example locations discovered." - exit 0 - fi - - if [[ "$DO_LIST" = true ]]; then - log "Discovered env locations (status shows if .env exists):" - for dir in "${targets[@]}"; do - if [[ -f "$dir/.env.example" ]]; then - if [[ -f "$dir/.env" ]]; then - printf '[OK] %s (has .env)\n' "$dir" - have_env=$((have_env+1)) - else - printf '[MISSING] %s (no .env)\n' "$dir" - missing_env=$((missing_env+1)) - missing_env_list+=("$dir") - fi - else - printf '[SKIP] %s (no .env.example)\n' "$dir" - skipped_no_example=$((skipped_no_example+1)) - skipped_list+=("$dir") - fi - done - total=${#targets[@]} - printf '\nSummary: %d discovered | %d with .env | %d missing .env | %d without .env.example\n' "$total" "$have_env" "$missing_env" "$skipped_no_example" - if [[ $missing_env -gt 0 ]]; then - printf 'Missing .env in:\n' - for d in "${missing_env_list[@]}"; do printf ' - %s\n' "$d"; done - fi - if [[ $skipped_no_example -gt 0 ]]; then - printf 'No .env.example in:\n' - for d in "${skipped_list[@]}"; do printf ' - %s\n' "$d"; done - fi - fi - - if [[ "$DO_RECREATE" = true ]]; then - log "Recreating .env from .env.example for ${#targets[@]} location(s)" - for dir in "${targets[@]}"; do - local ex="$dir/.env.example" - local env="$dir/.env" - if [[ ! -f "$ex" ]]; then - log "Skipping: no .env.example in $dir" - missing_example_count=$((missing_example_count+1)) - missing_example_list+=("$dir") - continue - fi - if [[ -f "$env" && "$FORCE" = false ]]; then - log "Skipping (exists): $env (use --force to overwrite)" - skipped_exists_count=$((skipped_exists_count+1)) - skipped_exists_list+=("$env") - continue - fi - - if [[ -f "$env" && "$FORCE" = true ]]; then - if [[ "$ASSUME_YES" = false ]]; then - printf 'About to overwrite %s. Create backup and continue? [y/N]: ' "$env" - read -r ans - case "$ans" in - [Yy]|[Yy][Ee][Ss]) ;; - *) log "Aborting overwrite for $env"; continue ;; - esac - fi - if [[ "$DRY_RUN" = true ]]; then - log "DRY-RUN: backup and overwrite $env from $ex" - overwritten_count=$((overwritten_count+1)) - overwritten_list+=("$env (dry-run)") - else - local backup - backup="$(backup_file "$env")" - log "Backed up $env -> $backup" - cp -f "$ex" "$env" - overwritten_count=$((overwritten_count+1)) - overwritten_list+=("$env") - backup_list+=("$backup") - fi - else - if [[ "$DRY_RUN" = true ]]; then - log "DRY-RUN: create $env from $ex" - created_count=$((created_count+1)) - created_list+=("$env (dry-run)") - else - cp -f "$ex" "$env" - created_count=$((created_count+1)) - created_list+=("$env") - fi - fi - done - # Recreate summary - printf '\nSummary: %d targets | %d created | %d overwritten | %d skipped (exists) | %d missing .env.example\n' \ - "${#targets[@]}" "$created_count" "$overwritten_count" "$skipped_exists_count" "$missing_example_count" - if [[ $created_count -gt 0 ]]; then - printf 'Created .env files:\n'; for f in "${created_list[@]}"; do printf ' - %s\n' "$f"; done - fi - if [[ $overwritten_count -gt 0 ]]; then - printf 'Overwritten .env files:\n'; for f in "${overwritten_list[@]}"; do printf ' - %s\n' "$f"; done - # Only show backups when not dry-run - if [[ ${#backup_list[@]} -gt 0 ]]; then - printf 'Backups created:\n'; for b in "${backup_list[@]}"; do printf ' - %s\n' "$b"; done - fi - fi - if [[ $skipped_exists_count -gt 0 ]]; then - printf 'Skipped (existing .env, use --force to overwrite):\n'; for f in "${skipped_exists_list[@]}"; do printf ' - %s\n' "$f"; done - fi - if [[ $missing_example_count -gt 0 ]]; then - printf 'Missing .env.example in:\n'; for d in "${missing_example_list[@]}"; do printf ' - %s\n' "$d"; done - fi - fi -} - -cmd_up() { - local FOLLOW_LOGS=true - local DRY_RUN=false - TARGET_STACKS=("${STACK_FILES[@]}") - - while [[ $# -gt 0 ]]; do - case "${1:-}" in - -n|--no-logs) - FOLLOW_LOGS=false; shift ;; - --dry-run) - DRY_RUN=true; shift ;; - -s|--stacks) - [[ $# -lt 2 ]] && { err "--stacks requires a value"; exit 2; } - set_target_stacks "${2:-}"; shift 2 ;; - -h|--help) - log "Deploy stacks and optionally follow logs. Options: -n/--no-logs, --dry-run, -s/--stacks (comma-separated: ${STACK_FILES[*]})"; exit 0 ;; - --) - shift; break ;; - -*) - printf '%s: unknown option for up: %s\n' "$SCRIPT_NAME" "$1" >&2; exit 2 ;; - *) - break ;; - esac - done - - check_command docker - ensure_swarm_info - ensure_traefik_network "$DRY_RUN" - - for stack in "${TARGET_STACKS[@]}"; do - local file - if ! file="$(find_stack_file "$stack")"; then - err "Stack file not found for '$stack' in stacks/ or repo root (.yml/.yaml) -- skipping" - continue - fi - # Render into a temporary sibling file to resolve service-level env vars in labels/commands/etc. - local render_file - render_file="$(render_compose_file "$file")" - if [[ "$DRY_RUN" = true ]]; then - log "DRY-RUN: would run: docker stack deploy -c $render_file $stack" - log "DRY-RUN: validating compose file: $render_file" - # Suppress the full rendered YAML output; keep warnings/errors on stderr visible - compose_config "$render_file" >/dev/null || true - else - log "Deploying stack: $stack (file: $render_file)" - docker stack deploy -c "$render_file" "$stack" - fi - done - - for stack in "${TARGET_STACKS[@]}"; do - log "Services for stack: $stack" - if [[ "$DRY_RUN" = true ]]; then - log "DRY-RUN: docker stack services $stack" - else - docker stack services "$stack" || log "Warning: failed to list services for $stack" - fi - done - - # Follow logs if requested - if [[ "$FOLLOW_LOGS" = true && "$DRY_RUN" = false ]]; then - cmd_logs "${DEFAULT_LOG_SERVICES[@]}" - else - log "Not following logs (FOLLOW_LOGS=$FOLLOW_LOGS, DRY_RUN=$DRY_RUN)" - fi -} - -cmd_down() { - local DRY_RUN=false - local REMOVE_NETWORK=false - local ASSUME_YES=false - TARGET_STACKS=("${STACK_FILES[@]}") - - while [[ $# -gt 0 ]]; do - case "${1:-}" in - --dry-run) - DRY_RUN=true; shift ;; - --remove-network) - REMOVE_NETWORK=true; shift ;; - -y|--yes) - ASSUME_YES=true; shift ;; - -s|--stacks) - [[ $# -lt 2 ]] && { err "--stacks requires a value"; exit 2; } - set_target_stacks "${2:-}"; shift 2 ;; - -h|--help) - log "Remove stacks. Options: --dry-run, --remove-network, -y/--yes, -s/--stacks (comma-separated: ${STACK_FILES[*]})"; exit 0 ;; - --) - shift; break ;; - -*) - printf '%s: unknown option for down: %s\n' "$SCRIPT_NAME" "$1" >&2; exit 2 ;; - *) - break ;; - esac - done - - check_command docker - - log "Stacks to remove: ${TARGET_STACKS[*]}" - if [[ "$ASSUME_YES" = false && "$DRY_RUN" = false ]]; then - printf "Are you sure you want to remove the above stacks? [y/N]: " - read -r ans - case "$ans" in - [Yy]|[Yy][Ee][Ss]) ;; - *) log "Aborting."; exit 0 ;; - esac - fi - - for stack in "${TARGET_STACKS[@]}"; do - if [[ "$DRY_RUN" = true ]]; then - log "DRY-RUN: docker stack rm $stack" - else - log "Removing stack: $stack" - docker stack rm "$stack" || log "Warning: failed to remove stack $stack or it may not exist" - fi - done - - if [[ "$REMOVE_NETWORK" = true ]]; then - if [[ "$DRY_RUN" = true ]]; then - log "DRY-RUN: docker network rm traefik-public" - else - log "Removing network: traefik-public" - docker network rm traefik-public || log "Warning: could not remove traefik-public (may not exist or may be in use)" - fi - fi -} - -cmd_status() { - check_command docker - TARGET_STACKS=("${STACK_FILES[@]}") - while [[ $# -gt 0 ]]; do - case "${1:-}" in - -s|--stacks) - [[ $# -lt 2 ]] && { err "--stacks requires a value"; exit 2; } - set_target_stacks "${2:-}"; shift 2 ;; - -h|--help) - log "List services. Options: -s/--stacks (comma-separated: ${STACK_FILES[*]})"; exit 0 ;; - --) - shift; break ;; - -*) - printf '%s: unknown option for status: %s\n' "$SCRIPT_NAME" "$1" >&2; exit 2 ;; - *) - break ;; - esac - done - - for stack in "${TARGET_STACKS[@]}"; do - log "Services for stack: $stack" - docker stack services "$stack" || log "Warning: failed to list services for $stack" - done -} - -cmd_logs() { - check_command docker - local services=("$@") - if [[ ${#services[@]} -eq 0 ]]; then - services=("${DEFAULT_LOG_SERVICES[@]}") - fi - local PIDS=() - cleanup() { - log "Cleaning up..." - for pid in "${PIDS[@]:-}"; do - if kill -0 "$pid" 2>/dev/null; then - kill "$pid" 2>/dev/null || true - fi - done - } - trap cleanup EXIT INT TERM - - for svc in "${services[@]}"; do - log "Following logs for service: $svc" - docker service logs -f "$svc" 2>&1 | sed "s/^/[$svc] /" & - PIDS+=("$!") - done - wait -} - -cmd_doctor() { - local FIX_NETWORK=false - local FIX_VOLUMES=false - while [[ $# -gt 0 ]]; do - case "${1:-}" in - --fix-network) - FIX_NETWORK=true; shift ;; - --fix-volumes) - FIX_VOLUMES=true; shift ;; - -h|--help) - log "Run preflight checks. Options: --fix-network (create traefik-public if missing), --fix-volumes (create missing external named volumes)"; exit 0 ;; - --) - shift; break ;; - -*) - printf '%s: unknown option for doctor: %s\n' "$SCRIPT_NAME" "$1" >&2; exit 2 ;; - *) - break ;; - esac - done - - # Basic tooling - check_command docker - if docker compose version >/dev/null 2>&1; then - log "Found: docker compose plugin" - elif command -v docker-compose >/dev/null 2>&1; then - log "Found: docker-compose (legacy)" - else - err "Missing both 'docker compose' and 'docker-compose'" - fi - - # Swarm state - ensure_swarm_info - - # Network - if docker network ls --format '{{.Name}}' | grep -qx "traefik-public"; then - log "OK: traefik-public network exists" - else - if [[ "$FIX_NETWORK" = true ]]; then - log "Creating missing traefik-public network (overlay, attachable)" - ensure_traefik_network false - else - err "Missing network 'traefik-public' — run: docker network create --driver=overlay --attachable traefik-public" - fi - fi - - # Stack files + validation - local overall_ok=true - for stack in "${STACK_FILES[@]}"; do - if file_path="$(find_stack_file "$stack")"; then - log "OK: found stack file for '$stack': $file_path" - # Attempt to render first for accurate validation - local rendered - rendered="$(render_compose_file "$file_path")" - local validated=false - if compose_config "$rendered" >/dev/null 2>&1; then - log "OK: '$stack' compose syntax valid (validated: $rendered)" - validated=true - else - err "Validation failed for '$stack' ($rendered)" - fi - - # Optionally ensure external named volumes exist - if [[ "$FIX_VOLUMES" = true ]]; then - # Extract external named volumes from the rendered file (best-effort awk/yq-less parsing) - # Look for pattern under top-level volumes: name: and external: true - # This is a heuristic and may miss exotic YAML, but works for our stacks. - local vol_names - vol_names=$(awk ' - /^volumes:/ {invol=1; next} - invol==1 && /^[^[:space:]]/ {invol=0} - invol==1 { - if ($0 ~ /^[[:space:]]{2,}[A-Za-z0-9_-]+:$/) { - if (keyname != "" && ext == 1) { - if (volname != "") print volname; else print keyname; - } - keyname=$1; sub(":", "", keyname); volname=""; ext=0; - } else if ($1 == "name:") { - volname=$2; - } else if ($1 == "external:" && $2 ~ /true/) { - ext=1; - } - } - END { if (keyname != "" && ext == 1) { if (volname != "") print volname; else print keyname; } } - ' "$rendered") || true - if [[ -n "$vol_names" ]]; then - while IFS= read -r vol; do - [[ -z "$vol" ]] && continue - if docker volume ls --format '{{.Name}}' | grep -qx "$vol"; then - log "OK: external volume exists: $vol" - else - log "Creating missing external volume: $vol" - docker volume create "$vol" >/dev/null 2>&1 || log "Warning: failed to create volume $vol" - fi - done <<< "$vol_names" - fi - fi - # Re-validate after creating volumes if initial validation failed - if [[ "$FIX_VOLUMES" = true && "$validated" = false ]]; then - if compose_config "$rendered" >/dev/null 2>&1; then - log "OK: '$stack' compose syntax valid after fixing volumes (validated: $rendered)" - validated=true - fi - fi - if [[ "$validated" = false ]]; then - overall_ok=false - fi - else - err "Missing stack file for '$stack' (looked in stacks/ and repo root)" - overall_ok=false - fi - done - - # .env hints for service directories that have docker-compose files - log "Scanning service folders for .env hints..." - local service_dirs - # GNU find on Linux supports -printf; if not available, this block may be skipped - if service_dirs=$(find "$SCRIPT_DIR" -mindepth 2 -maxdepth 3 -type f \( -name 'docker-compose.yml' -o -name 'docker-compose.yaml' \) -printf '%h\n' 2>/dev/null | sort -u); then - while IFS= read -r dir; do - [[ -z "$dir" ]] && continue - if [[ -f "$dir/.env.example" && ! -f "$dir/.env" ]]; then - log "NOTE: $dir has .env.example but no .env — copy it before deploying" - fi - done <<< "$service_dirs" - else - log "Skipping .env hint scan (find -printf not supported)" - fi - - # TLS dev cert hints for Traefik - local cert_dir="$SCRIPT_DIR/traefik/certs" - if [[ -d "$cert_dir" ]]; then - if [[ ! -f "$cert_dir/local-cert.pem" || ! -f "$cert_dir/local-key.pem" ]]; then - log "NOTE: Traefik dev certs not found in traefik/certs (local-cert.pem/local-key.pem). See stacks/README.md for mkcert instructions." - else - log "OK: Traefik dev certs present" - fi - fi - - if [[ "$overall_ok" = true ]]; then - log "Doctor checks completed: no blocking issues detected." - exit 0 - else - err "Doctor checks found issues. See messages above." - exit 1 - fi -} - -# Determine subcommand (default: up) -SUBCOMMAND="${1:-}" -case "$SUBCOMMAND" in - up|down|status|logs|doctor|env|help|-h|--help) - [[ $# -gt 0 ]] && shift || true ;; - *) - SUBCOMMAND="up" ;; +PY_CLI="$SCRIPT_DIR/tools/stackctl_cli.py" + +info() { printf "[info] %s\n" "$*"; } +warn() { printf "[warn] %s\n" "$*"; } +die() { printf "[error] %s\n" "$*" >&2; exit 1; } + +run_py() { + # run the Python CLI with forwarded args + python3 "$PY_CLI" "$@" +} + +check_py_cli() { + if [[ ! -f "$PY_CLI" ]]; then + die "Python CLI not found at $PY_CLI" + fi +} + +bootstrap() { + info "Copying .env.example files..." + # Create missing .env files from examples + find . -type f -name '.env.example' -print0 | while IFS= read -r -d '' src; do + dst="${src%.env.example}.env" + if [ ! -e "$dst" ]; then + info "create $dst" + cp "$src" "$dst" + fi + done + + if [ -f tools/requirements.txt ]; then + if [ -n "${VIRTUAL_ENV:-}" ]; then + info "Installing Python requirements..." + pip install -r tools/requirements.txt + else + warn "No Python venv active. Activate your venv before running bootstrap for Python deps." + fi + fi + + if command -v docker >/dev/null 2>&1; then + info "Docker version: $(docker --version)" + if docker info | grep -q 'Swarm: active'; then + info "Docker Swarm is active." + else + warn "Docker Swarm is not active. Run: docker swarm init" + fi + else + die "Docker not found. Please install Docker Desktop or Docker Engine." + fi + + printf "\n[bootstrap] Next steps:\n" + printf " - Review .env files and adjust as needed.\n" + printf " - Run ./stackctl.sh doctor --fix-network\n" + printf " - Run ./stackctl.sh up\n" + printf " - See tools/README.md for CLI usage.\n" +} + +check_py_cli + +cmd="${1:-}" +shift || true + +case "${cmd:-}" in + up|deploy ) + # Map legacy flags: --no-logs -> --no-follow-logs + args=() + while [[ $# -gt 0 ]]; do + case "$1" in + -n|--no-logs) + args+=("--no-follow-logs"); shift ;; + --dry-run|-s|--stacks|--env) + # pass-through + value when applicable + if [[ "$1" == "-s" || "$1" == "--stacks" || "$1" == "--env" ]]; then + args+=("$1" "${2:-}"); shift 2 || true + else + args+=("$1"); shift + fi ;; + *) args+=("$1"); shift ;; + esac + done + exec run_py deploy "${args[@]:-}" ;; + + down) + exec run_py down "$@" ;; + + status) + exec run_py status "$@" ;; + + logs) + exec run_py logs "$@" ;; + + env) + exec run_py env "$@" ;; + + doctor) + exec run_py doctor run "$@" ;; + + --bootstrap|bootstrap) + bootstrap + exit 0 ;; + + help|-h|--help) + exec run_py --help ;; + + "" ) + # No command supplied: show help (do not default to deploy) + exec run_py --help ;; + + *) + printf 'ERROR: unknown command "%s"\n\n' "${cmd:-}" >&2 + exec run_py --help ;; esac -case "$SUBCOMMAND" in - up) - cmd_up "$@" ;; - down) - cmd_down "$@" ;; - status) - cmd_status "$@" ;; - logs) - cmd_logs "$@" ;; - env) - cmd_env "$@" ;; - doctor) - cmd_doctor "$@" ;; - help|-h|--help) - print_usage ;; - *) - err "Unknown command: $SUBCOMMAND"; print_usage; exit 2 ;; -esac -exit 0 diff --git a/tools/README.md b/tools/README.md index 00dbdea..fb30f17 100644 --- a/tools/README.md +++ b/tools/README.md @@ -41,3 +41,106 @@ docker stack deploy -c .rendered/docker-compose.infrastructure.rendered.yml infr - This is a preprocessing step; paths remain relative to the original file location. - It does not evaluate command substitutions or complex expressions, only variable substitution. + +## Overlays (environment-specific overrides) + +For multi-environment deployments you can provide small, focused overlay YAML files that are deep-merged +onto the base `stacks/.yml` before interpolation. Overlays live under `environments//` and are +named `.overlay.yml` (or `.yaml`). Example overlay path: + +``` +environments/local/infrastructure.overlay.yml +environments/staging/infrastructure.overlay.yml +environments/prod/infrastructure.overlay.yml +``` + +How overlays are applied: +- The renderer first loads the base stack file from `stacks/.yml`. +- If an overlay exists, it is deep-merged into the base (nested mappings are merged, simple scalars/lists are replaced). +- The merged YAML is then passed to `tools/render_compose.py` which resolves `${VAR}` using service `env_file`s and environment. + +Simple examples + +1) Change Traefik host and TLS resolver for local testing (environments/local/infrastructure.overlay.yml): + +```yaml +services: + traefik: + environment: + HOST: "traefik.localhost" + CERT_RESOLVER: "local" + + grafana: + environment: + HOST: "grafana.localhost" +``` + +2) Point APISIX gateway to a local etcd for dev (environments/local/infrastructure.overlay.yml): + +```yaml +services: + apisix: + environment: + ETCD_HOSTS: "http://etcd:2379" + APISIX_ENABLE_PROMETHEUS: "true" + + etcd: + volumes: + - etcd-data:/var/lib/etcd + +volumes: + etcd-data: + driver: local +``` + +3) Production override example (environments/prod/infrastructure.overlay.yml): + +```yaml +services: + traefik: + environment: + HOST: "traefik.example.com" + CERT_RESOLVER: "letsencrypt" + deploy: + placement: + constraints: + - node.role == manager + + apisix: + environment: + ETCD_HOSTS: "https://etcd.prod.internal:2379" + deploy: + replicas: 3 + +volumes: + etcd-data: + external: true +``` + +Using overlays with the Python CLI + +Render and deploy a stack with the `local` overlay: + +```sh +python3 tools/stackctl_cli.py deploy --stacks infrastructure --env local +``` + +Or render only (writes to `./.rendered`): + +```sh +python3 tools/stackctl_cli.py render --stacks infrastructure --env local +``` + +Notes and tips +- Keep overlays small and focused: change only the values that differ between environments. +- Use overlays to drive Traefik hostnames, TLS settings, resource constraints, and scaling hints. +- For secrets, prefer encrypted files (SOPS) and use `tools/stackctl_cli.py secrets decrypt` during local setup. + +Secrets with SOPS +----------------- + +We recommend storing secrets as SOPS-encrypted YAML files (e.g., `.env.enc.yaml`) and keep the +repository free of plaintext `.env` files. See `docs/SOPS-usage.md` for a short guide on how to +create and decrypt SOPS files locally using the `stackctl_cli` helpers. + + diff --git a/tools/requirements.txt b/tools/requirements.txt index ffe9c1b..450b5de 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1 +1,5 @@ PyYAML>=6.0.1,<7 +typer>=0.12.3 +rich>=13.7.1 +mergedeep>=1.3.4 +pytest>=7.0.0 diff --git a/tools/stackctl_cli.py b/tools/stackctl_cli.py new file mode 100644 index 0000000..d4717ee --- /dev/null +++ b/tools/stackctl_cli.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python3 +""" +Local-Stack Python CLI for Swarm stacks + +Goals +- Provide a single, portable Python entrypoint to render, deploy, teardown, and inspect stacks +- Support multi-environment overlays and per-service env_file interpolation (reusing tools/render_compose.py) +- Add safe secret workflows via SOPS without exposing plaintext in VCS + +This CLI intentionally shells out to Docker where appropriate to avoid re-implementing Swarm semantics. + +Usage (examples): + python3 tools/stackctl_cli.py render --env local + python3 tools/stackctl_cli.py deploy --stacks infrastructure,observability --env local + python3 tools/stackctl_cli.py down --stacks platform + python3 tools/stackctl_cli.py env list + python3 tools/stackctl_cli.py secrets decrypt --in apisix/api-gateway/.env.enc.yaml --out apisix/api-gateway/.env + +Notes +- Rendered files are written to ./.rendered by default and are git-ignored in this repo. +- Overlays: if --env is provided, we will look for: + environments//.overlay.yml (or .yaml) and deep-merge onto stacks/.yml + Unknown overlays are ignored with a warning (non-fatal). +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import typer +from rich import print as rprint +from rich.console import Console +from rich.table import Table +import yaml + +# Optional dependency for deep merges; graceful fallback to a simple merge +try: + from mergedeep import merge as deep_merge # type: ignore +except Exception: # pragma: no cover + def deep_merge(dst, src, strategy=None): + """Minimal deep merge fallback: recursively overlay dicts, replace lists/scalars.""" + if not isinstance(dst, dict) or not isinstance(src, dict): + return src + for k, v in src.items(): + if k in dst and isinstance(dst[k], dict) and isinstance(v, dict): + deep_merge(dst[k], v) + else: + dst[k] = v + return dst + + +APP = typer.Typer(help="Local-Stack management CLI (Swarm)") +console = Console(stderr=True) + +REPO_ROOT = Path(__file__).resolve().parents[1] +STACKS_DIR = REPO_ROOT / "stacks" +RENDER_DIR = Path(os.environ.get("RENDER_DIR", REPO_ROOT / ".rendered")) +DEFAULT_STACKS = ["infrastructure", "observability", "platform"] +DEFAULT_LOG_SERVICES = [ + "observability_prometheus", + "observability_loki", + "infrastructure_traefik", +] + + +def require_cmd(cmd: str) -> None: + if shutil.which(cmd) is None: + raise typer.Exit(code=2) + + +def run(cmd: List[str], check: bool = True) -> subprocess.CompletedProcess: + """Run a command and stream output; returns CompletedProcess.""" + console.log(f"$ {' '.join(cmd)}") + return subprocess.run(cmd, check=check) + + +def stack_file_for(name: str) -> Optional[Path]: + for ext in ("yml", "yaml"): + cand = STACKS_DIR / f"{name}.{ext}" + if cand.is_file(): + return cand + return None + + +def overlay_file_for(env: Optional[str], stack: str) -> Optional[Path]: + if not env: + return None + base = REPO_ROOT / "environments" / env + for ext in ("yml", "yaml"): + cand = base / f"{stack}.overlay.{ext}" + if cand.is_file(): + return cand + return None + + +def load_yaml(path: Path) -> dict: + with path.open("r", encoding="utf-8") as fh: + return yaml.safe_load(fh) or {} + + +def save_yaml(path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + yaml.safe_dump(data, fh, sort_keys=False) + + +def render_with_env_interpolation(base_path: Path, out_path: Path, repo_root: Path) -> Path: + """Reuse the existing render_compose.py to resolve per-service env vars and absolutize paths.""" + render_script = REPO_ROOT / "tools" / "render_compose.py" + if not render_script.is_file(): + raise typer.BadParameter("tools/render_compose.py not found") + out_path.parent.mkdir(parents=True, exist_ok=True) + cmd = [ + sys.executable, + str(render_script), + "-i", + str(base_path), + "-o", + str(out_path), + "--repo-root", + str(repo_root), + ] + console.log(f"Rendering compose with env interpolation -> {out_path}") + subprocess.run(cmd, check=True) + return out_path + + +def deep_merge_overlay(base_data: dict, overlay_path: Optional[Path]) -> dict: + if overlay_path and overlay_path.is_file(): + overlay = load_yaml(overlay_path) + deep_merge(base_data, overlay) + return base_data + + +@APP.command() +def render( + stacks: str = typer.Option( + ",".join(DEFAULT_STACKS), "--stacks", "-s", help="Comma-separated list of stacks to render" + ), + env: Optional[str] = typer.Option(None, "--env", help="Environment overlay name (environments//...)"), + strict: bool = typer.Option(False, "--strict", help="Fail if unresolved ${VAR} remain after render"), +): + """Render selected stacks to ./.rendered with per-service env interpolation and optional overlays.""" + selected = [s.strip() for s in stacks.split(",") if s.strip()] + RENDER_DIR.mkdir(parents=True, exist_ok=True) + results: List[Tuple[str, Path]] = [] + for stack in selected: + src = stack_file_for(stack) + if not src: + console.print(f"[yellow]Skip[/yellow] stack '{stack}' (file not found)") + continue + # Apply overlay first (in-memory), then feed to render_compose for env substitution + base = load_yaml(src) + base = deep_merge_overlay(base, overlay_file_for(env, stack)) + # Write a temp merged file for interpolation + merged_tmp = RENDER_DIR / f"docker-compose.{stack}.merged.yml" + save_yaml(merged_tmp, base) + out_path = RENDER_DIR / f"docker-compose.{stack}.rendered.yml" + render_with_env_interpolation(merged_tmp, out_path, REPO_ROOT) + if strict: + text = out_path.read_text(encoding="utf-8") + # naive unresolved check + if "${" in text: + console.print(f"[red]Unresolved variables remain in {out_path}[/red]") + raise typer.Exit(code=3) + results.append((stack, out_path)) + + table = Table(title="Rendered Stacks") + table.add_column("Stack") + table.add_column("Path") + for name, path in results: + table.add_row(name, str(path)) + console.print(table) + + +def ensure_swarm() -> None: + try: + out = subprocess.check_output(["docker", "info", "--format", "{{.Swarm.LocalNodeState}}"], text=True) + except Exception: + console.print("[red]Docker not available[/red]") + raise typer.Exit(code=2) + if out.strip() != "active": + console.print("[yellow]Swarm is not active on this host. Run: docker swarm init[/yellow]") + + +def ensure_network() -> None: + try: + out = subprocess.check_output(["docker", "network", "ls", "--format", "{{.Name}}"], text=True) + if "traefik-public" not in out.splitlines(): + console.print("[cyan]Creating overlay network traefik-public[/cyan]") + subprocess.run(["docker", "network", "create", "--driver=overlay", "--attachable", "traefik-public"], check=False) + except Exception: + pass + + +@APP.command() +def deploy( + stacks: str = typer.Option( + ",".join(DEFAULT_STACKS), "--stacks", "-s", help="Comma-separated list of stacks to deploy" + ), + env: Optional[str] = typer.Option(None, "--env", help="Environment overlay name"), + dry_run: bool = typer.Option(False, "--dry-run", help="Only render and validate, do not deploy"), + follow_logs: bool = typer.Option( + False, + "--follow-logs/--no-follow-logs", + help="After deploy, follow logs for default services", + ), +): + """Render and deploy stacks via docker stack deploy.""" + ensure_swarm() + ensure_network() + selected = [s.strip() for s in stacks.split(",") if s.strip()] + # Render first + for stack in selected: + src = stack_file_for(stack) + if not src: + console.print(f"[yellow]Skip[/yellow] stack '{stack}' (file not found)") + continue + base = load_yaml(src) + base = deep_merge_overlay(base, overlay_file_for(env, stack)) + merged_tmp = RENDER_DIR / f"docker-compose.{stack}.merged.yml" + save_yaml(merged_tmp, base) + out_path = RENDER_DIR / f"docker-compose.{stack}.rendered.yml" + render_with_env_interpolation(merged_tmp, out_path, REPO_ROOT) + # Validate compose syntax + cmd = ["docker", "compose", "-f", str(out_path), "config"] + if dry_run: + console.print(f"[cyan]DRY-RUN[/cyan] validate: {' '.join(cmd)}") + subprocess.run(cmd, check=False) + else: + subprocess.run(cmd, check=False) + console.print(f"[green]Deploying[/green] stack: {stack}") + run(["docker", "stack", "deploy", "-c", str(out_path), stack], check=False) + + # Show services summary + if not dry_run: + for stack in selected: + console.print(f"\n[b]Services in {stack}[/b]") + run(["docker", "stack", "services", stack], check=False) + if follow_logs: + console.print("\n[cyan]Following logs for default services[/cyan]") + for svc in DEFAULT_LOG_SERVICES: + try: + console.print(f"[b]$ docker service logs -f {svc}[/b]") + run(["docker", "service", "logs", "-f", svc], check=False) + except KeyboardInterrupt: # pragma: no cover + break + + +@APP.command() +def logs(services: List[str] = typer.Argument(None, help="Services to follow (default preset)")): + """Follow logs for services (docker service logs -f).""" + svcs = services or DEFAULT_LOG_SERVICES + for svc in svcs: + console.print(f"[b]$ docker service logs -f {svc}[/b]") + try: + run(["docker", "service", "logs", "-f", svc], check=False) + except KeyboardInterrupt: # pragma: no cover + break + + +@APP.command() +def down( + stacks: str = typer.Option( + ",".join(DEFAULT_STACKS), "--stacks", "-s", help="Comma-separated list of stacks to remove" + ), + remove_network: bool = typer.Option(False, "--remove-network", help="Attempt to remove traefik-public after removal"), +): + """Remove stacks (docker stack rm).""" + selected = [s.strip() for s in stacks.split(",") if s.strip()] + for stack in selected: + console.print(f"[green]Removing[/green] stack: {stack}") + run(["docker", "stack", "rm", stack], check=False) + if remove_network: + run(["docker", "network", "rm", "traefik-public"], check=False) + + +@APP.command() +def status( + stacks: str = typer.Option( + ",".join(DEFAULT_STACKS), "--stacks", "-s", help="Comma-separated list of stacks" + ), +): + """List services for stacks.""" + selected = [s.strip() for s in stacks.split(",") if s.strip()] + for stack in selected: + console.print(f"\n[b]Services in {stack}[/b]") + run(["docker", "stack", "services", stack], check=False) + + +env_app = typer.Typer(help=".env management") +APP.add_typer(env_app, name="env") + + +def discover_env_example_dirs(root: Path) -> List[Path]: + return [p.parent for p in root.rglob(".env.example")] + + +@env_app.command("list") +def env_list(paths: Optional[str] = typer.Option(None, "--paths", help="Comma-separated paths to scan")): + """List directories that have .env.example and whether .env exists.""" + dirs: List[Path] = [] + if paths: + for tok in paths.split(","): + tok = tok.strip() + if not tok: + continue + p = (REPO_ROOT / tok).resolve() + if p.is_file(): + dirs.append(p.parent) + elif p.is_dir(): + dirs.append(p) + else: + dirs = discover_env_example_dirs(REPO_ROOT) + + rows: List[Tuple[str, str]] = [] + for d in sorted(set(dirs)): + ex = d / ".env.example" + cur = d / ".env" + if ex.is_file(): + status = "OK (.env present)" if cur.is_file() else "MISSING (.env)" + else: + status = "SKIP (no .env.example)" + rows.append((str(d.relative_to(REPO_ROOT)), status)) + + table = Table(title=".env scan") + table.add_column("Directory") + table.add_column("Status") + for a, b in rows: + table.add_row(a, b) + console.print(table) + + +@env_app.command("recreate") +def env_recreate( + force: bool = typer.Option(False, "--force", "-f", help="Overwrite existing .env after backup"), + yes: bool = typer.Option(False, "--yes", "-y", help="Non-interactive"), + paths: Optional[str] = typer.Option(None, "--paths", help="Comma-separated paths to process"), +): + """Create or overwrite .env from .env.example safely.""" + targets = discover_env_example_dirs(REPO_ROOT) if not paths else [ + ((REPO_ROOT / p.strip()).resolve()) for p in paths.split(",") if p.strip() + ] + created, overwritten, skipped, missing = [], [], [], [] + for d in targets: + ex, envp = d / ".env.example", d / ".env" + if not ex.is_file(): + missing.append(str(d)) + continue + if envp.exists() and not force: + skipped.append(str(envp)) + continue + if envp.exists() and force: + if not yes: + resp = input(f"Overwrite {envp}? [y/N]: ") + if resp.lower() not in ("y", "yes"): + skipped.append(str(envp)) + continue + backup = envp.with_suffix(envp.suffix + f".bak.{int(time.time())}") + shutil.copy2(envp, backup) + shutil.copy2(ex, envp) + overwritten.append(str(envp)) + else: + shutil.copy2(ex, envp) + created.append(str(envp)) + rprint({ + "created": created, + "overwritten": overwritten, + "skipped": skipped, + "missing_example": missing, + }) + + +secrets_app = typer.Typer(help="SOPS helper commands") +APP.add_typer(secrets_app, name="secrets") + + +@secrets_app.command("decrypt") +def secrets_decrypt( + input_path: Path = typer.Option(..., "--in", help="Encrypted input file (e.g., .env.enc.yaml)"), + output_path: Path = typer.Option(..., "--out", help="Destination plaintext file (e.g., .env)"), + overwrite: bool = typer.Option(False, "--force", help="Overwrite output if exists"), +): + """Decrypt a SOPS-managed file to a local plaintext file. Will never print secret contents.""" + if shutil.which("sops") is None: + console.print("[red]sops CLI not found. Install from https://github.com/mozilla/sops[/red]") + raise typer.Exit(code=2) + input_path = input_path.resolve() + output_path = output_path.resolve() + if output_path.exists() and not overwrite: + console.print(f"[yellow]Refusing to overwrite existing {output_path}. Use --force to overwrite.[/yellow]") + raise typer.Exit(code=1) + output_path.parent.mkdir(parents=True, exist_ok=True) + # Write directly to file to avoid secrets in stdout + with output_path.open("wb") as fh: + subprocess.run(["sops", "-d", str(input_path)], check=True, stdout=fh) + console.print(f"[green]Decrypted[/green] -> {output_path}") + + +@secrets_app.command("encrypt") +def secrets_encrypt( + input_path: Path = typer.Option(..., "--in", help="Plaintext input file (e.g., .env)"), + output_path: Optional[Path] = typer.Option(None, "--out", help="Encrypted output file (e.g., .env.enc.yaml)"), +): + """Encrypt a plaintext file using SOPS, honoring .sops.yaml policy in the repo root if present.""" + if shutil.which("sops") is None: + console.print("[red]sops CLI not found. Install from https://github.com/mozilla/sops[/red]") + raise typer.Exit(code=2) + input_path = input_path.resolve() + if output_path is None: + output_path = input_path.with_suffix(input_path.suffix + ".enc.yaml") + output_path = output_path.resolve() + run(["sops", "-e", str(input_path), "-o", str(output_path)], check=True) + console.print(f"[green]Encrypted[/green] -> {output_path}") + + +doctor_app = typer.Typer(help="Preflight checks") +APP.add_typer(doctor_app, name="doctor") + + +@doctor_app.command("run") +def doctor_run(fix_network: bool = typer.Option(False, "--fix-network")): + """Validate docker availability, swarm state, traefik-public network, and stack files.""" + # docker + if shutil.which("docker") is None: + console.print("[red]docker not found on PATH[/red]") + raise typer.Exit(code=2) + try: + out = subprocess.check_output(["docker", "compose", "version"], text=True) + console.print("compose: available") + except Exception: + console.print("compose: not available, will try docker-compose if needed") + ensure_swarm() + # network + try: + nets = subprocess.check_output(["docker", "network", "ls", "--format", "{{.Name}}"], text=True).splitlines() + if "traefik-public" in nets: + console.print("network: traefik-public exists") + else: + console.print("network: traefik-public missing") + if fix_network: + ensure_network() + except Exception: + pass + # stacks + for stack in DEFAULT_STACKS: + sf = stack_file_for(stack) + if sf: + console.print(f"stack: found {sf}") + # best-effort validation + subprocess.run(["docker", "compose", "-f", str(sf), "config"], check=False) + else: + console.print(f"[yellow]stack: missing file for {stack}[/yellow]") + + +def main() -> int: + APP() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tests/test_overlay_merge.py b/tools/tests/test_overlay_merge.py new file mode 100644 index 0000000..c1224c0 --- /dev/null +++ b/tools/tests/test_overlay_merge.py @@ -0,0 +1,51 @@ +from mergedeep import merge + + +def test_deep_merge_overlay(): + base = { + "services": { + "s": { + "environment": { + "A": "1", + "B": "2", + } + } + }, + "volumes": { + "data": {"driver": "local"} + }, + } + + overlay = { + "services": { + "s": { + "environment": { + "B": "override", + "C": "3", + } + } + }, + "volumes": { + "extra": {"driver": "local"} + }, + } + + expected = { + "services": { + "s": { + "environment": { + "A": "1", + "B": "override", + "C": "3", + } + } + }, + "volumes": { + "data": {"driver": "local"}, + "extra": {"driver": "local"}, + }, + } + + merged = base.copy() + merge(merged, overlay) + assert merged == expected