diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38e5e9d..dc64bef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,3 +17,20 @@ jobs: run: uv run ruff check - name: Format run: uv run ruff format --check + + helm: + name: Helm chart + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: azure/setup-helm@v4 + - name: Register dependency repos + run: | + helm repo add eoapi https://developmentseed.org/eoapi-k8s/ + helm repo add stac-manager https://stac-manager.ds.io/ + - name: Build chart dependencies + run: helm dependency build infrastructure/charts/eoapi-workshop + - name: Lint + run: helm lint infrastructure/charts/eoapi-workshop + - name: Render checks + run: ./infrastructure/charts/eoapi-workshop/tests/render-checks.sh diff --git a/.github/workflows/publish-workshop-image.yml b/.github/workflows/publish-workshop-image.yml new file mode 100644 index 0000000..204f86a --- /dev/null +++ b/.github/workflows/publish-workshop-image.yml @@ -0,0 +1,53 @@ +name: Publish workshop image + +# Builds the JupyterLab workshop image (Dockerfile.local + environment.yml) and +# pushes it to GHCR. Consumed by the Helm chart's per-participant Labs +# (infrastructure/charts/eoapi-workshop, values key `jupyter.image`). +on: + push: + branches: + - main + - foss4geu-helmchart # workshop branch — image is built from here for now + paths: + - Dockerfile.local + - environment.yml + - docs/** + - .github/workflows/publish-workshop-image.yml + workflow_dispatch: {} # manual run (once this workflow is on the default branch) + +jobs: + publish: + name: Build and push + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v5 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Image metadata + id: meta + uses: docker/metadata-action@v5 + with: + # Portable: resolves to ghcr.io/developmentseed/eoapi-workshop in this repo. + images: ghcr.io/${{ github.repository_owner }}/eoapi-workshop + tags: | + type=raw,value=latest + type=sha,format=long + type=ref,event=branch + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile.local + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index 5fea259..36dc738 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,8 @@ cdk.context.json node_modules/ config.yaml + +# Helm: pulled chart dependencies (rebuilt with `helm dependency update`). +# Chart.lock IS tracked for reproducible builds. +charts/*/charts/ +charts/**/*.tgz diff --git a/infrastructure/charts/eoapi-workshop/.gitignore b/infrastructure/charts/eoapi-workshop/.gitignore new file mode 100644 index 0000000..e2cdd2e --- /dev/null +++ b/infrastructure/charts/eoapi-workshop/.gitignore @@ -0,0 +1,6 @@ +# Host-specific install overrides generated by deploy.sh — never committed. +.deploy/ + +# Vendored chart dependencies — rebuilt from Chart.lock by `helm dependency +# build` (deploy.sh runs it). Chart.lock IS tracked; the archives are not. +charts/ diff --git a/infrastructure/charts/eoapi-workshop/.helmignore b/infrastructure/charts/eoapi-workshop/.helmignore new file mode 100644 index 0000000..f6bf03f --- /dev/null +++ b/infrastructure/charts/eoapi-workshop/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building Helm packages. +.DS_Store +.git/ +.gitignore +*.tmproj +*.swp +*.bak +*.orig +.idea/ +.vscode/ +# Local tooling / generated artifacts — not part of the packaged chart. +deploy.sh +.deploy/ diff --git a/infrastructure/charts/eoapi-workshop/Chart.lock b/infrastructure/charts/eoapi-workshop/Chart.lock new file mode 100644 index 0000000..63780a7 --- /dev/null +++ b/infrastructure/charts/eoapi-workshop/Chart.lock @@ -0,0 +1,9 @@ +dependencies: +- name: eoapi + repository: https://developmentseed.org/eoapi-k8s/ + version: 0.13.1 +- name: stac-manager + repository: https://stac-manager.ds.io/ + version: 1.0.3 +digest: sha256:c266058775f9745e48df835657208d23f5785875904049c880093faa5a7886bd +generated: "2026-07-01T17:27:13.34478+03:00" diff --git a/infrastructure/charts/eoapi-workshop/Chart.yaml b/infrastructure/charts/eoapi-workshop/Chart.yaml new file mode 100644 index 0000000..ea356bc --- /dev/null +++ b/infrastructure/charts/eoapi-workshop/Chart.yaml @@ -0,0 +1,27 @@ +apiVersion: v2 +name: eoapi-workshop +description: Minimal, docker-compose-aligned eoAPI deployment for the workshop (no observability/monitoring stack) +type: application +version: 0.1.0 +# appVersion tracks the eoAPI application shipped by the eoapi dependency below. +appVersion: "6.3.1" +icon: https://eoapi.dev/img/eoAPI.png +home: https://github.com/developmentseed/eoapi-workshop +sources: + - https://github.com/developmentseed/eoapi-workshop + - https://github.com/developmentseed/eoapi-k8s +dependencies: + # Upstream eoAPI chart. Its packaged .tgz vendors its own subcharts + # (postgrescluster, stac-auth-proxy, prometheus, grafana, knative, ...). + # Disabled components are turned off via values; their `condition` flags + # keep them from rendering, so only the workshop services are deployed. + # NOTE: devseed.com/eoapi-k8s/ 301-redirects to developmentseed.org/eoapi-k8s/. + - name: eoapi + version: 0.13.1 + repository: https://developmentseed.org/eoapi-k8s/ + # stac-manager: STAC collection/item editing UI (deployed the EOEPCA way — + # its published chart). Routed at /manager via the passthrough ingress. + - name: stac-manager + version: 1.0.3 + repository: https://stac-manager.ds.io/ + condition: stac-manager.enabled diff --git a/infrastructure/charts/eoapi-workshop/README.md b/infrastructure/charts/eoapi-workshop/README.md new file mode 100644 index 0000000..b0d906f --- /dev/null +++ b/infrastructure/charts/eoapi-workshop/README.md @@ -0,0 +1,158 @@ +# eoapi-workshop Helm chart + +A docker-compose-aligned Helm deployment of [eoAPI](https://eoapi.dev) for the +workshop: an *umbrella* chart over the upstream +[`eoapi`](https://github.com/developmentseed/eoapi-k8s) and +[`stac-manager`](https://github.com/developmentseed/stac-manager) charts, plus +per-participant **JupyterLab** environments — no observability/monitoring stack. + +Every service is served at the **root of its own subdomain** under a wildcard +domain (`*.`, default `eoapi-workshop.ds.io`). + +## What gets deployed + +| Component | Subdomain of `eoapi-workshop.ds.io` | Notes | +|---|---|---| +| STAC API (via stac-auth-proxy) | `stac.` | pgstac + stac-fastapi, fronted by the auth proxy | +| Raster (titiler-pgstac) | `raster.` | | +| Vector (tipg) | `vector.` | serves `features.ecoregions` (loaded by the features-loader Job) | +| STAC Browser | `browser.` | root-serving `radiantearth/stac-browser` | +| STAC Manager (editing UI) | `manager.` | `stac-manager` chart 1.0.3 | +| Mock OIDC server | `mock-oidc.` | test-only auth | +| JupyterLab × N | `lab-01.`…`lab-05.` | one isolated pod + PVC + token each | +| Database (pgstac) | in-cluster only | Crunchy `PostgresCluster` | + +Disabled (unlike upstream `experimental.yaml`): `multidim`, `docServer`, +`eoapi-notifier`, `knative`, `monitoring.*`, `observability.grafana`, autoscaling. + +## Contracts (read first) + +- **Wildcard DNS required** — `*.` must A-record to the ingress + LoadBalancer IP (check: `dig +short stac.eoapi-workshop.ds.io`). +- **Release name and namespace must both be `eoapi`** — the proxy's in-cluster + OIDC URL (`eoapi-mock-oidc-server.eoapi.svc…`) is derived from them. `deploy.sh` + defaults to this. +- **Test-only auth, http by default** — the mock OIDC ships `test-client` / + `test-secret` and reads are public (`DEFAULT_PUBLIC=true`). STAC Manager (and + Browser) *login/editing* needs a secure context, so enable `routing.tls` for + HTTPS; over http the UIs are browse/read-only. Not for production. + +## Prerequisites + +Kubernetes 1.23+ with an **NGINX ingress controller**, the **Crunchy Postgres +Operator (PGO)** (hard requirement — `postgrescluster` only reconciles if PGO/CRDs +are installed), Helm 3.8+, and the wildcard DNS above. `deploy.sh` installs the +two operators for you (unless `SKIP_PREREQS=1`). + +## Deploy + +`deploy.sh` installs prerequisites, generates host overrides (per-subdomain URLs + +a stable per-participant token), installs the release, waits for rollouts, and +verifies end-to-end. Idempotent — tokens/URLs stay stable across re-runs. + +```bash +cd infrastructure/charts/eoapi-workshop +./deploy.sh deploy # prerequisites + chart + verify +./deploy.sh verify # re-run endpoint/auth checks, print Lab URLs +./deploy.sh urls # print participant Lab URLs (+ tokens) +./deploy.sh teardown [--all] # remove release (--all also removes operators) +``` + +Env vars: `BASE_DOMAIN` (default `eoapi-workshop.ds.io`), `SKIP_PREREQS=1`, +`GHCR_USER`+`GHCR_TOKEN` (pull secret for a private image — see +[Participant JupyterLabs](#participant-jupyterlabs)). `RELEASE`/`NAMESPACE` must +stay `eoapi`. + +The pgstac DB is created asynchronously by PGO and seeded with sample STAC data, +so API pods may restart a few times before `Ready` on first install. + +To install without `deploy.sh`: `helm dependency update`, then `helm install eoapi +. -n eoapi --create-namespace` with a `-f` overrides file (generate one for a +non-default domain via `BASE_DOMAIN=… ./deploy.sh overrides`). + +## Routing + +All routing is one Ingress (`templates/subdomain-ingress.yaml`): a host rule per +service, each serving at `/` with no rewrite. The upstream path-based ingress is +off and each app serves at its subdomain root — stac/raster/vector with +`--root-path=`, proxy `ROOT_PATH=""`, browser via the root-serving +`radiantearth/stac-browser`, Labs without `--ServerApp.base_url`. Per-subdomain +URLs default to the workshop domain in `values.yaml`; `deploy.sh` rewrites them for +another `BASE_DOMAIN` via the gitignored `.deploy/overrides.yaml`. + +## Verify + +`./deploy.sh verify` checks every service subdomain, runs the auth test, and prints +the Lab URLs. Manually: + +```bash +kubectl -n eoapi get pods +curl -s http://stac.eoapi-workshop.ds.io/healthz # also raster. / vector. +curl -s http://stac.eoapi-workshop.ds.io/collections # sample items +# UIs: browser. manager. mock-oidc./.well-known/openid-configuration +``` + +## Participant JupyterLabs + +`jupyter.participants` (default `lab-01`…`lab-05`; edit for any N) → one Deployment ++ Service + PVC each at `.`, running the GHCR image +`ghcr.io/developmentseed/eoapi-workshop` (built by +`.github/workflows/publish-workshop-image.yml`). Each Lab gets the eoAPI endpoints ++ DB creds injected (from the `eoapi-pguser-eoapi` PGO secret) and an access token +(`./deploy.sh urls` prints them). + +- **Persistence:** notebooks come fresh from the image (`/home/jovyan/docs`) on + every start, so updates always appear; only `/home/jovyan/work` persists (save + work there — edits to the provided notebooks reset on restart). +- **Private image:** GHCR packages are private by default. Either make the package + public, or pass a pull token — `GHCR_USER= GHCR_TOKEN= + ./deploy.sh deploy` creates the `ghcr-pull` secret and wires it to the default + ServiceAccount before the Labs start. + +## Testing auth + +`stac-auth-proxy` fronts STAC at `stac.`: **GET is public, mutations +need a bearer token** from the mock OIDC server (`jq` required). + +```bash +b=eoapi-workshop.ds.io +curl -s -o/dev/null -w '%{http_code}\n' http://stac.$b/collections # 200 (public read) +curl -s -o/dev/null -w '%{http_code}\n' -X POST http://stac.$b/collections \ + -H 'Content-Type: application/json' -d '{}' # 401 (no token) +TOKEN=$(curl -s http://mock-oidc.$b/ \ + --data-raw 'username=testuser&scopes=openid+stac:read+stac:write' \ + -H 'Accept: application/json' | jq -r .token) +curl -s -o/dev/null -w '%{http_code}\n' -X POST http://stac.$b/collections \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' -d '{}' # NOT 401 +``` + +**401 without a token, non-401 with one** = working. If it stays 401, check +`kubectl -n eoapi logs deploy/eoapi-stac-auth-proxy` (usual cause: release/namespace +not `eoapi`). + +## Upgrade / uninstall + +```bash +./deploy.sh deploy # idempotent re-deploy (tokens preserved) +helm uninstall eoapi -n eoapi # or ./deploy.sh teardown +kubectl -n eoapi delete pvc --all # PVCs (DB + Lab work) are retained by design +``` + +## Notebook data + +The workshop notebooks (`docs/00`–`06`) run in the Labs against this deployment: +- `pgstacBootstrap.loadSamples` is **off** — the upstream sample collection + `noaa-emergency-response` is stored without a STAC `type` field and breaks + `pystac_client` (notebook 03). The notebooks create their own STAC data. +- the **features-loader Job** (`featuresLoader.enabled`) loads the NA CEC Level III + Ecoregions into `features.ecoregions`, and tipg is configured with + `TIPG_DB_SCHEMAS=["features","public"]`, so notebook 05 has vector data. + +## Limitations +- **UI login needs TLS** — STAC Manager / Browser OIDC login uses PKCE (needs + HTTPS); over http they're read-only. Enable `routing.tls`. (Browser's + `redirect_uri` also still derives from the apex host upstream.) +- **Capacity** — N always-on Labs at `limit 2 CPU / 4Gi` (default 5 ≈ ≤10 CPU / + 20Gi) + stac-manager's ~4Gi startup build + the backend. Size nodes to N. +- **Not production** — test auth, single 1-replica DB (5Gi), http. For production + use the CDK/AWS stack in [`DEPLOYMENT.md`](../../../DEPLOYMENT.md). diff --git a/infrastructure/charts/eoapi-workshop/deploy.sh b/infrastructure/charts/eoapi-workshop/deploy.sh new file mode 100755 index 0000000..109780e --- /dev/null +++ b/infrastructure/charts/eoapi-workshop/deploy.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +# +# Reproducible deploy for the eoapi-workshop chart (subdomain-per-service). +# +# Every service is served at the root of its own subdomain under a wildcard +# domain: stac. raster. vector. browser. manager. mock-oidc. lab-01..NN. of +# ${BASE_DOMAIN}. A wildcard DNS record `*.${BASE_DOMAIN}` must point at the +# ingress LoadBalancer. +# +# Usage: +# ./deploy.sh deploy # prerequisites + chart + verify (idempotent) +# ./deploy.sh verify # re-run endpoint/auth checks + print Lab URLs +# ./deploy.sh urls # print the participant Lab URLs (with tokens) +# ./deploy.sh overrides # (re)generate + print .deploy/overrides.yaml, no deploy +# ./deploy.sh teardown # remove the release, PVCs and namespace +# ./deploy.sh teardown --all # also remove ingress-nginx + PGO +# +# Env: +# RELEASE Helm release name (default: eoapi) -- see OIDC contract below +# NAMESPACE target namespace (default: eoapi) -- see OIDC contract below +# BASE_DOMAIN wildcard base domain (default: eoapi-workshop.ds.io) +# SKIP_PREREQS=1 skip the ingress-nginx + PGO install +# GHCR_TOKEN token with read:packages → create an imagePullSecret so the +# cluster can pull a PRIVATE workshop image (with GHCR_USER). +# Omit if the GHCR package is public. +# +# !!! OIDC CONTRACT !!! The proxy's OIDC_DISCOVERY_INTERNAL_URL is pinned to the +# Service DNS name eoapi-mock-oidc-server.eoapi.svc.cluster.local, derived from +# RELEASE + NAMESPACE. Both MUST stay "eoapi" or in-cluster OIDC discovery breaks. +set -euo pipefail + +RELEASE="${RELEASE:-eoapi}" +NAMESPACE="${NAMESPACE:-eoapi}" +BASE_DOMAIN="${BASE_DOMAIN:-eoapi-workshop.ds.io}" +CHART_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OVERRIDES="${CHART_DIR}/.deploy/overrides.yaml" + +log() { printf '\n\033[1;34m==> %s\033[0m\n' "$*" >&2; } + +install_prereqs() { + if [[ "${SKIP_PREREQS:-0}" == "1" ]]; then log "Skipping prerequisites (SKIP_PREREQS=1)"; return; fi + if kubectl get ingressclass nginx >/dev/null 2>&1; then + log "An 'nginx' ingressclass already exists — leaving ingress-nginx untouched" + else + log "Installing NGINX ingress controller" + helm upgrade --install ingress-nginx ingress-nginx \ + --repo https://kubernetes.github.io/ingress-nginx \ + --namespace ingress-nginx --create-namespace --wait --timeout 5m + fi + log "Installing Crunchy Postgres Operator (PGO)" + helm upgrade --install pgo oci://registry.developers.crunchydata.com/crunchydata/pgo \ + --namespace postgres-operator --create-namespace --wait --timeout 5m +} + +# Participant names, read from the rendered chart (single source of truth = values). +participant_names() { + helm template "$RELEASE" "$CHART_DIR" -n "$NAMESPACE" \ + --show-only templates/jupyter.yaml 2>/dev/null \ + | grep -oE "^ name: ${RELEASE}-[a-z0-9-]+" | sed "s/ name: ${RELEASE}-//" | sort -u +} + +# Reuse a participant's token from the existing overrides (idempotent URLs across +# re-deploys); prints nothing and returns 1 if not present. +existing_token() { # + [[ -f "$OVERRIDES" ]] || return 1 + local t; t="$(grep -E "name: $1, token:" "$OVERRIDES" 2>/dev/null | sed -E 's/.*token: "([^"]+)".*/\1/' | head -1)" + [[ -n "$t" ]] && printf '%s' "$t" +} + +# Host-specific overrides — derived, NEVER committed (gitignored .deploy/). +write_overrides() { + mkdir -p "$(dirname "$OVERRIDES")" + local tmp; tmp="$(mktemp)" + { + echo "# Generated by deploy.sh — DO NOT COMMIT. baseDomain=${BASE_DOMAIN}" + echo "routing:" + echo " baseDomain: \"${BASE_DOMAIN}\"" + echo "eoapi:" + echo " browser:" + echo " catalogUrl: \"http://stac.${BASE_DOMAIN}\"" + echo " oidcDiscoveryUrl: \"http://mock-oidc.${BASE_DOMAIN}/.well-known/openid-configuration\"" + # NOTE: stac-auth-proxy OIDC_DISCOVERY_URL is intentionally NOT overridden — + # it must stay the in-cluster URL (the proxy fetches JWKS from that origin; + # an external LB URL hairpins and fails). It is domain-independent. + echo " testing:" + echo " mockOidcServer:" + echo " extraEnv:" # list: restate in full + echo " - name: ISSUER" + echo " value: \"http://mock-oidc.${BASE_DOMAIN}\"" + echo " - name: SCOPES" + echo " value: \"stac:read,stac:write\"" + echo "stac-manager:" + echo " publicUrl: \"http://manager.${BASE_DOMAIN}\"" + echo " stacApi: \"http://stac.${BASE_DOMAIN}\"" + echo " stacBrowser: \"http://browser.${BASE_DOMAIN}\"" + echo " oidc:" + echo " authority: \"http://mock-oidc.${BASE_DOMAIN}\"" + echo "jupyter:" + echo " participants:" + local name tok + while read -r name; do + [[ -n "$name" ]] || continue + tok="$(existing_token "$name" || true)"; [[ -n "$tok" ]] || tok="$(openssl rand -hex 16)" + echo " - { name: ${name}, token: \"${tok}\" }" + done < <(participant_names) + } > "$tmp" + mv "$tmp" "$OVERRIDES" +} + +# Optional: let the cluster pull a PRIVATE workshop image. Set GHCR_TOKEN (a +# token with read:packages) + GHCR_USER to create an imagePullSecret and attach +# it to the namespace's default ServiceAccount (which the Labs use). Must run +# BEFORE the Lab pods are created so the secret is injected at creation time. +# Not needed if the GHCR package is public. +setup_pull_secret() { + if [[ -z "${GHCR_TOKEN:-}" ]]; then + log "No GHCR_TOKEN set — assuming the workshop image is public (skipping pull secret)" + return + fi + log "Creating GHCR pull secret + attaching it to the default ServiceAccount" + kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + kubectl -n "$NAMESPACE" create secret docker-registry ghcr-pull \ + --docker-server=ghcr.io --docker-username="${GHCR_USER:-$USER}" --docker-password="$GHCR_TOKEN" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + kubectl -n "$NAMESPACE" patch serviceaccount default \ + -p '{"imagePullSecrets":[{"name":"ghcr-pull"}]}' >/dev/null +} + +deploy_chart() { + log "Building chart dependencies" + # Register the dependency repos so `helm dependency build` can resolve them + # from Chart.lock on a fresh machine (the vendored .tgz are gitignored). + helm repo add eoapi https://developmentseed.org/eoapi-k8s/ --force-update >/dev/null 2>&1 || true + helm repo add stac-manager https://stac-manager.ds.io/ --force-update >/dev/null 2>&1 || true + helm repo update eoapi stac-manager >/dev/null 2>&1 || true + helm dependency build "$CHART_DIR" >/dev/null + log "Writing host overrides for ${BASE_DOMAIN} (tokens preserved across re-deploys)" + write_overrides + echo " overrides: ${OVERRIDES}" + setup_pull_secret # before helm upgrade, so new Lab pods inherit the secret + log "Deploying release '${RELEASE}' in namespace '${NAMESPACE}'" + helm upgrade --install "$RELEASE" "$CHART_DIR" \ + -n "$NAMESPACE" --create-namespace -f "$OVERRIDES" + log "Waiting for deployments (the database is created asynchronously by PGO)" + local d + for d in stac raster vector browser stac-auth-proxy mock-oidc-server stac-manager $(participant_names); do + kubectl -n "$NAMESPACE" rollout status "deploy/${RELEASE}-${d}" --timeout=300s || true + done +} + +# curl a URL until it returns the expected code (nginx warmup can lag). +_expect() { # -> echoes actual code, returns 0 if matched + local url="$1" want="$2" code="" _ + for _ in 1 2 3 4 5 6 7 8; do + code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 "$url" || true)" + [[ "$code" == "$want" ]] && break; sleep 3 + done + printf '%s' "$code" +} + +verify() { + local b="$BASE_DOMAIN" ok=1 code + log "Verifying service subdomains at *.$b" + declare -a checks=( + "http://stac.$b/healthz|200|stac" + "http://stac.$b/collections|200|stac collections" + "http://raster.$b/healthz|200|raster" + "http://vector.$b/healthz|200|vector" + "http://browser.$b/|200|browser" + "http://manager.$b/|200|manager" + "http://mock-oidc.$b/.well-known/openid-configuration|200|mock-oidc" + ) + local c url want name + for c in "${checks[@]}"; do + IFS='|' read -r url want name <<<"$c" + code="$(_expect "$url" "$want")" + printf ' %-16s %-52s %s\n' "$name" "$url" "$code" + [[ "$code" == "$want" ]] || ok=0 + done + + log "Verifying auth (expect 401 without a token, non-401 with one)" + local no_tok token with_tok + no_tok="$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \ + -X POST "http://stac.$b/collections" -H 'Content-Type: application/json' -d '{}' || true)" + token="$(curl -s --max-time 15 "http://mock-oidc.$b/" \ + --data-raw 'username=testuser&scopes=openid+stac:read+stac:write' \ + -H 'Accept: application/json' | sed -n 's/.*"token":"\([^"]*\)".*/\1/p')" + with_tok="$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 \ + -X POST "http://stac.$b/collections" -H "Authorization: Bearer ${token}" \ + -H 'Content-Type: application/json' -d '{}' || true)" + printf ' POST without token: %s with token: %s\n' "$no_tok" "$with_tok" + [[ "$no_tok" == "401" && "$with_tok" != "401" && -n "$with_tok" ]] || ok=0 + + print_urls + if [[ "$ok" == 1 ]]; then log "OK — services reachable and auth enforced."; else log "FAILED — see codes above"; exit 1; fi +} + +print_urls() { + log "Participant JupyterLab URLs" + local name tok + while read -r name; do + [[ -n "$name" ]] || continue + tok="$(existing_token "$name" || true)" + printf ' %-8s http://%s.%s/lab?token=%s\n' "$name" "$name" "$BASE_DOMAIN" "$tok" + done < <(participant_names) + printf ' %-8s http://manager.%s/\n' "manager" "$BASE_DOMAIN" + printf ' %-8s http://browser.%s/\n' "browser" "$BASE_DOMAIN" +} + +teardown() { + log "Uninstalling release '${RELEASE}'" + helm uninstall "$RELEASE" -n "$NAMESPACE" 2>/dev/null || true + kubectl -n "$NAMESPACE" delete pvc --all 2>/dev/null || true + kubectl delete namespace "$NAMESPACE" --timeout=180s 2>/dev/null || true + if [[ "${1:-}" == "--all" ]]; then + log "Removing prerequisites (ingress-nginx + PGO)" + helm uninstall ingress-nginx -n ingress-nginx 2>/dev/null || true + helm uninstall pgo -n postgres-operator 2>/dev/null || true + kubectl delete namespace ingress-nginx postgres-operator --timeout=180s 2>/dev/null || true + fi +} + +case "${1:-deploy}" in + deploy) install_prereqs; deploy_chart; verify ;; + verify) verify ;; + urls) print_urls ;; + overrides) write_overrides; echo "written: ${OVERRIDES}"; cat "$OVERRIDES" ;; + teardown) teardown "${2:-}" ;; + *) echo "Usage: $0 {deploy|verify|urls|teardown [--all]}" >&2; exit 2 ;; +esac diff --git a/infrastructure/charts/eoapi-workshop/templates/NOTES.txt b/infrastructure/charts/eoapi-workshop/templates/NOTES.txt new file mode 100644 index 0000000..f5709cf --- /dev/null +++ b/infrastructure/charts/eoapi-workshop/templates/NOTES.txt @@ -0,0 +1,29 @@ +eoAPI workshop — release {{ .Release.Name }} in namespace {{ .Release.Namespace }}. + +Every service is served at the root of its own subdomain under +*.{{ .Values.routing.baseDomain }} (requires a wildcard DNS record pointing at +the ingress LoadBalancer): + + STAC API http://stac.{{ .Values.routing.baseDomain }} + Raster http://raster.{{ .Values.routing.baseDomain }} + Vector http://vector.{{ .Values.routing.baseDomain }} + Browser http://browser.{{ .Values.routing.baseDomain }} +{{- $sm := index .Values "stac-manager" }} +{{- if and $sm $sm.enabled }} + Manager http://manager.{{ .Values.routing.baseDomain }} +{{- end }} + Mock OIDC http://mock-oidc.{{ .Values.routing.baseDomain }} (test-only auth) + +{{- if .Values.jupyter.enabled }} + +Participant JupyterLabs: +{{- range .Values.jupyter.participants }} + {{ .name }} http://{{ .name }}.{{ $.Values.routing.baseDomain }}/lab{{ if .token }}?token={{ .token }}{{ end }} +{{- end }} +{{- if not (first .Values.jupyter.participants).token }} + (tokens are empty — deploy with ./deploy.sh so each Lab gets an access token) +{{- end }} +{{- end }} + +Verify endpoints + auth and (re)print the Lab URLs: + ./deploy.sh verify diff --git a/infrastructure/charts/eoapi-workshop/templates/features-loader-job.yaml b/infrastructure/charts/eoapi-workshop/templates/features-loader-job.yaml new file mode 100644 index 0000000..f0442d1 --- /dev/null +++ b/infrastructure/charts/eoapi-workshop/templates/features-loader-job.yaml @@ -0,0 +1,122 @@ +{{/* +k8s equivalent of the docker-compose `features-loader` + `stac-loader`: load +the NA CEC Level III Ecoregions shapefile into features.ecoregions (notebook +05 / tipg) and the glad STAC collection from the MAAP STAC into pgstac +(notebook 04 §4.5 / titiler). Runs as a post-install/post-upgrade hook, +idempotently (each container skips if its data is already present). Uses the +PGO superuser secret to CREATE SCHEMA and grants read to all users so tipg can +serve it. Loads into the `eoapi` database (what tipg and pgstac serve). +*/}} +{{- if .Values.featuresLoader.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-features-loader + labels: + app: {{ .Release.Name }}-features-loader + app.kubernetes.io/component: workshop-features-loader + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "10" # after the DB / pgstac bootstrap + helm.sh/hook-delete-policy: before-hook-creation +spec: + backoffLimit: 3 + template: + metadata: + labels: + app: {{ .Release.Name }}-features-loader + spec: + restartPolicy: Never + containers: + - name: features-loader + image: {{ .Values.featuresLoader.image | quote }} + command: + - bash + - -c + - | + set -euo pipefail + apt-get update -qq && apt-get install -y -qq postgresql-client >/dev/null + for _ in $(seq 1 60); do pg_isready -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" && break; echo "waiting for database..."; sleep 5; done + if psql -tAc "select 1 from information_schema.tables where table_schema='features' and table_name='ecoregions'" | grep -q 1; then + echo "features.ecoregions already present — skipping load."; exit 0 + fi + psql -v ON_ERROR_STOP=1 -c "CREATE SCHEMA IF NOT EXISTS features;" + ogr2ogr -f PostgreSQL "PG:dbname=$PGDATABASE host=$PGHOST port=$PGPORT user=$PGUSER password=$PGPASSWORD" \ + {{ .Values.featuresLoader.shapefileUrl | quote }} \ + -nln features.ecoregions -t_srs EPSG:4326 \ + -lco GEOMETRY_NAME=geom -lco FID=id -lco PRECISION=NO -nlt PROMOTE_TO_MULTI + psql -v ON_ERROR_STOP=1 -c "GRANT USAGE ON SCHEMA features TO PUBLIC; GRANT SELECT ON ALL TABLES IN SCHEMA features TO PUBLIC;" + echo "features.ecoregions loaded." + env: + # Superuser creds (needed for CREATE SCHEMA); DB is pinned to `eoapi` + # (what tipg queries), not the secret's dbname. + - name: PGDATABASE + value: "eoapi" + - name: PGHOST + valueFrom: { secretKeyRef: { name: {{ .Release.Name }}-pguser-postgres, key: host } } + - name: PGPORT + valueFrom: { secretKeyRef: { name: {{ .Release.Name }}-pguser-postgres, key: port } } + - name: PGUSER + valueFrom: { secretKeyRef: { name: {{ .Release.Name }}-pguser-postgres, key: user } } + - name: PGPASSWORD + valueFrom: { secretKeyRef: { name: {{ .Release.Name }}-pguser-postgres, key: password } } + resources: + {{- toYaml .Values.featuresLoader.resources | nindent 12 }} + - name: stac-loader + image: {{ .Values.featuresLoader.stac.image | quote }} + command: + - bash + - -c + - | + set -euo pipefail + pip install -q "pypgstac[psycopg]=={{ .Values.featuresLoader.stac.pypgstacVersion }}" + python3 - <<'PY' + import json + import sys + import urllib.request + + from pypgstac.db import PgstacDB + from pypgstac.load import Loader, Methods + + SRC = {{ .Values.featuresLoader.stac.source | quote }} + COLLECTION = {{ .Values.featuresLoader.stac.collection | quote }} + LIMIT = {{ .Values.featuresLoader.stac.itemLimit }} + + db = PgstacDB() + if db.query_one( + "select 1 from pgstac.collections where id=%s", [COLLECTION] + ): + print(f"{COLLECTION} already present -- skipping load.") + sys.exit(0) + + urllib.request.urlretrieve( + f"{SRC}/collections/{COLLECTION}", "/tmp/collection.json" + ) + with urllib.request.urlopen( + f"{SRC}/search?collections={COLLECTION}&limit={LIMIT}" + ) as r: + features = json.load(r)["features"] + with open("/tmp/items.ndjson", "w") as f: + for feat in features: + f.write(json.dumps(feat) + "\n") + + loader = Loader(db=db) + loader.load_collections("/tmp/collection.json", Methods.upsert) + loader.load_items("/tmp/items.ndjson", Methods.upsert) + print(f"{COLLECTION}: collection + {len(features)} items loaded.") + PY + env: + - name: PGDATABASE + value: "eoapi" + - name: PGHOST + valueFrom: { secretKeyRef: { name: {{ .Release.Name }}-pguser-postgres, key: host } } + - name: PGPORT + valueFrom: { secretKeyRef: { name: {{ .Release.Name }}-pguser-postgres, key: port } } + - name: PGUSER + valueFrom: { secretKeyRef: { name: {{ .Release.Name }}-pguser-postgres, key: user } } + - name: PGPASSWORD + valueFrom: { secretKeyRef: { name: {{ .Release.Name }}-pguser-postgres, key: password } } + resources: + requests: { cpu: "100m", memory: "256Mi" } + limits: { cpu: "500m", memory: "1Gi" } +{{- end }} diff --git a/infrastructure/charts/eoapi-workshop/templates/jupyter.yaml b/infrastructure/charts/eoapi-workshop/templates/jupyter.yaml new file mode 100644 index 0000000..ea8ace4 --- /dev/null +++ b/infrastructure/charts/eoapi-workshop/templates/jupyter.yaml @@ -0,0 +1,139 @@ +{{/* +Per-participant JupyterLab environments (one isolated Deployment+Service+PVC +each), served at the ROOT of . and routed by +templates/subdomain-ingress.yaml (no path prefix, no base_url). Notebooks come +from the image; only /home/jovyan/work is persisted. Endpoints + DB creds are +injected to mirror the docker-compose `jupyterhub` service so the workshop +notebooks run unchanged. +*/}} +{{- if .Values.jupyter.enabled }} +{{- $root := . }} +{{- $j := .Values.jupyter }} +{{- $base := .Values.routing.baseDomain }} +{{- range $j.participants }} +{{- $name := .name }} +{{- $token := .token | default "" }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ $root.Release.Name }}-{{ $name }} + labels: + app: {{ $root.Release.Name }}-{{ $name }} + app.kubernetes.io/component: workshop-jupyterlab +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ $j.storage.size | quote }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $root.Release.Name }}-{{ $name }} + labels: + app: {{ $root.Release.Name }}-{{ $name }} + app.kubernetes.io/component: workshop-jupyterlab +spec: + selector: + app: {{ $root.Release.Name }}-{{ $name }} + ports: + - name: http + port: 8888 + targetPort: 8888 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $root.Release.Name }}-{{ $name }} + labels: + app: {{ $root.Release.Name }}-{{ $name }} + app.kubernetes.io/component: workshop-jupyterlab +spec: + replicas: 1 + strategy: + type: Recreate # RWO PVC: the new pod can't mount until the old one releases it + selector: + matchLabels: + app: {{ $root.Release.Name }}-{{ $name }} + template: + metadata: + labels: + app: {{ $root.Release.Name }}-{{ $name }} + spec: + securityContext: + fsGroup: 1000 # jovyan GID — makes the work PVC writable by the notebook user + # Notebooks live in the image at /home/jovyan/docs and come FRESH on every + # start, so image/notebook updates always appear. Only /home/jovyan/work is + # a persistent PVC — no seed initContainer, no home shadowing. (Trade-off: + # edits to the provided notebooks reset on pod restart; save work under work/.) + containers: + - name: jupyterlab + image: "{{ $j.image.repository }}:{{ $j.image.tag }}" + imagePullPolicy: {{ $j.image.pullPolicy }} + # args only (NO command): keep the image ENTRYPOINT (/entrypoint.sh + # activates the conda env) and just override the launch command. + # Served at the root of . — no base_url prefix. + args: + - jupyter + - lab + - --ServerApp.token={{ $token }} + - --ip=0.0.0.0 + - --port=8888 + - --no-browser + ports: + - name: http + containerPort: 8888 + env: + # eoAPI endpoints (in-cluster) — mirrors the docker-compose jupyterhub service. + - { name: STAC_API_ENDPOINT, value: "http://{{ $root.Release.Name }}-stac-auth-proxy:8080" } + - { name: STAC_AUTH_PROXY_ENDPOINT, value: "http://{{ $root.Release.Name }}-stac-auth-proxy:8080" } + - { name: TITILER_PGSTAC_API_ENDPOINT, value: "http://{{ $root.Release.Name }}-raster:8080" } + - { name: TIPG_API_ENDPOINT, value: "http://{{ $root.Release.Name }}-vector:8080" } + - { name: MOCK_OIDC_ENDPOINT, value: "http://{{ $root.Release.Name }}-mock-oidc-server:8080" } + - { name: STAC_BROWSER_ENDPOINT, value: "http://browser.{{ $base }}" } + # Browser-facing API URLs (external subdomains) — the notebooks' IFrame/ + # viewer cells hand these to the user's browser, which cannot reach the + # in-cluster Service DNS above. Server-side httpx calls keep the *_ENDPOINT + # vars; absent these (compose/2i2c) the notebooks fall back to .replace(). + - { name: STAC_API_BROWSER_URL, value: "http://stac.{{ $base }}" } + - { name: TITILER_BROWSER_URL, value: "http://raster.{{ $base }}" } + - { name: TIPG_BROWSER_URL, value: "http://vector.{{ $base }}" } + # DB creds from the PGO-generated secret, using the DIRECT primary keys + # (host/port) — NOT pgbouncer-*, whose transaction pooling breaks the + # DDL/COPY in 02-database.ipynb. Secret name is release-derived + # (release/namespace contract: both must be "eoapi"). + - name: PGHOST + valueFrom: { secretKeyRef: { name: {{ $root.Release.Name }}-pguser-eoapi, key: host } } + - name: PGPORT + valueFrom: { secretKeyRef: { name: {{ $root.Release.Name }}-pguser-eoapi, key: port } } + - name: PGDATABASE + valueFrom: { secretKeyRef: { name: {{ $root.Release.Name }}-pguser-eoapi, key: dbname } } + - name: PGUSER + valueFrom: { secretKeyRef: { name: {{ $root.Release.Name }}-pguser-eoapi, key: user } } + - name: PGPASSWORD + valueFrom: { secretKeyRef: { name: {{ $root.Release.Name }}-pguser-eoapi, key: password } } + volumeMounts: + - name: work + mountPath: /home/jovyan/work + # JupyterLab has no unauthenticated HTTP health endpoint → probe TCP. + # Generous startupProbe so the first (large) image pull doesn't trip liveness. + startupProbe: + tcpSocket: { port: 8888 } + periodSeconds: 10 + failureThreshold: 30 + readinessProbe: + tcpSocket: { port: 8888 } + periodSeconds: 10 + livenessProbe: + tcpSocket: { port: 8888 } + periodSeconds: 30 + failureThreshold: 3 + resources: + {{- toYaml $j.resources | nindent 12 }} + volumes: + - name: work + persistentVolumeClaim: + claimName: {{ $root.Release.Name }}-{{ $name }} +{{- end }} +{{- end }} diff --git a/infrastructure/charts/eoapi-workshop/templates/subdomain-ingress.yaml b/infrastructure/charts/eoapi-workshop/templates/subdomain-ingress.yaml new file mode 100644 index 0000000..56b2aa7 --- /dev/null +++ b/infrastructure/charts/eoapi-workshop/templates/subdomain-ingress.yaml @@ -0,0 +1,69 @@ +{{/* +Subdomain-per-service ingress. Each enabled service is exposed at +