diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..5d0dfa2 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,27 @@ +FROM mcr.microsoft.com/devcontainers/javascript-node:24-bookworm + +FROM --platform=linux/amd64 quay.io/openshift/origin-cli:5.1 AS openshift_cli + +FROM mcr.microsoft.com/devcontainers/javascript-node:24-bookworm + +ARG KUBECTL_VERSION=v1.33.3 +ARG HELM_VERSION=v3.18.4 + +COPY scripts/install-kubernetes-tools.sh /tmp/install-kubernetes-tools.sh +COPY --from=openshift_cli /usr/bin/oc /usr/local/bin/oc.amd64 + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl tar gzip; \ + chmod +x /usr/local/bin/oc.amd64; \ + printf '%s\n' '#!/bin/sh' \ + 'if [ "$(uname -m)" = "x86_64" ]; then' \ + ' exec /usr/local/bin/oc.amd64 "$@"' \ + 'fi' \ + 'echo "OpenShift oc 5.1 is only available for amd64 in this image build." >&2' \ + 'exit 1' > /usr/local/bin/oc; \ + chmod +x /usr/local/bin/oc; \ + rm -rf /var/lib/apt/lists/*; \ + chmod +x /tmp/install-kubernetes-tools.sh; \ + KUBECTL_VERSION="$KUBECTL_VERSION" HELM_VERSION="$HELM_VERSION" /tmp/install-kubernetes-tools.sh; \ + rm -f /tmp/install-kubernetes-tools.sh \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 4834aca..8ed9954 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,20 +1,32 @@ +name: lab-framework_devcontainer + services: # Update this to the name of the service you want to work with in your docker-compose.yml file lab-framework-core: - image: mcr.microsoft.com/devcontainers/javascript-node:24-bookworm - volumes: - # Update this to wherever you want VS Code to mount the folder of your project - - ..:/workspace:cached - + build: + context: .. + dockerfile: .devcontainer/Dockerfile # Overrides default command so things don't shut down after the process ends. command: /bin/sh -c "while sleep 1000; do :; done" ports: - "3000:3000" depends_on: - - mock-server + mock-server: + condition: service_started + k3s-single-node: + condition: service_healthy + required: false + environment: + KUBECONFIG: ${KUBECONFIG:-/home/node/.kube/k3s.yaml} + KUBERNETES_API_URL: ${KUBERNETES_API_URL:-https://k3s-single-node:6443} extra_hosts: - "host.docker.internal:host-gateway" + volumes: + # Update this to wherever you want VS Code to mount the folder of your project + - ..:/workspace:cached + - k3s-kubeconfig:/home/node/.kube:ro + - ../openshift:/home/node/.kube-openshift:ro mock-server: image: mockserver/mockserver @@ -22,9 +34,54 @@ services: - "80:1080" # for UDF mocks - "5123:1080" # for deployment identifier service mocks + k3s-single-node: + image: rancher/k3s:v1.33.3-k3s1 + profiles: ["k3s"] + privileged: true + cgroup: host + entrypoint: + - /bin/sh + - -c + command: + - >- + set -eu; + /bin/k3s server + --write-kubeconfig=/kubeconfig/k3s-source.yaml + --write-kubeconfig-mode=644 + --tls-san=k3s-single-node + --tls-san=host.docker.internal + --disable=traefik + --disable=servicelb + --kube-controller-manager-arg=cluster-signing-duration=87600h & + k3s_pid=$$!; + until [ -s /kubeconfig/k3s-source.yaml ]; do sleep 1; done; + sed 's|https://127.0.0.1:6443|https://k3s-single-node:6443|g' /kubeconfig/k3s-source.yaml > /kubeconfig/k3s.yaml; + cp /kubeconfig/k3s.yaml /kubeconfig/k3s-host.yaml; + sed -i 's|https://k3s-single-node:6443|https://host.docker.internal:6443|g' /kubeconfig/k3s-host.yaml; + wait "$$k3s_pid" + ports: + - "6443:6443" + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - k3s-data:/var/lib/rancher/k3s + - k3s-kubeconfig:/kubeconfig + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/run/docker.sock:/var/run/docker.sock + healthcheck: + test: ["CMD-SHELL", "KUBECONFIG=/kubeconfig/k3s-source.yaml k3s kubectl get --raw=/readyz >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 20s + volumes: cache: driver: local + k3s-data: + driver: local + k3s-kubeconfig: + driver: local networks: default: diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh index f2a88bf..6033013 100755 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -eu + # Get the docker assigned address of the host and insert it into the hosts file for use by UDF API mocks printf "\n`getent ahostsv4 host.docker.internal | grep STREAM | awk '{ print $1 ; exit }'` metadata.udf" | sudo tee -a /etc/hosts diff --git a/.env.example b/.env.example index 726c4ea..61dfece 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,9 @@ # Framework ENV Vars # DOCKER_API_URL=tcp://host.docker.internal:2375 # Set to use the Docker API (instead of docker.sock) when the framework makes docker calls. +# KUBERNETES_API_URL=https://k3s-single-node:6443 # Optional API endpoint for cluster-aware framework features. +# KUBECONFIG=/app/.kube/k3s.yaml # Kubeconfig mounted from the peer K3s service. +# For OpenShift, mount ./openshift/kubeconfig.yaml and set KUBECONFIG=/app/.kube-openshift/kubeconfig.yaml. +# For OpenShift, set KUBERNETES_API_URL to your cluster API endpoint. DEPLOYMENT_IDENTIFIER_RESPONSE_FIELD=petname REMOTE_DOCS_REPO_SERVER="https://raw.githubusercontent.com" REMOTE_DOCS_REPO_API_SERVER="https://api.github.com" diff --git a/.gitignore b/.gitignore index c2e1564..062fb38 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,6 @@ report.json # Used in lab content nginx-api.conf -ssl \ No newline at end of file +ssl +kubeconfig.yaml +kubeconfig diff --git a/Dockerfile b/Dockerfile index 842322d..7cec694 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,8 @@ -FROM node:24-alpine AS base +FROM node:24-bookworm-slim AS base # Install dependencies only when needed FROM base AS deps -# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed. -RUN apk add --no-cache libc6-compat WORKDIR /app # Install dependencies based on the preferred package manager @@ -35,12 +33,34 @@ RUN \ else echo "Lockfile not found." && exit 1; \ fi +FROM --platform=linux/amd64 quay.io/openshift/origin-cli:5.1 AS openshift_cli + # Production image, copy all the files and run next FROM base AS runner WORKDIR /app -# Add Docker CLI & cURL (for troubleshooting) -RUN apk add --no-cache docker-cli curl +ARG KUBECTL_VERSION=v1.33.3 +ARG HELM_VERSION=v3.18.4 + +COPY scripts/install-kubernetes-tools.sh /tmp/install-kubernetes-tools.sh +COPY --from=openshift_cli /usr/bin/oc /usr/local/bin/oc.amd64 + +# Add Docker CLI, kubectl, and Helm for container and Kubernetes workflows. +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends docker.io curl ca-certificates tar gzip; \ + chmod +x /usr/local/bin/oc.amd64; \ + printf '%s\n' '#!/bin/sh' \ + 'if [ "$(uname -m)" = "x86_64" ]; then' \ + ' exec /usr/local/bin/oc.amd64 "$@"' \ + 'fi' \ + 'echo "OpenShift oc 5.1 is only available for amd64 in this image build." >&2' \ + 'exit 1' > /usr/local/bin/oc; \ + chmod +x /usr/local/bin/oc; \ + rm -rf /var/lib/apt/lists/*; \ + chmod +x /tmp/install-kubernetes-tools.sh; \ + KUBECTL_VERSION="$KUBECTL_VERSION" HELM_VERSION="$HELM_VERSION" /tmp/install-kubernetes-tools.sh; \ + rm -f /tmp/install-kubernetes-tools.sh ENV NODE_ENV=production # Uncomment the following line in case you want to disable telemetry during runtime. diff --git a/README.md b/README.md index 243290c..928d0a3 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,26 @@ Open [http://localhost:3000](http://localhost:3000) with your browser to see the You can start editing the lab markdown page by modifying `app/docs/nginx-one.mdx`. The page auto-updates as you edit the file. +An optional Kubernetes peer cluster profile is available in the devcontainer Compose stack: + +```shell +# Rebuild devcontainer with default services (no optional k3s profile) +Dev Containers: Rebuild Container + +# Rebuild devcontainer with optional k3s peer cluster +COMPOSE_PROFILES=k3s Dev Containers: Rebuild Container +``` + +For OpenShift access in devcontainer mode, provide `./openshift/kubeconfig.yaml` in the repository workspace and set: + +1. `KUBECONFIG=/home/node/.kube-openshift/kubeconfig.yaml` +1. `KUBERNETES_API_URL=https://api.your-openshift.example:6443` + ## "Production" Docker Deployment -The lab framework runs in Docker, so the it must be installed in the host system. Additionally, the lab framework container will use the host's docker API to manage containers. The host system needs to expose the Docker API over a network so that the lab framework container can connect to it. This approach involves some setup and is more secure when properly configured, especially when using TLS for encrypted communication. +The lab framework runs in Docker, so Docker must be installed in the host system. The framework container will use the host's Docker API to manage containers, and the deployment may also include a peer single-node K3s cluster container that the framework can use for Kubernetes and Helm workflows. + +The host system needs to expose the Docker API over a network so that the lab framework container can connect to it. This approach involves some setup and is more secure when properly configured, especially when using TLS for encrypted communication. Assuming a host system of Ubuntu 24.04, docker can be installed using the following: @@ -87,7 +104,42 @@ sudo chmod 0666 /var/run/docker.sock ``` -A `docker-compose.yaml` file has been provided to orchestrate the lab framework containers. In order for the lab framework to create additional containers required by the lab author, the docker calls need to use the host's docker daemon. The above steps enable mounting docker.sock as a volume into the container. +The lab framework deployment uses Compose to orchestrate the application container, MockServer, and any peer infrastructure containers such as the K3s cluster. + +The K3s service is optional and is controlled by the `k3s` Compose profile: + +```shell +# Start the framework stack without K3s +docker compose up -d + +# Start the framework stack with the optional K3s peer cluster +docker compose --profile k3s up -d + +# Stop the framework stack started without K3s +docker compose down + +# Stop the framework stack started with the optional K3s peer cluster +docker compose --profile k3s down + +# Optional: remove volumes for a full reset +docker compose --profile k3s down -v +``` + +OpenShift access is also available for connecting the framework to an external OpenShift cluster using a kubeconfig file: + +```shell +# Prepare an OpenShift kubeconfig at this path: +# ./openshift/kubeconfig.yaml + +# Point the app at the OpenShift kubeconfig and API endpoint +export KUBECONFIG=/app/.kube-openshift/kubeconfig.yaml +export KUBERNETES_API_URL=https://api.your-openshift.example:6443 + +# Start the framework stack +docker compose up -d +``` + +In order for the lab framework to create additional containers required by the lab author, the docker calls need to use the host's docker daemon. The above steps enable mounting docker.sock as a volume into the container. If this isn't preferred (or will not work), the host's docker daemon has been configured to allow remote hosts to call the docker API. To use this, you will specify a custom host via the `-H` parameter in order to connect to the hosts Docker API. Example: @@ -95,6 +147,29 @@ If this isn't preferred (or will not work), the host's docker daemon has been co docker -H tcp://host.docker.internal:2375 pull ubuntu ``` +### Kubernetes Peer Cluster + +When the `k3s` profile is enabled, the deployment starts a single-node K3s cluster as a peer container in the same Compose network. The lab framework can connect to the cluster through the Kubernetes API, and host tools such as `kubectl` and `helm` may connect directly when port `6443` is published to the host. + +The intended operating model is trusted local-lab administration: + +1. The framework may use full cluster-admin privileges. +1. Namespace scoping is not required. +1. Kubeconfig and cluster credentials do not need to rotate on stack redeploy. +1. Cluster certificates should be issued with a multi-year validity window. +1. The K3s container may mount the host `docker.sock` if a lab workflow needs Docker-backed cluster integration. + +The framework and the K3s container should share a kubeconfig or equivalent access path so that Kubernetes operations, resource provisioning, and Helm chart installs can be driven from the application runtime. + +### OpenShift Option + +The framework image includes the OpenShift `oc` client pinned to v5.1. Compose mounts `./openshift` into the framework container at `/app/.kube-openshift` (and `/home/node/.kube-openshift` in devcontainer mode), so OpenShift access uses your provided kubeconfig directly. + +Use the following environment variables to point the framework at OpenShift: + +1. `KUBECONFIG=/app/.kube-openshift/kubeconfig.yaml` +1. `KUBERNETES_API_URL=https://api.your-openshift.example:6443` + ### Environment You will need to create your own `/.env` file to use remote MDX documents. Use the `/.env.example` as a template. diff --git a/docker-compose.yml b/docker-compose.yml index 9378916..34ab9da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,15 +7,72 @@ services: restart: always ports: - "3000:3000" + depends_on: + k3s-single-node: + condition: service_healthy + required: false + environment: + KUBECONFIG: ${KUBECONFIG:-/app/.kube/k3s.yaml} + KUBERNETES_API_URL: ${KUBERNETES_API_URL:-https://k3s-single-node:6443} extra_hosts: - "host.docker.internal:host-gateway" volumes: - "/var/run/docker.sock:/var/run/docker.sock" - "./.env:/app/.env" + - "k3s-kubeconfig:/app/.kube:ro" + - "./openshift:/app/.kube-openshift:ro" + + k3s-single-node: + image: rancher/k3s:v1.33.3-k3s1 + profiles: ["k3s"] + privileged: true + restart: always + command: + - /bin/sh + - -c + - | + set -eu + /bin/k3s server \ + --write-kubeconfig=/kubeconfig/k3s-source.yaml \ + --write-kubeconfig-mode=644 \ + --tls-san=k3s-single-node \ + --tls-san=host.docker.internal \ + --disable=traefik \ + --disable=servicelb \ + --kube-controller-manager-arg=cluster-signing-duration=87600h & + k3s_pid=$$! + + until [ -s /kubeconfig/k3s-source.yaml ]; do + sleep 1 + done + + sed 's|https://127.0.0.1:6443|https://k3s-single-node:6443|g' /kubeconfig/k3s-source.yaml > /kubeconfig/k3s.yaml + cp /kubeconfig/k3s.yaml /kubeconfig/k3s-host.yaml + sed -i 's|https://k3s-single-node:6443|https://host.docker.internal:6443|g' /kubeconfig/k3s-host.yaml + + wait "$$k3s_pid" + ports: + - "6443:6443" + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - "k3s-data:/var/lib/rancher/k3s" + - "k3s-kubeconfig:/kubeconfig" + - "/var/run/docker.sock:/var/run/docker.sock" + healthcheck: + test: ["CMD-SHELL", "KUBECONFIG=/kubeconfig/k3s-source.yaml k3s kubectl get --raw=/readyz >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 20s volumes: cache: driver: local + k3s-data: + driver: local + k3s-kubeconfig: + driver: local networks: default: diff --git a/openshift/.gitkeep b/openshift/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/install-kubernetes-tools.sh b/scripts/install-kubernetes-tools.sh new file mode 100644 index 0000000..b54780a --- /dev/null +++ b/scripts/install-kubernetes-tools.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -eu + +KUBECTL_VERSION="${KUBECTL_VERSION:-v1.33.3}" +HELM_VERSION="${HELM_VERSION:-v3.18.4}" + +case "$(uname -m)" in + x86_64) + bin_arch="amd64" + ;; + aarch64|arm64) + bin_arch="arm64" + ;; + *) + echo "Unsupported architecture: $(uname -m)" >&2 + exit 1 + ;; +esac + +curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${bin_arch}/kubectl" +chmod +x /usr/local/bin/kubectl + +curl -fsSL -o /tmp/helm.tar.gz "https://get.helm.sh/helm-${HELM_VERSION}-linux-${bin_arch}.tar.gz" +tar -xzf /tmp/helm.tar.gz -C /tmp +mv "/tmp/linux-${bin_arch}/helm" /usr/local/bin/helm +chmod +x /usr/local/bin/helm +rm -rf /tmp/helm.tar.gz "/tmp/linux-${bin_arch}" \ No newline at end of file