console: Enable e2e tests, previews, deploys and fix READMEs

.github/CODEOWNERS

@@ -15,6 +15,7 @@
 /bin/lint-versions                  @MaterializeInc/testing
 /ci                                 @MaterializeInc/testing
 /ci/test/lint-deps.toml
+/console                            @MaterializeInc/console
 /doc/user                           @MaterializeInc/docs
 /doc/developer/reference/compute    @MaterializeInc/cluster
 /doc/developer/reference/storage    @MaterializeInc/cluster

bin/ci-builder

@@ -25,7 +25,7 @@ cd "$(dirname "$0")/.."
 
 if [[ $# -lt 2 ]]
 then
-    echo "usage: $0 <command> <stable|nightly|min> [<args>...]
+    echo "usage: $0 <command> <stable|nightly|min|console> [<args>...]
 
 Manages the ci-builder Docker image, which contains the dependencies required
 to build, test, and deploy the code in this repository.
@@ -59,6 +59,10 @@ case "$flavor" in
         rust_version=nightly
         rust_date=/$NIGHTLY_RUST_DATE
         ;;
+    console)
+        docker_target=ci-builder-console
+        rust_version=$(sed -n 's/rust-version = "\(.*\)"/\1/p' Cargo.toml)
+        ;;
     *)
         printf "unknown CI builder flavor %q\n" "$flavor"
         exit 1
@@ -310,6 +314,16 @@ case "$cmd" in
                 --env DOCKERHUB_ACCESS_TOKEN
                 # For configuring the metadata store
                 --env EXTERNAL_METADATA_STORE
+                # For console
+                --env CONSOLE_E2E_TEST_STAGING_PASSWORD
+                --env CONSOLE_E2E_TEST_PRODUCTION_PASSWORD
+                --env E2E_TEST_STAGING_FRONTEGG_CLIENT_ID
+                --env E2E_TEST_STAGING_ORB_API_KEY
+                --env E2E_TEST_LICENSE_KEY
+                --env CLOUD_DEPLOY_KEY
+                --env VERCEL_TOKEN
+                --env VERCEL_ORG_ID
+                --env VERCEL_PROJECT_ID
             )
             for env in $(printenv | grep -E '^(BUILDKITE|MZCOMPOSE|CI)' | sed 's/=.*//'); do
                 args+=(--env "$env")

ci/builder/Dockerfile

@@ -44,7 +44,6 @@ RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-ge
     gdb \
     git \
     gnupg2 \
-    libxml2-16 \
     pigz \
     python3 \
     python3.13-venv \
@@ -146,6 +145,7 @@ RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-ge
     libclang-dev \
     libclang-rt-18-dev \
     libpq-dev \
+    libxml2-16 \
     lld \
     llvm \
     make \
@@ -172,7 +172,7 @@ COPY nodesource.asc .
 
 RUN gpg --dearmor < nodesource.asc > /etc/apt/keyrings/nodesource.gpg \
     && rm nodesource.asc \
-    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_25.x nodistro main" > /etc/apt/sources.list.d/nodesource.list \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_22.x nodistro main" > /etc/apt/sources.list.d/nodesource.list \
     && apt-get update \
     && apt-get install -y --no-install-recommends nodejs \
     && apt-get clean \
@@ -399,3 +399,158 @@ ENV HELM_PLUGINS=/usr/local/share/helm/plugins
 ENV CARGO_HOME=/cargo
 RUN mkdir /cargo && chmod 777 /cargo
 VOLUME /cargo
+
+# Stage 3: Build a lightweight CI Builder image for console/playwright jobs.
+FROM ubuntu:noble-20250529 AS ci-builder-console
+
+ARG ARCH_GCC
+ARG ARCH_GO
+
+WORKDIR /workdir
+
+ENV RUST_BACKTRACE=1
+ENV PYTHONUNBUFFERED=1
+ENV MZ_DEV_CI_BUILDER=1
+ENV DOCKER_BUILDKIT=1
+
+ARG XZ_OPT=-T0
+
+RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    dnsmasq \
+    docker.io \
+    eatmydata \
+    gettext-base \
+    git \
+    gnupg2 \
+    jq \
+    libatomic1 \
+    openssh-client \
+    perl \
+    python3 \
+    python3-venv \
+    python-is-python3 \
+    sudo \
+    libasound2t64 \
+    libatk1.0-0t64 \
+    libatk-bridge2.0-0t64 \
+    libatspi2.0-0t64 \
+    libcairo2 \
+    libcups2t64 \
+    libdbus-1-3 \
+    libdrm2 \
+    libgbm1 \
+    libnspr4 \
+    libnss3 \
+    libpango-1.0-0 \
+    libx11-6 \
+    libxcb1 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxkbcommon0 \
+    libxrandr2 \
+    gstreamer1.0-libav \
+    libcairo-gobject2 \
+    libenchant-2-2 \
+    libepoxy0 \
+    libevent-2.1-7t64 \
+    libflite1 \
+    libgdk-pixbuf-2.0-0 \
+    libgles2 \
+    libgstreamer1.0-0 \
+    libgstreamer-gl1.0-0 \
+    libgstreamer-plugins-bad1.0-0 \
+    libgstreamer-plugins-base1.0-0 \
+    libgtk-3-0t64 \
+    libharfbuzz-icu0 \
+    libhyphen0 \
+    libicu74 \
+    libjpeg-turbo8 \
+    liblcms2-2 \
+    libmanette-0.2-0 \
+    libopus0 \
+    libpangocairo-1.0-0 \
+    libsecret-1-0 \
+    libvpx9 \
+    libwebp7 \
+    libwebpdemux2 \
+    libwoff1 \
+    libxml2 \
+    libxslt1.1 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/* /usr/share/locale/* /var/cache/* /var/log/* \
+    && sed -i 's/^UID_MIN.*/UID_MIN\t500/' /etc/login.defs
+
+RUN curl -fsSL https://github.com/MaterializeInc/autouseradd/releases/download/1.3.0/autouseradd-1.3.0-$ARCH_GO.tar.gz \
+    | tar xz -C / --strip-components 1
+
+RUN mkdir -p /usr/local/lib/docker/cli-plugins \
+    && curl -fsSL https://github.com/docker/compose/releases/download/v5.0.1/docker-compose-linux-$ARCH_GCC > /usr/local/lib/docker/cli-plugins/docker-compose \
+    && chmod +x /usr/local/lib/docker/cli-plugins/docker-compose \
+    && curl -fsSL https://github.com/docker/buildx/releases/download/v0.30.1/buildx-v0.30.1.linux-$ARCH_GO > /usr/local/lib/docker/cli-plugins/docker-buildx \
+    && chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx
+
+COPY nodesource.asc .
+
+RUN gpg --dearmor < nodesource.asc > /etc/apt/keyrings/nodesource.gpg \
+    && rm nodesource.asc \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_22.x nodistro main" > /etc/apt/sources.list.d/nodesource.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends nodejs \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /usr/share/doc/nodejs \
+    && npm install -g corepack \
+    && corepack enable
+
+# TODO(def-) Upgrading kind/kubectl seems to cause Cloudtest failures
+RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \
+    && chmod +x /usr/local/bin/kind \
+    && if [ $ARCH_GO = amd64 ]; then echo 'c72eda46430f065fb45c5f70e7c957cc9209402ef309294821978677c8fb3284 /usr/local/bin/kind' | sha256sum --check; fi \
+    && if [ $ARCH_GO = arm64 ]; then echo '03d45095dbd9cc1689f179a3e5e5da24b77c2d1b257d7645abf1b4174bebcf2a /usr/local/bin/kind' | sha256sum --check; fi
+
+RUN curl -fsSL https://dl.k8s.io/release/v1.24.3/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \
+    && chmod +x /usr/local/bin/kubectl \
+    && if [ $ARCH_GO = amd64 ]; then echo '8a45348bdaf81d46caf1706c8bf95b3f431150554f47d444ffde89e8cdd712c1 /usr/local/bin/kubectl' | sha256sum --check; fi \
+    && if [ $ARCH_GO = arm64 ]; then echo 'bdad4d3063ddb7bfa5ecf17fb8b029d5d81d7d4ea1650e4369aafa13ed97149a /usr/local/bin/kubectl' | sha256sum --check; fi
+
+# Use Helm 3 (not 4) because the cloud repo's bin/kind-create uses relative
+# file paths for vendored charts, which Helm 4 rejects as invalid URLs.
+RUN curl -fsSL https://get.helm.sh/helm-v3.16.2-linux-$ARCH_GO.tar.gz > helm.tar.gz \
+    && if [ $ARCH_GO = amd64 ]; then echo '9318379b847e333460d33d291d4c088156299a26cd93d570a7f5d0c36e50b5bb helm.tar.gz' | sha256sum --check; fi \
+    && if [ $ARCH_GO = arm64 ]; then echo '1888301aeb7d08a03b6d9f4d2b73dcd09b89c41577e80e3455c113629fc657a4 helm.tar.gz' | sha256sum --check; fi \
+    && tar -xf helm.tar.gz -C /usr/local/bin --strip-components=1 linux-$ARCH_GO/helm \
+    && rm helm.tar.gz
+
+# Install just cargo (no rustc) — cloud scripts use it to read crate metadata.
+COPY rust.asc .
+
+ARG RUST_DATE
+ARG RUST_VERSION
+
+RUN gpg --import rust.asc \
+    && rm rust.asc \
+    && echo "trusted-key 85AB96E6FA1BE5FE" >> ~/.gnupg/gpg.conf \
+    && mkdir rust \
+    && curl -fsSL https://static.rust-lang.org/dist$RUST_DATE/rust-$RUST_VERSION-$ARCH_GCC-unknown-linux-gnu.tar.gz > rust.tar.gz \
+    && curl -fsSL https://static.rust-lang.org/dist$RUST_DATE/rust-$RUST_VERSION-$ARCH_GCC-unknown-linux-gnu.tar.gz.asc > rust.asc \
+    && gpg --verify rust.asc rust.tar.gz \
+    && tar -xzf rust.tar.gz -C rust --strip-components=1 \
+    && rust/install.sh --components=cargo \
+    && rm -rf rust.asc rust.tar.gz rust
+
+COPY ssh_known_hosts /etc/ssh/ssh_known_hosts
+
+# Remove Ubuntu user causing UID collisions.
+# https://bugs.launchpad.net/cloud-images/+bug/2005129
+RUN userdel -r ubuntu
+
+# Allow passwordless sudo for the CI user (created at runtime by autouseradd).
+RUN echo "ALL ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/ci \
+    && chmod 0440 /etc/sudoers.d/ci
+
+ENTRYPOINT ["autouseradd", "--user", "materialize"]

ci/deploy/console.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+set -euo pipefail
+
+cd console
+export COREPACK_ENABLE_DOWNLOAD_PROMPT=0
+export VERCEL_ENVIRONMENT=production
+export SENTRY_RELEASE="$BUILDKITE_COMMIT"
+corepack enable
+yarn install --immutable --network-timeout 30000
+
+npx vercel@latest pull --yes --environment="$VERCEL_ENVIRONMENT" --token="$VERCEL_TOKEN"
+npx vercel@latest build --token="$VERCEL_TOKEN" --prod
+
+npx vercel@latest deploy --prebuilt --prod --token="$VERCEL_TOKEN"

ci/deploy/pipeline.template.yml

@@ -47,3 +47,13 @@ steps:
       queue: linux-x86_64-small
     concurrency: 1
     concurrency_group: deploy/npm
+
+  - id: deploy-console
+    label: ":rocket: :vercel: Console"
+    command: bin/ci-builder run stable ci/deploy/console.sh
+    branches: main
+    timeout_in_minutes: 15
+    agents:
+      queue: linux-x86_64-small
+    concurrency: 1
+    concurrency_group: deploy/console

ci/mkpipeline.py

@@ -175,6 +175,7 @@ def fetch_hashes() -> None:
                 args.sanitizer,
                 lto,
             )
+            trim_ci_glue_exempt_steps(pipeline)
         else:
             print("Trimming unchanged steps from pipeline")
             trim_tests_pipeline(
@@ -501,8 +502,8 @@ def switch_jobs_to_aws(pipeline: Any, priority: int) -> None:
                 step["agents"]["queue"] = "linux-aarch64-medium"
 
         elif agent == "hetzner-aarch64-16cpu-32gb":
-            if "hetzner-x86-64-16cpu-32gb" not in stuck:
-                step["agents"]["queue"] = "hetzner-x86-64-16cpu-32gb"
+            if "hetzner-x86-64-12cpu-24gb" not in stuck:
+                step["agents"]["queue"] = "hetzner-x86-64-12cpu-24gb"
                 if step.get("depends_on") == "build-aarch64":
                     step["depends_on"] = "build-x86_64"
             else:
@@ -1036,6 +1037,15 @@ def have_paths_changed(globs: Iterable[str]) -> bool:
             raise RuntimeError("unreachable")
 
 
+def trim_ci_glue_exempt_steps(pipeline: Any) -> None:
+    for step in steps(pipeline):
+        if not step.get("ci_glue_exempt"):
+            continue
+        inputs = step.get("inputs", [])
+        if inputs and not have_paths_changed(inputs):
+            step["skip"] = "No changes in inputs"
+
+
 def remove_mz_specific_keys(pipeline: Any) -> None:
     """Remove the Materialize-specific keys from the configuration that are only used to inform how to trim the pipeline and for coverage runs."""
     for step in steps(pipeline):
@@ -1045,6 +1055,8 @@ def remove_mz_specific_keys(pipeline: Any) -> None:
             del step["coverage"]
         if "sanitizer" in step:
             del step["sanitizer"]
+        if "ci_glue_exempt" in step:
+            del step["ci_glue_exempt"]
         if (
             "timeout_in_minutes" not in step
             and "prompt" not in step

ci/mkpipeline.sh

@@ -27,7 +27,7 @@ bootstrap_steps=
 tmpfile=$(mktemp)
 
 for arch in x86_64 aarch64; do
-  for flavor in stable nightly min; do
+  for flavor in stable nightly min console; do
     (
       if ! MZ_DEV_CI_BUILDER_ARCH=$arch bin/ci-builder exists $flavor; then
         echo "$arch:$flavor" >> "$tmpfile"