diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 2c468fa341..b5f5efc270 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -86,56 +86,11 @@ jobs: if: ${{ env.BACKEND == 'rocksdb' }} run: | bash hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh - check_compose() { - local file="$1" - local server_policy="$2" - local hubble_policy="$3" - local rendered - rendered="$(mktemp)" - - if env -u HUGEGRAPH_ADMIN_PASSWORD \ - docker compose -f "$file" config -q >/dev/null 2>&1; then - echo "$file accepted an unset admin password" >&2 - return 1 - fi - if HUGEGRAPH_ADMIN_PASSWORD= \ - docker compose -f "$file" config -q >/dev/null 2>&1; then - echo "$file accepted an empty admin password" >&2 - return 1 - fi + bash docker/test-compose.sh render - HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ - docker compose -f "$file" config --format json > "$rendered" - jq -e \ - --arg server_policy "$server_policy" \ - --arg hubble_policy "$hubble_policy" ' - .services.server.pull_policy == $server_policy and - .services.hubble.pull_policy == $hubble_policy and - .services.server.environment.HG_SERVER_USE_PD == "true" and - .services.server.environment.HG_SERVER_CLUSTER == "hg" and - .services.server.environment.HG_SERVER_REST_URL == - "http://server:8080" and - .services.server.environment.HG_SERVER_INIT_STORE_ENABLED == - "false" and - .services.server.environment.PASSWORD == - "ci-test-password" and - (.services.server.environment.HG_SERVER_AUTH_TOKEN_SECRET != - null) and - .services.hubble.depends_on.server.condition == - "service_healthy" and - (.services.hubble.healthcheck.test[1] | - contains("http://127.0.0.1:8088/about") and - contains("\"status\":200") and - contains("\"name\":\"hugegraph-hubble\"")) and - any(.services.hubble.ports[]; - .target == 8088 and .published == "8088" and - .host_ip == "127.0.0.1") - ' "$rendered" >/dev/null - rm -f "$rendered" - } - - check_compose docker/docker-compose.yml always always - check_compose docker/docker-compose.dev.yml build missing + - name: Run Compose auth-on smoke tests + if: ${{ env.BACKEND == 'rocksdb' }} + run: bash docker/test-compose.sh smoke - name: Run check_port unit tests if: ${{ env.BACKEND == 'rocksdb' }} diff --git a/docker/README.md b/docker/README.md index f0992da823..99d92648fa 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,436 +1,361 @@ -# HugeGraph Docker Deployment +# HugeGraph Docker Compose -This directory contains Docker Compose files for running HugeGraph: +## Users -| File | Description | -|------|-------------| -| `docker-compose.yml` | PD, Store, Server, and Hubble using pre-built images | -| `docker-compose.dev.yml` | PD, Store, and Server built from source, plus Hubble | -| `docker-compose-3pd-3store-3server.yml` | 3-node distributed cluster (PD + Store + Server) | +### Choose a topology -## Prerequisites +Run commands in this directory: -- **Docker Engine** 20.10+ (or Docker Desktop 4.x+) -- **Docker Compose** v2 (included in Docker Desktop) -- **OpenSSL CLI** (used to generate the initial administrator password) -- **Memory**: Allocate at least **12 GB** to Docker Desktop (Settings → Resources → Memory). The 3-node cluster runs 9 JVM processes (3 PD + 3 Store + 3 Server) which are memory-intensive. Insufficient memory causes OOM kills that appear as silent Raft failures. +```bash +cd docker +``` -> [!IMPORTANT] -> The 12 GB minimum is for Docker Desktop. On Linux with native Docker, ensure the host has at least 12 GB of free memory. ---- +| Topology | Compose file | Services | When to use it | +| --- | --- | --- | --- | +| Standalone | `docker-compose.yml` | 1 RocksDB Server + 1 Hubble | Default; start here | +| Minimal HStore | `docker-compose-hstore.yml` | 1 PD + 1 Store + 1 Server + 1 Hubble | Distributed local development | +| HA | `docker-compose-3pd-3store-3server.yml` | 3 PD + 3 Store + 3 Server + 1 Hubble | Reference and evaluation | -## Single-Node Setup +Standalone uses `hugegraph/hugegraph:${HUGEGRAPH_VERSION:-latest}`. The HStore +topologies use the matching `hugegraph/pd`, `hugegraph/store`, and +`hugegraph/server` tags. Hubble is selected independently with +`${HUBBLE_IMAGE:-hugegraph/hubble:latest}`. -Two compose files run one PD, one Store, one Server, and one Hubble instance: +### Create the authentication environment -Create a Compose environment file once so every lifecycle command can resolve the required administrator password: +Create `.env` once. Replace `replace-with-your-password` with an administrator +password that you choose; the command generates and persists a random 32-byte +JWT secret. For this simple single-quoted format, do not use a password that +contains a single quote or newline. ```bash ( set -eu - cd docker - if [ -e .env ]; then - echo "docker/.env already exists; reusing it" - else - command -v openssl >/dev/null 2>&1 - admin_password="$(openssl rand -base64 12)" - if [ "${#admin_password}" -ne 16 ]; then - echo "Failed to generate a 16-character password" >&2 - exit 1 - fi - install -m 600 /dev/null .env - { - printf "HUGEGRAPH_ADMIN_PASSWORD='%s'\n" "${admin_password}" - } >> .env - unset admin_password - fi - chmod 600 .env - if ! env -u HUGEGRAPH_ADMIN_PASSWORD \ - docker compose -f docker-compose.yml config --quiet || - ! env -u HUGEGRAPH_ADMIN_PASSWORD \ - docker compose -f docker-compose.dev.yml config --quiet; then - echo "docker/.env is incomplete; repair or move it, then retry" >&2 + command -v openssl >/dev/null + jwt_secret="$(openssl rand -hex 32)" + test "${#jwt_secret}" -eq 64 + umask 077 + test ! -e .env || { + echo ".env already exists; edit it instead of overwriting it" >&2 exit 1 - fi + } + printf "HUGEGRAPH_ADMIN_PASSWORD='%s'\nHUGEGRAPH_AUTH_TOKEN_SECRET='%s'\n" \ + 'replace-with-your-password' "${jwt_secret}" > .env ) ``` -Compose automatically reads `docker/.env` for `up`, `ps`, `stop`, and `down`. The generated password is a 16-character, Compose-safe random value. The file is excluded from Git and Docker build contexts; keep its permissions restricted and source production credentials from your secret manager instead of committing them. +Do not commit `.env`. Keeping the same JWT secret preserves authentication +tokens when containers are recreated. For authenticated topologies with +multiple Server replicas, all replicas receive this same secret. -### Option A: Quick Start (pre-built images) +A non-empty `HUGEGRAPH_ADMIN_PASSWORD` enables Server authentication, and +Hubble detects that mode automatically. Omitting the variable or setting it to +an empty value disables authentication. Auth-off is only suitable for a +trusted local environment; never expose it to a public or untrusted network. +Hubble listens on host loopback by default. Set `HUBBLE_PUBLISH_HOST` only +behind an HTTPS reverse proxy and trusted network controls. -Uses pre-built images from Docker Hub. Best for **end users** who want to run HugeGraph quickly. Set `HUGEGRAPH_VERSION` to the same published release for PD, Store, Server, and Hubble. The authenticated PD/Hubble integration is not present in `1.7.x`; if no later compatible release is available, use Option B. +`HUGEGRAPH_ADMIN_PASSWORD` initializes the built-in `admin` account on its +first authenticated startup. Changing `.env` does not rotate an existing +administrator password; use the HugeGraph user API for credential changes. + +For the verification commands below, set the password in your current shell: ```bash -( - cd docker - HUGEGRAPH_VERSION='' \ - docker compose up -d -) +ADMIN_PASSWORD='the-same-password-used-in-.env' ``` -- Images: matching `hugegraph/pd`, `hugegraph/store`, `hugegraph/server`, and `hugegraph/hubble` tags from the selected compatible release -- `pull_policy: always` — always pulls the specified image tag - -> **Note**: Do not use `latest` to claim a reproducible deployment. Pin a compatible release tag and keep it unchanged for later lifecycle commands. -- PD healthcheck endpoint: `/v1/health` -- Hubble is available at `http://localhost:8088`; sign in as `admin` with the required `HUGEGRAPH_ADMIN_PASSWORD` -- Hubble binds to host loopback by default. Set `HUBBLE_PUBLISH_HOST` explicitly only behind an HTTPS reverse proxy and trusted network controls. -- Hubble uses PD discovery and the Docker-network Server address -- Server healthcheck endpoint: `/versions` +### Standalone -### Option B: Development Build (build from source) +This is the recommended quickstart. -Builds images locally from source Dockerfiles. Best for **developers** who want to test local changes. Build the matching `hugegraph-toolchain` Hubble source as `local/hugegraph-hubble:dev` before starting this stack. +Start: -The publishing pipeline uses [`docker/bake.hcl`](./bake.hcl) from the repository root to compile the Java reactor once and build the PD, Store, HStore Server, and standalone Server runtime images from that shared result. +```bash +docker compose -f docker-compose.yml up -d --wait +``` -Run Bake commands from the repository root. Use `--print` to inspect the resolved targets without building, or run the default group to build all four amd64/arm64 images. Loading both platforms under the same local tags requires Docker's containerd image store. +Status: ```bash -# Inspect the resolved build graph -docker buildx bake --file docker/bake.hcl --print +docker compose -f docker-compose.yml ps +``` + +Verify Server readiness, authentication, and Hubble: -# Build the default multi-platform target group -IMAGE_TAG=local docker buildx bake --file docker/bake.hcl +```bash +curl -fsS http://localhost:8080/versions +test "$(curl -sS -o /dev/null -w '%{http_code}' \ + http://localhost:8080/graphspaces/DEFAULT/graphs)" = 401 +test "$(curl -sS -u "admin:${ADMIN_PASSWORD}" -o /dev/null -w '%{http_code}' \ + http://localhost:8080/graphspaces/DEFAULT/graphs)" = 200 +curl -fsS http://localhost:8088/about ``` -Local source changes are included because Bake uses the current repository working tree as its build context. For routine development, override all targets to the host architecture so the four images still share one Maven build without requiring the multi-platform containerd image store. +Open `http://localhost:8088` and sign in as `admin` with the password from +`.env`. + +Stop containers while keeping them: ```bash -# x86_64 host -IMAGE_TAG=local docker buildx bake --file docker/bake.hcl --set '*.platform=linux/amd64' +docker compose -f docker-compose.yml stop +``` -# ARM64 host -IMAGE_TAG=local docker buildx bake --file docker/bake.hcl --set '*.platform=linux/arm64' +Remove containers and the network while keeping data: + +```bash +docker compose -f docker-compose.yml down ``` -These local commands can read existing Registry caches but do not publish images or write remote caches because `EXPORT_CACHE` defaults to `false`. +Delete containers, the network, and all topology data: ```bash -( - cd docker - HUBBLE_IMAGE=local/hugegraph-hubble:dev \ - HUBBLE_PULL_POLICY=never \ - docker compose -f docker-compose.dev.yml up -d -) +docker compose -f docker-compose.yml down -v ``` -- PD, Store, and Server images are built from this repository -- Hubble uses `HUBBLE_IMAGE` because its source is in `hugegraph-toolchain` -- Server entrypoint scripts are baked into the built image; Hubble mounts the Docker-local PD configuration -- PD healthcheck endpoint: `/v1/health` -- Otherwise identical env vars and structure to the quickstart file +### Minimal HStore -Use the same release tag for Option A lifecycle commands: +Start: ```bash -( - cd docker - export HUGEGRAPH_VERSION='' - docker compose ps - docker compose stop - docker compose down -) +docker compose -f docker-compose-hstore.yml up -d --wait ``` -Use the development Compose file for every Option B lifecycle command: +Status: ```bash -( - cd docker - docker compose -f docker-compose.dev.yml ps - docker compose -f docker-compose.dev.yml stop - docker compose -f docker-compose.dev.yml down -) +docker compose -f docker-compose-hstore.yml ps ``` -### Key Differences - -| | `docker-compose.yml` (quickstart) | `docker-compose.dev.yml` (dev build) | -|---|---|---| -| **Images** | Pull from Docker Hub | Build from source | -| **Who it's for** | End users | Developers | -| **Server pull_policy** | `always` | `build` | -| **Hubble pull_policy** | `always` | `never` in the workflow above (`missing` in the Compose file by default) | +Verify PD, Store, Server authentication, and Hubble: -**Verify** (both options): ```bash -curl http://localhost:8080/versions +curl -fsS http://localhost:8620/v1/health +curl -fsS http://localhost:8520/v1/health +curl -fsS http://localhost:8080/versions +test "$(curl -sS -o /dev/null -w '%{http_code}' \ + http://localhost:8080/graphspaces/DEFAULT/graphs)" = 401 +test "$(curl -sS -u "admin:${ADMIN_PASSWORD}" -o /dev/null -w '%{http_code}' \ + http://localhost:8080/graphspaces/DEFAULT/graphs)" = 200 curl -fsS http://localhost:8088/about ``` -To validate local images without Compose replacing them with remote `latest`: +Open `http://localhost:8088` and sign in as `admin` with the password from +`.env`. + +Stop containers while keeping them: ```bash -( - cd docker - HUGEGRAPH_SERVER_IMAGE=local/hugegraph-server:test \ - HUGEGRAPH_SERVER_PULL_POLICY=never \ - HUBBLE_IMAGE=local/hugegraph-hubble:test \ - HUBBLE_PULL_POLICY=never \ - docker compose up -d --wait -) +docker compose -f docker-compose-hstore.yml stop ``` ---- +Remove containers and the network while keeping data: + +```bash +docker compose -f docker-compose-hstore.yml down +``` -## 3-Node Cluster Quickstart +Delete containers, the network, and all topology data: ```bash -cd docker -HUGEGRAPH_VERSION=1.7.0 docker compose -f docker-compose-3pd-3store-3server.yml up -d +docker compose -f docker-compose-hstore.yml down -v +``` -# To stop and remove all data volumes (clean restart) -docker compose -f docker-compose-3pd-3store-3server.yml down -v +### HA reference + +The HA topology is resource-intensive. Running it locally is not required on +resource-constrained machines, but its Compose configuration must always render +successfully. This PR validates HA by rendering and static review only; it does +not start HA locally or in default CI. + +Start: + +```bash +docker compose -f docker-compose-3pd-3store-3server.yml up -d --wait ``` -**Startup ordering** is enforced via `depends_on` with `condition: service_healthy`: +Status: -1. **PD nodes** start first and must pass healthchecks (`/v1/health`) -2. **Store nodes** start after all PD nodes are healthy -3. **Server nodes** start after all Store nodes are healthy +```bash +docker compose -f docker-compose-3pd-3store-3server.yml ps +``` -This ensures PD and Store are healthy before the server starts. The server entrypoint still performs a best-effort partition wait after launch, so partition assignment may take a little longer. +Verify all published PD, Store, and Server endpoints, Server authentication, +and Hubble: + +```bash +for port in 8620 8621 8622; do + curl -fsS "http://localhost:${port}/v1/health" +done +for port in 8520 8521 8522; do + curl -fsS "http://localhost:${port}/v1/health" +done +for port in 8080 8081 8082; do + curl -fsS "http://localhost:${port}/versions" + test "$(curl -sS -o /dev/null -w '%{http_code}' \ + "http://localhost:${port}/graphspaces/DEFAULT/graphs")" = 401 + test "$(curl -sS -u "admin:${ADMIN_PASSWORD}" -o /dev/null \ + -w '%{http_code}' \ + "http://localhost:${port}/graphspaces/DEFAULT/graphs")" = 200 +done +curl -fsS http://localhost:8088/about +``` -**Verify the cluster is healthy**: +Open `http://localhost:8088` and sign in as `admin` with the password from +`.env`. + +Stop containers while keeping them: ```bash -# Check PD health -curl http://localhost:8620/v1/health +docker compose -f docker-compose-3pd-3store-3server.yml stop +``` -# Check Store health -curl http://localhost:8520/v1/health +Remove containers and the network while keeping data: -# Check Server (Graph API) -curl http://localhost:8080/versions +```bash +docker compose -f docker-compose-3pd-3store-3server.yml down +``` -# List registered stores via PD -curl http://localhost:8620/v1/stores +Delete containers, the network, and all topology data: -# List partitions -curl http://localhost:8620/v1/partitions +```bash +docker compose -f docker-compose-3pd-3store-3server.yml down -v ``` ---- - -## Environment Variable Reference - -Configuration is injected via environment variables. The old `docker/configs/application-pd*.yml` and `docker/configs/application-store*.yml` files are no longer used. - -### PD Environment Variables - -| Variable | Required | Default | Maps To (`application.yml`) | Description | -|----------|----------|---------|-----------------------------|-------------| -| `HG_PD_GRPC_HOST` | Yes | — | `grpc.host` | This node's hostname/IP for gRPC | -| `HG_PD_RAFT_ADDRESS` | Yes | — | `raft.address` | This node's Raft address (e.g. `pd0:8610`) | -| `HG_PD_RAFT_PEERS_LIST` | Yes | — | `raft.peers-list` | All PD peers (e.g. `pd0:8610,pd1:8610,pd2:8610`) | -| `HG_PD_INITIAL_STORE_LIST` | Yes | — | `pd.initial-store-list` | Expected stores (e.g. `store0:8500,store1:8500,store2:8500`) | -| `HG_PD_GRPC_PORT` | No | `8686` | `grpc.port` | gRPC server port | -| `HG_PD_REST_PORT` | No | `8620` | `server.port` | REST API port | -| `HG_PD_DATA_PATH` | No | `/hugegraph-pd/pd_data` | `pd.data-path` | Metadata storage path | -| `HG_PD_INITIAL_STORE_COUNT` | No | `1` | `pd.initial-store-count` | Min stores for cluster availability | - -**Deprecated aliases** (still work but log a warning): - -| Deprecated | Use Instead | -|------------|-------------| -| `GRPC_HOST` | `HG_PD_GRPC_HOST` | -| `RAFT_ADDRESS` | `HG_PD_RAFT_ADDRESS` | -| `RAFT_PEERS` | `HG_PD_RAFT_PEERS_LIST` | -| `PD_INITIAL_STORE_LIST` | `HG_PD_INITIAL_STORE_LIST` | - -### Store Environment Variables - -| Variable | Required | Default | Maps To (`application.yml`) | Description | -|----------|----------|---------|-----------------------------|-------------| -| `HG_STORE_PD_ADDRESS` | Yes | — | `pdserver.address` | PD gRPC addresses (e.g. `pd0:8686,pd1:8686,pd2:8686`) | -| `HG_STORE_GRPC_HOST` | Yes | — | `grpc.host` | This node's hostname (e.g. `store0`) | -| `HG_STORE_RAFT_ADDRESS` | Yes | — | `raft.address` | This node's Raft address (e.g. `store0:8510`) | -| `HG_STORE_GRPC_PORT` | No | `8500` | `grpc.port` | gRPC server port | -| `HG_STORE_REST_PORT` | No | `8520` | `server.port` | REST API port | -| `HG_STORE_DATA_PATH` | No | `/hugegraph-store/storage` | `app.data-path` | Data storage path | - -**Deprecated aliases** (still work but log a warning): - -| Deprecated | Use Instead | -|------------|-------------| -| `PD_ADDRESS` | `HG_STORE_PD_ADDRESS` | -| `GRPC_HOST` | `HG_STORE_GRPC_HOST` | -| `RAFT_ADDRESS` | `HG_STORE_RAFT_ADDRESS` | - -### Server Environment Variables - -| Variable | Required | Default | Maps To | Description | -|----------|----------|---------|-----------------------------|-------------| -| `HG_SERVER_BACKEND` | Yes | — | `backend` in `hugegraph.properties` | Storage backend (e.g. `hstore`) | -| `HG_SERVER_PD_PEERS` | Yes | — | `pd.peers` | PD cluster addresses (e.g. `pd0:8686,pd1:8686,pd2:8686`) | -| `HG_SERVER_CLUSTER` | No | — | `cluster` in `rest-server.properties` | PD discovery application name; single-node Compose uses `hg` to match Hubble | -| `HG_SERVER_USE_PD` | No | — | `usePD` in `rest-server.properties` | Enables Server PD registration and discovery | -| `HG_SERVER_REST_URL` | No | — | `restserver.url` | Address registered with PD and used by clients | -| `HG_SERVER_MIN_FREE_MEMORY` | No | — | `restserver.min_free_memory` | Minimum free-memory guard in MB; local Compose uses `0` | -| `HG_SERVER_AUTH_TOKEN_SECRET` | No | generated in auth mode | `auth.token_secret` | Shared JWT secret for REST and embedded Gremlin authentication; explicit values must be at least 32 bytes | -| `STORE_REST` | No | — | Used by `wait-partition.sh` | Store REST endpoint for partition verification (e.g. `store0:8520`) | -| `PASSWORD` | No | — | Enables auth and sets `auth.admin_pa` | Initial administrator password; disabled init-store does not read it from stdin, but the entrypoint still applies it to the PD bootstrap path | -| `HG_SERVER_INIT_STORE_ENABLED` | No | `true` | `init_store.enabled` in `rest-server.properties` | Set `false` in PD/HStore deployments so init-store skips local backend and admin initialization | - -> **The built-in authenticator with `HG_SERVER_INIT_STORE_ENABLED=false` requires `usePD=true` and an HStore-backed `auth.graph_store`, unless `auth.remote_url` delegates auth elsewhere.** With init-store skipped, the server creates the built-in admin in PD metadata, and only an HStore auth graph uses the PD-backed auth manager that can read that account. init-store exits non-zero when the combination is unusable, rather than leaving a server nobody can log in to. A custom `auth.authenticator` is exempt because it manages its own identities. -> -> `docker/init_complete` is written by init-store itself, and only after it has initialized. A skipped run therefore records nothing, whether it was disabled by the variable or by the property in a mounted `rest-server.properties`, so a later re-enable is still able to initialize. The marker only short-circuits re-initialization: init-store runs on every container start, and a disabled one performs the fail-closed check above first, so a marker left by an earlier release or an earlier enabled run cannot bypass it. -> -> The entrypoint maps **`PASSWORD` to `auth.admin_pa`** before init-store runs. A disabled init-store does not read the password from standard input, but the PD startup path uses the explicit `auth.admin_pa` value when it first creates the administrator. Changing it later does not rotate an existing password. - -The single-node Compose files also accept these deployment-level overrides: - -| Variable | Default | Description | -|----------|---------|-------------| -| `HUGEGRAPH_SERVER_IMAGE` | `hugegraph/server:` | Complete Server image reference | -| `HUGEGRAPH_SERVER_PULL_POLICY` | `always` (`build` for dev) | Server pull policy | -| `HUBBLE_IMAGE` | `hugegraph/hubble:` | Complete Hubble image reference | -| `HUBBLE_PULL_POLICY` | `always` (`missing` for dev) | Hubble pull policy | -| `HUBBLE_PUBLISH_HOST` | `127.0.0.1` | Hubble host bind address; remote access requires an HTTPS reverse proxy | -| `HUGEGRAPH_ADMIN_PASSWORD` | required (`docker/.env`) | Initial admin password; no public default is provided | -| `HUGEGRAPH_AUTH_TOKEN_SECRET` | generated | JWT signing secret; explicit values must be at least 32 bytes | - -When authentication is enabled and no token secret is supplied, the Server entrypoint generates a random secret and writes it to both authentication configurations. The value is reused on container restart while the container filesystem is preserved. To preserve tokens across container recreation, generate a compatible secret once and add it to the mode-600 `docker/.env`: +### Select image versions + +Set a HugeGraph release for Server, PD, and Store without changing Hubble: ```bash -( - set -euo pipefail - cd docker - secret_pattern='^[[:space:]]*(export[[:space:]]+)?HUGEGRAPH_AUTH_TOKEN_SECRET[[:space:]]*=' - secret_count="$(grep -Ec "${secret_pattern}" .env || true)" - case "${secret_count}" in - 0) - command -v openssl >/dev/null 2>&1 - token_secret="$(openssl rand -hex 32)" - LC_ALL=C - if (( ${#token_secret} != 64 )); then - echo "Failed to generate a 64-character token secret" >&2 - exit 1 - fi - printf "HUGEGRAPH_AUTH_TOKEN_SECRET='%s'\n" \ - "${token_secret}" >> .env - unset token_secret - echo "Generated HUGEGRAPH_AUTH_TOKEN_SECRET" - ;; - 1) - token_secret="$( - sed -nE \ - "s/${secret_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env - )" - LC_ALL=C - if (( ${#token_secret} < 32 )); then - echo "Existing token secret must use the documented single-quoted" \ - "format and contain at least 32 bytes; .env was not changed" >&2 - exit 1 - fi - unset token_secret - echo "HUGEGRAPH_AUTH_TOKEN_SECRET already exists; reusing it" - ;; - *) - echo "Duplicate HUGEGRAPH_AUTH_TOKEN_SECRET entries; repair .env" >&2 - exit 1 - ;; - esac - chmod 600 .env -) +HUGEGRAPH_VERSION=1.7.0 \ +docker compose -f docker-compose-hstore.yml up -d +``` + +Select Hubble independently: + +```bash +HUBBLE_IMAGE=hugegraph/hubble:latest \ +docker compose -f docker-compose.yml up -d ``` -The entrypoint rejects shorter explicit values before changing either Server configuration file. +The Hubble `latest` image is expected to work with HugeGraph Server 1.7 and +Server `latest`; compatibility with versions older than 1.7 is not promised. +Pin immutable image references when reproducibility is required. -**Deprecated aliases** (still work but log a warning): +### Data persistence -| Deprecated | Use Instead | -|------------|-------------| -| `BACKEND` | `HG_SERVER_BACKEND` | -| `PD_PEERS` | `HG_SERVER_PD_PEERS` | +Each topology creates its own normal Compose network and named volumes. No +network or volume needs to be created in advance. ---- +Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore +topologies keep PD and Store data in topology-local volumes. Hubble uses +`jdbc:h2:file:/hubble/data/hubble;DB_CLOSE_ON_EXIT=FALSE` and stores uploaded +files under `/hubble/data/upload-files`. -## Port Reference +`docker compose down` keeps named-volume data. `docker compose down -v` +intentionally deletes it. -The table below reflects the published host ports in `docker-compose-3pd-3store-3server.yml`. The single-node Compose file publishes `8620`, `8520`, `8080`, and Hubble `8088`; Hubble defaults to host loopback. +## Developers -| Service | Container Port | Host Port | Protocol | Purpose | -|---------|---------------|-----------|----------|---------| -| pd0 | 8620 | 8620 | HTTP | REST API | -| pd0 | 8686 | 8686 | gRPC | PD gRPC | -| pd0 | 8610 | — | TCP | Raft (internal only) | -| pd1 | 8620 | 8621 | HTTP | REST API | -| pd1 | 8686 | 8687 | gRPC | PD gRPC | -| pd2 | 8620 | 8622 | HTTP | REST API | -| pd2 | 8686 | 8688 | gRPC | PD gRPC | -| store0 | 8500 | 8500 | gRPC | Store gRPC | -| store0 | 8510 | 8510 | TCP | Raft | -| store0 | 8520 | 8520 | HTTP | REST API | -| store1 | 8500 | 8501 | gRPC | Store gRPC | -| store1 | 8510 | 8511 | TCP | Raft | -| store1 | 8520 | 8521 | HTTP | REST API | -| store2 | 8500 | 8502 | gRPC | Store gRPC | -| store2 | 8510 | 8512 | TCP | Raft | -| store2 | 8520 | 8522 | HTTP | REST API | -| server0 | 8080 | 8080 | HTTP | Graph API | -| server1 | 8080 | 8081 | HTTP | Graph API | -| server2 | 8080 | 8082 | HTTP | Graph API | +### Images and Compose files ---- +| Image | Build file | +| --- | --- | +| `hugegraph/hugegraph` (standalone RocksDB Server) | `hugegraph-server/Dockerfile` | +| `hugegraph/server` (HStore Server) | `hugegraph-server/Dockerfile-hstore` | +| `hugegraph/pd` | `hugegraph-pd/Dockerfile` | +| `hugegraph/store` | `hugegraph-store/Dockerfile` | -## Healthcheck Endpoints +Hubble is built from the separate HugeGraph Toolchain repository and is +selected here with `HUBBLE_IMAGE`. -| Service | Endpoint | Expected | -|---------|----------|----------| -| PD | `GET /v1/health` | `200 OK` | -| Store | `GET /v1/health` | `200 OK` | -| Server | `GET /versions` | `200 OK` with version JSON | -| Hubble | `GET /about` | `200` JSON with Hubble name and version | +The Compose mapping is intentionally small: ---- +- `docker-compose.yml` is the standalone user default. +- `docker-compose-hstore.yml` is the minimal 1 PD + 1 Store + 1 Server base. +- `docker-compose-3pd-3store-3server.yml` is the HA reference. +- `docker-compose.dev.yml` is a thin source-build override for the minimal + HStore topology. It does not duplicate runtime services, networks, volumes, + health checks, or Hubble. -## Troubleshooting +Build and start the minimal topology from local source: -### Containers Exiting or Restarting (OOM Kills) +```bash +docker compose \ + -f docker-compose-hstore.yml \ + -f docker-compose.dev.yml \ + up -d --build --wait +``` -**Symptom**: Containers exit with code 137, or restart loops. Raft logs show election timeouts. +Use both files for every later lifecycle command, for example: -**Cause**: Docker Desktop does not have enough memory. The 9 JVM processes require at least 12 GB. +```bash +docker compose \ + -f docker-compose-hstore.yml \ + -f docker-compose.dev.yml \ + down +``` -**Fix**: Docker Desktop → Settings → Resources → Memory → set to **12 GB** or higher. Restart Docker Desktop. +The development overlay builds `hugegraph/pd:dev`, `hugegraph/store:dev`, and +`hugegraph/server:dev`. To reuse those local images and a locally built Hubble +without pulling replacements: ```bash -# Check if containers were OOM killed -docker inspect hg-pd0 | grep -i oom -docker stats --no-stream +HUGEGRAPH_VERSION=dev \ +HUGEGRAPH_PULL_POLICY=never \ +HUBBLE_IMAGE=local/hugegraph-hubble:test \ +HUBBLE_PULL_POLICY=never \ +docker compose -f docker-compose-hstore.yml up -d --wait ``` -### Raft Leader Election Failure +### Hubble configuration + +The three small files under `conf/hubble/` contain only topology-specific +discovery settings and container paths: -**Symptom**: PD logs show repeated `Leader election timeout`. Store nodes cannot register. +- `conf/hubble/standalone.properties` uses direct Server mode. +- `conf/hubble/hstore.properties` uses one PD and one Store REST target. +- `conf/hubble/hstore-ha.properties` uses all three PD peers and all three + allowed Store REST targets. -**Cause**: PD nodes cannot reach each other on the Raft port (8610), or `HG_PD_RAFT_PEERS_LIST` is misconfigured. +Hubble detects Server authentication through the Server API. Do not add an +`auth.enabled` property or duplicate auth-on/auth-off configurations. -**Fix**: -1. Verify all PD containers are running: `docker compose -f docker-compose-3pd-3store-3server.yml ps` -2. Check PD logs: `docker logs hg-pd0` -3. Verify network connectivity: `docker exec hg-pd0 ping pd1` -4. Ensure `HG_PD_RAFT_PEERS_LIST` is identical on all PD nodes +### Render and smoke checks -### Partition Assignment Not Completing +Render every topology with auth-on inputs before submitting a change: -**Symptom**: Server starts but graph operations fail. Store logs show `partition not found`. +```bash +for file in \ + docker-compose.yml \ + docker-compose-hstore.yml \ + docker-compose-3pd-3store-3server.yml +do + docker compose -f "${file}" config --quiet +done +docker compose \ + -f docker-compose-hstore.yml \ + -f docker-compose.dev.yml \ + config --quiet +``` -**Cause**: PD has not finished assigning partitions to stores, or stores did not register successfully. +The HA render is mandatory even when local resources are insufficient to start +its ten containers. -**Fix**: -1. Check registered stores: `curl http://localhost:8620/v1/stores` -2. Check partition status: `curl http://localhost:8620/v1/partitions` -3. Wait for partition assignment (can take 1–3 minutes after all stores register) -4. Check server logs for the `wait-partition.sh` script output: `docker logs hg-server0` +Run focused auth-on smoke checks for standalone and minimal HStore with the +corresponding `up -d --wait`, status, authentication, Hubble `/about`, and +`down -v` commands from the Users section: -### Connection Refused Errors +```bash +bash test-compose.sh smoke +``` -**Symptom**: Stores cannot connect to PD, or Server cannot connect to Store. +Run the required local auth-off checks separately: -**Cause**: Services are using `127.0.0.1` instead of container hostnames, or the `hg-net` bridge network is misconfigured. +```bash +bash test-compose.sh smoke-auth-off +``` -**Fix**: Ensure all `HG_*` env vars use container hostnames (`pd0`, `store0`, etc.), not `127.0.0.1` or `localhost`. +The auth-off mode is intentionally excluded from the default CI matrix and must +remain on a trusted local machine. Both smoke modes remove only the isolated +Compose projects and volumes that they create. diff --git a/docker/conf/hubble/hstore-ha.properties b/docker/conf/hubble/hstore-ha.properties new file mode 100644 index 0000000000..a50c52c96f --- /dev/null +++ b/docker/conf/hubble/hstore-ha.properties @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +server.host=0.0.0.0 +cluster=hg +idc=docker +pd.enabled=true +server.direct_url=http://server0:8080 +pd.peers=pd0:8686,pd1:8686,pd2:8686 +pd.server=pd0:8620 +operations.store.allowed_targets=[http://store0:8520,http://store1:8520,http://store2:8520] +upload_file.location=/hubble/data/upload-files +dashboard.address= diff --git a/docker/hugegraph-hubble.properties b/docker/conf/hubble/hstore.properties similarity index 93% rename from docker/hugegraph-hubble.properties rename to docker/conf/hubble/hstore.properties index b44255cfcf..5de43c212f 100644 --- a/docker/hugegraph-hubble.properties +++ b/docker/conf/hubble/hstore.properties @@ -14,17 +14,12 @@ # limitations under the License. server.host=0.0.0.0 -server.port=8088 - cluster=hg idc=docker - pd.enabled=true server.direct_url=http://server:8080 pd.peers=pd:8686 pd.server=pd:8620 - operations.store.allowed_targets=[http://store:8520] - -# Dashboard is not part of this Compose stack. +upload_file.location=/hubble/data/upload-files dashboard.address= diff --git a/docker/conf/hubble/standalone.properties b/docker/conf/hubble/standalone.properties new file mode 100644 index 0000000000..2aab076218 --- /dev/null +++ b/docker/conf/hubble/standalone.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +server.host=0.0.0.0 +pd.enabled=false +server.direct_url=http://server:8080 +upload_file.location=/hubble/data/upload-files +dashboard.address= diff --git a/docker/docker-compose-3pd-3store-3server.yml b/docker/docker-compose-3pd-3store-3server.yml index fc7930351b..67ea85b51e 100644 --- a/docker/docker-compose-3pd-3store-3server.yml +++ b/docker/docker-compose-3pd-3store-3server.yml @@ -28,6 +28,7 @@ volumes: hg-store0-data: hg-store1-data: hg-store2-data: + hubble-data: # ── Shared service defaults ────────────────────────────────────────── x-pd-common: &pd-common @@ -58,6 +59,17 @@ x-store-common: &store-common retries: 40 start_period: 120s +x-server-environment: &server-environment + STORE_REST: store0:8520 + HG_SERVER_BACKEND: hstore + HG_SERVER_PD_PEERS: pd0:8686,pd1:8686,pd2:8686 + HG_SERVER_CLUSTER: hg + HG_SERVER_USE_PD: "true" + HG_SERVER_MIN_FREE_MEMORY: "0" + HG_SERVER_INIT_STORE_ENABLED: "false" + HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:-} + PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:-} + x-server-common: &server-common image: hugegraph/server:${HUGEGRAPH_VERSION:-latest} pull_policy: missing @@ -68,11 +80,9 @@ x-server-common: &server-common store1: { condition: service_healthy } store2: { condition: service_healthy } environment: - STORE_REST: store0:8520 - HG_SERVER_BACKEND: hstore - HG_SERVER_PD_PEERS: pd0:8686,pd1:8686,pd2:8686 + <<: *server-environment healthcheck: - test: ["CMD-SHELL", "curl -fsS http://localhost:8080/versions >/dev/null || exit 1"] + test: ["CMD-SHELL", "curl -fsS http://server0:8080/versions >/dev/null"] interval: 10s timeout: 5s retries: 30 @@ -188,15 +198,58 @@ services: container_name: hg-server0 hostname: server0 ports: ["8080:8080"] + environment: + <<: *server-environment + HG_SERVER_REST_URL: http://server0:8080 + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://server0:8080/versions >/dev/null"] server1: <<: *server-common container_name: hg-server1 hostname: server1 ports: ["8081:8080"] + environment: + <<: *server-environment + HG_SERVER_REST_URL: http://server1:8080 + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://server1:8080/versions >/dev/null"] server2: <<: *server-common container_name: hg-server2 hostname: server2 ports: ["8082:8080"] + environment: + <<: *server-environment + HG_SERVER_REST_URL: http://server2:8080 + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://server2:8080/versions >/dev/null"] + + hubble: + image: ${HUBBLE_IMAGE:-hugegraph/hubble:latest} + pull_policy: ${HUBBLE_PULL_POLICY:-missing} + restart: unless-stopped + networks: [hg-net] + depends_on: + server0: { condition: service_healthy } + server1: { condition: service_healthy } + server2: { condition: service_healthy } + environment: + SPRING_DATASOURCE_URL: jdbc:h2:file:/hubble/data/hubble;DB_CLOSE_ON_EXIT=FALSE + ports: + - "${HUBBLE_PUBLISH_HOST:-127.0.0.1}:8088:8088" + volumes: + - hubble-data:/hubble/data + - ./conf/hubble/hstore-ha.properties:/hubble/conf/hugegraph-hubble.properties:ro + healthcheck: + test: + - CMD-SHELL + - >- + body=$$(curl -fsS http://localhost:8088/about) && + printf '%s' "$$body" | grep -q '"status":200' && + printf '%s' "$$body" | grep -q '"name":"hugegraph-hubble"' + interval: 10s + timeout: 5s + retries: 30 + start_period: 60s diff --git a/docker/docker-compose-hstore.yml b/docker/docker-compose-hstore.yml new file mode 100644 index 0000000000..d201430692 --- /dev/null +++ b/docker/docker-compose-hstore.yml @@ -0,0 +1,131 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: hugegraph-hstore + +networks: + hg-net: + +volumes: + pd-data: + store-data: + hubble-data: + +services: + pd: + image: hugegraph/pd:${HUGEGRAPH_VERSION:-latest} + pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} + restart: unless-stopped + networks: [hg-net] + environment: + HG_PD_GRPC_HOST: pd + HG_PD_GRPC_PORT: "8686" + HG_PD_REST_PORT: "8620" + HG_PD_RAFT_ADDRESS: pd:8610 + HG_PD_RAFT_PEERS_LIST: pd:8610 + HG_PD_INITIAL_STORE_LIST: store:8500 + HG_PD_DATA_PATH: /hugegraph-pd/pd_data + ports: + - "8620:8620" + volumes: + - pd-data:/hugegraph-pd/pd_data + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8620/v1/health >/dev/null"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 30s + + store: + image: hugegraph/store:${HUGEGRAPH_VERSION:-latest} + pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} + restart: unless-stopped + networks: [hg-net] + depends_on: + pd: + condition: service_healthy + environment: + HG_STORE_PD_ADDRESS: pd:8686 + HG_STORE_GRPC_HOST: store + HG_STORE_GRPC_PORT: "8500" + HG_STORE_REST_PORT: "8520" + HG_STORE_RAFT_ADDRESS: store:8510 + HG_STORE_DATA_PATH: /hugegraph-store/storage + ports: + - "8520:8520" + volumes: + - store-data:/hugegraph-store/storage + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null"] + interval: 10s + timeout: 10s + retries: 30 + start_period: 60s + + server: + image: hugegraph/server:${HUGEGRAPH_VERSION:-latest} + pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} + restart: unless-stopped + networks: [hg-net] + depends_on: + store: + condition: service_healthy + environment: + HG_SERVER_BACKEND: hstore + HG_SERVER_PD_PEERS: pd:8686 + HG_SERVER_CLUSTER: hg + HG_SERVER_USE_PD: "true" + HG_SERVER_REST_URL: http://server:8080 + HG_SERVER_MIN_FREE_MEMORY: "0" + HG_SERVER_INIT_STORE_ENABLED: "false" + HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:-} + PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:-} + ports: + - "8080:8080" + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://server:8080/versions >/dev/null"] + interval: 10s + timeout: 5s + retries: 30 + start_period: 60s + + hubble: + image: ${HUBBLE_IMAGE:-hugegraph/hubble:latest} + pull_policy: ${HUBBLE_PULL_POLICY:-missing} + restart: unless-stopped + networks: [hg-net] + depends_on: + server: + condition: service_healthy + environment: + SPRING_DATASOURCE_URL: jdbc:h2:file:/hubble/data/hubble;DB_CLOSE_ON_EXIT=FALSE + ports: + - "${HUBBLE_PUBLISH_HOST:-127.0.0.1}:8088:8088" + volumes: + - hubble-data:/hubble/data + - ./conf/hubble/hstore.properties:/hubble/conf/hugegraph-hubble.properties:ro + healthcheck: + test: + - CMD-SHELL + - >- + body=$$(curl -fsS http://localhost:8088/about) && + printf '%s' "$$body" | grep -q '"status":200' && + printf '%s' "$$body" | grep -q '"name":"hugegraph-hubble"' + interval: 10s + timeout: 5s + retries: 30 + start_period: 60s diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 70089f488a..4da9c32e19 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -15,122 +15,28 @@ # limitations under the License. # -name: hugegraph-single - -networks: - hg-net: - driver: bridge - -volumes: - hg-pd-data: - hg-store-data: - services: pd: + image: hugegraph/pd:dev + pull_policy: build build: context: .. dockerfile: hugegraph-pd/Dockerfile - container_name: hg-pd - hostname: pd - restart: unless-stopped - networks: [hg-net] - environment: - HG_PD_GRPC_HOST: pd - HG_PD_GRPC_PORT: "8686" - HG_PD_REST_PORT: "8620" - HG_PD_RAFT_ADDRESS: pd:8610 - HG_PD_RAFT_PEERS_LIST: pd:8610 - HG_PD_INITIAL_STORE_LIST: store:8500 - HG_PD_DATA_PATH: /hugegraph-pd/pd_data - ports: - - "8620:8620" - volumes: - - hg-pd-data:/hugegraph-pd/pd_data healthcheck: - test: ["CMD-SHELL", "curl -fsS http://localhost:8620/v1/health >/dev/null || exit 1"] - interval: 10s - timeout: 5s - retries: 12 start_period: 20s store: + image: hugegraph/store:dev + pull_policy: build build: context: .. dockerfile: hugegraph-store/Dockerfile - container_name: hg-store - hostname: store - restart: unless-stopped - networks: [hg-net] - depends_on: - pd: - condition: service_healthy - environment: - HG_STORE_PD_ADDRESS: pd:8686 - HG_STORE_GRPC_HOST: store - HG_STORE_GRPC_PORT: "8500" - HG_STORE_REST_PORT: "8520" - HG_STORE_RAFT_ADDRESS: store:8510 - HG_STORE_DATA_PATH: /hugegraph-store/storage - ports: - - "8520:8520" - volumes: - - hg-store-data:/hugegraph-store/storage healthcheck: - test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] - interval: 10s - timeout: 10s - retries: 30 start_period: 30s server: - image: ${HUGEGRAPH_SERVER_IMAGE:-hugegraph/server:dev} - pull_policy: ${HUGEGRAPH_SERVER_PULL_POLICY:-build} + image: hugegraph/server:dev + pull_policy: build build: context: .. dockerfile: hugegraph-server/Dockerfile-hstore - container_name: hg-server - hostname: server - restart: unless-stopped - networks: [hg-net] - depends_on: - store: - condition: service_healthy - environment: - HG_SERVER_BACKEND: hstore - HG_SERVER_PD_PEERS: pd:8686 - HG_SERVER_CLUSTER: hg - HG_SERVER_USE_PD: "true" - HG_SERVER_REST_URL: http://server:8080 - HG_SERVER_MIN_FREE_MEMORY: "0" - HG_SERVER_INIT_STORE_ENABLED: "false" - HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:-} - PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:?Set a non-default admin password} - ports: - - "8080:8080" - healthcheck: - test: ["CMD-SHELL", "curl -fsS http://server:8080/versions >/dev/null || exit 1"] - interval: 10s - timeout: 5s - retries: 30 - start_period: 60s - - hubble: - image: ${HUBBLE_IMAGE:-hugegraph/hubble:latest} - pull_policy: ${HUBBLE_PULL_POLICY:-missing} - container_name: hg-hubble - hostname: hubble - restart: unless-stopped - networks: [hg-net] - depends_on: - server: - condition: service_healthy - ports: - - "${HUBBLE_PUBLISH_HOST:-127.0.0.1}:8088:8088" - volumes: - - ./hugegraph-hubble.properties:/hubble/conf/hugegraph-hubble.properties:ro - healthcheck: - test: ["CMD-SHELL", "body=$$(curl -fsS http://127.0.0.1:8088/about) && printf '%s' \"$$body\" | grep -q '\"status\":200' && printf '%s' \"$$body\" | grep -q '\"name\":\"hugegraph-hubble\"'"] - interval: 10s - timeout: 5s - retries: 30 - start_period: 60s diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0a7a7ea294..828ffe42b1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -14,117 +14,59 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: hugegraph-single + +name: hugegraph-standalone networks: hg-net: - driver: bridge volumes: - hg-pd-data: - hg-store-data: + server-data: + hubble-data: services: - - pd: - image: hugegraph/pd:${HUGEGRAPH_VERSION:-latest} - pull_policy: always - container_name: hg-pd - hostname: pd - restart: unless-stopped - networks: [hg-net] - environment: - HG_PD_GRPC_HOST: pd - HG_PD_GRPC_PORT: "8686" - HG_PD_REST_PORT: "8620" - HG_PD_RAFT_ADDRESS: pd:8610 - HG_PD_RAFT_PEERS_LIST: pd:8610 - HG_PD_INITIAL_STORE_LIST: store:8500 - HG_PD_DATA_PATH: /hugegraph-pd/pd_data - ports: - - "8620:8620" - volumes: - - hg-pd-data:/hugegraph-pd/pd_data - healthcheck: - test: ["CMD-SHELL", "curl -fsS http://localhost:8620/v1/health >/dev/null || exit 1"] - interval: 10s - timeout: 5s - retries: 12 - start_period: 30s - - store: - image: hugegraph/store:${HUGEGRAPH_VERSION:-latest} - pull_policy: always - container_name: hg-store - hostname: store - restart: unless-stopped - networks: [hg-net] - depends_on: - pd: - condition: service_healthy - environment: - HG_STORE_PD_ADDRESS: pd:8686 - HG_STORE_GRPC_HOST: store - HG_STORE_GRPC_PORT: "8500" - HG_STORE_REST_PORT: "8520" - HG_STORE_RAFT_ADDRESS: store:8510 - HG_STORE_DATA_PATH: /hugegraph-store/storage - ports: - - "8520:8520" - volumes: - - hg-store-data:/hugegraph-store/storage - healthcheck: - test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null || exit 1"] - interval: 10s - timeout: 10s - retries: 30 - start_period: 60s - server: - image: ${HUGEGRAPH_SERVER_IMAGE:-hugegraph/server:${HUGEGRAPH_VERSION:-latest}} - pull_policy: ${HUGEGRAPH_SERVER_PULL_POLICY:-always} - container_name: hg-server - hostname: server + image: hugegraph/hugegraph:${HUGEGRAPH_VERSION:-latest} + pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} restart: unless-stopped networks: [hg-net] - depends_on: - store: - condition: service_healthy environment: - HG_SERVER_BACKEND: hstore - HG_SERVER_PD_PEERS: pd:8686 - HG_SERVER_CLUSTER: hg - HG_SERVER_USE_PD: "true" - HG_SERVER_REST_URL: http://server:8080 - HG_SERVER_MIN_FREE_MEMORY: "0" - HG_SERVER_INIT_STORE_ENABLED: "false" + PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:-} HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:-} - PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:?Set a non-default admin password} + HG_SERVER_MIN_FREE_MEMORY: "0" ports: - "8080:8080" + volumes: + - server-data:/hugegraph-server/rocksdb-data healthcheck: - test: ["CMD-SHELL", "curl -fsS http://server:8080/versions >/dev/null || exit 1"] + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/versions >/dev/null"] interval: 10s timeout: 5s retries: 30 start_period: 60s hubble: - image: ${HUBBLE_IMAGE:-hugegraph/hubble:${HUGEGRAPH_VERSION:-latest}} - pull_policy: ${HUBBLE_PULL_POLICY:-always} - container_name: hg-hubble - hostname: hubble + image: ${HUBBLE_IMAGE:-hugegraph/hubble:latest} + pull_policy: ${HUBBLE_PULL_POLICY:-missing} restart: unless-stopped networks: [hg-net] depends_on: server: condition: service_healthy + environment: + SPRING_DATASOURCE_URL: jdbc:h2:file:/hubble/data/hubble;DB_CLOSE_ON_EXIT=FALSE ports: - "${HUBBLE_PUBLISH_HOST:-127.0.0.1}:8088:8088" volumes: - - ./hugegraph-hubble.properties:/hubble/conf/hugegraph-hubble.properties:ro + - hubble-data:/hubble/data + - ./conf/hubble/standalone.properties:/hubble/conf/hugegraph-hubble.properties:ro healthcheck: - test: ["CMD-SHELL", "body=$$(curl -fsS http://127.0.0.1:8088/about) && printf '%s' \"$$body\" | grep -q '\"status\":200' && printf '%s' \"$$body\" | grep -q '\"name\":\"hugegraph-hubble\"'"] + test: + - CMD-SHELL + - >- + body=$$(curl -fsS http://localhost:8088/about) && + printf '%s' "$$body" | grep -q '"status":200' && + printf '%s' "$$body" | grep -q '"name":"hugegraph-hubble"' interval: 10s timeout: 5s retries: 30 diff --git a/docker/test-compose.sh b/docker/test-compose.sh new file mode 100644 index 0000000000..7df96dfe23 --- /dev/null +++ b/docker/test-compose.sh @@ -0,0 +1,396 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +DOCKER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PASSWORD="ci-compose-password" +SECRET="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +VERSION="ci-version" +RENDER_HUBBLE_IMAGE="example.invalid/hugegraph/hubble:ci" +DATASOURCE="jdbc:h2:file:/hubble/data/hubble;DB_CLOSE_ON_EXIT=FALSE" +ACTIVE_PROJECT="" +ACTIVE_FILES=() +RENDER_DIR="" + +compose_auth() { + env HUGEGRAPH_VERSION="${VERSION}" \ + HUBBLE_IMAGE="${RENDER_HUBBLE_IMAGE}" \ + HUGEGRAPH_ADMIN_PASSWORD="${PASSWORD}" \ + HUGEGRAPH_AUTH_TOKEN_SECRET="${SECRET}" \ + docker compose "$@" +} + +render() { + local output="$1" + shift + compose_auth "$@" config --format json > "${output}" +} + +assert_file_property() { + local file="$1" + local property="$2" + grep -Fqx "${property}" "${file}" +} + +assert_common() { + local rendered="$1" + local services="$2" + local volumes="$3" + + jq -e \ + --arg password "${PASSWORD}" \ + --arg secret "${SECRET}" \ + --argjson services "${services}" \ + --argjson volumes "${volumes}" ' + (.services | keys) == $services and + (.volumes | keys) == $volumes and + (.networks | keys) == ["hg-net"] and + all(.networks[]; .external != true) and + all(.volumes[]; .external != true) and + all(.services[]; + (.networks | keys) == ["hg-net"] and + .healthcheck.test[0] == "CMD-SHELL") and + all( + [.services | to_entries[] | + select(.key | startswith("server")) | + .value.environment][]; + .PASSWORD == $password and + .HG_SERVER_AUTH_TOKEN_SECRET == $secret) + ' "${rendered}" >/dev/null +} + +assert_hubble() { + local rendered="$1" + local config="$2" + shift 2 + local dependencies + dependencies="$(printf '%s\n' "$@" | jq -Rsc 'split("\n")[:-1] | sort')" + + jq -e \ + --arg datasource "${DATASOURCE}" \ + --arg config "/docker/conf/hubble/${config}" \ + --argjson dependencies "${dependencies}" ' + .services.hubble.image == "example.invalid/hugegraph/hubble:ci" and + .services.hubble.pull_policy == "missing" and + .services.hubble.environment.SPRING_DATASOURCE_URL == $datasource and + any(.services.hubble.ports[]; + .target == 8088 and .published == "8088" and + .host_ip == "127.0.0.1") and + (.services.hubble.depends_on | keys | sort) == $dependencies and + all(.services.hubble.depends_on[]; + .condition == "service_healthy") and + any(.services.hubble.volumes[]; + .type == "volume" and .source == "hubble-data" and + .target == "/hubble/data") and + any(.services.hubble.volumes[]; + .type == "bind" and (.source | endswith($config)) and + .target == "/hubble/conf/hugegraph-hubble.properties" and + .read_only == true) and + (.services.hubble.healthcheck.test[1] | + contains("http://localhost:8088/about") and + contains("\"status\":200") and + contains("\"name\":\"hugegraph-hubble\"")) + ' "${rendered}" >/dev/null +} + +assert_standalone() { + local rendered="$1" + assert_common "${rendered}" \ + '["hubble","server"]' \ + '["hubble-data","server-data"]' + assert_hubble "${rendered}" "standalone.properties" server + jq -e ' + .services.server.image == "hugegraph/hugegraph:ci-version" and + .services.server.pull_policy == "missing" and + .services.server.healthcheck.test[1] == + "curl -fsS http://localhost:8080/versions >/dev/null" and + any(.services.server.volumes[]; + .source == "server-data" and + .target == "/hugegraph-server/rocksdb-data") + ' "${rendered}" >/dev/null + assert_file_property "${DOCKER_DIR}/conf/hubble/standalone.properties" \ + "pd.enabled=false" + assert_file_property "${DOCKER_DIR}/conf/hubble/standalone.properties" \ + "server.direct_url=http://server:8080" +} + +assert_hstore() { + local rendered="$1" + assert_common "${rendered}" \ + '["hubble","pd","server","store"]' \ + '["hubble-data","pd-data","store-data"]' + assert_hubble "${rendered}" "hstore.properties" server + jq -e ' + .services.pd.image == "hugegraph/pd:ci-version" and + .services.store.image == "hugegraph/store:ci-version" and + .services.server.image == "hugegraph/server:ci-version" and + all([.services.pd, .services.store, .services.server][]; + .pull_policy == "missing") and + .services.server.environment.HG_SERVER_BACKEND == "hstore" and + .services.server.environment.HG_SERVER_PD_PEERS == "pd:8686" and + .services.server.environment.HG_SERVER_USE_PD == "true" and + .services.server.environment.HG_SERVER_REST_URL == + "http://server:8080" and + .services.server.healthcheck.test[1] == + "curl -fsS http://server:8080/versions >/dev/null" and + any(.services.pd.volumes[]; + .source == "pd-data" and + .target == "/hugegraph-pd/pd_data") and + any(.services.store.volumes[]; + .source == "store-data" and + .target == "/hugegraph-store/storage") + ' "${rendered}" >/dev/null + assert_file_property "${DOCKER_DIR}/conf/hubble/hstore.properties" \ + "pd.peers=pd:8686" + assert_file_property "${DOCKER_DIR}/conf/hubble/hstore.properties" \ + "operations.store.allowed_targets=[http://store:8520]" +} + +assert_ha() { + local rendered="$1" + assert_common "${rendered}" \ + '["hubble","pd0","pd1","pd2","server0","server1","server2","store0","store1","store2"]' \ + '["hg-pd0-data","hg-pd1-data","hg-pd2-data","hg-store0-data","hg-store1-data","hg-store2-data","hubble-data"]' + assert_hubble "${rendered}" "hstore-ha.properties" \ + server0 server1 server2 + jq -e ' + all([.services.pd0, .services.pd1, .services.pd2][]; + .image == "hugegraph/pd:ci-version" and + .pull_policy == "missing") and + all([.services.store0, .services.store1, .services.store2][]; + .image == "hugegraph/store:ci-version" and + .pull_policy == "missing") and + all([.services.server0, .services.server1, .services.server2][]; + .image == "hugegraph/server:ci-version" and + .pull_policy == "missing" and + .environment.STORE_REST == "store0:8520" and + .environment.HG_SERVER_BACKEND == "hstore" and + .environment.HG_SERVER_PD_PEERS == + "pd0:8686,pd1:8686,pd2:8686" and + .environment.HG_SERVER_CLUSTER == "hg" and + .environment.HG_SERVER_USE_PD == "true" and + .environment.HG_SERVER_INIT_STORE_ENABLED == "false") and + [.services.server0.environment.HG_SERVER_REST_URL, + .services.server1.environment.HG_SERVER_REST_URL, + .services.server2.environment.HG_SERVER_REST_URL] == + ["http://server0:8080", + "http://server1:8080", + "http://server2:8080"] and + [.services.server0.healthcheck.test[1], + .services.server1.healthcheck.test[1], + .services.server2.healthcheck.test[1]] == + ["curl -fsS http://server0:8080/versions >/dev/null", + "curl -fsS http://server1:8080/versions >/dev/null", + "curl -fsS http://server2:8080/versions >/dev/null"] + ' "${rendered}" >/dev/null + assert_file_property "${DOCKER_DIR}/conf/hubble/hstore-ha.properties" \ + "pd.peers=pd0:8686,pd1:8686,pd2:8686" + assert_file_property "${DOCKER_DIR}/conf/hubble/hstore-ha.properties" \ + "operations.store.allowed_targets=[http://store0:8520,http://store1:8520,http://store2:8520]" +} + +assert_dev_override() { + local rendered="$1" + local override="$2" + assert_common "${rendered}" \ + '["hubble","pd","server","store"]' \ + '["hubble-data","pd-data","store-data"]' + jq -e ' + .services.pd.image == "hugegraph/pd:dev" and + .services.store.image == "hugegraph/store:dev" and + .services.server.image == "hugegraph/server:dev" and + all([.services.pd, .services.store, .services.server][]; + .pull_policy == "build" and .build != null) and + .services.hubble.image == "example.invalid/hugegraph/hubble:ci" and + .services.hubble.build == null + ' "${rendered}" >/dev/null + jq -e ' + (.services | keys) == ["pd","server","store"] and + (.networks | keys) == ["default"] and .volumes == null and + all(.services[]; + .build != null and .image != null and + .pull_policy == "build" and + .environment == null and + (.networks | keys) == ["default"] and + .volumes == null) + ' "${override}" >/dev/null +} + +cleanup() { + if [[ -n "${ACTIVE_PROJECT}" ]]; then + compose_active down -v --remove-orphans >/dev/null 2>&1 || true + fi + [[ -z "${RENDER_DIR}" ]] || rm -rf "${RENDER_DIR}" +} + +run_render() { + RENDER_DIR="$(mktemp -d)" + trap cleanup EXIT INT TERM + render "${RENDER_DIR}/standalone.json" \ + -f "${DOCKER_DIR}/docker-compose.yml" + render "${RENDER_DIR}/hstore.json" \ + -f "${DOCKER_DIR}/docker-compose-hstore.yml" + render "${RENDER_DIR}/ha.json" \ + -f "${DOCKER_DIR}/docker-compose-3pd-3store-3server.yml" + render "${RENDER_DIR}/dev.json" \ + -f "${DOCKER_DIR}/docker-compose-hstore.yml" \ + -f "${DOCKER_DIR}/docker-compose.dev.yml" + render "${RENDER_DIR}/override.json" \ + -f "${DOCKER_DIR}/docker-compose.dev.yml" + + assert_standalone "${RENDER_DIR}/standalone.json" + assert_hstore "${RENDER_DIR}/hstore.json" + assert_ha "${RENDER_DIR}/ha.json" + assert_dev_override "${RENDER_DIR}/dev.json" \ + "${RENDER_DIR}/override.json" + echo "Compose render contracts passed" +} + +compose_active() { + env HUGEGRAPH_VERSION="${HUGEGRAPH_VERSION:-latest}" \ + HUBBLE_IMAGE="${HUBBLE_IMAGE:-hugegraph/hubble:latest}" \ + HUGEGRAPH_ADMIN_PASSWORD="${PASSWORD}" \ + HUGEGRAPH_AUTH_TOKEN_SECRET="${SECRET}" \ + COMPOSE_PROGRESS=plain \ + docker compose -p "${ACTIVE_PROJECT}" "${ACTIVE_FILES[@]}" "$@" +} + +diagnose() { + compose_active ps || true + compose_active logs --no-color --tail 200 || true +} + +http_status() { + curl -sS -o /dev/null -w '%{http_code}' "$@" +} + +wait_hubble_mode() { + local expected_pd="$1" + local expected_auth="$2" + local response="" + local _ + for _ in {1..30}; do + response="$(curl -fsS http://localhost:8088/api/v1.3/config || true)" + if jq -e --argjson expected_pd "${expected_pd}" \ + --argjson expected_auth "${expected_auth}" ' + .status == 200 and + .data.pd_enabled == $expected_pd and + .data.auth_enabled == $expected_auth and + .data.server_capabilities_verified == true + ' <<<"${response}" >/dev/null 2>&1; then + return + fi + sleep 2 + done + echo "Hubble authentication detection failed: ${response}" >&2 + return 1 +} + +check_hubble_login() { + local response + response="$(curl -fsS -H "Content-Type: application/json" \ + --data "{\"user_name\":\"admin\",\"user_password\":\"${PASSWORD}\"}" \ + http://localhost:8088/api/v1.3/auth/login)" + jq -e '.status == 200 and .data.user_name == "admin"' \ + <<<"${response}" >/dev/null +} + +check_hubble_anonymous() { + curl -fsS http://localhost:8088/api/v1.3/auth/status | + jq -e '.status == 200 and .data.level == "ANONYMOUS"' >/dev/null + curl -fsS http://localhost:8088/api/v1.3/auth/context | + jq -e ' + .status == 200 and + .data.mode == "NON_AUTH" and + .data.role == "ANONYMOUS" + ' >/dev/null +} + +smoke() { + local name="$1" + local expected_pd="$2" + local expected_auth="$3" + shift 3 + ACTIVE_PROJECT="hg-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-$$-${name}" + ACTIVE_FILES=() + while (($#)); do + ACTIVE_FILES+=(-f "$1") + shift + done + + if ! compose_active up -d --wait --wait-timeout 600; then + diagnose + return 1 + fi + curl -fsS http://localhost:8080/versions >/dev/null + if [[ "${expected_auth}" == true ]]; then + [[ "$(http_status \ + http://localhost:8080/graphspaces/DEFAULT/graphs)" == 401 ]] + [[ "$(http_status -u "admin:${PASSWORD}" \ + http://localhost:8080/graphspaces/DEFAULT/graphs)" == 200 ]] + else + [[ "$(http_status \ + http://localhost:8080/graphspaces/DEFAULT/graphs)" == 200 ]] + fi + curl -fsS http://localhost:8088/about | + jq -e '.status == 200 and .data.name == "hugegraph-hubble"' >/dev/null + wait_hubble_mode "${expected_pd}" "${expected_auth}" + if [[ "${expected_auth}" == true ]]; then + check_hubble_login + else + check_hubble_anonymous + fi + compose_active down -v --remove-orphans + ACTIVE_PROJECT="" + ACTIVE_FILES=() + echo "Compose smoke passed: ${name}" +} + +run_smoke() { + trap cleanup EXIT INT TERM + smoke standalone false true "${DOCKER_DIR}/docker-compose.yml" + smoke hstore true true "${DOCKER_DIR}/docker-compose-hstore.yml" +} + +run_smoke_auth_off() { + PASSWORD="" + trap cleanup EXIT INT TERM + smoke standalone-anon false false "${DOCKER_DIR}/docker-compose.yml" + smoke hstore-anon true false "${DOCKER_DIR}/docker-compose-hstore.yml" +} + +case "${1:-}" in + render) + run_render + ;; + smoke) + run_smoke + ;; + smoke-auth-off) + run_smoke_auth_off + ;; + all) + run_render + run_smoke + ;; + *) + echo "Usage: $0 {render|smoke|smoke-auth-off|all}" >&2 + exit 2 + ;; +esac