From 400456c64fface1809f6e46d51b63160756842bc Mon Sep 17 00:00:00 2001 From: Jonas Thelemann Date: Thu, 6 Aug 2026 22:27:00 +0200 Subject: [PATCH 1/7] fix(postgraphile): correct sentry dsn environment variable name --- src/development/postgraphile/compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/development/postgraphile/compose.yaml b/src/development/postgraphile/compose.yaml index 51d8883d..ab2a5c6c 100644 --- a/src/development/postgraphile/compose.yaml +++ b/src/development/postgraphile/compose.yaml @@ -46,7 +46,7 @@ services: - source: postgraphile-owner-connection target: /run/environment-variables/POSTGRAPHILE_OWNER_CONNECTION - source: postgraphile-sentry-dsn - target: /run/environment-variables/POSTGRAPHILE_SENTRY_DSN + target: /run/environment-variables/SENTRY_DSN volumes: - ~~~/postgraphile/:/srv/app/ # dargstack:dev-only - ./configurations/jwtES256.key.pub:/run/environment-variables/POSTGRAPHILE_JWT_PUBLIC_KEY:ro From 57d636265b39204c9dfbd23ce54fe18933db07cf Mon Sep 17 00:00:00 2001 From: Minh Dang Hoang Date: Thu, 6 Aug 2026 14:33:03 +0200 Subject: [PATCH 2/7] feat(gizmosql): add --- artifacts/docs/README.md | 10 ++- src/development/gizmosql/compose.yaml | 58 ++++++++++++++ .../gizmosql/configurations/entrypoint.sh | 51 ++++++++++++ .../gizmosql/configurations/init.sql.template | 38 +++++++++ src/development/redpanda-connect/compose.yaml | 30 ++++++++ .../configurations/config.yaml | 77 +++++++++++++++++++ src/production/gizmosql/compose.yaml | 5 ++ src/production/redpanda-connect/compose.yaml | 5 ++ 8 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 src/development/gizmosql/compose.yaml create mode 100644 src/development/gizmosql/configurations/entrypoint.sh create mode 100644 src/development/gizmosql/configurations/init.sql.template create mode 100644 src/development/redpanda-connect/compose.yaml create mode 100644 src/development/redpanda-connect/configurations/config.yaml create mode 100644 src/production/gizmosql/compose.yaml create mode 100644 src/production/redpanda-connect/compose.yaml diff --git a/artifacts/docs/README.md b/artifacts/docs/README.md index 8e68816d..ef34911f 100644 --- a/artifacts/docs/README.md +++ b/artifacts/docs/README.md @@ -16,7 +16,7 @@ Services: adminer, cloudflared, portainer, portainer-agent, postgraphile, postgr ### event-streaming -Services: debezium, debezium-postgres-connector, postgres, redpanda, redpanda-console, sqitch, traefik +Services: debezium, debezium-postgres-connector, gizmosql, postgres, redpanda, redpanda-connect, redpanda-console, sqitch, traefik ### recommendation @@ -73,6 +73,10 @@ You cannot access the search engine via a web interface. You cannot access the ip geolocator via a web interface. +### gizmosql + +You can query the event stream's parquet lakehouse using any Arrow Flight SQL client at gizmosql:31337. + ### grafana You can access the observation dashboard at [grafana.app.localhost](https://grafana.app.localhost/). @@ -142,6 +146,10 @@ You cannot access the caching system via a web interface. You can access the event streaming platform's ui as described under `redpanda-console`. +### redpanda-connect + +You can track the event stream ingestion using the pipeline's readiness endpoint. + ### redpanda-console You can access the event streaming platform's ui at [redpanda.app.localhost](https://redpanda.app.localhost/). diff --git a/src/development/gizmosql/compose.yaml b/src/development/gizmosql/compose.yaml new file mode 100644 index 00000000..6d5b3e8b --- /dev/null +++ b/src/development/gizmosql/compose.yaml @@ -0,0 +1,58 @@ +secrets: + gizmosql-password: + # The gizmosql server's password. + file: ~~/artifacts/secrets/gizmosql-password.secret + r2-account-id: + # The event stream analytics' cloud storage account identifier. + file: ~~/artifacts/secrets/r2-account-id.secret + r2-aws-credentials-access-key: + # The event stream analytics' cloud storage secret access key. + file: ~~/artifacts/secrets/r2-aws-credentials-access-key.secret + r2-aws-credentials-access-key-id: + # The event stream analytics' cloud storage access key identifier. + file: ~~/artifacts/secrets/r2-aws-credentials-access-key-id.secret +services: + gizmosql: + # You can query the event stream's parquet lakehouse using any Arrow Flight SQL client. + command: /run/entrypoint.sh + deploy: + labels: + - dargstack.profiles=event-streaming + entrypoint: sh + environment: + GIZMOSQL_USERNAME: gizmosql + INIT_SQL_COMMANDS_FILE: /tmp/gizmosql-init.sql + PRINT_QUERIES: "1" + TLS_ENABLED: "0" + healthcheck: + test: ["CMD-SHELL", "GIZMOSQL_PASSWORD=$$(cat /run/environment-variables/GIZMOSQL_PASSWORD) gizmosql_client --host 127.0.0.1 --port 31337 --username gizmosql --command 'SELECT 1'"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + image: gizmodata/gizmosql:v1.35.1 + ports: # dargstack:dev-only + - 31337:31337 # dargstack:dev-only + secrets: + - source: gizmosql-password + target: /run/environment-variables/GIZMOSQL_PASSWORD + - source: r2-account-id + target: /run/environment-variables/R2_ACCOUNT_ID + - source: r2-aws-credentials-access-key-id + target: /run/environment-variables/R2_ACCESS_KEY_ID + - source: r2-aws-credentials-access-key + target: /run/environment-variables/R2_SECRET_ACCESS_KEY + volumes: + - ./configurations/entrypoint.sh:/run/entrypoint.sh:ro + - ./configurations/init.sql.template:/run/init.sql.template:ro +x-dargstack: + secrets: + gizmosql-password: + special_characters: false + type: random_string + r2-account-id: + type: third_party + r2-aws-credentials-access-key: + type: third_party + r2-aws-credentials-access-key-id: + type: third_party diff --git a/src/development/gizmosql/configurations/entrypoint.sh b/src/development/gizmosql/configurations/entrypoint.sh new file mode 100644 index 00000000..0aff68c2 --- /dev/null +++ b/src/development/gizmosql/configurations/entrypoint.sh @@ -0,0 +1,51 @@ +#!/bin/sh +set -eu + +# START of maevsi entrypoint script customization +ENVIRONMENT_VARIABLES_PATH="/run/environment-variables" + +is_valid_var_name() { + case "$1" in + ''|[!a-zA-Z_]*|*[!a-zA-Z0-9_]*) return 1 ;; + *) return 0 ;; + esac +} + +load_env_file() { + file="$1" + name=$(basename "$file") + is_valid_var_name "$name" || return 0 + value=$(cat "$file") + export "$name=$value" +} + +load_environment_variables() { + [ -d "$ENVIRONMENT_VARIABLES_PATH" ] || return 0 + set -- "$ENVIRONMENT_VARIABLES_PATH"/* + [ -e "$1" ] || return 0 + + for file in "$ENVIRONMENT_VARIABLES_PATH"/*; do + [ -f "$file" ] && load_env_file "$file" + done +} + +load_environment_variables +# END of maevsi entrypoint script customization + +# DuckDB in this image has no getenv() function, so the R2 values are rendered +# into init.sql. sed treats `&` and `\` specially in replacements, so escape +# them first (R2 keys are hex today, but this keeps arbitrary values safe). +escape_sed_replacement() { + printf '%s\n' "$1" | sed 's/[&\\]/\\&/g' +} + +R2_ACCOUNT_ID="$(escape_sed_replacement "${R2_ACCOUNT_ID}")" +R2_ACCESS_KEY_ID="$(escape_sed_replacement "${R2_ACCESS_KEY_ID}")" +R2_SECRET_ACCESS_KEY="$(escape_sed_replacement "${R2_SECRET_ACCESS_KEY}")" + +sed -e "s|__R2_ACCOUNT_ID__|${R2_ACCOUNT_ID}|g" \ + -e "s|__R2_ACCESS_KEY_ID__|${R2_ACCESS_KEY_ID}|g" \ + -e "s|__R2_SECRET_ACCESS_KEY__|${R2_SECRET_ACCESS_KEY}|g" \ + /run/init.sql.template > /tmp/gizmosql-init.sql + +exec /opt/gizmosql/scripts/start_gizmosql.sh diff --git a/src/development/gizmosql/configurations/init.sql.template b/src/development/gizmosql/configurations/init.sql.template new file mode 100644 index 00000000..24010123 --- /dev/null +++ b/src/development/gizmosql/configurations/init.sql.template @@ -0,0 +1,38 @@ +INSTALL httpfs; +LOAD httpfs; + +CREATE SECRET r2_secret (TYPE r2, KEY_ID '__R2_ACCESS_KEY_ID__', SECRET '__R2_SECRET_ACCESS_KEY__', ACCOUNT_ID '__R2_ACCOUNT_ID__'); + +CREATE SCHEMA IF NOT EXISTS analytics; + +-- The event stream's parquet lakehouse, written by redpanda-connect to R2 +-- (bucket "test", prefix "events/"). The raw parquet columns are strings, so +-- ids and timestamps are cast to proper types (try_cast keeps bad values NULL). +CREATE OR REPLACE VIEW analytics.events AS +SELECT + try_cast(id AS UUID) AS id, + name, + slug, + try_cast(start AS TIMESTAMPTZ) AS start, + try_cast("end" AS TIMESTAMPTZ) AS "end", + visibility, + is_archived, + is_in_person, + is_remote, + language, + url, + try_cast(address_id AS UUID) AS address_id, + guest_count_maximum, + try_cast(created_at AS TIMESTAMPTZ) AS created_at, + try_cast(created_by AS UUID) AS created_by, + operation +FROM read_parquet('r2://test/events/*.parquet'); + +-- Example aggregate over the lakehouse. +CREATE OR REPLACE VIEW analytics.events_daily AS +SELECT + try_cast(created_at AS DATE) AS day, + count(*) AS events +FROM read_parquet('r2://test/events/*.parquet') +GROUP BY 1 +ORDER BY 1; diff --git a/src/development/redpanda-connect/compose.yaml b/src/development/redpanda-connect/compose.yaml new file mode 100644 index 00000000..b41007db --- /dev/null +++ b/src/development/redpanda-connect/compose.yaml @@ -0,0 +1,30 @@ +secrets: + r2-aws-credentials: + # The event stream ingestion's s3 credentials file. + file: ~~/artifacts/secrets/r2-aws-credentials.secret +services: + redpanda-connect: + # You can track the event stream's parquet ingestion using the pipeline's readiness endpoint. + deploy: + labels: + - dargstack.profiles=event-streaming + environment: + AWS_SHARED_CREDENTIALS_FILE: /run/secrets/r2-aws-credentials + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://127.0.0.1:4195/ready"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + image: redpandadata/connect:4.103.1 + secrets: + - r2-aws-credentials + volumes: + - ./configurations/config.yaml:/connect.yaml:ro +x-dargstack: + secrets: + r2-aws-credentials: + template: | + [default] + aws_access_key_id = {{secret:r2-aws-credentials-access-key-id}} + aws_secret_access_key = {{secret:r2-aws-credentials-access-key}} diff --git a/src/development/redpanda-connect/configurations/config.yaml b/src/development/redpanda-connect/configurations/config.yaml new file mode 100644 index 00000000..7ba1a8e3 --- /dev/null +++ b/src/development/redpanda-connect/configurations/config.yaml @@ -0,0 +1,77 @@ +input: + kafka_franz: + seed_brokers: ["redpanda:9092"] + topics: ["vibetype.vibetype.event"] + consumer_group: "rp_connect_r2_ingest" +pipeline: + processors: + # Tombstone messages (Debezium deletes) carry no payload and must be dropped. + - bloblang: | + root = if this.payload.after.or(this.after) == null { deleted() } + - mapping: | + root = this.payload.after.or(this.after) + root.operation = this.payload.op.or(this.op) +output: + aws_s3: + bucket: test + endpoint: ${S3_ENDPOINT_URL} + region: weur + force_path_style_urls: true + credentials: + profile: default + path: 'events/${! timestamp_unix_nano() }-${! uuid_v4() }.parquet' + batching: + count: 10000 + period: 15s + processors: + - parquet_encode: + schema: + - name: id + type: UTF8 + optional: true + - name: name + type: UTF8 + optional: true + - name: slug + type: UTF8 + optional: true + - name: start + type: UTF8 + optional: true + - name: end + type: UTF8 + optional: true + - name: visibility + type: UTF8 + optional: true + - name: is_archived + type: BOOLEAN + optional: true + - name: is_in_person + type: BOOLEAN + optional: true + - name: is_remote + type: BOOLEAN + optional: true + - name: language + type: UTF8 + optional: true + - name: url + type: UTF8 + optional: true + - name: address_id + type: UTF8 + optional: true + - name: guest_count_maximum + type: INT64 + optional: true + - name: created_at + type: UTF8 + optional: true + - name: created_by + type: UTF8 + optional: true + - name: operation + type: UTF8 + optional: true + default_compression: zstd diff --git a/src/production/gizmosql/compose.yaml b/src/production/gizmosql/compose.yaml new file mode 100644 index 00000000..0d63a3f1 --- /dev/null +++ b/src/production/gizmosql/compose.yaml @@ -0,0 +1,5 @@ +services: + gizmosql: + deploy: + update_config: + order: stop-first diff --git a/src/production/redpanda-connect/compose.yaml b/src/production/redpanda-connect/compose.yaml new file mode 100644 index 00000000..4364f20d --- /dev/null +++ b/src/production/redpanda-connect/compose.yaml @@ -0,0 +1,5 @@ +services: + redpanda-connect: + deploy: + update_config: + order: stop-first From 089157f4e837a2a2608395d3de857dd464d2aced Mon Sep 17 00:00:00 2001 From: Minh Dang Hoang Date: Sat, 15 Aug 2026 23:52:10 +0200 Subject: [PATCH 3/7] chore: rename variables --- src/development/gizmosql/compose.yaml | 34 +++++++++---------- .../gizmosql/configurations/entrypoint.sh | 16 ++++----- .../gizmosql/configurations/init.sql.template | 4 +-- src/development/redpanda-connect/compose.yaml | 14 ++++---- .../configurations/config.yaml | 2 +- 5 files changed, 34 insertions(+), 36 deletions(-) diff --git a/src/development/gizmosql/compose.yaml b/src/development/gizmosql/compose.yaml index 6d5b3e8b..661caf43 100644 --- a/src/development/gizmosql/compose.yaml +++ b/src/development/gizmosql/compose.yaml @@ -2,18 +2,18 @@ secrets: gizmosql-password: # The gizmosql server's password. file: ~~/artifacts/secrets/gizmosql-password.secret - r2-account-id: + s3-account-id: # The event stream analytics' cloud storage account identifier. - file: ~~/artifacts/secrets/r2-account-id.secret - r2-aws-credentials-access-key: + file: ~~/artifacts/secrets/s3-account-id.secret + s3-aws-credentials-access-key: # The event stream analytics' cloud storage secret access key. - file: ~~/artifacts/secrets/r2-aws-credentials-access-key.secret - r2-aws-credentials-access-key-id: + file: ~~/artifacts/secrets/s3-aws-credentials-access-key.secret + s3-aws-credentials-access-key-id: # The event stream analytics' cloud storage access key identifier. - file: ~~/artifacts/secrets/r2-aws-credentials-access-key-id.secret + file: ~~/artifacts/secrets/s3-aws-credentials-access-key-id.secret services: gizmosql: - # You can query the event stream's parquet lakehouse using any Arrow Flight SQL client. + # You can query the event stream's parquet lakehouse using Grafana or any Arrow Flight SQL client. command: /run/entrypoint.sh deploy: labels: @@ -31,17 +31,15 @@ services: retries: 3 start_period: 60s image: gizmodata/gizmosql:v1.35.1 - ports: # dargstack:dev-only - - 31337:31337 # dargstack:dev-only secrets: - source: gizmosql-password target: /run/environment-variables/GIZMOSQL_PASSWORD - - source: r2-account-id - target: /run/environment-variables/R2_ACCOUNT_ID - - source: r2-aws-credentials-access-key-id - target: /run/environment-variables/R2_ACCESS_KEY_ID - - source: r2-aws-credentials-access-key - target: /run/environment-variables/R2_SECRET_ACCESS_KEY + - source: s3-account-id + target: /run/environment-variables/S3_ACCOUNT_ID + - source: s3-aws-credentials-access-key-id + target: /run/environment-variables/S3_ACCESS_KEY_ID + - source: s3-aws-credentials-access-key + target: /run/environment-variables/S3_SECRET_ACCESS_KEY volumes: - ./configurations/entrypoint.sh:/run/entrypoint.sh:ro - ./configurations/init.sql.template:/run/init.sql.template:ro @@ -50,9 +48,9 @@ x-dargstack: gizmosql-password: special_characters: false type: random_string - r2-account-id: + s3-account-id: type: third_party - r2-aws-credentials-access-key: + s3-aws-credentials-access-key: type: third_party - r2-aws-credentials-access-key-id: + s3-aws-credentials-access-key-id: type: third_party diff --git a/src/development/gizmosql/configurations/entrypoint.sh b/src/development/gizmosql/configurations/entrypoint.sh index 0aff68c2..b19cef3f 100644 --- a/src/development/gizmosql/configurations/entrypoint.sh +++ b/src/development/gizmosql/configurations/entrypoint.sh @@ -32,20 +32,20 @@ load_environment_variables() { load_environment_variables # END of maevsi entrypoint script customization -# DuckDB in this image has no getenv() function, so the R2 values are rendered +# DuckDB in this image has no getenv() function, so the S3 values are rendered # into init.sql. sed treats `&` and `\` specially in replacements, so escape -# them first (R2 keys are hex today, but this keeps arbitrary values safe). +# them first (S3 keys are hex today, but this keeps arbitrary values safe). escape_sed_replacement() { printf '%s\n' "$1" | sed 's/[&\\]/\\&/g' } -R2_ACCOUNT_ID="$(escape_sed_replacement "${R2_ACCOUNT_ID}")" -R2_ACCESS_KEY_ID="$(escape_sed_replacement "${R2_ACCESS_KEY_ID}")" -R2_SECRET_ACCESS_KEY="$(escape_sed_replacement "${R2_SECRET_ACCESS_KEY}")" +S3_ACCOUNT_ID="$(escape_sed_replacement "${S3_ACCOUNT_ID}")" +S3_ACCESS_KEY_ID="$(escape_sed_replacement "${S3_ACCESS_KEY_ID}")" +S3_SECRET_ACCESS_KEY="$(escape_sed_replacement "${S3_SECRET_ACCESS_KEY}")" -sed -e "s|__R2_ACCOUNT_ID__|${R2_ACCOUNT_ID}|g" \ - -e "s|__R2_ACCESS_KEY_ID__|${R2_ACCESS_KEY_ID}|g" \ - -e "s|__R2_SECRET_ACCESS_KEY__|${R2_SECRET_ACCESS_KEY}|g" \ +sed -e "s|__S3_ACCOUNT_ID__|${S3_ACCOUNT_ID}|g" \ + -e "s|__S3_ACCESS_KEY_ID__|${S3_ACCESS_KEY_ID}|g" \ + -e "s|__S3_SECRET_ACCESS_KEY__|${S3_SECRET_ACCESS_KEY}|g" \ /run/init.sql.template > /tmp/gizmosql-init.sql exec /opt/gizmosql/scripts/start_gizmosql.sh diff --git a/src/development/gizmosql/configurations/init.sql.template b/src/development/gizmosql/configurations/init.sql.template index 24010123..674cd9b8 100644 --- a/src/development/gizmosql/configurations/init.sql.template +++ b/src/development/gizmosql/configurations/init.sql.template @@ -1,11 +1,11 @@ INSTALL httpfs; LOAD httpfs; -CREATE SECRET r2_secret (TYPE r2, KEY_ID '__R2_ACCESS_KEY_ID__', SECRET '__R2_SECRET_ACCESS_KEY__', ACCOUNT_ID '__R2_ACCOUNT_ID__'); +CREATE SECRET s3_secret (TYPE r2, KEY_ID '__S3_ACCESS_KEY_ID__', SECRET '__S3_SECRET_ACCESS_KEY__', ACCOUNT_ID '__S3_ACCOUNT_ID__'); CREATE SCHEMA IF NOT EXISTS analytics; --- The event stream's parquet lakehouse, written by redpanda-connect to R2 +-- The event stream's parquet lakehouse, written by redpanda-connect to S3 -- (bucket "test", prefix "events/"). The raw parquet columns are strings, so -- ids and timestamps are cast to proper types (try_cast keeps bad values NULL). CREATE OR REPLACE VIEW analytics.events AS diff --git a/src/development/redpanda-connect/compose.yaml b/src/development/redpanda-connect/compose.yaml index b41007db..a54594da 100644 --- a/src/development/redpanda-connect/compose.yaml +++ b/src/development/redpanda-connect/compose.yaml @@ -1,7 +1,7 @@ secrets: - r2-aws-credentials: + s3-aws-credentials: # The event stream ingestion's s3 credentials file. - file: ~~/artifacts/secrets/r2-aws-credentials.secret + file: ~~/artifacts/secrets/s3-aws-credentials.secret services: redpanda-connect: # You can track the event stream's parquet ingestion using the pipeline's readiness endpoint. @@ -9,7 +9,7 @@ services: labels: - dargstack.profiles=event-streaming environment: - AWS_SHARED_CREDENTIALS_FILE: /run/secrets/r2-aws-credentials + AWS_SHARED_CREDENTIALS_FILE: /run/secrets/s3-aws-credentials healthcheck: test: ["CMD-SHELL", "wget -q --spider http://127.0.0.1:4195/ready"] interval: 30s @@ -18,13 +18,13 @@ services: start_period: 30s image: redpandadata/connect:4.103.1 secrets: - - r2-aws-credentials + - s3-aws-credentials volumes: - ./configurations/config.yaml:/connect.yaml:ro x-dargstack: secrets: - r2-aws-credentials: + s3-aws-credentials: template: | [default] - aws_access_key_id = {{secret:r2-aws-credentials-access-key-id}} - aws_secret_access_key = {{secret:r2-aws-credentials-access-key}} + aws_access_key_id = {{secret:s3-aws-credentials-access-key-id}} + aws_secret_access_key = {{secret:s3-aws-credentials-access-key}} diff --git a/src/development/redpanda-connect/configurations/config.yaml b/src/development/redpanda-connect/configurations/config.yaml index 7ba1a8e3..afe99800 100644 --- a/src/development/redpanda-connect/configurations/config.yaml +++ b/src/development/redpanda-connect/configurations/config.yaml @@ -2,7 +2,7 @@ input: kafka_franz: seed_brokers: ["redpanda:9092"] topics: ["vibetype.vibetype.event"] - consumer_group: "rp_connect_r2_ingest" + consumer_group: "rp_connect_s3_ingest" pipeline: processors: # Tombstone messages (Debezium deletes) carry no payload and must be dropped. From e988bc46285743b04ef8d7e7fe1d103f87e91986 Mon Sep 17 00:00:00 2001 From: Minh Dang Hoang Date: Sun, 16 Aug 2026 00:23:17 +0200 Subject: [PATCH 4/7] chore: add environment variable placeholder --- src/development/.env.template | 1 + 1 file changed, 1 insertion(+) diff --git a/src/development/.env.template b/src/development/.env.template index 546de5a7..45277401 100644 --- a/src/development/.env.template +++ b/src/development/.env.template @@ -1,4 +1,5 @@ RECCOOM_ADMIN_ACCOUNT_IDS= +S3_ENDPOINT_URL= TUSD_BUCKET=vibetype-images TUSD_ENDPOINT=http://minio:9000 TUSD_MAX_SIZE=10485760 From 8d4b6bc686fd3aae803a929d47c15b596519a33f Mon Sep 17 00:00:00 2001 From: Minh Dang Hoang Date: Sun, 16 Aug 2026 16:03:27 +0200 Subject: [PATCH 5/7] feat(gizmo): hook to grafana --- src/development/grafana/compose.yaml | 2 ++ .../provisioning/datasources/gizmosql.yaml | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 src/development/grafana/configurations/provisioning/datasources/gizmosql.yaml diff --git a/src/development/grafana/compose.yaml b/src/development/grafana/compose.yaml index fe445f9b..34023659 100644 --- a/src/development/grafana/compose.yaml +++ b/src/development/grafana/compose.yaml @@ -30,6 +30,7 @@ services: GF_DATABASE_PASSWORD__FILE: /run/secrets/postgres-role-service-grafana-password GF_DATABASE_TYPE: postgres GF_DATABASE_USER__FILE: /run/secrets/postgres-role-service-grafana-username + GF_INSTALL_PLUGINS: gizmodata-gizmosql-datasource GF_SECURITY_ADMIN_EMAIL__FILE: /run/secrets/grafana-admin-email GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana-admin-password GF_SECURITY_ADMIN_USER__FILE: /run/secrets/grafana-admin-user @@ -46,6 +47,7 @@ services: - grafana-admin-password - grafana-admin-user - grafana-discord-webhook + - gizmosql-password - postgres-db - postgres-role-service-grafana-password - postgres-role-service-grafana-username diff --git a/src/development/grafana/configurations/provisioning/datasources/gizmosql.yaml b/src/development/grafana/configurations/provisioning/datasources/gizmosql.yaml new file mode 100644 index 00000000..6e8c8086 --- /dev/null +++ b/src/development/grafana/configurations/provisioning/datasources/gizmosql.yaml @@ -0,0 +1,14 @@ +apiVersion: 1 + +datasources: + - access: proxy + editable: true + jsonData: + host: gizmosql + port: 31337 + useTLS: false + username: gizmosql + name: GizmoSQL + secureJsonData: + password: $__file{/run/secrets/gizmosql-password} + type: gizmodata-gizmosql-datasource From 6d9dc9f7064ad6237e7bd2d4b7375cd24d108407 Mon Sep 17 00:00:00 2001 From: Minh Dang Hoang Date: Sun, 16 Aug 2026 16:37:06 +0200 Subject: [PATCH 6/7] chore(docs): update docs and comments --- artifacts/docs/README.md | 4 ++-- src/development/gizmosql/compose.yaml | 2 +- src/development/redpanda-connect/compose.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/artifacts/docs/README.md b/artifacts/docs/README.md index ef34911f..3eb51bfd 100644 --- a/artifacts/docs/README.md +++ b/artifacts/docs/README.md @@ -75,7 +75,7 @@ You cannot access the ip geolocator via a web interface. ### gizmosql -You can query the event stream's parquet lakehouse using any Arrow Flight SQL client at gizmosql:31337. +You can query the event stream's parquet lakehouse using `grafana`. ### grafana @@ -148,7 +148,7 @@ You can access the event streaming platform's ui as described under `redpanda-co ### redpanda-connect -You can track the event stream ingestion using the pipeline's readiness endpoint. +You cannot access the event stream connector directly. ### redpanda-console diff --git a/src/development/gizmosql/compose.yaml b/src/development/gizmosql/compose.yaml index 661caf43..be687bf9 100644 --- a/src/development/gizmosql/compose.yaml +++ b/src/development/gizmosql/compose.yaml @@ -13,7 +13,7 @@ secrets: file: ~~/artifacts/secrets/s3-aws-credentials-access-key-id.secret services: gizmosql: - # You can query the event stream's parquet lakehouse using Grafana or any Arrow Flight SQL client. + # You can query the event stream's parquet lakehouse using `grafana`. command: /run/entrypoint.sh deploy: labels: diff --git a/src/development/redpanda-connect/compose.yaml b/src/development/redpanda-connect/compose.yaml index a54594da..4eee4bf0 100644 --- a/src/development/redpanda-connect/compose.yaml +++ b/src/development/redpanda-connect/compose.yaml @@ -4,7 +4,7 @@ secrets: file: ~~/artifacts/secrets/s3-aws-credentials.secret services: redpanda-connect: - # You can track the event stream's parquet ingestion using the pipeline's readiness endpoint. + # You cannot access the event stream connector directly. deploy: labels: - dargstack.profiles=event-streaming From 54fd4ee139173ed1678d9c2d01e6bbe1c962bf94 Mon Sep 17 00:00:00 2001 From: Minh Dang Hoang Date: Sun, 16 Aug 2026 19:16:25 +0200 Subject: [PATCH 7/7] chore: migrate secrets --- src/development/compose.yaml | 18 ++++++++++++++++++ src/development/gizmosql/compose.yaml | 15 --------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/development/compose.yaml b/src/development/compose.yaml index f5dcf3b3..335bc9c3 100644 --- a/src/development/compose.yaml +++ b/src/development/compose.yaml @@ -2,3 +2,21 @@ volumes: pnpm-data: # The node package manager's data. {} +secrets: + s3-account-id: + # The event stream analytics' cloud storage account identifier. + file: ~~/artifacts/secrets/s3-account-id.secret + s3-aws-credentials-access-key: + # The event stream analytics' cloud storage secret access key. + file: ~~/artifacts/secrets/s3-aws-credentials-access-key.secret + s3-aws-credentials-access-key-id: + # The event stream analytics' cloud storage access key identifier. + file: ~~/artifacts/secrets/s3-aws-credentials-access-key-id.secret +x-dargstack: + secrets: + s3-account-id: + type: third_party + s3-aws-credentials-access-key: + type: third_party + s3-aws-credentials-access-key-id: + type: third_party diff --git a/src/development/gizmosql/compose.yaml b/src/development/gizmosql/compose.yaml index be687bf9..0716b527 100644 --- a/src/development/gizmosql/compose.yaml +++ b/src/development/gizmosql/compose.yaml @@ -2,15 +2,6 @@ secrets: gizmosql-password: # The gizmosql server's password. file: ~~/artifacts/secrets/gizmosql-password.secret - s3-account-id: - # The event stream analytics' cloud storage account identifier. - file: ~~/artifacts/secrets/s3-account-id.secret - s3-aws-credentials-access-key: - # The event stream analytics' cloud storage secret access key. - file: ~~/artifacts/secrets/s3-aws-credentials-access-key.secret - s3-aws-credentials-access-key-id: - # The event stream analytics' cloud storage access key identifier. - file: ~~/artifacts/secrets/s3-aws-credentials-access-key-id.secret services: gizmosql: # You can query the event stream's parquet lakehouse using `grafana`. @@ -48,9 +39,3 @@ x-dargstack: gizmosql-password: special_characters: false type: random_string - s3-account-id: - type: third_party - s3-aws-credentials-access-key: - type: third_party - s3-aws-credentials-access-key-id: - type: third_party