From bdc175a50f34e17acce1eb58fb70aa1b29621dd9 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 29 Sep 2026 10:58:45 +0200 Subject: [PATCH 1/3] test(s3): replace MinIO with Garage MinIO no longer has images available. Note: Garage doesn't natively support TLS, and therefore needs a sidecar proxy to handle that (we could explore using tlsproxy, but nginx was a known quantity). --- .../kuttl/iceberg-hive/01_s3-connection.yaml | 12 +- .../kuttl/iceberg-hive/20-assert.yaml | 2 +- ...tall-minio.yaml => 20-install-garage.yaml} | 2 +- .../kuttl/iceberg-hive/20_garage.yaml | 159 +++++ .../kuttl/iceberg-hive/20_minio.yaml | 579 ------------------ .../kuttl/iceberg-hive/21-assert.yaml | 11 - .../iceberg-hive/21-install-minio-jobs.yaml | 5 - .../kuttl/iceberg-hive/21_minio_jobs.yaml | 116 ---- .../kuttl/iceberg-hive/33_hive.yaml.j2 | 2 +- .../kuttl/iceberg-hive/34_trino.yaml.j2 | 2 +- .../kuttl/iceberg-hive/50_nifi.yaml.j2 | 2 +- .../60_nifi-flow-with-kerberos.json | 6 +- .../60_nifi-flow-without-kerberos.json | 6 +- tests/templates/kuttl/iceberg-hive/README.md | 10 +- .../kuttl/iceberg-rest/01_s3-connection.yaml | 12 +- .../kuttl/iceberg-rest/20-assert.yaml | 2 +- ...tall-minio.yaml => 20-install-garage.yaml} | 2 +- .../kuttl/iceberg-rest/20_garage.yaml | 157 +++++ .../kuttl/iceberg-rest/20_minio.yaml | 579 ------------------ .../kuttl/iceberg-rest/21-assert.yaml | 11 - .../iceberg-rest/21-install-minio-jobs.yaml | 5 - .../kuttl/iceberg-rest/21_minio_jobs.yaml | 116 ---- .../kuttl/iceberg-rest/30_hive.yaml.j2 | 2 +- .../kuttl/iceberg-rest/31_trino.yaml.j2 | 2 +- .../kuttl/iceberg-rest/50_nifi.yaml.j2 | 2 +- .../kuttl/iceberg-rest/60_nifi-flow.json | 8 +- tests/templates/kuttl/iceberg-rest/README.md | 13 +- 27 files changed, 362 insertions(+), 1463 deletions(-) rename tests/templates/kuttl/iceberg-hive/{20-install-minio.yaml => 20-install-garage.yaml} (50%) create mode 100644 tests/templates/kuttl/iceberg-hive/20_garage.yaml delete mode 100644 tests/templates/kuttl/iceberg-hive/20_minio.yaml delete mode 100644 tests/templates/kuttl/iceberg-hive/21-assert.yaml delete mode 100644 tests/templates/kuttl/iceberg-hive/21-install-minio-jobs.yaml delete mode 100644 tests/templates/kuttl/iceberg-hive/21_minio_jobs.yaml rename tests/templates/kuttl/iceberg-rest/{20-install-minio.yaml => 20-install-garage.yaml} (50%) create mode 100644 tests/templates/kuttl/iceberg-rest/20_garage.yaml delete mode 100644 tests/templates/kuttl/iceberg-rest/20_minio.yaml delete mode 100644 tests/templates/kuttl/iceberg-rest/21-assert.yaml delete mode 100644 tests/templates/kuttl/iceberg-rest/21-install-minio-jobs.yaml delete mode 100644 tests/templates/kuttl/iceberg-rest/21_minio_jobs.yaml diff --git a/tests/templates/kuttl/iceberg-hive/01_s3-connection.yaml b/tests/templates/kuttl/iceberg-hive/01_s3-connection.yaml index 56c30d36..eeec9540 100644 --- a/tests/templates/kuttl/iceberg-hive/01_s3-connection.yaml +++ b/tests/templates/kuttl/iceberg-hive/01_s3-connection.yaml @@ -2,9 +2,9 @@ apiVersion: s3.stackable.tech/v1alpha1 kind: S3Connection metadata: - name: minio + name: garage spec: - host: "minio.${NAMESPACE}.svc.cluster.local" + host: "garage.${NAMESPACE}.svc.cluster.local" port: 9000 accessStyle: Path credentials: @@ -28,9 +28,11 @@ spec: apiVersion: v1 kind: Secret metadata: - name: minio-credentials + name: garage-credentials labels: secrets.stackable.tech/class: s3-credentials-class stringData: - accessKey: admin - secretKey: adminadmin + # Garage requires the GK<24 hex> key id format and a 64 hex char secret. + # Must match the GARAGE_DEFAULT_* variables in 20_garage.yaml. + accessKey: GK31c0ffee31c0ffee31c0ffee + secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef diff --git a/tests/templates/kuttl/iceberg-hive/20-assert.yaml b/tests/templates/kuttl/iceberg-hive/20-assert.yaml index 477bdd02..20a8dae0 100644 --- a/tests/templates/kuttl/iceberg-hive/20-assert.yaml +++ b/tests/templates/kuttl/iceberg-hive/20-assert.yaml @@ -6,7 +6,7 @@ timeout: 600 apiVersion: apps/v1 kind: Deployment metadata: - name: minio + name: garage status: readyReplicas: 1 replicas: 1 diff --git a/tests/templates/kuttl/iceberg-hive/20-install-minio.yaml b/tests/templates/kuttl/iceberg-hive/20-install-garage.yaml similarity index 50% rename from tests/templates/kuttl/iceberg-hive/20-install-minio.yaml rename to tests/templates/kuttl/iceberg-hive/20-install-garage.yaml index 985b51e8..0b085871 100644 --- a/tests/templates/kuttl/iceberg-hive/20-install-minio.yaml +++ b/tests/templates/kuttl/iceberg-hive/20-install-garage.yaml @@ -2,4 +2,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl -n $NAMESPACE apply -f 20_minio.yaml + - script: kubectl -n $NAMESPACE apply -f 20_garage.yaml diff --git a/tests/templates/kuttl/iceberg-hive/20_garage.yaml b/tests/templates/kuttl/iceberg-hive/20_garage.yaml new file mode 100644 index 00000000..7256c920 --- /dev/null +++ b/tests/templates/kuttl/iceberg-hive/20_garage.yaml @@ -0,0 +1,159 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: garage +data: + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ + garage.toml: | + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + db_engine = "lmdb" + + # Single node, no redundancy. Test data is disposable (emptyDir). + replication_factor = 1 + + # Throwaway value, this cluster is never joined by another node. + rpc_secret = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "127.0.0.1:3901" + + [s3_api] + # Garage rejects requests signed for a different region, so this must match + # the region the clients use. This test goes through Hadoop's s3a, which + # signs for us-east-2 when fs.s3a.endpoint.region is unset and the endpoint + # is not an AWS one. MinIO ignored the region, Garage does not. + s3_region = "us-east-2" + api_bind_addr = "[::]:3900" + + [admin] + api_bind_addr = "[::]:3903" + # Garage terminates no TLS on any endpoint, so nginx does it and forwards to + # Garage on loopback. Mounted over /etc/nginx/nginx.conf, hence the full file. + nginx.conf: | + events {} + http { + server { + listen 9000 ssl; + ssl_certificate /stackable/tls/tls.crt; + ssl_certificate_key /stackable/tls/tls.key; + + # Don't buffer or size-limit object uploads. + client_max_body_size 0; + + location / { + proxy_pass http://127.0.0.1:3900; + # Must be the original Host, it is part of the SigV4 signature. + proxy_set_header Host $http_host; + proxy_http_version 1.1; + proxy_buffering off; + proxy_request_buffering off; + } + } + } +--- +apiVersion: v1 +kind: Service +metadata: + name: garage +spec: + selector: + app: garage + ports: + - name: https + port: 9000 + targetPort: https +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: garage + labels: + app: garage +spec: + replicas: 1 + selector: + matchLabels: + app: garage + template: + metadata: + labels: + app: garage + spec: + containers: + - name: garage + image: oci.stackable.tech/stackable/dxflrs/garage:v2.4.1 + imagePullPolicy: IfNotPresent + # The image has no entrypoint. --single-node creates the cluster + # layout, --default-bucket creates the bucket and the access key from + # the GARAGE_DEFAULT_* variables below (implies --default-access-key). + command: + - /garage + - server + - --single-node + - --default-bucket + env: + - name: GARAGE_DEFAULT_BUCKET + value: demo + # Garage requires the GK<24 hex> key id format and a 64 hex char + # secret. These are also set in 01_s3-connection.yaml and in the + # NiFi flow (60_nifi-flow.json). + - name: GARAGE_DEFAULT_ACCESS_KEY + value: GK31c0ffee31c0ffee31c0ffee + - name: GARAGE_DEFAULT_SECRET_KEY + value: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef + ports: + - name: rpc + containerPort: 3901 + - name: admin + containerPort: 3903 + readinessProbe: + httpGet: + path: /health + port: admin + volumeMounts: + - name: config + mountPath: /etc/garage.toml + subPath: garage.toml + - name: data + mountPath: /var/lib/garage + resources: + requests: + cpu: 500m + memory: 512Mi + - name: nginx + image: docker.io/library/nginx:1.29-alpine + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 9000 + volumeMounts: + - name: config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: tls + mountPath: /stackable/tls + resources: + requests: + cpu: 100m + memory: 128Mi + volumes: + - name: config + configMap: + name: garage + - name: data + emptyDir: {} + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=garage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech diff --git a/tests/templates/kuttl/iceberg-hive/20_minio.yaml b/tests/templates/kuttl/iceberg-hive/20_minio.yaml deleted file mode 100644 index 3b9ffcbe..00000000 --- a/tests/templates/kuttl/iceberg-hive/20_minio.yaml +++ /dev/null @@ -1,579 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "minio-sa" ---- -apiVersion: v1 -kind: Secret -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -type: Opaque -data: - rootUser: "YWRtaW4=" - rootPassword: "YWRtaW5hZG1pbg==" ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -data: - initialize: |- - #!/bin/sh - set -e # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 - LIMIT=29 # Allow 30 attempts - set -e # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) - SECRET=$(cat /config/rootPassword) - set +e # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" - $MC_COMMAND - STATUS=$? - until [ $STATUS = 0 ]; do - ATTEMPTS=$(expr $ATTEMPTS + 1) - echo \"Failed attempts: $ATTEMPTS\" - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 - fi - sleep 2 # 1 second intervals between attempts - $MC_COMMAND - STATUS=$? - done - set -e # reset `e` as active - return 0 - } - - # checkBucketExists ($bucket) - # Check if the bucket exists, by using the exit code of `mc ls` - checkBucketExists() { - BUCKET=$1 - CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) - return $? - } - - # createBucket ($bucket, $policy, $purge) - # Ensure bucket exists, purging if asked to - createBucket() { - BUCKET=$1 - POLICY=$2 - PURGE=$3 - VERSIONING=$4 - OBJECTLOCKING=$5 - - # Purge the bucket, if set & exists - # Since PURGE is user input, check explicitly for `true` - if [ $PURGE = true ]; then - if checkBucketExists $BUCKET; then - echo "Purging bucket '$BUCKET'." - set +e # don't exit if this fails - ${MC} rm -r --force myminio/$BUCKET - set -e # reset `e` as active - else - echo "Bucket '$BUCKET' does not exist, skipping purge." - fi - fi - - # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) - if ! checkBucketExists $BUCKET; then - if [ ! -z $OBJECTLOCKING ]; then - if [ $OBJECTLOCKING = true ]; then - echo "Creating bucket with OBJECTLOCKING '$BUCKET'" - ${MC} mb --with-lock myminio/$BUCKET - elif [ $OBJECTLOCKING = false ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - fi - elif [ -z $OBJECTLOCKING ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - else - echo "Bucket '$BUCKET' already exists." - fi - fi - - # set versioning for bucket if objectlocking is disabled or not set - if [ $OBJECTLOCKING = false ]; then - if [ ! -z $VERSIONING ]; then - if [ $VERSIONING = true ]; then - echo "Enabling versioning for '$BUCKET'" - ${MC} version enable myminio/$BUCKET - elif [ $VERSIONING = false ]; then - echo "Suspending versioning for '$BUCKET'" - ${MC} version suspend myminio/$BUCKET - fi - fi - else - echo "Bucket '$BUCKET' versioning unchanged." - fi - - # At this point, the bucket should exist, skip checking for existence - # Set policy on the bucket - echo "Setting policy of bucket '$BUCKET' to '$POLICY'." - ${MC} anonymous set $POLICY myminio/$BUCKET - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - # Create the buckets - createBucket demo "public" false false false - - add-user: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkUserExists () - # Check if the user exists, by using the exit code of `mc admin user info` - checkUserExists() { - CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createUser ($policy) - createUser() { - POLICY=$1 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the user if it does not exist - if ! checkUserExists ; then - echo "Creating user '$USER'" - cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio - else - echo "User '$USER' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - - # set policy for user - if [ ! -z $POLICY -a $POLICY != " " ] ; then - echo "Adding policy '$POLICY' for '$USER'" - set +e ; # policy already attach errors out, allow it. - ${MC} admin policy attach myminio $POLICY --user=$USER - set -e - else - echo "User '$USER' has no policy attached." - fi - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - # Create the users - echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP - echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP - createUser consoleAdmin - - add-policy: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkPolicyExists ($policy) - # Check if the policy exists, by using the exit code of `mc admin policy info` - checkPolicyExists() { - POLICY=$1 - CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) - return $? - } - - # createPolicy($name, $filename) - createPolicy () { - NAME=$1 - FILENAME=$2 - - # Create the name if it does not exist - echo "Checking policy: $NAME (in /config/$FILENAME.json)" - if ! checkPolicyExists $NAME ; then - echo "Creating policy '$NAME'" - else - echo "Policy '$NAME' already exists." - fi - ${MC} admin policy create myminio $NAME /config/$FILENAME.json - - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - add-svcacct: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 2 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkSvcacctExists () - # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` - checkSvcacctExists() { - CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createSvcacct ($user) - createSvcacct () { - USER=$1 - FILENAME=$2 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the svcacct if it does not exist - if ! checkSvcacctExists ; then - echo "Creating svcacct '$SVCACCT'" - # Check if policy file is define - if [ -z $FILENAME ]; then - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER - else - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER - fi - else - echo "Svcacct '$SVCACCT' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - custom-command: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # runCommand ($@) - # Run custom mc command - runCommand() { - ${MC} "$@" - return $? - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: "10Gi" ---- -apiVersion: v1 -kind: Service -metadata: - name: minio-console - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9001 - protocol: TCP - targetPort: 9001 - selector: - app: minio - release: minio ---- -apiVersion: v1 -kind: Service -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - monitoring: "true" -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - app: minio - release: minio ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - stackable.tech/vendor: Stackable -spec: - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 - replicas: 1 - selector: - matchLabels: - app: minio - release: minio - template: - metadata: - name: minio - labels: - app: minio - release: minio - stackable.tech/vendor: Stackable - annotations: - checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 - checksum/config: ebea49cc4c1bfbd1b156a58bf770a776ff87fe199f642d31c2816b5515112e72 - spec: - securityContext: - fsGroupChangePolicy: OnRootMismatch - serviceAccountName: minio-sa - containers: - - name: minio - image: "quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # minio requires the TLS key pair to be specially named - # mkdir -p /etc/minio/certs - cp -v /etc/minio/original_certs/tls.crt /etc/minio/certs/public.crt - cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key - - /usr/bin/docker-entrypoint.sh minio server /export -S /etc/minio/certs/ --address :9000 --console-address :9001 - volumeMounts: - - name: minio-user - mountPath: "/tmp/credentials" - readOnly: true - - name: export - mountPath: /export - - mountPath: /etc/minio/original_certs - name: tls - - mountPath: /etc/minio/certs - name: certs - ports: - - name: https - containerPort: 9000 - - name: https-console - containerPort: 9001 - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: minio - key: rootUser - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio - key: rootPassword - - name: MINIO_PROMETHEUS_AUTH_TYPE - value: "public" - resources: - requests: - cpu: 1 - memory: 2Gi - securityContext: - readOnlyRootFilesystem: false - volumes: - - name: export - persistentVolumeClaim: - claimName: minio - - name: minio-user - secret: - secretName: minio - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs diff --git a/tests/templates/kuttl/iceberg-hive/21-assert.yaml b/tests/templates/kuttl/iceberg-hive/21-assert.yaml deleted file mode 100644 index 3895aff4..00000000 --- a/tests/templates/kuttl/iceberg-hive/21-assert.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 600 ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-post-job -status: - succeeded: 1 diff --git a/tests/templates/kuttl/iceberg-hive/21-install-minio-jobs.yaml b/tests/templates/kuttl/iceberg-hive/21-install-minio-jobs.yaml deleted file mode 100644 index d51dae4b..00000000 --- a/tests/templates/kuttl/iceberg-hive/21-install-minio-jobs.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: kubectl -n $NAMESPACE apply -f 21_minio_jobs.yaml diff --git a/tests/templates/kuttl/iceberg-hive/21_minio_jobs.yaml b/tests/templates/kuttl/iceberg-hive/21_minio_jobs.yaml deleted file mode 100644 index bd8f3ac4..00000000 --- a/tests/templates/kuttl/iceberg-hive/21_minio_jobs.yaml +++ /dev/null @@ -1,116 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-post-job - labels: - app: minio-post-job - chart: minio-5.4.0 - release: minio - heritage: Helm - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation -spec: - template: - metadata: - labels: - app: minio-job - release: minio - stackable.tech/vendor: Stackable - spec: - restartPolicy: OnFailure - volumes: - - name: etc-path - emptyDir: {} - - name: tmp - emptyDir: {} - - name: minio-configuration - projected: - sources: - - configMap: - name: minio - - secret: - name: minio - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs - serviceAccountName: minio-sa - containers: - - name: minio-make-bucket - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - - . /config/initialize - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi - - name: minio-make-user - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - - . /config/add-user - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi diff --git a/tests/templates/kuttl/iceberg-hive/33_hive.yaml.j2 b/tests/templates/kuttl/iceberg-hive/33_hive.yaml.j2 index df035847..93d7a526 100644 --- a/tests/templates/kuttl/iceberg-hive/33_hive.yaml.j2 +++ b/tests/templates/kuttl/iceberg-hive/33_hive.yaml.j2 @@ -21,7 +21,7 @@ spec: hdfs: configMap: hdfs s3: - reference: minio + reference: garage {% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} authentication: kerberos: diff --git a/tests/templates/kuttl/iceberg-hive/34_trino.yaml.j2 b/tests/templates/kuttl/iceberg-hive/34_trino.yaml.j2 index 949d9d4c..d6ccea82 100644 --- a/tests/templates/kuttl/iceberg-hive/34_trino.yaml.j2 +++ b/tests/templates/kuttl/iceberg-hive/34_trino.yaml.j2 @@ -11,7 +11,7 @@ spec: metastore: configMap: hive s3: - reference: minio + reference: garage hdfs: configMap: hdfs {% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} diff --git a/tests/templates/kuttl/iceberg-hive/50_nifi.yaml.j2 b/tests/templates/kuttl/iceberg-hive/50_nifi.yaml.j2 index 6eb1ec70..853b5e21 100644 --- a/tests/templates/kuttl/iceberg-hive/50_nifi.yaml.j2 +++ b/tests/templates/kuttl/iceberg-hive/50_nifi.yaml.j2 @@ -58,7 +58,7 @@ spec: {% endif %} jvmArgumentOverrides: add: - # Needed for NiFi to trust the minio cert + # Needed for NiFi to trust the Garage cert - -Djavax.net.ssl.trustStore=/stackable/keystore/truststore.p12 - -Djavax.net.ssl.trustStorePassword=secret - -Djavax.net.ssl.trustStoreType=PKCS12 diff --git a/tests/templates/kuttl/iceberg-hive/60_nifi-flow-with-kerberos.json b/tests/templates/kuttl/iceberg-hive/60_nifi-flow-with-kerberos.json index bf689ddd..ee43a13a 100644 --- a/tests/templates/kuttl/iceberg-hive/60_nifi-flow-with-kerberos.json +++ b/tests/templates/kuttl/iceberg-hive/60_nifi-flow-with-kerberos.json @@ -269,8 +269,8 @@ "default-credentials": "false", "profile-name": null, "Session Time": "3600", - "Access Key": "admin", - "Secret Key": "adminadmin" + "Access Key": "GK31c0ffee31c0ffee31c0ffee", + "Secret Key": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" }, "propertyDescriptors": { "Access Key": { @@ -654,7 +654,7 @@ "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml,/stackable/userdata/hive-config/hive-site.xml", "hive-metastore-uri": "thrift://hive-metastore:9083", - "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", + "s3-endpoint": "https://garage.${NAMESPACE}.svc.cluster.local:9000", "s3-path-style-access": "true", "warehouse-location": "s3a://demo/lakehouse" }, diff --git a/tests/templates/kuttl/iceberg-hive/60_nifi-flow-without-kerberos.json b/tests/templates/kuttl/iceberg-hive/60_nifi-flow-without-kerberos.json index 1a3d6c88..0aafbd49 100644 --- a/tests/templates/kuttl/iceberg-hive/60_nifi-flow-without-kerberos.json +++ b/tests/templates/kuttl/iceberg-hive/60_nifi-flow-without-kerberos.json @@ -269,8 +269,8 @@ "default-credentials": "false", "profile-name": null, "Session Time": "3600", - "Access Key": "admin", - "Secret Key": "adminadmin" + "Access Key": "GK31c0ffee31c0ffee31c0ffee", + "Secret Key": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" }, "propertyDescriptors": { "Access Key": { @@ -594,7 +594,7 @@ "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml", "hive-metastore-uri": "thrift://hive-metastore:9083", - "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", + "s3-endpoint": "https://garage.${NAMESPACE}.svc.cluster.local:9000", "s3-path-style-access": "true", "warehouse-location": "s3a://demo/lakehouse" }, diff --git a/tests/templates/kuttl/iceberg-hive/README.md b/tests/templates/kuttl/iceberg-hive/README.md index ce884d9e..97ce79ee 100644 --- a/tests/templates/kuttl/iceberg-hive/README.md +++ b/tests/templates/kuttl/iceberg-hive/README.md @@ -1,6 +1,6 @@ The file `60_nifi-flow.json` was exported from the NiFi UI. -*However*, we need to update some stuff, such as adding S3 credentials and templating the namespace of MinIO. +*However*, we need to update some stuff, such as adding S3 credentials and templating the namespace of Garage. TIP: I used `JSON: Sort Document` in VScode to somewhat have consistent formatting, which makes reading and diffs easier. @@ -17,8 +17,8 @@ index 09783fa..23c679f 100644 "profile-name": null, - "Session Time": "3600" + "Session Time": "3600", -+ "Access Key": "admin", -+ "Secret Key": "adminadmin" ++ "Access Key": "GK31c0ffee31c0ffee31c0ffee", ++ "Secret Key": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" }, "propertyDescriptors": { "Access Key": { @@ -26,8 +26,8 @@ index 09783fa..23c679f 100644 "properties": { "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", "hive-metastore-uri": "thrift://hive-metastore:9083", -- "s3-endpoint": "https://minio.kuttl-test-patient-tarpon.svc.cluster.local:9000", -+ "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", +- "s3-endpoint": "https://garage.kuttl-test-patient-tarpon.svc.cluster.local:9000", ++ "s3-endpoint": "https://garage.${NAMESPACE}.svc.cluster.local:9000", "s3-path-style-access": "true", "warehouse-location": "s3a://demo/lakehouse" }, diff --git a/tests/templates/kuttl/iceberg-rest/01_s3-connection.yaml b/tests/templates/kuttl/iceberg-rest/01_s3-connection.yaml index 56c30d36..eeec9540 100644 --- a/tests/templates/kuttl/iceberg-rest/01_s3-connection.yaml +++ b/tests/templates/kuttl/iceberg-rest/01_s3-connection.yaml @@ -2,9 +2,9 @@ apiVersion: s3.stackable.tech/v1alpha1 kind: S3Connection metadata: - name: minio + name: garage spec: - host: "minio.${NAMESPACE}.svc.cluster.local" + host: "garage.${NAMESPACE}.svc.cluster.local" port: 9000 accessStyle: Path credentials: @@ -28,9 +28,11 @@ spec: apiVersion: v1 kind: Secret metadata: - name: minio-credentials + name: garage-credentials labels: secrets.stackable.tech/class: s3-credentials-class stringData: - accessKey: admin - secretKey: adminadmin + # Garage requires the GK<24 hex> key id format and a 64 hex char secret. + # Must match the GARAGE_DEFAULT_* variables in 20_garage.yaml. + accessKey: GK31c0ffee31c0ffee31c0ffee + secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef diff --git a/tests/templates/kuttl/iceberg-rest/20-assert.yaml b/tests/templates/kuttl/iceberg-rest/20-assert.yaml index 477bdd02..20a8dae0 100644 --- a/tests/templates/kuttl/iceberg-rest/20-assert.yaml +++ b/tests/templates/kuttl/iceberg-rest/20-assert.yaml @@ -6,7 +6,7 @@ timeout: 600 apiVersion: apps/v1 kind: Deployment metadata: - name: minio + name: garage status: readyReplicas: 1 replicas: 1 diff --git a/tests/templates/kuttl/iceberg-rest/20-install-minio.yaml b/tests/templates/kuttl/iceberg-rest/20-install-garage.yaml similarity index 50% rename from tests/templates/kuttl/iceberg-rest/20-install-minio.yaml rename to tests/templates/kuttl/iceberg-rest/20-install-garage.yaml index 985b51e8..0b085871 100644 --- a/tests/templates/kuttl/iceberg-rest/20-install-minio.yaml +++ b/tests/templates/kuttl/iceberg-rest/20-install-garage.yaml @@ -2,4 +2,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl -n $NAMESPACE apply -f 20_minio.yaml + - script: kubectl -n $NAMESPACE apply -f 20_garage.yaml diff --git a/tests/templates/kuttl/iceberg-rest/20_garage.yaml b/tests/templates/kuttl/iceberg-rest/20_garage.yaml new file mode 100644 index 00000000..a89bf184 --- /dev/null +++ b/tests/templates/kuttl/iceberg-rest/20_garage.yaml @@ -0,0 +1,157 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: garage +data: + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ + garage.toml: | + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + db_engine = "lmdb" + + # Single node, no redundancy. Test data is disposable (emptyDir). + replication_factor = 1 + + # Throwaway value, this cluster is never joined by another node. + rpc_secret = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "127.0.0.1:3901" + + [s3_api] + # Garage rejects requests signed for a different region, so this must match + # the region the clients use (the S3Connection default, and the NiFi flow). + s3_region = "us-east-1" + api_bind_addr = "[::]:3900" + + [admin] + api_bind_addr = "[::]:3903" + # Garage terminates no TLS on any endpoint, so nginx does it and forwards to + # Garage on loopback. Mounted over /etc/nginx/nginx.conf, hence the full file. + nginx.conf: | + events {} + http { + server { + listen 9000 ssl; + ssl_certificate /stackable/tls/tls.crt; + ssl_certificate_key /stackable/tls/tls.key; + + # Don't buffer or size-limit object uploads. + client_max_body_size 0; + + location / { + proxy_pass http://127.0.0.1:3900; + # Must be the original Host, it is part of the SigV4 signature. + proxy_set_header Host $http_host; + proxy_http_version 1.1; + proxy_buffering off; + proxy_request_buffering off; + } + } + } +--- +apiVersion: v1 +kind: Service +metadata: + name: garage +spec: + selector: + app: garage + ports: + - name: https + port: 9000 + targetPort: https +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: garage + labels: + app: garage +spec: + replicas: 1 + selector: + matchLabels: + app: garage + template: + metadata: + labels: + app: garage + spec: + containers: + - name: garage + image: oci.stackable.tech/stackable/dxflrs/garage:v2.4.1 + imagePullPolicy: IfNotPresent + # The image has no entrypoint. --single-node creates the cluster + # layout, --default-bucket creates the bucket and the access key from + # the GARAGE_DEFAULT_* variables below (implies --default-access-key). + command: + - /garage + - server + - --single-node + - --default-bucket + env: + - name: GARAGE_DEFAULT_BUCKET + value: demo + # Garage requires the GK<24 hex> key id format and a 64 hex char + # secret. These are also set in 01_s3-connection.yaml and in the + # NiFi flow (60_nifi-flow.json). + - name: GARAGE_DEFAULT_ACCESS_KEY + value: GK31c0ffee31c0ffee31c0ffee + - name: GARAGE_DEFAULT_SECRET_KEY + value: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef + ports: + - name: rpc + containerPort: 3901 + - name: admin + containerPort: 3903 + readinessProbe: + httpGet: + path: /health + port: admin + volumeMounts: + - name: config + mountPath: /etc/garage.toml + subPath: garage.toml + - name: data + mountPath: /var/lib/garage + resources: + requests: + cpu: 500m + memory: 512Mi + - name: nginx + image: docker.io/library/nginx:1.29-alpine + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 9000 + volumeMounts: + - name: config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: tls + mountPath: /stackable/tls + resources: + requests: + cpu: 100m + memory: 128Mi + volumes: + - name: config + configMap: + name: garage + - name: data + emptyDir: {} + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=garage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech diff --git a/tests/templates/kuttl/iceberg-rest/20_minio.yaml b/tests/templates/kuttl/iceberg-rest/20_minio.yaml deleted file mode 100644 index 3b9ffcbe..00000000 --- a/tests/templates/kuttl/iceberg-rest/20_minio.yaml +++ /dev/null @@ -1,579 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "minio-sa" ---- -apiVersion: v1 -kind: Secret -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -type: Opaque -data: - rootUser: "YWRtaW4=" - rootPassword: "YWRtaW5hZG1pbg==" ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -data: - initialize: |- - #!/bin/sh - set -e # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 - LIMIT=29 # Allow 30 attempts - set -e # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) - SECRET=$(cat /config/rootPassword) - set +e # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" - $MC_COMMAND - STATUS=$? - until [ $STATUS = 0 ]; do - ATTEMPTS=$(expr $ATTEMPTS + 1) - echo \"Failed attempts: $ATTEMPTS\" - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 - fi - sleep 2 # 1 second intervals between attempts - $MC_COMMAND - STATUS=$? - done - set -e # reset `e` as active - return 0 - } - - # checkBucketExists ($bucket) - # Check if the bucket exists, by using the exit code of `mc ls` - checkBucketExists() { - BUCKET=$1 - CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) - return $? - } - - # createBucket ($bucket, $policy, $purge) - # Ensure bucket exists, purging if asked to - createBucket() { - BUCKET=$1 - POLICY=$2 - PURGE=$3 - VERSIONING=$4 - OBJECTLOCKING=$5 - - # Purge the bucket, if set & exists - # Since PURGE is user input, check explicitly for `true` - if [ $PURGE = true ]; then - if checkBucketExists $BUCKET; then - echo "Purging bucket '$BUCKET'." - set +e # don't exit if this fails - ${MC} rm -r --force myminio/$BUCKET - set -e # reset `e` as active - else - echo "Bucket '$BUCKET' does not exist, skipping purge." - fi - fi - - # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) - if ! checkBucketExists $BUCKET; then - if [ ! -z $OBJECTLOCKING ]; then - if [ $OBJECTLOCKING = true ]; then - echo "Creating bucket with OBJECTLOCKING '$BUCKET'" - ${MC} mb --with-lock myminio/$BUCKET - elif [ $OBJECTLOCKING = false ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - fi - elif [ -z $OBJECTLOCKING ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - else - echo "Bucket '$BUCKET' already exists." - fi - fi - - # set versioning for bucket if objectlocking is disabled or not set - if [ $OBJECTLOCKING = false ]; then - if [ ! -z $VERSIONING ]; then - if [ $VERSIONING = true ]; then - echo "Enabling versioning for '$BUCKET'" - ${MC} version enable myminio/$BUCKET - elif [ $VERSIONING = false ]; then - echo "Suspending versioning for '$BUCKET'" - ${MC} version suspend myminio/$BUCKET - fi - fi - else - echo "Bucket '$BUCKET' versioning unchanged." - fi - - # At this point, the bucket should exist, skip checking for existence - # Set policy on the bucket - echo "Setting policy of bucket '$BUCKET' to '$POLICY'." - ${MC} anonymous set $POLICY myminio/$BUCKET - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - # Create the buckets - createBucket demo "public" false false false - - add-user: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkUserExists () - # Check if the user exists, by using the exit code of `mc admin user info` - checkUserExists() { - CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createUser ($policy) - createUser() { - POLICY=$1 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the user if it does not exist - if ! checkUserExists ; then - echo "Creating user '$USER'" - cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio - else - echo "User '$USER' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - - # set policy for user - if [ ! -z $POLICY -a $POLICY != " " ] ; then - echo "Adding policy '$POLICY' for '$USER'" - set +e ; # policy already attach errors out, allow it. - ${MC} admin policy attach myminio $POLICY --user=$USER - set -e - else - echo "User '$USER' has no policy attached." - fi - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - # Create the users - echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP - echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP - createUser consoleAdmin - - add-policy: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkPolicyExists ($policy) - # Check if the policy exists, by using the exit code of `mc admin policy info` - checkPolicyExists() { - POLICY=$1 - CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) - return $? - } - - # createPolicy($name, $filename) - createPolicy () { - NAME=$1 - FILENAME=$2 - - # Create the name if it does not exist - echo "Checking policy: $NAME (in /config/$FILENAME.json)" - if ! checkPolicyExists $NAME ; then - echo "Creating policy '$NAME'" - else - echo "Policy '$NAME' already exists." - fi - ${MC} admin policy create myminio $NAME /config/$FILENAME.json - - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - add-svcacct: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 2 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkSvcacctExists () - # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` - checkSvcacctExists() { - CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createSvcacct ($user) - createSvcacct () { - USER=$1 - FILENAME=$2 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the svcacct if it does not exist - if ! checkSvcacctExists ; then - echo "Creating svcacct '$SVCACCT'" - # Check if policy file is define - if [ -z $FILENAME ]; then - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER - else - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER - fi - else - echo "Svcacct '$SVCACCT' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - custom-command: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # runCommand ($@) - # Run custom mc command - runCommand() { - ${MC} "$@" - return $? - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: "10Gi" ---- -apiVersion: v1 -kind: Service -metadata: - name: minio-console - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9001 - protocol: TCP - targetPort: 9001 - selector: - app: minio - release: minio ---- -apiVersion: v1 -kind: Service -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - monitoring: "true" -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - app: minio - release: minio ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - stackable.tech/vendor: Stackable -spec: - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 - replicas: 1 - selector: - matchLabels: - app: minio - release: minio - template: - metadata: - name: minio - labels: - app: minio - release: minio - stackable.tech/vendor: Stackable - annotations: - checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 - checksum/config: ebea49cc4c1bfbd1b156a58bf770a776ff87fe199f642d31c2816b5515112e72 - spec: - securityContext: - fsGroupChangePolicy: OnRootMismatch - serviceAccountName: minio-sa - containers: - - name: minio - image: "quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # minio requires the TLS key pair to be specially named - # mkdir -p /etc/minio/certs - cp -v /etc/minio/original_certs/tls.crt /etc/minio/certs/public.crt - cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key - - /usr/bin/docker-entrypoint.sh minio server /export -S /etc/minio/certs/ --address :9000 --console-address :9001 - volumeMounts: - - name: minio-user - mountPath: "/tmp/credentials" - readOnly: true - - name: export - mountPath: /export - - mountPath: /etc/minio/original_certs - name: tls - - mountPath: /etc/minio/certs - name: certs - ports: - - name: https - containerPort: 9000 - - name: https-console - containerPort: 9001 - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: minio - key: rootUser - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio - key: rootPassword - - name: MINIO_PROMETHEUS_AUTH_TYPE - value: "public" - resources: - requests: - cpu: 1 - memory: 2Gi - securityContext: - readOnlyRootFilesystem: false - volumes: - - name: export - persistentVolumeClaim: - claimName: minio - - name: minio-user - secret: - secretName: minio - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs diff --git a/tests/templates/kuttl/iceberg-rest/21-assert.yaml b/tests/templates/kuttl/iceberg-rest/21-assert.yaml deleted file mode 100644 index 3895aff4..00000000 --- a/tests/templates/kuttl/iceberg-rest/21-assert.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 600 ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-post-job -status: - succeeded: 1 diff --git a/tests/templates/kuttl/iceberg-rest/21-install-minio-jobs.yaml b/tests/templates/kuttl/iceberg-rest/21-install-minio-jobs.yaml deleted file mode 100644 index d51dae4b..00000000 --- a/tests/templates/kuttl/iceberg-rest/21-install-minio-jobs.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: kubectl -n $NAMESPACE apply -f 21_minio_jobs.yaml diff --git a/tests/templates/kuttl/iceberg-rest/21_minio_jobs.yaml b/tests/templates/kuttl/iceberg-rest/21_minio_jobs.yaml deleted file mode 100644 index bd8f3ac4..00000000 --- a/tests/templates/kuttl/iceberg-rest/21_minio_jobs.yaml +++ /dev/null @@ -1,116 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-post-job - labels: - app: minio-post-job - chart: minio-5.4.0 - release: minio - heritage: Helm - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation -spec: - template: - metadata: - labels: - app: minio-job - release: minio - stackable.tech/vendor: Stackable - spec: - restartPolicy: OnFailure - volumes: - - name: etc-path - emptyDir: {} - - name: tmp - emptyDir: {} - - name: minio-configuration - projected: - sources: - - configMap: - name: minio - - secret: - name: minio - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs - serviceAccountName: minio-sa - containers: - - name: minio-make-bucket - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - - . /config/initialize - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi - - name: minio-make-user - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - - . /config/add-user - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi diff --git a/tests/templates/kuttl/iceberg-rest/30_hive.yaml.j2 b/tests/templates/kuttl/iceberg-rest/30_hive.yaml.j2 index 07405993..4c0378c3 100644 --- a/tests/templates/kuttl/iceberg-rest/30_hive.yaml.j2 +++ b/tests/templates/kuttl/iceberg-rest/30_hive.yaml.j2 @@ -19,7 +19,7 @@ spec: database: hive credentialsSecretName: postgres-credentials s3: - reference: minio + reference: garage {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/iceberg-rest/31_trino.yaml.j2 b/tests/templates/kuttl/iceberg-rest/31_trino.yaml.j2 index 3cd9720d..04f9568f 100644 --- a/tests/templates/kuttl/iceberg-rest/31_trino.yaml.j2 +++ b/tests/templates/kuttl/iceberg-rest/31_trino.yaml.j2 @@ -9,7 +9,7 @@ spec: connector: iceberg: s3: - reference: minio + reference: garage configOverrides: iceberg.catalog.type: rest # We are using the headless service, as the hive-metastore service is missing port 9001 diff --git a/tests/templates/kuttl/iceberg-rest/50_nifi.yaml.j2 b/tests/templates/kuttl/iceberg-rest/50_nifi.yaml.j2 index 0b37cb02..78671c5d 100644 --- a/tests/templates/kuttl/iceberg-rest/50_nifi.yaml.j2 +++ b/tests/templates/kuttl/iceberg-rest/50_nifi.yaml.j2 @@ -28,7 +28,7 @@ spec: listenerClass: external-unstable jvmArgumentOverrides: add: - # Needed for NiFi to trust the minio cert + # Needed for NiFi to trust the Garage and Keycloak certs - -Djavax.net.ssl.trustStore=/stackable/keystore/truststore.p12 - -Djavax.net.ssl.trustStorePassword=secret - -Djavax.net.ssl.trustStoreType=PKCS12 diff --git a/tests/templates/kuttl/iceberg-rest/60_nifi-flow.json b/tests/templates/kuttl/iceberg-rest/60_nifi-flow.json index 45cc3e5e..e070d637 100644 --- a/tests/templates/kuttl/iceberg-rest/60_nifi-flow.json +++ b/tests/templates/kuttl/iceberg-rest/60_nifi-flow.json @@ -331,11 +331,11 @@ }, "properties": { "Authentication Strategy": "BASIC_CREDENTIALS", - "Access Key ID": "admin", - "Secret Access Key": "adminadmin", - "Endpoint URL": "https://minio.${NAMESPACE}.svc.cluster.local:9000", + "Access Key ID": "GK31c0ffee31c0ffee31c0ffee", + "Secret Access Key": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + "Endpoint URL": "https://garage.${NAMESPACE}.svc.cluster.local:9000", "Path Style Access": "true", - "Client Region": "us-east1" + "Client Region": "us-east-1" }, "propertyDescriptors": { "Authentication Strategy": { diff --git a/tests/templates/kuttl/iceberg-rest/README.md b/tests/templates/kuttl/iceberg-rest/README.md index d25c1dbb..fc8339e9 100644 --- a/tests/templates/kuttl/iceberg-rest/README.md +++ b/tests/templates/kuttl/iceberg-rest/README.md @@ -1,6 +1,6 @@ The file `60_nifi-flow.json` was exported from the NiFi UI. -*However*, we need to update some stuff, such as adding S3 credentials and templating the namespace of MinIO. +*However*, we need to update some stuff, such as adding S3 credentials and templating the namespace of Garage. TIP: I used `JSON: Sort Document` in VScode to somewhat have consistent formatting, which makes reading and diffs easier. @@ -15,11 +15,12 @@ index eb64241..6ead26e 100644 }, "properties": { "Authentication Strategy": "BASIC_CREDENTIALS", -- "Endpoint URL": "https://minio.kuttl-test-dear-bug.svc.cluster.local:9000", -+ "Access Key ID": "admin", -+ "Secret Access Key": "adminadmin", -+ "Endpoint URL": "https://minio.${NAMESPACE}.svc.cluster.local:9000", ++ "Access Key ID": "GK31c0ffee31c0ffee31c0ffee", ++ "Secret Access Key": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", +- "Endpoint URL": "https://garage.kuttl-test-dear-bug.svc.cluster.local:9000", ++ "Endpoint URL": "https://garage.${NAMESPACE}.svc.cluster.local:9000", "Path Style Access": "true", - "Client Region": "us-east1" +- "Client Region": "us-east1" ++ "Client Region": "us-east-1" }, ``` From 0e520d937168b6c5a6a915da9e6053166943d14d Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 29 Sep 2026 12:06:49 +0200 Subject: [PATCH 2/3] test(iceberg-hive): Use us-east-2 region which s3a defaults to --- .../templates/kuttl/iceberg-hive/01_s3-connection.yaml | 7 +++++++ tests/templates/kuttl/iceberg-hive/20_garage.yaml | 10 +++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/templates/kuttl/iceberg-hive/01_s3-connection.yaml b/tests/templates/kuttl/iceberg-hive/01_s3-connection.yaml index eeec9540..8fe1001e 100644 --- a/tests/templates/kuttl/iceberg-hive/01_s3-connection.yaml +++ b/tests/templates/kuttl/iceberg-hive/01_s3-connection.yaml @@ -6,6 +6,13 @@ metadata: spec: host: "garage.${NAMESPACE}.svc.cluster.local" port: 9000 + # The operators pass this on to the products (Trino: s3.region, Hive: + # fs.s3a.endpoint.region), so it has to match s3_region in 20_garage.yaml. + # Not the us-east-1 default: NiFi signs with us-east-2 here, which is the + # Hadoop S3 implementation's fallback for non-AWS endpoints and is more + # convoluted to configure from a flow, so the products follow it instead. + region: + name: us-east-2 accessStyle: Path credentials: secretClass: s3-credentials-class diff --git a/tests/templates/kuttl/iceberg-hive/20_garage.yaml b/tests/templates/kuttl/iceberg-hive/20_garage.yaml index 7256c920..c7a68c35 100644 --- a/tests/templates/kuttl/iceberg-hive/20_garage.yaml +++ b/tests/templates/kuttl/iceberg-hive/20_garage.yaml @@ -20,9 +20,13 @@ data: [s3_api] # Garage rejects requests signed for a different region, so this must match - # the region the clients use. This test goes through Hadoop's s3a, which - # signs for us-east-2 when fs.s3a.endpoint.region is unset and the endpoint - # is not an AWS one. MinIO ignored the region, Garage does not. + # the region every client signs with. That region is dictated by NiFi: the + # Iceberg catalog service reaches S3 through the Hadoop S3 implementation, + # which falls back to us-east-2 when fs.s3a.endpoint.region is unset and the + # endpoint is not an AWS one. Those settings are more convoluted to configure + # from a flow, so everything else follows suit instead, see the region in + # 01_s3-connection.yaml. NiFi drops the Hadoop S3 implementation after 2.7.0, + # after which this can become a provider agnostic name. s3_region = "us-east-2" api_bind_addr = "[::]:3900" From bcf0594cb66abcefd7cac353281d2e0a98d4cd25 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 29 Sep 2026 12:07:11 +0200 Subject: [PATCH 3/3] test(iceberg-rest): Use agnostic region-1 --- tests/templates/kuttl/iceberg-rest/01_s3-connection.yaml | 7 +++++++ tests/templates/kuttl/iceberg-rest/20_garage.yaml | 6 ++++-- tests/templates/kuttl/iceberg-rest/60_nifi-flow.json | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/templates/kuttl/iceberg-rest/01_s3-connection.yaml b/tests/templates/kuttl/iceberg-rest/01_s3-connection.yaml index eeec9540..15018bfc 100644 --- a/tests/templates/kuttl/iceberg-rest/01_s3-connection.yaml +++ b/tests/templates/kuttl/iceberg-rest/01_s3-connection.yaml @@ -6,6 +6,13 @@ metadata: spec: host: "garage.${NAMESPACE}.svc.cluster.local" port: 9000 + # The operators pass this on to the products (Trino: s3.region, Hive: + # fs.s3a.endpoint.region), so it has to match s3_region in 20_garage.yaml. + # The default (us-east-1) is not used, to keep the region the same everywhere: + # the Hadoop S3 implementation falls back to us-east-2 rather than us-east-1 + # when unset, and Garage rejects requests signed for a different region. + region: + name: region-1 accessStyle: Path credentials: secretClass: s3-credentials-class diff --git a/tests/templates/kuttl/iceberg-rest/20_garage.yaml b/tests/templates/kuttl/iceberg-rest/20_garage.yaml index a89bf184..3f53f723 100644 --- a/tests/templates/kuttl/iceberg-rest/20_garage.yaml +++ b/tests/templates/kuttl/iceberg-rest/20_garage.yaml @@ -20,8 +20,10 @@ data: [s3_api] # Garage rejects requests signed for a different region, so this must match - # the region the clients use (the S3Connection default, and the NiFi flow). - s3_region = "us-east-1" + # the region every client signs with (01_s3-connection.yaml for the products + # configured by the operators, and 60_nifi-flow.json for NiFi). + # The name is a provider agnostic one. + s3_region = "region-1" api_bind_addr = "[::]:3900" [admin] diff --git a/tests/templates/kuttl/iceberg-rest/60_nifi-flow.json b/tests/templates/kuttl/iceberg-rest/60_nifi-flow.json index e070d637..12195e54 100644 --- a/tests/templates/kuttl/iceberg-rest/60_nifi-flow.json +++ b/tests/templates/kuttl/iceberg-rest/60_nifi-flow.json @@ -335,7 +335,7 @@ "Secret Access Key": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", "Endpoint URL": "https://garage.${NAMESPACE}.svc.cluster.local:9000", "Path Style Access": "true", - "Client Region": "us-east-1" + "Client Region": "region-1" }, "propertyDescriptors": { "Authentication Strategy": {