Antalya 26.6 - Use new euro mirror for test datasets

.github/actions/runner_setup/action.yml

@@ -3,6 +3,13 @@ description: Setup environment
 runs:
   using: "composite"
   steps:
+    - name: Configure docker proxy and dataset mirror host
+      shell: bash
+      run: |
+          proxy_host=dockerhub-proxy.dockerhub-proxy-zone
+          if ! timeout 5 getent ahostsv4 "$proxy_host" >/dev/null; then
+            echo "65.108.242.32 $proxy_host" | sudo tee -a /etc/hosts
+          fi
     - name: Setup zram
       shell: bash
       run: |

ci/jobs/stress_job.py

@@ -1,6 +1,7 @@
 import csv
 import logging
 import os
+import socket
 import sys
 from pathlib import Path
 from typing import List, Tuple
@@ -169,12 +170,24 @@ def get_run_command(
     else:
         run_script = "/repo/tests/docker_scripts/stress_runner.sh"
 
+    # Nested docker does not inherit the runner /etc/hosts.
+    # --network=host would expose minio and azurite. Skip --add-host when the
+    # name is unknown (CI Tests e2e / local) rather than failing the job.
+    proxy_host = "dockerhub-proxy.dockerhub-proxy-zone"
+    add_host = ""
+    try:
+        proxy_ip = socket.getaddrinfo(proxy_host, None)[0][4][0]
+        add_host = f"--add-host={proxy_host}:{proxy_ip} "
+    except OSError:
+        logging.info("Could not resolve %s; not passing --add-host", proxy_host)
+
     cmd = (
         "docker run --cap-add=SYS_PTRACE "
         # For dmesg and sysctl
         "--privileged "
         # azurite-rs (in-process Azure Blob Storage emulator) needs many fds under parallel load
         "--ulimit nofile=1048576:1048576 "
+        f"{add_host}"
         # a static link, don't use S3_URL or S3_DOWNLOAD
         "-e S3_URL='https://s3.amazonaws.com/clickhouse-datasets' "
         "--tmpfs /tmp/clickhouse:mode=1777 "

ci/praktika/runner.py

@@ -5,6 +5,7 @@
 import os
 import re
 import shlex
+import socket
 import sys
 import traceback
 from pathlib import Path
@@ -446,6 +447,18 @@ def _run(
             )
             from_root = "root" in docker_settings
             settings = [s for s in docker_settings if s.startswith("-")]
+            # NOTE (strtgbb): FT/integration docker is not --network=host, so
+            # the runner /etc/hosts mapping for the web-disk proxy is invisible
+            # inside the container. Resolve on the host and pass --add-host.
+            # Skip if the name is not configured on this runner (e.g. CI Tests).
+            _proxy_host = "dockerhub-proxy.dockerhub-proxy-zone"
+            try:
+                _proxy_ip = socket.getaddrinfo(_proxy_host, None)[0][4][0]
+                settings.append(f"--add-host={_proxy_host}:{_proxy_ip}")
+            except OSError as e:
+                print(
+                    f"NOTE: skipping --add-host for {_proxy_host}: {e}"
+                )
             if ":" in job.run_in_docker:
                 docker_name, docker_tag = job.run_in_docker.split(":")
                 print(

tests/docker_scripts/create.sql

@@ -141,7 +141,7 @@ SAMPLE BY intHash32(UserID)
 SETTINGS
     table_disk = 1,
     disk = disk(type = cache, path = 'filesystem_caches/stateful/', max_size = '4G',
-    disk = disk(type = web, endpoint = 'https://clickhouse-datasets-web.s3.us-east-1.amazonaws.com/store/78e/78ebf6a1-d987-4579-b3ec-00c1a087b1f3/'));
+    disk = disk(type = web, endpoint = 'http://dockerhub-proxy.dockerhub-proxy-zone:6000/datasets-web/store/78e/78ebf6a1-d987-4579-b3ec-00c1a087b1f3/'));
 
 CREATE TABLE datasets.visits_v1
 (
@@ -334,4 +334,4 @@ SAMPLE BY intHash32(UserID)
 SETTINGS
     table_disk = 1,
     disk = disk(type = cache, path = 'filesystem_caches/stateful/', max_size = '4G',
-    disk = disk(type = web, endpoint = 'https://clickhouse-datasets-web.s3.us-east-1.amazonaws.com/store/513/5131f834-711f-4168-98a5-968b691a104b/'));
+    disk = disk(type = web, endpoint = 'http://dockerhub-proxy.dockerhub-proxy-zone:6000/datasets-web/store/513/5131f834-711f-4168-98a5-968b691a104b/'));

tests/docker_scripts/create_tpcds.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
 INIT_SQL="$REPO_ROOT/tests/benchmarks/tpc-ds/init.sql"
-S3_BASE="https://tpc-ds-sf1.s3.amazonaws.com"
+S3_BASE="http://dockerhub-proxy.dockerhub-proxy-zone:6000/tpc-ds-sf1"
 
 clickhouse-client --query "CREATE DATABASE IF NOT EXISTS tpcds"
 

tests/queries/0_stateless/04033_tpc_ds_q01.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q02.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q03.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q04.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q05.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q06.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q07.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q08.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q09.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q10.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q11.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q12.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q13.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q14.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q15.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q16.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q17.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 # Known issue: `stddev_samp` returns nan instead of NULL on single value (https://github.com/ClickHouse/ClickHouse/issues/94683).
 # Once the issue is fixed, update the .reference file.
 

tests/queries/0_stateless/04033_tpc_ds_q18.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q19.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q20.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

tests/queries/0_stateless/04033_tpc_ds_q21.sh

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check, no-parallel
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-flaky-check
 # no-fasttest: TPC-DS tables use web disk (S3) which is not available in fasttest.
 # no-random-settings: random session_timezone, query_plan_join_swap_table, etc. change query results.
 # no-replicated-database: the `tpcds` database is not created in DatabaseReplicated mode.
 # no-flaky-check: TPC-DS queries are too expensive for thread fuzzer.
-# no-parallel: heavy queries contend on the shared server under parallel FT jobs.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh