Skip to content

Commit fece835

Browse files
committed
fix(doris): harden Docker Iceberg S3 setup, pre-4.1 order_by gate, and unnamed STRUCT DDL
Signed-off-by: Bruce Arctor <5032356+brucearctor@users.noreply.github.com>
1 parent 8b36b0e commit fece835

8 files changed

Lines changed: 166 additions & 27 deletions

File tree

‎.github/scripts/wait-for-db.sh‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ doris_ready() {
119119
local timeout_secs="${DORIS_STARTUP_TIMEOUT_SECS:-300}"
120120
local deadline=$((SECONDS + timeout_secs))
121121
local remaining=0
122+
local probe_timeout=0
122123
local backend_registered=0
123124
local reg_out=""
124125

@@ -134,7 +135,8 @@ doris_ready() {
134135
echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris FE to become alive" >&2
135136
exit 1
136137
fi
137-
if timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "SHOW FRONTENDS\G" 2>/dev/null | grep -q "Alive: true"; then
138+
probe_timeout=$((remaining < 10 ? remaining : 10))
139+
if timeout "$probe_timeout" docker exec doris-fe mysql --connect-timeout=5 -h127.0.0.1 -P9030 -uroot -e "SHOW FRONTENDS\G" 2>/dev/null | grep -q "Alive: true"; then
138140
echo "Doris FE is alive"
139141
break
140142
fi
@@ -153,7 +155,8 @@ doris_ready() {
153155
echo "ERROR: Timed out after ${timeout_secs}s waiting for Doris backend registration: $reg_out" >&2
154156
exit 1
155157
fi
156-
if reg_out=$(timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '172.30.80.3:9050';" 2>&1); then
158+
probe_timeout=$((remaining < 10 ? remaining : 10))
159+
if reg_out=$(timeout "$probe_timeout" docker exec doris-fe mysql --connect-timeout=5 -h127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '172.30.80.3:9050';" 2>&1); then
157160
backend_registered=1
158161
elif echo "$reg_out" | grep -Eqi "already exists|already registered"; then
159162
backend_registered=1
@@ -172,7 +175,8 @@ doris_ready() {
172175
fi
173176
exit 1
174177
fi
175-
ALIVE_BACKENDS=$(timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "show backends \G" 2>/dev/null | grep -c "^ *Alive: true *$" || true)
178+
probe_timeout=$((remaining < 10 ? remaining : 10))
179+
ALIVE_BACKENDS=$(timeout "$probe_timeout" docker exec doris-fe mysql --connect-timeout=5 -h127.0.0.1 -P9030 -uroot -e "show backends \G" 2>/dev/null | grep -c "^ *Alive: true *$" || true)
176180

177181
# fallback value if failed to get number
178182
if ! [[ "$ALIVE_BACKENDS" =~ ^[0-9]+$ ]]; then
@@ -208,7 +212,8 @@ doris_ready() {
208212
echo "ERROR: Timed out after ${timeout_secs}s configuring Doris frontend: $cfg_out" >&2
209213
exit 1
210214
fi
211-
if cfg_out=$(timeout "${remaining}s" docker exec -i doris-fe mysql -h127.0.0.1 -P9030 -uroot -e "ADMIN SET FRONTEND CONFIG ('force_olap_table_replication_num' = '1');" 2>&1); then
215+
probe_timeout=$((remaining < 10 ? remaining : 10))
216+
if cfg_out=$(timeout "$probe_timeout" docker exec doris-fe mysql --connect-timeout=5 -h127.0.0.1 -P9030 -uroot -e "ADMIN SET FRONTEND CONFIG ('force_olap_table_replication_num' = '1');" 2>&1); then
212217
break
213218
fi
214219
if [ "$SECONDS" -ge "$deadline" ]; then

‎sqlmesh/core/config/connection.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2712,6 +2712,9 @@ def connect_db2(**kwargs: t.Any) -> t.Any:
27122712
return connect_db2
27132713

27142714

2715+
_doris_engine_import_validator = _get_engine_import_validator("pymysql", "doris", decorate=False)
2716+
2717+
27152718
class DorisConnectionConfig(ConnectionConfig):
27162719
"""Configuration for the Apache Doris connection.
27172720
@@ -2775,9 +2778,6 @@ class DorisConnectionConfig(ConnectionConfig):
27752778
DISPLAY_NAME: t.ClassVar[t.Literal["Apache Doris"]] = "Apache Doris"
27762779
DISPLAY_ORDER: t.ClassVar[t.Literal[20]] = 20
27772780
MIN_PYMYSQL_VERSION: t.ClassVar[t.Tuple[int, int, int]] = (1, 2, 0)
2778-
_base_engine_import_validator: t.ClassVar[t.Callable] = staticmethod(
2779-
_get_engine_import_validator("pymysql", "doris", decorate=False)
2780-
)
27812781

27822782
@classmethod
27832783
def _validate_pymysql_version(cls, pymysql_module: t.Any = None) -> None:
@@ -2828,7 +2828,7 @@ def _engine_import_validator(cls, data: t.Any) -> t.Any:
28282828
check_import = (
28292829
str_to_bool(str(data.get("check_import", True))) if isinstance(data, dict) else True
28302830
)
2831-
data = cls._base_engine_import_validator(cls, data)
2831+
data = _doris_engine_import_validator(cls, data)
28322832
if check_import:
28332833
cls._validate_pymysql_version()
28342834
return data

‎sqlmesh/core/dialect.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1113,6 +1113,33 @@ def doris_supports_map_from_arrays(version: t.Optional[t.Tuple[int, ...]] = None
11131113

11141114

11151115
def _doris_datatype_sql(self: Generator, expression: exp.DataType) -> str:
1116+
if (
1117+
expression.is_type(*exp.DataType.STRUCT_TYPES)
1118+
and expression.expressions
1119+
and any(
1120+
isinstance(elem, (exp.DataType, exp.DataTypeParam))
1121+
or (isinstance(elem, exp.ColumnDef) and elem.this is None)
1122+
for elem in expression.expressions
1123+
)
1124+
):
1125+
expression = expression.copy()
1126+
normalized_fields: t.List[exp.Expr] = []
1127+
for idx, elem in enumerate(expression.expressions, start=1):
1128+
if isinstance(elem, exp.DataType):
1129+
normalized_fields.append(
1130+
exp.ColumnDef(this=exp.to_identifier(f"col{idx}"), kind=elem)
1131+
)
1132+
elif isinstance(elem, exp.DataTypeParam) and isinstance(elem.this, exp.DataType):
1133+
normalized_fields.append(
1134+
exp.ColumnDef(this=exp.to_identifier(f"col{idx}"), kind=elem.this)
1135+
)
1136+
elif isinstance(elem, exp.ColumnDef) and elem.this is None:
1137+
elem_copy = elem.copy()
1138+
elem_copy.set("this", exp.to_identifier(f"col{idx}"))
1139+
normalized_fields.append(elem_copy)
1140+
else:
1141+
normalized_fields.append(elem)
1142+
expression.set("expressions", normalized_fields)
11161143
if expression.is_type(exp.DataType.Type.TIMESTAMPTZ) and not doris_supports_timestamptz(
11171144
getattr(self.dialect, "version", None)
11181145
):

‎sqlmesh/core/engine_adapter/doris.py‎

Lines changed: 37 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7208,6 +7208,12 @@ def _build_table_properties_exp(
72087208
"Doris table property 'order_by' is only supported on UNIQUE KEY tables. "
72097209
"For DUPLICATE KEY or AGGREGATE KEY tables, sort order is determined by the key columns."
72107210
)
7211+
doris_ver = self._get_doris_version()
7212+
if doris_ver is not None and doris_ver < (4, 1):
7213+
raise SQLMeshError(
7214+
f"Doris table property 'order_by' / 'cluster_by' (custom sort key on UNIQUE KEY tables) "
7215+
f"requires Apache Doris 4.1+ (detected Doris {doris_ver[0]}.{doris_ver[1]})."
7216+
)
72117217
mow_val = self._extract_str_value(
72127218
table_properties_copy.get("enable_unique_key_merge_on_write")
72137219
)
@@ -7624,6 +7630,18 @@ def _to_sql(self, expression: exp.Expr, quote: bool = True, **kwargs: t.Any) ->
76247630
expression = cast_node
76257631
else:
76267632
ts_node.replace(cast_node)
7633+
if any(
7634+
not (
7635+
isinstance(m.args.get("keys"), exp.Array)
7636+
and isinstance(m.args.get("values"), exp.Array)
7637+
)
7638+
and (m.args.get("keys") is not None or m.args.get("values") is not None)
7639+
for m in expression.find_all(exp.Map)
7640+
):
7641+
self._get_doris_version()
7642+
doris_version = getattr(self, "_doris_version", None)
7643+
if doris_version and "dialect" not in kwargs:
7644+
kwargs["dialect"] = f"{self.dialect}, version={'.'.join(str(p) for p in doris_version)}"
76277645
if expression.find(exp.DataType):
76287646
expression = expression.copy()
76297647
for dtype in reversed(list(expression.find_all(exp.DataType))):
@@ -7654,9 +7672,13 @@ def _to_sql(self, expression: exp.Expr, quote: bool = True, **kwargs: t.Any) ->
76547672
dtype.set("expressions", [exp.DataTypeParam(this=exp.Literal.number(6))])
76557673
elif dtype.is_type("struct"):
76567674
new_field_sqls: t.List[str] = []
7657-
for field in dtype.expressions:
7675+
for idx, field in enumerate(dtype.expressions, start=1):
76587676
if isinstance(field, exp.ColumnDef) and field.args.get("kind"):
7659-
field_id = field.this.copy()
7677+
field_id = (
7678+
field.this.copy()
7679+
if isinstance(field.this, exp.Expr)
7680+
else exp.to_identifier(f"col{idx}")
7681+
)
76607682
if quote and isinstance(field_id, exp.Identifier):
76617683
field_id.set("quoted", True)
76627684
field_name_sql = super()._to_sql(
@@ -7666,6 +7688,19 @@ def _to_sql(self, expression: exp.Expr, quote: bool = True, **kwargs: t.Any) ->
76667688
field.args["kind"], quote=quote, pretty=False, **kwargs
76677689
)
76687690
new_field_sqls.append(f"{field_name_sql}: {field_kind_sql}")
7691+
elif isinstance(field, exp.DataType) or (
7692+
isinstance(field, exp.DataTypeParam)
7693+
and isinstance(field.this, exp.DataType)
7694+
):
7695+
raw_kind = field.this if isinstance(field, exp.DataTypeParam) else field
7696+
field_id = exp.to_identifier(f"col{idx}", quoted=quote)
7697+
field_name_sql = super()._to_sql(
7698+
field_id, quote=False, pretty=False, **kwargs
7699+
)
7700+
field_kind_sql = super()._to_sql(
7701+
raw_kind, quote=quote, pretty=False, **kwargs
7702+
)
7703+
new_field_sqls.append(f"{field_name_sql}: {field_kind_sql}")
76697704
else:
76707705
new_field_sqls.append(
76717706
super()._to_sql(field, quote=quote, pretty=False, **kwargs)
@@ -7678,18 +7713,6 @@ def _to_sql(self, expression: exp.Expr, quote: bool = True, **kwargs: t.Any) ->
76787713
expression = struct_replacement
76797714
else:
76807715
dtype.replace(struct_replacement)
7681-
if any(
7682-
not (
7683-
isinstance(m.args.get("keys"), exp.Array)
7684-
and isinstance(m.args.get("values"), exp.Array)
7685-
)
7686-
and (m.args.get("keys") is not None or m.args.get("values") is not None)
7687-
for m in expression.find_all(exp.Map)
7688-
):
7689-
self._get_doris_version()
7690-
doris_version = getattr(self, "_doris_version", None)
7691-
if doris_version and "dialect" not in kwargs:
7692-
kwargs["dialect"] = f"{self.dialect}, version={'.'.join(str(p) for p in doris_version)}"
76937716
return super()._to_sql(expression, quote=quote, **kwargs)
76947717

76957718
def _build_distributed_by_exp(

‎tests/core/engine_adapter/integration/docker/compose.doris.yaml‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,50 @@
11
# SPDX-License-Identifier: Apache-2.0
22
services:
3+
doris-minio:
4+
image: cgr.dev/chainguard/minio@sha256:039800e64ec7247d2fde7cff3697e964f6fe20b6d7d2c46aa7d82cc63355d512
5+
container_name: doris-minio
6+
hostname: doris-minio
7+
command: ["server", "/data", "--console-address", ":9001"]
8+
environment:
9+
MINIO_ROOT_USER: minio
10+
MINIO_ROOT_PASSWORD: minio123
11+
tmpfs:
12+
- /data:size=256m,mode=1777
13+
networks:
14+
doris_net:
15+
ipv4_address: 172.30.80.4
16+
17+
doris-minio-init:
18+
image: cgr.dev/chainguard/minio-client:latest-dev@sha256:fb635b967f5f32424391150dac4985cc1aea78fb0a1ce031c01952cf661a13ec
19+
container_name: doris-minio-init
20+
depends_on:
21+
doris-minio:
22+
condition: service_started
23+
entrypoint: ["/bin/sh", "-c"]
24+
command:
25+
- |
26+
set -eu
27+
attempt=0
28+
until mc alias set local http://172.30.80.4:9000 minio minio123 >/dev/null 2>&1 && mc ready local >/dev/null 2>&1; do
29+
attempt=$$((attempt + 1))
30+
if [ "$$attempt" -ge 30 ]; then
31+
echo 'MinIO did not become ready after 30 attempts' >&2
32+
exit 1
33+
fi
34+
sleep 1
35+
done
36+
mc mb --ignore-existing local/warehouse
37+
networks:
38+
doris_net:
39+
ipv4_address: 172.30.80.5
40+
341
doris-fe:
442
image: apache/doris:fe-4.1.5
543
container_name: doris-fe
644
hostname: doris-fe
45+
depends_on:
46+
doris-minio-init:
47+
condition: service_completed_successfully
748
entrypoint:
849
- /bin/bash
950
- -c
@@ -17,7 +58,6 @@ services:
1758
- "${DORIS_BIND_ADDRESS:-127.0.0.1}:8030:8030"
1859
- "${DORIS_BIND_ADDRESS:-127.0.0.1}:9030:9030"
1960
volumes:
20-
- iceberg-warehouse:/tmp/doris_iceberg_warehouse
2161
- paimon-warehouse:/tmp/doris_paimon_warehouse
2262
networks:
2363
doris_net:
@@ -46,14 +86,12 @@ services:
4686
- "${DORIS_BIND_ADDRESS:-127.0.0.1}:8040:8040"
4787
- "${DORIS_BIND_ADDRESS:-127.0.0.1}:9050:9050"
4888
volumes:
49-
- iceberg-warehouse:/tmp/doris_iceberg_warehouse
5089
- paimon-warehouse:/tmp/doris_paimon_warehouse
5190
networks:
5291
doris_net:
5392
ipv4_address: 172.30.80.3
5493

5594
volumes:
56-
iceberg-warehouse:
5795
paimon-warehouse:
5896

5997
networks:

‎tests/core/engine_adapter/integration/test_integration_doris.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,12 @@ def test_doris_iceberg_local_catalog(ctx: TestContext):
4949
CREATE CATALOG IF NOT EXISTS `{catalog_name}` PROPERTIES (
5050
'type' = 'iceberg',
5151
'iceberg.catalog.type' = 'hadoop',
52-
'warehouse' = 'file:///tmp/doris_iceberg_warehouse/{catalog_name}'
52+
'warehouse' = 's3://warehouse/iceberg/{catalog_name}',
53+
's3.endpoint' = 'http://172.30.80.4:9000',
54+
's3.access_key' = 'minio',
55+
's3.secret_key' = 'minio123',
56+
's3.region' = 'us-east-1',
57+
'use_path_style' = 'true'
5358
)
5459
""",
5560
quote_identifiers=False,

‎tests/core/engine_adapter/test_doris.py‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -717,6 +717,7 @@ def test_create_table_olap_key_prefix_reordering_and_order_by(
717717
make_mocked_engine_adapter: t.Callable[..., DorisEngineAdapter],
718718
):
719719
adapter = make_mocked_engine_adapter(DorisEngineAdapter)
720+
adapter._doris_version = (4, 1, 5)
720721
adapter.create_table(
721722
"events",
722723
target_columns_to_types={
@@ -2426,7 +2427,7 @@ def test_unannotated_view_column_comments_and_starrocks_property_parity(
24262427

24272428
# 2. Parenthesized string forms for unique_key and order_by, structured distributed_by with `columns=...`,
24282429
# and tuple generic properties (e.g. excluded_trigger_tables = ('db.t1', 'db.t2')).
2429-
adapter._doris_version = (2, 1, 10)
2430+
adapter._doris_version = (4, 1, 5)
24302431
adapter.create_table(
24312432
"db.paren_keys_tbl",
24322433
target_columns_to_types={
@@ -6413,6 +6414,36 @@ def _cursor_init_internal(cur: t.Any) -> None:
64136414
v415_adapter.cursor.fetchone.return_value = ("Doris version doris-4.1.5-rc02", "5.7.99")
64146415
with pytest.raises(SQLMeshError, match="MAP_FROM_ARRAYS requires Doris 4.2\\+"):
64156416
v415_adapter._to_sql(exp.Map(keys=exp.column("k_arr"), values=exp.column("v_arr")))
6417+
v408_adapter = make_mocked_engine_adapter(DorisEngineAdapter)
6418+
v408_adapter.cursor.fetchone.return_value = ("Doris version doris-4.0.8-rc01", "5.7.99")
6419+
with pytest.raises(SQLMeshError, match="requires Apache Doris 4\\.1\\+"):
6420+
v408_adapter.create_table(
6421+
"db.uk_orderby",
6422+
{"id": exp.DataType.build("INT"), "val": exp.DataType.build("INT")},
6423+
table_properties={
6424+
"unique_key": exp.Tuple(expressions=[exp.column("id")]),
6425+
"order_by": exp.Tuple(expressions=[exp.column("val")]),
6426+
},
6427+
)
6428+
unnamed_struct_adapter = make_mocked_engine_adapter(DorisEngineAdapter)
6429+
unnamed_struct_dtype = exp.DataType(
6430+
this=exp.DataType.Type.STRUCT,
6431+
expressions=[
6432+
exp.DataType.build("INT", dialect="doris"),
6433+
exp.ColumnDef(this=None, kind=exp.DataType.build("VARCHAR(32)", dialect="doris")),
6434+
],
6435+
nested=True,
6436+
)
6437+
unnamed_struct_adapter.create_table(
6438+
"db.pos_struct_tbl",
6439+
{
6440+
"id": exp.DataType.build("INT", dialect="doris"),
6441+
"s_pos": unnamed_struct_dtype,
6442+
},
6443+
)
6444+
assert to_sql_calls(unnamed_struct_adapter) == [
6445+
"CREATE TABLE IF NOT EXISTS `db`.`pos_struct_tbl` (`id` INT, `s_pos` STRUCT<`col1`: INT, `col2`: VARCHAR(32)>)"
6446+
]
64166447
assert to_sql_calls(existing_ref_adapter) == []
64176448

64186449
# 14. `_gen_config` appends the Doris `state_connection` even when custom settings are supplied (e.g. DLT template).

‎tests/core/test_dialect.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1835,6 +1835,16 @@ def test_doris_match_struct_and_map_expressions():
18351835
)
18361836
assert aliased_eq_struct_ast.sql("doris") == "SELECT NAMED_STRUCT('is_eq', a = b, 'col2', c)"
18371837

1838+
unnamed_struct_dtype = exp.DataType(
1839+
this=exp.DataType.Type.STRUCT,
1840+
expressions=[
1841+
exp.DataType.build("INT", dialect="doris"),
1842+
exp.ColumnDef(this=None, kind=exp.DataType.build("VARCHAR(32)", dialect="doris")),
1843+
],
1844+
nested=True,
1845+
)
1846+
assert unnamed_struct_dtype.sql("doris") == "STRUCT<col1: INT, col2: VARCHAR(32)>"
1847+
18381848
k_arr = exp.column("k_arr")
18391849
k_arr.type = exp.DataType.build("ARRAY<INT>", dialect="doris")
18401850
v_arr = exp.column("v_arr")

0 commit comments

Comments
 (0)