Skip to content

Commit 54852e7

Browse files
authored
fix(snowflake): use ALTER ICEBERG TABLE to register column comments on Iceberg tables (#6038)
Signed-off-by: Guillem G <guillem.gimenez@titanos.tv>
1 parent ec513a7 commit 54852e7

5 files changed

Lines changed: 85 additions & 2 deletions

File tree

‎sqlmesh/core/engine_adapter/base.py‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -833,7 +833,9 @@ def _create_table_from_columns(
833833
and self.COMMENT_CREATION_TABLE.is_comment_command_only
834834
and self.comments_enabled
835835
):
836-
self._create_column_comments(table_name, column_descriptions)
836+
self._create_column_comments(
837+
table_name, column_descriptions, table_format=kwargs.get("table_format")
838+
)
837839

838840
def _build_schema_exp(
839841
self,
@@ -990,7 +992,11 @@ def _create_table_from_source_queries(
990992
):
991993
self._create_table_comment(table_name, table_description)
992994
if column_descriptions and schema is None and self.comments_enabled:
993-
self._create_column_comments(table_name, column_descriptions)
995+
self._create_column_comments(
996+
table_name,
997+
column_descriptions,
998+
table_format=None if table_kind else kwargs.get("table_format"),
999+
)
9941000

9951001
def _create_table(
9961002
self,
@@ -3083,7 +3089,18 @@ def _create_column_comments(
30833089
column_comments: t.Dict[str, str],
30843090
table_kind: str = "TABLE",
30853091
materialized_view: bool = False,
3092+
table_format: t.Optional[str] = None,
30863093
) -> None:
3094+
"""Registers column comments with a post-creation command.
3095+
3096+
Args:
3097+
table_name: The name of the table or view.
3098+
column_comments: Mapping between the column name and its comment.
3099+
table_kind: The kind of object being commented on, `TABLE` or `VIEW`.
3100+
materialized_view: Whether the view is materialized.
3101+
table_format: The table format of the table, if any. Engines that require
3102+
format-specific DDL to alter a table use it to derive `table_kind`.
3103+
"""
30873104
table = exp.to_table(table_name)
30883105

30893106
for col, comment in column_comments.items():

‎sqlmesh/core/engine_adapter/bigquery.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -819,6 +819,7 @@ def _create_column_comments(
819819
column_comments: t.Dict[str, str],
820820
table_kind: str = "TABLE",
821821
materialized_view: bool = False,
822+
table_format: t.Optional[str] = None,
822823
) -> None:
823824
if not (table_kind == "VIEW" and materialized_view):
824825
table = self._get_table(table_name)

‎sqlmesh/core/engine_adapter/mysql.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ def _create_column_comments(
131131
column_comments: t.Dict[str, str],
132132
table_kind: str = "TABLE",
133133
materialized_view: bool = False,
134+
table_format: t.Optional[str] = None,
134135
) -> None:
135136
table = exp.to_table(table_name)
136137
table_sql = table.sql(dialect=self.dialect, identify=True)

‎sqlmesh/core/engine_adapter/snowflake.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -637,13 +637,20 @@ def _create_column_comments(
637637
column_comments: t.Dict[str, str],
638638
table_kind: str = "TABLE",
639639
materialized_view: bool = False,
640+
table_format: t.Optional[str] = None,
640641
) -> None:
641642
"""
642643
Reference: https://docs.snowflake.com/en/sql-reference/sql/alter-table-column#syntax
644+
Reference: https://docs.snowflake.com/en/sql-reference/sql/alter-iceberg-table#syntax
643645
"""
644646
if not column_comments:
645647
return
646648

649+
# Snowflake rejects `ALTER TABLE` for Iceberg tables, it requires
650+
# `ALTER ICEBERG TABLE` instead
651+
if table_format and table_kind == "TABLE":
652+
table_kind = f"{table_format.upper()} TABLE"
653+
647654
table = exp.to_table(table_name)
648655
table_sql = self._to_sql(table)
649656

‎tests/core/engine_adapter/test_snowflake.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,38 @@ def test_multiple_column_comments(make_mocked_engine_adapter: t.Callable, mocker
309309
]
310310

311311

312+
def test_column_comments_iceberg(make_mocked_engine_adapter: t.Callable):
313+
adapter = make_mocked_engine_adapter(SnowflakeEngineAdapter)
314+
315+
adapter._create_column_comments(
316+
"test_table",
317+
{"a": "a column description", "b": "b column description"},
318+
table_format="iceberg",
319+
)
320+
321+
assert to_sql_calls(adapter) == [
322+
"""ALTER ICEBERG TABLE "test_table" ALTER COLUMN "a" COMMENT 'a column description', COLUMN "b" COMMENT 'b column description'""",
323+
]
324+
325+
326+
def test_ctas_column_comments_iceberg(make_mocked_engine_adapter: t.Callable):
327+
adapter = make_mocked_engine_adapter(SnowflakeEngineAdapter)
328+
329+
# The column types are unknown, so the comments can't be inlined into the CTAS
330+
# schema definition and are registered with a post-creation ALTER instead
331+
adapter.ctas(
332+
"test_table",
333+
parse_one("SELECT a, b FROM source_table"),
334+
table_format="iceberg",
335+
column_descriptions={"a": "a column description"},
336+
)
337+
338+
assert to_sql_calls(adapter) == [
339+
"""CREATE ICEBERG TABLE IF NOT EXISTS "test_table" AS SELECT "a", "b" FROM "source_table\"""",
340+
"""ALTER ICEBERG TABLE "test_table" ALTER COLUMN "a" COMMENT 'a column description'""",
341+
]
342+
343+
312344
def test_sync_grants_config(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture):
313345
adapter = make_mocked_engine_adapter(SnowflakeEngineAdapter)
314346
relation = normalize_identifiers(
@@ -637,6 +669,31 @@ def test_create_managed_table(make_mocked_engine_adapter: t.Callable, mocker: Mo
637669
]
638670

639671

672+
@pytest.mark.parametrize("table_format", [None, "iceberg"])
673+
def test_create_managed_table_column_comments(
674+
make_mocked_engine_adapter: t.Callable, table_format: t.Optional[str]
675+
):
676+
adapter = make_mocked_engine_adapter(SnowflakeEngineAdapter)
677+
678+
# Unknown column types require a post-creation comment command.
679+
adapter.create_managed_table(
680+
table_name="test_table",
681+
query=parse_one("SELECT a, b FROM source_table"),
682+
table_format=table_format,
683+
table_properties={
684+
"target_lag": exp.Literal.string("20 minutes"),
685+
"warehouse": exp.to_identifier("test_warehouse"),
686+
},
687+
column_descriptions={"a": "a column description", "b": "b column description"},
688+
)
689+
690+
table_kind = "DYNAMIC ICEBERG TABLE" if table_format else "DYNAMIC TABLE"
691+
assert to_sql_calls(adapter) == [
692+
f'CREATE OR REPLACE {table_kind} "test_table" TARGET_LAG=\'20 minutes\' WAREHOUSE="test_warehouse" AS SELECT "a", "b" FROM "source_table"',
693+
"""ALTER TABLE "test_table" ALTER COLUMN "a" COMMENT 'a column description', COLUMN "b" COMMENT 'b column description'""",
694+
]
695+
696+
640697
def test_drop_managed_table(make_mocked_engine_adapter: t.Callable, mocker: MockerFixture):
641698
adapter = make_mocked_engine_adapter(SnowflakeEngineAdapter)
642699

0 commit comments

Comments
 (0)