Skip to content

Commit 4ca3a67

Browse files
committed
fix(postgres): harden COPY staging cleanup, JSON fallback, and CR quoting
- Wrap DataFrame replace_query, ctas, merge, and _scd_type_2 calls in a single transaction so COPY staging DDL, target writes, and cleanup commit or roll back atomically - Protect COMMENT ON statements inside active transactions with a SAVEPOINT so swallowed comment permission/policy failures do not leave psycopg2 in PQTRANS_INERROR and silently roll back DataFrame loads on commit - Track standalone staging table commits in _df_to_source_queries, roll back aborted 25P02 transactions before retrying DROP TABLE, and never mask propagating caller errors with secondary drop failures - Fall back to VALUES when non-TEXT staging columns target JSON or JSONB while preserving COPY for TEXT JSON strings - Use CRLF CSV row terminators in _copy_df so Python 3.9 csv.QUOTE_MINIMAL quotes bare carriage returns, and clarify staging table behavior in Postgres docs Signed-off-by: Vamsi-klu <nrvamsi13@gmail.com>
1 parent 889bcc2 commit 4ca3a67

4 files changed

Lines changed: 825 additions & 25 deletions

File tree

‎docs/integrations/engines/postgres.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,12 @@ pip install "sqlmesh[postgres]"
2626

2727
### DataFrame loading
2828

29-
When using the `psycopg2` driver, SQLMesh loads pandas DataFrames (such as those returned by Python models and used for seeds) with `COPY ... FROM STDIN (FORMAT csv)` instead of batched `INSERT ... VALUES` statements, which is much faster for large DataFrames. The data is copied into a temporary staging table, which is created in the same schema as the target table and then cast into the target table's column types. The staging table is dropped afterwards, including when the load fails. Because it is created in the target's schema, the connecting user needs the `CREATE` privilege on that schema, as for other SQLMesh DDL.
29+
When using the `psycopg2` driver, SQLMesh loads pandas DataFrames (such as those returned by Python models and used for seeds) with `COPY ... FROM STDIN (FORMAT csv)` instead of batched `INSERT ... VALUES` statements, which is much faster for large DataFrames. The data is copied into a regular schema-qualified staging table (`__temp_...`, rather than a session-local PostgreSQL `TEMP` table) created in the same schema as the target table and then cast into the target table's column types. SQLMesh DataFrame loads run inside a transaction and drop the staging table at the end of the load (or roll back its creation if the load fails inside the transaction); if a staging query is run outside an enclosing transaction, default schema privileges apply while the table exists and an abrupt connection drop or process crash may leave a `__temp_*` table behind that operators can remove. Because the staging table is created in the target's schema, the connecting user needs the `CREATE` privilege on that schema, as for other SQLMesh DDL.
3030

3131
No configuration is required and there is no option to disable this behavior. SQLMesh automatically falls back to `INSERT ... VALUES` when:
3232

3333
- A column has a type or values that cannot be loaded with `COPY` identically. Supported columns are integers (up to 64 bits signed, up to 32 bits unsigned), `float64`, booleans, timestamps (with or without time zone), and strings. Other types, such as `float32`, 64-bit unsigned integers, `Decimal`, dates, bytes, dicts, lists, and categoricals, trigger the fallback.
34-
- A column's staging type is incompatible with the target column's cast (for example, an integer or numeric DataFrame column targeting a `BOOLEAN`, `BIT`, or `OID`/`REG*` type).
34+
- A column's staging type is incompatible with the target column's cast (for example, a non-text DataFrame column targeting `JSON` or `JSONB`, or an integer or numeric column targeting `BOOLEAN`, `BIT`, or `OID`/`REG*`).
3535
- The DataFrame has a single column and one of its values is `\.`.
3636
- The driver does not support `copy_expert` (for example, a driver other than `psycopg2`).
3737

‎sqlmesh/core/engine_adapter/postgres.py‎

Lines changed: 190 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
from __future__ import annotations
22

3+
import contextlib
34
import csv
45
import io
56
import logging
67
import re
8+
import sys
79
import typing as t
810
from functools import cached_property, partial
911
from sqlglot import exp
@@ -170,20 +172,28 @@ def _df_to_source_queries(
170172
)
171173

172174
staging_table = self._get_temp_table(target_table or "pandas")
175+
committed_standalone = False
173176

174177
def query_factory() -> Query:
178+
nonlocal committed_standalone
175179
# The factory can be called multiple times. The staging table is only loaded once, which
176180
# assumes that every call would have produced the same result.
177181
if not self.table_exists(staging_table):
182+
committed_standalone = False
183+
created_in_outer_tx = self._connection_pool.is_transaction_active
178184
try:
179185
with self.transaction():
180186
self.create_table(staging_table, staging_columns_to_types)
181187
self._copy_df(staging_table, ordered_df, null_marker, batch_size)
188+
if not created_in_outer_tx:
189+
committed_standalone = True
182190
except Exception:
183191
# Callers such as `replace_query` invoke the factory outside of `with source_query`,
184192
# so `cleanup_func` won't run and a committed staging table would leak.
185193
try:
186-
self._drop_copy_staging_table(staging_table)
194+
self._drop_copy_staging_table(
195+
staging_table, committed_standalone=committed_standalone
196+
)
187197
except Exception:
188198
logger.warning("Failed to drop staging table %s", staging_table.sql())
189199
raise
@@ -194,7 +204,9 @@ def query_factory() -> Query:
194204
return [
195205
SourceQuery(
196206
query_factory=query_factory,
197-
cleanup_func=lambda: self._drop_copy_staging_table(staging_table),
207+
cleanup_func=lambda: self._drop_copy_staging_table(
208+
staging_table, committed_standalone=committed_standalone
209+
),
198210
)
199211
]
200212

@@ -259,17 +271,51 @@ def _check_identifier_length(self, expression: exp.Expr) -> None:
259271
f"Identifier name '{name}' (length {name_bytes} bytes) exceeds {self.dialect.capitalize()}'s max identifier limit of {self.MAX_IDENTIFIER_LENGTH} bytes"
260272
)
261273

262-
def _drop_copy_staging_table(self, staging_table: exp.Table) -> None:
274+
def _drop_copy_staging_table(
275+
self, staging_table: exp.Table, *, committed_standalone: bool = False
276+
) -> None:
277+
exc_propagating = sys.exc_info()[0] is not None
263278
try:
264279
self.drop_table(staging_table)
265280
except Exception as ex:
266-
# The cleanup runs while an exception is propagating. If it left the transaction aborted
267-
# the DROP fails with "current transaction is aborted", which would hide the original
268-
# error. The staging table was created in that transaction, so its rollback removes it.
269281
sqlstate = getattr(ex, "pgcode", None) or getattr(ex, "sqlstate", None)
270-
if sqlstate != "25P02" or not self._connection_pool.is_transaction_active:
271-
raise
272-
logger.debug("Skipped dropping %s from an aborted transaction", staging_table.sql())
282+
if (
283+
sqlstate == "25P02"
284+
and self._connection_pool.is_transaction_active
285+
and not committed_standalone
286+
):
287+
# The staging table was created in the active (aborted) outer transaction, so its
288+
# rollback removes it without masking the caller's error.
289+
logger.debug("Skipped dropping %s from an aborted transaction", staging_table.sql())
290+
return
291+
if sqlstate == "25P02" and committed_standalone:
292+
# The staging table was already committed in its own transaction before a subsequent
293+
# statement aborted the connection's transaction, so rollback alone will not drop it.
294+
was_in_tx = self._connection_pool.is_transaction_active
295+
try:
296+
self._connection_pool.rollback()
297+
with self.transaction():
298+
self.drop_table(staging_table)
299+
if was_in_tx and not exc_propagating:
300+
self._connection_pool.begin()
301+
return
302+
except Exception:
303+
if exc_propagating:
304+
logger.warning(
305+
"Failed to drop staging table %s after rolling back aborted transaction",
306+
staging_table.sql(),
307+
exc_info=True,
308+
)
309+
return
310+
raise
311+
if exc_propagating:
312+
logger.warning(
313+
"Failed to drop staging table %s",
314+
staging_table.sql(),
315+
exc_info=True,
316+
)
317+
return
318+
raise
273319

274320
@staticmethod
275321
def _is_oid_or_reg_type(data_type: exp.DataType) -> bool:
@@ -381,6 +427,10 @@ def _get_copy_staging_columns_to_types(
381427
staging_type = None
382428
if staging_type is None:
383429
return None
430+
if staging_type != "TEXT" and target_type.is_type(
431+
exp.DataType.Type.JSON, exp.DataType.Type.JSONB
432+
):
433+
return None
384434
staging_columns_to_types[col_name] = exp.DataType.build(
385435
staging_type, dialect=PostgresEngineAdapter.DIALECT
386436
)
@@ -408,12 +458,130 @@ def _copy_df(
408458
index=False,
409459
na_rep=null_marker,
410460
quoting=csv.QUOTE_MINIMAL,
411-
lineterminator="\n",
461+
lineterminator="\r\n",
412462
)
413463
buffer.seek(0)
414464
# A text buffer is encoded by the driver with the connection's client_encoding
415465
self.cursor.copy_expert(copy_sql, buffer)
416466

467+
def replace_query(
468+
self,
469+
table_name: TableName,
470+
query_or_df: QueryOrDF,
471+
target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
472+
table_description: t.Optional[str] = None,
473+
column_descriptions: t.Optional[t.Dict[str, str]] = None,
474+
source_columns: t.Optional[t.List[str]] = None,
475+
supports_replace_table_override: t.Optional[bool] = None,
476+
**kwargs: t.Any,
477+
) -> None:
478+
import pandas as pd
479+
480+
with self.transaction(condition=isinstance(query_or_df, pd.DataFrame)):
481+
return super().replace_query(
482+
table_name,
483+
query_or_df,
484+
target_columns_to_types=target_columns_to_types,
485+
table_description=table_description,
486+
column_descriptions=column_descriptions,
487+
source_columns=source_columns,
488+
supports_replace_table_override=supports_replace_table_override,
489+
**kwargs,
490+
)
491+
492+
def ctas(
493+
self,
494+
table_name: TableName,
495+
query_or_df: QueryOrDF,
496+
target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None,
497+
exists: bool = True,
498+
table_description: t.Optional[str] = None,
499+
column_descriptions: t.Optional[t.Dict[str, str]] = None,
500+
source_columns: t.Optional[t.List[str]] = None,
501+
**kwargs: t.Any,
502+
) -> None:
503+
import pandas as pd
504+
505+
with self.transaction(condition=isinstance(query_or_df, pd.DataFrame)):
506+
return super().ctas(
507+
table_name,
508+
query_or_df,
509+
target_columns_to_types=target_columns_to_types,
510+
exists=exists,
511+
table_description=table_description,
512+
column_descriptions=column_descriptions,
513+
source_columns=source_columns,
514+
**kwargs,
515+
)
516+
517+
def _scd_type_2(
518+
self,
519+
target_table: TableName,
520+
source_table: QueryOrDF,
521+
*args: t.Any,
522+
**kwargs: t.Any,
523+
) -> None:
524+
import pandas as pd
525+
526+
with self.transaction(condition=isinstance(source_table, pd.DataFrame)):
527+
super()._scd_type_2(target_table, source_table, *args, **kwargs)
528+
529+
@contextlib.contextmanager
530+
def _comment_savepoint(self) -> t.Iterator[None]:
531+
if (
532+
self._connection_pool.is_transaction_active is not True
533+
or not self.SUPPORTS_TRANSACTIONS
534+
):
535+
yield
536+
return
537+
538+
savepoint = "_sqlmesh_comment"
539+
self.execute(f"SAVEPOINT {savepoint}")
540+
try:
541+
yield
542+
except Exception:
543+
try:
544+
self.execute(f"ROLLBACK TO SAVEPOINT {savepoint}")
545+
self.execute(f"RELEASE SAVEPOINT {savepoint}")
546+
except Exception:
547+
logger.debug("Failed to roll back savepoint %s", savepoint, exc_info=True)
548+
raise
549+
else:
550+
self.execute(f"RELEASE SAVEPOINT {savepoint}")
551+
552+
def _create_table_comment(
553+
self, table_name: TableName, table_comment: str, table_kind: str = "TABLE"
554+
) -> None:
555+
table = exp.to_table(table_name)
556+
try:
557+
with self._comment_savepoint():
558+
self.execute(self._build_create_comment_table_exp(table, table_comment, table_kind))
559+
except Exception:
560+
logger.warning(
561+
f"Table comment for '{table.alias_or_name}' not registered - this may be due to limited permissions",
562+
exc_info=True,
563+
)
564+
565+
def _create_column_comments(
566+
self,
567+
table_name: TableName,
568+
column_comments: t.Dict[str, str],
569+
table_kind: str = "TABLE",
570+
materialized_view: bool = False,
571+
) -> None:
572+
table = exp.to_table(table_name)
573+
for col, comment in column_comments.items():
574+
try:
575+
with self._comment_savepoint():
576+
self.execute(
577+
self._build_create_comment_column_exp(table, col, comment, table_kind)
578+
)
579+
except Exception:
580+
logger.warning(
581+
f"Column comments for column '{col}' in table '{table.alias_or_name}' not registered - this may be due to limited permissions",
582+
exc_info=True,
583+
)
584+
417585
def _create_table_like(
418586
self,
419587
target_table_name: TableName,
@@ -448,18 +616,21 @@ def merge(
448616
source_columns: t.Optional[t.List[str]] = None,
449617
**kwargs: t.Any,
450618
) -> None:
619+
import pandas as pd
620+
451621
# Merge isn't supported until Postgres 15
452622
major, minor = self.server_version
453623
merge_impl = super().merge if major >= 15 else partial(logical_merge, self)
454-
merge_impl( # type: ignore
455-
target_table,
456-
source_table,
457-
target_columns_to_types,
458-
unique_key,
459-
when_matched=when_matched,
460-
merge_filter=merge_filter,
461-
source_columns=source_columns,
462-
)
624+
with self.transaction(condition=isinstance(source_table, pd.DataFrame)):
625+
merge_impl( # type: ignore
626+
target_table,
627+
source_table,
628+
target_columns_to_types,
629+
unique_key,
630+
when_matched=when_matched,
631+
merge_filter=merge_filter,
632+
source_columns=source_columns,
633+
)
463634

464635
@cached_property
465636
def server_version(self) -> t.Tuple[int, int]:

0 commit comments

Comments
 (0)