diff --git a/gridsight-ext/src/gridsight_gat/_partitions.py b/gridsight-ext/src/gridsight_gat/_partitions.py new file mode 100644 index 0000000..55b4041 --- /dev/null +++ b/gridsight-ext/src/gridsight_gat/_partitions.py @@ -0,0 +1,34 @@ +"""Partition layout helpers (no GridSight SDK import, so tests can load +this file on its own).""" + +from __future__ import annotations + + +def transpose_day(wide, ts_cols: list[str]): + """Turn entity rows x hour columns into the partition layout the web + client reads: a ``datetime`` column (one row per hour) plus one float64 + column per entity, entities sorted by id. Duplicate entity ids keep the + first row (as PIVOT ... USING first(val) did).""" + import numpy as np + import pyarrow as pa + + ids = wide.column("entity_id").to_pylist() + first: dict[str, int] = {} + for i, e in enumerate(ids): + if e is not None and e not in first: + first[e] = i + order = sorted(first) + rows = np.fromiter((first[e] for e in order), dtype=np.int64, count=len(order)) + mat = np.column_stack( + [ + wide.column(c).to_numpy(zero_copy_only=False).astype("float64") + for c in ts_cols + ] + )[ + rows + ] # entities x hours; NaN where the stored value is NULL + arrays = [pa.array(ts_cols, type=pa.string())] + arrays += [ + pa.array(mat[j], type=pa.float64(), from_pandas=True) for j in range(len(order)) + ] + return pa.Table.from_arrays(arrays, names=["datetime", *order]) diff --git a/gridsight-ext/src/gridsight_gat/extension.py b/gridsight-ext/src/gridsight_gat/extension.py index 99bc8c5..639cc43 100644 --- a/gridsight-ext/src/gridsight_gat/extension.py +++ b/gridsight-ext/src/gridsight_gat/extension.py @@ -15,6 +15,7 @@ import io import json +import re import logging import os import threading @@ -322,6 +323,16 @@ def _schema_name(project: str, scenario: str, model: str) -> str: return f"{project}__{scenario}__{model}".replace("-", "_").replace(" ", "_").lower() +def _view_prefix(project: str, scenario: str) -> str: + """Prefix of the per-scenario DuckDB views (``ext_gat.__``). + + Project and scenario names are user text ("Summer peak 2019"), so every + character that is not valid in an unquoted SQL identifier becomes ``_``. + Unquoted names keep the views usable in chart templates and in SQL that + an agent writes.""" + return re.sub(r"\W", "_", f"{project}_{scenario}") + + _GAT_DB_RESOLVED: Path | None = None @@ -398,6 +409,7 @@ def _gat_db_path() -> str: from gridsight_gat._gateway import _DuckDBGateway +from ._partitions import transpose_day class GATExtension(GridSightExtensionBase): @@ -3526,20 +3538,18 @@ def get_dataset_partition(self, dataset_name: str, date_str: str) -> bytes: f"No data for date {date_str} in {dataset_name}" ) - # Transpose via UNPIVOT + PIVOT - unpivot_cols = ", ".join(f'"{c}"' for c in ts_cols) - sql = ( - f"WITH unpivoted AS (" - f" SELECT entity_id, ts AS datetime, CAST(val AS DOUBLE) AS val " - f" FROM gat_db.{schema}.{table_name} " - f" UNPIVOT (val FOR ts IN ({unpivot_cols}))" - f" WHERE ts >= '{date_str}' AND ts < '{date_str}T99'" - f") " - f"PIVOT unpivoted ON entity_id USING first(val) " - f"ORDER BY datetime" + # Read the day's columns as stored (entity rows x hour columns) + # and transpose in Arrow. A DuckDB UNPIVOT + PIVOT gives the same + # table, but pivoting on ~10k entity ids (line flows) allocated + # several GB that DuckDB then keeps, so the server's memory grew + # from ~0.5 GB to ~7 GB on the first partition request. + select_cols = ", ".join(f'CAST("{c}" AS DOUBLE) AS "{c}"' for c in ts_cols) + wide = self._db.fetch_arrow( + f"SELECT CAST(entity_id AS VARCHAR) AS entity_id, {select_cols} " + f"FROM gat_db.{schema}.{table_name}" ) - table = self._db.fetch_arrow(sql) + table = transpose_day(wide, ts_cols) # Serialize to Arrow IPC (no lock needed) sink = io.BytesIO() @@ -3644,7 +3654,7 @@ def register_duckdb_tables(self, conn: Any) -> list[DuckDBTableInfo]: # has a chance to issue concurrent gat_db queries. for s in scenarios: schema = s["schema_name"] - prefix = f"{s['project']}_{s['scenario']}".replace("-", "_") + prefix = _view_prefix(s["project"], s["scenario"]) try: self._ensure_catmap_display_group(schema, prefix) except Exception as e: @@ -3656,7 +3666,7 @@ def register_duckdb_tables(self, conn: Any) -> list[DuckDBTableInfo]: for s in scenarios: schema = s["schema_name"] - prefix = f"{s['project']}_{s['scenario']}".replace("-", "_") + prefix = _view_prefix(s["project"], s["scenario"]) gat_tables = self._get_schema_tables(schema) for table_name in gat_tables: @@ -3702,7 +3712,7 @@ def get_ai_schema(self) -> AISchema | None: tables: list[AITableSchema] = [] for s in scenarios: schema = s["schema_name"] - prefix = f"{s['project']}_{s['scenario']}".replace("-", "_") + prefix = _view_prefix(s["project"], s["scenario"]) gat_tables = self._get_schema_tables(schema) for table_name in ("generation", "load", "line_flow"): @@ -3801,7 +3811,7 @@ def _list_chart_specs_locked(self) -> list[ChartSpec]: for s in scenarios: schema = s["schema_name"] - prefix = f"{s['project']}_{s['scenario']}".replace("-", "_") + prefix = _view_prefix(s["project"], s["scenario"]) # Track where this scenario's specs start so we can stamp # project/scenario on them in one shot at the end (lets the # frontend filter the charts menu by active scenario). diff --git a/gridsight-ext/tests/test_partitions.py b/gridsight-ext/tests/test_partitions.py new file mode 100644 index 0000000..8f9e046 --- /dev/null +++ b/gridsight-ext/tests/test_partitions.py @@ -0,0 +1,80 @@ +"""``transpose_day`` must give the same partition as the PIVOT query it +replaced (one ``datetime`` row per hour, one float64 column per entity, +entities sorted, first row wins for duplicate ids), without the PIVOT's +memory cost on wide tables.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import duckdb +import pytest + +_PATH = Path(__file__).resolve().parents[1] / "src" / "gridsight_gat" / "_partitions.py" +_spec = importlib.util.spec_from_file_location("_partitions", _PATH) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +transpose_day = _mod.transpose_day + +DATE = "2019-08-15" +HOURS = [f"{DATE}T{h:02d}:00:00" for h in range(3)] + + +@pytest.fixture() +def con(): + c = duckdb.connect() + cols = ", ".join(f'"{h}" FLOAT' for h in HOURS + ["2019-08-16T00:00:00"]) + c.execute(f"CREATE TABLE flows (entity_id VARCHAR, {cols})") + rows = [ + ("line-b", 1.5, None, -2.0, 9.0), + ("line-a", 0.0, 3.25, 4.0, 9.0), + ("line-c", -1.0, -1.0, None, 9.0), + ("line-a", 99.0, 99.0, 99.0, 9.0), # duplicate id: first row wins + ] + c.executemany("INSERT INTO flows VALUES (?, ?, ?, ?, ?)", rows) + return c + + +def _arrow(c, sql): + res = c.execute(sql) + return ( + res.to_arrow_table() + if hasattr(res, "to_arrow_table") + else res.fetch_arrow_table() + ) + + +def test_matches_pivot(con): + up = ", ".join(f'"{h}"' for h in HOURS) + old = _arrow( + con, + f"WITH u AS (SELECT entity_id, ts AS datetime, CAST(val AS DOUBLE) AS val " + f"FROM flows UNPIVOT INCLUDE NULLS (val FOR ts IN ({up}))) " + f"PIVOT u ON entity_id USING first(val) ORDER BY datetime", + ) + sel = ", ".join(f'CAST("{h}" AS DOUBLE) AS "{h}"' for h in HOURS) + wide = _arrow( + con, f"SELECT CAST(entity_id AS VARCHAR) AS entity_id, {sel} FROM flows" + ) + new = transpose_day(wide, HOURS) + + assert new.column_names == ["datetime", "line-a", "line-b", "line-c"] + assert new.column_names == old.column_names + assert new.column("datetime").to_pylist() == HOURS + assert new.column("line-a").to_pylist() == [0.0, 3.25, 4.0] + assert new.column("line-b").to_pylist() == [1.5, None, -2.0] + assert new.column("line-c").to_pylist() == [-1.0, -1.0, None] + for name in ("line-b", "line-c"): + assert new.column(name).to_pylist() == old.column(name).to_pylist() + + +def test_empty_table(con): + con.execute("DELETE FROM flows") + sel = ", ".join(f'CAST("{h}" AS DOUBLE) AS "{h}"' for h in HOURS) + wide = _arrow( + con, f"SELECT CAST(entity_id AS VARCHAR) AS entity_id, {sel} FROM flows" + ) + new = transpose_day(wide, HOURS) + assert new.column_names == ["datetime"] + assert new.num_rows == len(HOURS)