Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions gridsight-ext/src/gridsight_gat/_partitions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
"""Partition layout helpers (no GridSight SDK import, so tests can load
this file on its own)."""

from __future__ import annotations


def transpose_day(wide, ts_cols: list[str]):
"""Turn entity rows x hour columns into the partition layout the web
client reads: a ``datetime`` column (one row per hour) plus one float64
column per entity, entities sorted by id. Duplicate entity ids keep the
first row (as PIVOT ... USING first(val) did)."""
import numpy as np
import pyarrow as pa

ids = wide.column("entity_id").to_pylist()
first: dict[str, int] = {}
for i, e in enumerate(ids):
if e is not None and e not in first:
first[e] = i
order = sorted(first)
rows = np.fromiter((first[e] for e in order), dtype=np.int64, count=len(order))
mat = np.column_stack(
[
wide.column(c).to_numpy(zero_copy_only=False).astype("float64")
for c in ts_cols
]
)[
rows
] # entities x hours; NaN where the stored value is NULL
arrays = [pa.array(ts_cols, type=pa.string())]
arrays += [
pa.array(mat[j], type=pa.float64(), from_pandas=True) for j in range(len(order))
]
return pa.Table.from_arrays(arrays, names=["datetime", *order])
42 changes: 26 additions & 16 deletions gridsight-ext/src/gridsight_gat/extension.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import io
import json
import re
import logging
import os
import threading
Expand Down Expand Up @@ -322,6 +323,16 @@ def _schema_name(project: str, scenario: str, model: str) -> str:
return f"{project}__{scenario}__{model}".replace("-", "_").replace(" ", "_").lower()


def _view_prefix(project: str, scenario: str) -> str:
"""Prefix of the per-scenario DuckDB views (``ext_gat.<prefix>__<table>``).

Project and scenario names are user text ("Summer peak 2019"), so every
character that is not valid in an unquoted SQL identifier becomes ``_``.
Unquoted names keep the views usable in chart templates and in SQL that
an agent writes."""
return re.sub(r"\W", "_", f"{project}_{scenario}")


_GAT_DB_RESOLVED: Path | None = None


Expand Down Expand Up @@ -398,6 +409,7 @@ def _gat_db_path() -> str:


from gridsight_gat._gateway import _DuckDBGateway
from ._partitions import transpose_day


class GATExtension(GridSightExtensionBase):
Expand Down Expand Up @@ -3526,20 +3538,18 @@ def get_dataset_partition(self, dataset_name: str, date_str: str) -> bytes:
f"No data for date {date_str} in {dataset_name}"
)

# Transpose via UNPIVOT + PIVOT
unpivot_cols = ", ".join(f'"{c}"' for c in ts_cols)
sql = (
f"WITH unpivoted AS ("
f" SELECT entity_id, ts AS datetime, CAST(val AS DOUBLE) AS val "
f" FROM gat_db.{schema}.{table_name} "
f" UNPIVOT (val FOR ts IN ({unpivot_cols}))"
f" WHERE ts >= '{date_str}' AND ts < '{date_str}T99'"
f") "
f"PIVOT unpivoted ON entity_id USING first(val) "
f"ORDER BY datetime"
# Read the day's columns as stored (entity rows x hour columns)
# and transpose in Arrow. A DuckDB UNPIVOT + PIVOT gives the same
# table, but pivoting on ~10k entity ids (line flows) allocated
# several GB that DuckDB then keeps, so the server's memory grew
# from ~0.5 GB to ~7 GB on the first partition request.
select_cols = ", ".join(f'CAST("{c}" AS DOUBLE) AS "{c}"' for c in ts_cols)
wide = self._db.fetch_arrow(
f"SELECT CAST(entity_id AS VARCHAR) AS entity_id, {select_cols} "
f"FROM gat_db.{schema}.{table_name}"
)

table = self._db.fetch_arrow(sql)
table = transpose_day(wide, ts_cols)

# Serialize to Arrow IPC (no lock needed)
sink = io.BytesIO()
Expand Down Expand Up @@ -3644,7 +3654,7 @@ def register_duckdb_tables(self, conn: Any) -> list[DuckDBTableInfo]:
# has a chance to issue concurrent gat_db queries.
for s in scenarios:
schema = s["schema_name"]
prefix = f"{s['project']}_{s['scenario']}".replace("-", "_")
prefix = _view_prefix(s["project"], s["scenario"])
try:
self._ensure_catmap_display_group(schema, prefix)
except Exception as e:
Expand All @@ -3656,7 +3666,7 @@ def register_duckdb_tables(self, conn: Any) -> list[DuckDBTableInfo]:

for s in scenarios:
schema = s["schema_name"]
prefix = f"{s['project']}_{s['scenario']}".replace("-", "_")
prefix = _view_prefix(s["project"], s["scenario"])
gat_tables = self._get_schema_tables(schema)

for table_name in gat_tables:
Expand Down Expand Up @@ -3702,7 +3712,7 @@ def get_ai_schema(self) -> AISchema | None:
tables: list[AITableSchema] = []
for s in scenarios:
schema = s["schema_name"]
prefix = f"{s['project']}_{s['scenario']}".replace("-", "_")
prefix = _view_prefix(s["project"], s["scenario"])
gat_tables = self._get_schema_tables(schema)

for table_name in ("generation", "load", "line_flow"):
Expand Down Expand Up @@ -3801,7 +3811,7 @@ def _list_chart_specs_locked(self) -> list[ChartSpec]:

for s in scenarios:
schema = s["schema_name"]
prefix = f"{s['project']}_{s['scenario']}".replace("-", "_")
prefix = _view_prefix(s["project"], s["scenario"])
# Track where this scenario's specs start so we can stamp
# project/scenario on them in one shot at the end (lets the
# frontend filter the charts menu by active scenario).
Expand Down
80 changes: 80 additions & 0 deletions gridsight-ext/tests/test_partitions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
"""``transpose_day`` must give the same partition as the PIVOT query it
replaced (one ``datetime`` row per hour, one float64 column per entity,
entities sorted, first row wins for duplicate ids), without the PIVOT's
memory cost on wide tables."""

from __future__ import annotations

import importlib.util
from pathlib import Path

import duckdb
import pytest

_PATH = Path(__file__).resolve().parents[1] / "src" / "gridsight_gat" / "_partitions.py"
_spec = importlib.util.spec_from_file_location("_partitions", _PATH)
_mod = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(_mod)
transpose_day = _mod.transpose_day

DATE = "2019-08-15"
HOURS = [f"{DATE}T{h:02d}:00:00" for h in range(3)]


@pytest.fixture()
def con():
c = duckdb.connect()
cols = ", ".join(f'"{h}" FLOAT' for h in HOURS + ["2019-08-16T00:00:00"])
c.execute(f"CREATE TABLE flows (entity_id VARCHAR, {cols})")
rows = [
("line-b", 1.5, None, -2.0, 9.0),
("line-a", 0.0, 3.25, 4.0, 9.0),
("line-c", -1.0, -1.0, None, 9.0),
("line-a", 99.0, 99.0, 99.0, 9.0), # duplicate id: first row wins
]
c.executemany("INSERT INTO flows VALUES (?, ?, ?, ?, ?)", rows)
return c


def _arrow(c, sql):
res = c.execute(sql)
return (
res.to_arrow_table()
if hasattr(res, "to_arrow_table")
else res.fetch_arrow_table()
)


def test_matches_pivot(con):
up = ", ".join(f'"{h}"' for h in HOURS)
old = _arrow(
con,
f"WITH u AS (SELECT entity_id, ts AS datetime, CAST(val AS DOUBLE) AS val "
f"FROM flows UNPIVOT INCLUDE NULLS (val FOR ts IN ({up}))) "
f"PIVOT u ON entity_id USING first(val) ORDER BY datetime",
)
sel = ", ".join(f'CAST("{h}" AS DOUBLE) AS "{h}"' for h in HOURS)
wide = _arrow(
con, f"SELECT CAST(entity_id AS VARCHAR) AS entity_id, {sel} FROM flows"
)
new = transpose_day(wide, HOURS)

assert new.column_names == ["datetime", "line-a", "line-b", "line-c"]
assert new.column_names == old.column_names
assert new.column("datetime").to_pylist() == HOURS
assert new.column("line-a").to_pylist() == [0.0, 3.25, 4.0]
assert new.column("line-b").to_pylist() == [1.5, None, -2.0]
assert new.column("line-c").to_pylist() == [-1.0, -1.0, None]
for name in ("line-b", "line-c"):
assert new.column(name).to_pylist() == old.column(name).to_pylist()


def test_empty_table(con):
con.execute("DELETE FROM flows")
sel = ", ".join(f'CAST("{h}" AS DOUBLE) AS "{h}"' for h in HOURS)
wide = _arrow(
con, f"SELECT CAST(entity_id AS VARCHAR) AS entity_id, {sel} FROM flows"
)
new = transpose_day(wide, HOURS)
assert new.column_names == ["datetime"]
assert new.num_rows == len(HOURS)
Loading