diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 89c303af..ed4acb0b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,3 +125,63 @@ jobs: - name: Run type checker (mypy) run: | mypy . + + no-extras: + # The base install must work without polars, dask, tqdm and deprecation. + # This job catches a hard import reintroduced into the core path. + name: Core install without optional dependencies + needs: [build] + runs-on: ubuntu-latest + + steps: + - name: Set up Python 3.12 + uses: actions/setup-python@v7 + with: + python-version: 3.12 + + - name: Download package + uses: actions/download-artifact@v8 + with: + name: Packages + path: dist + + - name: Install the wheel with no extras + run: | + python -m pip install uv + uv pip install --system "$(ls dist/*.whl)" + uv pip install --system highspy + + - name: Smoke test the core install + run: | + python - <<'PY' + import importlib.util + + # The wheel with no extras must not pull in any of the optional extras. + for optional in ("polars", "dask", "tqdm"): + assert importlib.util.find_spec(optional) is None, optional + + import pandas as pd + + from linopy import Model + + m = Model() + t = pd.RangeIndex(6, name="t") + x = m.add_variables(lower=0, coords=[t], name="x") + y = m.add_variables(lower=0, coords=[t], name="y") + m.add_constraints(x + y >= 1) + m.add_objective(2 * x + 3 * y) + + # xarray's ffill/bfill import bottleneck directly, with no numpy + # fallback, so bottleneck has to stay a declared dependency. + x.ffill("t") + x.bfill("t") + + status, condition = m.solve(io_api="direct") + assert status == "ok", (status, condition) + # One x per time step at cost 2, since x + y >= 1 and 2x + 3y. + assert abs(float(m.objective.value) - 2.0 * len(t)) < 1e-9 + + m.to_file("smoke.mps", io_api="mps") + + print("core install smoke test passed") + PY diff --git a/doc/prerequisites.rst b/doc/prerequisites.rst index b8f1ea82..f50e918a 100644 --- a/doc/prerequisites.rst +++ b/doc/prerequisites.rst @@ -25,6 +25,37 @@ or conda install -c conda-forge linopy +Optional dependencies +--------------------- + +The base install pulls numpy, scipy, xarray, pandas, packaging and bottleneck. +Three features need an extra: + +``linopy[polars]`` + The LP file writer and the ``to_polars()`` export methods. This is required + by the default ``Model.solve()``, which writes an LP file. Install it if you + want the default solve path or any polars export: + + .. code:: bash + + uv pip install "linopy[polars]" + + Without it, ``Model.solve()`` raises ``ImportError``. Pass + ``io_api="direct"`` to solve without writing a file, or ``io_api="mps"`` to + write an MPS file instead (the MPS writer uses highspy). + +``linopy[dask]`` + Chunked (dask-backed) arrays, for example via ``Variable.chunk()``. Plain + numpy and xarray models do not need it. + +``linopy[progress]`` + Progress bars while writing LP files. Without it, LP export runs silently. + +.. code:: bash + + uv pip install "linopy[polars,dask,progress]" + + Install a solver ---------------- diff --git a/linopy/_polars.py b/linopy/_polars.py new file mode 100644 index 00000000..113c3cb6 --- /dev/null +++ b/linopy/_polars.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +""" +Placeholder used in place of :mod:`polars` when it is not installed. + +Polars backs the ``to_polars()`` export methods and the LP file writer, which +the default ``Model.solve()`` uses. It is an optional dependency, so every +module that touches it imports it as:: + + try: + import polars as pl + except ImportError: + from linopy._polars import missing_polars as pl + +Attribute access on the placeholder raises :class:`ImportError` with an install +hint, so the failure surfaces at the first real use rather than at import time. +""" + +from typing import Any + +_INSTALL_HINT = ( + "polars is required for this feature. Install it with `pip install linopy[polars]`." +) + + +class _MissingPolars: + """Stand-in for the ``polars`` module; every attribute access raises.""" + + def __getattr__(self, name: str) -> Any: + # Tools probe dunders such as ``__wrapped__`` and expect AttributeError. + # Raising ImportError there breaks pytest's doctest collection. + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + raise ImportError(_INSTALL_HINT) + + +missing_polars = _MissingPolars() diff --git a/linopy/alignment.py b/linopy/alignment.py index 1f521fd9..0113484c 100644 --- a/linopy/alignment.py +++ b/linopy/alignment.py @@ -24,11 +24,10 @@ from collections import Counter from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence from functools import partial -from typing import TYPE_CHECKING, Any, Literal, NamedTuple, overload +from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast, overload import numpy as np import pandas as pd -import polars as pl import scipy.sparse from numpy import arange from xarray import Coordinates, DataArray, Dataset, broadcast @@ -50,7 +49,12 @@ enforce_aux_conflict, first_mismatched_dim, ) -from linopy.types import UNLABELED_TYPES, CoordsLike, DimsLike +from linopy.types import ( + POLARS_SERIES_TYPES, + UNLABELED_TYPES, + CoordsLike, + DimsLike, +) def as_constant(other: Any) -> Any: @@ -416,8 +420,11 @@ def as_dataarray( arr = pandas_to_dataarray(arr, coords=coords, dims=dims, **kwargs) elif isinstance(arr, np.ndarray): arr = numpy_to_dataarray(arr, coords=coords, dims=dims, **kwargs) - elif isinstance(arr, pl.Series): - arr = numpy_to_dataarray(arr.to_numpy(), coords=coords, dims=dims, **kwargs) + elif isinstance(arr, POLARS_SERIES_TYPES): + # mypy cannot narrow `arr` through the runtime polars type tuple. + arr = numpy_to_dataarray( + cast(Any, arr).to_numpy(), coords=coords, dims=dims, **kwargs + ) elif isinstance(arr, np.number | int | float | str | bool | list): if isinstance(arr, np.number): arr = float(arr) @@ -438,7 +445,7 @@ def as_dataarray( pd.DataFrame, np.ndarray, DataArray, - pl.Series, + *POLARS_SERIES_TYPES, ] supported_types_str = ", ".join([t.__name__ for t in supported_types]) raise TypeError( diff --git a/linopy/common.py b/linopy/common.py index 16b1da0c..f4ba1726 100644 --- a/linopy/common.py +++ b/linopy/common.py @@ -17,15 +17,18 @@ import numpy as np import pandas as pd -import polars as pl from numpy import nan, signedinteger from numpy.typing import DTypeLike -from polars.datatypes import DataTypeClass from xarray import DataArray, Dataset, apply_ufunc, broadcast from xarray import align as xr_align from xarray.core import indexing from xarray.namedarray.utils import is_dict_like +try: + import polars as pl +except ImportError: + from linopy._polars import missing_polars as pl # type: ignore[no-redef] + from linopy.alignment import _as_index from linopy.config import options from linopy.constants import ( @@ -148,34 +151,6 @@ def check_has_nulls(df: pd.DataFrame, name: str) -> None: raise ValueError(f"Fields {name} contains nan's in field(s) {fields}") -def infer_schema_polars(ds: Dataset) -> dict[str, DataTypeClass]: - """ - Infer the polars data schema from a xarray dataset. - - Args: - ---- - ds (polars.DataFrame): The Polars DataFrame for which to infer the schema. - - Returns: - ------- - dict: A dictionary mapping column names to their corresponding Polars data types. - """ - schema: dict[str, DataTypeClass] = {} - for name, array in ds.items(): - name = str(name) - if np.issubdtype(array.dtype, np.integer): - schema[name] = pl.Int32 if array.dtype.itemsize <= 4 else pl.Int64 - elif np.issubdtype(array.dtype, np.floating): - schema[name] = pl.Float64 - elif np.issubdtype(array.dtype, np.bool_): - schema[name] = pl.Boolean - elif np.issubdtype(array.dtype, np.object_): - schema[name] = pl.Object - else: - schema[name] = pl.Utf8 - return schema - - def to_polars(ds: Dataset, **kwargs: Any) -> pl.DataFrame: """ Convert an xarray Dataset to a polars DataFrame. diff --git a/linopy/constraints.py b/linopy/constraints.py index 50014bf9..1a798c2f 100644 --- a/linopy/constraints.py +++ b/linopy/constraints.py @@ -32,7 +32,6 @@ import numpy as np import pandas as pd -import polars as pl import scipy.sparse import xarray as xr from numpy import array, ndarray @@ -41,6 +40,11 @@ from xarray.core.indexes import Indexes from xarray.core.utils import Frozen +try: + import polars as pl +except ImportError: + from linopy._polars import missing_polars as pl # type: ignore[no-redef] + from linopy import expressions, variables from linopy.alignment import as_dataarray, broadcast_to_coords from linopy.common import ( diff --git a/linopy/expressions.py b/linopy/expressions.py index 197482ad..40829baa 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -36,7 +36,6 @@ import numpy as np import pandas as pd -import polars as pl import scipy import xarray as xr import xarray.core.groupby @@ -60,6 +59,11 @@ from types import EllipsisType, NotImplementedType +try: + import polars as pl +except ImportError: + from linopy._polars import missing_polars as pl # type: ignore[no-redef] + from linopy import constraints, variables from linopy.alignment import ( _matmul_operand_to_dataarray, diff --git a/linopy/io.py b/linopy/io.py index 80568f18..e56ee351 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -21,10 +21,18 @@ import numpy as np import pandas as pd -import polars as pl import scipy.sparse import xarray as xr -from tqdm import tqdm + +try: + import polars as pl +except ImportError: + from linopy._polars import missing_polars as pl # type: ignore[no-redef] + +try: + from tqdm import tqdm as _tqdm +except ImportError: + _tqdm = None from linopy import solvers from linopy.common import ( @@ -59,6 +67,13 @@ TQDM_COLOR = "#80bfff" +def tqdm(iterable: Any, **kwargs: Any) -> Any: + """Wrap in a progress bar when tqdm is installed, otherwise pass through.""" + if _tqdm is None: + return iterable + return _tqdm(iterable, **kwargs) + + name_sanitizer = str.maketrans("-+*^[] ", "_______") @@ -190,7 +205,7 @@ def get_printers_scalar( m: Model, explicit_coordinate_names: bool = False ) -> tuple[Callable, Callable]: """ - Get batch printer functions for numpy label arrays (non-polars). + Get batch printer functions for numpy label arrays. Returns two callables that take an int64 numpy array of labels and return a list of name strings. @@ -217,10 +232,10 @@ def print_constraints(labels: np.ndarray) -> list[str]: else: def print_variables(labels: np.ndarray) -> list[str]: - return ("x" + pl.Series(labels).cast(pl.String)).to_list() + return np.char.add("x", labels.astype(str)).tolist() def print_constraints(labels: np.ndarray) -> list[str]: - return ("c" + pl.Series(labels).cast(pl.String)).to_list() + return np.char.add("c", labels.astype(str)).tolist() return print_variables, print_constraints @@ -712,6 +727,20 @@ def to_lp_file( progress: bool = True, explicit_coordinate_names: bool = False, ) -> None: + """ + Write out a model to an LP file. + + Requires the optional ``polars`` dependency. Progress bars require ``tqdm``; + without it the export runs silently. + """ + try: + import polars # noqa: F401 + except ImportError as exc: + raise ImportError( + "LP file export requires polars. Install it with " + "`pip install linopy[polars]`, or pass io_api='direct' to solve()." + ) from exc + with open(fn, mode="wb") as f: start = time.time() variable_scaling = variable_scaling_lookup(m) diff --git a/linopy/model.py b/linopy/model.py index 30b42090..d705a87d 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -20,7 +20,6 @@ import numpy as np import pandas as pd import xarray as xr -from deprecation import deprecated from numpy import inf from pandas.core.frame import DataFrame from pandas.core.series import Series @@ -2815,9 +2814,6 @@ def print_infeasibilities(self, display_max_terms: int | None = None) -> None: ) print(self.format_infeasibilities(display_max_terms=display_max_terms)) - @deprecated( - details="Use `compute_infeasibilities`/`format_infeasibilities` instead." - ) def compute_set_of_infeasible_constraints(self) -> Dataset: """ Compute a set of infeasible constraints. @@ -2825,11 +2821,20 @@ def compute_set_of_infeasible_constraints(self) -> Dataset: This function requires that the model was solved with `gurobi`, `xpress` or `highs` and the termination condition was infeasible. + .. deprecated:: + Use :meth:`compute_infeasibilities`/:meth:`format_infeasibilities` instead. + Returns ------- labels : xr.DataArray Labels of the infeasible constraints. Labels with value -1 are not in the set. """ + warn( + "`Model.compute_set_of_infeasible_constraints` is deprecated. Use " + "`Model.compute_infeasibilities`/`Model.format_infeasibilities` instead.", + DeprecationWarning, + stacklevel=2, + ) labels = self.compute_infeasibilities() cons = self.constraints.labels.isin(np.array(labels)) subset = self.constraints.labels.where(cons, -1) diff --git a/linopy/objective.py b/linopy/objective.py index f3801ce9..5e56c645 100644 --- a/linopy/objective.py +++ b/linopy/objective.py @@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any import numpy as np -import polars as pl from numpy import float64 from pandas.core.frame import DataFrame from scipy.sparse._csc import csc_matrix @@ -20,6 +19,11 @@ from xarray.core.dataarray import DataArray from xarray.core.dataset import Dataset from xarray.core.indexes import Indexes + +try: + import polars as pl +except ImportError: + from linopy._polars import missing_polars as pl # type: ignore[no-redef] from xarray.core.utils import Frozen from linopy import expressions diff --git a/linopy/types.py b/linopy/types.py index 21a8dddd..7ff6a9f1 100644 --- a/linopy/types.py +++ b/linopy/types.py @@ -5,11 +5,15 @@ from typing import TYPE_CHECKING, TypeAlias, Union, get_args import numpy -import polars as pl from pandas import DataFrame, Index, Series from xarray import DataArray from xarray.core.coordinates import DataArrayCoordinates, DatasetCoordinates +try: + import polars as pl +except ImportError: + pl = None # type: ignore[assignment] + if TYPE_CHECKING: from linopy.constraints import ( AnonymousScalarConstraint, @@ -27,20 +31,38 @@ ) DimsLike: TypeAlias = str | Iterable[Hashable] -ConstantLike: TypeAlias = ( - int - | float - | numpy.floating - | numpy.integer - | numpy.ndarray - | DataArray - | Series - | DataFrame - | pl.Series -) -CONSTANT_TYPES: tuple[type, ...] = get_args(ConstantLike) +# polars is optional. When it is missing, `pl` is None and the polars entries +# drop out of the aliases and the runtime type tuples below, so `isinstance` +# checks keep working and simply never match a polars input. +POLARS_SERIES_TYPES: tuple[type, ...] = (pl.Series,) if pl is not None else () -UnlabeledLike: TypeAlias = numpy.ndarray | list | pl.Series +if pl is None: + ConstantLike: TypeAlias = ( + int + | float + | numpy.floating + | numpy.integer + | numpy.ndarray + | DataArray + | Series + | DataFrame + ) + UnlabeledLike: TypeAlias = numpy.ndarray | list +else: + ConstantLike: TypeAlias = ( # type: ignore[misc,no-redef] + int + | float + | numpy.floating + | numpy.integer + | numpy.ndarray + | DataArray + | Series + | DataFrame + | pl.Series + ) + UnlabeledLike: TypeAlias = numpy.ndarray | list | pl.Series # type: ignore[misc,no-redef] + +CONSTANT_TYPES: tuple[type, ...] = get_args(ConstantLike) UNLABELED_TYPES = get_args(UnlabeledLike) SignLike: TypeAlias = str | numpy.ndarray | DataArray | Series | DataFrame diff --git a/linopy/variables.py b/linopy/variables.py index 912aaeec..95ce1a3f 100644 --- a/linopy/variables.py +++ b/linopy/variables.py @@ -21,7 +21,6 @@ import numpy as np import pandas as pd -import polars as pl from numpy import floating, int64, issubdtype, ndarray, str_ from pandas.core.frame import DataFrame from xarray import DataArray, Dataset, broadcast @@ -30,6 +29,11 @@ from xarray.core.types import JoinOptions from xarray.core.utils import Frozen +try: + import polars as pl +except ImportError: + from linopy._polars import missing_polars as pl # type: ignore[no-redef] + import linopy.expressions as expressions from linopy.alignment import as_dataarray, broadcast_to_coords from linopy.common import ( diff --git a/pyproject.toml b/pyproject.toml index b9c0c051..2e999405 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,15 +30,12 @@ requires-python = ">=3.11" dependencies = [ "numpy", "scipy", - "bottleneck", - "toolz", - "numexpr", "xarray>=2024.2.0", - "dask>=0.18.0", - "polars>=1.31.1", - "tqdm", - "deprecation", + "pandas", "packaging", + # Variable.ffill/bfill go through xarray's push, which imports bottleneck + # directly with no numpy fallback. + "bottleneck", ] [project.urls] @@ -46,6 +43,19 @@ Homepage = "https://github.com/PyPSA/linopy" Source = "https://github.com/PyPSA/linopy" [project.optional-dependencies] +# Needed for the LP file writer (the default `Model.solve()` export path) and +# for the `to_polars()` export methods. +polars = [ + "polars>=1.31.1", +] +# Needed only for chunked (dask-backed) arrays, e.g. `Variable.chunk()`. +dask = [ + "dask>=0.18.0", +] +# Progress bars while writing LP files. +progress = [ + "tqdm", +] oetc = [ "google-cloud-storage", "requests", @@ -67,6 +77,8 @@ docs = [ "ipykernel==7.3.0", "matplotlib==3.11.0", "highspy>=1.7.1", + # The example notebooks call solve()/to_file(), which need the LP writer. + "linopy[polars]", ] dev = [ "pytest", @@ -81,6 +93,7 @@ dev = [ "highspy", "jupyter", "matplotlib", + "linopy[polars,dask,progress]", ] # Perf-relevant deps pinned exactly so run-to-run deltas reflect linopy # changes, not dependency bumps. diff --git a/test/test_io.py b/test/test_io.py index dabee6f5..0a963dcb 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -8,6 +8,7 @@ import importlib.util import json import pickle +import sys from collections.abc import Callable from pathlib import Path from typing import Any @@ -18,6 +19,7 @@ import pytest import xarray as xr +import linopy from linopy import LESS_EQUAL, Model, available_solvers, read_netcdf from linopy.constants import FACTOR_DIM from linopy.constraints import Constraint @@ -688,6 +690,48 @@ def test_to_file_invalid(model: Model, tmp_path: Path) -> None: model.to_file(fn) +def test_to_file_lp_progress_without_tqdm( + model: Model, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The progress bars are cosmetic: without tqdm the LP file is unchanged.""" + monkeypatch.setattr(linopy.io, "_tqdm", None) + + quiet = tmp_path / "quiet.lp" + loud = tmp_path / "loud.lp" + model.to_file(quiet, io_api="lp", progress=False) + model.to_file(loud, io_api="lp", progress=True) + + assert loud.read_bytes() == quiet.read_bytes() + + +def test_to_file_lp_without_polars( + model: Model, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """LP export names the missing extra instead of failing inside polars.""" + monkeypatch.setitem(sys.modules, "polars", None) + + with pytest.raises(ImportError, match=r"linopy\[polars\]"): + model.to_file(tmp_path / "test.lp", io_api="lp") + + +def test_missing_polars_placeholder() -> None: + """The polars stand-in raises on first attribute access, not on import.""" + from linopy._polars import missing_polars + + with pytest.raises(ImportError, match=r"linopy\[polars\]"): + missing_polars.DataFrame + + +def test_missing_polars_placeholder_dunder_probe() -> None: + """Dunder probes must raise AttributeError, not the install hint.""" + from linopy._polars import missing_polars + + with pytest.raises(AttributeError): + missing_polars.__wrapped__ + + assert hasattr(missing_polars, "__class__") + + @pytest.mark.skipif("gurobi" not in available_solvers, reason="Gurobipy not installed") def test_to_gurobipy(model: Model) -> None: gm = model.to_gurobipy()