From 129d119a1046be4e7cecd03bca413a0923d5a90d Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 14 Jul 2026 11:48:48 +0200 Subject: [PATCH 01/15] perf(int32): auto-widen labels to int64 on overflow instead of raising Replace the int32 overflow ValueError in add_variables/add_constraints with a monotonic, sticky widen of options["label_dtype"] to int64. Once a model crosses ~2.1 billion labels the process uses int64 labels everywhere (surviving options.reset()), so no cast site can silently narrow an int64 label back to int32. Mixed int32/int64 arrays upcast on concat, so previously-allocated int32 batches stay valid. --- doc/release_notes.rst | 2 +- linopy/config.py | 5 +++++ linopy/model.py | 22 +++++++--------------- test/test_dtypes.py | 27 +++++++++++++++++++++------ 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index f8464a7d2..9ca573dc2 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,7 +19,7 @@ Upcoming Version *Other* -* Default internal integer labels to ``int32`` (configurable via ``linopy.options["label_dtype"]``, set to ``np.int64`` for the old behavior), cutting memory ~25% and speeding up model build 10-35%. Raises ``ValueError`` if labels exceed the int32 maximum. +* Default internal integer labels to ``int32`` (configurable via ``linopy.options["label_dtype"]``, set to ``np.int64`` for the old behavior), cutting memory ~25% and speeding up model build 10-35%. Labels auto-widen to ``int64`` once a model exceeds the int32 maximum (~2.1 billion labels); the widening is permanent for the process and can be reverted with ``linopy.options["label_dtype"] = np.int32``. * ``add_variables(binary=True, ...)`` now accepts ``lower``/``upper`` bounds, as long as they are 0 or 1. Previously binary bounds could only be set via the ``.lower``/``.upper`` setters after creation. (https://github.com/PyPSA/linopy/issues/776) * ``add_piecewise_formulation`` gained an ``active_fill`` parameter that gates a partial ``active`` (defined over a subset of the indexed dimension, or masked) as always-active (``1``) or always-off (``0``); without it, a partial ``active`` — which was previously zeroed silently — now raises. Useful when one formulation mixes gated and ungated entities (e.g. committable and non-committable units sharing a ``status``). ``active_fill`` is transitional and will be removed once v1 semantics make ``active.reindex(coords).fillna(value)`` sufficient. (https://github.com/PyPSA/linopy/issues/796) diff --git a/linopy/config.py b/linopy/config.py index 6a28d43fe..13a5002ec 100644 --- a/linopy/config.py +++ b/linopy/config.py @@ -46,6 +46,11 @@ def get_value(self, name: str) -> Any: else: raise KeyError(f"{name} is not a valid setting.") + def widen_label_dtype(self) -> None: + """Permanently widen ``label_dtype`` to ``int64`` (monotonic, survives ``reset``).""" + self._defaults["label_dtype"] = np.int64 + self._current_values["label_dtype"] = np.int64 + def reset(self) -> None: self._current_values = self._defaults.copy() diff --git a/linopy/model.py b/linopy/model.py index b1477b276..f4fa6a365 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -825,12 +825,8 @@ def add_variables( start = self._xCounter end = start + data.labels.size - label_dtype = options["label_dtype"] - if end > np.iinfo(label_dtype).max: - raise ValueError( - f"Number of labels ({end}) exceeds the maximum value for " - f"{label_dtype.__name__} ({np.iinfo(label_dtype).max})." - ) + if end > np.iinfo(options["label_dtype"]).max: + options.widen_label_dtype() data.labels.values = np.arange( start, end, dtype=options["label_dtype"] ).reshape(data.labels.shape) @@ -978,15 +974,11 @@ def _allocate_constraint_labels( """Assign label ranges from the constraint counter and apply an optional mask.""" start = self._cCounter end = start + data.labels.size - label_dtype = options["label_dtype"] - if end > np.iinfo(label_dtype).max: - raise ValueError( - f"Number of labels ({end}) exceeds the maximum value for " - f"{label_dtype.__name__} ({np.iinfo(label_dtype).max})." - ) - data.labels.values = np.arange(start, end, dtype=label_dtype).reshape( - data.labels.shape - ) + if end > np.iinfo(options["label_dtype"]).max: + options.widen_label_dtype() + data.labels.values = np.arange( + start, end, dtype=options["label_dtype"] + ).reshape(data.labels.shape) self._cCounter += data.labels.size if mask is not None: data.labels.values = np.where(mask.values, data.labels.values, -1) diff --git a/test/test_dtypes.py b/test/test_dtypes.py index b30c7eac3..fae6b8368 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -45,19 +45,34 @@ def test_solve_with_int32_labels() -> None: assert m.objective.value == pytest.approx(25.0) -def test_overflow_guard_variables() -> None: +@pytest.fixture +def restore_label_dtype(): + yield + options._defaults["label_dtype"] = np.int32 + options["label_dtype"] = np.int32 + + +def test_auto_widen_variables(restore_label_dtype) -> None: m = Model() m._xCounter = np.iinfo(np.int32).max - 1 - with pytest.raises(ValueError, match="exceeds the maximum"): - m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + assert x.labels.dtype == np.int64 + assert options["label_dtype"] == np.int64 -def test_overflow_guard_constraints() -> None: +def test_auto_widen_constraints(restore_label_dtype) -> None: m = Model() x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") m._cCounter = np.iinfo(np.int32).max - 1 - with pytest.raises(ValueError, match="exceeds the maximum"): - m.add_constraints(x >= 0, name="c") + m.add_constraints(x >= 0, name="c") + assert m.constraints["c"].labels.dtype == np.int64 + assert options["label_dtype"] == np.int64 + + +def test_auto_widen_is_sticky(restore_label_dtype) -> None: + options.widen_label_dtype() + options.reset() + assert options["label_dtype"] == np.int64 def test_label_dtype_option_int64() -> None: From 067fd2a7d612cdda7533527848faa17e3962fd94 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 14 Jul 2026 12:03:26 +0200 Subject: [PATCH 02/15] perf(int32): re-widen label dtype when loading an int64 model read_netcdf reconstructs label arrays directly and restores the counters without going through the add_variables/add_constraints widen path, so a model that had auto-widened to int64 would load with int64 label arrays while the process option stayed int32 -- the silent-downcast corner, reachable via read_netcdf and solve(remote=...). Widen the option on load whenever a restored counter exceeds the int32 maximum. --- linopy/io.py | 4 ++++ test/test_dtypes.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/linopy/io.py b/linopy/io.py index bccbdecf7..6692d1a40 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -26,6 +26,7 @@ from linopy import solvers from linopy.common import to_polars +from linopy.config import options from linopy.constants import CONCAT_DIM, SOS_DIM_ATTR, SOS_TYPE_ATTR from linopy.objective import Objective @@ -1119,6 +1120,9 @@ def get_prefix(ds: xr.Dataset, prefix: str) -> xr.Dataset: if k in ds.attrs: setattr(m, k, ds.attrs[k]) + if max(m._xCounter, m._cCounter) > np.iinfo(np.int32).max: + options.widen_label_dtype() + if "_relaxed_registry" in ds.attrs: m._relaxed_registry = json.loads(ds.attrs["_relaxed_registry"]) diff --git a/test/test_dtypes.py b/test/test_dtypes.py index fae6b8368..872674beb 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -75,6 +75,25 @@ def test_auto_widen_is_sticky(restore_label_dtype) -> None: assert options["label_dtype"] == np.int64 +def test_auto_widen_survives_netcdf(restore_label_dtype, tmp_path) -> None: + from linopy import read_netcdf + + m = Model() + m._xCounter = np.iinfo(np.int32).max - 1 + x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + m.add_constraints(x >= 0, name="c") + path = tmp_path / "widened.nc" + m.to_netcdf(path) + + options._defaults["label_dtype"] = np.int32 + options["label_dtype"] = np.int32 + loaded = read_netcdf(path) + + assert options["label_dtype"] == np.int64 + assert loaded.variables["x"].labels.dtype == np.int64 + assert (2 * loaded.variables["x"]).vars.dtype == np.int64 + + def test_label_dtype_option_int64() -> None: with options: options["label_dtype"] = np.int64 From d3388580ca2435e841dc44b6c7918a45c96af3d0 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 14 Jul 2026 12:13:56 +0200 Subject: [PATCH 03/15] perf(int32): warn on auto-widen, recommend explicit int64 for large models Address review: keep auto-widening a convenience but emit a UserWarning at the int32->int64 transition, advising users to set label_dtype=int64 upfront (avoids the mid-build upcast, faster and lighter). --- doc/release_notes.rst | 2 +- linopy/config.py | 12 ++++++++++++ test/test_dtypes.py | 6 ++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 9ca573dc2..f5ecdbbe9 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,7 +19,7 @@ Upcoming Version *Other* -* Default internal integer labels to ``int32`` (configurable via ``linopy.options["label_dtype"]``, set to ``np.int64`` for the old behavior), cutting memory ~25% and speeding up model build 10-35%. Labels auto-widen to ``int64`` once a model exceeds the int32 maximum (~2.1 billion labels); the widening is permanent for the process and can be reverted with ``linopy.options["label_dtype"] = np.int32``. +* Default internal integer labels to ``int32`` (configurable via ``linopy.options["label_dtype"]``, set to ``np.int64`` for the old behavior), cutting memory ~25% and speeding up model build 10-35%. Labels auto-widen to ``int64`` once a model exceeds the int32 maximum (~2.1 billion labels), emitting a ``UserWarning`` that recommends setting ``linopy.options["label_dtype"] = np.int64`` explicitly for large models; the widening is permanent for the process and can be reverted with ``linopy.options["label_dtype"] = np.int32``. * ``add_variables(binary=True, ...)`` now accepts ``lower``/``upper`` bounds, as long as they are 0 or 1. Previously binary bounds could only be set via the ``.lower``/``.upper`` setters after creation. (https://github.com/PyPSA/linopy/issues/776) * ``add_piecewise_formulation`` gained an ``active_fill`` parameter that gates a partial ``active`` (defined over a subset of the indexed dimension, or masked) as always-active (``1``) or always-off (``0``); without it, a partial ``active`` — which was previously zeroed silently — now raises. Useful when one formulation mixes gated and ungated entities (e.g. committable and non-committable units sharing a ``status``). ``active_fill`` is transitional and will be removed once v1 semantics make ``active.reindex(coords).fillna(value)`` sufficient. (https://github.com/PyPSA/linopy/issues/796) diff --git a/linopy/config.py b/linopy/config.py index 13a5002ec..38d4a2e1e 100644 --- a/linopy/config.py +++ b/linopy/config.py @@ -7,6 +7,7 @@ from __future__ import annotations +import warnings from typing import Any import numpy as np @@ -48,8 +49,19 @@ def get_value(self, name: str) -> Any: def widen_label_dtype(self) -> None: """Permanently widen ``label_dtype`` to ``int64`` (monotonic, survives ``reset``).""" + transitioned = self._current_values["label_dtype"] != np.int64 self._defaults["label_dtype"] = np.int64 self._current_values["label_dtype"] = np.int64 + if transitioned: + warnings.warn( + "Auto-widened the internal label dtype to int64: the model " + "exceeded the int32 label limit (~2.1 billion). Set " + "linopy.options['label_dtype'] = np.int64 explicitly when large " + "models are expected -- it avoids the mid-build upcast and is " + "faster and lighter on memory.", + UserWarning, + stacklevel=2, + ) def reset(self) -> None: self._current_values = self._defaults.copy() diff --git a/test/test_dtypes.py b/test/test_dtypes.py index 872674beb..50fa892a0 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -55,7 +55,8 @@ def restore_label_dtype(): def test_auto_widen_variables(restore_label_dtype) -> None: m = Model() m._xCounter = np.iinfo(np.int32).max - 1 - x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + with pytest.warns(UserWarning, match="Auto-widened"): + x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") assert x.labels.dtype == np.int64 assert options["label_dtype"] == np.int64 @@ -87,7 +88,8 @@ def test_auto_widen_survives_netcdf(restore_label_dtype, tmp_path) -> None: options._defaults["label_dtype"] = np.int32 options["label_dtype"] = np.int32 - loaded = read_netcdf(path) + with pytest.warns(UserWarning, match="Auto-widened"): + loaded = read_netcdf(path) assert options["label_dtype"] == np.int64 assert loaded.variables["x"].labels.dtype == np.int64 From 15b6a2f4731b88f10dc5fa1aa97adbacb1144a5b Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 14 Jul 2026 12:32:08 +0200 Subject: [PATCH 04/15] test(int32): add type annotations to auto-widen tests for mypy --- test/test_dtypes.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/test_dtypes.py b/test/test_dtypes.py index 50fa892a0..965d68fcb 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -1,5 +1,8 @@ """Tests for int32 default label dtype.""" +from collections.abc import Generator +from pathlib import Path + import numpy as np import pytest @@ -46,13 +49,13 @@ def test_solve_with_int32_labels() -> None: @pytest.fixture -def restore_label_dtype(): +def restore_label_dtype() -> Generator[None, None, None]: yield options._defaults["label_dtype"] = np.int32 options["label_dtype"] = np.int32 -def test_auto_widen_variables(restore_label_dtype) -> None: +def test_auto_widen_variables(restore_label_dtype: None) -> None: m = Model() m._xCounter = np.iinfo(np.int32).max - 1 with pytest.warns(UserWarning, match="Auto-widened"): @@ -61,7 +64,7 @@ def test_auto_widen_variables(restore_label_dtype) -> None: assert options["label_dtype"] == np.int64 -def test_auto_widen_constraints(restore_label_dtype) -> None: +def test_auto_widen_constraints(restore_label_dtype: None) -> None: m = Model() x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") m._cCounter = np.iinfo(np.int32).max - 1 @@ -70,13 +73,13 @@ def test_auto_widen_constraints(restore_label_dtype) -> None: assert options["label_dtype"] == np.int64 -def test_auto_widen_is_sticky(restore_label_dtype) -> None: +def test_auto_widen_is_sticky(restore_label_dtype: None) -> None: options.widen_label_dtype() options.reset() assert options["label_dtype"] == np.int64 -def test_auto_widen_survives_netcdf(restore_label_dtype, tmp_path) -> None: +def test_auto_widen_survives_netcdf(restore_label_dtype: None, tmp_path: Path) -> None: from linopy import read_netcdf m = Model() From e2152970730ed3fc7ddb5f7bd7be02ec9d42b2af Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:04:38 +0200 Subject: [PATCH 05/15] chore: trigger CI Co-Authored-By: Claude Fable 5 From 163ddb3f57c31395f2fd48702170eb1ea34e5f90 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:41:11 +0200 Subject: [PATCH 06/15] refactor(int32): make label-dtype widening per-model instead of process-global Store the label dtype on the Model (snapshot of options['label_dtype'] at creation, exposed as Model.label_dtype) and widen only that model to int64 on overflow. Every cast site reads the dtype from its object's model reference; save_join takes it as an explicit fill_dtype parameter. This removes the sticky global state: the option and other models are unaffected by one model's widening, reset() keeps its usual semantics, and the widened dtype travels with the model through netCDF and pickle round-trips instead of relying on process-global stickiness. Co-Authored-By: Claude Fable 5 --- doc/release_notes.rst | 2 +- linopy/common.py | 10 ++++-- linopy/config.py | 17 ---------- linopy/constraints.py | 10 +++--- linopy/expressions.py | 16 +++++---- linopy/io.py | 3 +- linopy/model.py | 47 ++++++++++++++++++++------ linopy/variables.py | 11 ++++--- test/test_dtypes.py | 76 +++++++++++++++++++++++++++++-------------- 9 files changed, 117 insertions(+), 75 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index f5ecdbbe9..4dda20f4e 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,7 +19,7 @@ Upcoming Version *Other* -* Default internal integer labels to ``int32`` (configurable via ``linopy.options["label_dtype"]``, set to ``np.int64`` for the old behavior), cutting memory ~25% and speeding up model build 10-35%. Labels auto-widen to ``int64`` once a model exceeds the int32 maximum (~2.1 billion labels), emitting a ``UserWarning`` that recommends setting ``linopy.options["label_dtype"] = np.int64`` explicitly for large models; the widening is permanent for the process and can be reverted with ``linopy.options["label_dtype"] = np.int32``. +* Default internal integer labels to ``int32`` (configurable via ``linopy.options["label_dtype"]``, set to ``np.int64`` for the old behavior), cutting memory ~25% and speeding up model build 10-35%. Each model snapshots the option at creation (exposed as ``Model.label_dtype``) and auto-widens itself to ``int64`` once its labels exceed the int32 maximum (~2.1 billion), emitting a ``UserWarning`` that recommends setting ``linopy.options["label_dtype"] = np.int64`` upfront for large models. The widening is per-model and travels with the model through netCDF and pickle round-trips; other models and the global option are unaffected. * ``add_variables(binary=True, ...)`` now accepts ``lower``/``upper`` bounds, as long as they are 0 or 1. Previously binary bounds could only be set via the ``.lower``/``.upper`` setters after creation. (https://github.com/PyPSA/linopy/issues/776) * ``add_piecewise_formulation`` gained an ``active_fill`` parameter that gates a partial ``active`` (defined over a subset of the indexed dimension, or masked) as always-active (``1``) or always-off (``0``); without it, a partial ``active`` — which was previously zeroed silently — now raises. Useful when one formulation mixes gated and ungated entities (e.g. committable and non-committable units sharing a ``status``). ``active_fill`` is transitional and will be removed once v1 semantics make ``active.reindex(coords).fillna(value)`` sufficient. (https://github.com/PyPSA/linopy/issues/796) diff --git a/linopy/common.py b/linopy/common.py index 2c45c999c..49cbb4a99 100644 --- a/linopy/common.py +++ b/linopy/common.py @@ -18,6 +18,7 @@ import pandas as pd import polars as pl from numpy import nan, signedinteger +from numpy.typing import DTypeLike from polars.datatypes import DataTypeClass from xarray import DataArray, Dataset, apply_ufunc, broadcast from xarray import align as xr_align @@ -292,9 +293,12 @@ def maybe_group_terms_polars(df: pl.DataFrame) -> pl.DataFrame: return df.select(keys + ["coeffs"] + rest) -def save_join(*dataarrays: DataArray, integer_dtype: bool = False) -> Dataset: +def save_join(*dataarrays: DataArray, fill_dtype: DTypeLike | None = None) -> Dataset: """ Join multiple xarray Dataarray's to a Dataset and warn if coordinates are not equal. + + If ``fill_dtype`` is given, values filled in by an outer join are set to -1 + and the arrays are cast to that dtype (used for integer label arrays). """ try: arrs = xr_align(*dataarrays, join="exact") @@ -304,8 +308,8 @@ def save_join(*dataarrays: DataArray, integer_dtype: bool = False) -> Dataset: UserWarning, ) arrs = xr_align(*dataarrays, join="outer") - if integer_dtype: - arrs = tuple([ds.fillna(-1).astype(options["label_dtype"]) for ds in arrs]) + if fill_dtype is not None: + arrs = tuple([ds.fillna(-1).astype(fill_dtype) for ds in arrs]) return Dataset({ds.name: ds for ds in arrs}) diff --git a/linopy/config.py b/linopy/config.py index 38d4a2e1e..6a28d43fe 100644 --- a/linopy/config.py +++ b/linopy/config.py @@ -7,7 +7,6 @@ from __future__ import annotations -import warnings from typing import Any import numpy as np @@ -47,22 +46,6 @@ def get_value(self, name: str) -> Any: else: raise KeyError(f"{name} is not a valid setting.") - def widen_label_dtype(self) -> None: - """Permanently widen ``label_dtype`` to ``int64`` (monotonic, survives ``reset``).""" - transitioned = self._current_values["label_dtype"] != np.int64 - self._defaults["label_dtype"] = np.int64 - self._current_values["label_dtype"] = np.int64 - if transitioned: - warnings.warn( - "Auto-widened the internal label dtype to int64: the model " - "exceeded the int32 label limit (~2.1 billion). Set " - "linopy.options['label_dtype'] = np.int64 explicitly when large " - "models are expected -- it avoids the mid-build upcast and is " - "faster and lighter on memory.", - UserWarning, - stacklevel=2, - ) - def reset(self) -> None: self._current_values = self._defaults.copy() diff --git a/linopy/constraints.py b/linopy/constraints.py index ceef45c9d..3bfd5ccd2 100644 --- a/linopy/constraints.py +++ b/linopy/constraints.py @@ -752,7 +752,7 @@ def _to_dataset(self, nterm: int) -> Dataset: # Map active row i -> flat position in full shape via con_labels active_positions = self.active_positions coeffs_2d = np.zeros((full_size, nterm), dtype=csr.dtype) - vars_2d = np.full((full_size, nterm), -1, dtype=options["label_dtype"]) + vars_2d = np.full((full_size, nterm), -1, dtype=self._model._label_dtype) if csr.nnz > 0: row_indices = np.repeat(active_positions, counts) term_cols = np.arange(csr.nnz) - np.repeat(csr.indptr[:-1], counts) @@ -776,7 +776,7 @@ def _to_dataset(self, nterm: int) -> Dataset: ) ds = Dataset({"coeffs": coeffs_da, "vars": vars_da}) if self._cindex is not None: - labels_flat = np.full(full_size, -1, dtype=options["label_dtype"]) + labels_flat = np.full(full_size, -1, dtype=self._model._label_dtype) labels_flat[active_positions] = self._con_labels ds = assign_multiindex_safe( ds, @@ -1933,7 +1933,7 @@ def labels(self) -> Dataset: """ return save_join( *[v.labels.rename(k) for k, v in self.items()], - integer_dtype=True, + fill_dtype=self.model._label_dtype, ) @property @@ -1954,7 +1954,7 @@ def rename_term_dim(ds: DataArray) -> DataArray: return save_join( *[rename_term_dim(v.vars.rename(k)) for k, v in self.items()], - integer_dtype=True, + fill_dtype=self.model._label_dtype, ) @property @@ -2191,7 +2191,7 @@ def flat(self) -> pd.DataFrame: df = pd.concat(dfs, ignore_index=True) unique_labels = df.labels.unique() map_labels = pd.Series( - np.arange(len(unique_labels), dtype=options["label_dtype"]), + np.arange(len(unique_labels), dtype=self.model._label_dtype), index=unique_labels, ) df["key"] = df.labels.map(map_labels) diff --git a/linopy/expressions.py b/linopy/expressions.py index dd8c5c6c1..3d7fde769 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -510,6 +510,9 @@ def to_polars(self) -> pl.DataFrame: ... def __init__(self, data: Dataset | Any | None, model: Model) -> None: from linopy.model import Model + if not isinstance(model, Model): + raise ValueError("model must be an instance of linopy.Model") + if data is None: da = xr.DataArray([], dims=[TERM_DIM]) data = Dataset({"coeffs": da, "vars": da, "const": 0.0}) @@ -533,7 +536,7 @@ def __init__(self, data: Dataset | Any | None, model: Model) -> None: if np.issubdtype(data.vars, np.floating): data = assign_multiindex_safe( - data, vars=data.vars.fillna(-1).astype(options["label_dtype"]) + data, vars=data.vars.fillna(-1).astype(model._label_dtype) ) if not np.issubdtype(data.coeffs, np.floating): data["coeffs"].values = data.coeffs.values.astype(float) @@ -561,9 +564,6 @@ def __init__(self, data: Dataset | Any | None, model: Model) -> None: # TODO: add a warning here, routines should be safe against this data = data.drop_vars(drop_dims) - if not isinstance(model, Model): - raise ValueError("model must be an instance of linopy.Model") - self._model = model self._data = cast(Dataset, data) @@ -1618,7 +1618,9 @@ def sanitize(self) -> Self: linopy.LinearExpression """ if not np.issubdtype(self.vars.dtype, np.integer): - return self.assign(vars=self.vars.fillna(-1).astype(options["label_dtype"])) + return self.assign( + vars=self.vars.fillna(-1).astype(self.model._label_dtype) + ) return self @@ -2022,12 +2024,12 @@ def _simplify_row(vars_row: np.ndarray, coeffs_row: np.ndarray) -> np.ndarray: # Combined has dimensions (.., CV_DIM, TERM_DIM) # Drop terms where all vars are -1 (i.e., empty terms across all coordinates) - vars = combined.isel({CV_DIM: 0}).astype(options["label_dtype"]) + vars = combined.isel({CV_DIM: 0}).astype(self.model._label_dtype) non_empty_terms = (vars != -1).any(dim=[d for d in vars.dims if d != TERM_DIM]) combined = combined.isel({TERM_DIM: non_empty_terms}) # Extract vars and coeffs from the combined result - vars = combined.isel({CV_DIM: 0}).astype(options["label_dtype"]) + vars = combined.isel({CV_DIM: 0}).astype(self.model._label_dtype) coeffs = combined.isel({CV_DIM: 1}) # Create new dataset with simplified data diff --git a/linopy/io.py b/linopy/io.py index 6692d1a40..d1557122f 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -26,7 +26,6 @@ from linopy import solvers from linopy.common import to_polars -from linopy.config import options from linopy.constants import CONCAT_DIM, SOS_DIM_ATTR, SOS_TYPE_ATTR from linopy.objective import Objective @@ -1121,7 +1120,7 @@ def get_prefix(ds: xr.Dataset, prefix: str) -> xr.Dataset: setattr(m, k, ds.attrs[k]) if max(m._xCounter, m._cCounter) > np.iinfo(np.int32).max: - options.widen_label_dtype() + m._label_dtype = np.int64 if "_relaxed_registry" in ds.attrs: m._relaxed_registry = json.loads(ds.attrs["_relaxed_registry"]) diff --git a/linopy/model.py b/linopy/model.py index f4fa6a365..eeb33af89 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -140,6 +140,7 @@ class Model: _termination_condition: str _xCounter: int _cCounter: int + _label_dtype: type[np.signedinteger] _varnameCounter: int _connameCounter: int _pwlCounter: int @@ -163,6 +164,7 @@ class Model: # TODO: move counters to Variables and Constraints class "_xCounter", "_cCounter", + "_label_dtype", "_varnameCounter", "_connameCounter", "_pwlCounter", @@ -223,6 +225,7 @@ def __init__( ------- linopy.Model """ + self._label_dtype: type[np.signedinteger] = options["label_dtype"] self._variables: Variables = Variables({}, model=self) self._constraints: Constraints = Constraints({}, model=self) self._objective: Objective = Objective(LinearExpression(None, self), self) @@ -495,6 +498,30 @@ def solver_dir(self, value: str | Path) -> None: raise TypeError("'solver_dir' must path-like.") self._solver_dir = Path(value) + @property + def label_dtype(self) -> type[np.signedinteger]: + """ + Integer dtype used for this model's variable and constraint labels. + + Snapshot of ``linopy.options["label_dtype"]`` taken at model creation, + automatically widened to ``int64`` once the labels outgrow int32. + """ + return self._label_dtype + + def _widen_label_dtype(self) -> None: + """Widen this model's label dtype to ``int64`` (monotonic, never narrows).""" + if self._label_dtype == np.int64: + return + self._label_dtype = np.int64 + warnings.warn( + "The model exceeded the int32 label limit (~2.1 billion labels); " + "its label dtype was widened to int64. Set " + "linopy.options['label_dtype'] = np.int64 before building large " + "models to avoid the mid-build upcast.", + UserWarning, + stacklevel=3, + ) + @property def dataset_attrs(self) -> list[str]: return ["parameters"] @@ -825,11 +852,11 @@ def add_variables( start = self._xCounter end = start + data.labels.size - if end > np.iinfo(options["label_dtype"]).max: - options.widen_label_dtype() - data.labels.values = np.arange( - start, end, dtype=options["label_dtype"] - ).reshape(data.labels.shape) + if end > np.iinfo(self._label_dtype).max: + self._widen_label_dtype() + data.labels.values = np.arange(start, end, dtype=self._label_dtype).reshape( + data.labels.shape + ) self._xCounter += data.labels.size if mask is not None: @@ -974,11 +1001,11 @@ def _allocate_constraint_labels( """Assign label ranges from the constraint counter and apply an optional mask.""" start = self._cCounter end = start + data.labels.size - if end > np.iinfo(options["label_dtype"]).max: - options.widen_label_dtype() - data.labels.values = np.arange( - start, end, dtype=options["label_dtype"] - ).reshape(data.labels.shape) + if end > np.iinfo(self._label_dtype).max: + self._widen_label_dtype() + data.labels.values = np.arange(start, end, dtype=self._label_dtype).reshape( + data.labels.shape + ) self._cCounter += data.labels.size if mask is not None: data.labels.values = np.where(mask.values, data.labels.values, -1) diff --git a/linopy/variables.py b/linopy/variables.py index 54679524a..39a1c760a 100644 --- a/linopy/variables.py +++ b/linopy/variables.py @@ -1273,7 +1273,7 @@ def ffill(self, dim: str, limit: None = None) -> Variable: .fillna(self._fill_value) ) return self.assign_multiindex_safe( - labels=data.labels.astype(options["label_dtype"]) + labels=data.labels.astype(self.model._label_dtype) ) def bfill(self, dim: str, limit: None = None) -> Variable: @@ -1301,7 +1301,7 @@ def bfill(self, dim: str, limit: None = None) -> Variable: .map(DataArray.bfill, dim=dim, limit=limit) .fillna(self._fill_value) ) - return self.assign(labels=data.labels.astype(options["label_dtype"])) + return self.assign(labels=data.labels.astype(self.model._label_dtype)) def sanitize(self) -> Variable: """ @@ -1313,7 +1313,7 @@ def sanitize(self) -> Variable: """ if issubdtype(self.labels.dtype, floating): return self.assign( - labels=self.labels.fillna(-1).astype(options["label_dtype"]) + labels=self.labels.fillna(-1).astype(self.model._label_dtype) ) return self @@ -1726,7 +1726,8 @@ def labels(self) -> Dataset: Get the labels of all variables. """ return save_join( - *[v.labels.rename(k) for k, v in self.items()], integer_dtype=True + *[v.labels.rename(k) for k, v in self.items()], + fill_dtype=self.model._label_dtype, ) @property @@ -2037,7 +2038,7 @@ def flat(self) -> pd.DataFrame: df = pd.concat([self[k].flat for k in self], ignore_index=True) unique_labels = df.labels.unique() map_labels = pd.Series( - np.arange(len(unique_labels), dtype=options["label_dtype"]), + np.arange(len(unique_labels), dtype=self.model._label_dtype), index=unique_labels, ) df["key"] = df.labels.map(map_labels) diff --git a/test/test_dtypes.py b/test/test_dtypes.py index 965d68fcb..765300446 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -1,6 +1,5 @@ """Tests for int32 default label dtype.""" -from collections.abc import Generator from pathlib import Path import numpy as np @@ -48,57 +47,84 @@ def test_solve_with_int32_labels() -> None: assert m.objective.value == pytest.approx(25.0) -@pytest.fixture -def restore_label_dtype() -> Generator[None, None, None]: - yield - options._defaults["label_dtype"] = np.int32 - options["label_dtype"] = np.int32 - - -def test_auto_widen_variables(restore_label_dtype: None) -> None: +def test_auto_widen_variables() -> None: m = Model() m._xCounter = np.iinfo(np.int32).max - 1 - with pytest.warns(UserWarning, match="Auto-widened"): + with pytest.warns(UserWarning, match="widened to int64"): x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") assert x.labels.dtype == np.int64 - assert options["label_dtype"] == np.int64 + assert m.label_dtype == np.int64 + # the widening is per-model: the global option and other models are untouched + assert options["label_dtype"] == np.int32 + other = Model() + y = other.add_variables(lower=0, upper=1, coords=[range(5)], name="y") + assert y.labels.dtype == np.int32 -def test_auto_widen_constraints(restore_label_dtype: None) -> None: +def test_auto_widen_constraints() -> None: m = Model() x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") m._cCounter = np.iinfo(np.int32).max - 1 - m.add_constraints(x >= 0, name="c") + with pytest.warns(UserWarning, match="widened to int64"): + m.add_constraints(x >= 0, name="c") assert m.constraints["c"].labels.dtype == np.int64 - assert options["label_dtype"] == np.int64 + assert m.label_dtype == np.int64 + assert options["label_dtype"] == np.int32 -def test_auto_widen_is_sticky(restore_label_dtype: None) -> None: - options.widen_label_dtype() - options.reset() - assert options["label_dtype"] == np.int64 +def test_widen_applies_to_expressions() -> None: + m = Model() + m._xCounter = np.iinfo(np.int32).max - 1 + with pytest.warns(UserWarning, match="widened to int64"): + x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + assert (2 * x + 1).vars.dtype == np.int64 -def test_auto_widen_survives_netcdf(restore_label_dtype: None, tmp_path: Path) -> None: +def test_label_dtype_is_model_snapshot() -> None: + with options: + options["label_dtype"] = np.int64 + m = Model() + # resetting the option does not narrow an existing model + assert options["label_dtype"] == np.int32 + assert m.label_dtype == np.int64 + x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + assert x.labels.dtype == np.int64 + + +def test_auto_widen_survives_netcdf(tmp_path: Path) -> None: from linopy import read_netcdf m = Model() m._xCounter = np.iinfo(np.int32).max - 1 - x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + with pytest.warns(UserWarning, match="widened to int64"): + x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") m.add_constraints(x >= 0, name="c") path = tmp_path / "widened.nc" m.to_netcdf(path) - options._defaults["label_dtype"] = np.int32 - options["label_dtype"] = np.int32 - with pytest.warns(UserWarning, match="Auto-widened"): - loaded = read_netcdf(path) + loaded = read_netcdf(path) - assert options["label_dtype"] == np.int64 + assert loaded.label_dtype == np.int64 + assert options["label_dtype"] == np.int32 assert loaded.variables["x"].labels.dtype == np.int64 assert (2 * loaded.variables["x"]).vars.dtype == np.int64 +def test_auto_widen_survives_pickle() -> None: + import pickle + + m = Model() + m._xCounter = np.iinfo(np.int32).max - 1 + with pytest.warns(UserWarning, match="widened to int64"): + x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + m.add_constraints(x >= 0, name="c") + + loaded = pickle.loads(pickle.dumps(m)) + + assert loaded.label_dtype == np.int64 + assert (2 * loaded.variables["x"]).vars.dtype == np.int64 + + def test_label_dtype_option_int64() -> None: with options: options["label_dtype"] = np.int64 From d4cf445c06fba6e0960dfc939e927b2c4c791a00 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:50:25 +0200 Subject: [PATCH 07/15] feat(int32): accept label_dtype as Model() argument Model(label_dtype=np.int64) is more direct than mutating the global option before construction; None falls back to options['label_dtype'] so embedders that construct the Model internally keep a process-wide default. Co-Authored-By: Claude Fable 5 --- doc/release_notes.rst | 2 +- linopy/model.py | 29 ++++++++++++++++++++++------- test/test_dtypes.py | 15 +++++++++++++++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4dda20f4e..0a40b807e 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,7 +19,7 @@ Upcoming Version *Other* -* Default internal integer labels to ``int32`` (configurable via ``linopy.options["label_dtype"]``, set to ``np.int64`` for the old behavior), cutting memory ~25% and speeding up model build 10-35%. Each model snapshots the option at creation (exposed as ``Model.label_dtype``) and auto-widens itself to ``int64`` once its labels exceed the int32 maximum (~2.1 billion), emitting a ``UserWarning`` that recommends setting ``linopy.options["label_dtype"] = np.int64`` upfront for large models. The widening is per-model and travels with the model through netCDF and pickle round-trips; other models and the global option are unaffected. +* Default internal integer labels to ``int32``, cutting memory ~25% and speeding up model build 10-35%. The dtype is set per model via ``Model(label_dtype=np.int64)`` (exposed as ``Model.label_dtype``, defaulting to ``linopy.options["label_dtype"]``) and each model auto-widens itself to ``int64`` once its labels exceed the int32 maximum (~2.1 billion), emitting a ``UserWarning`` that recommends passing ``label_dtype=np.int64`` upfront for large models. The widening is per-model and travels with the model through netCDF and pickle round-trips; other models and the global option are unaffected. * ``add_variables(binary=True, ...)`` now accepts ``lower``/``upper`` bounds, as long as they are 0 or 1. Previously binary bounds could only be set via the ``.lower``/``.upper`` setters after creation. (https://github.com/PyPSA/linopy/issues/776) * ``add_piecewise_formulation`` gained an ``active_fill`` parameter that gates a partial ``active`` (defined over a subset of the indexed dimension, or masked) as always-active (``1``) or always-off (``0``); without it, a partial ``active`` — which was previously zeroed silently — now raises. Useful when one formulation mixes gated and ungated entities (e.g. committable and non-committable units sharing a ``status``). ``active_fill`` is transitional and will be removed once v1 semantics make ``active.reindex(coords).fillna(value)`` sufficient. (https://github.com/PyPSA/linopy/issues/796) diff --git a/linopy/model.py b/linopy/model.py index eeb33af89..e340dbf10 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -35,7 +35,7 @@ replace_by_map, to_path, ) -from linopy.config import options +from linopy.config import _VALID_LABEL_DTYPES, options from linopy.constants import ( GREATER_EQUAL, HELPER_DIMS, @@ -191,6 +191,7 @@ def __init__( auto_mask: bool = False, freeze_constraints: bool = False, set_names_in_solver_io: bool = True, + label_dtype: type[np.signedinteger] | None = None, ) -> None: """ Initialize the linopy model. @@ -220,12 +221,25 @@ def __init__( set_names_in_solver_io : bool Whether direct solver exports should include variable and constraint names by default. The default is True. + label_dtype : np.int32 or np.int64, optional + Integer dtype used for the model's variable and constraint labels. + ``np.int32`` halves label memory but caps a model at ~2.1 billion + labels; the model widens itself to ``np.int64`` automatically when + that limit is hit. Pass ``np.int64`` upfront for very large models + to avoid the mid-build upcast. The default None takes + ``linopy.options["label_dtype"]`` (``np.int32``). Returns ------- linopy.Model """ - self._label_dtype: type[np.signedinteger] = options["label_dtype"] + if label_dtype is None: + label_dtype = options["label_dtype"] + elif label_dtype not in _VALID_LABEL_DTYPES: + raise ValueError( + f"label_dtype must be one of {_VALID_LABEL_DTYPES}, got {label_dtype}" + ) + self._label_dtype: type[np.signedinteger] = label_dtype self._variables: Variables = Variables({}, model=self) self._constraints: Constraints = Constraints({}, model=self) self._objective: Objective = Objective(LinearExpression(None, self), self) @@ -503,8 +517,9 @@ def label_dtype(self) -> type[np.signedinteger]: """ Integer dtype used for this model's variable and constraint labels. - Snapshot of ``linopy.options["label_dtype"]`` taken at model creation, - automatically widened to ``int64`` once the labels outgrow int32. + Set via the ``label_dtype`` argument of ``Model()`` (defaulting to + ``linopy.options["label_dtype"]``), and automatically widened to + ``int64`` once the labels outgrow int32. """ return self._label_dtype @@ -515,9 +530,9 @@ def _widen_label_dtype(self) -> None: self._label_dtype = np.int64 warnings.warn( "The model exceeded the int32 label limit (~2.1 billion labels); " - "its label dtype was widened to int64. Set " - "linopy.options['label_dtype'] = np.int64 before building large " - "models to avoid the mid-build upcast.", + "its label dtype was widened to int64. Pass label_dtype=np.int64 " + "to Model() when building large models to avoid the mid-build " + "upcast.", UserWarning, stacklevel=3, ) diff --git a/test/test_dtypes.py b/test/test_dtypes.py index 765300446..481a4a903 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -80,6 +80,21 @@ def test_widen_applies_to_expressions() -> None: assert (2 * x + 1).vars.dtype == np.int64 +def test_label_dtype_init_arg() -> None: + m = Model(label_dtype=np.int64) + assert m.label_dtype == np.int64 + x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + assert x.labels.dtype == np.int64 + assert (2 * x + 1).vars.dtype == np.int64 + # the global option is untouched + assert options["label_dtype"] == np.int32 + + +def test_label_dtype_init_arg_rejects_invalid() -> None: + with pytest.raises(ValueError, match="label_dtype must be one of"): + Model(label_dtype=np.float64) # type: ignore[arg-type] + + def test_label_dtype_is_model_snapshot() -> None: with options: options["label_dtype"] = np.int64 From f7323afca3f99881dc8bac60a882541837e93cbe Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:05:21 +0200 Subject: [PATCH 08/15] refactor: deduplicate label allocation and dtype validation Extract Model._allocate_labels (shared by variable and constraint label assignment) and config.validate_label_dtype (shared by the option setter and Model.__init__). Co-Authored-By: Claude Fable 5 --- linopy/config.py | 13 +++++++++---- linopy/model.py | 24 ++++++++++++------------ test/test_dtypes.py | 3 +-- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/linopy/config.py b/linopy/config.py index 6a28d43fe..cb8df4a88 100644 --- a/linopy/config.py +++ b/linopy/config.py @@ -14,6 +14,13 @@ _VALID_LABEL_DTYPES = {np.int32, np.int64} +def validate_label_dtype(value: Any) -> None: + if value not in _VALID_LABEL_DTYPES: + raise ValueError( + f"label_dtype must be one of {_VALID_LABEL_DTYPES}, got {value}" + ) + + class OptionSettings: """Runtime configuration knobs (e.g. display widths). Use as a context manager or set values directly via ``options(key=value)``.""" @@ -34,10 +41,8 @@ def set_value(self, **kwargs: Any) -> None: for k, v in kwargs.items(): if k not in self._defaults: raise KeyError(f"{k} is not a valid setting.") - if k == "label_dtype" and v not in _VALID_LABEL_DTYPES: - raise ValueError( - f"label_dtype must be one of {_VALID_LABEL_DTYPES}, got {v}" - ) + if k == "label_dtype": + validate_label_dtype(v) self._current_values[k] = v def get_value(self, name: str) -> Any: diff --git a/linopy/model.py b/linopy/model.py index e340dbf10..00137ab4a 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -35,7 +35,7 @@ replace_by_map, to_path, ) -from linopy.config import _VALID_LABEL_DTYPES, options +from linopy.config import options, validate_label_dtype from linopy.constants import ( GREATER_EQUAL, HELPER_DIMS, @@ -235,10 +235,8 @@ def __init__( """ if label_dtype is None: label_dtype = options["label_dtype"] - elif label_dtype not in _VALID_LABEL_DTYPES: - raise ValueError( - f"label_dtype must be one of {_VALID_LABEL_DTYPES}, got {label_dtype}" - ) + else: + validate_label_dtype(label_dtype) self._label_dtype: type[np.signedinteger] = label_dtype self._variables: Variables = Variables({}, model=self) self._constraints: Constraints = Constraints({}, model=self) @@ -534,9 +532,15 @@ def _widen_label_dtype(self) -> None: "to Model() when building large models to avoid the mid-build " "upcast.", UserWarning, - stacklevel=3, + stacklevel=4, ) + def _allocate_labels(self, start: int, end: int) -> np.ndarray: + """Return the label range ``[start, end)``, widening the dtype on overflow.""" + if end > np.iinfo(self._label_dtype).max: + self._widen_label_dtype() + return np.arange(start, end, dtype=self._label_dtype) + @property def dataset_attrs(self) -> list[str]: return ["parameters"] @@ -867,9 +871,7 @@ def add_variables( start = self._xCounter end = start + data.labels.size - if end > np.iinfo(self._label_dtype).max: - self._widen_label_dtype() - data.labels.values = np.arange(start, end, dtype=self._label_dtype).reshape( + data.labels.values = self._allocate_labels(start, end).reshape( data.labels.shape ) self._xCounter += data.labels.size @@ -1016,9 +1018,7 @@ def _allocate_constraint_labels( """Assign label ranges from the constraint counter and apply an optional mask.""" start = self._cCounter end = start + data.labels.size - if end > np.iinfo(self._label_dtype).max: - self._widen_label_dtype() - data.labels.values = np.arange(start, end, dtype=self._label_dtype).reshape( + data.labels.values = self._allocate_labels(start, end).reshape( data.labels.shape ) self._cCounter += data.labels.size diff --git a/test/test_dtypes.py b/test/test_dtypes.py index 481a4a903..515243f62 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -1,5 +1,6 @@ """Tests for int32 default label dtype.""" +import pickle from pathlib import Path import numpy as np @@ -126,8 +127,6 @@ def test_auto_widen_survives_netcdf(tmp_path: Path) -> None: def test_auto_widen_survives_pickle() -> None: - import pickle - m = Model() m._xCounter = np.iinfo(np.int32).max - 1 with pytest.warns(UserWarning, match="widened to int64"): From aaecbf6bf4c57b3c79d349ac4eedbc4bf0600f92 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:09:52 +0200 Subject: [PATCH 09/15] docs: trim label-dtype release note to user-facing behavior Co-Authored-By: Claude Fable 5 --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 0a40b807e..4f754e543 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,7 +19,7 @@ Upcoming Version *Other* -* Default internal integer labels to ``int32``, cutting memory ~25% and speeding up model build 10-35%. The dtype is set per model via ``Model(label_dtype=np.int64)`` (exposed as ``Model.label_dtype``, defaulting to ``linopy.options["label_dtype"]``) and each model auto-widens itself to ``int64`` once its labels exceed the int32 maximum (~2.1 billion), emitting a ``UserWarning`` that recommends passing ``label_dtype=np.int64`` upfront for large models. The widening is per-model and travels with the model through netCDF and pickle round-trips; other models and the global option are unaffected. +* Default internal integer labels to ``int32``, cutting memory ~25% and speeding up model build 10-35%. Models exceeding the int32 maximum (~2.1 billion labels) widen to ``int64`` automatically with a ``UserWarning``; pass ``Model(label_dtype=np.int64)`` or set ``linopy.options["label_dtype"]`` upfront to avoid the mid-build upcast. * ``add_variables(binary=True, ...)`` now accepts ``lower``/``upper`` bounds, as long as they are 0 or 1. Previously binary bounds could only be set via the ``.lower``/``.upper`` setters after creation. (https://github.com/PyPSA/linopy/issues/776) * ``add_piecewise_formulation`` gained an ``active_fill`` parameter that gates a partial ``active`` (defined over a subset of the indexed dimension, or masked) as always-active (``1``) or always-off (``0``); without it, a partial ``active`` — which was previously zeroed silently — now raises. Useful when one formulation mixes gated and ungated entities (e.g. committable and non-committable units sharing a ``status``). ``active_fill`` is transitional and will be removed once v1 semantics make ``active.reindex(coords).fillna(value)`` sufficient. (https://github.com/PyPSA/linopy/issues/796) From 70ec680d330b0114b87b400cac155ce3e7dbc23f Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:17:03 +0200 Subject: [PATCH 10/15] refactor!: drop linopy.options['label_dtype'] in favor of the Model() argument The option predated the constructor argument; with per-model storage it only duplicated the knob. Embedders forward constructor kwargs (PyPSA: n.optimize(model_kwargs={'label_dtype': np.int64})), so no process-wide setting is needed. config.py returns to its pre-int32 state. Co-Authored-By: Claude Fable 5 --- doc/release_notes.rst | 2 +- linopy/config.py | 14 -------------- linopy/model.py | 25 +++++++++++-------------- test/test_dtypes.py | 38 +++----------------------------------- 4 files changed, 15 insertions(+), 64 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4f754e543..6a41bc736 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,7 +19,7 @@ Upcoming Version *Other* -* Default internal integer labels to ``int32``, cutting memory ~25% and speeding up model build 10-35%. Models exceeding the int32 maximum (~2.1 billion labels) widen to ``int64`` automatically with a ``UserWarning``; pass ``Model(label_dtype=np.int64)`` or set ``linopy.options["label_dtype"]`` upfront to avoid the mid-build upcast. +* Default internal integer labels to ``int32``, cutting memory ~25% and speeding up model build 10-35%. Models exceeding the int32 maximum (~2.1 billion labels) widen to ``int64`` automatically with a ``UserWarning``; pass ``Model(label_dtype=np.int64)`` upfront to avoid the mid-build upcast. * ``add_variables(binary=True, ...)`` now accepts ``lower``/``upper`` bounds, as long as they are 0 or 1. Previously binary bounds could only be set via the ``.lower``/``.upper`` setters after creation. (https://github.com/PyPSA/linopy/issues/776) * ``add_piecewise_formulation`` gained an ``active_fill`` parameter that gates a partial ``active`` (defined over a subset of the indexed dimension, or masked) as always-active (``1``) or always-off (``0``); without it, a partial ``active`` — which was previously zeroed silently — now raises. Useful when one formulation mixes gated and ungated entities (e.g. committable and non-committable units sharing a ``status``). ``active_fill`` is transitional and will be removed once v1 semantics make ``active.reindex(coords).fillna(value)`` sufficient. (https://github.com/PyPSA/linopy/issues/796) diff --git a/linopy/config.py b/linopy/config.py index cb8df4a88..5d269c4e7 100644 --- a/linopy/config.py +++ b/linopy/config.py @@ -9,17 +9,6 @@ from typing import Any -import numpy as np - -_VALID_LABEL_DTYPES = {np.int32, np.int64} - - -def validate_label_dtype(value: Any) -> None: - if value not in _VALID_LABEL_DTYPES: - raise ValueError( - f"label_dtype must be one of {_VALID_LABEL_DTYPES}, got {value}" - ) - class OptionSettings: """Runtime configuration knobs (e.g. display widths). Use as a context manager or set values directly via ``options(key=value)``.""" @@ -41,8 +30,6 @@ def set_value(self, **kwargs: Any) -> None: for k, v in kwargs.items(): if k not in self._defaults: raise KeyError(f"{k} is not a valid setting.") - if k == "label_dtype": - validate_label_dtype(v) self._current_values[k] = v def get_value(self, name: str) -> Any: @@ -75,5 +62,4 @@ def __repr__(self) -> str: options = OptionSettings( display_max_rows=14, display_max_terms=6, - label_dtype=np.int32, ) diff --git a/linopy/model.py b/linopy/model.py index 00137ab4a..545720f31 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -35,7 +35,6 @@ replace_by_map, to_path, ) -from linopy.config import options, validate_label_dtype from linopy.constants import ( GREATER_EQUAL, HELPER_DIMS, @@ -191,7 +190,7 @@ def __init__( auto_mask: bool = False, freeze_constraints: bool = False, set_names_in_solver_io: bool = True, - label_dtype: type[np.signedinteger] | None = None, + label_dtype: type[np.signedinteger] = np.int32, ) -> None: """ Initialize the linopy model. @@ -223,20 +222,19 @@ def __init__( constraint names by default. The default is True. label_dtype : np.int32 or np.int64, optional Integer dtype used for the model's variable and constraint labels. - ``np.int32`` halves label memory but caps a model at ~2.1 billion - labels; the model widens itself to ``np.int64`` automatically when - that limit is hit. Pass ``np.int64`` upfront for very large models - to avoid the mid-build upcast. The default None takes - ``linopy.options["label_dtype"]`` (``np.int32``). + The default ``np.int32`` halves label memory but caps a model at + ~2.1 billion labels; the model widens itself to ``np.int64`` + automatically when that limit is hit. Pass ``np.int64`` upfront + for very large models to avoid the mid-build upcast. Returns ------- linopy.Model """ - if label_dtype is None: - label_dtype = options["label_dtype"] - else: - validate_label_dtype(label_dtype) + if label_dtype not in (np.int32, np.int64): + raise ValueError( + f"label_dtype must be np.int32 or np.int64, got {label_dtype}" + ) self._label_dtype: type[np.signedinteger] = label_dtype self._variables: Variables = Variables({}, model=self) self._constraints: Constraints = Constraints({}, model=self) @@ -515,9 +513,8 @@ def label_dtype(self) -> type[np.signedinteger]: """ Integer dtype used for this model's variable and constraint labels. - Set via the ``label_dtype`` argument of ``Model()`` (defaulting to - ``linopy.options["label_dtype"]``), and automatically widened to - ``int64`` once the labels outgrow int32. + Set via the ``label_dtype`` argument of ``Model()`` and automatically + widened to ``int64`` once the labels outgrow int32. """ return self._label_dtype diff --git a/test/test_dtypes.py b/test/test_dtypes.py index 515243f62..65889950a 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -7,11 +7,10 @@ import pytest from linopy import Model -from linopy.config import options def test_default_label_dtype_is_int32() -> None: - assert options["label_dtype"] == np.int32 + assert Model().label_dtype == np.int32 def test_variable_labels_are_int32() -> None: @@ -55,8 +54,7 @@ def test_auto_widen_variables() -> None: x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") assert x.labels.dtype == np.int64 assert m.label_dtype == np.int64 - # the widening is per-model: the global option and other models are untouched - assert options["label_dtype"] == np.int32 + # the widening is per-model: other models are untouched other = Model() y = other.add_variables(lower=0, upper=1, coords=[range(5)], name="y") assert y.labels.dtype == np.int32 @@ -70,7 +68,6 @@ def test_auto_widen_constraints() -> None: m.add_constraints(x >= 0, name="c") assert m.constraints["c"].labels.dtype == np.int64 assert m.label_dtype == np.int64 - assert options["label_dtype"] == np.int32 def test_widen_applies_to_expressions() -> None: @@ -87,26 +84,13 @@ def test_label_dtype_init_arg() -> None: x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") assert x.labels.dtype == np.int64 assert (2 * x + 1).vars.dtype == np.int64 - # the global option is untouched - assert options["label_dtype"] == np.int32 def test_label_dtype_init_arg_rejects_invalid() -> None: - with pytest.raises(ValueError, match="label_dtype must be one of"): + with pytest.raises(ValueError, match="label_dtype must be"): Model(label_dtype=np.float64) # type: ignore[arg-type] -def test_label_dtype_is_model_snapshot() -> None: - with options: - options["label_dtype"] = np.int64 - m = Model() - # resetting the option does not narrow an existing model - assert options["label_dtype"] == np.int32 - assert m.label_dtype == np.int64 - x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") - assert x.labels.dtype == np.int64 - - def test_auto_widen_survives_netcdf(tmp_path: Path) -> None: from linopy import read_netcdf @@ -121,7 +105,6 @@ def test_auto_widen_survives_netcdf(tmp_path: Path) -> None: loaded = read_netcdf(path) assert loaded.label_dtype == np.int64 - assert options["label_dtype"] == np.int32 assert loaded.variables["x"].labels.dtype == np.int64 assert (2 * loaded.variables["x"]).vars.dtype == np.int64 @@ -137,18 +120,3 @@ def test_auto_widen_survives_pickle() -> None: assert loaded.label_dtype == np.int64 assert (2 * loaded.variables["x"]).vars.dtype == np.int64 - - -def test_label_dtype_option_int64() -> None: - with options: - options["label_dtype"] = np.int64 - m = Model() - x = m.add_variables(lower=0, upper=10, coords=[range(5)], name="x") - assert x.labels.dtype == np.int64 - expr = 2 * x + 1 - assert expr.vars.dtype == np.int64 - - -def test_label_dtype_rejects_invalid() -> None: - with pytest.raises(ValueError, match="label_dtype must be one of"): - options["label_dtype"] = np.float64 From 5d61e9c4da30949727454f0cf9c73cdb1ef1103f Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:58:56 +0200 Subject: [PATCH 11/15] refactor(int32): expose per-model dtypes as a read-only mapping Address review: generalize the per-model label-dtype accessor into a `Model.dtypes` property returning a read-only mapping (`MappingProxyType`) with a single `"labels"` field for now, extensible to more fields later. The constructor takes `dtypes={"labels": ...}` in place of `label_dtype=`; unknown keys and unsupported dtypes are rejected. Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/release_notes.rst | 2 +- linopy/model.py | 46 ++++++++++++++++++++++++++++--------------- test/test_dtypes.py | 30 +++++++++++++++++++--------- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 6a41bc736..65b65f069 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,7 +19,7 @@ Upcoming Version *Other* -* Default internal integer labels to ``int32``, cutting memory ~25% and speeding up model build 10-35%. Models exceeding the int32 maximum (~2.1 billion labels) widen to ``int64`` automatically with a ``UserWarning``; pass ``Model(label_dtype=np.int64)`` upfront to avoid the mid-build upcast. +* Default internal integer labels to ``int32``, cutting memory ~25% and speeding up model build 10-35%. Models exceeding the int32 maximum (~2.1 billion labels) widen to ``int64`` automatically with a ``UserWarning``; pass ``Model(dtypes={"labels": np.int64})`` upfront to avoid the mid-build upcast. The per-model dtypes are exposed read-only via ``Model.dtypes``. * ``add_variables(binary=True, ...)`` now accepts ``lower``/``upper`` bounds, as long as they are 0 or 1. Previously binary bounds could only be set via the ``.lower``/``.upper`` setters after creation. (https://github.com/PyPSA/linopy/issues/776) * ``add_piecewise_formulation`` gained an ``active_fill`` parameter that gates a partial ``active`` (defined over a subset of the indexed dimension, or masked) as always-active (``1``) or always-off (``0``); without it, a partial ``active`` — which was previously zeroed silently — now raises. Useful when one formulation mixes gated and ungated entities (e.g. committable and non-committable units sharing a ``status``). ``active_fill`` is transitional and will be removed once v1 semantics make ``active.reindex(coords).fillna(value)`` sufficient. (https://github.com/PyPSA/linopy/issues/796) diff --git a/linopy/model.py b/linopy/model.py index 545720f31..ed9a0960e 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -13,6 +13,7 @@ from collections.abc import Callable, Mapping, Sequence from pathlib import Path from tempfile import NamedTemporaryFile, gettempdir +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Literal, overload from warnings import warn @@ -190,7 +191,7 @@ def __init__( auto_mask: bool = False, freeze_constraints: bool = False, set_names_in_solver_io: bool = True, - label_dtype: type[np.signedinteger] = np.int32, + dtypes: Mapping[str, type[np.signedinteger]] | None = None, ) -> None: """ Initialize the linopy model. @@ -220,20 +221,31 @@ def __init__( set_names_in_solver_io : bool Whether direct solver exports should include variable and constraint names by default. The default is True. - label_dtype : np.int32 or np.int64, optional - Integer dtype used for the model's variable and constraint labels. - The default ``np.int32`` halves label memory but caps a model at - ~2.1 billion labels; the model widens itself to ``np.int64`` - automatically when that limit is hit. Pass ``np.int64`` upfront - for very large models to avoid the mid-build upcast. + dtypes : mapping, optional + Integer dtypes used for the model's data, as a mapping of field + name to numpy dtype, exposed read-only as ``Model.dtypes``. + Currently the only supported field is ``"labels"`` (the variable + and constraint labels), e.g. ``Model(dtypes={"labels": np.int64})``. + The default label dtype ``np.int32`` halves label memory but caps a + model at ~2.1 billion labels; the model widens itself to + ``np.int64`` automatically when that limit is hit. Pass + ``np.int64`` upfront for very large models to avoid the mid-build + upcast. Returns ------- linopy.Model """ + dtypes = dict(dtypes) if dtypes is not None else {} + unknown = set(dtypes) - {"labels"} + if unknown: + raise ValueError( + f"dtypes only supports the key 'labels', got unknown {sorted(unknown)}" + ) + label_dtype = dtypes.get("labels", np.int32) if label_dtype not in (np.int32, np.int64): raise ValueError( - f"label_dtype must be np.int32 or np.int64, got {label_dtype}" + f"dtypes['labels'] must be np.int32 or np.int64, got {label_dtype}" ) self._label_dtype: type[np.signedinteger] = label_dtype self._variables: Variables = Variables({}, model=self) @@ -509,14 +521,16 @@ def solver_dir(self, value: str | Path) -> None: self._solver_dir = Path(value) @property - def label_dtype(self) -> type[np.signedinteger]: + def dtypes(self) -> Mapping[str, type[np.signedinteger]]: """ - Integer dtype used for this model's variable and constraint labels. + Integer dtypes used for this model's data, as a read-only mapping. - Set via the ``label_dtype`` argument of ``Model()`` and automatically - widened to ``int64`` once the labels outgrow int32. + Currently exposes a single field, ``"labels"`` -- the dtype of the + model's variable and constraint labels. Set via the ``dtypes`` argument + of ``Model()`` and, for ``"labels"``, widened to ``int64`` automatically + once the labels outgrow int32. More fields may be added in the future. """ - return self._label_dtype + return MappingProxyType({"labels": self._label_dtype}) def _widen_label_dtype(self) -> None: """Widen this model's label dtype to ``int64`` (monotonic, never narrows).""" @@ -525,9 +539,9 @@ def _widen_label_dtype(self) -> None: self._label_dtype = np.int64 warnings.warn( "The model exceeded the int32 label limit (~2.1 billion labels); " - "its label dtype was widened to int64. Pass label_dtype=np.int64 " - "to Model() when building large models to avoid the mid-build " - "upcast.", + "its label dtype was widened to int64. Pass " + 'dtypes={"labels": np.int64} to Model() when building large models ' + "to avoid the mid-build upcast.", UserWarning, stacklevel=4, ) diff --git a/test/test_dtypes.py b/test/test_dtypes.py index 65889950a..e09cd3fd7 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -10,7 +10,7 @@ def test_default_label_dtype_is_int32() -> None: - assert Model().label_dtype == np.int32 + assert Model().dtypes["labels"] == np.int32 def test_variable_labels_are_int32() -> None: @@ -53,7 +53,7 @@ def test_auto_widen_variables() -> None: with pytest.warns(UserWarning, match="widened to int64"): x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") assert x.labels.dtype == np.int64 - assert m.label_dtype == np.int64 + assert m.dtypes["labels"] == np.int64 # the widening is per-model: other models are untouched other = Model() y = other.add_variables(lower=0, upper=1, coords=[range(5)], name="y") @@ -67,7 +67,7 @@ def test_auto_widen_constraints() -> None: with pytest.warns(UserWarning, match="widened to int64"): m.add_constraints(x >= 0, name="c") assert m.constraints["c"].labels.dtype == np.int64 - assert m.label_dtype == np.int64 + assert m.dtypes["labels"] == np.int64 def test_widen_applies_to_expressions() -> None: @@ -79,16 +79,28 @@ def test_widen_applies_to_expressions() -> None: def test_label_dtype_init_arg() -> None: - m = Model(label_dtype=np.int64) - assert m.label_dtype == np.int64 + m = Model(dtypes={"labels": np.int64}) + assert m.dtypes["labels"] == np.int64 x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") assert x.labels.dtype == np.int64 assert (2 * x + 1).vars.dtype == np.int64 def test_label_dtype_init_arg_rejects_invalid() -> None: - with pytest.raises(ValueError, match="label_dtype must be"): - Model(label_dtype=np.float64) # type: ignore[arg-type] + with pytest.raises(ValueError, match="dtypes\\['labels'\\] must be"): + Model(dtypes={"labels": np.float64}) # type: ignore[dict-item] + + +def test_dtypes_init_arg_rejects_unknown_key() -> None: + with pytest.raises(ValueError, match="only supports the key 'labels'"): + Model(dtypes={"coeffs": np.int64}) # type: ignore[dict-item] + + +def test_dtypes_is_read_only_mapping() -> None: + dtypes = Model().dtypes + assert set(dtypes) == {"labels"} + with pytest.raises(TypeError): + dtypes["labels"] = np.int64 # type: ignore[index] def test_auto_widen_survives_netcdf(tmp_path: Path) -> None: @@ -104,7 +116,7 @@ def test_auto_widen_survives_netcdf(tmp_path: Path) -> None: loaded = read_netcdf(path) - assert loaded.label_dtype == np.int64 + assert loaded.dtypes["labels"] == np.int64 assert loaded.variables["x"].labels.dtype == np.int64 assert (2 * loaded.variables["x"]).vars.dtype == np.int64 @@ -118,5 +130,5 @@ def test_auto_widen_survives_pickle() -> None: loaded = pickle.loads(pickle.dumps(m)) - assert loaded.label_dtype == np.int64 + assert loaded.dtypes["labels"] == np.int64 assert (2 * loaded.variables["x"]).vars.dtype == np.int64 From 2fd1c8d9e7cacfeb77ca6b850ca3db023b89462d Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:12:10 +0200 Subject: [PATCH 12/15] refactor(int32): store per-model dtypes in a single _dtypes dict Drop the redundant _label_dtype field: the model now holds one `_dtypes: dict[DtypeKey, type]` as the single source of truth, and `Model.dtypes` returns a read-only view over it. Keys are typed with a `DtypeKey = Literal["labels"]` alias; the constructor validates against it via `get_args`. All internal cast sites read `model._dtypes["labels"]`. Co-Authored-By: Claude Opus 4.8 (1M context) --- linopy/constraints.py | 10 +++++----- linopy/expressions.py | 8 ++++---- linopy/io.py | 2 +- linopy/model.py | 31 ++++++++++++++++++------------- linopy/variables.py | 10 +++++----- test/test_dtypes.py | 2 +- 6 files changed, 34 insertions(+), 29 deletions(-) diff --git a/linopy/constraints.py b/linopy/constraints.py index 3bfd5ccd2..dbd2d2eeb 100644 --- a/linopy/constraints.py +++ b/linopy/constraints.py @@ -752,7 +752,7 @@ def _to_dataset(self, nterm: int) -> Dataset: # Map active row i -> flat position in full shape via con_labels active_positions = self.active_positions coeffs_2d = np.zeros((full_size, nterm), dtype=csr.dtype) - vars_2d = np.full((full_size, nterm), -1, dtype=self._model._label_dtype) + vars_2d = np.full((full_size, nterm), -1, dtype=self._model._dtypes["labels"]) if csr.nnz > 0: row_indices = np.repeat(active_positions, counts) term_cols = np.arange(csr.nnz) - np.repeat(csr.indptr[:-1], counts) @@ -776,7 +776,7 @@ def _to_dataset(self, nterm: int) -> Dataset: ) ds = Dataset({"coeffs": coeffs_da, "vars": vars_da}) if self._cindex is not None: - labels_flat = np.full(full_size, -1, dtype=self._model._label_dtype) + labels_flat = np.full(full_size, -1, dtype=self._model._dtypes["labels"]) labels_flat[active_positions] = self._con_labels ds = assign_multiindex_safe( ds, @@ -1933,7 +1933,7 @@ def labels(self) -> Dataset: """ return save_join( *[v.labels.rename(k) for k, v in self.items()], - fill_dtype=self.model._label_dtype, + fill_dtype=self.model._dtypes["labels"], ) @property @@ -1954,7 +1954,7 @@ def rename_term_dim(ds: DataArray) -> DataArray: return save_join( *[rename_term_dim(v.vars.rename(k)) for k, v in self.items()], - fill_dtype=self.model._label_dtype, + fill_dtype=self.model._dtypes["labels"], ) @property @@ -2191,7 +2191,7 @@ def flat(self) -> pd.DataFrame: df = pd.concat(dfs, ignore_index=True) unique_labels = df.labels.unique() map_labels = pd.Series( - np.arange(len(unique_labels), dtype=self.model._label_dtype), + np.arange(len(unique_labels), dtype=self.model._dtypes["labels"]), index=unique_labels, ) df["key"] = df.labels.map(map_labels) diff --git a/linopy/expressions.py b/linopy/expressions.py index 3d7fde769..63ac295f9 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -536,7 +536,7 @@ def __init__(self, data: Dataset | Any | None, model: Model) -> None: if np.issubdtype(data.vars, np.floating): data = assign_multiindex_safe( - data, vars=data.vars.fillna(-1).astype(model._label_dtype) + data, vars=data.vars.fillna(-1).astype(model._dtypes["labels"]) ) if not np.issubdtype(data.coeffs, np.floating): data["coeffs"].values = data.coeffs.values.astype(float) @@ -1619,7 +1619,7 @@ def sanitize(self) -> Self: """ if not np.issubdtype(self.vars.dtype, np.integer): return self.assign( - vars=self.vars.fillna(-1).astype(self.model._label_dtype) + vars=self.vars.fillna(-1).astype(self.model._dtypes["labels"]) ) return self @@ -2024,12 +2024,12 @@ def _simplify_row(vars_row: np.ndarray, coeffs_row: np.ndarray) -> np.ndarray: # Combined has dimensions (.., CV_DIM, TERM_DIM) # Drop terms where all vars are -1 (i.e., empty terms across all coordinates) - vars = combined.isel({CV_DIM: 0}).astype(self.model._label_dtype) + vars = combined.isel({CV_DIM: 0}).astype(self.model._dtypes["labels"]) non_empty_terms = (vars != -1).any(dim=[d for d in vars.dims if d != TERM_DIM]) combined = combined.isel({TERM_DIM: non_empty_terms}) # Extract vars and coeffs from the combined result - vars = combined.isel({CV_DIM: 0}).astype(self.model._label_dtype) + vars = combined.isel({CV_DIM: 0}).astype(self.model._dtypes["labels"]) coeffs = combined.isel({CV_DIM: 1}) # Create new dataset with simplified data diff --git a/linopy/io.py b/linopy/io.py index d1557122f..e4859f69e 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -1120,7 +1120,7 @@ def get_prefix(ds: xr.Dataset, prefix: str) -> xr.Dataset: setattr(m, k, ds.attrs[k]) if max(m._xCounter, m._cCounter) > np.iinfo(np.int32).max: - m._label_dtype = np.int64 + m._dtypes["labels"] = np.int64 if "_relaxed_registry" in ds.attrs: m._relaxed_registry = json.loads(ds.attrs["_relaxed_registry"]) diff --git a/linopy/model.py b/linopy/model.py index ed9a0960e..d5814602f 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -14,7 +14,7 @@ from pathlib import Path from tempfile import NamedTemporaryFile, gettempdir from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Literal, overload +from typing import TYPE_CHECKING, Any, Literal, get_args, overload from warnings import warn import numpy as np @@ -112,6 +112,10 @@ logger = logging.getLogger(__name__) +#: Keys of the per-model :attr:`Model.dtypes` mapping. Extend the ``Literal`` +#: as more configurable dtypes are exposed on the model. +DtypeKey = Literal["labels"] + class Model: """ @@ -140,7 +144,7 @@ class Model: _termination_condition: str _xCounter: int _cCounter: int - _label_dtype: type[np.signedinteger] + _dtypes: dict[DtypeKey, type[np.signedinteger]] _varnameCounter: int _connameCounter: int _pwlCounter: int @@ -164,7 +168,7 @@ class Model: # TODO: move counters to Variables and Constraints class "_xCounter", "_cCounter", - "_label_dtype", + "_dtypes", "_varnameCounter", "_connameCounter", "_pwlCounter", @@ -191,7 +195,7 @@ def __init__( auto_mask: bool = False, freeze_constraints: bool = False, set_names_in_solver_io: bool = True, - dtypes: Mapping[str, type[np.signedinteger]] | None = None, + dtypes: Mapping[DtypeKey, type[np.signedinteger]] | None = None, ) -> None: """ Initialize the linopy model. @@ -237,17 +241,18 @@ def __init__( linopy.Model """ dtypes = dict(dtypes) if dtypes is not None else {} - unknown = set(dtypes) - {"labels"} + unknown = set(dtypes) - set(get_args(DtypeKey)) if unknown: raise ValueError( - f"dtypes only supports the key 'labels', got unknown {sorted(unknown)}" + f"dtypes only supports the keys {list(get_args(DtypeKey))}, got " + f"unknown {sorted(unknown)}" ) label_dtype = dtypes.get("labels", np.int32) if label_dtype not in (np.int32, np.int64): raise ValueError( f"dtypes['labels'] must be np.int32 or np.int64, got {label_dtype}" ) - self._label_dtype: type[np.signedinteger] = label_dtype + self._dtypes: dict[DtypeKey, type[np.signedinteger]] = {"labels": label_dtype} self._variables: Variables = Variables({}, model=self) self._constraints: Constraints = Constraints({}, model=self) self._objective: Objective = Objective(LinearExpression(None, self), self) @@ -521,7 +526,7 @@ def solver_dir(self, value: str | Path) -> None: self._solver_dir = Path(value) @property - def dtypes(self) -> Mapping[str, type[np.signedinteger]]: + def dtypes(self) -> Mapping[DtypeKey, type[np.signedinteger]]: """ Integer dtypes used for this model's data, as a read-only mapping. @@ -530,13 +535,13 @@ def dtypes(self) -> Mapping[str, type[np.signedinteger]]: of ``Model()`` and, for ``"labels"``, widened to ``int64`` automatically once the labels outgrow int32. More fields may be added in the future. """ - return MappingProxyType({"labels": self._label_dtype}) + return MappingProxyType(self._dtypes) def _widen_label_dtype(self) -> None: """Widen this model's label dtype to ``int64`` (monotonic, never narrows).""" - if self._label_dtype == np.int64: + if self._dtypes["labels"] == np.int64: return - self._label_dtype = np.int64 + self._dtypes["labels"] = np.int64 warnings.warn( "The model exceeded the int32 label limit (~2.1 billion labels); " "its label dtype was widened to int64. Pass " @@ -548,9 +553,9 @@ def _widen_label_dtype(self) -> None: def _allocate_labels(self, start: int, end: int) -> np.ndarray: """Return the label range ``[start, end)``, widening the dtype on overflow.""" - if end > np.iinfo(self._label_dtype).max: + if end > np.iinfo(self._dtypes["labels"]).max: self._widen_label_dtype() - return np.arange(start, end, dtype=self._label_dtype) + return np.arange(start, end, dtype=self._dtypes["labels"]) @property def dataset_attrs(self) -> list[str]: diff --git a/linopy/variables.py b/linopy/variables.py index 39a1c760a..c2e247bb0 100644 --- a/linopy/variables.py +++ b/linopy/variables.py @@ -1273,7 +1273,7 @@ def ffill(self, dim: str, limit: None = None) -> Variable: .fillna(self._fill_value) ) return self.assign_multiindex_safe( - labels=data.labels.astype(self.model._label_dtype) + labels=data.labels.astype(self.model._dtypes["labels"]) ) def bfill(self, dim: str, limit: None = None) -> Variable: @@ -1301,7 +1301,7 @@ def bfill(self, dim: str, limit: None = None) -> Variable: .map(DataArray.bfill, dim=dim, limit=limit) .fillna(self._fill_value) ) - return self.assign(labels=data.labels.astype(self.model._label_dtype)) + return self.assign(labels=data.labels.astype(self.model._dtypes["labels"])) def sanitize(self) -> Variable: """ @@ -1313,7 +1313,7 @@ def sanitize(self) -> Variable: """ if issubdtype(self.labels.dtype, floating): return self.assign( - labels=self.labels.fillna(-1).astype(self.model._label_dtype) + labels=self.labels.fillna(-1).astype(self.model._dtypes["labels"]) ) return self @@ -1727,7 +1727,7 @@ def labels(self) -> Dataset: """ return save_join( *[v.labels.rename(k) for k, v in self.items()], - fill_dtype=self.model._label_dtype, + fill_dtype=self.model._dtypes["labels"], ) @property @@ -2038,7 +2038,7 @@ def flat(self) -> pd.DataFrame: df = pd.concat([self[k].flat for k in self], ignore_index=True) unique_labels = df.labels.unique() map_labels = pd.Series( - np.arange(len(unique_labels), dtype=self.model._label_dtype), + np.arange(len(unique_labels), dtype=self.model._dtypes["labels"]), index=unique_labels, ) df["key"] = df.labels.map(map_labels) diff --git a/test/test_dtypes.py b/test/test_dtypes.py index e09cd3fd7..1284d6548 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -92,7 +92,7 @@ def test_label_dtype_init_arg_rejects_invalid() -> None: def test_dtypes_init_arg_rejects_unknown_key() -> None: - with pytest.raises(ValueError, match="only supports the key 'labels'"): + with pytest.raises(ValueError, match="only supports the keys"): Model(dtypes={"coeffs": np.int64}) # type: ignore[dict-item] From f64d1bf46b2cdf5a7a6ba9bf523804a6643938a9 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:18:05 +0200 Subject: [PATCH 13/15] refactor(int32): extract dtypes validation into _resolve_dtypes helper Move the `dtypes` argument validation out of `__init__` into a small `Model._resolve_dtypes` staticmethod that merges the argument onto the defaults in one loop, rejecting unknown keys and non-int32/64 dtypes. `__init__` now just assigns its result. Co-Authored-By: Claude Opus 4.8 (1M context) --- linopy/model.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/linopy/model.py b/linopy/model.py index d5814602f..6232ac194 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -187,6 +187,25 @@ class Model: "__weakref__", ) + @staticmethod + def _resolve_dtypes( + dtypes: Mapping[DtypeKey, type[np.signedinteger]] | None, + ) -> dict[DtypeKey, type[np.signedinteger]]: + """Validate the ``dtypes`` argument and merge it onto the defaults.""" + resolved: dict[DtypeKey, type[np.signedinteger]] = {"labels": np.int32} + for key, dtype in (dtypes or {}).items(): + if key not in get_args(DtypeKey): + raise ValueError( + f"dtypes only supports the keys {list(get_args(DtypeKey))}, " + f"got unknown key {key!r}" + ) + if dtype not in (np.int32, np.int64): + raise ValueError( + f"dtypes[{key!r}] must be np.int32 or np.int64, got {dtype}" + ) + resolved[key] = dtype + return resolved + def __init__( self, solver_dir: str | None = None, @@ -240,19 +259,9 @@ def __init__( ------- linopy.Model """ - dtypes = dict(dtypes) if dtypes is not None else {} - unknown = set(dtypes) - set(get_args(DtypeKey)) - if unknown: - raise ValueError( - f"dtypes only supports the keys {list(get_args(DtypeKey))}, got " - f"unknown {sorted(unknown)}" - ) - label_dtype = dtypes.get("labels", np.int32) - if label_dtype not in (np.int32, np.int64): - raise ValueError( - f"dtypes['labels'] must be np.int32 or np.int64, got {label_dtype}" - ) - self._dtypes: dict[DtypeKey, type[np.signedinteger]] = {"labels": label_dtype} + self._dtypes: dict[DtypeKey, type[np.signedinteger]] = self._resolve_dtypes( + dtypes + ) self._variables: Variables = Variables({}, model=self) self._constraints: Constraints = Constraints({}, model=self) self._objective: Objective = Objective(LinearExpression(None, self), self) From cc1252b34bcfdb0af54f1ecc3efa12e52b62182d Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:20:07 +0200 Subject: [PATCH 14/15] style(int32): drop Sphinx comment on DtypeKey alias The alias name is self-explanatory; a Literal alias cannot carry a real docstring anyway. Co-Authored-By: Claude Opus 4.8 (1M context) --- linopy/model.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/linopy/model.py b/linopy/model.py index 6232ac194..f9b550194 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -112,8 +112,6 @@ logger = logging.getLogger(__name__) -#: Keys of the per-model :attr:`Model.dtypes` mapping. Extend the ``Literal`` -#: as more configurable dtypes are exposed on the model. DtypeKey = Literal["labels"] From e63d92b248050c3bffed639e32c70e922322455b Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:21:44 +0200 Subject: [PATCH 15/15] docs(int32): trim dtypes docstrings and release note Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/release_notes.rst | 2 +- linopy/model.py | 24 ++++++++++-------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 65b65f069..665a5023e 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,7 +19,7 @@ Upcoming Version *Other* -* Default internal integer labels to ``int32``, cutting memory ~25% and speeding up model build 10-35%. Models exceeding the int32 maximum (~2.1 billion labels) widen to ``int64`` automatically with a ``UserWarning``; pass ``Model(dtypes={"labels": np.int64})`` upfront to avoid the mid-build upcast. The per-model dtypes are exposed read-only via ``Model.dtypes``. +* Default internal integer labels to ``int32``, cutting memory ~25% and speeding up model build 10-35%. Models exceeding the int32 maximum (~2.1 billion labels) widen to ``int64`` automatically with a ``UserWarning``; pass ``Model(dtypes={"labels": np.int64})`` upfront to avoid the mid-build upcast (exposed read-only via ``Model.dtypes``). * ``add_variables(binary=True, ...)`` now accepts ``lower``/``upper`` bounds, as long as they are 0 or 1. Previously binary bounds could only be set via the ``.lower``/``.upper`` setters after creation. (https://github.com/PyPSA/linopy/issues/776) * ``add_piecewise_formulation`` gained an ``active_fill`` parameter that gates a partial ``active`` (defined over a subset of the indexed dimension, or masked) as always-active (``1``) or always-off (``0``); without it, a partial ``active`` — which was previously zeroed silently — now raises. Useful when one formulation mixes gated and ungated entities (e.g. committable and non-committable units sharing a ``status``). ``active_fill`` is transitional and will be removed once v1 semantics make ``active.reindex(coords).fillna(value)`` sufficient. (https://github.com/PyPSA/linopy/issues/796) diff --git a/linopy/model.py b/linopy/model.py index f9b550194..f814dbf11 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -243,15 +243,12 @@ def __init__( Whether direct solver exports should include variable and constraint names by default. The default is True. dtypes : mapping, optional - Integer dtypes used for the model's data, as a mapping of field - name to numpy dtype, exposed read-only as ``Model.dtypes``. - Currently the only supported field is ``"labels"`` (the variable - and constraint labels), e.g. ``Model(dtypes={"labels": np.int64})``. - The default label dtype ``np.int32`` halves label memory but caps a - model at ~2.1 billion labels; the model widens itself to - ``np.int64`` automatically when that limit is hit. Pass - ``np.int64`` upfront for very large models to avoid the mid-build - upcast. + Integer dtypes for the model's data, exposed read-only as + ``Model.dtypes``. Only ``"labels"`` is supported, e.g. + ``Model(dtypes={"labels": np.int64})``. The default ``np.int32`` + halves label memory but caps the model at ~2.1 billion labels, + after which it widens to ``np.int64`` automatically; pass + ``np.int64`` upfront to avoid that mid-build upcast. Returns ------- @@ -535,12 +532,11 @@ def solver_dir(self, value: str | Path) -> None: @property def dtypes(self) -> Mapping[DtypeKey, type[np.signedinteger]]: """ - Integer dtypes used for this model's data, as a read-only mapping. + Read-only mapping of the model's integer dtypes. - Currently exposes a single field, ``"labels"`` -- the dtype of the - model's variable and constraint labels. Set via the ``dtypes`` argument - of ``Model()`` and, for ``"labels"``, widened to ``int64`` automatically - once the labels outgrow int32. More fields may be added in the future. + Currently holds only ``"labels"``, the dtype of the variable and + constraint labels, which widens to ``int64`` automatically once the + labels outgrow int32. """ return MappingProxyType(self._dtypes)