From 129d119a1046be4e7cecd03bca413a0923d5a90d Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 14 Jul 2026 11:48:48 +0200 Subject: [PATCH 1/5] perf(int32): auto-widen labels to int64 on overflow instead of raising Replace the int32 overflow ValueError in add_variables/add_constraints with a monotonic, sticky widen of options["label_dtype"] to int64. Once a model crosses ~2.1 billion labels the process uses int64 labels everywhere (surviving options.reset()), so no cast site can silently narrow an int64 label back to int32. Mixed int32/int64 arrays upcast on concat, so previously-allocated int32 batches stay valid. --- doc/release_notes.rst | 2 +- linopy/config.py | 5 +++++ linopy/model.py | 22 +++++++--------------- test/test_dtypes.py | 27 +++++++++++++++++++++------ 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index f8464a7d2..9ca573dc2 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,7 +19,7 @@ Upcoming Version *Other* -* Default internal integer labels to ``int32`` (configurable via ``linopy.options["label_dtype"]``, set to ``np.int64`` for the old behavior), cutting memory ~25% and speeding up model build 10-35%. Raises ``ValueError`` if labels exceed the int32 maximum. +* Default internal integer labels to ``int32`` (configurable via ``linopy.options["label_dtype"]``, set to ``np.int64`` for the old behavior), cutting memory ~25% and speeding up model build 10-35%. Labels auto-widen to ``int64`` once a model exceeds the int32 maximum (~2.1 billion labels); the widening is permanent for the process and can be reverted with ``linopy.options["label_dtype"] = np.int32``. * ``add_variables(binary=True, ...)`` now accepts ``lower``/``upper`` bounds, as long as they are 0 or 1. Previously binary bounds could only be set via the ``.lower``/``.upper`` setters after creation. (https://github.com/PyPSA/linopy/issues/776) * ``add_piecewise_formulation`` gained an ``active_fill`` parameter that gates a partial ``active`` (defined over a subset of the indexed dimension, or masked) as always-active (``1``) or always-off (``0``); without it, a partial ``active`` — which was previously zeroed silently — now raises. Useful when one formulation mixes gated and ungated entities (e.g. committable and non-committable units sharing a ``status``). ``active_fill`` is transitional and will be removed once v1 semantics make ``active.reindex(coords).fillna(value)`` sufficient. (https://github.com/PyPSA/linopy/issues/796) diff --git a/linopy/config.py b/linopy/config.py index 6a28d43fe..13a5002ec 100644 --- a/linopy/config.py +++ b/linopy/config.py @@ -46,6 +46,11 @@ def get_value(self, name: str) -> Any: else: raise KeyError(f"{name} is not a valid setting.") + def widen_label_dtype(self) -> None: + """Permanently widen ``label_dtype`` to ``int64`` (monotonic, survives ``reset``).""" + self._defaults["label_dtype"] = np.int64 + self._current_values["label_dtype"] = np.int64 + def reset(self) -> None: self._current_values = self._defaults.copy() diff --git a/linopy/model.py b/linopy/model.py index b1477b276..f4fa6a365 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -825,12 +825,8 @@ def add_variables( start = self._xCounter end = start + data.labels.size - label_dtype = options["label_dtype"] - if end > np.iinfo(label_dtype).max: - raise ValueError( - f"Number of labels ({end}) exceeds the maximum value for " - f"{label_dtype.__name__} ({np.iinfo(label_dtype).max})." - ) + if end > np.iinfo(options["label_dtype"]).max: + options.widen_label_dtype() data.labels.values = np.arange( start, end, dtype=options["label_dtype"] ).reshape(data.labels.shape) @@ -978,15 +974,11 @@ def _allocate_constraint_labels( """Assign label ranges from the constraint counter and apply an optional mask.""" start = self._cCounter end = start + data.labels.size - label_dtype = options["label_dtype"] - if end > np.iinfo(label_dtype).max: - raise ValueError( - f"Number of labels ({end}) exceeds the maximum value for " - f"{label_dtype.__name__} ({np.iinfo(label_dtype).max})." - ) - data.labels.values = np.arange(start, end, dtype=label_dtype).reshape( - data.labels.shape - ) + if end > np.iinfo(options["label_dtype"]).max: + options.widen_label_dtype() + data.labels.values = np.arange( + start, end, dtype=options["label_dtype"] + ).reshape(data.labels.shape) self._cCounter += data.labels.size if mask is not None: data.labels.values = np.where(mask.values, data.labels.values, -1) diff --git a/test/test_dtypes.py b/test/test_dtypes.py index b30c7eac3..fae6b8368 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -45,19 +45,34 @@ def test_solve_with_int32_labels() -> None: assert m.objective.value == pytest.approx(25.0) -def test_overflow_guard_variables() -> None: +@pytest.fixture +def restore_label_dtype(): + yield + options._defaults["label_dtype"] = np.int32 + options["label_dtype"] = np.int32 + + +def test_auto_widen_variables(restore_label_dtype) -> None: m = Model() m._xCounter = np.iinfo(np.int32).max - 1 - with pytest.raises(ValueError, match="exceeds the maximum"): - m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + assert x.labels.dtype == np.int64 + assert options["label_dtype"] == np.int64 -def test_overflow_guard_constraints() -> None: +def test_auto_widen_constraints(restore_label_dtype) -> None: m = Model() x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") m._cCounter = np.iinfo(np.int32).max - 1 - with pytest.raises(ValueError, match="exceeds the maximum"): - m.add_constraints(x >= 0, name="c") + m.add_constraints(x >= 0, name="c") + assert m.constraints["c"].labels.dtype == np.int64 + assert options["label_dtype"] == np.int64 + + +def test_auto_widen_is_sticky(restore_label_dtype) -> None: + options.widen_label_dtype() + options.reset() + assert options["label_dtype"] == np.int64 def test_label_dtype_option_int64() -> None: From 067fd2a7d612cdda7533527848faa17e3962fd94 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 14 Jul 2026 12:03:26 +0200 Subject: [PATCH 2/5] perf(int32): re-widen label dtype when loading an int64 model read_netcdf reconstructs label arrays directly and restores the counters without going through the add_variables/add_constraints widen path, so a model that had auto-widened to int64 would load with int64 label arrays while the process option stayed int32 -- the silent-downcast corner, reachable via read_netcdf and solve(remote=...). Widen the option on load whenever a restored counter exceeds the int32 maximum. --- linopy/io.py | 4 ++++ test/test_dtypes.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/linopy/io.py b/linopy/io.py index bccbdecf7..6692d1a40 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -26,6 +26,7 @@ from linopy import solvers from linopy.common import to_polars +from linopy.config import options from linopy.constants import CONCAT_DIM, SOS_DIM_ATTR, SOS_TYPE_ATTR from linopy.objective import Objective @@ -1119,6 +1120,9 @@ def get_prefix(ds: xr.Dataset, prefix: str) -> xr.Dataset: if k in ds.attrs: setattr(m, k, ds.attrs[k]) + if max(m._xCounter, m._cCounter) > np.iinfo(np.int32).max: + options.widen_label_dtype() + if "_relaxed_registry" in ds.attrs: m._relaxed_registry = json.loads(ds.attrs["_relaxed_registry"]) diff --git a/test/test_dtypes.py b/test/test_dtypes.py index fae6b8368..872674beb 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -75,6 +75,25 @@ def test_auto_widen_is_sticky(restore_label_dtype) -> None: assert options["label_dtype"] == np.int64 +def test_auto_widen_survives_netcdf(restore_label_dtype, tmp_path) -> None: + from linopy import read_netcdf + + m = Model() + m._xCounter = np.iinfo(np.int32).max - 1 + x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + m.add_constraints(x >= 0, name="c") + path = tmp_path / "widened.nc" + m.to_netcdf(path) + + options._defaults["label_dtype"] = np.int32 + options["label_dtype"] = np.int32 + loaded = read_netcdf(path) + + assert options["label_dtype"] == np.int64 + assert loaded.variables["x"].labels.dtype == np.int64 + assert (2 * loaded.variables["x"]).vars.dtype == np.int64 + + def test_label_dtype_option_int64() -> None: with options: options["label_dtype"] = np.int64 From d3388580ca2435e841dc44b6c7918a45c96af3d0 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 14 Jul 2026 12:13:56 +0200 Subject: [PATCH 3/5] perf(int32): warn on auto-widen, recommend explicit int64 for large models Address review: keep auto-widening a convenience but emit a UserWarning at the int32->int64 transition, advising users to set label_dtype=int64 upfront (avoids the mid-build upcast, faster and lighter). --- doc/release_notes.rst | 2 +- linopy/config.py | 12 ++++++++++++ test/test_dtypes.py | 6 ++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 9ca573dc2..f5ecdbbe9 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,7 +19,7 @@ Upcoming Version *Other* -* Default internal integer labels to ``int32`` (configurable via ``linopy.options["label_dtype"]``, set to ``np.int64`` for the old behavior), cutting memory ~25% and speeding up model build 10-35%. Labels auto-widen to ``int64`` once a model exceeds the int32 maximum (~2.1 billion labels); the widening is permanent for the process and can be reverted with ``linopy.options["label_dtype"] = np.int32``. +* Default internal integer labels to ``int32`` (configurable via ``linopy.options["label_dtype"]``, set to ``np.int64`` for the old behavior), cutting memory ~25% and speeding up model build 10-35%. Labels auto-widen to ``int64`` once a model exceeds the int32 maximum (~2.1 billion labels), emitting a ``UserWarning`` that recommends setting ``linopy.options["label_dtype"] = np.int64`` explicitly for large models; the widening is permanent for the process and can be reverted with ``linopy.options["label_dtype"] = np.int32``. * ``add_variables(binary=True, ...)`` now accepts ``lower``/``upper`` bounds, as long as they are 0 or 1. Previously binary bounds could only be set via the ``.lower``/``.upper`` setters after creation. (https://github.com/PyPSA/linopy/issues/776) * ``add_piecewise_formulation`` gained an ``active_fill`` parameter that gates a partial ``active`` (defined over a subset of the indexed dimension, or masked) as always-active (``1``) or always-off (``0``); without it, a partial ``active`` — which was previously zeroed silently — now raises. Useful when one formulation mixes gated and ungated entities (e.g. committable and non-committable units sharing a ``status``). ``active_fill`` is transitional and will be removed once v1 semantics make ``active.reindex(coords).fillna(value)`` sufficient. (https://github.com/PyPSA/linopy/issues/796) diff --git a/linopy/config.py b/linopy/config.py index 13a5002ec..38d4a2e1e 100644 --- a/linopy/config.py +++ b/linopy/config.py @@ -7,6 +7,7 @@ from __future__ import annotations +import warnings from typing import Any import numpy as np @@ -48,8 +49,19 @@ def get_value(self, name: str) -> Any: def widen_label_dtype(self) -> None: """Permanently widen ``label_dtype`` to ``int64`` (monotonic, survives ``reset``).""" + transitioned = self._current_values["label_dtype"] != np.int64 self._defaults["label_dtype"] = np.int64 self._current_values["label_dtype"] = np.int64 + if transitioned: + warnings.warn( + "Auto-widened the internal label dtype to int64: the model " + "exceeded the int32 label limit (~2.1 billion). Set " + "linopy.options['label_dtype'] = np.int64 explicitly when large " + "models are expected -- it avoids the mid-build upcast and is " + "faster and lighter on memory.", + UserWarning, + stacklevel=2, + ) def reset(self) -> None: self._current_values = self._defaults.copy() diff --git a/test/test_dtypes.py b/test/test_dtypes.py index 872674beb..50fa892a0 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -55,7 +55,8 @@ def restore_label_dtype(): def test_auto_widen_variables(restore_label_dtype) -> None: m = Model() m._xCounter = np.iinfo(np.int32).max - 1 - x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") + with pytest.warns(UserWarning, match="Auto-widened"): + x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") assert x.labels.dtype == np.int64 assert options["label_dtype"] == np.int64 @@ -87,7 +88,8 @@ def test_auto_widen_survives_netcdf(restore_label_dtype, tmp_path) -> None: options._defaults["label_dtype"] = np.int32 options["label_dtype"] = np.int32 - loaded = read_netcdf(path) + with pytest.warns(UserWarning, match="Auto-widened"): + loaded = read_netcdf(path) assert options["label_dtype"] == np.int64 assert loaded.variables["x"].labels.dtype == np.int64 From 15b6a2f4731b88f10dc5fa1aa97adbacb1144a5b Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 14 Jul 2026 12:32:08 +0200 Subject: [PATCH 4/5] test(int32): add type annotations to auto-widen tests for mypy --- test/test_dtypes.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/test_dtypes.py b/test/test_dtypes.py index 50fa892a0..965d68fcb 100644 --- a/test/test_dtypes.py +++ b/test/test_dtypes.py @@ -1,5 +1,8 @@ """Tests for int32 default label dtype.""" +from collections.abc import Generator +from pathlib import Path + import numpy as np import pytest @@ -46,13 +49,13 @@ def test_solve_with_int32_labels() -> None: @pytest.fixture -def restore_label_dtype(): +def restore_label_dtype() -> Generator[None, None, None]: yield options._defaults["label_dtype"] = np.int32 options["label_dtype"] = np.int32 -def test_auto_widen_variables(restore_label_dtype) -> None: +def test_auto_widen_variables(restore_label_dtype: None) -> None: m = Model() m._xCounter = np.iinfo(np.int32).max - 1 with pytest.warns(UserWarning, match="Auto-widened"): @@ -61,7 +64,7 @@ def test_auto_widen_variables(restore_label_dtype) -> None: assert options["label_dtype"] == np.int64 -def test_auto_widen_constraints(restore_label_dtype) -> None: +def test_auto_widen_constraints(restore_label_dtype: None) -> None: m = Model() x = m.add_variables(lower=0, upper=1, coords=[range(5)], name="x") m._cCounter = np.iinfo(np.int32).max - 1 @@ -70,13 +73,13 @@ def test_auto_widen_constraints(restore_label_dtype) -> None: assert options["label_dtype"] == np.int64 -def test_auto_widen_is_sticky(restore_label_dtype) -> None: +def test_auto_widen_is_sticky(restore_label_dtype: None) -> None: options.widen_label_dtype() options.reset() assert options["label_dtype"] == np.int64 -def test_auto_widen_survives_netcdf(restore_label_dtype, tmp_path) -> None: +def test_auto_widen_survives_netcdf(restore_label_dtype: None, tmp_path: Path) -> None: from linopy import read_netcdf m = Model() From e2152970730ed3fc7ddb5f7bd7be02ec9d42b2af Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:04:38 +0200 Subject: [PATCH 5/5] chore: trigger CI Co-Authored-By: Claude Fable 5