From 78b8ed366e4907e3ca6718fd7bfda2c3bc287113 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 16:57:34 +0000 Subject: [PATCH 01/13] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/uv-pre-commit: 0.11.19 → 0.11.21](https://github.com/astral-sh/uv-pre-commit/compare/0.11.19...0.11.21) - [github.com/astral-sh/ruff-pre-commit: v0.15.16 → v0.15.17](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.16...v0.15.17) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b931f51..2f9b6dd8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/astral-sh/uv-pre-commit # uv version. - rev: 0.11.19 + rev: 0.11.21 hooks: # Dependency management - id: uv-lock @@ -47,7 +47,7 @@ repos: # Python Linting & Formatting with Ruff - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.15.16" + rev: "v0.15.17" hooks: - id: ruff name: ruff (linter) From 768f6bb2d6fd08a8e03db19bf12b83a369ef9a8e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 16:56:35 +0000 Subject: [PATCH 02/13] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/uv-pre-commit: 0.11.21 → 0.11.23](https://github.com/astral-sh/uv-pre-commit/compare/0.11.21...0.11.23) - [github.com/astral-sh/ruff-pre-commit: v0.15.17 → v0.15.18](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.17...v0.15.18) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2f9b6dd8..5f170d2a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/astral-sh/uv-pre-commit # uv version. - rev: 0.11.21 + rev: 0.11.23 hooks: # Dependency management - id: uv-lock @@ -47,7 +47,7 @@ repos: # Python Linting & Formatting with Ruff - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.15.17" + rev: "v0.15.18" hooks: - id: ruff name: ruff (linter) From 25ea39f590ed25cf13a264ded41b9bdf17596a32 Mon Sep 17 00:00:00 2001 From: Davey Elder Date: Wed, 24 Jun 2026 17:37:20 -0400 Subject: [PATCH 03/13] Load early retirements from previous planning periods Signed-off-by: Davey Elder --- temoa/core/model.py | 4 ++++ temoa/data_io/component_manifest.py | 8 ++++++++ temoa/data_io/hybrid_loader.py | 31 +++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/temoa/core/model.py b/temoa/core/model.py index 6c492e2a..a288b83e 100755 --- a/temoa/core/model.py +++ b/temoa/core/model.py @@ -383,6 +383,10 @@ def __init__(self, *args: object, **kwargs: object) -> None: self.capacity_to_activity = Param(self.regional_indices, self.tech_all, default=1) self.existing_capacity = Param(self.regional_indices, self.tech_exist, self.vintage_exist) + # This is needed to handle past retirements in myopic mode. Maybe it will find other uses. + self.retired_existing_capacity = Param( + self.regional_indices, self.time_exist, self.tech_exist, self.vintage_exist, default=0 + ) # Dev Note: The below is temporarily useful for passing down to validator to find # set violations diff --git a/temoa/data_io/component_manifest.py b/temoa/data_io/component_manifest.py index e22fdf67..12426048 100644 --- a/temoa/data_io/component_manifest.py +++ b/temoa/data_io/component_manifest.py @@ -288,6 +288,14 @@ def build_manifest(model: TemoaModel) -> list[LoadItem]: is_period_filtered=False, # Custom loader handles all logic is_table_required=False, ), + LoadItem( + component=model.retired_existing_capacity, + table='output_retired_capacity', + columns=['region', 'period', 'tech', 'vintage', 'cap_early'], + custom_loader_name='_load_retired_existing_capacity', + is_period_filtered=False, # Custom loader handles all logic + is_table_required=False, + ), LoadItem( component=model.cost_invest, table='cost_invest', diff --git a/temoa/data_io/hybrid_loader.py b/temoa/data_io/hybrid_loader.py index a4e2ce4d..d47a3635 100644 --- a/temoa/data_io/hybrid_loader.py +++ b/temoa/data_io/hybrid_loader.py @@ -630,6 +630,37 @@ def _load_existing_capacity( tech_exist_data = sorted({(row[1],) for row in rows_to_load}) self._load_component_data(data, model.tech_exist, tech_exist_data) + def _load_retired_existing_capacity( + self, + data: dict[str, object], + raw_data: Sequence[tuple[object, ...]], + filtered_data: Sequence[tuple[object, ...]], + ) -> None: + """ + Handles different queries for myopic vs. standard runs and also + populates the `tech_exist` set. + """ + model = TemoaModel() + cur = self.con.cursor() + mi = self.myopic_index + + if not mi: + # for now, we only use this in myopic mode + return + + rows_to_load = [] + prev_period_res = cur.execute( + 'SELECT MAX(period) FROM time_period WHERE period < ?', (mi.base_year,) + ).fetchone() + prev_period = prev_period_res[0] if prev_period_res else -1 + rows_to_load = cur.execute( + 'SELECT region, period, tech, vintage, cap_early FROM output_retired_capacity WHERE ' + 'period <= ? AND scenario = ? AND cap_early > 0 ', + (prev_period, self.config.scenario), + ).fetchall() + + self._load_component_data(data, model.retired_existing_capacity, rows_to_load) + # --- Singleton and Configuration-based Components --- def _load_global_discount_rate( self, From 1e798864c8b3b65e50af9d290426ac9cbf1ecb1e Mon Sep 17 00:00:00 2001 From: Davey Elder Date: Wed, 24 Jun 2026 17:55:26 -0400 Subject: [PATCH 04/13] Adjust existing capacity for past early retirement in myopic Signed-off-by: Davey Elder --- temoa/components/capacity.py | 9 ++++++--- temoa/components/limits.py | 9 +++++++-- temoa/components/technology.py | 11 ++++++++--- temoa/components/utils.py | 17 +++++++++++++++++ 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/temoa/components/capacity.py b/temoa/components/capacity.py index 1383e09f..82e55312 100644 --- a/temoa/components/capacity.py +++ b/temoa/components/capacity.py @@ -18,7 +18,7 @@ from deprecated import deprecated from pyomo.environ import value -from .utils import get_capacity_factor +from .utils import get_adjusted_existing_capacity, get_capacity_factor if TYPE_CHECKING: from temoa.core.model import TemoaModel @@ -292,7 +292,9 @@ def annual_retirement_constraint( # Exact EOL. No v_capacity or v_retired_capacity for this period. if p == model.time_optimize.first(): # Must be existing capacity. Apply survival curve to existing cap - cap_begin = model.existing_capacity[r, t, v] * model.lifetime_survival_curve[r, p, t, v] + cap_begin = get_adjusted_existing_capacity(model, r, t, v) * value( + model.lifetime_survival_curve[r, p, t, v] + ) else: # Get previous capacity and continue survival curve p_prev = model.time_optimize.prev(p) @@ -545,8 +547,9 @@ def adjusted_capacity_constraint( the time when that retirement occurred (treated here as at the beginning of each period). """ + built_capacity: ExprLike if v in model.time_exist: - built_capacity = value(model.existing_capacity[r, t, v]) + built_capacity = get_adjusted_existing_capacity(model, r, t, v) else: built_capacity = model.v_new_capacity[r, t, v] diff --git a/temoa/components/limits.py b/temoa/components/limits.py index f7f5d5d9..4d35e8b6 100644 --- a/temoa/components/limits.py +++ b/temoa/components/limits.py @@ -20,7 +20,12 @@ import temoa.components.geography as geography import temoa.components.technology as technology -from temoa.components.utils import Operator, get_variable_efficiency, operator_expression +from temoa.components.utils import ( + Operator, + get_adjusted_existing_capacity, + get_variable_efficiency, + operator_expression, +) if TYPE_CHECKING: from pyomo.core import Expression @@ -1046,7 +1051,7 @@ def limit_growth_capacity( # Adjust in-line for past PLF because we are constraining available capacity p_prev = model.time_exist.last() capacity_prev = sum( - value(model.existing_capacity[_r, _t, _v]) + get_adjusted_existing_capacity(model, _r, _t, _v) * min(1.0, (_v + value(model.lifetime_process[_r, _t, _v]) - p_prev) / (p - p_prev)) for _r, _t, _v in model.existing_capacity.sparse_keys() if _r in regions diff --git a/temoa/components/technology.py b/temoa/components/technology.py index e9cc254d..404e2cb2 100644 --- a/temoa/components/technology.py +++ b/temoa/components/technology.py @@ -17,6 +17,8 @@ from pyomo.environ import value +from temoa.components.utils import get_adjusted_existing_capacity + if TYPE_CHECKING: from collections.abc import Iterable @@ -426,12 +428,15 @@ def check_existing_capacity(model: TemoaModel) -> None: continue if t not in model.tech_all: continue + if get_adjusted_existing_capacity(model, r, t, v) <= 0: + # It was just retired earlier + continue life = value(model.lifetime_process[r, t, v]) if (r, t, v) not in model.process_periods and v + life > model.time_optimize.first(): msg = ( f'Existing capacity {r, t, v} with lifetime {life} and capacity {cap} ' 'should extend into future periods but it is not in process periods. ' - 'Was it included in the Efficiency table?' + 'May be missing from Efficiency table or too small to carry forward ' + 'if running in myopic mode.' ) - logger.error(msg) - raise ValueError(msg) + logger.warning(msg) diff --git a/temoa/components/utils.py b/temoa/components/utils.py index 943b9018..daa2b421 100644 --- a/temoa/components/utils.py +++ b/temoa/components/utils.py @@ -87,3 +87,20 @@ def get_capacity_factor( if model.is_capacity_factor_process[r, t, v]: return value(model.capacity_factor_process[r, s, d, t, v]) return value(model.capacity_factor_tech[r, s, d, t]) + + +def get_adjusted_existing_capacity( + model: TemoaModel, r: Region, t: Technology, v: Vintage +) -> float: + """ + Returns the built existing capacity adjusted for any early retirements. + + Needed for early retirements in myopic mode. Takes into account survival curves + and any early retirements that may have occurred prior to this planning step. + """ + capacity_adjustment = sum( + value(model.retired_existing_capacity[r, _p, t, v]) + / (value(model.lifetime_survival_curve[r, _p, t, v]) or 1) + for _p in model.time_exist + ) + return value(model.existing_capacity[r, t, v]) - capacity_adjustment From 39cd539ff049338d26a30e2bbdd22d69d5bd4f4d Mon Sep 17 00:00:00 2001 From: Davey Elder Date: Wed, 24 Jun 2026 17:56:27 -0400 Subject: [PATCH 05/13] Fix handling of existing capacity for p0 Signed-off-by: Davey Elder --- temoa/components/limits.py | 3 +-- temoa/components/time.py | 5 +++++ temoa/core/model.py | 4 +++- temoa/data_io/component_manifest.py | 1 - 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/temoa/components/limits.py b/temoa/components/limits.py index 4d35e8b6..857cb148 100644 --- a/temoa/components/limits.py +++ b/temoa/components/limits.py @@ -1048,11 +1048,10 @@ def limit_growth_capacity( if p == model.time_optimize.first(): # First future period. Grab available capacity in last existing period - # Adjust in-line for past PLF because we are constraining available capacity p_prev = model.time_exist.last() capacity_prev = sum( get_adjusted_existing_capacity(model, _r, _t, _v) - * min(1.0, (_v + value(model.lifetime_process[_r, _t, _v]) - p_prev) / (p - p_prev)) + * value(model.process_life_frac[_r, p_prev, _t, _v]) for _r, _t, _v in model.existing_capacity.sparse_keys() if _r in regions and _t in techs diff --git a/temoa/components/time.py b/temoa/components/time.py index 0877a40b..c34a6bed 100644 --- a/temoa/components/time.py +++ b/temoa/components/time.py @@ -196,6 +196,11 @@ def init_set_vintage_optimize(model: TemoaModel) -> list[int]: def param_period_length(model: TemoaModel, p: Period) -> int: """Rule to calculate the length of each optimization period in years.""" + if model.time_exist and p == model.time_exist.last(): + # Need this for one specific use case (capacity growth constraints) + return model.time_future.first() - model.time_exist.last() + elif p in model.time_exist: + return -1 # Period length is not defined for existing periods except the last periods: list[int] = sorted(model.time_future) i: int = periods.index(p) return periods[i + 1] - periods[i] diff --git a/temoa/core/model.py b/temoa/core/model.py index a288b83e..ac7146b3 100755 --- a/temoa/core/model.py +++ b/temoa/core/model.py @@ -335,7 +335,9 @@ def __init__(self, *args: object, **kwargs: object) -> None: # Define time-related parameters # Basic period construction self.time_sequencing = Set() # How do states carry between time segments? - self.period_length = Param(self.time_optimize, initialize=time.param_period_length) + self.period_length = Param( + self.time_optimize | self.time_exist, initialize=time.param_period_length + ) self.days_per_period = Param(domain=PositiveReals, default=365.0) self.time_of_day_hours = Param(self.time_of_day, domain=PositiveReals, default=1.0) self.segment_fraction_per_season = Param(self.time_season) diff --git a/temoa/data_io/component_manifest.py b/temoa/data_io/component_manifest.py index 12426048..72688e3b 100644 --- a/temoa/data_io/component_manifest.py +++ b/temoa/data_io/component_manifest.py @@ -445,7 +445,6 @@ def build_manifest(model: TemoaModel) -> list[LoadItem]: component=model.lifetime_survival_curve, table='lifetime_survival_curve', columns=['region', 'period', 'tech', 'vintage', 'fraction'], - validator_name='viable_rtv', validation_map=(0, 2, 3), is_period_filtered=False, is_table_required=False, From 19b6d47b464fb38776407111fddea00efc27e6ef Mon Sep 17 00:00:00 2001 From: Davey Elder Date: Fri, 26 Jun 2026 10:04:31 -0400 Subject: [PATCH 06/13] Pull lifetimes data for previously retired existing capacities for accounting purposes Signed-off-by: Davey Elder --- temoa/components/technology.py | 27 +++++++---- temoa/components/time.py | 5 --- temoa/core/model.py | 11 +++-- temoa/data_io/component_manifest.py | 8 ++-- temoa/data_io/hybrid_loader.py | 70 ++++++++++++++++++++++++++++- 5 files changed, 95 insertions(+), 26 deletions(-) diff --git a/temoa/components/technology.py b/temoa/components/technology.py index 404e2cb2..4e594b52 100644 --- a/temoa/components/technology.py +++ b/temoa/components/technology.py @@ -17,8 +17,6 @@ from pyomo.environ import value -from temoa.components.utils import get_adjusted_existing_capacity - if TYPE_CHECKING: from collections.abc import Iterable @@ -54,7 +52,23 @@ def model_process_life_indices( the periods in which a process is active, distinct from TechLifeFracIndices that returns indices only for processes that EOL mid-period. """ - return model.active_activity_rptv + indices = { + (r, model.time_exist.last(), t, v) for r, t, v in model.existing_capacity.sparse_keys() + } + indices = indices | model.active_activity_rptv + + return indices + + +def lifetime_tech_indices(model: TemoaModel) -> set[tuple[Region, Technology]]: + """ + Based on the efficiency parameter's indices, this function returns the set of + process indices that may be specified in the lifetime_tech parameter. + """ + indices = {(r, t) for r, _, t, _, _ in set(model.efficiency.sparse_keys())} + indices = indices | {(r, t) for r, t, _ in model.existing_capacity.sparse_keys()} + + return indices def lifetime_process_indices(model: TemoaModel) -> set[tuple[Region, Technology, Vintage]]: @@ -62,7 +76,7 @@ def lifetime_process_indices(model: TemoaModel) -> set[tuple[Region, Technology, Based on the efficiency parameter's indices, this function returns the set of process indices that may be specified in the lifetime_process parameter. """ - indices = {(r, t, v) for r, i, t, v, o in set(model.efficiency.sparse_keys())} + indices = {(r, t, v) for r, _, t, v, _ in set(model.efficiency.sparse_keys())} indices = indices | set(model.existing_capacity.sparse_keys()) return indices @@ -428,14 +442,11 @@ def check_existing_capacity(model: TemoaModel) -> None: continue if t not in model.tech_all: continue - if get_adjusted_existing_capacity(model, r, t, v) <= 0: - # It was just retired earlier - continue life = value(model.lifetime_process[r, t, v]) if (r, t, v) not in model.process_periods and v + life > model.time_optimize.first(): msg = ( f'Existing capacity {r, t, v} with lifetime {life} and capacity {cap} ' - 'should extend into future periods but it is not in process periods. ' + 'should extend into future periods but is not an active process. ' 'May be missing from Efficiency table or too small to carry forward ' 'if running in myopic mode.' ) diff --git a/temoa/components/time.py b/temoa/components/time.py index c34a6bed..cdbb05db 100644 --- a/temoa/components/time.py +++ b/temoa/components/time.py @@ -184,11 +184,6 @@ def init_set_time_optimize(model: TemoaModel) -> list[int]: return sorted(model.time_future)[:-1] -def init_set_vintage_exist(model: TemoaModel) -> list[int]: - """Initializes the `vintage_exist` set.""" - return sorted(model.time_exist) - - def init_set_vintage_optimize(model: TemoaModel) -> list[int]: """Initializes the `vintage_optimize` set.""" return sorted(model.time_optimize) diff --git a/temoa/core/model.py b/temoa/core/model.py index ac7146b3..fc223db3 100755 --- a/temoa/core/model.py +++ b/temoa/core/model.py @@ -212,9 +212,9 @@ def __init__(self, *args: object, **kwargs: object) -> None: ordered=True, initialize=time.init_set_time_optimize, within=self.time_future ) # Define time period vintages to track capacity installation - self.vintage_exist = Set(ordered=True, initialize=time.init_set_vintage_exist) + self.vintage_exist = Set(ordered=True) self.vintage_optimize = Set(ordered=True, initialize=time.init_set_vintage_optimize) - self.vintage_all = Set(initialize=self.time_exist | self.time_optimize) + self.vintage_all = Set(initialize=self.vintage_exist | self.time_optimize) # Perform some basic validation on the specified time periods. self.validate_time = BuildAction(rule=time.validate_time) @@ -437,9 +437,8 @@ def __init__(self, *args: object, **kwargs: object) -> None: default=1, ) - self.lifetime_tech = Param( - self.regional_indices, self.tech_all, default=TemoaModel.default_lifetime_tech - ) + self.lifetime_tech_rt = Set(dimen=2, initialize=technology.lifetime_tech_indices) + self.lifetime_tech = Param(self.lifetime_tech_rt, default=TemoaModel.default_lifetime_tech) self.lifetime_process_rtv = Set(dimen=3, initialize=technology.lifetime_process_indices) self.lifetime_process = Param( @@ -449,7 +448,7 @@ def __init__(self, *args: object, **kwargs: object) -> None: self.lifetime_survival_curve = Param( self.regional_indices, Integers, - self.tech_all, + self.tech_all | self.tech_exist, self.vintage_all, default=technology.get_default_survival, validate=validate_0to1, diff --git a/temoa/data_io/component_manifest.py b/temoa/data_io/component_manifest.py index 72688e3b..a5848c2e 100644 --- a/temoa/data_io/component_manifest.py +++ b/temoa/data_io/component_manifest.py @@ -427,8 +427,7 @@ def build_manifest(model: TemoaModel) -> list[LoadItem]: component=model.lifetime_tech, table='lifetime_tech', columns=['region', 'tech', 'lifetime'], - validator_name='viable_rt', - validation_map=(0, 1), + custom_loader_name='_load_lifetime_tech', is_period_filtered=False, is_table_required=False, ), @@ -436,8 +435,7 @@ def build_manifest(model: TemoaModel) -> list[LoadItem]: component=model.lifetime_process, table='lifetime_process', columns=['region', 'tech', 'vintage', 'lifetime'], - validator_name='viable_rtv', - validation_map=(0, 1, 2), + custom_loader_name='_load_lifetime_process', is_period_filtered=False, is_table_required=False, ), @@ -445,7 +443,7 @@ def build_manifest(model: TemoaModel) -> list[LoadItem]: component=model.lifetime_survival_curve, table='lifetime_survival_curve', columns=['region', 'period', 'tech', 'vintage', 'fraction'], - validation_map=(0, 2, 3), + custom_loader_name='_load_lifetime_survival_curve', is_period_filtered=False, is_table_required=False, ), diff --git a/temoa/data_io/hybrid_loader.py b/temoa/data_io/hybrid_loader.py index d47a3635..71024b5b 100644 --- a/temoa/data_io/hybrid_loader.py +++ b/temoa/data_io/hybrid_loader.py @@ -24,6 +24,7 @@ import time from collections import defaultdict from logging import getLogger +from operator import itemgetter from sqlite3 import Connection, Cursor, OperationalError from typing import TYPE_CHECKING, cast @@ -98,6 +99,7 @@ def __init__(self, db_connection: Connection, config: TemoaConfig) -> None: self.manager: CommodityNetworkManager | None = None self.efficiency_values: list[tuple[object, ...]] = [] self.data: dict[str, object] | None = None + self.tech_exist_data: set[str] = set() # --- Viable sets for source-trace filtering --- self.viable_techs: ViableSet | None = None @@ -629,6 +631,9 @@ def _load_existing_capacity( if rows_to_load: tech_exist_data = sorted({(row[1],) for row in rows_to_load}) self._load_component_data(data, model.tech_exist, tech_exist_data) + self.tech_exist_data = {row[1] for row in rows_to_load} + vintage_exist_data = sorted({(row[2],) for row in rows_to_load}) + self._load_component_data(data, model.vintage_exist, vintage_exist_data) def _load_retired_existing_capacity( self, @@ -637,8 +642,7 @@ def _load_retired_existing_capacity( filtered_data: Sequence[tuple[object, ...]], ) -> None: """ - Handles different queries for myopic vs. standard runs and also - populates the `tech_exist` set. + Only needed in myopic to bring past early retirement decisions forward """ model = TemoaModel() cur = self.con.cursor() @@ -661,6 +665,68 @@ def _load_retired_existing_capacity( self._load_component_data(data, model.retired_existing_capacity, rows_to_load) + # --- Lifetime components --- + def _load_lifetime_tech( + self, + data: dict[str, object], + raw_data: Sequence[tuple[object, ...]], + filtered_data: Sequence[tuple[object, ...]], + ) -> None: + """Loads the lifetime_tech component.""" + model = TemoaModel() + cur = self.con.cursor() + rows_to_load = cur.execute('SELECT region, tech, lifetime FROM lifetime_tech').fetchall() + tech_getter = itemgetter(1) + if self.viable_techs: + valid_techs = self.viable_techs.members | self.tech_exist_data + rows_to_load = [item for item in rows_to_load if tech_getter(item) in valid_techs] + self._load_component_data(data, model.lifetime_tech, rows_to_load) + + def _load_lifetime_process( + self, + data: dict[str, object], + raw_data: Sequence[tuple[object, ...]], + filtered_data: Sequence[tuple[object, ...]], + ) -> None: + """Loads the lifetime_process component.""" + model = TemoaModel() + cur = self.con.cursor() + mi = self.myopic_index + + if mi: + rows_to_load = cur.execute( + 'SELECT region, tech, vintage, lifetime FROM lifetime_process WHERE vintage <= ?', + (mi.last_demand_year,), + ).fetchall() + else: + rows_to_load = cur.execute( + 'SELECT region, tech, vintage, lifetime FROM lifetime_process' + ).fetchall() + self._load_component_data(data, model.lifetime_process, rows_to_load) + + def _load_lifetime_survival_curve( + self, + data: dict[str, object], + raw_data: Sequence[tuple[object, ...]], + filtered_data: Sequence[tuple[object, ...]], + ) -> None: + """Loads the lifetime_survival_curve component.""" + model = TemoaModel() + cur = self.con.cursor() + mi = self.myopic_index + + if mi: + rows_to_load = cur.execute( + 'SELECT region, period, tech, vintage, fraction FROM lifetime_survival_curve ' + 'WHERE vintage <= ?', + (mi.last_demand_year,), + ).fetchall() + else: + rows_to_load = cur.execute( + 'SELECT region, period, tech, vintage, fraction FROM lifetime_survival_curve' + ).fetchall() + self._load_component_data(data, model.lifetime_survival_curve, rows_to_load) + # --- Singleton and Configuration-based Components --- def _load_global_discount_rate( self, From 0c1884774fca76cbb6384ae5427f28ca10b14767 Mon Sep 17 00:00:00 2001 From: Davey Elder Date: Fri, 26 Jun 2026 10:05:09 -0400 Subject: [PATCH 07/13] Actually load growthrate constraints Signed-off-by: Davey Elder --- temoa/core/model.py | 13 ++++--- temoa/data_io/component_manifest.py | 60 +++++++++++++++++++++++++++++ temoa/data_io/hybrid_loader.py | 4 ++ temoa/data_io/loader_manifest.py | 1 + 4 files changed, 72 insertions(+), 6 deletions(-) diff --git a/temoa/core/model.py b/temoa/core/model.py index fc223db3..85f48ea3 100755 --- a/temoa/core/model.py +++ b/temoa/core/model.py @@ -14,6 +14,7 @@ from pyomo.core import BuildCheck, Set, Var from pyomo.environ import ( AbstractModel, + Any, BuildAction, Constraint, Integers, @@ -626,22 +627,22 @@ def __init__(self, *args: object, **kwargs: object) -> None: ) self.limit_growth_capacity = Param( - self.regional_global_indices, self.tech_or_group, self.operator + self.regional_global_indices, self.tech_or_group, self.operator, within=Any ) self.limit_degrowth_capacity = Param( - self.regional_global_indices, self.tech_or_group, self.operator + self.regional_global_indices, self.tech_or_group, self.operator, within=Any ) self.limit_growth_new_capacity = Param( - self.regional_global_indices, self.tech_or_group, self.operator + self.regional_global_indices, self.tech_or_group, self.operator, within=Any ) self.limit_degrowth_new_capacity = Param( - self.regional_global_indices, self.tech_or_group, self.operator + self.regional_global_indices, self.tech_or_group, self.operator, within=Any ) self.limit_growth_new_capacity_delta = Param( - self.regional_global_indices, self.tech_or_group, self.operator + self.regional_global_indices, self.tech_or_group, self.operator, within=Any ) self.limit_degrowth_new_capacity_delta = Param( - self.regional_global_indices, self.tech_or_group, self.operator + self.regional_global_indices, self.tech_or_group, self.operator, within=Any ) self.limit_emission_constraint_rpe = Set( diff --git a/temoa/data_io/component_manifest.py b/temoa/data_io/component_manifest.py index a5848c2e..7a97e2ba 100644 --- a/temoa/data_io/component_manifest.py +++ b/temoa/data_io/component_manifest.py @@ -654,6 +654,66 @@ def build_manifest(model: TemoaModel) -> list[LoadItem]: validation_map=(0, 1, 2), is_table_required=False, ), + LoadItem( + component=model.limit_growth_capacity, + table='limit_growth_capacity', + columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'], + index_length=3, + validator_name='viable_rt', + validation_map=(0, 1), + is_period_filtered=False, + is_table_required=False, + ), + LoadItem( + component=model.limit_growth_new_capacity, + table='limit_growth_new_capacity', + columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'], + index_length=3, + validator_name='viable_rt', + validation_map=(0, 1), + is_period_filtered=False, + is_table_required=False, + ), + LoadItem( + component=model.limit_growth_new_capacity_delta, + table='limit_growth_new_capacity_delta', + columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'], + index_length=3, + validator_name='viable_rt', + validation_map=(0, 1), + is_period_filtered=False, + is_table_required=False, + ), + LoadItem( + component=model.limit_degrowth_capacity, + table='limit_degrowth_capacity', + columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'], + index_length=3, + validator_name='viable_rt', + validation_map=(0, 1), + is_period_filtered=False, + is_table_required=False, + ), + LoadItem( + component=model.limit_degrowth_new_capacity, + table='limit_degrowth_new_capacity', + columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'], + index_length=3, + validator_name='viable_rt', + validation_map=(0, 1), + is_period_filtered=False, + is_table_required=False, + ), + LoadItem( + component=model.limit_degrowth_new_capacity_delta, + table='limit_degrowth_new_capacity_delta', + columns=['region', 'tech_or_group', 'operator', 'rate', 'seed'], + index_length=3, + validator_name='viable_rt', + validation_map=(0, 1), + is_period_filtered=False, + is_table_required=False, + ), LoadItem( component=model.limit_resource, table='limit_resource', diff --git a/temoa/data_io/hybrid_loader.py b/temoa/data_io/hybrid_loader.py index 71024b5b..9452f2e3 100644 --- a/temoa/data_io/hybrid_loader.py +++ b/temoa/data_io/hybrid_loader.py @@ -233,6 +233,10 @@ def create_data_dict(self, myopic_index: MyopicIndex | None = None) -> dict[str, for item in self.manifest: # 1. Fetch data from the database raw_data = self._fetch_data(cur, item, myopic_index) + if item.index_length: + raw_data = [ + (*row[0 : item.index_length], row[item.index_length :]) for row in raw_data + ] # 2. Validate/filter data filtered_data = self._filter_data(raw_data, item, use_raw_data) diff --git a/temoa/data_io/loader_manifest.py b/temoa/data_io/loader_manifest.py index ad6ce930..5723d9e9 100644 --- a/temoa/data_io/loader_manifest.py +++ b/temoa/data_io/loader_manifest.py @@ -50,6 +50,7 @@ class LoadItem: component: ComponentType table: str columns: list[str] + index_length: int | None = None validator_name: str | None = None validation_map: tuple[int, ...] = field(default_factory=tuple) where_clause: str | None = None From df2f4e3bc9d0ea2ae0dbc2f18814f627e3d4a4c5 Mon Sep 17 00:00:00 2001 From: Davey Elder Date: Fri, 26 Jun 2026 10:31:48 -0400 Subject: [PATCH 08/13] Add a broad stress test for myopic that includes survival curves, early retirement, and growth rate constraints all together Signed-off-by: Davey Elder --- tests/conftest.py | 6 + tests/legacy_test_values.py | 3 +- tests/test_full_runs.py | 45 ++++ .../config_myopic_capacities.toml | 23 ++ tests/testing_data/mediumville.sql | 1 - tests/testing_data/myopic_capacities.sql | 242 ++++++++++++++++++ 6 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 tests/testing_configs/config_myopic_capacities.toml create mode 100644 tests/testing_data/myopic_capacities.sql diff --git a/tests/conftest.py b/tests/conftest.py index 3f813d5b..0e263b9c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -88,6 +88,7 @@ def refresh_databases() -> None: ('mediumville.sql', 'mediumville.sqlite'), ('seasonal_storage.sql', 'seasonal_storage.sqlite'), ('survival_curve.sql', 'survival_curve.sqlite'), + ('myopic_capacities.sql', 'myopic_capacities.sqlite'), ('annualised_demand.sql', 'annualised_demand.sqlite'), # Feature tests (separate for temporal consistency) ('emissions.sql', 'emissions.sqlite'), @@ -207,6 +208,11 @@ def system_test_run( silent=True, ) + # Allow parametrized tests to override myopic settings per case. + myopic_overrides = request.param.get('myopic') + if myopic_overrides is not None: + config.myopic_inputs = {**(config.myopic_inputs or {}), **myopic_overrides} + sequencer = TemoaSequencer(config=config) sequencer.start() diff --git a/tests/legacy_test_values.py b/tests/legacy_test_values.py index af31669c..3fb7d2e1 100644 --- a/tests/legacy_test_values.py +++ b/tests/legacy_test_values.py @@ -68,7 +68,8 @@ class ExpectedVals(Enum): # added 2025/06/12 prior to addition of dynamic reserve margin # reduced 2025/06/16 after fixing bug in db ExpectedVals.OBJ_VALUE: 7035.7275, - ExpectedVals.EFF_DOMAIN_SIZE: 2800, + # halved after realising mediumville had an unused existing time period + ExpectedVals.EFF_DOMAIN_SIZE: 1400, ExpectedVals.EFF_INDEX_SIZE: 18, # increased after reviving RampSeason constraints # reduced 2025/07/25 by 24 after annualising demands diff --git a/tests/test_full_runs.py b/tests/test_full_runs.py index 45ca7d1b..5decd266 100644 --- a/tests/test_full_runs.py +++ b/tests/test_full_runs.py @@ -36,6 +36,24 @@ mc_files = [{'name': 'utopia mc', 'filename': 'config_utopia_mc.toml'}] stochastic_files = [{'name': 'stochastic utopia', 'filename': 'config_utopia_stochastic.toml'}] +myopic_stress_tests = [ + { + 'name': ( + f'myopic capacities | {"evolving" if evolving else "non-evolving"}' + f' | view={view_depth} step={step_size}' + ), + 'filename': 'config_myopic_capacities.toml', + 'myopic': { + 'view_depth': view_depth, + 'step_size': step_size, + 'evolving': evolving, + }, + } + for evolving in (False, True) + for view_depth in [1, 3, 6] + for step_size in range(1, view_depth + 1, 2) +] + @pytest.mark.parametrize( 'system_test_run', @@ -129,6 +147,33 @@ def test_myopic_utopia( ) +@pytest.mark.parametrize( + 'system_test_run', + argvalues=myopic_stress_tests, + indirect=True, + ids=[d['name'] for d in myopic_stress_tests], +) +def test_myopic_stress_tests( + system_test_run: tuple[str, SolverResults | None, TemoaModel | None, TemoaSequencer], +) -> None: + """ + The idea of these is that they should be tightly constrained so that if anything + is wrong the model will fail to find a feasible solution. Use lots of equality constraints + """ + _, _, _, sequencer = system_test_run + import contextlib + + with contextlib.closing(sqlite3.connect(sequencer.config.output_database)) as con: + cur = con.cursor() + res = cur.execute('SELECT SUM(total_system_cost) FROM main.output_objective').fetchone() + obj = res[0] + # This part is just a very rough check on the objective function. Constraints inside the + # model are extremely tight so any other changes will lead to infeasibility + assert obj == pytest.approx(32, abs=1), ( + 'objective function value did not match expected for myopic stress test' + ) + + @pytest.mark.parametrize( 'system_test_run', argvalues=stochastic_files, diff --git a/tests/testing_configs/config_myopic_capacities.toml b/tests/testing_configs/config_myopic_capacities.toml new file mode 100644 index 00000000..3ab763c4 --- /dev/null +++ b/tests/testing_configs/config_myopic_capacities.toml @@ -0,0 +1,23 @@ +scenario = "test myopic capacities" +scenario_mode = "myopic" +input_database = "tests/testing_outputs/myopic_capacities.sqlite" +output_database = "tests/testing_outputs/myopic_capacities.sqlite" +neos = false +solver_name = "appsi_highs" +save_excel = true +save_duals = true +save_lp_file = false +time_sequencing = "seasonal_timeslices" +days_per_period = 365 +reserve_margin = "static" + +[MGA] +slack = 0.1 +iterations = 4 +weight = "integer" + +[myopic] +view_depth = 1 +step_size = 1 +evolving = false +evolution_script = '' diff --git a/tests/testing_data/mediumville.sql b/tests/testing_data/mediumville.sql index 9df6b3f7..2f253a8c 100644 --- a/tests/testing_data/mediumville.sql +++ b/tests/testing_data/mediumville.sql @@ -181,7 +181,6 @@ REPLACE INTO "technology_type" VALUES('pb','baseload production technology'); REPLACE INTO "technology_type" VALUES('ps','storage production technology'); REPLACE INTO "time_of_day" VALUES(1,'d1',12,NULL); REPLACE INTO "time_of_day" VALUES(2,'d2',12,NULL); -REPLACE INTO "time_period" VALUES(1,2020,'e'); REPLACE INTO "time_period" VALUES(2,2025,'f'); REPLACE INTO "time_period" VALUES(3,2030,'f'); REPLACE INTO "time_period_type" VALUES('e','existing vintages'); diff --git a/tests/testing_data/myopic_capacities.sql b/tests/testing_data/myopic_capacities.sql new file mode 100644 index 00000000..f561e073 --- /dev/null +++ b/tests/testing_data/myopic_capacities.sql @@ -0,0 +1,242 @@ +REPLACE INTO metadata VALUES('DB_MAJOR',4,''); +REPLACE INTO metadata VALUES('DB_MINOR',0,''); +REPLACE INTO metadata_real VALUES('global_discount_rate',0.05000000000000000277,'Discount Rate for future costs'); +REPLACE INTO metadata_real VALUES('default_loan_rate',0.05000000000000000277,'Default Loan Rate if not specified in loan_rate table'); +REPLACE INTO sector_label VALUES('energy',NULL); +REPLACE INTO commodity VALUES('source','s',NULL,NULL); +REPLACE INTO commodity VALUES('demand','d',NULL,NULL); +REPLACE INTO commodity_type VALUES('p','physical commodity'); +REPLACE INTO commodity_type VALUES('a','annual commodity'); +REPLACE INTO commodity_type VALUES('e','emissions commodity'); +REPLACE INTO commodity_type VALUES('d','demand commodity'); +REPLACE INTO commodity_type VALUES('s','source commodity'); +REPLACE INTO commodity_type VALUES('w','waste commodity'); +REPLACE INTO commodity_type VALUES('wa','waste annual commodity'); +REPLACE INTO commodity_type VALUES('wp','waste physical commodity'); +REPLACE INTO cost_fixed VALUES('region',2025,'tech_ancient',1994,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2025,'tech_old',2010,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2030,'tech_old',2010,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2035,'tech_old',2010,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2040,'tech_old',2010,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2025,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2030,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2035,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2040,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2045,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2050,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2030,'tech_future',2030,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2035,'tech_future',2035,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2040,'tech_future',2040,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2045,'tech_future',2045,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2050,'tech_future',2050,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2035,'tech_future',2030,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2040,'tech_future',2035,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2045,'tech_future',2040,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2050,'tech_future',2045,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2040,'tech_future',2030,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2045,'tech_future',2035,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2050,'tech_future',2040,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2045,'tech_future',2030,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2050,'tech_future',2035,1.0,NULL,NULL); +REPLACE INTO cost_fixed VALUES('region',2050,'tech_future',2030,1.0,NULL,NULL); +REPLACE INTO cost_invest VALUES('region','tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_invest VALUES('region','tech_future',2030,1.0,NULL,NULL); +REPLACE INTO cost_invest VALUES('region','tech_future',2035,1.0,NULL,NULL); +REPLACE INTO cost_invest VALUES('region','tech_future',2040,1.0,NULL,NULL); +REPLACE INTO cost_invest VALUES('region','tech_future',2045,1.0,NULL,NULL); +REPLACE INTO cost_invest VALUES('region','tech_future',2050,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2025,'tech_ancient',1994,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2025,'tech_old',2010,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2030,'tech_old',2010,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2035,'tech_old',2010,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2040,'tech_old',2010,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2025,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2030,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2035,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2040,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2045,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2050,'tech_current',2025,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2030,'tech_future',2030,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2035,'tech_future',2035,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2040,'tech_future',2040,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2045,'tech_future',2045,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2050,'tech_future',2050,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2035,'tech_future',2030,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2040,'tech_future',2035,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2045,'tech_future',2040,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2050,'tech_future',2045,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2040,'tech_future',2030,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2045,'tech_future',2035,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2050,'tech_future',2040,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2045,'tech_future',2030,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2050,'tech_future',2035,1.0,NULL,NULL); +REPLACE INTO cost_variable VALUES('region',2050,'tech_future',2030,1.0,NULL,NULL); +REPLACE INTO demand VALUES('region',2025,'demand',1.0,NULL,NULL); +REPLACE INTO demand VALUES('region',2030,'demand',1.0,NULL,NULL); +REPLACE INTO demand VALUES('region',2035,'demand',1.0,NULL,NULL); +REPLACE INTO demand VALUES('region',2040,'demand',1.0,NULL,NULL); +REPLACE INTO demand VALUES('region',2045,'demand',1.0,NULL,NULL); +REPLACE INTO demand VALUES('region',2050,'demand',1.0,NULL,NULL); +REPLACE INTO efficiency VALUES('region','source','tech_old',2010,'demand',1.0,NULL,NULL); +REPLACE INTO efficiency VALUES('region','source','tech_current',2025,'demand',1.0,NULL,NULL); +REPLACE INTO efficiency VALUES('region','source','tech_future',2030,'demand',1.0,NULL,NULL); +REPLACE INTO efficiency VALUES('region','source','tech_future',2035,'demand',1.0,NULL,NULL); +REPLACE INTO efficiency VALUES('region','source','tech_future',2040,'demand',1.0,NULL,NULL); +REPLACE INTO efficiency VALUES('region','source','tech_future',2045,'demand',1.0,NULL,NULL); +REPLACE INTO efficiency VALUES('region','source','tech_future',2050,'demand',1.0,NULL,NULL); +REPLACE INTO efficiency VALUES('region','source','tech_ancient',1994,'demand',1.0,NULL,NULL); +REPLACE INTO efficiency VALUES('region','source','tech_retire',2010,'demand',1.0,NULL,NULL); +REPLACE INTO existing_capacity VALUES('region','tech_ancient',1990,3.0,NULL,NULL); +REPLACE INTO existing_capacity VALUES('region','tech_old',2010,0.6999999999999999556,NULL,NULL); +REPLACE INTO existing_capacity VALUES('region','tech_ancient',1994,3.0,NULL,NULL); +REPLACE INTO existing_capacity VALUES('region','tech_retire',2010,1.0,NULL,NULL); +REPLACE INTO lifetime_tech VALUES('region','tech_ancient',35.0,NULL,NULL); +REPLACE INTO lifetime_tech VALUES('region','tech_old',35.0,NULL,NULL); +REPLACE INTO lifetime_tech VALUES('region','tech_current',35.0,NULL,NULL); +REPLACE INTO lifetime_tech VALUES('region','tech_future',35.0,NULL,NULL); +REPLACE INTO lifetime_tech VALUES('region','tech_retire',35.0,NULL,NULL); +REPLACE INTO operator VALUES('e','equal to'); +REPLACE INTO operator VALUES('le','less than or equal to'); +REPLACE INTO operator VALUES('ge','greater than or equal to'); +REPLACE INTO limit_growth_capacity VALUES('region','tech_ancient','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_growth_capacity VALUES('region','tech_old','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_growth_capacity VALUES('region','tech_current','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_growth_capacity VALUES('region','tech_future','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_degrowth_capacity VALUES('region','tech_ancient','le',1.0,2.0,NULL,NULL); +REPLACE INTO limit_degrowth_capacity VALUES('region','tech_old','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_degrowth_capacity VALUES('region','tech_current','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_degrowth_capacity VALUES('region','tech_future','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_growth_new_capacity VALUES('region','tech_ancient','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_growth_new_capacity VALUES('region','tech_old','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_growth_new_capacity VALUES('region','tech_current','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_growth_new_capacity VALUES('region','tech_future','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_degrowth_new_capacity VALUES('region','tech_ancient','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_degrowth_new_capacity VALUES('region','tech_old','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_degrowth_new_capacity VALUES('region','tech_current','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_degrowth_new_capacity VALUES('region','tech_future','le',1.0,1.0,NULL,NULL); +REPLACE INTO limit_growth_new_capacity_delta VALUES('region','tech_ancient','le',0.0,2.0,NULL,NULL); +REPLACE INTO limit_growth_new_capacity_delta VALUES('region','tech_old','le',0.0,2.0,NULL,NULL); +REPLACE INTO limit_growth_new_capacity_delta VALUES('region','tech_current','le',0.0,2.0,NULL,NULL); +REPLACE INTO limit_growth_new_capacity_delta VALUES('region','tech_future','le',0.0,2.0,NULL,NULL); +REPLACE INTO limit_degrowth_new_capacity_delta VALUES('region','tech_ancient','le',0.0,2.0,NULL,NULL); +REPLACE INTO limit_degrowth_new_capacity_delta VALUES('region','tech_old','le',0.0,2.0,NULL,NULL); +REPLACE INTO limit_degrowth_new_capacity_delta VALUES('region','tech_current','le',0.0,2.0,NULL,NULL); +REPLACE INTO limit_degrowth_new_capacity_delta VALUES('region','tech_future','le',0.0,2.0,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2025,'tech_ancient','e',0.04800000000000000099,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2025,'tech_current','e',0.6520000000000000239,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2030,'tech_current','e',0.5823319839999999692,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2035,'tech_current','e',0.4500000000000000111,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2040,'tech_current','e',0.25,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2030,'tech_future','e',0.2964180160000000063,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2035,'tech_future','e',0.5100000000000000088,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2040,'tech_future','e',0.7349999999999999867,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2045,'tech_future','e',1.0,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2050,'tech_future','e',1.0,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2025,'tech_old','e',0.2999910000000000076,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2030,'tech_old','e',0.1212499999999999967,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2035,'tech_old','e',0.04000000000000000083,NULL,NULL); +REPLACE INTO limit_activity VALUES('region',2040,'tech_old','e',0.01499999999999999945,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2025,'tech_ancient','e',0.04800000000000000099,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2025,'tech_current','e',0.6520000000000000239,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2030,'tech_current','e',0.5823319839999999692,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2035,'tech_current','e',0.4500000000000000111,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2040,'tech_current','e',0.25,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2045,'tech_current','e',0.0,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2050,'tech_current','e',0.0,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2030,'tech_future','e',0.2964180160000000063,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2035,'tech_future','e',0.5100000000000000088,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2040,'tech_future','e',0.7349999999999999867,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2045,'tech_future','e',1.0,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2050,'tech_future','e',1.0,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2025,'tech_old','e',0.2999999999999999889,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2025,'tech_retire','e',9.000000000000000228e-06,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2030,'tech_old','e',0.1212499999999999967,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2035,'tech_old','e',0.04000000000000000083,NULL,NULL); +REPLACE INTO limit_capacity VALUES('region',2040,'tech_old','e',0.01499999999999999945,NULL,NULL); +REPLACE INTO limit_new_capacity VALUES('region','tech_current',2025,'e',0.659919028340081093,NULL,NULL); +REPLACE INTO limit_new_capacity VALUES('region','tech_future',2030,'e',0.3000182348178138114,NULL,NULL); +REPLACE INTO limit_new_capacity VALUES('region','tech_future',2035,'e',0.232573854939435165,NULL,NULL); +REPLACE INTO limit_new_capacity VALUES('region','tech_future',2040,'e',0.2884229446031822408,NULL,NULL); +REPLACE INTO limit_new_capacity VALUES('region','tech_future',2045,'e',0.4110596210476472612,NULL,NULL); +REPLACE INTO limit_new_capacity VALUES('region','tech_future',2050,'e',0.2251165192346591404,NULL,NULL); +REPLACE INTO region VALUES('region',NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',1990,'tech_ancient',1990,1.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2025,'tech_ancient',1990,0.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2010,'tech_old',2010,1.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2015,'tech_old',2010,0.969999999999999974,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2020,'tech_old',2010,0.8800000000000001154,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2025,'tech_old',2010,0.6199999999999999956,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2030,'tech_old',2010,0.2700000000000000177,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2035,'tech_old',2010,0.08000000000000000166,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2045,'tech_old',2010,0.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2025,'tech_current',2025,1.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2030,'tech_current',2025,0.969999999999999974,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2035,'tech_current',2025,0.8800000000000001154,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2040,'tech_current',2025,0.6199999999999999956,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2045,'tech_current',2025,0.2700000000000000177,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2050,'tech_current',2025,0.08000000000000000166,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2060,'tech_current',2025,0.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2030,'tech_future',2030,1.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2035,'tech_future',2030,0.969999999999999974,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2040,'tech_future',2030,0.8800000000000001154,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2045,'tech_future',2030,0.6199999999999999956,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2050,'tech_future',2030,0.2700000000000000177,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2055,'tech_future',2030,0.08000000000000000166,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2065,'tech_future',2030,0.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2035,'tech_future',2035,1.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2040,'tech_future',2035,0.969999999999999974,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2045,'tech_future',2035,0.8800000000000001154,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2050,'tech_future',2035,0.6199999999999999956,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2055,'tech_future',2035,0.2700000000000000177,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2060,'tech_future',2035,0.08000000000000000166,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2070,'tech_future',2035,0.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2040,'tech_future',2040,1.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2045,'tech_future',2040,0.969999999999999974,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2050,'tech_future',2040,0.8800000000000001154,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2055,'tech_future',2040,0.6199999999999999956,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2060,'tech_future',2040,0.2700000000000000177,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2065,'tech_future',2040,0.08000000000000000166,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2075,'tech_future',2040,0.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2045,'tech_future',2045,1.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2050,'tech_future',2045,0.969999999999999974,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2055,'tech_future',2045,0.8800000000000001154,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2060,'tech_future',2045,0.6199999999999999956,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2065,'tech_future',2045,0.2700000000000000177,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2070,'tech_future',2045,0.08000000000000000166,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2080,'tech_future',2045,0.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2050,'tech_future',2050,1.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2055,'tech_future',2050,0.969999999999999974,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2060,'tech_future',2050,0.8800000000000001154,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2065,'tech_future',2050,0.6199999999999999956,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2070,'tech_future',2050,0.2700000000000000177,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2075,'tech_future',2050,0.08000000000000000166,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2085,'tech_future',2050,0.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',1994,'tech_ancient',1994,1.0,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',1999,'tech_ancient',1994,0.969999999999999974,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2004,'tech_ancient',1994,0.8800000000000000044,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2009,'tech_ancient',1994,0.6199999999999999956,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2014,'tech_ancient',1994,0.2700000000000000177,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2019,'tech_ancient',1994,0.08000000000000000166,NULL); +REPLACE INTO lifetime_survival_curve VALUES('region',2029,'tech_ancient',1994,0.0,NULL); +REPLACE INTO technology_type VALUES('p','production technology'); +REPLACE INTO technology_type VALUES('pb','baseload production technology'); +REPLACE INTO technology_type VALUES('ps','storage production technology'); +REPLACE INTO time_of_day VALUES(0,'d',24.0,NULL); +REPLACE INTO time_period VALUES(-3,1990,'e'); +REPLACE INTO time_period VALUES(-2,1994,'e'); +REPLACE INTO time_period VALUES(-1,2010,'e'); +REPLACE INTO time_period VALUES(0,2025,'f'); +REPLACE INTO time_period VALUES(1,2030,'f'); +REPLACE INTO time_period VALUES(2,2035,'f'); +REPLACE INTO time_period VALUES(3,2040,'f'); +REPLACE INTO time_period VALUES(4,2045,'f'); +REPLACE INTO time_period VALUES(5,2050,'f'); +REPLACE INTO time_period VALUES(6,2055,'f'); +REPLACE INTO time_period_type VALUES('e','existing vintages'); +REPLACE INTO time_period_type VALUES('f','future'); +REPLACE INTO technology VALUES('tech_ancient','p','energy',NULL,NULL,0,0,0,0,1,0,0,0,NULL); +REPLACE INTO technology VALUES('tech_old','p','energy',NULL,NULL,0,0,0,0,1,0,0,0,NULL); +REPLACE INTO technology VALUES('tech_current','p','energy',NULL,NULL,0,0,0,0,1,0,0,0,NULL); +REPLACE INTO technology VALUES('tech_future','p','energy',NULL,NULL,0,0,0,0,1,0,0,0,NULL); +REPLACE INTO technology VALUES('tech_retire','p','energy',NULL,NULL,0,0,0,0,1,0,0,0,NULL); +REPLACE INTO time_season VALUES(0,'s',1.0,NULL); From c76808f638335a7ab880e0970a5a3465d8c91f77 Mon Sep 17 00:00:00 2001 From: Davey Elder Date: Fri, 26 Jun 2026 10:45:20 -0400 Subject: [PATCH 09/13] Update test set hashes Signed-off-by: Davey Elder --- tests/testing_data/mediumville_sets.json | 7 ++++--- tests/testing_data/test_system_sets.json | 3 ++- tests/testing_data/utopia_sets.json | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/testing_data/mediumville_sets.json b/tests/testing_data/mediumville_sets.json index 82bf63af..c86baf5c 100644 --- a/tests/testing_data/mediumville_sets.json +++ b/tests/testing_data/mediumville_sets.json @@ -31,6 +31,7 @@ "flow_in_storage_rpsditvo": "ff680754a218ee843941e77bd07cbc5e5afd74e2b7b29a76ea86621624e8c0fd", "flow_var_annual_rpitvo": "6f5c569787fbf1e0a4076011f75fcd50a840bac85bb57b026f4fc5682739f888", "flow_var_rpsditvo": "e8c35c3b7b1f0c0e10e6f53e94d0c6b8a884c0dd048675cbeb9e7df33794161b", + "lifetime_tech_rt": "6786f2e9acdccce907ebad50d2ea70481bff72a749253e78168d29b480b07f65", "lifetime_process_rtv": "212639322f29aa7687eb9962fb9d74ef60961d720a2a753e2af53473b9e1648f", "limit_activity_constraint_rpt": "90461349933d0b32167abdca242233004b66212b57ed57213dce6f6818203963", "limit_activity_share_constraint_rpgg": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", @@ -101,7 +102,7 @@ "tech_uncap": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "tech_upramping": "ddcf6ff7665c2a8acc4dff1b43655fae1b5a265135cbee18cec638df4e954346", "tech_with_capacity": "24f75b6f705a36033456d02638e7c50667908c45c474151fb8490666d928c63f", - "time_exist": "91a2461c25439830d94bf4b3d3a3b020343f75e74561a913b1f972b2ac42e943", + "time_exist": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "time_future": "6b150df8e12ac4dd15396b52f304c7935a41d2cc4da498186552819973171389", "time_manual": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "time_of_day": "f9bbfe9130c510cba59a13e8b385b4d0206196abbdfe8032b995502bb7215f76", @@ -109,7 +110,7 @@ "time_season": "1e413891065e24bbf66543d4747ff12b1c65bf23c4ff9857b0bf0520eccd7f21", "time_season_sequential": "1e413891065e24bbf66543d4747ff12b1c65bf23c4ff9857b0bf0520eccd7f21", "time_sequencing": "91f69c8abab9959c1f8c90f5aaa56db29bccc67e37d12673ab41c54e4179d7ca", - "vintage_all": "947dd08ad4812a98649b4306e4a0ca1b51374d86f1a8e27c3a8af3b189bcc0f6", - "vintage_exist": "91a2461c25439830d94bf4b3d3a3b020343f75e74561a913b1f972b2ac42e943", + "vintage_all": "9c9380fb50cd4f4f9e2032bd9a18645ae4fc1d30335672c62c897bcb9e099ba5", + "vintage_exist": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "vintage_optimize": "9c9380fb50cd4f4f9e2032bd9a18645ae4fc1d30335672c62c897bcb9e099ba5" } diff --git a/tests/testing_data/test_system_sets.json b/tests/testing_data/test_system_sets.json index 35cd0b4c..6d73af97 100644 --- a/tests/testing_data/test_system_sets.json +++ b/tests/testing_data/test_system_sets.json @@ -31,6 +31,7 @@ "flow_in_storage_rpsditvo": "8c97fbeacbca1a772ba04f63d0a82f7703500f83c0dc602114b17c64ad4d2e28", "flow_var_annual_rpitvo": "f98f5f41c5cba8ce2a894734abbc5e90310b695bee3c24c52acd8b4800c5ab85", "flow_var_rpsditvo": "784d98e451a8dcf0122f475c2e229162d99e403bdea85f7c608a3d86e850db44", + "lifetime_tech_rt": "984cc175a7aa58e77bb626e3237ca9bf625f5d9b15250870bef1cddc5f57f642", "lifetime_process_rtv": "5033502364848a3a3f295f1b3d051531ecd5c1e5f8bbaecd61afcd18181225ab", "limit_activity_constraint_rpt": "2561103e7e6dd3dee3ba5832290ab5e933f4af08661dff4f2e661b6f4ffefc86", "limit_activity_share_constraint_rpgg": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", @@ -63,7 +64,7 @@ "new_capacity_var_rtv": "bc0ec9e7f812410cb2af924cbc99a31c6e99cd95fc2de26193daad38f33cc132", "operator": "74d830836f1399fb336a0432dde7d7bd36cffa3ff76b1c42d7945350cfb9bf91", "ordered_season_sequential": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", - "process_life_frac_rptv": "5501652d0145dbf7c2e8c006aabd3dbb85ca64ed5caf1d8f45a1957274af81ca", + "process_life_frac_rptv": "d1b75ede9f90899b1c1cbc56489bf9d12c52abc6c0466eb5fe4dccaf6f3be800", "ramp_down_day_constraint_rpsdtv": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "ramp_down_season_constraint_rpsstv": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "ramp_up_day_constraint_rpsdtv": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", diff --git a/tests/testing_data/utopia_sets.json b/tests/testing_data/utopia_sets.json index 628c177d..53d8b2a8 100644 --- a/tests/testing_data/utopia_sets.json +++ b/tests/testing_data/utopia_sets.json @@ -31,6 +31,7 @@ "flow_in_storage_rpsditvo": "99eb864a26adcb6633c95462649bca3ef7096f67682702915d7769b54b5de386", "flow_var_annual_rpitvo": "4053866a304af0a6b9a1d293ddaf358dfee5b170dcb01d41e6bc52437908a98a", "flow_var_rpsditvo": "350739a59b14969da094cf1c95524134b9c9acd0989710456a8dcc6596d0a401", + "lifetime_tech_rt": "7b3703e1f021594993a54c55427a599ed03fc512cf34826b7a7670eb93edfb9a", "lifetime_process_rtv": "2cfc288b15f25957dfc70f6396d97ad655ecbed91c5a11582329749f1fb3dbd7", "limit_activity_constraint_rpt": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "limit_activity_share_constraint_rpgg": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", @@ -63,7 +64,7 @@ "new_capacity_var_rtv": "d4f1cc8b432075001befddab648d54d1f82a29099236948845393a193b6add5b", "operator": "74d830836f1399fb336a0432dde7d7bd36cffa3ff76b1c42d7945350cfb9bf91", "ordered_season_sequential": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", - "process_life_frac_rptv": "37ad263f78986f4d38c63278a0d1a5a85b3eb8b833e8bfabbbea254eaf3a3efb", + "process_life_frac_rptv": "63d17957ee2bebf664ffa6a39cea5e16ec65ad07db1df24dcea6b15bb9ebf589", "ramp_down_day_constraint_rpsdtv": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "ramp_down_season_constraint_rpsstv": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "ramp_up_day_constraint_rpsdtv": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", From b65fd62eba9a7b1f1e178c09af2a8484fd6181b7 Mon Sep 17 00:00:00 2001 From: Davey Elder Date: Fri, 26 Jun 2026 14:01:08 -0400 Subject: [PATCH 10/13] Improve custom loader filtering for lifetime data Signed-off-by: Davey Elder --- temoa/data_io/hybrid_loader.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/temoa/data_io/hybrid_loader.py b/temoa/data_io/hybrid_loader.py index 9452f2e3..7ccf56cf 100644 --- a/temoa/data_io/hybrid_loader.py +++ b/temoa/data_io/hybrid_loader.py @@ -44,6 +44,7 @@ from temoa.core.config import TemoaConfig from temoa.data_io.loader_manifest import LoadItem + from temoa.types.core_types import Region, Technology, Vintage logger = getLogger(__name__) @@ -99,7 +100,8 @@ def __init__(self, db_connection: Connection, config: TemoaConfig) -> None: self.manager: CommodityNetworkManager | None = None self.efficiency_values: list[tuple[object, ...]] = [] self.data: dict[str, object] | None = None - self.tech_exist_data: set[str] = set() + self.viable_existing_rt: set[tuple[Region, Technology]] = set() + self.viable_existing_rtv: set[tuple[Region, Technology, Vintage]] = set() # --- Viable sets for source-trace filtering --- self.viable_techs: ViableSet | None = None @@ -635,10 +637,14 @@ def _load_existing_capacity( if rows_to_load: tech_exist_data = sorted({(row[1],) for row in rows_to_load}) self._load_component_data(data, model.tech_exist, tech_exist_data) - self.tech_exist_data = {row[1] for row in rows_to_load} vintage_exist_data = sorted({(row[2],) for row in rows_to_load}) self._load_component_data(data, model.vintage_exist, vintage_exist_data) + # Collect existing capacity data indices + self.viable_existing_techs = {row[1] for row in rows_to_load} + self.viable_existing_rt = {(row[0], row[1]) for row in rows_to_load} + self.viable_existing_rtv = {(row[0], row[1], row[2]) for row in rows_to_load} + def _load_retired_existing_capacity( self, data: dict[str, object], @@ -680,10 +686,10 @@ def _load_lifetime_tech( model = TemoaModel() cur = self.con.cursor() rows_to_load = cur.execute('SELECT region, tech, lifetime FROM lifetime_tech').fetchall() - tech_getter = itemgetter(1) - if self.viable_techs: - valid_techs = self.viable_techs.members | self.tech_exist_data - rows_to_load = [item for item in rows_to_load if tech_getter(item) in valid_techs] + rt_getter = itemgetter(0, 1) + if self.viable_rt: + valid_rt = self.viable_rt.members | self.viable_existing_rt + rows_to_load = [item for item in rows_to_load if rt_getter(item) in valid_rt] self._load_component_data(data, model.lifetime_tech, rows_to_load) def _load_lifetime_process( @@ -706,6 +712,10 @@ def _load_lifetime_process( rows_to_load = cur.execute( 'SELECT region, tech, vintage, lifetime FROM lifetime_process' ).fetchall() + rtv_getter = itemgetter(0, 1, 2) + if self.viable_rtv: + valid_rtv = self.viable_rtv.member_tuples | self.viable_existing_rtv + rows_to_load = [item for item in rows_to_load if rtv_getter(item) in valid_rtv] self._load_component_data(data, model.lifetime_process, rows_to_load) def _load_lifetime_survival_curve( @@ -729,6 +739,10 @@ def _load_lifetime_survival_curve( rows_to_load = cur.execute( 'SELECT region, period, tech, vintage, fraction FROM lifetime_survival_curve' ).fetchall() + rtv_getter = itemgetter(0, 2, 3) + if self.viable_rtv: + valid_rtv = self.viable_rtv.member_tuples | self.viable_existing_rtv + rows_to_load = [item for item in rows_to_load if rtv_getter(item) in valid_rtv] self._load_component_data(data, model.lifetime_survival_curve, rows_to_load) # --- Singleton and Configuration-based Components --- From 674c54da2f4efcab4d006a9b4005fb29e077a2b9 Mon Sep 17 00:00:00 2001 From: Davey Elder Date: Fri, 26 Jun 2026 14:01:30 -0400 Subject: [PATCH 11/13] Update existing capacity check to specifically look for tiny dropped capacities Signed-off-by: Davey Elder --- temoa/components/technology.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/temoa/components/technology.py b/temoa/components/technology.py index 4e594b52..9957db97 100644 --- a/temoa/components/technology.py +++ b/temoa/components/technology.py @@ -17,6 +17,8 @@ from pyomo.environ import value +from temoa.components.utils import get_adjusted_existing_capacity + if TYPE_CHECKING: from collections.abc import Iterable @@ -440,14 +442,18 @@ def check_existing_capacity(model: TemoaModel) -> None: ) logger.warning(msg) continue - if t not in model.tech_all: + adjusted_cap = get_adjusted_existing_capacity(model, r, t, v) + if adjusted_cap <= 0.0: + # Was retired in a previous period (myopic mode) continue + p = model.time_optimize.first() life = value(model.lifetime_process[r, t, v]) - if (r, t, v) not in model.process_periods and v + life > model.time_optimize.first(): + if (r, t, v) not in model.process_periods and v + life > p: + surviving_cap = adjusted_cap * value(model.lifetime_survival_curve[r, p, t, v]) msg = ( - f'Existing capacity {r, t, v} with lifetime {life} and capacity {cap} ' - 'should extend into future periods but is not an active process. ' - 'May be missing from Efficiency table or too small to carry forward ' - 'if running in myopic mode.' + f'Existing capacity {r, t, v} with lifetime {life} and surviving capacity ' + f'{surviving_cap} should extend into future periods but is not an active ' + 'process. It may be missing from the Efficiency table or have too little ' + 'capacity to output and carry forward if running in myopic mode.' ) logger.warning(msg) From bdd3a66f81d5ad3c519e6c8c5840535940125a59 Mon Sep 17 00:00:00 2001 From: Davey Elder Date: Fri, 26 Jun 2026 14:14:12 -0400 Subject: [PATCH 12/13] Try to fix python 3.12 type error Signed-off-by: Davey Elder --- tests/test_full_runs.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/test_full_runs.py b/tests/test_full_runs.py index 5decd266..c71f9b4c 100644 --- a/tests/test_full_runs.py +++ b/tests/test_full_runs.py @@ -5,7 +5,7 @@ import logging import sqlite3 from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypedDict import pytest from pyomo.core import Constraint, Var @@ -36,7 +36,20 @@ mc_files = [{'name': 'utopia mc', 'filename': 'config_utopia_mc.toml'}] stochastic_files = [{'name': 'stochastic utopia', 'filename': 'config_utopia_stochastic.toml'}] -myopic_stress_tests = [ + +class MyopicSettings(TypedDict): + view_depth: int + step_size: int + evolving: bool + + +class MyopicStressCase(TypedDict): + name: str + filename: str + myopic: MyopicSettings + + +myopic_stress_tests: list[MyopicStressCase] = [ { 'name': ( f'myopic capacities | {"evolving" if evolving else "non-evolving"}' From b48e1d8a534de7d9e19799f6a02c8cf58297ea42 Mon Sep 17 00:00:00 2001 From: Davey Elder Date: Fri, 26 Jun 2026 14:22:21 -0400 Subject: [PATCH 13/13] Check if output retirement table exists Signed-off-by: Davey Elder --- temoa/data_io/hybrid_loader.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/temoa/data_io/hybrid_loader.py b/temoa/data_io/hybrid_loader.py index 7ccf56cf..f657ed68 100644 --- a/temoa/data_io/hybrid_loader.py +++ b/temoa/data_io/hybrid_loader.py @@ -654,6 +654,13 @@ def _load_retired_existing_capacity( """ Only needed in myopic to bring past early retirement decisions forward """ + if not self.table_exists('output_retired_capacity'): + logger.info( + "Table 'output_retired_capacity' not found. Skipping loading " + 'of retired existing capacity.' + ) + return + model = TemoaModel() cur = self.con.cursor() mi = self.myopic_index