diff --git a/schemas/input/commodity_constraints.yaml b/schemas/input/commodity_constraints.yaml new file mode 100644 index 000000000..6a65446b1 --- /dev/null +++ b/schemas/input/commodity_constraints.yaml @@ -0,0 +1,34 @@ +$schema: https://specs.frictionlessdata.io/schemas/table-schema.json +description: | + Specifies the limits on the total amount of consumption/production of a given + commodity in a given region, year and time slice(s). + +fields: + - name: commodity_id + type: string + description: The commodity to which this constraint applies + - name: region_id + type: string + description: The region in which this constraint applies + - name: balance_type + type: string + description: The type of balance to which this is applied + notes: + Valid options are "cons" and "prod"; the "net" option is not valid here, as net + production is already constrained by the commodity balance constraints. + - name: years + type: string + description: The year(s) to which this entry applies + - name: time_slice + type: string + description: The time slice(s) to which this entry applies + notes: | + Can be a single time slice (e.g. `winter.day`), a whole season (e.g. `winter`) or `annual`, + representing the whole year + - name: limits + type: string + description: Lower and upper limits on the value of the commodity + notes: + A string in the format `min..max`, where `min` and `max` are decimal numbers + (e.g. "0.12..78.9"). Either `min` or `max` can be omitted (e.g "0.12.." or + "..78.9"), which will set the corresponding limit to 0 or infinite, respectively. diff --git a/src/commodity.rs b/src/commodity.rs index a7fa92edf..585cf5fdc 100644 --- a/src/commodity.rs +++ b/src/commodity.rs @@ -6,6 +6,7 @@ use crate::units::{Flow, MoneyPerFlow}; use indexmap::IndexMap; use serde::Deserialize; use std::collections::HashMap; +use std::ops::RangeInclusive; use std::sync::Arc; define_id_type! {CommodityID, "commodity ID"} @@ -16,6 +17,9 @@ pub type CommodityMap = IndexMap>; /// A map of [`MoneyPerFlow`]s, keyed by region ID, year and time slice ID for a specific levy pub type CommodityLevyMap = HashMap<(RegionID, u32, TimeSliceID), MoneyPerFlow>; +/// A map of vectors of [`CommodityConstraint`]s, keyed by region ID and year +pub type CommodityConstraintsMap = HashMap<(RegionID, u32), Vec>; + /// A map of demand values, keyed by region ID, year and time slice selection pub type DemandMap = HashMap<(RegionID, u32, TimeSliceSelection), Flow>; @@ -53,6 +57,11 @@ pub struct Commodity { /// `time_slice_level` field. E.g. if the `time_slice_level` is seasonal, then there will be /// keys representing each season (and not e.g. individual time slices). pub demand: DemandMap, + /// Constraints for this commodity for different combinations of region, year and time slice. + /// + /// May be empty if there are no constraints for this commodity, otherwise there must be entries + /// for every combination of parameters. + pub constraints: CommodityConstraintsMap, /// Units for this commodity represented as a string e.g Petajoules, Tonnes /// This is only used for validation purposes. pub units: String, @@ -115,6 +124,19 @@ pub enum PricingStrategy { Unpriced, } +/// A constraint imposed on commodity values +#[derive(PartialEq, Debug, Clone)] +pub struct CommodityConstraint { + /// The balance type for the commodity constraint + pub balance_type: BalanceType, + /// The time slice selection for the commodity constraint + pub ts_selection: TimeSliceSelection, + /// The range of values the commodity is constrained to lie between + pub limits: RangeInclusive, +} + +impl CommodityConstraint {} + #[cfg(test)] mod tests { use super::*; diff --git a/src/fixture.rs b/src/fixture.rs index 1d215148a..420983e37 100644 --- a/src/fixture.rs +++ b/src/fixture.rs @@ -6,7 +6,8 @@ use crate::agent::{ }; use crate::asset::{Asset, AssetCapacity, AssetPool, AssetRef}; use crate::commodity::{ - Commodity, CommodityID, CommodityLevyMap, CommodityType, DemandMap, PricingStrategy, + Commodity, CommodityConstraintsMap, CommodityID, CommodityLevyMap, CommodityType, DemandMap, + PricingStrategy, }; use crate::patch::{FilePatch, ModelPatch}; use crate::process::{ @@ -157,6 +158,7 @@ pub fn svd_commodity() -> Commodity { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), } } @@ -172,6 +174,7 @@ pub fn sed_commodity() -> Commodity { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), } } @@ -187,6 +190,7 @@ pub fn other_commodity() -> Commodity { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), } } diff --git a/src/input/commodity.rs b/src/input/commodity.rs index b9209aab1..e3d503fc9 100644 --- a/src/input/commodity.rs +++ b/src/input/commodity.rs @@ -2,8 +2,8 @@ use super::{input_err_msg, read_csv}; use crate::ISSUES_URL; use crate::commodity::{ - BalanceType, Commodity, CommodityID, CommodityLevyMap, CommodityMap, CommodityType, DemandMap, - PricingStrategy, + BalanceType, Commodity, CommodityConstraintsMap, CommodityID, CommodityLevyMap, CommodityMap, + CommodityType, DemandMap, PricingStrategy, }; use crate::model::{ALLOW_DANGEROUS_OPTION_NAME, dangerous_model_options_enabled}; use crate::region::RegionID; @@ -16,6 +16,8 @@ use std::path::Path; mod levy; use levy::read_commodity_levies; +mod constraints; +use constraints::read_commodity_constraints; mod demand; use demand::read_demand; mod demand_slicing; @@ -55,6 +57,15 @@ pub fn read_commodities( let commodities = read_commodities_file(model_dir)?; let commodity_ids = commodities.keys().cloned().collect(); + // Read constraints table + let mut commodity_constraints = read_commodity_constraints( + model_dir, + &commodities, + region_ids, + time_slice_info, + milestone_years, + )?; + // Read costs table let mut costs = read_commodity_levies( model_dir, @@ -88,6 +99,9 @@ pub fn read_commodities( if let Some(demand) = demand.remove(&id) { commodity.demand = demand; } + if let Some(commodity_constraints) = commodity_constraints.remove(&id) { + commodity.constraints = commodity_constraints; + } (id, commodity.into()) }) @@ -120,6 +134,7 @@ where levies_prod: CommodityLevyMap::default(), levies_cons: CommodityLevyMap::default(), demand: DemandMap::default(), + constraints: CommodityConstraintsMap::default(), units: commodity_raw.units, }; @@ -206,6 +221,7 @@ mod tests { levies_prod: CommodityLevyMap::default(), levies_cons: CommodityLevyMap::default(), demand: DemandMap::default(), + constraints: CommodityConstraintsMap::default(), units: "PJ".into(), } } diff --git a/src/input/commodity/constraints.rs b/src/input/commodity/constraints.rs new file mode 100644 index 000000000..0c0401f5e --- /dev/null +++ b/src/input/commodity/constraints.rs @@ -0,0 +1,328 @@ +//! Code for reading commodity constraints from a CSV file. +use super::super::{input_err_msg, read_csv_optional}; +use crate::commodity::{ + BalanceType, Commodity, CommodityConstraint, CommodityConstraintsMap, CommodityID, + CommodityType, +}; +use crate::id::{GetIDValue, IDCollection}; +use crate::input::{parse_range, parse_year_str}; +use crate::region::RegionID; +use crate::time_slice::TimeSliceInfo; +use crate::units::Flow; +use anyhow::{Context, Result, ensure}; +use indexmap::{IndexMap, IndexSet}; +use serde::Deserialize; +use std::collections::HashMap; +use std::path::Path; + +const COMMODITY_CONSTRAINTS_FILE_NAME: &str = "commodity_constraints.csv"; + +/// Constraints for each commodity +#[derive(PartialEq, Debug, Deserialize)] +struct CommodityConstraintRaw { + /// Unique identifier for the commodity + commodity_id: String, + /// Region id + region_id: String, + /// Type of balance + balance_type: BalanceType, + /// The year(s) to which the constraint applies + years: String, + /// The time slice to which the constraint applies + time_slice: String, + /// Limits on the value of the commodity + limits: String, +} + +impl CommodityConstraintRaw { + fn validate(&self) -> Result<()> { + // Net production already constrained by commodity balance constraints + ensure!( + self.balance_type != BalanceType::Net, + "Balance type cannot be 'net' for commodity constraints" + ); + + Ok(()) + } +} + +/// Read the commodity constraints CSV file. +/// +/// # Arguments +/// +/// * `model_dir` - Folder containing model configuration files +/// * `commodities` - The commodities in the model +/// * `region_ids` - All possible region IDs +/// * `time_slice_info` - Information about time slices +/// * `milestone_years` - All milestone years +/// +/// # Returns +/// +/// A `HashMap` mapping commodity IDs to their +/// commodity-constraints maps, or an error. +pub fn read_commodity_constraints( + model_dir: &Path, + commodities: &IndexMap, + region_ids: &IndexSet, + time_slice_info: &TimeSliceInfo, + milestone_years: &[u32], +) -> Result> { + let file_path = model_dir.join(COMMODITY_CONSTRAINTS_FILE_NAME); + let commodity_constraints_csv = read_csv_optional(&file_path)?; + read_commodity_constraints_from_iter( + commodity_constraints_csv, + commodities, + region_ids, + time_slice_info, + milestone_years, + ) + .with_context(|| input_err_msg(&file_path)) +} + +/// Process raw commodity-constraint records into a constraints map. +/// +/// # Arguments +/// +/// * `iter` - Iterator over `CommodityConstraintRaw` records +/// * `commodities` - The commodoties in the model +/// * `region_ids` - All possible region IDs +/// * `time_slice_info` - Information about time slices +/// * `milestone_years` - All milestone years +/// +/// # Returns +/// +/// A `HashMap` mapping commodity IDs to their +/// commodity-constraints maps, or an error. +fn read_commodity_constraints_from_iter( + iter: I, + commodities: &IndexMap, + region_ids: &IndexSet, + time_slice_info: &TimeSliceInfo, + milestone_years: &[u32], +) -> Result> +where + I: Iterator, +{ + let mut map: HashMap = HashMap::new(); + + for record in iter { + record.validate()?; + + // Extract fields from record + let (commodity_id, commodity) = commodities.get_id_value(&record.commodity_id)?; + ensure!( + commodity.kind != CommodityType::ServiceDemand, + "SVD commodities are not permitted to have commodity constraints" + ); + let region_id = region_ids.get_id(&record.region_id)?; + let years = parse_year_str(&record.years, milestone_years)?; + let ts_selection = time_slice_info.get_selection(&record.time_slice)?; + let limits = parse_range(&record.limits, Flow(0.0)..=Flow(f64::INFINITY)) + .with_context(|| format!("Could not parse constraint range: {}", record.limits))?; + + // For each record, store that constraint per year + let commodity_map = map.entry(commodity_id.clone()).or_default(); + for year in &years { + let constraint = CommodityConstraint { + balance_type: record.balance_type.clone(), + ts_selection: ts_selection.clone(), + limits: limits.clone(), + }; + commodity_map + .entry((region_id.clone(), *year)) + .and_modify(|constraints| constraints.push(constraint.clone())) + .or_insert(vec![constraint]); + } + } + Ok(map) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixture::assert_error; + use crate::input::commodity::{COMMODITY_FILE_NAME, read_commodities_file}; + use crate::time_slice::{TimeSliceID, TimeSliceSelection}; + use crate::units::Year; + use float_cmp::assert_approx_eq; + use std::fs; + use tempfile::tempdir; + + fn validate_raw_constraint(balance_type: BalanceType) -> Result<()> { + let constraint = CommodityConstraintRaw { + commodity_id: "test_commodity".into(), + region_id: "test_region".into(), + balance_type, + years: "2020".into(), + time_slice: "annual".into(), + limits: "1.2..2.3".into(), + }; + constraint.validate() + } + + #[test] + fn validate_constraints_valid() { + let valid = validate_raw_constraint(BalanceType::Production); + valid.unwrap(); + } + + #[test] + fn validate_constraints_invalid() { + // Invalid balance_type specified + let invalid = validate_raw_constraint(BalanceType::Net); + assert_error!( + invalid, + "Balance type cannot be 'net' for commodity constraints" + ); + } + + #[test] + fn read_commodity_constraints_success() -> Result<()> { + // Create a model dir and write simple CSV files + let dir = tempdir()?; + let model_dir = dir.path(); + + // Create simple commodity constraints CSV file + let constraints_csv = concat!( + "commodity_id,region_id,balance_type,years,time_slice,limits\n", + "ELCTRI,GBR,cons,2030,summer,12.34..56.78\n", + "CO2EMT,GBR,cons,2030,winter,..9.99\n", + "CO2EMT,GBR,prod,2030,summer,9.99..\n", + ); + fs::write( + model_dir.join(COMMODITY_CONSTRAINTS_FILE_NAME), + constraints_csv, + )?; + + // Create simple commodities CSV to simplify creating `Commodity`s + let commodities_csv = concat!( + "id,description,type,time_slice_level,units\n", + "ELCTRI,Electricity,sed,season,PJ\n", + "CO2EMT,CO2 emitted,oth,season,ktCO2\n", + ); + fs::write(model_dir.join(COMMODITY_FILE_NAME), commodities_csv)?; + + // Create basic model inputs + let commodities = read_commodities_file(model_dir).unwrap(); + + let mut region_ids: IndexSet = IndexSet::new(); + region_ids.insert(RegionID::from("GBR")); + + let time_slice1 = TimeSliceID { + season: "summer".into(), + time_of_day: "day".into(), + }; + let time_slice2 = TimeSliceID { + season: "summer".into(), + time_of_day: "night".into(), + }; + let time_slice3 = TimeSliceID { + season: "winter".into(), + time_of_day: "day".into(), + }; + let time_slice4 = TimeSliceID { + season: "winter".into(), + time_of_day: "night".into(), + }; + let time_slice_info = TimeSliceInfo { + seasons: [("summer".into(), Year(0.5)), ("winter".into(), Year(0.5))].into(), + times_of_day: ["day".into(), "night".into()].into(), + time_slices: [ + (time_slice1.clone(), Year(0.25)), + (time_slice2.clone(), Year(0.25)), + (time_slice3.clone(), Year(0.25)), + (time_slice4.clone(), Year(0.25)), + ] + .into(), + }; + + let milestone_years = vec![2030]; + + // Create the constraints map + let constraints_map = read_commodity_constraints( + model_dir, + &commodities, + ®ion_ids, + &time_slice_info, + &milestone_years, + )?; + + // Check the constraints map contains the expected constraint, keyed by the expected + // commodity id + assert!(constraints_map.contains_key(&CommodityID::from("ELCTRI"))); + assert!(constraints_map.contains_key(&CommodityID::from("CO2EMT"))); + + // ELCTRI constraint + let elctri_constraint = &constraints_map[&CommodityID::from("ELCTRI")]; + let elctri_gbr_2030 = elctri_constraint + .get(&(RegionID::from("GBR"), 2030)) + .unwrap(); + assert_eq!(elctri_gbr_2030[0].balance_type, BalanceType::Consumption); + assert_eq!( + elctri_gbr_2030[0].ts_selection, + TimeSliceSelection::Season("summer".into()), + ); + assert_approx_eq!(f64, elctri_gbr_2030[0].limits.start().value(), 12.34); + assert_approx_eq!(f64, elctri_gbr_2030[0].limits.end().value(), 56.78); + + // CO2EMT constraints + let co2emt_constraint = &constraints_map[&CommodityID::from("CO2EMT")]; + let co2emt_gbr_2030 = co2emt_constraint + .get(&(RegionID::from("GBR"), 2030)) + .unwrap(); + assert_eq!(co2emt_gbr_2030[0].balance_type, BalanceType::Consumption); + assert_eq!( + co2emt_gbr_2030[0].ts_selection, + TimeSliceSelection::Season("winter".into()), + ); + assert_approx_eq!(f64, co2emt_gbr_2030[0].limits.start().value(), 0.0); + assert_approx_eq!(f64, co2emt_gbr_2030[0].limits.end().value(), 9.99); + + assert_eq!(co2emt_gbr_2030[1].balance_type, BalanceType::Production); + assert_eq!( + co2emt_gbr_2030[1].ts_selection, + TimeSliceSelection::Season("summer".into()), + ); + assert_approx_eq!(f64, co2emt_gbr_2030[1].limits.start().value(), 9.99); + assert_approx_eq!(f64, co2emt_gbr_2030[1].limits.end().value(), f64::INFINITY); + + Ok(()) + } + + #[test] + fn read_commodity_constraints_fails_with_invalid_csv() -> Result<()> { + // Create a model dir and write invalid CSV content to force + // read_commodity_constraints_from_iter failure + let dir = tempdir()?; + let model_dir = dir.path(); + + // Create invalid commodity constraints CSV content + let file_path = model_dir.join(COMMODITY_CONSTRAINTS_FILE_NAME); + fs::write(&file_path, "invalid,commodity,constraints\nbad,row\n")?; + + // Create empty model inputs + let commodities: IndexMap = IndexMap::new(); + let region_ids: IndexSet = IndexSet::new(); + let time_slice_info = TimeSliceInfo::default(); + let milestone_years: Vec = vec![2020, 2030]; + + // Try to create the constraints map + let result = read_commodity_constraints( + model_dir, + &commodities, + ®ion_ids, + &time_slice_info, + &milestone_years, + ); + + // Check failure and file path present in error message + assert!(result.is_err()); + let err_text = format!("{:#}", result.unwrap_err()); + assert!( + err_text.contains(COMMODITY_CONSTRAINTS_FILE_NAME), + "error message should include file name context, got: {err_text}" + ); + + Ok(()) + } +} diff --git a/src/input/process/flow.rs b/src/input/process/flow.rs index 69a323308..6248039ed 100644 --- a/src/input/process/flow.rs +++ b/src/input/process/flow.rs @@ -389,8 +389,9 @@ fn validate_secondary_flows( #[cfg(test)] mod tests { use super::*; - use crate::commodity::Commodity; - use crate::commodity::{CommodityLevyMap, DemandMap, PricingStrategy}; + use crate::commodity::{ + Commodity, CommodityConstraintsMap, CommodityLevyMap, DemandMap, PricingStrategy, + }; use crate::fixture::{ assert_error, assert_validate_fails_with_simple, assert_validate_ok_simple, other_commodity, process, sed_commodity, svd_commodity, @@ -445,6 +446,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), } } @@ -460,6 +462,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "tonnes".into(), } } diff --git a/src/process.rs b/src/process.rs index 48245a83f..2c9221b69 100644 --- a/src/process.rs +++ b/src/process.rs @@ -543,7 +543,9 @@ impl ProcessInvestmentConstraint { #[cfg(test)] mod tests { use super::*; - use crate::commodity::{CommodityLevyMap, CommodityType, DemandMap, PricingStrategy}; + use crate::commodity::{ + CommodityConstraintsMap, CommodityLevyMap, CommodityType, DemandMap, PricingStrategy, + }; use crate::fixture::{assert_error, region_id, time_slice, time_slice_info2}; use crate::time_slice::TimeSliceLevel; use crate::time_slice::TimeSliceSelection; @@ -611,6 +613,7 @@ mod tests { levies_prod, levies_cons, demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), }) } @@ -632,6 +635,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: levies, demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), }) } @@ -653,6 +657,7 @@ mod tests { levies_prod: levies, levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), }) } @@ -676,6 +681,7 @@ mod tests { levies_prod, levies_cons, demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), }) } @@ -691,6 +697,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), }) } @@ -707,6 +714,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), }), coeff: FlowPerActivity(1.0), @@ -730,6 +738,7 @@ mod tests { levies_prod: levies, levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), }), coeff: FlowPerActivity(1.0), @@ -753,6 +762,7 @@ mod tests { levies_prod: levies, levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), }), coeff: FlowPerActivity(1.0), @@ -1021,6 +1031,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + constraints: CommodityConstraintsMap::new(), units: "PJ".into(), });