From 51d4b1a606a506ef9c6f8fc78368b394bb846a52 Mon Sep 17 00:00:00 2001 From: Clair Mould <86794332+clmould@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:22:02 +0100 Subject: [PATCH 1/5] WIP but copied validate_input() to InDat, renamed to update_obsolete(), and got it working to update the obsolete vars when running process, includes pdbs --- process/core/init.py | 13 +++- process/core/io/in_dat/base.py | 134 ++++++++++++++++++++++++++++++++- process/main.py | 127 ++----------------------------- 3 files changed, 149 insertions(+), 125 deletions(-) diff --git a/process/core/init.py b/process/core/init.py index 1b7e6dc508..5046658b25 100644 --- a/process/core/init.py +++ b/process/core/init.py @@ -14,6 +14,7 @@ from process.core import constants, process_output from process.core.exceptions import ProcessValidationError from process.core.input import parse_input_file +from process.core.io.in_dat.base import InDat from process.core.solver import iteration_variables from process.core.solver.constraints import ConstraintManager from process.data_structure.blanket_variables import BlktModelTypes @@ -50,7 +51,7 @@ from process.core.model import DataStructure -def init_process(data: DataStructure): +def init_process(data: DataStructure, update_obsolete: bool = False): """Routine that calls the initialisation routines This routine calls the main initialisation routines that set @@ -59,9 +60,17 @@ def init_process(data: DataStructure): """ # Initialise the program variables iteration_variables.initialise_iteration_variables(data) - # Creating and open the files MFile and OUTFile process_output.OutputFileManager.open_files(data.globals.output_prefix) + import ipdb + + ipdb.set_trace() + # TODO use InDat(filename) instead here? + # Use InDat class to read in IN.DAT, update obsolete and + # parse input file + filename = data.globals.output_prefix + "IN.DAT" + # Check for and, if requested, update obsolete variables + InDat(filename=filename, update_obsolete=update_obsolete) # Input any desired new initial values inputs = parse_input_file(data) diff --git a/process/core/io/in_dat/base.py b/process/core/io/in_dat/base.py index 32dc649e66..66c04a4b4b 100644 --- a/process/core/io/in_dat/base.py +++ b/process/core/io/in_dat/base.py @@ -14,6 +14,7 @@ from re import sub from process.core.exceptions import ProcessValidationError +from process.core.io import obsolete_vars as ov from process.core.io.data_structure_dicts import get_dicts from process.core.solver.constraints import ConstraintManager from process.core.solver.iteration_variables import ITERATION_VARIABLES @@ -1027,9 +1028,10 @@ class InDat: - Writing IN.DAT files - Storing information in dictionary for use in other codes - Alterations to IN.DAT + - Updating obsolete variables """ - def __init__(self, filename="IN.DAT", start_line=0): + def __init__(self, filename="IN.DAT", start_line=0, update_obsolete: bool = False): """Initialise class Parameters @@ -1038,9 +1040,18 @@ def __init__(self, filename="IN.DAT", start_line=0): Name of input IN.DAT start_line: Line to start reading from + update_obsolete: + Whether to update obsolete variables in the IN.DAT or not """ self.filename = filename self.start_line = start_line + self.update_obsolete = update_obsolete + import ipdb + + ipdb.set_trace() + # Update obsolete variables if requested + if self.update_obsolete: + self.update_obsolete_variables() # Initialise parameters self.in_dat_lines = [] @@ -1632,6 +1643,127 @@ def write_in_dat(self, output_filename="new_IN.DAT"): # Write parameters write_parameters(self.data, output) + def update_obsolete_variables(self): + """Checks the input IN.DAT file for any obsolete variables in the OBS_VARS dict + contained within obsolete_variables.py. + If obsolete variables are found, and if `update_obsolete` is set to True, + they are either removed or replaced by their updated names as specified + in the OBS_VARS dictionary. + + Raises + ------ + ValueError + If obsolete variables are present in the input file and update_obsolete + is False. + """ + obsolete_variables = ov.OBS_VARS + obsolete_vars_help_message = ov.OBS_VARS_HELP + + filename = self.filename + variables_in_in_dat = [] + modified_lines = [] + changes_made = [] # To store details of the changes + + with open(filename) as file: + for line in file: + # Skip comment lines or lines without an assignment + if line.startswith("*") or "=" not in line: + modified_lines.append(line) + continue + + # Extract the variable name before the separator + raw_variable_name = line.split("=", 1)[0].strip() + # handle cases where the variable name might have parentheses + variable_name = ( + raw_variable_name.split("(", 1)[0] + if "(" in raw_variable_name + else raw_variable_name + ) + + # Check if the variable is obsolete and needs replacing + if variable_name in obsolete_variables: + replacement = obsolete_variables.get(variable_name) + if self.update_obsolete: + # Prepare replacement or removal + if replacement is None: + # If no replacement is defined, comment out the line + modified_lines.append(f"* Obsolete: {line}") + changes_made.append( + f"Commented out obsolete variable: {variable_name}" + ) + else: + if isinstance(replacement, list): + # Raise an error if replacement is a list + replacement_str = ", ".join(replacement) + raise ValueError( + f"The variable '{variable_name}' is obsolete and " + "should be replaced by the following variables: " + f"{replacement_str}. " + "Please set their values accordingly." + ) + # Replace obsolete variable + modified_line = line.replace(variable_name, replacement, 1) + modified_lines.append( + f"* Replaced '{variable_name}' with " + f"'{replacement}'\n{modified_line}" + ) + changes_made.append( + f"Replaced '{variable_name}' with '{replacement}'" + ) + variables_in_in_dat.append(variable_name) + else: + # If replacement is False, add the line as-is + modified_lines.append(line) + variables_in_in_dat.append(variable_name) + else: + modified_lines.append(line) + + obs_vars_in_in_dat = [ + var for var in variables_in_in_dat if var in obsolete_variables + ] + + if obs_vars_in_in_dat: + if self.update_obsolete: + # If update_obsolete is True, write the modified content to the file + with open(filename, "w") as file: + file.writelines(modified_lines) + print( + "The IN.DAT file has been updated to replace or " + "comment out obsolete variables." + ) + print("Summary of changes made:") + for change in changes_made: + print(f" - {change}") + else: + # Only print the report if update_obsolete is False + message = ( + "The IN.DAT file contains obsolete variables " + "from the OBS_VARS dictionary. " + "The obsolete variables in your IN.DAT file are: " + f"{obs_vars_in_in_dat}. " + "Either remove these or replace them with " + "their updated variable names. " + "Use the --update-obsolete flag for this " + "to be done automatically." + ) + for obs_var in obs_vars_in_in_dat: + replacement = obsolete_variables.get(obs_var) + if replacement is None: + message += ( + f"\n\n{obs_var} is an obsolete variable " + "and needs to be removed." + ) + else: + message += ( + f"\n\n{obs_var} is an obsolete variable " + f"and needs to be replaced by {replacement}." + ) + message += f" {obsolete_vars_help_message.get(obs_var, '')}" + raise ValueError(message) + + else: + print("The IN.DAT file does not contain any obsolete variables.") + @property def number_of_constraints(self): """ diff --git a/process/main.py b/process/main.py index c136a6e9ef..d046f9b57e 100644 --- a/process/main.py +++ b/process/main.py @@ -39,7 +39,6 @@ import process # noqa: F401 from process.core import constants, init -from process.core.io import obsolete_vars as ov from process.core.io.cli_tools import LazyGroup, help_opt, indat_opt from process.core.io.mfile import MFile from process.core.io.plot import plot_sankey_plotly, plot_summary @@ -330,11 +329,13 @@ def __init__( """ self.input_file = Path(input_file) self.data = data_structure or DataStructure() + import ipdb - self.validate_input(update_obsolete) + ipdb.set_trace() + self.update_obsolete = update_obsolete self.init_module_vars() self.set_filenames(filepath_out) - self.initialise() + self.initialise() # in here does init_process self.models = Models(self.data) self.solver = solver @@ -427,7 +428,7 @@ def initialise(self): initialise_imprad(self.data) # Reads in input file - init.init_process(self.data) + init.init_process(self.data, self.update_obsolete) # Order optimisation parameters (arbitrary order in input file) # Ensures consistency and makes output comparisons more straightforward @@ -491,124 +492,6 @@ def append_input(self): mfile_file.write("***********************************************") mfile_file.writelines(input_lines) - def validate_input(self, replace_obsolete: bool = False): - """Checks the input IN.DAT file for any obsolete variables in the OBS_VARS dict - contained within obsolete_variables.py. - If obsolete variables are found, and if `replace_obsolete` is set to True, - they are either removed or replaced by their updated names as specified - in the OBS_VARS dictionary. - - Raises - ------ - ValueError - If obsolete variables are present in the input file. - """ - obsolete_variables = ov.OBS_VARS - obsolete_vars_help_message = ov.OBS_VARS_HELP - - filename = self.input_file - variables_in_in_dat = [] - modified_lines = [] - changes_made = [] # To store details of the changes - - with open(filename) as file: - for line in file: - # Skip comment lines or lines without an assignment - if line.startswith("*") or "=" not in line: - modified_lines.append(line) - continue - - # Extract the variable name before the separator - raw_variable_name = line.split("=", 1)[0].strip() - # handle cases where the variable name might have parentheses - variable_name = ( - raw_variable_name.split("(", 1)[0] - if "(" in raw_variable_name - else raw_variable_name - ) - - # Check if the variable is obsolete and needs replacing - if variable_name in obsolete_variables: - replacement = obsolete_variables.get(variable_name) - if replace_obsolete: - # Prepare replacement or removal - if replacement is None: - # If no replacement is defined, comment out the line - modified_lines.append(f"* Obsolete: {line}") - changes_made.append( - f"Commented out obsolete variable: {variable_name}" - ) - else: - if isinstance(replacement, list): - # Raise an error if replacement is a list - replacement_str = ", ".join(replacement) - raise ValueError( - f"The variable '{variable_name}' is obsolete and " - "should be replaced by the following variables: " - f"{replacement_str}. " - "Please set their values accordingly." - ) - # Replace obsolete variable - modified_line = line.replace(variable_name, replacement, 1) - modified_lines.append( - f"* Replaced '{variable_name}' with " - f"'{replacement}'\n{modified_line}" - ) - changes_made.append( - f"Replaced '{variable_name}' with '{replacement}'" - ) - variables_in_in_dat.append(variable_name) - else: - # If replacement is False, add the line as-is - modified_lines.append(line) - variables_in_in_dat.append(variable_name) - else: - modified_lines.append(line) - - obs_vars_in_in_dat = [ - var for var in variables_in_in_dat if var in obsolete_variables - ] - - if obs_vars_in_in_dat: - if replace_obsolete: - # If replace_obsolete is True, write the modified content to the file - with open(filename, "w") as file: - file.writelines(modified_lines) - print( - "The IN.DAT file has been updated to replace or " - "comment out obsolete variables." - ) - print("Summary of changes made:") - for change in changes_made: - print(f" - {change}") - else: - # Only print the report if replace_obsolete is False - message = ( - "The IN.DAT file contains obsolete variables " - "from the OBS_VARS dictionary. " - "The obsolete variables in your IN.DAT file are: " - f"{obs_vars_in_in_dat}. " - "Either remove these or replace them with " - "their updated variable names. " - ) - for obs_var in obs_vars_in_in_dat: - replacement = obsolete_variables.get(obs_var) - if replacement is None: - message += ( - f"\n\n{obs_var} is an obsolete variable " - "and needs to be removed." - ) - else: - message += ( - f"\n\n{obs_var} is an obsolete variable " - f"and needs to be replaced by {replacement}." - ) - message += f" {obsolete_vars_help_message.get(obs_var, '')}" - raise ValueError(message) - - else: - print("The IN.DAT file does not contain any obsolete variables.") - def validate_user_model(self): """Checks that a user-created model has been injected correctly From 7d888c18fa393f83320c7a72193eab509c41f647 Mon Sep 17 00:00:00 2001 From: Clair Mould <86794332+clmould@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:26:26 +0100 Subject: [PATCH 2/5] Remove unnecessary init_module_vars() from main.py --- process/main.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/process/main.py b/process/main.py index d046f9b57e..713678b35a 100644 --- a/process/main.py +++ b/process/main.py @@ -333,7 +333,7 @@ def __init__( ipdb.set_trace() self.update_obsolete = update_obsolete - self.init_module_vars() + logging_model_handler.clear_logs() self.set_filenames(filepath_out) self.initialise() # in here does init_process self.models = Models(self.data) @@ -349,15 +349,6 @@ def run(self): self.finish() self.append_input() - @staticmethod - def init_module_vars(): - """Initialise all module variables in the Fortran. - - This "resets" all module variables to their initialised values, so each - new run doesn't have any side-effects from previous runs. - """ - logging_model_handler.clear_logs() - def set_filenames(self, filepath_out): """Validate the input filename and create other filenames from it.""" filepath = Path(filepath_out or self.input_file) From e97b517e434cf87551442a3406c69780954a0a6f Mon Sep 17 00:00:00 2001 From: Clair Mould <86794332+clmould@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:30:34 +0100 Subject: [PATCH 3/5] comment out ipdb --- process/core/init.py | 12 ++++++++---- process/core/io/in_dat/base.py | 4 ++-- process/main.py | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/process/core/init.py b/process/core/init.py index 5046658b25..d49a9982a6 100644 --- a/process/core/init.py +++ b/process/core/init.py @@ -62,19 +62,23 @@ def init_process(data: DataStructure, update_obsolete: bool = False): iteration_variables.initialise_iteration_variables(data) # Creating and open the files MFile and OUTFile process_output.OutputFileManager.open_files(data.globals.output_prefix) - import ipdb + # import ipdb - ipdb.set_trace() + # ipdb.set_trace() # TODO use InDat(filename) instead here? # Use InDat class to read in IN.DAT, update obsolete and # parse input file filename = data.globals.output_prefix + "IN.DAT" # Check for and, if requested, update obsolete variables - InDat(filename=filename, update_obsolete=update_obsolete) + in_dat = InDat(filename=filename, update_obsolete=update_obsolete) # Input any desired new initial values - inputs = parse_input_file(data) + # if comment this out, everything has its default value from data_structure files + # so need InDat to + inputs = parse_input_file(data) # want to absorb into InDat() + import ipdb + ipdb.set_trace() # Set active constraints set_active_constraints(data) diff --git a/process/core/io/in_dat/base.py b/process/core/io/in_dat/base.py index 66c04a4b4b..dee84c3fe4 100644 --- a/process/core/io/in_dat/base.py +++ b/process/core/io/in_dat/base.py @@ -1046,9 +1046,9 @@ def __init__(self, filename="IN.DAT", start_line=0, update_obsolete: bool = Fals self.filename = filename self.start_line = start_line self.update_obsolete = update_obsolete - import ipdb + # import ipdb - ipdb.set_trace() + # ipdb.set_trace() # Update obsolete variables if requested if self.update_obsolete: self.update_obsolete_variables() diff --git a/process/main.py b/process/main.py index 713678b35a..20e208857a 100644 --- a/process/main.py +++ b/process/main.py @@ -329,9 +329,9 @@ def __init__( """ self.input_file = Path(input_file) self.data = data_structure or DataStructure() - import ipdb + # import ipdb - ipdb.set_trace() + # ipdb.set_trace() self.update_obsolete = update_obsolete logging_model_handler.clear_logs() self.set_filenames(filepath_out) From e6f104d49309247f5cb0dfee3f6c21e402c9454f Mon Sep 17 00:00:00 2001 From: Clair Mould <86794332+clmould@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:31:04 +0100 Subject: [PATCH 4/5] rename to check_obsolete_variables, and make sure it runs at correct time --- process/core/io/in_dat/base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/process/core/io/in_dat/base.py b/process/core/io/in_dat/base.py index dee84c3fe4..058170ae10 100644 --- a/process/core/io/in_dat/base.py +++ b/process/core/io/in_dat/base.py @@ -1050,8 +1050,7 @@ def __init__(self, filename="IN.DAT", start_line=0, update_obsolete: bool = Fals # ipdb.set_trace() # Update obsolete variables if requested - if self.update_obsolete: - self.update_obsolete_variables() + self.check_obsolete_variables() # Initialise parameters self.in_dat_lines = [] @@ -1643,7 +1642,7 @@ def write_in_dat(self, output_filename="new_IN.DAT"): # Write parameters write_parameters(self.data, output) - def update_obsolete_variables(self): + def check_obsolete_variables(self): """Checks the input IN.DAT file for any obsolete variables in the OBS_VARS dict contained within obsolete_variables.py. If obsolete variables are found, and if `update_obsolete` is set to True, From eda15dc9d80b4c6a652c6cb5d6dee7a92fdba5bf Mon Sep 17 00:00:00 2001 From: Clair Mould <86794332+clmould@users.noreply.github.com> Date: Thu, 13 Aug 2026 17:19:47 +0100 Subject: [PATCH 5/5] comment --- process/core/init.py | 4 ++-- process/core/io/in_dat/base.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/process/core/init.py b/process/core/init.py index d49a9982a6..e679adf1b4 100644 --- a/process/core/init.py +++ b/process/core/init.py @@ -76,9 +76,9 @@ def init_process(data: DataStructure, update_obsolete: bool = False): # if comment this out, everything has its default value from data_structure files # so need InDat to inputs = parse_input_file(data) # want to absorb into InDat() - import ipdb + # import ipdb - ipdb.set_trace() + # ipdb.set_trace() # Set active constraints set_active_constraints(data) diff --git a/process/core/io/in_dat/base.py b/process/core/io/in_dat/base.py index 058170ae10..6670bee6e9 100644 --- a/process/core/io/in_dat/base.py +++ b/process/core/io/in_dat/base.py @@ -1049,7 +1049,7 @@ def __init__(self, filename="IN.DAT", start_line=0, update_obsolete: bool = Fals # import ipdb # ipdb.set_trace() - # Update obsolete variables if requested + # Check for obsolete variables and update if requested self.check_obsolete_variables() # Initialise parameters