Skip to content

4174 only read indats using the indat class #4523

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
clmould wants to merge 5 commits into
base: main
Choose a base branch
from
+154 −136
Draft

4174 only read indats using the indat class #4523

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
51d4b1a
WIP but copied validate_input() to InDat, renamed to update_obsolete(…
clmould Aug 12, 2026
7d888c1
Remove unnecessary init_module_vars() from main.py
clmould Aug 12, 2026
e97b517
comment out ipdb
clmould Aug 13, 2026
e6f104d
rename to check_obsolete_variables, and make sure it runs at correct …
clmould Aug 13, 2026
eda15dc
comment
clmould Aug 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions process/core/init.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from process.core import constants, process_output
from process.core.exceptions import ProcessValidationError
from process.core.input import parse_input_file
from process.core.io.in_dat.base import InDat
from process.core.solver import iteration_variables
from process.core.solver.constraints import ConstraintManager
from process.data_structure.blanket_variables import BlktModelTypes
Expand Down Expand Up @@ -50,7 +51,7 @@
from process.core.model import DataStructure


def init_process(data: DataStructure):
def init_process(data: DataStructure, update_obsolete: bool = False):
"""Routine that calls the initialisation routines

This routine calls the main initialisation routines that set
Expand All @@ -59,13 +60,25 @@ def init_process(data: DataStructure):
"""
# Initialise the program variables
iteration_variables.initialise_iteration_variables(data)

# Creating and open the files MFile and OUTFile
process_output.OutputFileManager.open_files(data.globals.output_prefix)
# import ipdb

# ipdb.set_trace()
# TODO use InDat(filename) instead here?
# Use InDat class to read in IN.DAT, update obsolete and
# parse input file
filename = data.globals.output_prefix + "IN.DAT"
# Check for and, if requested, update obsolete variables
in_dat = InDat(filename=filename, update_obsolete=update_obsolete)

# Input any desired new initial values
inputs = parse_input_file(data)
# if comment this out, everything has its default value from data_structure files
# so need InDat to
inputs = parse_input_file(data) # want to absorb into InDat()
# import ipdb

# ipdb.set_trace()
# Set active constraints
set_active_constraints(data)

Expand Down
133 changes: 132 additions & 1 deletion process/core/io/in_dat/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from re import sub

from process.core.exceptions import ProcessValidationError
from process.core.io import obsolete_vars as ov
from process.core.io.data_structure_dicts import get_dicts
from process.core.solver.constraints import ConstraintManager
from process.core.solver.iteration_variables import ITERATION_VARIABLES
Expand Down Expand Up @@ -1027,9 +1028,10 @@ class InDat:
- Writing IN.DAT files
- Storing information in dictionary for use in other codes
- Alterations to IN.DAT
- Updating obsolete variables
"""

def __init__(self, filename="IN.DAT", start_line=0):
def __init__(self, filename="IN.DAT", start_line=0, update_obsolete: bool = False):
"""Initialise class

Parameters
Expand All @@ -1038,9 +1040,17 @@ def __init__(self, filename="IN.DAT", start_line=0):
Name of input IN.DAT
start_line:
Line to start reading from
update_obsolete:
Whether to update obsolete variables in the IN.DAT or not
"""
self.filename = filename
self.start_line = start_line
self.update_obsolete = update_obsolete
# import ipdb

# ipdb.set_trace()
# Check for obsolete variables and update if requested
self.check_obsolete_variables()

# Initialise parameters
self.in_dat_lines = []
Expand Down Expand Up @@ -1632,6 +1642,127 @@ def write_in_dat(self, output_filename="new_IN.DAT"):
# Write parameters
write_parameters(self.data, output)

def check_obsolete_variables(self):
"""Checks the input IN.DAT file for any obsolete variables in the OBS_VARS dict
contained within obsolete_variables.py.
If obsolete variables are found, and if `update_obsolete` is set to True,
they are either removed or replaced by their updated names as specified
in the OBS_VARS dictionary.

Raises
------
ValueError
If obsolete variables are present in the input file and update_obsolete
is False.
"""
obsolete_variables = ov.OBS_VARS
obsolete_vars_help_message = ov.OBS_VARS_HELP

filename = self.filename
variables_in_in_dat = []
modified_lines = []
changes_made = [] # To store details of the changes

with open(filename) as file:
for line in file:
# Skip comment lines or lines without an assignment
if line.startswith("*") or "=" not in line:
modified_lines.append(line)
continue

# Extract the variable name before the separator
raw_variable_name = line.split("=", 1)[0].strip()
# handle cases where the variable name might have parentheses
variable_name = (
raw_variable_name.split("(", 1)[0]
if "(" in raw_variable_name
else raw_variable_name
)

# Check if the variable is obsolete and needs replacing
if variable_name in obsolete_variables:
replacement = obsolete_variables.get(variable_name)
if self.update_obsolete:
# Prepare replacement or removal
if replacement is None:
# If no replacement is defined, comment out the line
modified_lines.append(f"* Obsolete: {line}")
changes_made.append(
f"Commented out obsolete variable: {variable_name}"
)
else:
if isinstance(replacement, list):
# Raise an error if replacement is a list
replacement_str = ", ".join(replacement)
raise ValueError(
f"The variable '{variable_name}' is obsolete and "
"should be replaced by the following variables: "
f"{replacement_str}. "
"Please set their values accordingly."
)
# Replace obsolete variable
modified_line = line.replace(variable_name, replacement, 1)
modified_lines.append(
f"* Replaced '{variable_name}' with "
f"'{replacement}'\n{modified_line}"
)
changes_made.append(
f"Replaced '{variable_name}' with '{replacement}'"
)
variables_in_in_dat.append(variable_name)
else:
# If replacement is False, add the line as-is
modified_lines.append(line)
variables_in_in_dat.append(variable_name)
else:
modified_lines.append(line)

obs_vars_in_in_dat = [
var for var in variables_in_in_dat if var in obsolete_variables
]

if obs_vars_in_in_dat:
if self.update_obsolete:
# If update_obsolete is True, write the modified content to the file
with open(filename, "w") as file:
file.writelines(modified_lines)
print(
"The IN.DAT file has been updated to replace or "
"comment out obsolete variables."
)
print("Summary of changes made:")
for change in changes_made:
print(f" - {change}")
else:
# Only print the report if update_obsolete is False
message = (
"The IN.DAT file contains obsolete variables "
"from the OBS_VARS dictionary. "
"The obsolete variables in your IN.DAT file are: "
f"{obs_vars_in_in_dat}. "
"Either remove these or replace them with "
"their updated variable names. "
"Use the --update-obsolete flag for this "
"to be done automatically."
)
for obs_var in obs_vars_in_in_dat:
replacement = obsolete_variables.get(obs_var)
if replacement is None:
message += (
f"\n\n{obs_var} is an obsolete variable "
"and needs to be removed."
)
else:
message += (
f"\n\n{obs_var} is an obsolete variable "
f"and needs to be replaced by {replacement}."
)
message += f" {obsolete_vars_help_message.get(obs_var, '')}"
raise ValueError(message)

else:
print("The IN.DAT file does not contain any obsolete variables.")

@property
def number_of_constraints(self):
"""
Expand Down
138 changes: 6 additions & 132 deletions process/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@

import process # noqa: F401
from process.core import constants, init
from process.core.io import obsolete_vars as ov
from process.core.io.cli_tools import LazyGroup, help_opt, indat_opt
from process.core.io.mfile import MFile
from process.core.io.plot import plot_sankey_plotly, plot_summary
Expand Down Expand Up @@ -330,11 +329,13 @@ def __init__(
"""
self.input_file = Path(input_file)
self.data = data_structure or DataStructure()
# import ipdb

self.validate_input(update_obsolete)
self.init_module_vars()
# ipdb.set_trace()
self.update_obsolete = update_obsolete
logging_model_handler.clear_logs()
self.set_filenames(filepath_out)
self.initialise()
self.initialise() # in here does init_process
self.models = Models(self.data)
self.solver = solver

Expand All @@ -348,15 +349,6 @@ def run(self):
self.finish()
self.append_input()

@staticmethod
def init_module_vars():
"""Initialise all module variables in the Fortran.

This "resets" all module variables to their initialised values, so each
new run doesn't have any side-effects from previous runs.
"""
logging_model_handler.clear_logs()

def set_filenames(self, filepath_out):
"""Validate the input filename and create other filenames from it."""
filepath = Path(filepath_out or self.input_file)
Expand Down Expand Up @@ -427,7 +419,7 @@ def initialise(self):

initialise_imprad(self.data)
# Reads in input file
init.init_process(self.data)
init.init_process(self.data, self.update_obsolete)

# Order optimisation parameters (arbitrary order in input file)
# Ensures consistency and makes output comparisons more straightforward
Expand Down Expand Up @@ -491,124 +483,6 @@ def append_input(self):
mfile_file.write("***********************************************")
mfile_file.writelines(input_lines)

def validate_input(self, replace_obsolete: bool = False):
"""Checks the input IN.DAT file for any obsolete variables in the OBS_VARS dict
contained within obsolete_variables.py.
If obsolete variables are found, and if `replace_obsolete` is set to True,
they are either removed or replaced by their updated names as specified
in the OBS_VARS dictionary.

Raises
------
ValueError
If obsolete variables are present in the input file.
"""
obsolete_variables = ov.OBS_VARS
obsolete_vars_help_message = ov.OBS_VARS_HELP

filename = self.input_file
variables_in_in_dat = []
modified_lines = []
changes_made = [] # To store details of the changes

with open(filename) as file:
for line in file:
# Skip comment lines or lines without an assignment
if line.startswith("*") or "=" not in line:
modified_lines.append(line)
continue

# Extract the variable name before the separator
raw_variable_name = line.split("=", 1)[0].strip()
# handle cases where the variable name might have parentheses
variable_name = (
raw_variable_name.split("(", 1)[0]
if "(" in raw_variable_name
else raw_variable_name
)

# Check if the variable is obsolete and needs replacing
if variable_name in obsolete_variables:
replacement = obsolete_variables.get(variable_name)
if replace_obsolete:
# Prepare replacement or removal
if replacement is None:
# If no replacement is defined, comment out the line
modified_lines.append(f"* Obsolete: {line}")
changes_made.append(
f"Commented out obsolete variable: {variable_name}"
)
else:
if isinstance(replacement, list):
# Raise an error if replacement is a list
replacement_str = ", ".join(replacement)
raise ValueError(
f"The variable '{variable_name}' is obsolete and "
"should be replaced by the following variables: "
f"{replacement_str}. "
"Please set their values accordingly."
)
# Replace obsolete variable
modified_line = line.replace(variable_name, replacement, 1)
modified_lines.append(
f"* Replaced '{variable_name}' with "
f"'{replacement}'\n{modified_line}"
)
changes_made.append(
f"Replaced '{variable_name}' with '{replacement}'"
)
variables_in_in_dat.append(variable_name)
else:
# If replacement is False, add the line as-is
modified_lines.append(line)
variables_in_in_dat.append(variable_name)
else:
modified_lines.append(line)

obs_vars_in_in_dat = [
var for var in variables_in_in_dat if var in obsolete_variables
]

if obs_vars_in_in_dat:
if replace_obsolete:
# If replace_obsolete is True, write the modified content to the file
with open(filename, "w") as file:
file.writelines(modified_lines)
print(
"The IN.DAT file has been updated to replace or "
"comment out obsolete variables."
)
print("Summary of changes made:")
for change in changes_made:
print(f" - {change}")
else:
# Only print the report if replace_obsolete is False
message = (
"The IN.DAT file contains obsolete variables "
"from the OBS_VARS dictionary. "
"The obsolete variables in your IN.DAT file are: "
f"{obs_vars_in_in_dat}. "
"Either remove these or replace them with "
"their updated variable names. "
)
for obs_var in obs_vars_in_in_dat:
replacement = obsolete_variables.get(obs_var)
if replacement is None:
message += (
f"\n\n{obs_var} is an obsolete variable "
"and needs to be removed."
)
else:
message += (
f"\n\n{obs_var} is an obsolete variable "
f"and needs to be replaced by {replacement}."
)
message += f" {obsolete_vars_help_message.get(obs_var, '')}"
raise ValueError(message)

else:
print("The IN.DAT file does not contain any obsolete variables.")

def validate_user_model(self):
"""Checks that a user-created model has been injected correctly

Expand Down
Loading