From 37094641e2a8c63e03284824ccdccf4a439878b9 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 20 May 2026 14:05:26 +0100 Subject: [PATCH 01/17] More noticeable dividers between different database table categories --- src/murfey/util/db.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 5ba6691d6..eb92f48fa 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -10,7 +10,9 @@ from sqlmodel import Enum, Field, Relationship, SQLModel, create_engine """ +======================================================================================= GENERAL +======================================================================================= """ @@ -173,7 +175,9 @@ class ImagingSite(SQLModel, table=True): # type: ignore """ +======================================================================================= TEM SESSION AND PROCESSING WORKFLOW +======================================================================================= """ @@ -1072,7 +1076,9 @@ class CryoemInitialModel(SQLModel, table=True): # type: ignore """ +======================================================================================= FUNCTIONS +======================================================================================= """ From 84ad3b4dd31070f3382bba928b078e9776d354ba Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 20 May 2026 14:08:19 +0100 Subject: [PATCH 02/17] Moved 'number_from_name' into new module 'murfey.util.fib' --- src/murfey/client/contexts/fib.py | 22 +++------------------- src/murfey/util/fib.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 19 deletions(-) create mode 100644 src/murfey/util/fib.py diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index 874fc6b94..0b06370d2 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -import re import threading import xml.etree.ElementTree as ET from dataclasses import dataclass, field @@ -11,6 +10,7 @@ from murfey.client.context import Context from murfey.client.instance_environment import MurfeyInstanceEnvironment from murfey.util.client import capture_post +from murfey.util.fib import number_from_name from murfey.util.models import ( LamellaSiteInfo, MillingStepInfo, @@ -24,22 +24,6 @@ lock = threading.Lock() -def _number_from_name(name: str) -> int: - """ - In the AutoTEM and Maps workflows for the FIB, the sites and images are - auto-incremented with parenthesised numbers (e.g. "Lamella (2)"), with - the first site/image typically not having a number. - - This function extracts the number from the file name, and returns 1 if - no such number is found. - """ - return ( - int(match.group(1)) - if (match := re.search(r"^[\w\s]+\((\d+)\)$", name)) is not None - else 1 - ) - - T = TypeVar("T") @@ -416,7 +400,7 @@ def _parse_autotem_metadata(self, file: Path): if (site_name := _parse_xml_text(site, "Name", str)) is None: logger.warning("Current site doesn't have a name") continue - site_num = _number_from_name(site_name) + site_num = number_from_name(site_name) site_info = LamellaSiteInfo( project_name=project_name, site_name=site_name, @@ -555,7 +539,7 @@ def _make_drift_correction_gif( parts = file.parts try: lamella_name = parts[parts.index("Sites") + 1] - lamella_number = _number_from_name(lamella_name) + lamella_number = number_from_name(lamella_name) except Exception: logger.warning( f"Could not extract metadata from file {file}", exc_info=True diff --git a/src/murfey/util/fib.py b/src/murfey/util/fib.py new file mode 100644 index 000000000..7a5efdadb --- /dev/null +++ b/src/murfey/util/fib.py @@ -0,0 +1,21 @@ +""" +General functinos specific to the FIB workflow +""" + +import re + + +def number_from_name(name: str) -> int: + """ + In the AutoTEM and Maps workflows for the FIB, the sites and images are + auto-incremented with parenthesised numbers (e.g. "Lamella (2)"), with + the first site/image typically not having a number. + + This function extracts the number from the file name, and returns 1 if + no such number is found. + """ + return ( + int(match.group(1)) + if (match := re.search(r"^[\w\s]+\((\d+)\)$", name)) is not None + else 1 + ) From 5e6b5be0f855e5c081ffa88eb99e2416619c6e46 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 20 May 2026 18:22:09 +0100 Subject: [PATCH 03/17] Adjust logic to register only the latest FIB atlas acquired for a FIB sample slot --- src/murfey/workflows/fib/register_atlas.py | 53 ++++++++++++++-------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/src/murfey/workflows/fib/register_atlas.py b/src/murfey/workflows/fib/register_atlas.py index ebed1a198..958197b35 100644 --- a/src/murfey/workflows/fib/register_atlas.py +++ b/src/murfey/workflows/fib/register_atlas.py @@ -9,6 +9,7 @@ from sqlmodel import Session, select import murfey.util.db as MurfeyDB +from murfey.util.fib import number_from_name logger = logging.getLogger("murfey.workflows.fib.register_atlas") @@ -151,34 +152,48 @@ def _register_fib_imaging_site( Register FIB atlas in Murfey database or update existing entry. """ # Create new entry if one doesn't already exist - if not ( + if ( fib_imaging_site := murfey_db.exec( select(MurfeyDB.ImagingSite) .where(MurfeyDB.ImagingSite.session_id == session_id) - .where(MurfeyDB.ImagingSite.image_path == str(metadata.file)) + .where(MurfeyDB.ImagingSite.site_name == metadata.site_name) + .where(MurfeyDB.ImagingSite.data_type == "atlas") ).one_or_none() - ): + ) is None: fib_imaging_site = MurfeyDB.ImagingSite( session_id=session_id, + site_name=metadata.site_name, image_path=str(metadata.file), data_type="atlas", ) - # Add/update entries - fib_imaging_site.site_name = metadata.site_name - fib_imaging_site.pos_x = metadata.pos_x - fib_imaging_site.pos_y = metadata.pos_y - fib_imaging_site.pos_z = metadata.pos_z - fib_imaging_site.rotation = float(np.rad2deg(metadata.rotation)) - fib_imaging_site.tilt_alpha = float(np.rad2deg(metadata.tilt_alpha)) - fib_imaging_site.tilt_beta = float(np.rad2deg(metadata.tilt_beta)) - fib_imaging_site.len_x = metadata.len_x - fib_imaging_site.len_y = metadata.len_y - fib_imaging_site.image_pixels_x = metadata.pixels_x - fib_imaging_site.image_pixels_y = metadata.pixels_y - fib_imaging_site.image_pixel_size = metadata.pixel_size - - murfey_db.add(fib_imaging_site) - murfey_db.commit() + + # Check if the entry is new or newer than the current stored one + incoming_number = number_from_name(metadata.file.stem) + # Handle empty string + if not fib_imaging_site.image_path: + current_number = 0 + # Read 'maps' atlases in one way + elif "maps" in (curr_path := Path(fib_imaging_site.image_path)).parts: + current_number = number_from_name(curr_path.stem) + else: + current_number = 0 + # Update if incoming one is newer + if incoming_number > current_number: + fib_imaging_site.image_path = str(metadata.file) + fib_imaging_site.pos_x = metadata.pos_x + fib_imaging_site.pos_y = metadata.pos_y + fib_imaging_site.pos_z = metadata.pos_z + fib_imaging_site.rotation = float(np.rad2deg(metadata.rotation)) + fib_imaging_site.tilt_alpha = float(np.rad2deg(metadata.tilt_alpha)) + fib_imaging_site.tilt_beta = float(np.rad2deg(metadata.tilt_beta)) + fib_imaging_site.len_x = metadata.len_x + fib_imaging_site.len_y = metadata.len_y + fib_imaging_site.image_pixels_x = metadata.pixels_x + fib_imaging_site.image_pixels_y = metadata.pixels_y + fib_imaging_site.image_pixel_size = metadata.pixel_size + + murfey_db.add(fib_imaging_site) + murfey_db.commit() def run( From ff3a3abe1dbde71cd1c392a29a9bf4b4a8642cc2 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 20 May 2026 18:32:41 +0100 Subject: [PATCH 04/17] Updated DB test to check that only one file was registered --- tests/workflows/fib/test_register_atlas.py | 68 +++++++++++++--------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/tests/workflows/fib/test_register_atlas.py b/tests/workflows/fib/test_register_atlas.py index 3b2600b51..2eecb6570 100644 --- a/tests/workflows/fib/test_register_atlas.py +++ b/tests/workflows/fib/test_register_atlas.py @@ -291,8 +291,10 @@ def test_run_with_db( visit_dir: Path, murfey_db_session: Session, ): - test_file = ( - visit_dir / "maps/LayersData/Layer/Electron Snapshot/Electron Snapshot.tiff" + test_files = ( + visit_dir / "maps/LayersData/Layer/Electron Snapshot/Electron Snapshot.tiff", + visit_dir + / "maps/LayersData/Layer/Electron Snapshot/Electron Snapshot (2).tiff", ) # Add a test visit to the database @@ -310,33 +312,45 @@ def test_run_with_db( murfey_db_session.commit() # Mock the metadata returned from the image file - mock_metadata = FIBAtlasMetadata( - visit_name=visit_name, - file=test_file, - voltage=2000, - shift_x=0, - shift_y=0, - len_x=0.003072, - len_y=0.002048, - pos_x=0.003, - pos_y=0.0003, - pos_z=0.01, - rotation=-1.309, - tilt_alpha=0.8, - tilt_beta=0, - pixels_x=3072, - pixels_y=2048, - pixel_size_x=1e-6, - pixel_size_y=1e-6, - ) + mock_metadata = [ + FIBAtlasMetadata( + visit_name=visit_name, + file=test_file, + voltage=2000, + shift_x=0, + shift_y=0, + len_x=0.003072, + len_y=0.002048, + pos_x=0.003, + pos_y=0.0003, + pos_z=0.01, + rotation=-1.309, + tilt_alpha=0.8, + tilt_beta=0, + pixels_x=3072, + pixels_y=2048, + pixel_size_x=1e-6, + pixel_size_y=1e-6, + ) + for test_file in test_files + ] mocker.patch( "murfey.workflows.fib.register_atlas._parse_metadata", - return_value=mock_metadata, + side_effect=mock_metadata, ) # Run the function and check that it's run through to completion - assert run( - session_id=session_id, - file=test_file, - murfey_db=murfey_db_session, - ) + for test_file in test_files: + run( + session_id=session_id, + file=test_file, + murfey_db=murfey_db_session, + ) + search_results = murfey_db_session.exec( + select(MurfeyDB.ImagingSite).where( + MurfeyDB.ImagingSite.session_id == session_id + ) + ).all() + + assert len(search_results) == 1 + assert search_results[0].image_path == str(mock_metadata[-1].file) From cb0a651a7ef31689b8a53dd07b49970152f2a2d2 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 21 May 2026 09:59:10 +0100 Subject: [PATCH 05/17] Migrated 'number_from_name' test to its own test module --- tests/client/contexts/test_fib.py | 19 ------------------- tests/util/test_fib_util.py | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 19 deletions(-) create mode 100644 tests/util/test_fib_util.py diff --git a/tests/client/contexts/test_fib.py b/tests/client/contexts/test_fib.py index f13c552f4..218a669c5 100644 --- a/tests/client/contexts/test_fib.py +++ b/tests/client/contexts/test_fib.py @@ -14,7 +14,6 @@ FIBImage, _file_transferred_to, _get_source, - _number_from_name, _parse_boolean, ) from murfey.util.models import LamellaSiteInfo @@ -392,24 +391,6 @@ def fib_maps_images(visit_dir: Path): # ------------------------------------------------------------------------------------- -@pytest.mark.parametrize( - "test_params", - ( # File name | Expected number - # AutoTEM examples - ("Lamella", 1), - ("Lamella (2)", 2), - ("Lamella (12)", 12), - # Maps examples - ("Electron Snapshot", 1), - ("Electron Snapshot (3)", 3), - ("Electron Snapshot (21)", 21), - ), -) -def test_number_from_name(test_params: tuple[str, int]): - name, number = test_params - assert _number_from_name(name) == number - - @pytest.mark.parametrize( "test_params", ( # Input | Expected output diff --git a/tests/util/test_fib_util.py b/tests/util/test_fib_util.py new file mode 100644 index 000000000..6a8d9a73b --- /dev/null +++ b/tests/util/test_fib_util.py @@ -0,0 +1,21 @@ +import pytest + +from murfey.util.fib import number_from_name + + +@pytest.mark.parametrize( + "test_params", + ( # File name | Expected number + # AutoTEM examples + ("Lamella", 1), + ("Lamella (2)", 2), + ("Lamella (12)", 12), + # Maps examples + ("Electron Snapshot", 1), + ("Electron Snapshot (3)", 3), + ("Electron Snapshot (21)", 21), + ), +) +def test_number_from_name(test_params: tuple[str, int]): + name, number = test_params + assert number_from_name(name) == number From 94d95a8a930faf14309a95f7a93a2ab40b4744f6 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 21 May 2026 11:35:27 +0100 Subject: [PATCH 06/17] Updated registratino logic so that 'add' and 'commit' always happen at the end of the function, and both new and old entries are populated or updated correctly --- src/murfey/workflows/fib/register_atlas.py | 66 +++++++++++++--------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/src/murfey/workflows/fib/register_atlas.py b/src/murfey/workflows/fib/register_atlas.py index 958197b35..f24360fdb 100644 --- a/src/murfey/workflows/fib/register_atlas.py +++ b/src/murfey/workflows/fib/register_atlas.py @@ -151,7 +151,26 @@ def _register_fib_imaging_site( """ Register FIB atlas in Murfey database or update existing entry. """ - # Create new entry if one doesn't already exist + + def _update_entry( + site: MurfeyDB.ImagingSite, + metadata: FIBAtlasMetadata, + ): + site.image_path = str(metadata.file) + site.pos_x = metadata.pos_x + site.pos_y = metadata.pos_y + site.pos_z = metadata.pos_z + site.rotation = float(np.rad2deg(metadata.rotation)) + site.tilt_alpha = float(np.rad2deg(metadata.tilt_alpha)) + site.tilt_beta = float(np.rad2deg(metadata.tilt_beta)) + site.len_x = metadata.len_x + site.len_y = metadata.len_y + site.image_pixels_x = metadata.pixels_x + site.image_pixels_y = metadata.pixels_y + site.image_pixel_size = metadata.pixel_size + + return site + if ( fib_imaging_site := murfey_db.exec( select(MurfeyDB.ImagingSite) @@ -160,40 +179,31 @@ def _register_fib_imaging_site( .where(MurfeyDB.ImagingSite.data_type == "atlas") ).one_or_none() ) is None: + # Create new entry if one doesn't already exist fib_imaging_site = MurfeyDB.ImagingSite( session_id=session_id, site_name=metadata.site_name, image_path=str(metadata.file), data_type="atlas", ) - - # Check if the entry is new or newer than the current stored one - incoming_number = number_from_name(metadata.file.stem) - # Handle empty string - if not fib_imaging_site.image_path: - current_number = 0 - # Read 'maps' atlases in one way - elif "maps" in (curr_path := Path(fib_imaging_site.image_path)).parts: - current_number = number_from_name(curr_path.stem) + fib_imaging_site = _update_entry(fib_imaging_site, metadata) else: - current_number = 0 - # Update if incoming one is newer - if incoming_number > current_number: - fib_imaging_site.image_path = str(metadata.file) - fib_imaging_site.pos_x = metadata.pos_x - fib_imaging_site.pos_y = metadata.pos_y - fib_imaging_site.pos_z = metadata.pos_z - fib_imaging_site.rotation = float(np.rad2deg(metadata.rotation)) - fib_imaging_site.tilt_alpha = float(np.rad2deg(metadata.tilt_alpha)) - fib_imaging_site.tilt_beta = float(np.rad2deg(metadata.tilt_beta)) - fib_imaging_site.len_x = metadata.len_x - fib_imaging_site.len_y = metadata.len_y - fib_imaging_site.image_pixels_x = metadata.pixels_x - fib_imaging_site.image_pixels_y = metadata.pixels_y - fib_imaging_site.image_pixel_size = metadata.pixel_size - - murfey_db.add(fib_imaging_site) - murfey_db.commit() + # Check if the entry is new or newer than the current stored one + incoming_number = number_from_name(metadata.file.stem) + # Handle empty string + if not fib_imaging_site.image_path: + current_number = 0 + # Read 'maps' atlases in one way + elif "maps" in (curr_path := Path(fib_imaging_site.image_path)).parts: + current_number = number_from_name(curr_path.stem) + else: + current_number = 0 + # Update if incoming one is newer + if incoming_number > current_number: + fib_imaging_site = _update_entry(fib_imaging_site, metadata) + + murfey_db.add(fib_imaging_site) + murfey_db.commit() def run( From 409f9a4a97871a22e7d1edafba972c37b4b47a27 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 21 May 2026 11:38:09 +0100 Subject: [PATCH 07/17] Fixed broken test --- tests/workflows/fib/test_register_atlas.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/workflows/fib/test_register_atlas.py b/tests/workflows/fib/test_register_atlas.py index 2eecb6570..44c69c28f 100644 --- a/tests/workflows/fib/test_register_atlas.py +++ b/tests/workflows/fib/test_register_atlas.py @@ -7,6 +7,7 @@ from sqlmodel import Session, select import murfey.util.db as MurfeyDB +import murfey.workflows.fib.register_atlas from murfey.workflows.fib.register_atlas import FIBAtlasMetadata, _parse_metadata, run session_id = 10 @@ -334,10 +335,14 @@ def test_run_with_db( ) for test_file in test_files ] - mocker.patch( + mock_parse = mocker.patch( "murfey.workflows.fib.register_atlas._parse_metadata", side_effect=mock_metadata, ) + spy_register = mocker.spy( + murfey.workflows.fib.register_atlas, + "_register_fib_imaging_site", + ) # Run the function and check that it's run through to completion for test_file in test_files: @@ -346,11 +351,13 @@ def test_run_with_db( file=test_file, murfey_db=murfey_db_session, ) + assert mock_parse.call_count == len(test_files) + assert spy_register.call_count == len(test_files) + search_results = murfey_db_session.exec( select(MurfeyDB.ImagingSite).where( MurfeyDB.ImagingSite.session_id == session_id ) ).all() - assert len(search_results) == 1 assert search_results[0].image_path == str(mock_metadata[-1].file) From fe1869231f5b4d66762a289c35bac4b46b244736 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 21 May 2026 17:32:44 +0100 Subject: [PATCH 08/17] Added logic to register data collection group and atlas rows in ISPyB using FIB atlas data --- src/murfey/workflows/fib/register_atlas.py | 153 ++++++++++++++++++--- 1 file changed, 132 insertions(+), 21 deletions(-) diff --git a/src/murfey/workflows/fib/register_atlas.py b/src/murfey/workflows/fib/register_atlas.py index f24360fdb..251b97616 100644 --- a/src/murfey/workflows/fib/register_atlas.py +++ b/src/murfey/workflows/fib/register_atlas.py @@ -1,6 +1,8 @@ import logging +import traceback import xml.etree.ElementTree as ET from functools import cached_property +from importlib.metadata import entry_points from pathlib import Path import numpy as np @@ -153,23 +155,23 @@ def _register_fib_imaging_site( """ def _update_entry( - site: MurfeyDB.ImagingSite, + imaging_site: MurfeyDB.ImagingSite, metadata: FIBAtlasMetadata, ): - site.image_path = str(metadata.file) - site.pos_x = metadata.pos_x - site.pos_y = metadata.pos_y - site.pos_z = metadata.pos_z - site.rotation = float(np.rad2deg(metadata.rotation)) - site.tilt_alpha = float(np.rad2deg(metadata.tilt_alpha)) - site.tilt_beta = float(np.rad2deg(metadata.tilt_beta)) - site.len_x = metadata.len_x - site.len_y = metadata.len_y - site.image_pixels_x = metadata.pixels_x - site.image_pixels_y = metadata.pixels_y - site.image_pixel_size = metadata.pixel_size - - return site + imaging_site.image_path = str(metadata.file) + imaging_site.pos_x = metadata.pos_x + imaging_site.pos_y = metadata.pos_y + imaging_site.pos_z = metadata.pos_z + imaging_site.rotation = float(np.rad2deg(metadata.rotation)) + imaging_site.tilt_alpha = float(np.rad2deg(metadata.tilt_alpha)) + imaging_site.tilt_beta = float(np.rad2deg(metadata.tilt_beta)) + imaging_site.len_x = metadata.len_x + imaging_site.len_y = metadata.len_y + imaging_site.image_pixels_x = metadata.pixels_x + imaging_site.image_pixels_y = metadata.pixels_y + imaging_site.image_pixel_size = metadata.pixel_size + + return imaging_site if ( fib_imaging_site := murfey_db.exec( @@ -205,6 +207,93 @@ def _update_entry( murfey_db.add(fib_imaging_site) murfey_db.commit() + return fib_imaging_site + + +def _register_dcg_and_atlas( + session_id: int, + instrument_name: str, + visit_name: str, + imaging_site: MurfeyDB.ImagingSite, + metadata: FIBAtlasMetadata, + murfey_db: Session, +): + proposal_code = "".join(char for char in visit_name.split("-")[0] if char.isalpha()) + proposal_number = "".join( + char for char in visit_name.split("-")[0] if char.isdigit() + ) + visit_number = visit_name.split("-")[-1] + + # Register using thumbnail values if they are provided + if ( + imaging_site.thumbnail_path is not None + and imaging_site.thumbnail_pixel_size is not None + ): + atlas_name: str | None = imaging_site.thumbnail_path + atlas_pixel_size: float | None = imaging_site.thumbnail_pixel_size + else: + atlas_name = imaging_site.image_path + atlas_pixel_size = imaging_site.image_pixel_size + + if dcg_search := murfey_db.exec( + select(MurfeyDB.DataCollectionGroup) + .where(MurfeyDB.DataCollectionGroup.session_id == session_id) + .where(MurfeyDB.DataCollectionGroup.tag == imaging_site.site_name) + ).all(): + dcg_entry = dcg_search[0] + atlas_message = { + "session_id": session_id, + "dcgid": dcg_entry.id, + "atlas_id": dcg_entry.atlas_id, + "atlas": atlas_name, + "atlas_pixel_size": atlas_pixel_size, + "sample": dcg_entry.sample, + } + if entry_point_result := entry_points( + group="murfey.workflows", name="atlas_update" + ): + (workflow,) = entry_point_result + _ = workflow.load()( + message=atlas_message, + murfey_db=murfey_db, + ) + else: + logger.warning("No workflow found for 'atlas_update'") + else: + dcg_message = { + "microscope": instrument_name, + "proposal_code": proposal_code, + "proposal_number": proposal_number, + "visit_number": visit_number, + "session_id": session_id, + "tag": imaging_site.site_name, + "experiment_type_id": 46, + "atlas": atlas_name, + "atlas_pixel_size": atlas_pixel_size, + "sample": metadata.slot_number, + } + if entry_point_result := entry_points( + group="murfey.workflows", name="data_collection_group" + ): + (workflow,) = entry_point_result + # Register grid square + _ = workflow.load()( + message=dcg_message, + murfey_db=murfey_db, + ) + else: + logger.warning("No workflow found for 'data_collection_group'") + dcg_entry = murfey_db.exec( + select(MurfeyDB.DataCollectionGroup) + .where(MurfeyDB.DataCollectionGroup.session_id == session_id) + .where(MurfeyDB.DataCollectionGroup.tag == imaging_site.site_name) + ).one() + + imaging_site.dcg_id = dcg_entry.id + imaging_site.dcg_name = dcg_entry.tag + murfey_db.add(imaging_site) + murfey_db.commit() + def run( session_id: int, @@ -213,28 +302,30 @@ def run( ): # Outer try-finally block to ensure database connection closes try: - # Load visit information try: - session_entry = murfey_db.exec( + # Load visit information + murfey_session = murfey_db.exec( select(MurfeyDB.Session).where(MurfeyDB.Session.id == session_id) ).one() - visit_name = session_entry.visit + visit_name = murfey_session.visit except Exception: logger.error( "Exception encountered while querying Murfey database", exc_info=True ) return False - # Extract metadata from Electron Snapshot image try: + # Extract metadata from Electron Snapshot image metadata = _parse_metadata(file, visit_name) except Exception: logger.error(f"Error extracting metadata from file {file}", exc_info=True) return False - # Register imaging site in Murfey, or update existing one try: - _register_fib_imaging_site(session_id, metadata, murfey_db) + # Register imaging site in Murfey, or update existing one + fib_imaging_site = _register_fib_imaging_site( + session_id, metadata, murfey_db + ) logger.info( f"Registered FIB atlas image {file} for slot {metadata.slot_number} in Murfey database" ) @@ -244,6 +335,26 @@ def run( exc_info=True, ) return False + + try: + # Register data collection group and atlas in ISPyB + _register_dcg_and_atlas( + session_id=session_id, + instrument_name=murfey_session.instrument_name, + visit_name=murfey_session.visit, + imaging_site=fib_imaging_site, + metadata=metadata, + murfey_db=murfey_db, + ) + except Exception: + # Log error but allow workflow to proceed + logger.error( + "Exception encountered when registering data collection group for FIB workflow " + f"for {metadata.site_name!r}: \n" + f"{traceback.format_exc()}" + ) + return True + finally: murfey_db.close() From 06d9cace7df9f8e9b67d36fe9f593ea6bbbbf3d3 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 21 May 2026 18:00:06 +0100 Subject: [PATCH 09/17] Added tests for the ISPyB database registration components of the workflow --- tests/workflows/fib/test_register_atlas.py | 83 ++++++++++++++++++++-- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/tests/workflows/fib/test_register_atlas.py b/tests/workflows/fib/test_register_atlas.py index 44c69c28f..1adbfc0a9 100644 --- a/tests/workflows/fib/test_register_atlas.py +++ b/tests/workflows/fib/test_register_atlas.py @@ -2,17 +2,21 @@ from pathlib import Path from unittest.mock import MagicMock +import ispyb.sqlalchemy as ISPyBDB import pytest from pytest_mock import MockerFixture -from sqlmodel import Session, select +from sqlalchemy import select as sa_select +from sqlalchemy.orm import Session as SQLAlchemySession +from sqlmodel import Session as SQLModelSession, select as sm_select import murfey.util.db as MurfeyDB import murfey.workflows.fib.register_atlas from murfey.workflows.fib.register_atlas import FIBAtlasMetadata, _parse_metadata, run +from tests.conftest import ExampleVisit session_id = 10 -visit_name = "cm12345-6" -instrument_name = "test_instrument" +visit_name = f"{ExampleVisit.proposal_code}{ExampleVisit.proposal_number}-{ExampleVisit.visit_number}" +instrument_name = ExampleVisit.instrument_name @pytest.fixture @@ -290,7 +294,9 @@ def test_register_fib_imaging_site(): def test_run_with_db( mocker: MockerFixture, visit_dir: Path, - murfey_db_session: Session, + murfey_db_session: SQLModelSession, + ispyb_db_session: SQLAlchemySession, + mock_ispyb_credentials, ): test_files = ( visit_dir / "maps/LayersData/Layer/Electron Snapshot/Electron Snapshot.tiff", @@ -301,7 +307,7 @@ def test_run_with_db( # Add a test visit to the database if not ( session_entry := murfey_db_session.exec( - select(MurfeyDB.Session).where(MurfeyDB.Session.id == session_id) + sm_select(MurfeyDB.Session).where(MurfeyDB.Session.id == session_id) ).one_or_none() ): session_entry = MurfeyDB.Session(id=session_id) @@ -312,6 +318,36 @@ def test_run_with_db( murfey_db_session.add(session_entry) murfey_db_session.commit() + # Mock the ISPyB connection where the TransportManager class is located + mock_security_config = MagicMock() + mock_security_config.ispyb_credentials = mock_ispyb_credentials + mocker.patch( + "murfey.server.ispyb.get_security_config", + return_value=mock_security_config, + ) + mocker.patch( + "murfey.server.ispyb.ISPyBSession", + return_value=ispyb_db_session, + ) + + # Mock the ISPYB connection when registering data collection group + mocker.patch( + "murfey.workflows.register_data_collection_group.ISPyBSession", + return_value=ispyb_db_session, + ) + + # Patch the TransportManager object in the workflows called + from murfey.server.ispyb import TransportManager + + mocker.patch( + "murfey.workflows.register_data_collection_group._transport_object", + new=TransportManager("PikaTransport"), + ) + mocker.patch( + "murfey.workflows.register_atlas_update._transport_object", + new=TransportManager("PikaTransport"), + ) + # Mock the metadata returned from the image file mock_metadata = [ FIBAtlasMetadata( @@ -354,10 +390,45 @@ def test_run_with_db( assert mock_parse.call_count == len(test_files) assert spy_register.call_count == len(test_files) + # Murfey's ImagingSite should have an entry search_results = murfey_db_session.exec( - select(MurfeyDB.ImagingSite).where( + sm_select(MurfeyDB.ImagingSite).where( MurfeyDB.ImagingSite.session_id == session_id ) ).all() assert len(search_results) == 1 assert search_results[0].image_path == str(mock_metadata[-1].file) + + # Murfey's DataCollectionGroup should have an entry + murfey_dcg_search = murfey_db_session.exec( + sm_select(MurfeyDB.DataCollectionGroup).where( + MurfeyDB.DataCollectionGroup.session_id == session_id + ) + ).all() + assert len(murfey_dcg_search) == 1 + + # ISPyB's DataCollectionGroup should have an entry + murfey_dcg = murfey_dcg_search[0] + ispyb_dcg_search = ( + ispyb_db_session.execute( + sa_select(ISPyBDB.DataCollectionGroup).where( + ISPyBDB.DataCollectionGroup.dataCollectionGroupId == murfey_dcg.id + ) + ) + .scalars() + .all() + ) + assert len(ispyb_dcg_search) == 1 + + # Atlas should have an entry + ispyb_dcg = ispyb_dcg_search[0] + ispyb_atlas_search = ( + ispyb_db_session.execute( + sa_select(ISPyBDB.Atlas).where( + ISPyBDB.Atlas.dataCollectionGroupId == ispyb_dcg.dataCollectionGroupId + ) + ) + .scalars() + .all() + ) + assert len(ispyb_atlas_search) == 1 From 89f34b0977513b881433667bd8d99fa8c7a76a59 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 22 May 2026 08:51:27 +0100 Subject: [PATCH 10/17] Module import can occur in the test function instead, since it's only needed for one function --- tests/workflows/fib/test_register_atlas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/workflows/fib/test_register_atlas.py b/tests/workflows/fib/test_register_atlas.py index 1adbfc0a9..27f0af52a 100644 --- a/tests/workflows/fib/test_register_atlas.py +++ b/tests/workflows/fib/test_register_atlas.py @@ -10,7 +10,6 @@ from sqlmodel import Session as SQLModelSession, select as sm_select import murfey.util.db as MurfeyDB -import murfey.workflows.fib.register_atlas from murfey.workflows.fib.register_atlas import FIBAtlasMetadata, _parse_metadata, run from tests.conftest import ExampleVisit @@ -349,6 +348,8 @@ def test_run_with_db( ) # Mock the metadata returned from the image file + import murfey.workflows.fib.register_atlas + mock_metadata = [ FIBAtlasMetadata( visit_name=visit_name, From 032ea3b345e1c897b64ad87fd52e9c5efee56a29 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 28 May 2026 16:26:48 +0100 Subject: [PATCH 11/17] Updated the FIB atlas registration workflow so that it is passed to and run through the RabbitMQ feedback queue instead of being loaded directly --- src/murfey/server/api/workflow_fib.py | 26 ++++++------ src/murfey/workflows/fib/register_atlas.py | 46 +++++++++++++++------- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/src/murfey/server/api/workflow_fib.py b/src/murfey/server/api/workflow_fib.py index f40629fa0..afe42e619 100644 --- a/src/murfey/server/api/workflow_fib.py +++ b/src/murfey/server/api/workflow_fib.py @@ -1,7 +1,6 @@ import json import logging import os -from importlib.metadata import entry_points from pathlib import Path import numpy as np @@ -11,6 +10,7 @@ from sqlmodel import Session, select import murfey.util.db as MurfeyDB +from murfey.server import _transport_object from murfey.server.api.auth import validate_instrument_token from murfey.server.murfey_db import murfey_db from murfey.util import sanitise_path @@ -36,20 +36,16 @@ def register_fib_atlas( fib_atlas_info: FIBAtlasInfo, db: Session = murfey_db, ): - # See if the relevant workflow is available - if not ( - workflow_search := list( - entry_points(group="murfey.workflows", name="fib.register_atlas") - ) - ): - raise RuntimeError("Unable to find Murfey workflow to register FIB atlas") - workflow = workflow_search[0] - - # Run the workflow - workflow.load()( - session_id=session_id, - file=fib_atlas_info.file, - murfey_db=db, + if _transport_object is None: + logger.error("No Transport Manager object was set up") + return None + _transport_object.send( + _transport_object.feedback_queue, + { + "register": "fib.register_atlas", + "session_id": session_id, + "atlas_file": str(fib_atlas_info.file), + }, ) diff --git a/src/murfey/workflows/fib/register_atlas.py b/src/murfey/workflows/fib/register_atlas.py index 251b97616..3e46b0b2b 100644 --- a/src/murfey/workflows/fib/register_atlas.py +++ b/src/murfey/workflows/fib/register_atlas.py @@ -4,6 +4,7 @@ from functools import cached_property from importlib.metadata import entry_points from pathlib import Path +from typing import Any import numpy as np import PIL.Image @@ -295,51 +296,68 @@ def _register_dcg_and_atlas( murfey_db.commit() +class FIBAtlasRegistrationInfo(BaseModel): + register: str + session_id: int + atlas_file: Path + + def run( - session_id: int, - file: Path, + message: dict[str, Any], murfey_db: Session, ): # Outer try-finally block to ensure database connection closes try: + try: + # Validate incoming message + fib_info = FIBAtlasRegistrationInfo(**message) + except Exception: + logger.error("Could not validate incoming message", exc_info=True) + return {"success": False, "requeue": False} + try: # Load visit information murfey_session = murfey_db.exec( - select(MurfeyDB.Session).where(MurfeyDB.Session.id == session_id) + select(MurfeyDB.Session).where( + MurfeyDB.Session.id == fib_info.session_id + ) ).one() visit_name = murfey_session.visit except Exception: logger.error( "Exception encountered while querying Murfey database", exc_info=True ) - return False + return {"success": False, "requeue": False} try: # Extract metadata from Electron Snapshot image - metadata = _parse_metadata(file, visit_name) + metadata = _parse_metadata(fib_info.atlas_file, visit_name) except Exception: - logger.error(f"Error extracting metadata from file {file}", exc_info=True) - return False + logger.error( + f"Error extracting metadata from file {fib_info.atlas_file}", + exc_info=True, + ) + return {"success": False, "requeue": False} try: # Register imaging site in Murfey, or update existing one fib_imaging_site = _register_fib_imaging_site( - session_id, metadata, murfey_db + fib_info.session_id, metadata, murfey_db ) logger.info( - f"Registered FIB atlas image {file} for slot {metadata.slot_number} in Murfey database" + f"Registered FIB atlas image {fib_info.atlas_file} for slot {metadata.slot_number} in Murfey database" ) except Exception: logger.error( - f"Error registering FIB atlas image {file} in Murfey database", + f"Error registering FIB atlas image {fib_info.atlas_file} in Murfey database", exc_info=True, ) - return False + return {"success": False, "requeue": False} try: # Register data collection group and atlas in ISPyB _register_dcg_and_atlas( - session_id=session_id, + session_id=fib_info.session_id, instrument_name=murfey_session.instrument_name, visit_name=murfey_session.visit, imaging_site=fib_imaging_site, @@ -353,8 +371,8 @@ def run( f"for {metadata.site_name!r}: \n" f"{traceback.format_exc()}" ) - - return True + return {"success": False, "requeue": True} + return {"success": True, "requeue": False} finally: murfey_db.close() From e0386173a55f868a9de53fdd78993699e9b6340a Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 28 May 2026 16:36:22 +0100 Subject: [PATCH 12/17] Updated Pydantic model name and removed DB dependency for 'register_fib_atlas' API endpoint --- src/murfey/server/api/workflow_fib.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/murfey/server/api/workflow_fib.py b/src/murfey/server/api/workflow_fib.py index afe42e619..577a11a4d 100644 --- a/src/murfey/server/api/workflow_fib.py +++ b/src/murfey/server/api/workflow_fib.py @@ -26,15 +26,14 @@ ) -class FIBAtlasInfo(BaseModel): - file: Path | None = None +class FIBAtlasFile(BaseModel): + file: Path @router.post("/sessions/{session_id}/register_atlas") def register_fib_atlas( session_id: int, - fib_atlas_info: FIBAtlasInfo, - db: Session = murfey_db, + fib_atlas: FIBAtlasFile, ): if _transport_object is None: logger.error("No Transport Manager object was set up") @@ -44,7 +43,7 @@ def register_fib_atlas( { "register": "fib.register_atlas", "session_id": session_id, - "atlas_file": str(fib_atlas_info.file), + "atlas_file": str(fib_atlas.file), }, ) From ec7cf48e29e961a4c29a109208dd6211eaa8cca5 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 28 May 2026 16:50:23 +0100 Subject: [PATCH 13/17] Updated test for 'register_fib_atlas' API endpoint --- tests/server/api/test_workflow_fib.py | 74 +++++++++++++++------------ 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/tests/server/api/test_workflow_fib.py b/tests/server/api/test_workflow_fib.py index 3afdb935f..28b37ae29 100644 --- a/tests/server/api/test_workflow_fib.py +++ b/tests/server/api/test_workflow_fib.py @@ -7,58 +7,64 @@ from pytest_mock import MockerFixture from murfey.server.api.workflow_fib import ( - FIBAtlasInfo, + FIBAtlasFile, FIBGIFParameters, make_gif, register_fib_atlas, ) +@pytest.mark.parametrize( + "has_transport_object", + ( + True, + False, + ), +) def test_register_fib_atlas( mocker: MockerFixture, tmp_path: Path, + has_transport_object: bool, ): - # Mock the databse instance - mock_db = MagicMock() - - # Patch out the entry point being called - mock_register_fib_atlas = mocker.patch("murfey.workflows.fib.register_atlas.run") - + # Set up the variables session_id = 1 - fib_atlas_info = FIBAtlasInfo(**{"file": str(tmp_path / "dummy")}) + fib_atlas = FIBAtlasFile(**{"file": str(tmp_path / "dummy")}) + + # Mock the logger + mock_logger = mocker.patch("murfey.server.api.workflow_fib.logger") + + # Mock the tranposrt object + if has_transport_object: + mock_transport_object = MagicMock() + mock_transport_object.feedback_queue = "dummy" + mocker.patch( + "murfey.server.api.workflow_fib._transport_object", + mock_transport_object, + ) + else: + mocker.patch( + "murfey.server.api.workflow_fib._transport_object", + None, + ) # Run the function and check that the expected calls were made register_fib_atlas( session_id=session_id, - fib_atlas_info=fib_atlas_info, - db=mock_db, - ) - mock_register_fib_atlas.assert_called_once_with( - session_id=session_id, - file=fib_atlas_info.file, - murfey_db=mock_db, + fib_atlas=fib_atlas, ) - -def test_register_fib_atlas_no_entry_point( - mocker: MockerFixture, - tmp_path: Path, -): - # Mock out entry_points to return an empty list - mocker.patch("murfey.server.api.workflow_fib.entry_points", return_value=[]) - - # Mock the databse instance - mock_db = MagicMock() - - fib_atlas_info = FIBAtlasInfo(**{"file": str(tmp_path / "dummy")}) - - # Patch out the entry point being called - with pytest.raises(RuntimeError): - register_fib_atlas( - session_id=1, - fib_atlas_info=fib_atlas_info, - db=mock_db, + # Check that the expected calls were made + if has_transport_object: + mock_transport_object.send.assert_called_with( + "dummy", + { + "register": "fib.register_atlas", + "session_id": session_id, + "atlas_file": str(fib_atlas.file), + }, ) + else: + mock_logger.error.assert_called_with("No Transport Manager object was set up") @pytest.mark.asyncio From c778394f786bec78dc1f70c15d5f3b65f8585d48 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 28 May 2026 16:51:16 +0100 Subject: [PATCH 14/17] Updated test for 'register_atlas' FIB workflow --- tests/workflows/fib/test_register_atlas.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/workflows/fib/test_register_atlas.py b/tests/workflows/fib/test_register_atlas.py index 27f0af52a..7714f6861 100644 --- a/tests/workflows/fib/test_register_atlas.py +++ b/tests/workflows/fib/test_register_atlas.py @@ -384,8 +384,11 @@ def test_run_with_db( # Run the function and check that it's run through to completion for test_file in test_files: run( - session_id=session_id, - file=test_file, + message={ + "register": "fib.register_atlas", + "session_id": session_id, + "atlas_file": str(test_file), + }, murfey_db=murfey_db_session, ) assert mock_parse.call_count == len(test_files) From 29e8668cf35f2fc984b1d93da82790820fc39603 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 29 May 2026 10:52:15 +0100 Subject: [PATCH 15/17] Add minimal logic needed to create, save, and register thumbnail images of the FIB atlases --- src/murfey/workflows/fib/register_atlas.py | 76 ++++++++++++++++++++-- 1 file changed, 69 insertions(+), 7 deletions(-) diff --git a/src/murfey/workflows/fib/register_atlas.py b/src/murfey/workflows/fib/register_atlas.py index 3e46b0b2b..0a028c6be 100644 --- a/src/murfey/workflows/fib/register_atlas.py +++ b/src/murfey/workflows/fib/register_atlas.py @@ -1,4 +1,5 @@ import logging +import os import traceback import xml.etree.ElementTree as ET from functools import cached_property @@ -12,6 +13,7 @@ from sqlmodel import Session, select import murfey.util.db as MurfeyDB +from murfey.util.config import get_machine_config from murfey.util.fib import number_from_name logger = logging.getLogger("murfey.workflows.fib.register_atlas") @@ -25,6 +27,7 @@ class FIBAtlasMetadata(BaseModel): visit_name: str file: Path + thumbnail_path: Path | None = None # Acceleration voltage voltage: float # Beam shifts @@ -79,16 +82,24 @@ def slot_number(self) -> int: # mypy doesn't support decorators on @property @computed_field # type: ignore @cached_property - def site_name(self) -> str: + def project_name(self) -> str: """ - Create a site name for the current image based on the project name - and its slot number. This assumes a specific folder structure of - {visit_name}/maps/{project_name} + Extract the project name from the file path. This assumes a specific + folder structure of '{visit_name}/maps/{project_name}'. """ path_parts = self.file.parts visit_idx = path_parts.index(self.visit_name) - project_name = path_parts[visit_idx + 2] # {visit}/maps/{project_name} - return f"{project_name}--slot_{self.slot_number}" + return path_parts[visit_idx + 2] # {visit}/maps/{project_name} + + # mypy doesn't support decorators on @property + @computed_field # type: ignore + @cached_property + def site_name(self) -> str: + """ + Create a site name for the current image based on the project name + and its slot number. + """ + return f"{self.project_name}--slot_{self.slot_number}" def _parse_metadata(file: Path, visit_name: str): @@ -146,6 +157,35 @@ def _parse_metadata(file: Path, visit_name: str): ) +def _make_thumbnail(file: Path, metadata: FIBAtlasMetadata, visit_name: str, mode: int): + img = PIL.Image.open(file) + img.thumbnail((512, 512)) + + # Find visit directory path + visit_idx = file.parts.index(visit_name) + visit_dir = list(reversed(file.parents))[visit_idx] + + # Construct processed directory and set permissions + processed_dir = visit_dir / "processed" + processed_dir.mkdir(exist_ok=True) + os.chmod(processed_dir, mode=mode) + + # Construct path to thumbnail + image_number = number_from_name(file.stem) + save_path = ( + processed_dir + / metadata.project_name + / f"grid_{metadata.slot_number}" + / "atlas" + / f"atlas_{str(image_number).zfill(2)}.png" + ) + save_path.parent.mkdir(parents=True, exist_ok=True) + + # Save the thumbnail + img.save(save_path) + return save_path + + def _register_fib_imaging_site( session_id: int, metadata: FIBAtlasMetadata, @@ -172,6 +212,13 @@ def _update_entry( imaging_site.image_pixels_y = metadata.pixels_y imaging_site.image_pixel_size = metadata.pixel_size + if metadata.thumbnail_path is not None: + scale = 512 / (max(metadata.pixels_x, metadata.pixels_y) or 1) + imaging_site.thumbnail_path = str(metadata.thumbnail_path) + imaging_site.thumbnail_pixels_x = int(round(metadata.pixels_x * scale)) or 1 + imaging_site.thumbnail_pixels_y = int(round(metadata.pixels_y * scale)) or 1 + imaging_site.thumbnail_pixel_size = metadata.pixel_size / scale + return imaging_site if ( @@ -202,7 +249,7 @@ def _update_entry( else: current_number = 0 # Update if incoming one is newer - if incoming_number > current_number: + if incoming_number >= current_number: fib_imaging_site = _update_entry(fib_imaging_site, metadata) murfey_db.add(fib_imaging_site) @@ -323,6 +370,8 @@ def run( ) ).one() visit_name = murfey_session.visit + instrument_name = murfey_session.instrument_name + machine_config = get_machine_config(instrument_name)[instrument_name] except Exception: logger.error( "Exception encountered while querying Murfey database", exc_info=True @@ -339,6 +388,19 @@ def run( ) return {"success": False, "requeue": False} + try: + # Make a thumbnail of the image and update metadata accordingly + metadata.thumbnail_path = _make_thumbnail( + file=metadata.file, + metadata=metadata, + visit_name=visit_name, + mode=machine_config.mkdir_chmod, + ) + except Exception: + logger.warning( + f"Error creating thumbnail of file {fib_info.atlas_file}", exc_info=True + ) + try: # Register imaging site in Murfey, or update existing one fib_imaging_site = _register_fib_imaging_site( From 6eb841655ee4eb8cbc7f6c543e020d03d07eb7fc Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 29 May 2026 11:06:53 +0100 Subject: [PATCH 16/17] Added minimal changes needed to test the '_make_thumbnail' function --- tests/workflows/fib/test_register_atlas.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/workflows/fib/test_register_atlas.py b/tests/workflows/fib/test_register_atlas.py index 7714f6861..455a62ad4 100644 --- a/tests/workflows/fib/test_register_atlas.py +++ b/tests/workflows/fib/test_register_atlas.py @@ -3,6 +3,8 @@ from unittest.mock import MagicMock import ispyb.sqlalchemy as ISPyBDB +import numpy as np +import PIL.Image import pytest from pytest_mock import MockerFixture from sqlalchemy import select as sa_select @@ -381,6 +383,12 @@ def test_run_with_db( "_register_fib_imaging_site", ) + # Mock 'PIL.Image.open' and create a test image + mocker.patch( + "murfey.workflows.fib.register_atlas.PIL.Image.open", + return_value=PIL.Image.fromarray(np.ones((2048, 1152), dtype=np.uint8)), + ) + # Run the function and check that it's run through to completion for test_file in test_files: run( @@ -436,3 +444,7 @@ def test_run_with_db( .all() ) assert len(ispyb_atlas_search) == 1 + ispyb_atlas_entry = ispyb_atlas_search[0] + assert ispyb_atlas_entry.atlasImage.endswith( + f"atlas_{str(mock_metadata[-1].slot_number).zfill(2)}.png" + ) From 6164a9d961b90d946a6d7697a151c8abdddacb86 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 29 May 2026 11:55:51 +0100 Subject: [PATCH 17/17] Do not manually chmod 'processed' directory; just inherit permissions as normal --- src/murfey/workflows/fib/register_atlas.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/murfey/workflows/fib/register_atlas.py b/src/murfey/workflows/fib/register_atlas.py index 0a028c6be..03df5dc19 100644 --- a/src/murfey/workflows/fib/register_atlas.py +++ b/src/murfey/workflows/fib/register_atlas.py @@ -1,5 +1,4 @@ import logging -import os import traceback import xml.etree.ElementTree as ET from functools import cached_property @@ -13,7 +12,6 @@ from sqlmodel import Session, select import murfey.util.db as MurfeyDB -from murfey.util.config import get_machine_config from murfey.util.fib import number_from_name logger = logging.getLogger("murfey.workflows.fib.register_atlas") @@ -157,7 +155,7 @@ def _parse_metadata(file: Path, visit_name: str): ) -def _make_thumbnail(file: Path, metadata: FIBAtlasMetadata, visit_name: str, mode: int): +def _make_thumbnail(file: Path, metadata: FIBAtlasMetadata, visit_name: str): img = PIL.Image.open(file) img.thumbnail((512, 512)) @@ -165,12 +163,8 @@ def _make_thumbnail(file: Path, metadata: FIBAtlasMetadata, visit_name: str, mod visit_idx = file.parts.index(visit_name) visit_dir = list(reversed(file.parents))[visit_idx] - # Construct processed directory and set permissions - processed_dir = visit_dir / "processed" - processed_dir.mkdir(exist_ok=True) - os.chmod(processed_dir, mode=mode) - # Construct path to thumbnail + processed_dir = visit_dir / "processed" image_number = number_from_name(file.stem) save_path = ( processed_dir @@ -370,8 +364,6 @@ def run( ) ).one() visit_name = murfey_session.visit - instrument_name = murfey_session.instrument_name - machine_config = get_machine_config(instrument_name)[instrument_name] except Exception: logger.error( "Exception encountered while querying Murfey database", exc_info=True @@ -394,7 +386,6 @@ def run( file=metadata.file, metadata=metadata, visit_name=visit_name, - mode=machine_config.mkdir_chmod, ) except Exception: logger.warning(