From a1086f3353e553442a9ff5a95ba230c320c562aa Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 29 Sep 2026 18:36:09 +0100 Subject: [PATCH 01/15] Added logic to register lamella images as GridSquare entries in ISPyB --- .../fib/register_lamella_evaluation_image.py | 203 +++++++++++++++++- 1 file changed, 198 insertions(+), 5 deletions(-) diff --git a/src/murfey/workflows/fib/register_lamella_evaluation_image.py b/src/murfey/workflows/fib/register_lamella_evaluation_image.py index ac3c7416d..20af9de44 100644 --- a/src/murfey/workflows/fib/register_lamella_evaluation_image.py +++ b/src/murfey/workflows/fib/register_lamella_evaluation_image.py @@ -1,5 +1,6 @@ import json import logging +import math import re from datetime import datetime from pathlib import Path @@ -9,9 +10,10 @@ from pydantic import BaseModel from sqlmodel import Session as SQLModelSession, select +import murfey.server import murfey.util.db as MurfeyDB from murfey.util.config import get_machine_config -from murfey.util.models import FIBImageMetadata +from murfey.util.models import FIBImageMetadata, GridSquareParameters from murfey.workflows.fib.shared import ( parse_image_metadata, populate_fib_imaging_site_entry, @@ -178,6 +180,185 @@ def _register_dcg( murfey_db.commit() +def _register_grid_square( + session_id: int, + imaging_site: MurfeyDB.ImagingSite, + site_number: int, + murfey_db: SQLModelSession, +): + """ + Helper function to create a GridSquare entry in ISPyB if one doesn't already + exist, and to link it to the corresopnding ImagingSite entry. + """ + # Early exits if values are missing/not configured + if murfey.server._transport_object is None: + raise RuntimeError("No TransportManager object was set up") + dcg_name = imaging_site.dcg_name + if dcg_name is None: + raise ValueError( + f"'dcg_name' field in ImagingSite entry for {imaging_site.image_path} is empty" + ) + + # Check if an atlas has been registered + atlas_search = murfey_db.exec( + select(MurfeyDB.ImagingSite) + .where(MurfeyDB.ImagingSite.session_id == session_id) + .where(MurfeyDB.ImagingSite.dcg_name == dcg_name) + .where(MurfeyDB.ImagingSite.data_type == "atlas") + .order_by(MurfeyDB.ImagingSite.id) # Sort in ascending insertion order + ).all() + if not atlas_search: + logger.info( + f"No atlas has been registered for data collection group {dcg_name!r} yet" + ) + return + atlas = atlas_search[-1] + + # Check if the atlas has the required values for the GridSquare registration + if not ( + atlas.pos_x is not None + and atlas.pos_y is not None + and atlas.pos_z is not None + and atlas.rotation is not None + and atlas.tilt_alpha is not None + and atlas.len_x is not None + and atlas.len_y is not None + and atlas.thumbnail_pixels_x is not None + and atlas.thumbnail_pixels_y is not None + ): + logger.warning(f"Atlas {atlas.image_path} not populated with required values") + return + atlas_x1 = atlas.pos_x + (atlas.len_x / 2) + atlas_y0 = atlas.pos_y - (atlas.len_y / 2) + + # Check that imaging site has the required values for registration + if not ( + imaging_site.pos_x is not None + and imaging_site.pos_y is not None + and imaging_site.pos_z is not None + and imaging_site.rotation is not None + and imaging_site.tilt_alpha is not None + and imaging_site.len_x is not None + and imaging_site.len_y is not None + ): + logger.warning( + f"ImagingSite for {imaging_site.image_path} not populated with required values" + ) + return + + # Transform the imaging site coordinates into the atlas' frame of reference + # NOTE: This will require further investigation and tweaking, given the + # many axes and centres of rotation present in the FIB stage system. + # We start with a simple 2D rotation for now, and will adjust it as we observe + # the alignment accuracy + theta = math.radians(imaging_site.rotation - atlas.rotation) + sin = math.sin(theta) + cos = math.cos(theta) + x_transformed = (imaging_site.pos_x * cos) - (imaging_site.pos_y * sin) + y_transformed = (imaging_site.pos_x * sin) + (imaging_site.pos_y * cos) + + # Find the pixel coordinates of the image on the atlas + # NOTE: On the atlas image, positive directions are LEFT (x) and DOWN (y) + x_mid_px = int( + round((atlas_x1 - x_transformed) / atlas.len_x * atlas.thumbnail_pixels_x) or 1 + ) + y_mid_px = int( + round((y_transformed - atlas_y0) / atlas.len_y * atlas.thumbnail_pixels_y) or 1 + ) + + # Find the pixel width and height of the lamella image on the atlas + width_scaled = int( + round((imaging_site.len_x / atlas.len_x) * atlas.thumbnail_pixels_x) or 1 + ) + height_scaled = int( + round((imaging_site.len_y / atlas.len_y) * atlas.thumbnail_pixels_y) or 1 + ) + + # Populate GridSquareParameters model + grid_square_params = GridSquareParameters( + tag=dcg_name, + x_location=x_transformed, + x_location_scaled=x_mid_px, + y_location=y_transformed, + y_location_scaled=y_mid_px, + readout_area_x=imaging_site.image_pixels_x, + readout_area_y=imaging_site.image_pixels_y, + thumbnail_size_x=imaging_site.thumbnail_pixels_x, + thumbnail_size_y=imaging_site.thumbnail_pixels_y, + width=imaging_site.image_pixels_x, + width_scaled=width_scaled, + height=imaging_site.image_pixels_y, + height_scaled=height_scaled, + x_stage_position=x_transformed, + y_stage_position=y_transformed, + pixel_size=imaging_site.image_pixel_size, + image=imaging_site.thumbnail_path, + ) + + # Register or update the grid square entry as required + if grid_square_entry := murfey_db.exec( + select(MurfeyDB.GridSquare) + .where(MurfeyDB.GridSquare.name == site_number) + .where(MurfeyDB.GridSquare.session_id == session_id) + .where(MurfeyDB.GridSquare.tag == grid_square_params.tag) + ).one_or_none(): + # Update existing grid square entry on Murfey + grid_square_entry.x_location = grid_square_params.x_location + grid_square_entry.y_location = grid_square_params.y_location + grid_square_entry.x_stage_position = grid_square_params.x_stage_position + grid_square_entry.y_stage_position = grid_square_params.y_stage_position + grid_square_entry.readout_area_x = grid_square_params.readout_area_x + grid_square_entry.readout_area_y = grid_square_params.readout_area_y + grid_square_entry.thumbnail_size_x = grid_square_params.thumbnail_size_x + grid_square_entry.thumbnail_size_y = grid_square_params.thumbnail_size_y + grid_square_entry.pixel_size = grid_square_params.pixel_size + grid_square_entry.image = grid_square_params.image + + # Update existing entry on ISPyB + murfey.server._transport_object.do_update_grid_square( + grid_square_id=grid_square_entry.id, + grid_square_parameters=grid_square_params, + ) + else: + # Look up data collection group for current series + dcg_entry = murfey_db.exec( + select(MurfeyDB.DataCollectionGroup) + .where(MurfeyDB.DataCollectionGroup.session_id == session_id) + .where(MurfeyDB.DataCollectionGroup.tag == grid_square_params.tag) + ).one() + # Register to ISPyB + grid_square_ispyb_result = ( + murfey.server._transport_object.do_insert_grid_square( + atlas_id=dcg_entry.atlas_id, + grid_square_id=site_number, + grid_square_parameters=grid_square_params, + ) + ) + # Create matching record in Murfey + grid_square_entry = MurfeyDB.GridSquare( + id=grid_square_ispyb_result.get("return_value", None), + name=site_number, + session_id=session_id, + tag=grid_square_params.tag, + x_location=grid_square_params.x_location, + y_location=grid_square_params.y_location, + x_stage_position=grid_square_params.x_stage_position, + y_stage_position=grid_square_params.y_stage_position, + readout_area_x=grid_square_params.readout_area_x, + readout_area_y=grid_square_params.readout_area_y, + thumbnail_size_x=grid_square_params.thumbnail_size_x, + thumbnail_size_y=grid_square_params.thumbnail_size_y, + pixel_size=grid_square_params.pixel_size, + image=grid_square_params.image, + ) + murfey_db.add(grid_square_entry) + + # Add grid square ID to existing CLEM image series entry + imaging_site.grid_square_id = grid_square_entry.id + murfey_db.add(imaging_site) + murfey_db.commit() + + class FIBLamellaImageInfo(BaseModel): session_id: int lamella_image_file: Path @@ -217,6 +398,10 @@ def run( rotation_offset=rotation_offset, ), ) + if metadata.lamella_number is None: + raise ValueError( + f"No lamella number associated with lamella image {fib_info.lamella_image_file}" + ) logger.info( "Extracted the following metadata from the image:\n" f"{json.dumps(metadata.model_dump(), indent=2, default=str)}" @@ -231,10 +416,6 @@ def run( # Register imaging site to Murfey, or update existing one fib_img_site = _register_fib_imaging_site(fib_info.session_id, metadata, murfey_db) - logger.info( - f"Registered lamella evaluation image {fib_info.lamella_image_file} " - f"for slot {metadata.slot_number} in Murfey database" - ) # Register data collection group and atlas in ISPyB _register_dcg( @@ -245,4 +426,16 @@ def run( murfey_db=murfey_db, ) + # Register grid square in ISPyB + _register_grid_square( + session_id=fib_info.session_id, + imaging_site=fib_img_site, + site_number=metadata.lamella_number, + murfey_db=murfey_db, + ) + + logger.info( + f"Registered lamella evaluation image {fib_info.lamella_image_file} " + f"for slot {metadata.slot_number} in Murfey database" + ) return {"success": True} From 885f9e4e6519d175ae47a14d896b6f6e5eef17e5 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 29 Sep 2026 18:46:56 +0100 Subject: [PATCH 02/15] Updated test to check for GridSquare registration in Murfey and ISPyB databases --- .../test_register_lamella_evaluation_image.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index 0d67c0fbe..3a1434c4e 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -355,3 +355,28 @@ def test_run_with_db( .all() ) assert len(ispyb_atlas_search) == 1 + + # ISPyB's GridSquare should have an entry + ispyb_atlas = ispyb_atlas_search[0] + ispyb_gs_search = ( + ispyb_db_session.execute( + sa_select(ISPyBDB.GridSquare).where( + ISPyBDB.GridSquare.atlasId == ispyb_atlas.atlasId + ) + ) + .scalars() + .all() + ) + assert len(ispyb_gs_search) == 1 + + # Murfey's GridSquare should also have an entry + murfey_gs_search = murfey_db_session.exec( + sm_select(MurfeyDB.GridSquare).where( + MurfeyDB.GridSquare.session_id == session_id + ) + ).all() + assert len(murfey_gs_search) == 1 + # Check that it's populated correctly + murfey_gs = murfey_gs_search[0] + assert murfey_gs.tag == f"{visit_name}/grid_2" + assert murfey_gs.name == 1 From d350e75b43f0ca35138c4ced9a27d9d0e6432279 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 14:27:11 +0100 Subject: [PATCH 03/15] Parametrise test and add logic to register a test atlas ImagingSite as needed --- .../test_register_lamella_evaluation_image.py | 155 ++++++++++++------ 1 file changed, 109 insertions(+), 46 deletions(-) diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index 3a1434c4e..c42bb71ef 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -14,8 +14,8 @@ import murfey.workflows.fib.register_lamella_evaluation_image from murfey.server.ispyb import TransportManager from murfey.util.config import MachineConfig +from murfey.util.models import FIBImageMetadata from murfey.workflows.fib.register_lamella_evaluation_image import ( - FIBImageMetadata, _register_fib_imaging_site, run, ) @@ -147,13 +147,28 @@ def test_register_fib_imaging_site_with_db( assert registered_site.image_path == str(file) +@pytest.mark.parametrize( + "test_params", + ( # Atlas registered | Slot number | Lamella number + (True, 1, 1), + (False, 2, 2), + ), +) def test_run_with_db( mocker: MockerFixture, + test_params: tuple[bool, int, int], visit_dir: Path, murfey_db_session: SQLModelSession, ispyb_db_session: SQLAlchemySession, mock_ispyb_credentials, ): + # Unpack test params + atlas_registered, slot_number, lamella_number = test_params + + # Construct expected tag and site name + expected_dcg_name = f"{visit_name}/grid_{slot_number}" + expected_site_name = f"{expected_dcg_name}/lamella_{lamella_number}" + # Register a Session for this test if not ( murfey_session := murfey_db_session.exec( @@ -166,8 +181,60 @@ def test_run_with_db( murfey_session.instrument_name = instrument_name murfey_db_session.add(murfey_session) + + # Create placeholder metadata dictionary + pixel_size = 1e-6 + metadata_dict = { + "voltage": 2000, + "shift_x": 0, + "shift_y": 0, + "len_x": 0.001500, + "len_y": 0.001000, + "pos_x": 0.003 * (-1 if slot_number > 1 else 1), + "pos_y": 0.003, + "pos_z": 0.01, + "rotation": 1.833, + "slot_number": slot_number, + "lamella_number": lamella_number, + "tilt_alpha": 0, + "tilt_beta": 0, + "pixels_x": 1500, + "pixels_y": 1000, + "pixel_size_x": pixel_size, + "pixel_size_y": pixel_size, + } + + # Create and populate an ImagingSite entry for the atlas if toggled + if atlas_registered: + atlas_metadata_dict = metadata_dict.copy() + atlas_metadata_dict["len_x"] = 0.002400 + atlas_metadata_dict["len_y"] = 0.001600 + atlas_metadata_dict["pixels_x"] = atlas_metadata_dict["len_x"] / pixel_size + atlas_metadata_dict["pixels_y"] = atlas_metadata_dict["len_y"] / pixel_size + + atlas_metadata = FIBImageMetadata( + visit_name=visit_name, + file=visit_dir / "some_file.tif", + **atlas_metadata_dict, + ) + + atlas_entry = MurfeyDB.ImagingSite( + session_id=session_id, + site_name=expected_dcg_name, + image_path=str(atlas_metadata.file), + data_type="atlas", + ) + populate_fib_imaging_site_entry(atlas_entry, atlas_metadata) + murfey_db_session.add(atlas_entry) + + # Commit all needed changes murfey_db_session.commit() + # Mock the logger + mock_logger = mocker.patch( + "murfey.workflows.fib.register_lamella_evaluation_image.logger" + ) + # Mock the machine config machine_config = MachineConfig( calibrations={ @@ -201,17 +268,24 @@ def test_run_with_db( ) # Create the test image files and their thumbnails + lamella_folder = "Lamella" + if lamella_number > 1: + lamella_folder += f" ({lamella_number})" raw_lamella_dir = ( visit_dir / "autotem" / visit_name / "Sites" - / "Lamella" + / lamella_folder / "LamellaEvaluationImages" ) raw_lamella_dir.mkdir(parents=True, exist_ok=True) processed_dir = ( - visit_dir / "processed" / visit_name / "grid_2" / "lamella_evaluation_images" + visit_dir + / "processed" + / visit_name + / f"grid_{slot_number}" + / "lamella_evaluation_images" ) processed_dir.mkdir(parents=True, exist_ok=True) @@ -227,30 +301,13 @@ def test_run_with_db( timestamp, step_name = file_name.split("_drift_corrected_image_") step_name = step_name.split(" - ")[0].replace(" ", "_").lower() - thumbnail = processed_dir / f"lamella_1_{timestamp}_{step_name}.png" + thumbnail = ( + processed_dir / f"lamella_{lamella_number}_{timestamp}_{step_name}.png" + ) thumbnail.touch() thumbnails.append(thumbnail) # Mock the expected metadata returns - metadata_dict = { - "voltage": 2000, - "shift_x": 0, - "shift_y": 0, - "len_x": 0.003072, - "len_y": 0.002048, - "pos_x": -0.003, - "pos_y": 0.003, - "pos_z": 0.01, - "rotation": 1.833, - "slot_number": 2, - "lamella_number": 1, - "tilt_alpha": 0, - "tilt_beta": 0, - "pixels_x": 1500, - "pixels_y": 1000, - "pixel_size_x": 1e-6, - "pixel_size_y": 1e-6, - } mock_parse = mocker.patch( "murfey.workflows.fib.register_lamella_evaluation_image.parse_image_metadata", return_value=metadata_dict, @@ -313,8 +370,8 @@ def test_run_with_db( ) # Site name should have been constructed correctly - assert imaging_site.site_name == f"{visit_name}/grid_2/lamella_1" - assert imaging_site.dcg_name == f"{visit_name}/grid_2" + assert imaging_site.dcg_name == expected_dcg_name + assert imaging_site.site_name == expected_site_name # Murfey's DataCollectionGroup should have an entry murfey_dcg_search = murfey_db_session.exec( @@ -326,7 +383,7 @@ def test_run_with_db( # Check that the Murfey DataCollectionGroup entry was populated correctly murfey_dcg = murfey_dcg_search[0] - assert murfey_dcg.tag == f"{visit_name}/grid_2" + assert murfey_dcg.tag == expected_dcg_name # ISPyB's DataCollectionGroup should have an entry ispyb_dcg_search = ( @@ -356,27 +413,33 @@ def test_run_with_db( ) assert len(ispyb_atlas_search) == 1 - # ISPyB's GridSquare should have an entry - ispyb_atlas = ispyb_atlas_search[0] - ispyb_gs_search = ( - ispyb_db_session.execute( - sa_select(ISPyBDB.GridSquare).where( - ISPyBDB.GridSquare.atlasId == ispyb_atlas.atlasId + # GridSquare should be registered if an atlas ImagingSite exists + if atlas_registered: + # ISPyB's GridSquare should have an entry + ispyb_atlas = ispyb_atlas_search[0] + ispyb_gs_search = ( + ispyb_db_session.execute( + sa_select(ISPyBDB.GridSquare).where( + ISPyBDB.GridSquare.atlasId == ispyb_atlas.atlasId + ) ) + .scalars() + .all() ) - .scalars() - .all() - ) - assert len(ispyb_gs_search) == 1 + assert len(ispyb_gs_search) == 1 - # Murfey's GridSquare should also have an entry - murfey_gs_search = murfey_db_session.exec( - sm_select(MurfeyDB.GridSquare).where( - MurfeyDB.GridSquare.session_id == session_id + # Murfey's GridSquare should also have an entry + murfey_gs_search = murfey_db_session.exec( + sm_select(MurfeyDB.GridSquare).where( + MurfeyDB.GridSquare.session_id == session_id + ) + ).all() + assert len(murfey_gs_search) == 1 + # Check that it's populated correctly + murfey_gs = murfey_gs_search[0] + assert murfey_gs.tag == expected_dcg_name + assert murfey_gs.name == 1 + else: + mock_logger.info.assert_any_call( + f"No atlas has been registered for data collection group {expected_dcg_name!r} yet" ) - ).all() - assert len(murfey_gs_search) == 1 - # Check that it's populated correctly - murfey_gs = murfey_gs_search[0] - assert murfey_gs.tag == f"{visit_name}/grid_2" - assert murfey_gs.name == 1 From 252528e5889f5dbf4b9c8b09aec9647904191909 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 14:44:40 +0100 Subject: [PATCH 04/15] Ensure ints --- tests/workflows/fib/test_register_lamella_evaluation_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index c42bb71ef..32f65db83 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -209,8 +209,8 @@ def test_run_with_db( atlas_metadata_dict = metadata_dict.copy() atlas_metadata_dict["len_x"] = 0.002400 atlas_metadata_dict["len_y"] = 0.001600 - atlas_metadata_dict["pixels_x"] = atlas_metadata_dict["len_x"] / pixel_size - atlas_metadata_dict["pixels_y"] = atlas_metadata_dict["len_y"] / pixel_size + atlas_metadata_dict["pixels_x"] = int(atlas_metadata_dict["len_x"] / pixel_size) + atlas_metadata_dict["pixels_y"] = int(atlas_metadata_dict["len_y"] / pixel_size) atlas_metadata = FIBImageMetadata( visit_name=visit_name, From a3c5fc1eaf87ba50b66438d19bc5c888290edaee Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 14:56:31 +0100 Subject: [PATCH 05/15] Placeholder atlas ImagingSite entry needs 'dcg_name' filled --- tests/workflows/fib/test_register_lamella_evaluation_image.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index 32f65db83..afccdcbed 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -222,6 +222,7 @@ def test_run_with_db( session_id=session_id, site_name=expected_dcg_name, image_path=str(atlas_metadata.file), + dcg_name=expected_dcg_name, data_type="atlas", ) populate_fib_imaging_site_entry(atlas_entry, atlas_metadata) From 47c98122cbfa22680571f23717f76bcfaaf01b48 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 15:23:54 +0100 Subject: [PATCH 06/15] Return updated ImagingSite entries from helper functions --- .../fib/register_lamella_evaluation_image.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/murfey/workflows/fib/register_lamella_evaluation_image.py b/src/murfey/workflows/fib/register_lamella_evaluation_image.py index 20af9de44..ed00a596a 100644 --- a/src/murfey/workflows/fib/register_lamella_evaluation_image.py +++ b/src/murfey/workflows/fib/register_lamella_evaluation_image.py @@ -179,6 +179,8 @@ def _register_dcg( murfey_db.add(imaging_site) murfey_db.commit() + return imaging_site + def _register_grid_square( session_id: int, @@ -211,7 +213,7 @@ def _register_grid_square( logger.info( f"No atlas has been registered for data collection group {dcg_name!r} yet" ) - return + return imaging_site atlas = atlas_search[-1] # Check if the atlas has the required values for the GridSquare registration @@ -227,7 +229,7 @@ def _register_grid_square( and atlas.thumbnail_pixels_y is not None ): logger.warning(f"Atlas {atlas.image_path} not populated with required values") - return + return imaging_site atlas_x1 = atlas.pos_x + (atlas.len_x / 2) atlas_y0 = atlas.pos_y - (atlas.len_y / 2) @@ -244,7 +246,7 @@ def _register_grid_square( logger.warning( f"ImagingSite for {imaging_site.image_path} not populated with required values" ) - return + return imaging_site # Transform the imaging site coordinates into the atlas' frame of reference # NOTE: This will require further investigation and tweaking, given the @@ -358,6 +360,8 @@ def _register_grid_square( murfey_db.add(imaging_site) murfey_db.commit() + return imaging_site + class FIBLamellaImageInfo(BaseModel): session_id: int @@ -418,7 +422,7 @@ def run( fib_img_site = _register_fib_imaging_site(fib_info.session_id, metadata, murfey_db) # Register data collection group and atlas in ISPyB - _register_dcg( + fib_img_site = _register_dcg( session_id=fib_info.session_id, instrument_name=instrument_name, visit_name=visit_name, @@ -427,7 +431,7 @@ def run( ) # Register grid square in ISPyB - _register_grid_square( + fib_img_site = _register_grid_square( session_id=fib_info.session_id, imaging_site=fib_img_site, site_number=metadata.lamella_number, From 6cfde2dd280bb405bdc5e736ed592a5762f6de4e Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 15:30:02 +0100 Subject: [PATCH 07/15] Verbose logging while tests are failing --- .../fib/register_lamella_evaluation_image.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/murfey/workflows/fib/register_lamella_evaluation_image.py b/src/murfey/workflows/fib/register_lamella_evaluation_image.py index ed00a596a..9584e0696 100644 --- a/src/murfey/workflows/fib/register_lamella_evaluation_image.py +++ b/src/murfey/workflows/fib/register_lamella_evaluation_image.py @@ -179,6 +179,9 @@ def _register_dcg( murfey_db.add(imaging_site) murfey_db.commit() + logger.info( + f"Return ImagingSite with values: {json.dumps(imaging_site.model_dump(), indent=2, default=str)}" + ) return imaging_site @@ -216,6 +219,10 @@ def _register_grid_square( return imaging_site atlas = atlas_search[-1] + logger.info( + f"Found atlas ImagingSite: {json.dumps(atlas.model_dump(), indent=2, default=str)}" + ) + # Check if the atlas has the required values for the GridSquare registration if not ( atlas.pos_x is not None @@ -316,6 +323,10 @@ def _register_grid_square( grid_square_entry.pixel_size = grid_square_params.pixel_size grid_square_entry.image = grid_square_params.image + logger.info( + f"Updated Murfey GridSquare entry: {json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" + ) + # Update existing entry on ISPyB murfey.server._transport_object.do_update_grid_square( grid_square_id=grid_square_entry.id, @@ -353,6 +364,9 @@ def _register_grid_square( pixel_size=grid_square_params.pixel_size, image=grid_square_params.image, ) + logger.info( + f"Creating new Murfey GridSquare entry: {json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" + ) murfey_db.add(grid_square_entry) # Add grid square ID to existing CLEM image series entry @@ -360,6 +374,9 @@ def _register_grid_square( murfey_db.add(imaging_site) murfey_db.commit() + logger.info( + f"Updated ImagingSite after GridSquare registration: {json.dumps(imaging_site.model_dump(), indent=2, default=str)}" + ) return imaging_site From bcc6cafd6c6bb99d3a121494942acf6ceb6ce71d Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 15:42:13 +0100 Subject: [PATCH 08/15] Updated log levels --- .../fib/register_lamella_evaluation_image.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/murfey/workflows/fib/register_lamella_evaluation_image.py b/src/murfey/workflows/fib/register_lamella_evaluation_image.py index 9584e0696..2767adaf9 100644 --- a/src/murfey/workflows/fib/register_lamella_evaluation_image.py +++ b/src/murfey/workflows/fib/register_lamella_evaluation_image.py @@ -179,8 +179,9 @@ def _register_dcg( murfey_db.add(imaging_site) murfey_db.commit() - logger.info( - f"Return ImagingSite with values: {json.dumps(imaging_site.model_dump(), indent=2, default=str)}" + logger.error( + "Returning updated ImagingSite with values after DCG registration: \n" + f"{json.dumps(imaging_site.model_dump(), indent=2, default=str)}" ) return imaging_site @@ -323,8 +324,9 @@ def _register_grid_square( grid_square_entry.pixel_size = grid_square_params.pixel_size grid_square_entry.image = grid_square_params.image - logger.info( - f"Updated Murfey GridSquare entry: {json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" + logger.error( + "Updated Murfey GridSquare entry:\n" + f"{json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" ) # Update existing entry on ISPyB @@ -364,8 +366,9 @@ def _register_grid_square( pixel_size=grid_square_params.pixel_size, image=grid_square_params.image, ) - logger.info( - f"Creating new Murfey GridSquare entry: {json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" + logger.error( + "Creating new Murfey GridSquare entry:\n" + f"{json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" ) murfey_db.add(grid_square_entry) @@ -374,8 +377,9 @@ def _register_grid_square( murfey_db.add(imaging_site) murfey_db.commit() - logger.info( - f"Updated ImagingSite after GridSquare registration: {json.dumps(imaging_site.model_dump(), indent=2, default=str)}" + logger.error( + "Updated ImagingSite after GridSquare registration:\n" + f"{json.dumps(imaging_site.model_dump(), indent=2, default=str)}" ) return imaging_site From 89059f767425d5beb3a50f5f89c164ac5bb1731b Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 15:53:30 +0100 Subject: [PATCH 09/15] Revert "Updated log levels" This reverts commit bcc6cafd6c6bb99d3a121494942acf6ceb6ce71d. --- .../fib/register_lamella_evaluation_image.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/murfey/workflows/fib/register_lamella_evaluation_image.py b/src/murfey/workflows/fib/register_lamella_evaluation_image.py index 2767adaf9..9584e0696 100644 --- a/src/murfey/workflows/fib/register_lamella_evaluation_image.py +++ b/src/murfey/workflows/fib/register_lamella_evaluation_image.py @@ -179,9 +179,8 @@ def _register_dcg( murfey_db.add(imaging_site) murfey_db.commit() - logger.error( - "Returning updated ImagingSite with values after DCG registration: \n" - f"{json.dumps(imaging_site.model_dump(), indent=2, default=str)}" + logger.info( + f"Return ImagingSite with values: {json.dumps(imaging_site.model_dump(), indent=2, default=str)}" ) return imaging_site @@ -324,9 +323,8 @@ def _register_grid_square( grid_square_entry.pixel_size = grid_square_params.pixel_size grid_square_entry.image = grid_square_params.image - logger.error( - "Updated Murfey GridSquare entry:\n" - f"{json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" + logger.info( + f"Updated Murfey GridSquare entry: {json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" ) # Update existing entry on ISPyB @@ -366,9 +364,8 @@ def _register_grid_square( pixel_size=grid_square_params.pixel_size, image=grid_square_params.image, ) - logger.error( - "Creating new Murfey GridSquare entry:\n" - f"{json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" + logger.info( + f"Creating new Murfey GridSquare entry: {json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" ) murfey_db.add(grid_square_entry) @@ -377,9 +374,8 @@ def _register_grid_square( murfey_db.add(imaging_site) murfey_db.commit() - logger.error( - "Updated ImagingSite after GridSquare registration:\n" - f"{json.dumps(imaging_site.model_dump(), indent=2, default=str)}" + logger.info( + f"Updated ImagingSite after GridSquare registration: {json.dumps(imaging_site.model_dump(), indent=2, default=str)}" ) return imaging_site From bbaf91830bb5d1d1e35ccc30001830d47f963edd Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 15:53:43 +0100 Subject: [PATCH 10/15] Revert "Verbose logging while tests are failing" This reverts commit 6cfde2dd280bb405bdc5e736ed592a5762f6de4e. --- .../fib/register_lamella_evaluation_image.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/murfey/workflows/fib/register_lamella_evaluation_image.py b/src/murfey/workflows/fib/register_lamella_evaluation_image.py index 9584e0696..ed00a596a 100644 --- a/src/murfey/workflows/fib/register_lamella_evaluation_image.py +++ b/src/murfey/workflows/fib/register_lamella_evaluation_image.py @@ -179,9 +179,6 @@ def _register_dcg( murfey_db.add(imaging_site) murfey_db.commit() - logger.info( - f"Return ImagingSite with values: {json.dumps(imaging_site.model_dump(), indent=2, default=str)}" - ) return imaging_site @@ -219,10 +216,6 @@ def _register_grid_square( return imaging_site atlas = atlas_search[-1] - logger.info( - f"Found atlas ImagingSite: {json.dumps(atlas.model_dump(), indent=2, default=str)}" - ) - # Check if the atlas has the required values for the GridSquare registration if not ( atlas.pos_x is not None @@ -323,10 +316,6 @@ def _register_grid_square( grid_square_entry.pixel_size = grid_square_params.pixel_size grid_square_entry.image = grid_square_params.image - logger.info( - f"Updated Murfey GridSquare entry: {json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" - ) - # Update existing entry on ISPyB murfey.server._transport_object.do_update_grid_square( grid_square_id=grid_square_entry.id, @@ -364,9 +353,6 @@ def _register_grid_square( pixel_size=grid_square_params.pixel_size, image=grid_square_params.image, ) - logger.info( - f"Creating new Murfey GridSquare entry: {json.dumps(grid_square_entry.model_dump(), indent=2, default=str)}" - ) murfey_db.add(grid_square_entry) # Add grid square ID to existing CLEM image series entry @@ -374,9 +360,6 @@ def _register_grid_square( murfey_db.add(imaging_site) murfey_db.commit() - logger.info( - f"Updated ImagingSite after GridSquare registration: {json.dumps(imaging_site.model_dump(), indent=2, default=str)}" - ) return imaging_site From 33fcdfee8a25e34a90e55a9b770a188e7eca31a7 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 15:57:52 +0100 Subject: [PATCH 11/15] Patch ISPyBSession at source --- .../fib/test_register_lamella_evaluation_image.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index afccdcbed..93b9dcf39 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -254,13 +254,7 @@ def test_run_with_db( ) mocker.patch( "murfey.server.ispyb.ISPyBSession", - return_value=ispyb_db_session, - ) - - # Mock the ISPYB connection when registering data collection group - mocker.patch( - "murfey.workflows.register_data_collection_group.ISPyBSession", - return_value=ispyb_db_session, + new=ispyb_db_session, ) # Patch the TransportManager object in the workflows called From 828538f4d01e1f133abc19d557793cbbcedfbf6e Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 16:07:00 +0100 Subject: [PATCH 12/15] Use 'return_value' instead of 'new' when patching --- tests/workflows/fib/test_register_lamella_evaluation_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index 93b9dcf39..c3fafef0d 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -254,7 +254,7 @@ def test_run_with_db( ) mocker.patch( "murfey.server.ispyb.ISPyBSession", - new=ispyb_db_session, + return_value=ispyb_db_session, ) # Patch the TransportManager object in the workflows called @@ -433,7 +433,7 @@ def test_run_with_db( # Check that it's populated correctly murfey_gs = murfey_gs_search[0] assert murfey_gs.tag == expected_dcg_name - assert murfey_gs.name == 1 + assert murfey_gs.name == lamella_number else: mock_logger.info.assert_any_call( f"No atlas has been registered for data collection group {expected_dcg_name!r} yet" From 63b9583928f38e5e0de258e6bce9eb4cd5ba5ee8 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 16:10:27 +0100 Subject: [PATCH 13/15] Revert "Use 'return_value' instead of 'new' when patching" This reverts commit 828538f4d01e1f133abc19d557793cbbcedfbf6e. --- tests/workflows/fib/test_register_lamella_evaluation_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index c3fafef0d..93b9dcf39 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -254,7 +254,7 @@ def test_run_with_db( ) mocker.patch( "murfey.server.ispyb.ISPyBSession", - return_value=ispyb_db_session, + new=ispyb_db_session, ) # Patch the TransportManager object in the workflows called @@ -433,7 +433,7 @@ def test_run_with_db( # Check that it's populated correctly murfey_gs = murfey_gs_search[0] assert murfey_gs.tag == expected_dcg_name - assert murfey_gs.name == lamella_number + assert murfey_gs.name == 1 else: mock_logger.info.assert_any_call( f"No atlas has been registered for data collection group {expected_dcg_name!r} yet" From 1f573ee366760953ba36427216d7f9232c0cc2b6 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 16:10:35 +0100 Subject: [PATCH 14/15] Revert "Patch ISPyBSession at source" This reverts commit 33fcdfee8a25e34a90e55a9b770a188e7eca31a7. --- .../fib/test_register_lamella_evaluation_image.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index 93b9dcf39..afccdcbed 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -254,7 +254,13 @@ def test_run_with_db( ) mocker.patch( "murfey.server.ispyb.ISPyBSession", - new=ispyb_db_session, + return_value=ispyb_db_session, + ) + + # Mock the ISPYB connection when registering data collection group + mocker.patch( + "murfey.workflows.register_data_collection_group.ISPyBSession", + return_value=ispyb_db_session, ) # Patch the TransportManager object in the workflows called From fa81e283bdd6d5a2ee454fd019ac6ca7c7cff8ff Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 30 Sep 2026 16:21:26 +0100 Subject: [PATCH 15/15] Placeholder thumbnail image needed for atlas ImagingSite entry --- .../workflows/fib/test_register_lamella_evaluation_image.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/workflows/fib/test_register_lamella_evaluation_image.py b/tests/workflows/fib/test_register_lamella_evaluation_image.py index afccdcbed..436dad6b5 100644 --- a/tests/workflows/fib/test_register_lamella_evaluation_image.py +++ b/tests/workflows/fib/test_register_lamella_evaluation_image.py @@ -194,10 +194,10 @@ def test_run_with_db( "pos_y": 0.003, "pos_z": 0.01, "rotation": 1.833, - "slot_number": slot_number, - "lamella_number": lamella_number, "tilt_alpha": 0, "tilt_beta": 0, + "slot_number": slot_number, + "lamella_number": lamella_number, "pixels_x": 1500, "pixels_y": 1000, "pixel_size_x": pixel_size, @@ -215,6 +215,7 @@ def test_run_with_db( atlas_metadata = FIBImageMetadata( visit_name=visit_name, file=visit_dir / "some_file.tif", + thumbnail_path=visit_dir / "some_thumbnail.png", **atlas_metadata_dict, )