From ad436f26b4360bbdb27c4bd73059e5484dae4da6 Mon Sep 17 00:00:00 2001 From: frederick Date: Thu, 1 Oct 2026 14:51:14 +0100 Subject: [PATCH 1/4] adding code to actually attempt create the artifiacts from the ptyrex-recon --- e02/templates/ptyrex_basic.yaml | 110 +++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 10 deletions(-) diff --git a/e02/templates/ptyrex_basic.yaml b/e02/templates/ptyrex_basic.yaml index 7ddce2d..cc2107b 100644 --- a/e02/templates/ptyrex_basic.yaml +++ b/e02/templates/ptyrex_basic.yaml @@ -124,23 +124,113 @@ spec: value: "{{`{{workflow.parameters.config_json}}`}}" outputs: artifacts: - - name: ptyrex-output - path: /tmp/ptyrex_recon.tif + - name: ptyrex-object-modulus + path: /tmp/object_modulus.jpeg + - name: ptyrex-object-phase + path: /tmp/object_phase.jpeg + - name: ptyrex-probe-modulus + path: /tmp/probe_modulus.jpeg + - name: ptyrex-probe-phase + path: /tmp/probe_phase.jpeg archive: none: {} script: - image: ghcr.io/diamondlightsource/httomo:latest + image: gitlab.diamond.ac.uk:5050/scisoft/ptychography/dimtools/mib2x + imagePullPolicy: Always + env: + - name: BLOSC_NTHREADS + value: "{{ `{{inputs.parameters.nprocs}}` }}" + - name: OMP_NUM_THREADS + value: "{{ `{{inputs.parameters.nprocs}}` }}" + - name: M2X_DEBUG + value: "{{ `{{inputs.parameters.DEBUG}}` }}" volumeMounts: - - name: session - mountPath: "{{ `{{workflow.parameters.visitdir}}` }}" - - name: tmpdir - mountPath: /tmp - command: [/opt/conda/bin/python] + - name: tmpdir + mountPath: /tmp + - name: session + mountPath: "{{ `{{workflow.parameters.visitdir}}` }}" + command: [python] source: | - #Aim: find the ptyrex_recon file and move a copy of object phase to tmp + #Aim: find the ptyrex_recon file and then load corresponding hdf file and + #create a jpeg of object/probe phase/modulus and display as artifacts + import json + import h5py + import glob + import numpy + import numpy as np + from PIL import Image print('###start of log###') - print("{{`{{inputs.parameters.config_json}}`}}") + print('json path: %s' % (test_path)) print('###end of log###') + + #find the output path of the recon from loading the json used in the recon + json_path = test_path + with open(json_path,'r') as f: + json_data = json.load(f) + outdir = json_data['process']['save_dir'] + prefix = json_data['process']['save_prefix'] + if outdir == '': + outdir = '/'.join(json_path.split('/')[:-1]) + '/' + + #search dir for the most recent reconstruction note this might be incorrect + #when multiple reconstructions are occuring at once + search_string = outdir + prefix +'**.hdf' + hdf_list = glob.glob(search_string) + hdf_list.sort() + chosen_recon = hdf_list[-1] + print(f'saving this recon to jpeg: {chosen_recon}...') + + #load obj and probe + with h5py.File(chosen_recon,'r') as f: + obj = np.squeeze(f['/entry_1/process_1/output_1/object'][()]) + probe = np.squeeze(f['/entry_1/process_1/output_1/probe'][()]) + + #check probe shape and slice and reshape if needed + print(f'probe shape: {probe.shape}') + if len(probe.shape) == 3: + probe = np.reshape(probe,(probe.shape[0]*probe.shape[1],probe.shape[2])).T + elif len(probe.shape) == 4: + probe = probe[:,0,:,:] + probe = np.reshape(probe,(probe.shape[0]*probe.shape[1],probe.shape[2])).T + + #convert unsigned 8 for jpeg format + obj_abs = np.abs(obj) - np.amin(np.abs(obj)) + obj_abs = np.uint8(255*obj_abs/np.amax(obj_abs)) + obj_ang = np.angle(obj) - np.amin(np.angle(obj)) + obj_ang = np.uint8(255*obj_ang/np.amax(obj_ang)) + + probe_abs = np.abs(probe) - np.amin(np.abs(probe)) + probe_abs = np.uint8(255*probe_abs/np.amax(probe_abs)) + probe_ang = np.angle(probe) - np.amin(np.angle(probe)) + probe_ang = np.uint8(255*probe_ang/np.amax(probe_ang)) + + #determine outpaths + obj_modulus_path = '/tmp/object_modulus.jpeg' + obj_phase_path = '/tmp/object_phase.jpeg' + probe_modulus_path = '/tmp/probe_modulus.jpeg' + probe_phase_path = '/tmp/probe_phase.jpeg' + + #save probe and object (save and modulus) as JPEG's + obj_mj = Image.fromarray(obj_abs) + obj_mj.save(obj_modulus_path) + obj_pj = Image.fromarray(obj_ang) + obj_pj.save(obj_phase_path) + probe_mj = Image.fromarray(probe_abs) + probe_mj.save(probe_modulus_path) + probe_pj = Image.fromarray(probe_ang) + probe_pj.save(probe_phase_path) + + + podSpecPatch: | + containers: + - name: main + resources: + requests: + cpu: "{{ `{{inputs.parameters.nprocs}}` }}" + memory: "{{ `{{inputs.parameters.memory}}` }}" + limits: + cpu: "{{ `{{inputs.parameters.nprocs}}` }}" + memory: "{{ `{{inputs.parameters.memory}}` }}" From 543e9818003aab84830750f33cbfb1cae1f8959a Mon Sep 17 00:00:00 2001 From: frederick Date: Thu, 1 Oct 2026 14:59:05 +0100 Subject: [PATCH 2/4] setting number threads to one --- e02/templates/ptyrex_basic.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/e02/templates/ptyrex_basic.yaml b/e02/templates/ptyrex_basic.yaml index cc2107b..10e8254 100644 --- a/e02/templates/ptyrex_basic.yaml +++ b/e02/templates/ptyrex_basic.yaml @@ -139,11 +139,9 @@ spec: imagePullPolicy: Always env: - name: BLOSC_NTHREADS - value: "{{ `{{inputs.parameters.nprocs}}` }}" + value: 1 - name: OMP_NUM_THREADS - value: "{{ `{{inputs.parameters.nprocs}}` }}" - - name: M2X_DEBUG - value: "{{ `{{inputs.parameters.DEBUG}}` }}" + value: 1 volumeMounts: - name: tmpdir mountPath: /tmp @@ -220,7 +218,6 @@ spec: probe_pj = Image.fromarray(probe_ang) probe_pj.save(probe_phase_path) - podSpecPatch: | containers: - name: main From c63c9444a71791971c1a88ae6f9d9e96f72df981 Mon Sep 17 00:00:00 2001 From: frederick Date: Thu, 1 Oct 2026 15:04:16 +0100 Subject: [PATCH 3/4] fixing error by making 1 into a string --- e02/notebooks/Ptycho_calibration.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e02/notebooks/Ptycho_calibration.ipynb b/e02/notebooks/Ptycho_calibration.ipynb index 034df4e..6ea0bbe 100644 --- a/e02/notebooks/Ptycho_calibration.ipynb +++ b/e02/notebooks/Ptycho_calibration.ipynb @@ -415,9 +415,9 @@ ], "metadata": { "kernelspec": { - "display_name": "", + "display_name": "ePSIC 3.13 [User - epsic3.13]", "language": "python", - "name": "" + "name": "conda-env-User_-_epsic3.13-epsic-py313" }, "language_info": { "codemirror_mode": { @@ -429,7 +429,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.13.12" } }, "nbformat": 4, From f4ad977adfa856a6e7ec143861d1a70c9ce3367c Mon Sep 17 00:00:00 2001 From: frederick Date: Thu, 1 Oct 2026 15:31:01 +0100 Subject: [PATCH 4/4] sorting out env of the recon-artifact step --- e02/templates/ptyrex_basic.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/e02/templates/ptyrex_basic.yaml b/e02/templates/ptyrex_basic.yaml index 10e8254..0ff3228 100644 --- a/e02/templates/ptyrex_basic.yaml +++ b/e02/templates/ptyrex_basic.yaml @@ -122,6 +122,10 @@ spec: parameters: - name: config_json value: "{{`{{workflow.parameters.config_json}}`}}" + - name: memory + value: "{{`{{ workflow.parameters.memory }}`}}" + - name: nprocs + value: "{{`{{workflow.parameters.nprocs}}`}}" outputs: artifacts: - name: ptyrex-object-modulus @@ -139,9 +143,9 @@ spec: imagePullPolicy: Always env: - name: BLOSC_NTHREADS - value: 1 + value: "{{ `{{inputs.parameters.nprocs}}` }}" - name: OMP_NUM_THREADS - value: 1 + value: "{{ `{{inputs.parameters.nprocs}}` }}" volumeMounts: - name: tmpdir mountPath: /tmp