Skip to content

Alignment done #121

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
daurer merged 4 commits into from
Feb 28, 2026
Merged

Alignment done #121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
30e2711
Added initial alignment step
daurer Feb 28, 2026
4d3419b
Merge branch 'main' into xrf_tomo
daurer Feb 28, 2026
f28f556
Changed kernelspec on the notebooks
daurer Feb 28, 2026
d374618
Added alignment step
daurer Feb 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions i14/notebooks/xrf_tomo_align_and_normalise.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,21 +418,12 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.10 - EPSIC [DLS Conda]",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "conda-env-DLS_Conda-epsic3.10-kernel.json"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
"name": ""
}
},
"nbformat": 4,
Expand Down
34 changes: 18 additions & 16 deletions i14/notebooks/xrf_tomo_align_initial.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"editable": true,
"slideshow": {
"slide_type": ""
},
Expand All @@ -91,7 +90,13 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"editable": true,
"slideshow": {
"slide_type": ""
},
"tags": []
},
"outputs": [],
"source": [
"assets_folder = pathlib.Path(\"./_assets\")\n",
Expand Down Expand Up @@ -291,12 +296,18 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"editable": true,
"slideshow": {
"slide_type": ""
},
"tags": []
},
"outputs": [],
"source": [
"maxv = np.max(to_align)\n",
"minv = np.min(to_align)\n",
"output_file = assets_folder / f\"{inpath_nexus.stem}_align_stack.gif\"\n",
"output_file = assets_folder / \"align_stack.gif\"\n",
"image = []\n",
"for i in range(len(to_align)):\n",
" data = to_align[i] - minv\n",
Expand Down Expand Up @@ -362,21 +373,12 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.10 - EPSIC [DLS Conda]",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "conda-env-DLS_Conda-epsic3.10-kernel.json"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
"name": ""
}
},
"nbformat": 4,
Expand Down
16 changes: 2 additions & 14 deletions i14/notebooks/xrf_tomo_plotting.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,21 +81,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.10 - EPSIC [DLS Conda]",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "conda-env-DLS_Conda-epsic3.10-kernel.json"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
"name": "python3"
}
},
"nbformat": 4,
Expand Down
160 changes: 138 additions & 22 deletions i14/templates/xrf-tomo-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ spec:
- name: outputFolder
value: "testing"
- name: multiScan
value: '[{"multiScan":{"start":423844,"end":423947,"excluded":[]}}]'
value: '[{"multiScan":{"start":423844,"end":423947,"excluded":"423908-423909,423911,423916-423917"}}]'
- name: multiEdge
value: '[{"edgeElement":"Pb","edgeTransition":"La"},{"edgeElement":"Br","edgeTransition":"Ka"}]'
- name: elementToAlign
Expand Down Expand Up @@ -64,12 +64,6 @@ spec:
echo '{{ .Files.Get "notebooks/xrf_tomo_align_initial.ipynb" | b64enc }}' | base64 -d > $OUTPUT/align_initial.ipynb
echo '{{ .Files.Get "notebooks/xrf_tomo_align_and_normalise.ipynb" | b64enc }}' | base64 -d > $OUTPUT/align_and_normalise.ipynb
echo '{{ .Files.Get "notebooks/xrf_tomo_plotting.ipynb" | b64enc }}' | base64 -d > $OUTPUT/plotting.ipynb
echo "{{`{{ workflow.parameters.multiScan }}`}}"
echo "{{`{{ workflow.parameters.multiEdge }}`}}"
echo "{{`{{ workflow.parameters.elementToAlign }}`}}"-"{{`{{ workflow.parameters.transitionToAlign }}`}}"
echo "{{`{{ workflow.parameters.alignmentSection }}`}}"
echo "{{`{{ workflow.parameters.alignmentBand }}`}}"
echo "{{`{{ workflow.parameters.normalise }}`}}"
volumes:
- name: session
hostPath:
Expand All @@ -79,7 +73,7 @@ spec:
- name: xrf-tomo-stack-projections
inputs:
parameters:
- name: edge
- name: element
- name: transition
script:
image: ghcr.io/diamondlightsource/nxstacker:latest
Expand All @@ -97,7 +91,7 @@ spec:

SCANS="{{`{{= sprig.join(',', map(sprig.mustFromJson(workflow.parameters.multiScan), { string(#.multiScan.start) + '-' + string(#.multiScan.end) } )) }}`}}"
EXCLUDE="{{`{{= sprig.join(',', filter(map(sprig.mustFromJson(workflow.parameters.multiScan), { #.multiScan.excluded } ), { len(#) > 0 })) }}`}}"
EDGE="{{`{{ inputs.parameters.edge }}`}}"
ELEMENT="{{`{{ inputs.parameters.element }}`}}"
TRANSITION="{{`{{ inputs.parameters.transition }}`}}"
VISIT="{{`{{ workflow.parameters.visitdir }}`}}"
OUTPUT="{{`{{ workflow.parameters.outputFolder }}`}}"
Expand All @@ -107,8 +101,9 @@ spec:
cmd+=("--proj-dir=$VISIT/processed")
[[ -n $SCANS ]] && cmd+=("--from-scan=$SCANS")
[[ -n $EXCLUDE ]] && cmd+=("--exclude-scan=$EXCLUDE")
cmd+=("--transition=$EDGE-$TRANSITION")
cmd+=("--transition=$ELEMENT-$TRANSITION")
cmd+=("--nxtomo-dir=$VISIT/processing/workflows/$OUTPUT")
cmd+=("--sort-by-angle")

echo "Executing: ${cmd[*]}"
exec "${cmd[@]}"
Expand All @@ -121,23 +116,144 @@ spec:

- name: xrf-tomo-generate-shifts
script:
image: busybox
command: ["/bin/sh", "-c"]
args:
- echo "{{`{{ workflow.parameters.elementToAlign }}`}}";
- echo "{{`{{ workflow.parameters.transitionToAlign }}`}}";
- echo "{{`{{ workflow.parameters.outputFolder }}`}}"
image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1
env:
- name: MPLCONFIGDIR
value: /tmp
command: [bash]
source: |
VISIT="{{`{{ workflow.parameters.visitdir }}`}}"
OUTPUT=$VISIT/processing/workflows/"{{`{{ workflow.parameters.outputFolder }}`}}"
EDGE="{{`{{ workflow.parameters.elementToAlign }}`}}"-"{{`{{ workflow.parameters.transitionToAlign }}`}}"
START="{{`{{= string(first(sprig.mustFromJson(workflow.parameters.multiScan)).multiScan['start']) }}`}}"
END="{{`{{= string(first(sprig.mustFromJson(workflow.parameters.multiScan)).multiScan['end']) }}`}}"

cd $OUTPUT
python -m papermill \
$OUTPUT/align_initial.ipynb $OUTPUT/align_initial_with_params.ipynb \
-p inpath "tomo_xrf_${START}_${END}_${EDGE}.nxs" \
-p outpath_nexus "${EDGE}_aligned_initial.nxs" \
-p outpath_shifts "${EDGE}_shifts.txt" \
-p alignment_section "{{`{{ workflow.parameters.alignmentSection }}`}}" \
-p alignment_band "{{`{{ workflow.parameters.alignmentBand }}`}}"

python -m jupyter nbconvert --to html --execute --output align_initial --output-dir $OUTPUT/_assets --embed-images $OUTPUT/align_initial_with_params.ipynb
volumeMounts:
- name: session
mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"
- name: tmp
mountPath: /tmp
outputs:
artifacts:
- name: align_initial
path: "{{`{{ workflow.parameters.visitdir }}`}}/processing/workflows/{{`{{ workflow.parameters.outputFolder }}`}}/_assets/align_initial.html"
archive:
none: {}
- name: aligned_stack
path: "{{`{{ workflow.parameters.visitdir }}`}}/processing/workflows/{{`{{ workflow.parameters.outputFolder }}`}}/_assets/align_stack.gif"
archive:
none: {}
podSpecPatch: |
containers:
- name: main
resources:
requests:
cpu: 1
memory: 10Gi
limits:
cpu: 1
memory: 10Gi
tolerations:
- key: nodegroup
operator: Equal
value: workflows
effect: NoSchedule
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: nodegroup
operator: In
values:
- workflows
volumes:
- name: session
hostPath:
path: "{{`{{ workflow.parameters.visitdir }}`}}"
type: Directory

- name: xrf-tomo-align-and-normalise
inputs:
parameters:
- name: edge
script:
image: busybox
command: ["/bin/sh", "-c"]
args:
- echo "{{`{{ inputs.parameters.edge }}`}}";
- echo "{{`{{ workflow.parameters.outputFolder }}`}}"
image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1
env:
- name: MPLCONFIGDIR
value: /tmp
command: [bash]
source: |
VISIT="{{`{{ workflow.parameters.visitdir }}`}}"
OUTPUT=$VISIT/processing/workflows/"{{`{{ workflow.parameters.outputFolder }}`}}"
ELEMENT="{{`{{= sprig.mustFromJson(inputs.parameters.edge).edgeElement }}`}}"
TRANSITION="{{`{{= sprig.mustFromJson(inputs.parameters.edge).edgeTransition }}`}}"
EDGE2ALIGN="{{`{{ workflow.parameters.elementToAlign }}`}}"-"{{`{{ workflow.parameters.transitionToAlign }}`}}"
START="{{`{{= string(first(sprig.mustFromJson(workflow.parameters.multiScan)).multiScan['start']) }}`}}"
END="{{`{{= string(first(sprig.mustFromJson(workflow.parameters.multiScan)).multiScan['end']) }}`}}"

cd $OUTPUT
python -m papermill \
$OUTPUT/align_and_normalise.ipynb $OUTPUT/align_and_normalise_with_params.ipynb \
-p inpath "tomo_xrf_${START}_${END}_${ELEMENT}-${TRANSITION}.nxs" \
-p inpath_shifts "${EDGE2ALIGN}_shifts.txt"
-p outpath_nexus "${ELEMENT}-${TRANSITION}_aligned.nxs" \
-p normalise "{{`{{ workflow.parameters.normalise }}`}}" \

python -m jupyter nbconvert --to html --execute --output align_and_normalise --output-dir $OUTPUT/_assets --embed-images $OUTPUT/align_and_normalise_with_params.ipynb
volumeMounts:
- name: session
mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"
- name: tmp
mountPath: /tmp
outputs:
artifacts:
- name: align_initial
path: "{{`{{ workflow.parameters.visitdir }}`}}/processing/workflows/{{`{{ workflow.parameters.outputFolder }}`}}/_assets/align_and_normalise.html"
archive:
none: {}
podSpecPatch: |
containers:
- name: main
resources:
requests:
cpu: 1
memory: 10Gi
limits:
cpu: 1
memory: 10Gi
tolerations:
- key: nodegroup
operator: Equal
value: workflows
effect: NoSchedule
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: nodegroup
operator: In
values:
- workflows
volumes:
- name: session
hostPath:
path: "{{`{{ workflow.parameters.visitdir }}`}}"
type: Directory


- name: xrf-tomo-reconstruct
inputs:
Expand Down Expand Up @@ -192,7 +308,7 @@ spec:
dependencies: [setup]
arguments:
parameters:
- name: edge
- name: element
value: "{{`{{ item.edgeElement }}`}}"
- name: transition
value: "{{`{{ item.edgeTransition }}`}}"
Expand Down