diff --git a/i14/notebooks/xrf_tomo_align_and_normalise.ipynb b/i14/notebooks/xrf_tomo_align_and_normalise.ipynb index d5fa8c7..db68971 100644 --- a/i14/notebooks/xrf_tomo_align_and_normalise.ipynb +++ b/i14/notebooks/xrf_tomo_align_and_normalise.ipynb @@ -418,21 +418,12 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - EPSIC [DLS Conda]", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "conda-env-DLS_Conda-epsic3.10-kernel.json" + "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" + "name": "" } }, "nbformat": 4, diff --git a/i14/notebooks/xrf_tomo_align_initial.ipynb b/i14/notebooks/xrf_tomo_align_initial.ipynb index 2e139a0..14d9897 100644 --- a/i14/notebooks/xrf_tomo_align_initial.ipynb +++ b/i14/notebooks/xrf_tomo_align_initial.ipynb @@ -65,7 +65,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "editable": true, "slideshow": { "slide_type": "" }, @@ -91,7 +90,13 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [], "source": [ "assets_folder = pathlib.Path(\"./_assets\")\n", @@ -291,12 +296,18 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [], "source": [ "maxv = np.max(to_align)\n", "minv = np.min(to_align)\n", - "output_file = assets_folder / f\"{inpath_nexus.stem}_align_stack.gif\"\n", + "output_file = assets_folder / \"align_stack.gif\"\n", "image = []\n", "for i in range(len(to_align)):\n", " data = to_align[i] - minv\n", @@ -362,21 +373,12 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - EPSIC [DLS Conda]", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "conda-env-DLS_Conda-epsic3.10-kernel.json" + "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" + "name": "" } }, "nbformat": 4, diff --git a/i14/notebooks/xrf_tomo_plotting.ipynb b/i14/notebooks/xrf_tomo_plotting.ipynb index c22da03..a121839 100644 --- a/i14/notebooks/xrf_tomo_plotting.ipynb +++ b/i14/notebooks/xrf_tomo_plotting.ipynb @@ -81,21 +81,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - EPSIC [DLS Conda]", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "conda-env-DLS_Conda-epsic3.10-kernel.json" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" + "name": "python3" } }, "nbformat": 4, diff --git a/i14/templates/xrf-tomo-template.yaml b/i14/templates/xrf-tomo-template.yaml index d4402f9..96f950a 100644 --- a/i14/templates/xrf-tomo-template.yaml +++ b/i14/templates/xrf-tomo-template.yaml @@ -25,7 +25,7 @@ spec: - name: outputFolder value: "testing" - name: multiScan - value: '[{"multiScan":{"start":423844,"end":423947,"excluded":[]}}]' + value: '[{"multiScan":{"start":423844,"end":423947,"excluded":"423908-423909,423911,423916-423917"}}]' - name: multiEdge value: '[{"edgeElement":"Pb","edgeTransition":"La"},{"edgeElement":"Br","edgeTransition":"Ka"}]' - name: elementToAlign @@ -64,12 +64,6 @@ spec: echo '{{ .Files.Get "notebooks/xrf_tomo_align_initial.ipynb" | b64enc }}' | base64 -d > $OUTPUT/align_initial.ipynb echo '{{ .Files.Get "notebooks/xrf_tomo_align_and_normalise.ipynb" | b64enc }}' | base64 -d > $OUTPUT/align_and_normalise.ipynb echo '{{ .Files.Get "notebooks/xrf_tomo_plotting.ipynb" | b64enc }}' | base64 -d > $OUTPUT/plotting.ipynb - echo "{{`{{ workflow.parameters.multiScan }}`}}" - echo "{{`{{ workflow.parameters.multiEdge }}`}}" - echo "{{`{{ workflow.parameters.elementToAlign }}`}}"-"{{`{{ workflow.parameters.transitionToAlign }}`}}" - echo "{{`{{ workflow.parameters.alignmentSection }}`}}" - echo "{{`{{ workflow.parameters.alignmentBand }}`}}" - echo "{{`{{ workflow.parameters.normalise }}`}}" volumes: - name: session hostPath: @@ -79,7 +73,7 @@ spec: - name: xrf-tomo-stack-projections inputs: parameters: - - name: edge + - name: element - name: transition script: image: ghcr.io/diamondlightsource/nxstacker:latest @@ -97,7 +91,7 @@ spec: SCANS="{{`{{= sprig.join(',', map(sprig.mustFromJson(workflow.parameters.multiScan), { string(#.multiScan.start) + '-' + string(#.multiScan.end) } )) }}`}}" EXCLUDE="{{`{{= sprig.join(',', filter(map(sprig.mustFromJson(workflow.parameters.multiScan), { #.multiScan.excluded } ), { len(#) > 0 })) }}`}}" - EDGE="{{`{{ inputs.parameters.edge }}`}}" + ELEMENT="{{`{{ inputs.parameters.element }}`}}" TRANSITION="{{`{{ inputs.parameters.transition }}`}}" VISIT="{{`{{ workflow.parameters.visitdir }}`}}" OUTPUT="{{`{{ workflow.parameters.outputFolder }}`}}" @@ -107,8 +101,9 @@ spec: cmd+=("--proj-dir=$VISIT/processed") [[ -n $SCANS ]] && cmd+=("--from-scan=$SCANS") [[ -n $EXCLUDE ]] && cmd+=("--exclude-scan=$EXCLUDE") - cmd+=("--transition=$EDGE-$TRANSITION") + cmd+=("--transition=$ELEMENT-$TRANSITION") cmd+=("--nxtomo-dir=$VISIT/processing/workflows/$OUTPUT") + cmd+=("--sort-by-angle") echo "Executing: ${cmd[*]}" exec "${cmd[@]}" @@ -121,23 +116,144 @@ spec: - name: xrf-tomo-generate-shifts script: - image: busybox - command: ["/bin/sh", "-c"] - args: - - echo "{{`{{ workflow.parameters.elementToAlign }}`}}"; - - echo "{{`{{ workflow.parameters.transitionToAlign }}`}}"; - - echo "{{`{{ workflow.parameters.outputFolder }}`}}" + image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1 + env: + - name: MPLCONFIGDIR + value: /tmp + command: [bash] + source: | + VISIT="{{`{{ workflow.parameters.visitdir }}`}}" + OUTPUT=$VISIT/processing/workflows/"{{`{{ workflow.parameters.outputFolder }}`}}" + EDGE="{{`{{ workflow.parameters.elementToAlign }}`}}"-"{{`{{ workflow.parameters.transitionToAlign }}`}}" + START="{{`{{= string(first(sprig.mustFromJson(workflow.parameters.multiScan)).multiScan['start']) }}`}}" + END="{{`{{= string(first(sprig.mustFromJson(workflow.parameters.multiScan)).multiScan['end']) }}`}}" + + cd $OUTPUT + python -m papermill \ + $OUTPUT/align_initial.ipynb $OUTPUT/align_initial_with_params.ipynb \ + -p inpath "tomo_xrf_${START}_${END}_${EDGE}.nxs" \ + -p outpath_nexus "${EDGE}_aligned_initial.nxs" \ + -p outpath_shifts "${EDGE}_shifts.txt" \ + -p alignment_section "{{`{{ workflow.parameters.alignmentSection }}`}}" \ + -p alignment_band "{{`{{ workflow.parameters.alignmentBand }}`}}" + + python -m jupyter nbconvert --to html --execute --output align_initial --output-dir $OUTPUT/_assets --embed-images $OUTPUT/align_initial_with_params.ipynb + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmp + mountPath: /tmp + outputs: + artifacts: + - name: align_initial + path: "{{`{{ workflow.parameters.visitdir }}`}}/processing/workflows/{{`{{ workflow.parameters.outputFolder }}`}}/_assets/align_initial.html" + archive: + none: {} + - name: aligned_stack + path: "{{`{{ workflow.parameters.visitdir }}`}}/processing/workflows/{{`{{ workflow.parameters.outputFolder }}`}}/_assets/align_stack.gif" + archive: + none: {} + podSpecPatch: | + containers: + - name: main + resources: + requests: + cpu: 1 + memory: 10Gi + limits: + cpu: 1 + memory: 10Gi + tolerations: + - key: nodegroup + operator: Equal + value: workflows + effect: NoSchedule + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: nodegroup + operator: In + values: + - workflows + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory - name: xrf-tomo-align-and-normalise inputs: parameters: - name: edge script: - image: busybox - command: ["/bin/sh", "-c"] - args: - - echo "{{`{{ inputs.parameters.edge }}`}}"; - - echo "{{`{{ workflow.parameters.outputFolder }}`}}" + image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1 + env: + - name: MPLCONFIGDIR + value: /tmp + command: [bash] + source: | + VISIT="{{`{{ workflow.parameters.visitdir }}`}}" + OUTPUT=$VISIT/processing/workflows/"{{`{{ workflow.parameters.outputFolder }}`}}" + ELEMENT="{{`{{= sprig.mustFromJson(inputs.parameters.edge).edgeElement }}`}}" + TRANSITION="{{`{{= sprig.mustFromJson(inputs.parameters.edge).edgeTransition }}`}}" + EDGE2ALIGN="{{`{{ workflow.parameters.elementToAlign }}`}}"-"{{`{{ workflow.parameters.transitionToAlign }}`}}" + START="{{`{{= string(first(sprig.mustFromJson(workflow.parameters.multiScan)).multiScan['start']) }}`}}" + END="{{`{{= string(first(sprig.mustFromJson(workflow.parameters.multiScan)).multiScan['end']) }}`}}" + + cd $OUTPUT + python -m papermill \ + $OUTPUT/align_and_normalise.ipynb $OUTPUT/align_and_normalise_with_params.ipynb \ + -p inpath "tomo_xrf_${START}_${END}_${ELEMENT}-${TRANSITION}.nxs" \ + -p inpath_shifts "${EDGE2ALIGN}_shifts.txt" + -p outpath_nexus "${ELEMENT}-${TRANSITION}_aligned.nxs" \ + -p normalise "{{`{{ workflow.parameters.normalise }}`}}" \ + + python -m jupyter nbconvert --to html --execute --output align_and_normalise --output-dir $OUTPUT/_assets --embed-images $OUTPUT/align_and_normalise_with_params.ipynb + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmp + mountPath: /tmp + outputs: + artifacts: + - name: align_initial + path: "{{`{{ workflow.parameters.visitdir }}`}}/processing/workflows/{{`{{ workflow.parameters.outputFolder }}`}}/_assets/align_and_normalise.html" + archive: + none: {} + podSpecPatch: | + containers: + - name: main + resources: + requests: + cpu: 1 + memory: 10Gi + limits: + cpu: 1 + memory: 10Gi + tolerations: + - key: nodegroup + operator: Equal + value: workflows + effect: NoSchedule + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: nodegroup + operator: In + values: + - workflows + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory + - name: xrf-tomo-reconstruct inputs: @@ -192,7 +308,7 @@ spec: dependencies: [setup] arguments: parameters: - - name: edge + - name: element value: "{{`{{ item.edgeElement }}`}}" - name: transition value: "{{`{{ item.edgeTransition }}`}}"