From 2ed4770930737e69324703049f7953669dcd89f8 Mon Sep 17 00:00:00 2001 From: Thomas Wilder Date: Wed, 19 Nov 2025 17:43:39 +0200 Subject: [PATCH 1/6] First commits: a blank recipe but named for purpose; a copy of my_little_diagnostic.py into diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py --- .../southern_ocean/diagnostic_acc_sigma_tr.py | 137 ++++++++++++++++++ esmvaltool/recipes/recipe_so_acc_sigma_tr.yml | 0 2 files changed, 137 insertions(+) create mode 100644 esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py create mode 100644 esmvaltool/recipes/recipe_so_acc_sigma_tr.yml diff --git a/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py b/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py new file mode 100644 index 0000000000..32dcaa0f91 --- /dev/null +++ b/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py @@ -0,0 +1,137 @@ +""" +Look at this module for guidance how to write your own. + +Read the README_PERSONAL_DIAGNOSTIC file associated with this example; + +Module for personal diagnostics (example). +Internal imports from exmvaltool work e.g.: + +from esmvalcore.preprocessor import regrid +from esmvaltool.diag_scripts.shared.supermeans import get_supermean + +Pipe output through logger; + +Please consult the documentation for help with esmvaltool's functionalities +and best coding practices. +""" +# place your module imports here: + +# operating system manipulations (e.g. path constructions) +import os + +# to manipulate iris cubes +import iris +import matplotlib.pyplot as plt +from esmvalcore.preprocessor import area_statistics + +# import internal esmvaltool modules here +from esmvaltool.diag_scripts.shared import group_metadata, run_diagnostic + + +def _plot_time_series(cfg, cube, dataset): + """ + Example of personal diagnostic plotting function. + + Arguments: + cfg - nested dictionary of metadata + cube - the cube to plot + dataset - name of the dataset to plot + + Returns: + string; makes some time-series plots + + Note: this function is private; remove the '_' + so you can make it public. + """ + # custom local paths for e.g. plots are supported - + # here is an example + # root_dir = '/group_workspaces/jasmin2/cmip6_prep/' # edit as per need + # out_path = 'esmvaltool_users/valeriu/' # edit as per need + # local_path = os.path.join(root_dir, out_path) + # but one can use the already defined esmvaltool output paths + local_path = cfg["plot_dir"] + + # do the plotting dance + plt.plot(cube.data, label=dataset) + plt.xlabel("Time (months)") + plt.ylabel("Area average") + plt.title("Time series at (ground level - first level)") + plt.tight_layout() + plt.grid() + plt.legend() + png_name = "Time_series_" + dataset + ".png" + plt.savefig(os.path.join(local_path, png_name)) + plt.close() + + # no need to brag :) + return "I made some plots!" + + +def run_my_diagnostic(cfg): + """ + Simple example of a diagnostic. + + This is a basic (and rather esotherical) diagnostic that firstly + loads the needed model data as iris cubes, performs a difference between + values at ground level and first vertical level, then squares the + result. + + Before plotting, we grab the squared result (not all operations on cubes) + and apply an area average on it. This is a useful example of how to use + standard esmvalcore.preprocessor functionality within a diagnostic, and + especially after a certain (custom) diagnostic has been run and the user + needs to perform an operation that is already part of the preprocessor + standard library of functions. + + The user will implement their own (custom) diagnostics, but this + example shows that once the preprocessor has finished a whole lot of + user-specific metrics can be computed as part of the diagnostic, + and then plotted in various manners. + + Arguments: + cfg - nested dictionary of metadata + + Returns: + string; runs the user diagnostic + + """ + # assemble the data dictionary keyed by dataset name + # this makes use of the handy group_metadata function that + # orders the data by 'dataset'; the resulting dictionary is + # keyed on datasets e.g. dict = {'MPI-ESM-LR': [var1, var2...]} + # where var1, var2 are dicts holding all needed information per variable + my_files_dict = group_metadata(cfg["input_data"].values(), "dataset") + + # iterate over key(dataset) and values(list of vars) + for key, value in my_files_dict.items(): + # load the cube from data files only + # using a single variable here so just grab the first (and only) + # list element + cube = iris.load_cube(value[0]["filename"]) + + # the first data analysis bit: simple cube difference: + # perform a difference between ground and first levels + diff_cube = cube[:, 0, :, :] - cube[:, 1, :, :] + # square the difference'd cube just for fun + squared_cube = diff_cube**2.0 + + # the second data analysis bit (slightly more advanced): + # compute an area average over the squared cube + # to apply the area average use a preprocessor function + # rather than writing your own function + area_avg_cube = area_statistics(squared_cube, "mean") + + # finalize your analysis by plotting a time series of the + # diffed, squared and area averaged cube; call the plot function: + _plot_time_series(cfg, area_avg_cube, key) + + # that's it, we're done! + return "I am done with my first ESMValTool diagnostic!" + + +if __name__ == "__main__": + # always use run_diagnostic() to get the config (the preprocessor + # nested dictionary holding all the needed information) + with run_diagnostic() as config: + # list here the functions that need to run + run_my_diagnostic(config) diff --git a/esmvaltool/recipes/recipe_so_acc_sigma_tr.yml b/esmvaltool/recipes/recipe_so_acc_sigma_tr.yml new file mode 100644 index 0000000000..e69de29bb2 From ad5b9453fdc35fba8db0175a1482819083496769 Mon Sep 17 00:00:00 2001 From: Thomas Wilder Date: Tue, 25 Nov 2025 15:56:16 +0200 Subject: [PATCH 2/6] added some modules and defined some functions --- .../southern_ocean/diagnostic_acc_sigma_tr.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py b/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py index 32dcaa0f91..8162a91187 100644 --- a/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py +++ b/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py @@ -15,6 +15,11 @@ and best coding practices. """ # place your module imports here: +import gsw +import xarray as xr +import numpy as np + +import cmocean # operating system manipulations (e.g. path constructions) import os @@ -27,7 +32,25 @@ # import internal esmvaltool modules here from esmvaltool.diag_scripts.shared import group_metadata, run_diagnostic +# my functions +def _plot_transect(): + + pass + +def _compute_sigma(): + pass + +def _extract_transect(): + + pass + +def main(): + pass + + + +# example functions def _plot_time_series(cfg, cube, dataset): """ Example of personal diagnostic plotting function. From 959033823915cf7a640e52f631db198c06f72fa7 Mon Sep 17 00:00:00 2001 From: Thomas Wilder Date: Thu, 27 Nov 2025 18:50:06 +0200 Subject: [PATCH 3/6] added code to def_main. --- .../southern_ocean/diagnostic_acc_sigma_tr.py | 219 ++++++++++-------- 1 file changed, 117 insertions(+), 102 deletions(-) diff --git a/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py b/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py index 8162a91187..f9239bd09c 100644 --- a/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py +++ b/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py @@ -21,6 +21,8 @@ import cmocean +import logging + # operating system manipulations (e.g. path constructions) import os @@ -32,6 +34,10 @@ # import internal esmvaltool modules here from esmvaltool.diag_scripts.shared import group_metadata, run_diagnostic +# This part sends debug statements to stdout +logger = logging.getLogger(os.path.basename(__file__)) +logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) + # my functions def _plot_transect(): @@ -44,112 +50,121 @@ def _extract_transect(): pass -def main(): +def main(cfg): + + # assemble the data dictionary keyed by dataset name + input_data = cfg["input_data"].values() + my_files_dict = group_metadata(input_data, "dataset") + + # load xarray dataset + ds = xr.open_dataset(value[0]["filename"]) + + pass -# example functions -def _plot_time_series(cfg, cube, dataset): - """ - Example of personal diagnostic plotting function. - - Arguments: - cfg - nested dictionary of metadata - cube - the cube to plot - dataset - name of the dataset to plot - - Returns: - string; makes some time-series plots - - Note: this function is private; remove the '_' - so you can make it public. - """ - # custom local paths for e.g. plots are supported - - # here is an example - # root_dir = '/group_workspaces/jasmin2/cmip6_prep/' # edit as per need - # out_path = 'esmvaltool_users/valeriu/' # edit as per need - # local_path = os.path.join(root_dir, out_path) - # but one can use the already defined esmvaltool output paths - local_path = cfg["plot_dir"] - - # do the plotting dance - plt.plot(cube.data, label=dataset) - plt.xlabel("Time (months)") - plt.ylabel("Area average") - plt.title("Time series at (ground level - first level)") - plt.tight_layout() - plt.grid() - plt.legend() - png_name = "Time_series_" + dataset + ".png" - plt.savefig(os.path.join(local_path, png_name)) - plt.close() - - # no need to brag :) - return "I made some plots!" - - -def run_my_diagnostic(cfg): - """ - Simple example of a diagnostic. - - This is a basic (and rather esotherical) diagnostic that firstly - loads the needed model data as iris cubes, performs a difference between - values at ground level and first vertical level, then squares the - result. - - Before plotting, we grab the squared result (not all operations on cubes) - and apply an area average on it. This is a useful example of how to use - standard esmvalcore.preprocessor functionality within a diagnostic, and - especially after a certain (custom) diagnostic has been run and the user - needs to perform an operation that is already part of the preprocessor - standard library of functions. - - The user will implement their own (custom) diagnostics, but this - example shows that once the preprocessor has finished a whole lot of - user-specific metrics can be computed as part of the diagnostic, - and then plotted in various manners. - - Arguments: - cfg - nested dictionary of metadata - - Returns: - string; runs the user diagnostic - - """ - # assemble the data dictionary keyed by dataset name - # this makes use of the handy group_metadata function that - # orders the data by 'dataset'; the resulting dictionary is - # keyed on datasets e.g. dict = {'MPI-ESM-LR': [var1, var2...]} - # where var1, var2 are dicts holding all needed information per variable - my_files_dict = group_metadata(cfg["input_data"].values(), "dataset") - - # iterate over key(dataset) and values(list of vars) - for key, value in my_files_dict.items(): - # load the cube from data files only - # using a single variable here so just grab the first (and only) - # list element - cube = iris.load_cube(value[0]["filename"]) - - # the first data analysis bit: simple cube difference: - # perform a difference between ground and first levels - diff_cube = cube[:, 0, :, :] - cube[:, 1, :, :] - # square the difference'd cube just for fun - squared_cube = diff_cube**2.0 - - # the second data analysis bit (slightly more advanced): - # compute an area average over the squared cube - # to apply the area average use a preprocessor function - # rather than writing your own function - area_avg_cube = area_statistics(squared_cube, "mean") - - # finalize your analysis by plotting a time series of the - # diffed, squared and area averaged cube; call the plot function: - _plot_time_series(cfg, area_avg_cube, key) - - # that's it, we're done! - return "I am done with my first ESMValTool diagnostic!" +# # example functions +# def _plot_time_series(cfg, cube, dataset): +# """ +# Example of personal diagnostic plotting function. + +# Arguments: +# cfg - nested dictionary of metadata +# cube - the cube to plot +# dataset - name of the dataset to plot + +# Returns: +# string; makes some time-series plots + +# Note: this function is private; remove the '_' +# so you can make it public. +# """ +# # custom local paths for e.g. plots are supported - +# # here is an example +# # root_dir = '/group_workspaces/jasmin2/cmip6_prep/' # edit as per need +# # out_path = 'esmvaltool_users/valeriu/' # edit as per need +# # local_path = os.path.join(root_dir, out_path) +# # but one can use the already defined esmvaltool output paths +# local_path = cfg["plot_dir"] + +# # do the plotting dance +# plt.plot(cube.data, label=dataset) +# plt.xlabel("Time (months)") +# plt.ylabel("Area average") +# plt.title("Time series at (ground level - first level)") +# plt.tight_layout() +# plt.grid() +# plt.legend() +# png_name = "Time_series_" + dataset + ".png" +# plt.savefig(os.path.join(local_path, png_name)) +# plt.close() + +# # no need to brag :) +# return "I made some plots!" + + +# def run_my_diagnostic(cfg): +# """ +# Simple example of a diagnostic. + +# This is a basic (and rather esotherical) diagnostic that firstly +# loads the needed model data as iris cubes, performs a difference between +# values at ground level and first vertical level, then squares the +# result. + +# Before plotting, we grab the squared result (not all operations on cubes) +# and apply an area average on it. This is a useful example of how to use +# standard esmvalcore.preprocessor functionality within a diagnostic, and +# especially after a certain (custom) diagnostic has been run and the user +# needs to perform an operation that is already part of the preprocessor +# standard library of functions. + +# The user will implement their own (custom) diagnostics, but this +# example shows that once the preprocessor has finished a whole lot of +# user-specific metrics can be computed as part of the diagnostic, +# and then plotted in various manners. + +# Arguments: +# cfg - nested dictionary of metadata + +# Returns: +# string; runs the user diagnostic + +# """ +# # assemble the data dictionary keyed by dataset name +# # this makes use of the handy group_metadata function that +# # orders the data by 'dataset'; the resulting dictionary is +# # keyed on datasets e.g. dict = {'MPI-ESM-LR': [var1, var2...]} +# # where var1, var2 are dicts holding all needed information per variable +# my_files_dict = group_metadata(cfg["input_data"].values(), "dataset") + +# # iterate over key(dataset) and values(list of vars) +# for key, value in my_files_dict.items(): +# # load the cube from data files only +# # using a single variable here so just grab the first (and only) +# # list element +# cube = iris.load_cube(value[0]["filename"]) + +# # the first data analysis bit: simple cube difference: +# # perform a difference between ground and first levels +# diff_cube = cube[:, 0, :, :] - cube[:, 1, :, :] +# # square the difference'd cube just for fun +# squared_cube = diff_cube**2.0 + +# # the second data analysis bit (slightly more advanced): +# # compute an area average over the squared cube +# # to apply the area average use a preprocessor function +# # rather than writing your own function +# area_avg_cube = area_statistics(squared_cube, "mean") + +# # finalize your analysis by plotting a time series of the +# # diffed, squared and area averaged cube; call the plot function: +# _plot_time_series(cfg, area_avg_cube, key) + +# # that's it, we're done! +# return "I am done with my first ESMValTool diagnostic!" if __name__ == "__main__": @@ -157,4 +172,4 @@ def run_my_diagnostic(cfg): # nested dictionary holding all the needed information) with run_diagnostic() as config: # list here the functions that need to run - run_my_diagnostic(config) + main(config) From 51db6469c71c052581cda621868adc3f55f2b480 Mon Sep 17 00:00:00 2001 From: Thomas Wilder Date: Thu, 27 Nov 2025 18:50:55 +0200 Subject: [PATCH 4/6] a recipe with details added: description, dataset, variables. need to add preprocessors. --- esmvaltool/recipes/recipe_so_drakepassage.yml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 esmvaltool/recipes/recipe_so_drakepassage.yml diff --git a/esmvaltool/recipes/recipe_so_drakepassage.yml b/esmvaltool/recipes/recipe_so_drakepassage.yml new file mode 100644 index 0000000000..065a83d1c4 --- /dev/null +++ b/esmvaltool/recipes/recipe_so_drakepassage.yml @@ -0,0 +1,38 @@ +# ESMValTool +# recipe_so_acc_sigma_tr.yml +--- +documentation: + description: + Creates a transect through Drake Passage of zonal velocity with + the sigma contour. + title: + Drake Passage zonal velocity transect with sigma contour + + authors: + - wilder_thomas + +datasets: + - dataset: HadGEM3-GC31-LL + +diagnostics: + # ------------------------------------------------------------------------ + # Drake Passage zonal velocity transect with sigma contour + # -------------------------------------------------------------------------------- + drake_passage_transect: + description: Transect of zonal velocity and sigma + variables: + thetao: # Temperature ocean + ensemble: r1i1p1f3 + exp: historical + grid: gn + mip: Omon + project: CMIP6 + timerange: 2000/2001 + so: # Salinity ocean + ensemble: r1i1p1f3 + exp: historical + grid: gn + mip: Omon + project: CMIP6 + timerange: 2000/2001 + scripts: From 3c721cd108320535aa0adc1ac66db9f2ab93d04c Mon Sep 17 00:00:00 2001 From: Thomas Wilder Date: Fri, 28 Nov 2025 18:42:22 +0200 Subject: [PATCH 5/6] - renamed the recipe file. - current recipe working with basic python diagnostic. --- esmvaltool/recipes/recipe_so_acc_sigma_tr.yml | 0 esmvaltool/recipes/recipe_so_drakepassage.yml | 32 +++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) delete mode 100644 esmvaltool/recipes/recipe_so_acc_sigma_tr.yml diff --git a/esmvaltool/recipes/recipe_so_acc_sigma_tr.yml b/esmvaltool/recipes/recipe_so_acc_sigma_tr.yml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/esmvaltool/recipes/recipe_so_drakepassage.yml b/esmvaltool/recipes/recipe_so_drakepassage.yml index 065a83d1c4..886f81ac5e 100644 --- a/esmvaltool/recipes/recipe_so_drakepassage.yml +++ b/esmvaltool/recipes/recipe_so_drakepassage.yml @@ -9,16 +9,25 @@ documentation: Drake Passage zonal velocity transect with sigma contour authors: - - wilder_thomas + - demora_lee datasets: - dataset: HadGEM3-GC31-LL +preprocessors: + # ------------------------------------------------------------------------ + # Preprocessor settings for Drake Passage transect + # ------------------------------------------------------------------------ + time_mean: + climate_statistics: + operator: 'mean' + period: 'full' + diagnostics: # ------------------------------------------------------------------------ # Drake Passage zonal velocity transect with sigma contour # -------------------------------------------------------------------------------- - drake_passage_transect: + transect: description: Transect of zonal velocity and sigma variables: thetao: # Temperature ocean @@ -28,11 +37,16 @@ diagnostics: mip: Omon project: CMIP6 timerange: 2000/2001 - so: # Salinity ocean - ensemble: r1i1p1f3 - exp: historical - grid: gn - mip: Omon - project: CMIP6 - timerange: 2000/2001 + preprocessor: time_mean + # so: # Salinity ocean + # ensemble: r1i1p1f3 + # exp: historical + # grid: gn + # mip: Omon + # project: CMIP6 + # timerange: 2000/2001 + # preprocessor: time_mean + scripts: + script1: + script: /scratch/project_465001823/twilder/ESMValTool/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py From 777a843e36cd8dc8c8dfd0a530e2865eb8a847c9 Mon Sep 17 00:00:00 2001 From: Thomas Wilder Date: Fri, 28 Nov 2025 18:44:07 +0200 Subject: [PATCH 6/6] - removed some imports and added a log message to indicate it runs without error. --- .../southern_ocean/diagnostic_acc_sigma_tr.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py b/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py index f9239bd09c..149a70b96e 100644 --- a/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py +++ b/esmvaltool/diag_scripts/southern_ocean/diagnostic_acc_sigma_tr.py @@ -15,16 +15,17 @@ and best coding practices. """ # place your module imports here: -import gsw +# import gsw import xarray as xr import numpy as np -import cmocean +# import cmocean import logging # operating system manipulations (e.g. path constructions) import os +import sys # to manipulate iris cubes import iris @@ -52,12 +53,16 @@ def _extract_transect(): def main(cfg): - # assemble the data dictionary keyed by dataset name - input_data = cfg["input_data"].values() - my_files_dict = group_metadata(input_data, "dataset") + # # assemble the data dictionary keyed by dataset name + # input_data = cfg["input_data"].values() + # my_files_dict = group_metadata(input_data, "dataset") - # load xarray dataset - ds = xr.open_dataset(value[0]["filename"]) + # logger.info('my_files_dict: %s', my_files_dict) + + logger.info('we have lift off!') + + # # load xarray dataset + # ds = xr.open_dataset(value[0]["filename"]) pass