From 92744dcee068c0d532ef71a708b3e6152ee461bc Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Tue, 26 May 2026 23:34:27 +0200 Subject: [PATCH 01/16] In StepsBlendingNowcaster, add an option to compute a single blended member. This will relax the constraint that n_ens_members >= n_model_members (NWP members). Useful when parallelising blending via a process pool or MPI. --- pysteps/blending/steps.py | 51 ++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/pysteps/blending/steps.py b/pysteps/blending/steps.py index fc1530f0..402ea624 100644 --- a/pysteps/blending/steps.py +++ b/pysteps/blending/steps.py @@ -92,9 +92,9 @@ class StepsBlendingConfig: Time step of the motion vectors (minutes). Required if vel_pert_method is not None or mask_method is 'incremental'. n_ens_members: int - The number of ensemble members to generate. This number should always be - equal to or larger than the number of NWP ensemble members / number of - NWP models. + The number of ensemble members to generate. When ``single_member_mode`` + is False (default), this must be equal to or larger than the number of + NWP ensemble members / number of NWP models. n_cascade_levels: int, optional The number of cascade levels to use. Defaults to 6, see issue #385 on GitHub. @@ -102,6 +102,12 @@ class StepsBlendingConfig: Check if NWP models/members should be used individually, or if all of them are blended together per nowcast ensemble member. Standard set to false. + single_member_mode: bool, optional + If True, relax the constraint that ``n_ens_members >= n_model_members``. + Use this when parallelising blending via a process pool or MPI, where + each worker computes a subset of the full ensemble. In that scenario each + worker should receive the correctly-sliced ``precip_models`` and + ``velocity_models`` for its assigned members. Defaults to False. extrapolation_method: str, optional Name of the extrapolation method to use. See the documentation of :py:mod:`pysteps.extrapolation.interface`. @@ -316,6 +322,7 @@ class StepsBlendingConfig: velocity_perturbation_kwargs: dict[str, Any] = field(default_factory=dict) climatology_kwargs: dict[str, Any] = field(default_factory=dict) mask_kwargs: dict[str, Any] = field(default_factory=dict) + single_member_mode: bool = False measure_time: bool = False callback: Any | None = None return_output: bool = True @@ -1834,11 +1841,18 @@ def __find_nowcast_NWP_combination(self, t): self.__state.velocity_models_timestep = self.__velocity_models[ :, t, :, :, : ].astype(np.float64, copy=False) - # Make sure the number of model members is not larger than or equal to n_ens_members + # Make sure the number of model members is not larger than n_ens_members + # (unless single_member_mode is enabled for process-pool / MPI parallelism). n_model_members = self.__state.precip_models_cascades_timestep.shape[0] - if n_model_members > self.__config.n_ens_members: + if ( + n_model_members > self.__config.n_ens_members + and not self.__config.single_member_mode + ): raise ValueError( - "The number of NWP model members is larger than the given number of ensemble members. n_model_members <= n_ens_members." + "The number of NWP model members is larger than the given number of " + "ensemble members. Either increase n_ens_members, reduce the number of " + "NWP members, or set single_member_mode=True when computing individual " + "members in parallel (process pool / MPI)." ) # Check if NWP models/members should be used individually, or if all of @@ -1852,9 +1866,16 @@ def __find_nowcast_NWP_combination(self, t): ].astype(np.float64, copy=False) n_ens_members_provided = self.__precip_nowcast.shape[0] - if n_ens_members_provided > self.__config.n_ens_members: + if ( + n_ens_members_provided > self.__config.n_ens_members + and not self.__config.single_member_mode + ): raise ValueError( - "The number of nowcast ensemble members provided is larger than the given number of ensemble members requested. n_ens_members_provided <= n_ens_members." + "The number of nowcast ensemble members provided is larger than the " + "given number of ensemble members requested. Either increase " + "n_ens_members, reduce the number of provided nowcast members, or set " + "single_member_mode=True when computing individual members in parallel " + "(process pool / MPI)." ) n_ens_members_max = self.__config.n_ens_members @@ -3348,6 +3369,7 @@ def forecast( precip_nowcast=None, n_cascade_levels=6, blend_nwp_members=False, + single_member_mode=False, precip_thr=None, norain_thr=0.0, kmperpixel=None, @@ -3435,9 +3457,9 @@ def forecast( issuetime: datetime is issued. n_ens_members: int - The number of ensemble members to generate. This number should always be - equal to or larger than the number of NWP ensemble members / number of - NWP models. + The number of ensemble members to generate. When ``single_member_mode`` + is False (default), this must be equal to or larger than the number of + NWP ensemble members / number of NWP models. precip_nowcast: array-like, optional Optional input with array of shape (n_ens_members,timestep+1,m,n) containing and external nowcast as input to the blending. If precip_nowcast is provided, @@ -3457,6 +3479,12 @@ def forecast( Check if NWP models/members should be used individually, or if all of them are blended together per nowcast ensemble member. Standard set to false. + single_member_mode: bool, optional + If True, relax the constraint that ``n_ens_members >= n_model_members``. + Use this when parallelising blending via a process pool or MPI, where + each worker computes a subset of the full ensemble. Each worker should + receive the correctly-sliced ``precip_models`` and ``velocity_models`` + for its assigned members. Defaults to False. precip_thr: float, optional Specifies the threshold value for minimum observable precipitation intensity. Required if mask_method is not None or conditional is True. @@ -3702,6 +3730,7 @@ def forecast( n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, blend_nwp_members=blend_nwp_members, + single_member_mode=single_member_mode, precip_threshold=precip_thr, norain_threshold=norain_thr, kmperpixel=kmperpixel, From f193b9a6b9e0f5ebf3fed2511aef99578269c011 Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Tue, 26 May 2026 23:41:33 +0200 Subject: [PATCH 02/16] Remove duplicate documentation in steps blending. --- pysteps/blending/steps.py | 317 +++++++++----------------------------- 1 file changed, 71 insertions(+), 246 deletions(-) diff --git a/pysteps/blending/steps.py b/pysteps/blending/steps.py index 402ea624..5dc76d54 100644 --- a/pysteps/blending/steps.py +++ b/pysteps/blending/steps.py @@ -3407,6 +3407,12 @@ def forecast( Generate a blended nowcast ensemble by using the Short-Term Ensemble Prediction System (STEPS) method. + This is a convenience wrapper around :class:`StepsBlendingNowcaster` that + accepts flat keyword arguments instead of a :class:`StepsBlendingConfig` + object. All configuration parameters are documented in + :class:`StepsBlendingConfig`; only the data-input arguments unique to this + function are described here. + Parameters ---------- precip: array-like @@ -3414,267 +3420,82 @@ def forecast( ordered by timestamp from oldest to newest. The time steps between the inputs are assumed to be regular. precip_models: array-like - Either raw (NWP) model forecast data or decomposed (NWP) model forecast data. - If you supply decomposed data, it needs to be an array of shape - (n_models,timesteps+1) containing, per timestep (t=0 to lead time here) and - per (NWP) model or model ensemble member, a dictionary with a list of cascades - obtained by calling a method implemented in :py:mod:`pysteps.cascade.decomposition`. - If you supply the original (NWP) model forecast data, it needs to be an array of shape - (n_models,timestep+1,m,n) containing precipitation (or other) fields, which will - then be decomposed in this function. + Either raw (NWP) model forecast data or decomposed (NWP) model forecast + data. If you supply decomposed data, it needs to be an array of shape + (n_models,timesteps+1) containing, per timestep (t=0 to lead time here) + and per (NWP) model or model ensemble member, a dictionary with a list of + cascades obtained by calling a method implemented in + :py:mod:`pysteps.cascade.decomposition`. + If you supply the original (NWP) model forecast data, it needs to be an + array of shape (n_models,timestep+1,m,n) containing precipitation (or + other) fields, which will then be decomposed in this function. Depending on your use case it can be advantageous to decompose the model forecasts outside beforehand, as this slightly reduces calculation times. This is possible with :py:func:`pysteps.blending.utils.decompose_NWP`, :py:func:`pysteps.blending.utils.compute_store_nwp_motion`, and - :py:func:`pysteps.blending.utils.load_NWP`. However, if you have a lot of (NWP) model - members (e.g. 1 model member per nowcast member), this can lead to excessive memory - usage. + :py:func:`pysteps.blending.utils.load_NWP`. However, if you have a lot of + (NWP) model members (e.g. 1 model member per nowcast member), this can + lead to excessive memory usage. - To further reduce memory usage, both this array and the ``velocity_models`` array - can be given as float32. They will then be converted to float64 before computations - to minimize loss in precision. + To further reduce memory usage, both this array and the + ``velocity_models`` array can be given as float32. They will then be + converted to float64 before computations to minimise loss in precision. - In case of one (deterministic) model as input, add an extra dimension to make sure - precip_models is four dimensional prior to calling this function. + In case of one (deterministic) model as input, add an extra dimension to + make sure precip_models is four dimensional prior to calling this + function. velocity: array-like - Array of shape (2,m,n) containing the x- and y-components of the advection - field. The velocities are assumed to represent one time step between the - inputs. All values are required to be finite. + Array of shape (2,m,n) containing the x- and y-components of the + advection field. The velocities are assumed to represent one time step + between the inputs. All values are required to be finite. velocity_models: array-like - Array of shape (n_models,timestep,2,m,n) containing the x- and y-components - of the advection field for the (NWP) model field per forecast lead time. - All values are required to be finite. To reduce memory usage, this array can - be given as float32. They will then be converted to float64 before computations - to minimize loss in precision. + Array of shape (n_models,timestep,2,m,n) containing the x- and + y-components of the advection field for the (NWP) model field per + forecast lead time. All values are required to be finite. To reduce + memory usage, this array can be given as float32. timesteps: int or list of floats Number of time steps to forecast or a list of time steps for which the forecasts are computed (relative to the input time step). The elements of the list are required to be in ascending order. timestep: float - Time step of the motion vectors (minutes). Required if vel_pert_method is - not None or mask_method is 'incremental'. + Time step of the motion vectors (minutes). issuetime: datetime - is issued. + Issue time of the forecast. n_ens_members: int - The number of ensemble members to generate. When ``single_member_mode`` - is False (default), this must be equal to or larger than the number of - NWP ensemble members / number of NWP models. + Passed to :class:`StepsBlendingConfig` as ``n_ens_members``. precip_nowcast: array-like, optional - Optional input with array of shape (n_ens_members,timestep+1,m,n) containing - and external nowcast as input to the blending. If precip_nowcast is provided, - the autoregression step and advection step will be omitted for the - extrapolation cascade of the blending procedure and instead, precip_nowcast - will be used as estimate. Defaults to None (which is the standard STEPS) - method described in :cite:`Imhoff2023`. - Note that nowcasting_method should be set to 'external_nowcast' if - precip_nowcast is not None. - Note that in the current setup, only a deterministic precip_nowcast model can - be provided and only one ensemble member (without noise generation) is - returned. This will change soon. - n_cascade_levels: int, optional - The number of cascade levels to use. Defaults to 6, - see issue #385 on GitHub. - blend_nwp_members: bool - Check if NWP models/members should be used individually, or if all of - them are blended together per nowcast ensemble member. Standard set to - false. - single_member_mode: bool, optional - If True, relax the constraint that ``n_ens_members >= n_model_members``. - Use this when parallelising blending via a process pool or MPI, where - each worker computes a subset of the full ensemble. Each worker should - receive the correctly-sliced ``precip_models`` and ``velocity_models`` - for its assigned members. Defaults to False. - precip_thr: float, optional - Specifies the threshold value for minimum observable precipitation - intensity. Required if mask_method is not None or conditional is True. - norain_thr: float - Specifies the threshold value for the fraction of rainy (see above) pixels - in the radar rainfall field below which we consider there to be no rain. - Depends on the amount of clutter typically present. - Standard set to 0.0 - kmperpixel: float, optional - Spatial resolution of the input data (kilometers/pixel). Required if - vel_pert_method is not None or mask_method is 'incremental'. - extrap_method: str, optional - Name of the extrapolation method to use. See the documentation of - :py:mod:`pysteps.extrapolation.interface`. - decomp_method: {'fft'}, optional - Name of the cascade decomposition method to use. See the documentation - of :py:mod:`pysteps.cascade.interface`. - bandpass_filter_method: {'gaussian', 'uniform'}, optional - Name of the bandpass filter method to use with the cascade decomposition. - See the documentation of :py:mod:`pysteps.cascade.interface`. - nowcasting_method: {'steps', 'external_nowcast'}, - Name of the nowcasting method used to generate the nowcasts. If an external - nowcast is provided, the script will use this as input and bypass the - autoregression and advection of the extrapolation cascade. Defaults to 'steps', - which follows the method described in :cite:`Imhoff2023`. Note, if - nowcasting_method is 'external_nowcast', precip_nowcast cannot be None. - noise_method: {'parametric','nonparametric','ssft','nested',None}, optional - Name of the noise generator to use for perturbating the precipitation - field. See the documentation of :py:mod:`pysteps.noise.interface`. If set to None, - no noise is generated. - noise_stddev_adj: {'auto','fixed',None}, optional - Optional adjustment for the standard deviations of the noise fields added - to each cascade level. This is done to compensate incorrect std. dev. - estimates of casace levels due to presence of no-rain areas. 'auto'=use - the method implemented in :py:func:`pysteps.noise.utils.compute_noise_stddev_adjs`. - 'fixed'= use the formula given in :cite:`BPS2006` (eq. 6), None=disable - noise std. dev adjustment. - ar_order: int, optional - The order of the autoregressive model to use. Must be >= 1. - vel_pert_method: {'bps',None}, optional - Name of the noise generator to use for perturbing the advection field. See - the documentation of :py:mod:`pysteps.noise.interface`. If set to None, the advection - field is not perturbed. - weights_method: {'bps','spn'}, optional - The calculation method of the blending weights. Options are the method - by :cite:`BPS2006` and the covariance-based method by :cite:`SPN2013`. - Defaults to bps. - timestep_start_full_nwp_weight: int, optional. - The timestep, which should be smaller than timesteps, at which a linear - transition takes place from the calculated weights to full (1.0) NWP weight - (and zero extrapolation and noise weight) to ensure the blending - procedure becomes equal to the NWP forecast(s) at the last timestep - of the blending procedure. If not provided, the blending stick to the - theoretical weights provided by the chosen weights_method for a given - lead time and skill of each blending component. - conditional: bool, optional - If set to True, compute the statistics of the precipitation field - conditionally by excluding pixels where the values are below the threshold - precip_thr. - probmatching_method: {'cdf','mean',None}, optional - Method for matching the statistics of the forecast field with those of - the most recently observed one. 'cdf'=map the forecast CDF to the observed - one, 'mean'=adjust only the conditional mean value of the forecast field - in precipitation areas, None=no matching applied. Using 'mean' requires - that mask_method is not None. - mask_method: {'obs','incremental',None}, optional - The method to use for masking no precipitation areas in the forecast field. - The masked pixels are set to the minimum value of the observations. - 'obs' = apply precip_thr to the most recently observed precipitation intensity - field, 'incremental' = iteratively buffer the mask with a certain rate - (currently it is 1 km/min), None=no masking. - resample_distribution: bool, optional - Method to resample the distribution from the extrapolation and NWP cascade as input - for the probability matching. Not resampling these distributions may lead to losing - some extremes when the weight of both the extrapolation and NWP cascade is similar. - Defaults to True. - smooth_radar_mask_range: int, Default is 0. - Method to smooth the transition between the radar-NWP-noise blend and the NWP-noise - blend near the edge of the radar domain (radar mask), where the radar data is either - not present anymore or is not reliable. If set to 0 (grid cells), this generates a - normal forecast without smoothing. To create a smooth mask, this range should be a - positive value, representing a buffer band of a number of pixels by which the mask - is cropped and smoothed. The smooth radar mask removes the hard edges between NWP - and radar in the final blended product. Typically, a value between 50 and 100 km - can be used. 80 km generally gives good results. - callback: function, optional - Optional function that is called after computation of each time step of - the nowcast. The function takes one argument: a three-dimensional array - of shape (n_ens_members,h,w), where h and w are the height and width - of the input field precip, respectively. This can be used, for instance, - writing the outputs into files. - return_output: bool, optional - Set to False to disable returning the outputs as numpy arrays. This can - save memory if the intermediate results are written to output files using - the callback function. - seed: int, optional - Optional seed number for the random generators. - num_workers: int, optional - The number of workers to use for parallel computation. Applicable if dask - is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it - is advisable to disable OpenMP by setting the environment variable - OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous - threads. - fft_method: str, optional - A string defining the FFT method to use (see FFT methods in - :py:func:`pysteps.utils.interface.get_method`). - Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed, - the recommended method is 'pyfftw'. - domain: {"spatial", "spectral"} - If "spatial", all computations are done in the spatial domain (the - classical STEPS model). If "spectral", the AR(2) models and stochastic - perturbations are applied directly in the spectral domain to reduce - memory footprint and improve performance :cite:`PCH2019b`. - outdir_path_skill: string, optional - Path to folder where the historical skill are stored. Defaults to - path_workdir from rcparams. If no path is given, './tmp' will be used. - extrap_kwargs: dict, optional - Optional dictionary containing keyword arguments for the extrapolation - method. See the documentation of :py:func:`pysteps.extrapolation.interface`. - filter_kwargs: dict, optional - Optional dictionary containing keyword arguments for the filter method. - See the documentation of :py:mod:`pysteps.cascade.bandpass_filters`. - noise_kwargs: dict, optional - Optional dictionary containing keyword arguments for the initializer of - the noise generator. See the documentation of :py:mod:`pysteps.noise.fftgenerators`. - vel_pert_kwargs: dict, optional - Optional dictionary containing keyword arguments 'p_par' and 'p_perp' for - the initializer of the velocity perturbator. The choice of the optimal - parameters depends on the domain and the used optical flow method. - - Default parameters from :cite:`BPS2006`: - p_par = [10.88, 0.23, -7.68] - p_perp = [5.76, 0.31, -2.72] - - Parameters fitted to the data (optical flow/domain): - - darts/fmi: - p_par = [13.71259667, 0.15658963, -16.24368207] - p_perp = [8.26550355, 0.17820458, -9.54107834] - - darts/mch: - p_par = [24.27562298, 0.11297186, -27.30087471] - p_perp = [-7.80797846e+01, -3.38641048e-02, 7.56715304e+01] - - darts/fmi+mch: - p_par = [16.55447057, 0.14160448, -19.24613059] - p_perp = [14.75343395, 0.11785398, -16.26151612] - - lucaskanade/fmi: - p_par = [2.20837526, 0.33887032, -2.48995355] - p_perp = [2.21722634, 0.32359621, -2.57402761] - - lucaskanade/mch: - p_par = [2.56338484, 0.3330941, -2.99714349] - p_perp = [1.31204508, 0.3578426, -1.02499891] - - lucaskanade/fmi+mch: - p_par = [2.31970635, 0.33734287, -2.64972861] - p_perp = [1.90769947, 0.33446594, -2.06603662] - - vet/fmi: - p_par = [0.25337388, 0.67542291, 11.04895538] - p_perp = [0.02432118, 0.99613295, 7.40146505] - - vet/mch: - p_par = [0.5075159, 0.53895212, 7.90331791] - p_perp = [0.68025501, 0.41761289, 4.73793581] - - vet/fmi+mch: - p_par = [0.29495222, 0.62429207, 8.6804131 ] - p_perp = [0.23127377, 0.59010281, 5.98180004] - - fmi=Finland, mch=Switzerland, fmi+mch=both pooled into the same data set - - The above parameters have been fitted by using run_vel_pert_analysis.py - and fit_vel_pert_params.py located in the scripts directory. - - See :py:mod:`pysteps.noise.motion` for additional documentation. - clim_kwargs: dict, optional - Optional dictionary containing keyword arguments for the climatological - skill file. Arguments can consist of: 'outdir_path', 'n_models' - (the number of NWP models) and 'window_length' (the minimum number of - days the clim file should have, otherwise the default is used). - mask_kwargs: dict - Optional dictionary containing mask keyword arguments 'mask_f', - 'mask_rim' and 'max_mask_rim', the factor defining the the mask - increment and the (maximum) rim size, respectively. - The mask increment is defined as mask_f*timestep/kmperpixel. - measure_time: bool - If set to True, measure, print and return the computation time. + Optional array of shape (n_ens_members,timestep+1,m,n) containing an + external nowcast. When provided, the autoregression and advection step + are skipped for the extrapolation cascade and this field is used instead. + Requires ``nowcasting_method='external_nowcast'``. Defaults to None. + + Configuration parameters + ------------------------ + All remaining keyword arguments correspond directly to fields in + :class:`StepsBlendingConfig`, with the following name differences: + + ======================== ================================ + ``forecast()`` argument ``StepsBlendingConfig`` field + ======================== ================================ + ``precip_thr`` ``precip_threshold`` + ``norain_thr`` ``norain_threshold`` + ``extrap_method`` ``extrapolation_method`` + ``decomp_method`` ``decomposition_method`` + ``vel_pert_method`` ``velocity_perturbation_method`` + ``extrap_kwargs`` ``extrapolation_kwargs`` + ``vel_pert_kwargs`` ``velocity_perturbation_kwargs`` + ``clim_kwargs`` ``climatology_kwargs`` + ======================== ================================ + + All other arguments (``n_cascade_levels``, ``blend_nwp_members``, + ``single_member_mode``, ``noise_method``, ``noise_stddev_adj``, + ``ar_order``, ``weights_method``, ``timestep_start_full_nwp_weight``, + ``conditional``, ``probmatching_method``, ``mask_method``, + ``resample_distribution``, ``smooth_radar_mask_range``, ``seed``, + ``num_workers``, ``fft_method``, ``domain``, ``outdir_path_skill``, + ``filter_kwargs``, ``noise_kwargs``, ``mask_kwargs``, ``callback``, + ``return_output``, ``measure_time``) are passed through unchanged. Returns ------- @@ -3690,8 +3511,12 @@ def forecast( See also -------- - :py:mod:`pysteps.extrapolation.interface`, :py:mod:`pysteps.cascade.interface`, - :py:mod:`pysteps.noise.interface`, :py:func:`pysteps.noise.utils.compute_noise_stddev_adjs` + :class:`StepsBlendingConfig`, + :class:`StepsBlendingNowcaster`, + :py:mod:`pysteps.extrapolation.interface`, + :py:mod:`pysteps.cascade.interface`, + :py:mod:`pysteps.noise.interface`, + :py:func:`pysteps.noise.utils.compute_noise_stddev_adjs` References ---------- From 5dcabb2d40ab5eeec591bfac292509d478675ffa Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Wed, 27 May 2026 10:12:29 +0200 Subject: [PATCH 03/16] Add tests pysteps blending single member mode (used for parallellizing at process level). --- pysteps/tests/test_blending_steps.py | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pysteps/tests/test_blending_steps.py b/pysteps/tests/test_blending_steps.py index abfa5522..32c57265 100644 --- a/pysteps/tests/test_blending_steps.py +++ b/pysteps/tests/test_blending_steps.py @@ -75,6 +75,13 @@ (1, 10, 5, 8,'external_nowcast_ens', "incremental", "cdf", False, "bps", True, 5, False, False, 0, False, None, None, 5) ] +# Extend every existing entry with single_member_mode=False, then add cases where +# n_models > n_ens_members is only valid because single_member_mode=True. +steps_arg_values = [t + (False,) for t in steps_arg_values] + [ + (3, 3, 1, 6, 'steps', None, None, False, "spn", True, 1, False, False, 0, False, None, None, None, True), + (3, 3, 1, 6, 'steps', "incremental", "cdf", False, "bps", False, 1, False, False, 80, False, None, None, None, True), +] + # fmt:on @@ -125,6 +132,7 @@ def run_and_assert_forecast( "vel_pert_method", "max_mask_rim", "timestep_start_full_nwp_weight", + "single_member_mode", ) @@ -148,6 +156,7 @@ def test_steps_blending( vel_pert_method, max_mask_rim, timestep_start_full_nwp_weight, + single_member_mode, ): pytest.importorskip("cv2") @@ -398,6 +407,7 @@ def test_steps_blending( n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, blend_nwp_members=blend_nwp_members, + single_member_mode=single_member_mode, precip_thr=metadata["threshold"], kmperpixel=1.0, extrap_method="semilagrangian", @@ -554,3 +564,30 @@ def test_steps_blending_partial_zero_radar(ar_order): converter=converter, metadata=metadata, ) + + +def test_single_member_mode_raises_without_flag(): + """Without single_member_mode, n_model_members > n_ens_members must raise ValueError.""" + pytest.importorskip("cv2") + with pytest.raises(ValueError, match="single_member_mode"): + test_steps_blending( + n_models=3, + timesteps=3, + n_ens_members=1, + n_cascade_levels=6, + nowcasting_method="steps", + mask_method=None, + probmatching_method=None, + blend_nwp_members=False, + weights_method="spn", + decomposed_nwp=True, + expected_n_ens_members=1, + zero_radar=False, + zero_nwp=False, + smooth_radar_mask_range=0, + resample_distribution=False, + vel_pert_method=None, + max_mask_rim=None, + timestep_start_full_nwp_weight=None, + single_member_mode=False, + ) From 62383296a49153f2b0f01ee55220e91060904a28 Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Wed, 27 May 2026 12:00:08 +0200 Subject: [PATCH 04/16] Refactor StepsBlendingConfig to replace single_member_mode with model_index_offset for improved parallel processing. --- pysteps/blending/steps.py | 70 ++++++++++++++-------------- pysteps/tests/test_blending_steps.py | 25 +++++----- 2 files changed, 48 insertions(+), 47 deletions(-) diff --git a/pysteps/blending/steps.py b/pysteps/blending/steps.py index 5dc76d54..6f456796 100644 --- a/pysteps/blending/steps.py +++ b/pysteps/blending/steps.py @@ -92,9 +92,8 @@ class StepsBlendingConfig: Time step of the motion vectors (minutes). Required if vel_pert_method is not None or mask_method is 'incremental'. n_ens_members: int - The number of ensemble members to generate. When ``single_member_mode`` - is False (default), this must be equal to or larger than the number of - NWP ensemble members / number of NWP models. + The number of ensemble members to generate. Must be equal to or larger + than the number of NWP ensemble members / number of NWP models. n_cascade_levels: int, optional The number of cascade levels to use. Defaults to 6, see issue #385 on GitHub. @@ -102,12 +101,14 @@ class StepsBlendingConfig: Check if NWP models/members should be used individually, or if all of them are blended together per nowcast ensemble member. Standard set to false. - single_member_mode: bool, optional - If True, relax the constraint that ``n_ens_members >= n_model_members``. - Use this when parallelising blending via a process pool or MPI, where - each worker computes a subset of the full ensemble. In that scenario each - worker should receive the correctly-sliced ``precip_models`` and - ``velocity_models`` for its assigned members. Defaults to False. + model_index_offset: int, optional + Global index of the first model in ``precip_models``. Use this when + parallelising blending via a process pool or MPI: slice + ``precip_models[i:i+n]`` and ``velocity_models[i:i+n]`` for each worker + and set ``model_index_offset=i``. This ensures the climatological skill + regression file for the correct global model index is used. Workers with + ``model_index_offset > 0`` skip writing to the skill file (the + coordinating process is responsible for that). Defaults to 0. extrapolation_method: str, optional Name of the extrapolation method to use. See the documentation of :py:mod:`pysteps.extrapolation.interface`. @@ -322,7 +323,7 @@ class StepsBlendingConfig: velocity_perturbation_kwargs: dict[str, Any] = field(default_factory=dict) climatology_kwargs: dict[str, Any] = field(default_factory=dict) mask_kwargs: dict[str, Any] = field(default_factory=dict) - single_member_mode: bool = False + model_index_offset: int = 0 measure_time: bool = False callback: Any | None = None return_output: bool = True @@ -1841,18 +1842,16 @@ def __find_nowcast_NWP_combination(self, t): self.__state.velocity_models_timestep = self.__velocity_models[ :, t, :, :, : ].astype(np.float64, copy=False) - # Make sure the number of model members is not larger than n_ens_members - # (unless single_member_mode is enabled for process-pool / MPI parallelism). + # Make sure the number of model members is not larger than n_ens_members. + # For process-pool / MPI parallelism, slice precip_models and velocity_models + # per worker (e.g. precip_models[i:i+1]) and set model_index_offset=i. n_model_members = self.__state.precip_models_cascades_timestep.shape[0] - if ( - n_model_members > self.__config.n_ens_members - and not self.__config.single_member_mode - ): + if n_model_members > self.__config.n_ens_members: raise ValueError( "The number of NWP model members is larger than the given number of " - "ensemble members. Either increase n_ens_members, reduce the number of " - "NWP members, or set single_member_mode=True when computing individual " - "members in parallel (process pool / MPI)." + "ensemble members. Either increase n_ens_members or reduce the number " + "of NWP members. For process-pool / MPI parallelism, slice " + "precip_models and velocity_models per worker and set model_index_offset." ) # Check if NWP models/members should be used individually, or if all of @@ -1866,16 +1865,11 @@ def __find_nowcast_NWP_combination(self, t): ].astype(np.float64, copy=False) n_ens_members_provided = self.__precip_nowcast.shape[0] - if ( - n_ens_members_provided > self.__config.n_ens_members - and not self.__config.single_member_mode - ): + if n_ens_members_provided > self.__config.n_ens_members: raise ValueError( "The number of nowcast ensemble members provided is larger than the " "given number of ensemble members requested. Either increase " - "n_ens_members, reduce the number of provided nowcast members, or set " - "single_member_mode=True when computing individual members in parallel " - "(process pool / MPI)." + "n_ens_members or reduce the number of provided nowcast members." ) n_ens_members_max = self.__config.n_ens_members @@ -2093,12 +2087,15 @@ def __determine_skill_for_current_timestep(self, t): ) # Save this in the climatological skill file - blending.clim.save_skill( - current_skill=self.__params.rho_nwp_models, - validtime=self.__issuetime, - outdir_path=self.__config.outdir_path_skill, - **self.__params.climatology_kwargs, - ) + # Workers with model_index_offset > 0 must not write to the shared + # skill file to avoid race conditions and shape mismatches. + if self.__config.model_index_offset == 0: + blending.clim.save_skill( + current_skill=self.__params.rho_nwp_models, + validtime=self.__issuetime, + outdir_path=self.__config.outdir_path_skill, + **self.__params.climatology_kwargs, + ) if t > 0: # Determine the skill of the components for lead time (t0 + t) # First for the extrapolation component. Only calculate it when t > 0. @@ -2140,7 +2137,10 @@ def __determine_NWP_skill_for_next_timestep(self, t, j, worker_state): lt=(t * int(self.__config.timestep)), correlations=self.__params.rho_nwp_models[j], outdir_path=self.__config.outdir_path_skill, - n_model=worker_state.mapping_list_NWP_member_to_ensemble_member[j], + n_model=( + worker_state.mapping_list_NWP_member_to_ensemble_member[j] + + self.__config.model_index_offset + ), skill_kwargs=self.__params.climatology_kwargs, ) # Concatenate rho_extrap_cascade and rho_nwp @@ -3369,7 +3369,7 @@ def forecast( precip_nowcast=None, n_cascade_levels=6, blend_nwp_members=False, - single_member_mode=False, + model_index_offset=0, precip_thr=None, norain_thr=0.0, kmperpixel=None, @@ -3555,7 +3555,7 @@ def forecast( n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, blend_nwp_members=blend_nwp_members, - single_member_mode=single_member_mode, + model_index_offset=model_index_offset, precip_threshold=precip_thr, norain_threshold=norain_thr, kmperpixel=kmperpixel, diff --git a/pysteps/tests/test_blending_steps.py b/pysteps/tests/test_blending_steps.py index 32c57265..b3a131b0 100644 --- a/pysteps/tests/test_blending_steps.py +++ b/pysteps/tests/test_blending_steps.py @@ -75,11 +75,12 @@ (1, 10, 5, 8,'external_nowcast_ens', "incremental", "cdf", False, "bps", True, 5, False, False, 0, False, None, None, 5) ] -# Extend every existing entry with single_member_mode=False, then add cases where -# n_models > n_ens_members is only valid because single_member_mode=True. -steps_arg_values = [t + (False,) for t in steps_arg_values] + [ - (3, 3, 1, 6, 'steps', None, None, False, "spn", True, 1, False, False, 0, False, None, None, None, True), - (3, 3, 1, 6, 'steps', "incremental", "cdf", False, "bps", False, 1, False, False, 80, False, None, None, None, True), +# Extend every existing entry with model_index_offset=0 (default, no offset). +# Add two entries that exercise the process-pool worker pattern: +# n_models=1, n_ens_members=1, model_index_offset=0 (worker holds 1 sliced model). +steps_arg_values = [t + (0,) for t in steps_arg_values] + [ + (1, 3, 1, 6, 'steps', None, None, False, "spn", True, 1, False, False, 0, False, None, None, None, 0), + (1, 3, 1, 6, 'steps', "incremental", "cdf", False, "bps", False, 1, False, False, 80, False, None, None, None, 0), ] # fmt:on @@ -132,7 +133,7 @@ def run_and_assert_forecast( "vel_pert_method", "max_mask_rim", "timestep_start_full_nwp_weight", - "single_member_mode", + "model_index_offset", ) @@ -156,7 +157,7 @@ def test_steps_blending( vel_pert_method, max_mask_rim, timestep_start_full_nwp_weight, - single_member_mode, + model_index_offset, ): pytest.importorskip("cv2") @@ -407,7 +408,7 @@ def test_steps_blending( n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, blend_nwp_members=blend_nwp_members, - single_member_mode=single_member_mode, + model_index_offset=model_index_offset, precip_thr=metadata["threshold"], kmperpixel=1.0, extrap_method="semilagrangian", @@ -566,10 +567,10 @@ def test_steps_blending_partial_zero_radar(ar_order): ) -def test_single_member_mode_raises_without_flag(): - """Without single_member_mode, n_model_members > n_ens_members must raise ValueError.""" +def test_raises_when_n_models_exceeds_n_ens_members(): + """n_model_members > n_ens_members must raise ValueError (user forgot to slice).""" pytest.importorskip("cv2") - with pytest.raises(ValueError, match="single_member_mode"): + with pytest.raises(ValueError, match="slice"): test_steps_blending( n_models=3, timesteps=3, @@ -589,5 +590,5 @@ def test_single_member_mode_raises_without_flag(): vel_pert_method=None, max_mask_rim=None, timestep_start_full_nwp_weight=None, - single_member_mode=False, + model_index_offset=0, ) From 4958692791fc1653c6810952b4bad9d34c2b27c1 Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Wed, 27 May 2026 12:09:44 +0200 Subject: [PATCH 05/16] Improve vague error message in steps blending; adapt tests accordingly. --- pysteps/blending/steps.py | 9 +++++---- pysteps/tests/test_blending_steps.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pysteps/blending/steps.py b/pysteps/blending/steps.py index 6f456796..13131ca3 100644 --- a/pysteps/blending/steps.py +++ b/pysteps/blending/steps.py @@ -1848,10 +1848,11 @@ def __find_nowcast_NWP_combination(self, t): n_model_members = self.__state.precip_models_cascades_timestep.shape[0] if n_model_members > self.__config.n_ens_members: raise ValueError( - "The number of NWP model members is larger than the given number of " - "ensemble members. Either increase n_ens_members or reduce the number " - "of NWP members. For process-pool / MPI parallelism, slice " - "precip_models and velocity_models per worker and set model_index_offset." + f"Number of NWP models ({n_model_members}) exceeds ensemble size ({self.__config.n_ens_members}). " + f"\n\nQuick fix: Increase n_ens_members to at least {n_model_members}, or reduce NWP models to {self.__config.n_ens_members}." + f"\n\nFor parallel processing: Slice precip_models and velocity_models per worker " + f"(e.g., precip_models[i:i+n]) and set model_index_offset to the global index of the first model. " + f"This allows each worker to process one model independently. See documentation for details." ) # Check if NWP models/members should be used individually, or if all of diff --git a/pysteps/tests/test_blending_steps.py b/pysteps/tests/test_blending_steps.py index b3a131b0..d0e7f050 100644 --- a/pysteps/tests/test_blending_steps.py +++ b/pysteps/tests/test_blending_steps.py @@ -570,7 +570,7 @@ def test_steps_blending_partial_zero_radar(ar_order): def test_raises_when_n_models_exceeds_n_ens_members(): """n_model_members > n_ens_members must raise ValueError (user forgot to slice).""" pytest.importorskip("cv2") - with pytest.raises(ValueError, match="slice"): + with pytest.raises(ValueError, match="[Ss]lice"): test_steps_blending( n_models=3, timesteps=3, From 695af3e298d160a3baa03b9c55b0707920a5617e Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Wed, 27 May 2026 14:37:39 +0200 Subject: [PATCH 06/16] Add complevel argument to netcdf exporter + tests (#550) * Add optional argument complevel to initialize_forecast_exporter_netcdf. Setting this to a lower value significantly speeds up netcdf writing * Add tests with different complevel (0-9). --- pysteps/io/exporters.py | 9 +++++++-- pysteps/tests/test_exporters.py | 27 +++++++++++++++++++-------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/pysteps/io/exporters.py b/pysteps/io/exporters.py index 5b1ec4d7..2719b92e 100644 --- a/pysteps/io/exporters.py +++ b/pysteps/io/exporters.py @@ -381,6 +381,7 @@ def initialize_forecast_exporter_netcdf( fill_value=None, scale_factor=None, offset=None, + complevel=9, **kwargs, ): """ @@ -429,6 +430,10 @@ def initialize_forecast_exporter_netcdf( offset: float, optional The offset to offset the data as: store_value = scale_factor * precipitation_value + offset. Defaults to None. + complevel: int, optional + Compression level for zlib compression, ranging from 0 (no + compression) to 9 (slowest, most compression). Higher values + reduce file size but increase write time. Defaults to 9. Other Parameters ---------------- @@ -612,7 +617,7 @@ def initialize_forecast_exporter_netcdf( dimensions=("ens_number", "time", "y", "x"), compression="zlib", zlib=True, - complevel=9, + complevel=complevel, fill_value=fill_value, ) else: @@ -622,7 +627,7 @@ def initialize_forecast_exporter_netcdf( dimensions=("time", "y", "x"), compression="zlib", zlib=True, - complevel=9, + complevel=complevel, fill_value=fill_value, ) diff --git a/pysteps/tests/test_exporters.py b/pysteps/tests/test_exporters.py index 10e87d46..79e0135b 100644 --- a/pysteps/tests/test_exporters.py +++ b/pysteps/tests/test_exporters.py @@ -25,16 +25,19 @@ "scale_factor", "offset", "n_timesteps", + "complevel", ) exporter_arg_values = [ - (1, None, np.float32, None, None, None, 3), - (1, "timestep", np.float32, 65535, None, None, 3), - (2, None, np.float32, 65535, None, None, 3), - (2, None, np.float32, 65535, None, None, [1, 2, 4]), - (2, "timestep", np.float32, None, None, None, 3), - (2, "timestep", np.float32, None, None, None, [1, 2, 4]), - (2, "member", np.float64, None, 0.01, 1.0, 3), + (1, None, np.float32, None, None, None, 3, 1), + (1, None, np.float32, None, None, None, 3, 0), + (1, None, np.float32, None, None, None, 3, 6), + (1, "timestep", np.float32, 65535, None, None, 3, 9), + (2, None, np.float32, 65535, None, None, 3, 2), + (2, None, np.float32, 65535, None, None, [1, 2, 4], 2), + (2, "timestep", np.float32, None, None, None, 3, 2), + (2, "timestep", np.float32, None, None, None, [1, 2, 4], 2), + (2, "member", np.float64, None, 0.01, 1.0, 3, 2), ] @@ -58,7 +61,14 @@ def test_get_geotiff_filename(): @pytest.mark.parametrize(exporter_arg_names, exporter_arg_values) def test_io_export_netcdf_one_member_one_time_step( - n_ens_members, incremental, datatype, fill_value, scale_factor, offset, n_timesteps + n_ens_members, + incremental, + datatype, + fill_value, + scale_factor, + offset, + n_timesteps, + complevel, ): """ Test the export netcdf. @@ -95,6 +105,7 @@ def test_io_export_netcdf_one_member_one_time_step( fill_value=fill_value, scale_factor=scale_factor, offset=offset, + complevel=complevel, ) if n_ens_members > 1: From cf0d129ea1afb090ed88db684b498df66517ed28 Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Wed, 27 May 2026 17:52:34 +0200 Subject: [PATCH 07/16] Check past skill dimensions in save_skill and calc_clim_skill. Disregard stored skill if they don't match. They will get overwritten eventually. --- pysteps/blending/clim.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pysteps/blending/clim.py b/pysteps/blending/clim.py index 82656b27..75399966 100644 --- a/pysteps/blending/clim.py +++ b/pysteps/blending/clim.py @@ -121,6 +121,7 @@ def save_skill( # Append skill to the list of the past X daily averages. if ( past_skill is not None + and past_skill.ndim >= 3 and past_skill.shape[2] == n_cascade_levels and past_skill.shape[1] == skill_today["mean_skill"].shape[0] ): @@ -189,6 +190,13 @@ def calc_clim_skill( # past_skill has dimensions date x model x scale_level x .... if past_skill_file.is_file(): past_skill = np.load(past_skill_file) + # Don't use stored file if the shape no longer matches the requested configuration. + if ( + past_skill.ndim < 3 + or past_skill.shape[1] != n_models + or past_skill.shape[2] != n_cascade_levels + ): + past_skill = np.array(None) else: past_skill = np.array(None) # check if there is enough data to compute the climatological skill From cb27f2bc15845fadd8294bbb7f13d06f4ae0296f Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Wed, 27 May 2026 18:27:41 +0200 Subject: [PATCH 08/16] Add write_skill flag and avoid error-prone dependence on model_index_offset to determine this. --- pysteps/blending/steps.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pysteps/blending/steps.py b/pysteps/blending/steps.py index 13131ca3..d598683a 100644 --- a/pysteps/blending/steps.py +++ b/pysteps/blending/steps.py @@ -105,10 +105,13 @@ class StepsBlendingConfig: Global index of the first model in ``precip_models``. Use this when parallelising blending via a process pool or MPI: slice ``precip_models[i:i+n]`` and ``velocity_models[i:i+n]`` for each worker - and set ``model_index_offset=i``. This ensures the climatological skill - regression file for the correct global model index is used. Workers with - ``model_index_offset > 0`` skip writing to the skill file (the - coordinating process is responsible for that). Defaults to 0. + and set ``model_index_offset=i``. This shifts the climatological skill + lookup to the correct global model index. Defaults to 0. + write_skill: bool, optional + If True, update the rolling climatological skill file at each timestep. + Set to False for workers that should not write skill (e.g. duplicate + workers in a process pool that share a skill directory with a designated + writer). Defaults to True. extrapolation_method: str, optional Name of the extrapolation method to use. See the documentation of :py:mod:`pysteps.extrapolation.interface`. @@ -324,6 +327,7 @@ class StepsBlendingConfig: climatology_kwargs: dict[str, Any] = field(default_factory=dict) mask_kwargs: dict[str, Any] = field(default_factory=dict) model_index_offset: int = 0 + write_skill: bool = True measure_time: bool = False callback: Any | None = None return_output: bool = True @@ -2088,9 +2092,7 @@ def __determine_skill_for_current_timestep(self, t): ) # Save this in the climatological skill file - # Workers with model_index_offset > 0 must not write to the shared - # skill file to avoid race conditions and shape mismatches. - if self.__config.model_index_offset == 0: + if self.__config.write_skill: blending.clim.save_skill( current_skill=self.__params.rho_nwp_models, validtime=self.__issuetime, @@ -3371,6 +3373,7 @@ def forecast( n_cascade_levels=6, blend_nwp_members=False, model_index_offset=0, + write_skill=True, precip_thr=None, norain_thr=0.0, kmperpixel=None, @@ -3557,6 +3560,7 @@ def forecast( n_cascade_levels=n_cascade_levels, blend_nwp_members=blend_nwp_members, model_index_offset=model_index_offset, + write_skill=write_skill, precip_threshold=precip_thr, norain_threshold=norain_thr, kmperpixel=kmperpixel, From 026b2319476a00b0392a3771eeea80697f9c37ab Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Wed, 27 May 2026 22:41:46 +0200 Subject: [PATCH 09/16] Correctly compute n_model also when blending multiple NWP members. --- pysteps/blending/steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysteps/blending/steps.py b/pysteps/blending/steps.py index d598683a..3072ca6f 100644 --- a/pysteps/blending/steps.py +++ b/pysteps/blending/steps.py @@ -2125,7 +2125,7 @@ def __determine_NWP_skill_for_next_timestep(self, t, j, worker_state): lt=(t * int(self.__config.timestep)), correlations=self.__params.rho_nwp_models[model_index], outdir_path=self.__config.outdir_path_skill, - n_model=model_index, + n_model=model_index + self.__config.model_index_offset, skill_kwargs=self.__params.climatology_kwargs, ) rho_nwp_forecast.append(rho_value) From 6dd888d60ec2461f0270441b4263f491a30ff304 Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Wed, 27 May 2026 22:48:00 +0200 Subject: [PATCH 10/16] Add steps blending test with nonzero model_index_offset Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pysteps/tests/test_blending_steps.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pysteps/tests/test_blending_steps.py b/pysteps/tests/test_blending_steps.py index d0e7f050..fef59aae 100644 --- a/pysteps/tests/test_blending_steps.py +++ b/pysteps/tests/test_blending_steps.py @@ -76,11 +76,13 @@ ] # Extend every existing entry with model_index_offset=0 (default, no offset). -# Add two entries that exercise the process-pool worker pattern: -# n_models=1, n_ens_members=1, model_index_offset=0 (worker holds 1 sliced model). +# Add worker-pattern entries that exercise both the default and a non-zero +# model_index_offset path: +# - n_models=1, n_ens_members=1, model_index_offset=0 (worker holds 1 sliced model) +# - n_models=1, n_ens_members=1, model_index_offset=2 (worker holds a later sliced model) steps_arg_values = [t + (0,) for t in steps_arg_values] + [ (1, 3, 1, 6, 'steps', None, None, False, "spn", True, 1, False, False, 0, False, None, None, None, 0), - (1, 3, 1, 6, 'steps', "incremental", "cdf", False, "bps", False, 1, False, False, 80, False, None, None, None, 0), + (1, 3, 1, 6, 'steps', "incremental", "cdf", False, "bps", False, 1, False, False, 80, False, None, None, None, 2), ] # fmt:on From af454a043cfb47ab2b819abb8834008cd5dea3be Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Wed, 27 May 2026 22:58:12 +0200 Subject: [PATCH 11/16] Remove out-of-sync list of blended forecast arguments from documentation. --- pysteps/blending/steps.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pysteps/blending/steps.py b/pysteps/blending/steps.py index 3072ca6f..9d239a62 100644 --- a/pysteps/blending/steps.py +++ b/pysteps/blending/steps.py @@ -3492,14 +3492,6 @@ def forecast( ``clim_kwargs`` ``climatology_kwargs`` ======================== ================================ - All other arguments (``n_cascade_levels``, ``blend_nwp_members``, - ``single_member_mode``, ``noise_method``, ``noise_stddev_adj``, - ``ar_order``, ``weights_method``, ``timestep_start_full_nwp_weight``, - ``conditional``, ``probmatching_method``, ``mask_method``, - ``resample_distribution``, ``smooth_radar_mask_range``, ``seed``, - ``num_workers``, ``fft_method``, ``domain``, ``outdir_path_skill``, - ``filter_kwargs``, ``noise_kwargs``, ``mask_kwargs``, ``callback``, - ``return_output``, ``measure_time``) are passed through unchanged. Returns ------- From 83664b429d190c94c540c3565a0dd9682912b931 Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Thu, 28 May 2026 00:00:24 +0200 Subject: [PATCH 12/16] Remove the model_index_offset which has become superfluous now that we use separate directories per NWP member for the skill + adapt tests. --- pysteps/blending/steps.py | 20 +- pysteps/tests/test_blending_steps.py | 1237 ++++++++++++++++++++++++-- 2 files changed, 1167 insertions(+), 90 deletions(-) diff --git a/pysteps/blending/steps.py b/pysteps/blending/steps.py index 9d239a62..2a50113c 100644 --- a/pysteps/blending/steps.py +++ b/pysteps/blending/steps.py @@ -101,12 +101,6 @@ class StepsBlendingConfig: Check if NWP models/members should be used individually, or if all of them are blended together per nowcast ensemble member. Standard set to false. - model_index_offset: int, optional - Global index of the first model in ``precip_models``. Use this when - parallelising blending via a process pool or MPI: slice - ``precip_models[i:i+n]`` and ``velocity_models[i:i+n]`` for each worker - and set ``model_index_offset=i``. This shifts the climatological skill - lookup to the correct global model index. Defaults to 0. write_skill: bool, optional If True, update the rolling climatological skill file at each timestep. Set to False for workers that should not write skill (e.g. duplicate @@ -326,7 +320,6 @@ class StepsBlendingConfig: velocity_perturbation_kwargs: dict[str, Any] = field(default_factory=dict) climatology_kwargs: dict[str, Any] = field(default_factory=dict) mask_kwargs: dict[str, Any] = field(default_factory=dict) - model_index_offset: int = 0 write_skill: bool = True measure_time: bool = False callback: Any | None = None @@ -1848,14 +1841,14 @@ def __find_nowcast_NWP_combination(self, t): ].astype(np.float64, copy=False) # Make sure the number of model members is not larger than n_ens_members. # For process-pool / MPI parallelism, slice precip_models and velocity_models - # per worker (e.g. precip_models[i:i+1]) and set model_index_offset=i. + # per worker (e.g. precip_models[i:i+1]) and use a per-worker skill directory. n_model_members = self.__state.precip_models_cascades_timestep.shape[0] if n_model_members > self.__config.n_ens_members: raise ValueError( f"Number of NWP models ({n_model_members}) exceeds ensemble size ({self.__config.n_ens_members}). " f"\n\nQuick fix: Increase n_ens_members to at least {n_model_members}, or reduce NWP models to {self.__config.n_ens_members}." f"\n\nFor parallel processing: Slice precip_models and velocity_models per worker " - f"(e.g., precip_models[i:i+n]) and set model_index_offset to the global index of the first model. " + f"(e.g., precip_models[i:i+n]) and give each worker its own outdir_path_skill sub-directory. " f"This allows each worker to process one model independently. See documentation for details." ) @@ -2125,7 +2118,7 @@ def __determine_NWP_skill_for_next_timestep(self, t, j, worker_state): lt=(t * int(self.__config.timestep)), correlations=self.__params.rho_nwp_models[model_index], outdir_path=self.__config.outdir_path_skill, - n_model=model_index + self.__config.model_index_offset, + n_model=model_index, skill_kwargs=self.__params.climatology_kwargs, ) rho_nwp_forecast.append(rho_value) @@ -2140,10 +2133,7 @@ def __determine_NWP_skill_for_next_timestep(self, t, j, worker_state): lt=(t * int(self.__config.timestep)), correlations=self.__params.rho_nwp_models[j], outdir_path=self.__config.outdir_path_skill, - n_model=( - worker_state.mapping_list_NWP_member_to_ensemble_member[j] - + self.__config.model_index_offset - ), + n_model=worker_state.mapping_list_NWP_member_to_ensemble_member[j], skill_kwargs=self.__params.climatology_kwargs, ) # Concatenate rho_extrap_cascade and rho_nwp @@ -3372,7 +3362,6 @@ def forecast( precip_nowcast=None, n_cascade_levels=6, blend_nwp_members=False, - model_index_offset=0, write_skill=True, precip_thr=None, norain_thr=0.0, @@ -3551,7 +3540,6 @@ def forecast( n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, blend_nwp_members=blend_nwp_members, - model_index_offset=model_index_offset, write_skill=write_skill, precip_threshold=precip_thr, norain_threshold=norain_thr, diff --git a/pysteps/tests/test_blending_steps.py b/pysteps/tests/test_blending_steps.py index fef59aae..ff7627d9 100644 --- a/pysteps/tests/test_blending_steps.py +++ b/pysteps/tests/test_blending_steps.py @@ -8,85 +8,1136 @@ import pysteps from pysteps import blending, cascade -# fmt:off steps_arg_values = [ - (1, 3, 4, 8, 'steps', None, None, False, "spn", True, 4, False, False, 0, False, None, None, None), - (1, 3, 4, 8,'steps', "obs", None, False, "spn", True, 4, False, False, 0, False, None, None, None), - (1, 3, 4, 8,'steps', "incremental", None, False, "spn", True, 4, False, False, 0, False, None, None, None), - (1, 3, 4, 8,'steps', None, "mean", False, "spn", True, 4, False, False, 0, False, None, None, None), - (1, 3, 4, 8,'steps', None, "mean", False, "spn", True, 4, False, False, 0, True, None, None, None), - (1, 3, 4, 8,'steps', None, "cdf", False, "spn", True, 4, False, False, 0, False, None, None, None), - (1, [1, 2, 3], 4, 8,'steps', None, "cdf", False, "spn", True, 4, False, False, 0, False, None, None, None), - (1, 3, 4, 8,'steps', "incremental", "cdf", False, "spn", True, 4, False, False, 0, False, None, None, None), - (1, 3, 4, 6,'steps', "incremental", "cdf", False, "bps", True, 4, False, False, 0, False, None, None, None), - (1, 3, 4, 6,'steps', "incremental", "cdf", False, "bps", False, 4, False, False, 0, False, None, None, None), - (1, 3, 4, 6,'steps', "incremental", "cdf", False, "bps", False, 4, False, False, 0, True, None, None, None), - (1, 3, 4, 9,'steps', "incremental", "cdf", False, "spn", True, 4, False, False, 0, False, None, None, None), - (2, 3, 10, 8,'steps', "incremental", "cdf", False, "spn", True, 10, False, False, 0, False, None, None, None), - (5, 3, 5, 8,'steps', "incremental", "cdf", False, "spn", True, 5, False, False, 0, False, None, None, None), - (1, 10, 1, 8,'steps', "incremental", "cdf", False, "spn", True, 1, False, False, 0, False, None, None, None), - (2, 3, 2, 8,'steps', "incremental", "cdf", True, "spn", True, 2, False, False, 0, False, None, None, None), - (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, False, False, 0, False, None, None, None), - (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, False, False, 0, False, "bps", None, None), + ( + 1, + 3, + 4, + 8, + "steps", + None, + None, + False, + "spn", + True, + 4, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 3, + 4, + 8, + "steps", + "obs", + None, + False, + "spn", + True, + 4, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 3, + 4, + 8, + "steps", + "incremental", + None, + False, + "spn", + True, + 4, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 3, + 4, + 8, + "steps", + None, + "mean", + False, + "spn", + True, + 4, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 3, + 4, + 8, + "steps", + None, + "mean", + False, + "spn", + True, + 4, + False, + False, + 0, + True, + None, + None, + None, + ), + ( + 1, + 3, + 4, + 8, + "steps", + None, + "cdf", + False, + "spn", + True, + 4, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + [1, 2, 3], + 4, + 8, + "steps", + None, + "cdf", + False, + "spn", + True, + 4, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 3, + 4, + 8, + "steps", + "incremental", + "cdf", + False, + "spn", + True, + 4, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 3, + 4, + 6, + "steps", + "incremental", + "cdf", + False, + "bps", + True, + 4, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 3, + 4, + 6, + "steps", + "incremental", + "cdf", + False, + "bps", + False, + 4, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 3, + 4, + 6, + "steps", + "incremental", + "cdf", + False, + "bps", + False, + 4, + False, + False, + 0, + True, + None, + None, + None, + ), + ( + 1, + 3, + 4, + 9, + "steps", + "incremental", + "cdf", + False, + "spn", + True, + 4, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 2, + 3, + 10, + 8, + "steps", + "incremental", + "cdf", + False, + "spn", + True, + 10, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 5, + 3, + 5, + 8, + "steps", + "incremental", + "cdf", + False, + "spn", + True, + 5, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 10, + 1, + 8, + "steps", + "incremental", + "cdf", + False, + "spn", + True, + 1, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 2, + 3, + 2, + 8, + "steps", + "incremental", + "cdf", + True, + "spn", + True, + 2, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 3, + 6, + 8, + "steps", + None, + None, + False, + "spn", + True, + 6, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 3, + 6, + 8, + "steps", + None, + None, + False, + "spn", + True, + 6, + False, + False, + 0, + False, + "bps", + None, + None, + ), # Test the case where the radar image contains no rain. - (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, True, False, 0, False, None, None, None), - (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, True, False, 0, False, None, None, None), - (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, True, False, 0, True, None, None, None), + ( + 1, + 3, + 6, + 8, + "steps", + None, + None, + False, + "spn", + True, + 6, + True, + False, + 0, + False, + None, + None, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "incremental", + "cdf", + False, + "spn", + False, + 5, + True, + False, + 0, + False, + None, + None, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "incremental", + "cdf", + False, + "spn", + False, + 5, + True, + False, + 0, + True, + None, + None, + None, + ), # Test the case where the NWP fields contain no rain. - (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, False, True, 0, False, None, None, None), - (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, False, True, 0, True, None, None, None), + ( + 1, + 3, + 6, + 8, + "steps", + None, + None, + False, + "spn", + True, + 6, + False, + True, + 0, + False, + None, + None, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "incremental", + "cdf", + False, + "spn", + False, + 5, + False, + True, + 0, + True, + None, + None, + None, + ), # Test the case where both the radar image and the NWP fields contain no rain. - (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, True, True, 0, False, None, None, None), - (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, True, True, 0, False, None, None, None), - (5, 3, 5, 6,'steps', "obs", "mean", True, "spn", True, 5, True, True, 0, False, None, None, None), + ( + 1, + 3, + 6, + 8, + "steps", + None, + None, + False, + "spn", + True, + 6, + True, + True, + 0, + False, + None, + None, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "incremental", + "cdf", + False, + "spn", + False, + 5, + True, + True, + 0, + False, + None, + None, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "obs", + "mean", + True, + "spn", + True, + 5, + True, + True, + 0, + False, + None, + None, + None, + ), # Test cases where we apply timestep_start_full_nwp_weight - (1, 10, 2, 6,'steps', "incremental", "cdf", False, "bps", False, 2, False, False, 0, True, None, None, 5), - (1, 10, 2, 6,'steps', "incremental", "cdf", False, "spn", False, 2, False, False, 0, False, None, None, 5), + ( + 1, + 10, + 2, + 6, + "steps", + "incremental", + "cdf", + False, + "bps", + False, + 2, + False, + False, + 0, + True, + None, + None, + 5, + ), + ( + 1, + 10, + 2, + 6, + "steps", + "incremental", + "cdf", + False, + "spn", + False, + 2, + False, + False, + 0, + False, + None, + None, + 5, + ), # Test for smooth radar mask - (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, False, False, 80, False, None, None, None), - (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, False, False, 80, False, None, None, None), - (5, 3, 5, 6,'steps', "obs", "mean", False, "spn", False, 5, False, False, 80, False, None, None, None), - (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, False, True, 80, False, None, None, None), - (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, True, False, 80, True, None, None, None), - (5, 3, 5, 6,'steps', "obs", "mean", False, "spn", False, 5, True, True, 80, False, None, None, None), - (5, [1, 2, 3], 5, 6,'steps', "obs", "mean", False, "spn", False, 5, True, True, 80, False, None, None, None), - (5, [1, 3], 5, 6,'steps', "obs", "mean", False, "spn", False, 5, True, True, 80, False, None, None, None), + ( + 1, + 3, + 6, + 8, + "steps", + None, + None, + False, + "spn", + True, + 6, + False, + False, + 80, + False, + None, + None, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "incremental", + "cdf", + False, + "spn", + False, + 5, + False, + False, + 80, + False, + None, + None, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "obs", + "mean", + False, + "spn", + False, + 5, + False, + False, + 80, + False, + None, + None, + None, + ), + ( + 1, + 3, + 6, + 8, + "steps", + None, + None, + False, + "spn", + True, + 6, + False, + True, + 80, + False, + None, + None, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "incremental", + "cdf", + False, + "spn", + False, + 5, + True, + False, + 80, + True, + None, + None, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "obs", + "mean", + False, + "spn", + False, + 5, + True, + True, + 80, + False, + None, + None, + None, + ), + ( + 5, + [1, 2, 3], + 5, + 6, + "steps", + "obs", + "mean", + False, + "spn", + False, + 5, + True, + True, + 80, + False, + None, + None, + None, + ), + ( + 5, + [1, 3], + 5, + 6, + "steps", + "obs", + "mean", + False, + "spn", + False, + 5, + True, + True, + 80, + False, + None, + None, + None, + ), # Test the usage of a max_mask_rim in the mask_kwargs - (1, 3, 6, 8,'steps', None, None, False, "bps", True, 6, False, False, 80, False, None, 40, None), - (5, 3, 5, 6,'steps', "obs", "mean", False, "bps", False, 5, False, False, 80, False, None, 40, None), - (5, 3, 5, 6,'steps', "incremental", "cdf", False, "bps", False, 5, False, False, 80, False, None, 25, None), - (5, 3, 5, 6,'steps', "incremental", "cdf", False, "bps", False, 5, False, False, 80, False, None, 40, None), - (5, 3, 5, 6,'steps', "incremental", "cdf", False, "bps", False, 5, False, False, 80, False, None, 60, None), - #Test the externally provided nowcast - (1, 10, 1, 8,'external_nowcast_det', None, None, False, "spn", True, 1, False, False, 0, False, None, None, None), - (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "bps", True, 1, False, False, 0, False, None, None, None), - (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "spn", True, 1, False, False, 80, False, None, None, None), - (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "bps", True, 1, True, False, 0, False, None, None, None), - (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "spn", True, 1, False, True, 0, False, None, None, None), - (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "bps", True, 1, True, True, 0, False, None, None, None), - (1, 10, 1, 8,'external_nowcast_det', "incremental", "cdf", False, "spn", True, 1, False, False, 0, True, None, None, None), - (1, 10, 1, 8,'external_nowcast_det', "incremental", "obs", False, "bps", True, 1, False, False, 0, False, None, None, None), - (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "bps", True, 1, False, False, 0, False, None, None, 5), - (5, 10, 5, 8,'external_nowcast_ens', "incremental", None, False, "spn", True, 5, False, False, 0, False, None, None, None), - (5, 10, 5, 8,'external_nowcast_ens', "incremental", None, False, "spn", True, 5, False, False, 0, False, None, None, None), - (1, 10, 5, 8,'external_nowcast_ens', "incremental", None, False, "spn", True, 5, False, False, 0, False, None, None, None), - (1, 10, 1, 8,'external_nowcast_ens', "incremental", "cdf", False, "bps", True, 5, False, False, 0, False, None, None, None), - (5, 10, 1, 8,'external_nowcast_ens', "incremental", "obs", False, "spn", True, 5, False, False, 0, False, None, None, None), - (1, 10, 5, 8,'external_nowcast_ens', "incremental", "cdf", False, "bps", True, 5, False, False, 0, False, None, None, 5) + ( + 1, + 3, + 6, + 8, + "steps", + None, + None, + False, + "bps", + True, + 6, + False, + False, + 80, + False, + None, + 40, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "obs", + "mean", + False, + "bps", + False, + 5, + False, + False, + 80, + False, + None, + 40, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "incremental", + "cdf", + False, + "bps", + False, + 5, + False, + False, + 80, + False, + None, + 25, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "incremental", + "cdf", + False, + "bps", + False, + 5, + False, + False, + 80, + False, + None, + 40, + None, + ), + ( + 5, + 3, + 5, + 6, + "steps", + "incremental", + "cdf", + False, + "bps", + False, + 5, + False, + False, + 80, + False, + None, + 60, + None, + ), + # Test the externally provided nowcast + ( + 1, + 10, + 1, + 8, + "external_nowcast_det", + None, + None, + False, + "spn", + True, + 1, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 10, + 1, + 8, + "external_nowcast_det", + "incremental", + None, + False, + "bps", + True, + 1, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 10, + 1, + 8, + "external_nowcast_det", + "incremental", + None, + False, + "spn", + True, + 1, + False, + False, + 80, + False, + None, + None, + None, + ), + ( + 1, + 10, + 1, + 8, + "external_nowcast_det", + "incremental", + None, + False, + "bps", + True, + 1, + True, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 10, + 1, + 8, + "external_nowcast_det", + "incremental", + None, + False, + "spn", + True, + 1, + False, + True, + 0, + False, + None, + None, + None, + ), + ( + 1, + 10, + 1, + 8, + "external_nowcast_det", + "incremental", + None, + False, + "bps", + True, + 1, + True, + True, + 0, + False, + None, + None, + None, + ), + ( + 1, + 10, + 1, + 8, + "external_nowcast_det", + "incremental", + "cdf", + False, + "spn", + True, + 1, + False, + False, + 0, + True, + None, + None, + None, + ), + ( + 1, + 10, + 1, + 8, + "external_nowcast_det", + "incremental", + "obs", + False, + "bps", + True, + 1, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 10, + 1, + 8, + "external_nowcast_det", + "incremental", + None, + False, + "bps", + True, + 1, + False, + False, + 0, + False, + None, + None, + 5, + ), + ( + 5, + 10, + 5, + 8, + "external_nowcast_ens", + "incremental", + None, + False, + "spn", + True, + 5, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 5, + 10, + 5, + 8, + "external_nowcast_ens", + "incremental", + None, + False, + "spn", + True, + 5, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 10, + 5, + 8, + "external_nowcast_ens", + "incremental", + None, + False, + "spn", + True, + 5, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 10, + 1, + 8, + "external_nowcast_ens", + "incremental", + "cdf", + False, + "bps", + True, + 5, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 5, + 10, + 1, + 8, + "external_nowcast_ens", + "incremental", + "obs", + False, + "spn", + True, + 5, + False, + False, + 0, + False, + None, + None, + None, + ), + ( + 1, + 10, + 5, + 8, + "external_nowcast_ens", + "incremental", + "cdf", + False, + "bps", + True, + 5, + False, + False, + 0, + False, + None, + None, + 5, + ), ] -# Extend every existing entry with model_index_offset=0 (default, no offset). -# Add worker-pattern entries that exercise both the default and a non-zero -# model_index_offset path: -# - n_models=1, n_ens_members=1, model_index_offset=0 (worker holds 1 sliced model) -# - n_models=1, n_ens_members=1, model_index_offset=2 (worker holds a later sliced model) -steps_arg_values = [t + (0,) for t in steps_arg_values] + [ - (1, 3, 1, 6, 'steps', None, None, False, "spn", True, 1, False, False, 0, False, None, None, None, 0), - (1, 3, 1, 6, 'steps', "incremental", "cdf", False, "bps", False, 1, False, False, 80, False, None, None, None, 2), -] - -# fmt:on - def run_and_assert_forecast( precip, forecast_kwargs, expected_n_ens_members, n_timesteps, converter, metadata @@ -135,7 +1186,6 @@ def run_and_assert_forecast( "vel_pert_method", "max_mask_rim", "timestep_start_full_nwp_weight", - "model_index_offset", ) @@ -159,7 +1209,7 @@ def test_steps_blending( vel_pert_method, max_mask_rim, timestep_start_full_nwp_weight, - model_index_offset, + write_skill=True, ): pytest.importorskip("cv2") @@ -410,7 +1460,7 @@ def test_steps_blending( n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, blend_nwp_members=blend_nwp_members, - model_index_offset=model_index_offset, + write_skill=write_skill, precip_thr=metadata["threshold"], kmperpixel=1.0, extrap_method="semilagrangian", @@ -569,6 +1619,46 @@ def test_steps_blending_partial_zero_radar(ar_order): ) +@pytest.mark.parametrize( + "write_skill", + [ + True, # designated skill writer (one per NWP model in the pool) + False, # duplicate worker: forecast runs but skill is not written + ], +) +def test_steps_blending_single_process(write_skill): + """Simulate one process-pool worker holding a single NWP model slice. + + This mirrors the one-per-proc mode in multiprocessing.py: each worker + receives one model's precip/velocity slice, its own outdir_path_skill + sub-directory, and write_skill=True only for the designated skill writer + (one per NWP model). With 2 NWP models and 2 STEPS members, both + workers are skill writers (one each). The forecast output shape must + be correct regardless of the write_skill flag. + """ + test_steps_blending( + n_models=1, + timesteps=3, + n_ens_members=1, + n_cascade_levels=6, + nowcasting_method="steps", + mask_method=None, + probmatching_method=None, + blend_nwp_members=False, + weights_method="spn", + decomposed_nwp=True, + expected_n_ens_members=1, + zero_radar=False, + zero_nwp=False, + smooth_radar_mask_range=0, + resample_distribution=False, + vel_pert_method=None, + max_mask_rim=None, + timestep_start_full_nwp_weight=None, + write_skill=write_skill, + ) + + def test_raises_when_n_models_exceeds_n_ens_members(): """n_model_members > n_ens_members must raise ValueError (user forgot to slice).""" pytest.importorskip("cv2") @@ -592,5 +1682,4 @@ def test_raises_when_n_models_exceeds_n_ens_members(): vel_pert_method=None, max_mask_rim=None, timestep_start_full_nwp_weight=None, - model_index_offset=0, ) From 85047ad76552d50e9b5b55ddf58dc13108970538 Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Thu, 28 May 2026 00:05:09 +0200 Subject: [PATCH 13/16] Restore requirements in documentation of steps blending forecast method. --- pysteps/blending/steps.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pysteps/blending/steps.py b/pysteps/blending/steps.py index 2a50113c..e8871417 100644 --- a/pysteps/blending/steps.py +++ b/pysteps/blending/steps.py @@ -3452,11 +3452,14 @@ def forecast( forecasts are computed (relative to the input time step). The elements of the list are required to be in ascending order. timestep: float - Time step of the motion vectors (minutes). + Time step of the motion vectors (minutes). Required if + ``vel_pert_method`` is not ``None`` or ``mask_method`` is + ``'incremental'``. issuetime: datetime Issue time of the forecast. n_ens_members: int - Passed to :class:`StepsBlendingConfig` as ``n_ens_members``. + The number of ensemble members to generate. Must be equal to or larger + than the number of NWP ensemble members / number of NWP models. precip_nowcast: array-like, optional Optional array of shape (n_ens_members,timestep+1,m,n) containing an external nowcast. When provided, the autoregression and advection step From 3e696922d4bb10c7af100506b095855ecde773b6 Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Thu, 28 May 2026 00:14:59 +0200 Subject: [PATCH 14/16] Restore old formatting of tests after terrible accident due to removed fmt: guards --- pysteps/tests/test_blending_steps.py | 1180 ++------------------------ 1 file changed, 59 insertions(+), 1121 deletions(-) diff --git a/pysteps/tests/test_blending_steps.py b/pysteps/tests/test_blending_steps.py index ff7627d9..78d3824b 100644 --- a/pysteps/tests/test_blending_steps.py +++ b/pysteps/tests/test_blending_steps.py @@ -8,1135 +8,73 @@ import pysteps from pysteps import blending, cascade +# fmt:off steps_arg_values = [ - ( - 1, - 3, - 4, - 8, - "steps", - None, - None, - False, - "spn", - True, - 4, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 3, - 4, - 8, - "steps", - "obs", - None, - False, - "spn", - True, - 4, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 3, - 4, - 8, - "steps", - "incremental", - None, - False, - "spn", - True, - 4, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 3, - 4, - 8, - "steps", - None, - "mean", - False, - "spn", - True, - 4, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 3, - 4, - 8, - "steps", - None, - "mean", - False, - "spn", - True, - 4, - False, - False, - 0, - True, - None, - None, - None, - ), - ( - 1, - 3, - 4, - 8, - "steps", - None, - "cdf", - False, - "spn", - True, - 4, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - [1, 2, 3], - 4, - 8, - "steps", - None, - "cdf", - False, - "spn", - True, - 4, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 3, - 4, - 8, - "steps", - "incremental", - "cdf", - False, - "spn", - True, - 4, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 3, - 4, - 6, - "steps", - "incremental", - "cdf", - False, - "bps", - True, - 4, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 3, - 4, - 6, - "steps", - "incremental", - "cdf", - False, - "bps", - False, - 4, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 3, - 4, - 6, - "steps", - "incremental", - "cdf", - False, - "bps", - False, - 4, - False, - False, - 0, - True, - None, - None, - None, - ), - ( - 1, - 3, - 4, - 9, - "steps", - "incremental", - "cdf", - False, - "spn", - True, - 4, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 2, - 3, - 10, - 8, - "steps", - "incremental", - "cdf", - False, - "spn", - True, - 10, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 5, - 3, - 5, - 8, - "steps", - "incremental", - "cdf", - False, - "spn", - True, - 5, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 10, - 1, - 8, - "steps", - "incremental", - "cdf", - False, - "spn", - True, - 1, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 2, - 3, - 2, - 8, - "steps", - "incremental", - "cdf", - True, - "spn", - True, - 2, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 3, - 6, - 8, - "steps", - None, - None, - False, - "spn", - True, - 6, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 3, - 6, - 8, - "steps", - None, - None, - False, - "spn", - True, - 6, - False, - False, - 0, - False, - "bps", - None, - None, - ), + (1, 3, 4, 8, 'steps', None, None, False, "spn", True, 4, False, False, 0, False, None, None, None), + (1, 3, 4, 8,'steps', "obs", None, False, "spn", True, 4, False, False, 0, False, None, None, None), + (1, 3, 4, 8,'steps', "incremental", None, False, "spn", True, 4, False, False, 0, False, None, None, None), + (1, 3, 4, 8,'steps', None, "mean", False, "spn", True, 4, False, False, 0, False, None, None, None), + (1, 3, 4, 8,'steps', None, "mean", False, "spn", True, 4, False, False, 0, True, None, None, None), + (1, 3, 4, 8,'steps', None, "cdf", False, "spn", True, 4, False, False, 0, False, None, None, None), + (1, [1, 2, 3], 4, 8,'steps', None, "cdf", False, "spn", True, 4, False, False, 0, False, None, None, None), + (1, 3, 4, 8,'steps', "incremental", "cdf", False, "spn", True, 4, False, False, 0, False, None, None, None), + (1, 3, 4, 6,'steps', "incremental", "cdf", False, "bps", True, 4, False, False, 0, False, None, None, None), + (1, 3, 4, 6,'steps', "incremental", "cdf", False, "bps", False, 4, False, False, 0, False, None, None, None), + (1, 3, 4, 6,'steps', "incremental", "cdf", False, "bps", False, 4, False, False, 0, True, None, None, None), + (1, 3, 4, 9,'steps', "incremental", "cdf", False, "spn", True, 4, False, False, 0, False, None, None, None), + (2, 3, 10, 8,'steps', "incremental", "cdf", False, "spn", True, 10, False, False, 0, False, None, None, None), + (5, 3, 5, 8,'steps', "incremental", "cdf", False, "spn", True, 5, False, False, 0, False, None, None, None), + (1, 10, 1, 8,'steps', "incremental", "cdf", False, "spn", True, 1, False, False, 0, False, None, None, None), + (2, 3, 2, 8,'steps', "incremental", "cdf", True, "spn", True, 2, False, False, 0, False, None, None, None), + (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, False, False, 0, False, None, None, None), + (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, False, False, 0, False, "bps", None, None), # Test the case where the radar image contains no rain. - ( - 1, - 3, - 6, - 8, - "steps", - None, - None, - False, - "spn", - True, - 6, - True, - False, - 0, - False, - None, - None, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "incremental", - "cdf", - False, - "spn", - False, - 5, - True, - False, - 0, - False, - None, - None, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "incremental", - "cdf", - False, - "spn", - False, - 5, - True, - False, - 0, - True, - None, - None, - None, - ), + (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, True, False, 0, False, None, None, None), + (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, True, False, 0, False, None, None, None), + (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, True, False, 0, True, None, None, None), # Test the case where the NWP fields contain no rain. - ( - 1, - 3, - 6, - 8, - "steps", - None, - None, - False, - "spn", - True, - 6, - False, - True, - 0, - False, - None, - None, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "incremental", - "cdf", - False, - "spn", - False, - 5, - False, - True, - 0, - True, - None, - None, - None, - ), + (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, False, True, 0, False, None, None, None), + (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, False, True, 0, True, None, None, None), # Test the case where both the radar image and the NWP fields contain no rain. - ( - 1, - 3, - 6, - 8, - "steps", - None, - None, - False, - "spn", - True, - 6, - True, - True, - 0, - False, - None, - None, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "incremental", - "cdf", - False, - "spn", - False, - 5, - True, - True, - 0, - False, - None, - None, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "obs", - "mean", - True, - "spn", - True, - 5, - True, - True, - 0, - False, - None, - None, - None, - ), + (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, True, True, 0, False, None, None, None), + (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, True, True, 0, False, None, None, None), + (5, 3, 5, 6,'steps', "obs", "mean", True, "spn", True, 5, True, True, 0, False, None, None, None), # Test cases where we apply timestep_start_full_nwp_weight - ( - 1, - 10, - 2, - 6, - "steps", - "incremental", - "cdf", - False, - "bps", - False, - 2, - False, - False, - 0, - True, - None, - None, - 5, - ), - ( - 1, - 10, - 2, - 6, - "steps", - "incremental", - "cdf", - False, - "spn", - False, - 2, - False, - False, - 0, - False, - None, - None, - 5, - ), + (1, 10, 2, 6,'steps', "incremental", "cdf", False, "bps", False, 2, False, False, 0, True, None, None, 5), + (1, 10, 2, 6,'steps', "incremental", "cdf", False, "spn", False, 2, False, False, 0, False, None, None, 5), # Test for smooth radar mask - ( - 1, - 3, - 6, - 8, - "steps", - None, - None, - False, - "spn", - True, - 6, - False, - False, - 80, - False, - None, - None, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "incremental", - "cdf", - False, - "spn", - False, - 5, - False, - False, - 80, - False, - None, - None, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "obs", - "mean", - False, - "spn", - False, - 5, - False, - False, - 80, - False, - None, - None, - None, - ), - ( - 1, - 3, - 6, - 8, - "steps", - None, - None, - False, - "spn", - True, - 6, - False, - True, - 80, - False, - None, - None, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "incremental", - "cdf", - False, - "spn", - False, - 5, - True, - False, - 80, - True, - None, - None, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "obs", - "mean", - False, - "spn", - False, - 5, - True, - True, - 80, - False, - None, - None, - None, - ), - ( - 5, - [1, 2, 3], - 5, - 6, - "steps", - "obs", - "mean", - False, - "spn", - False, - 5, - True, - True, - 80, - False, - None, - None, - None, - ), - ( - 5, - [1, 3], - 5, - 6, - "steps", - "obs", - "mean", - False, - "spn", - False, - 5, - True, - True, - 80, - False, - None, - None, - None, - ), + (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, False, False, 80, False, None, None, None), + (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, False, False, 80, False, None, None, None), + (5, 3, 5, 6,'steps', "obs", "mean", False, "spn", False, 5, False, False, 80, False, None, None, None), + (1, 3, 6, 8,'steps', None, None, False, "spn", True, 6, False, True, 80, False, None, None, None), + (5, 3, 5, 6,'steps', "incremental", "cdf", False, "spn", False, 5, True, False, 80, True, None, None, None), + (5, 3, 5, 6,'steps', "obs", "mean", False, "spn", False, 5, True, True, 80, False, None, None, None), + (5, [1, 2, 3], 5, 6,'steps', "obs", "mean", False, "spn", False, 5, True, True, 80, False, None, None, None), + (5, [1, 3], 5, 6,'steps', "obs", "mean", False, "spn", False, 5, True, True, 80, False, None, None, None), # Test the usage of a max_mask_rim in the mask_kwargs - ( - 1, - 3, - 6, - 8, - "steps", - None, - None, - False, - "bps", - True, - 6, - False, - False, - 80, - False, - None, - 40, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "obs", - "mean", - False, - "bps", - False, - 5, - False, - False, - 80, - False, - None, - 40, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "incremental", - "cdf", - False, - "bps", - False, - 5, - False, - False, - 80, - False, - None, - 25, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "incremental", - "cdf", - False, - "bps", - False, - 5, - False, - False, - 80, - False, - None, - 40, - None, - ), - ( - 5, - 3, - 5, - 6, - "steps", - "incremental", - "cdf", - False, - "bps", - False, - 5, - False, - False, - 80, - False, - None, - 60, - None, - ), - # Test the externally provided nowcast - ( - 1, - 10, - 1, - 8, - "external_nowcast_det", - None, - None, - False, - "spn", - True, - 1, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 10, - 1, - 8, - "external_nowcast_det", - "incremental", - None, - False, - "bps", - True, - 1, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 10, - 1, - 8, - "external_nowcast_det", - "incremental", - None, - False, - "spn", - True, - 1, - False, - False, - 80, - False, - None, - None, - None, - ), - ( - 1, - 10, - 1, - 8, - "external_nowcast_det", - "incremental", - None, - False, - "bps", - True, - 1, - True, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 10, - 1, - 8, - "external_nowcast_det", - "incremental", - None, - False, - "spn", - True, - 1, - False, - True, - 0, - False, - None, - None, - None, - ), - ( - 1, - 10, - 1, - 8, - "external_nowcast_det", - "incremental", - None, - False, - "bps", - True, - 1, - True, - True, - 0, - False, - None, - None, - None, - ), - ( - 1, - 10, - 1, - 8, - "external_nowcast_det", - "incremental", - "cdf", - False, - "spn", - True, - 1, - False, - False, - 0, - True, - None, - None, - None, - ), - ( - 1, - 10, - 1, - 8, - "external_nowcast_det", - "incremental", - "obs", - False, - "bps", - True, - 1, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 10, - 1, - 8, - "external_nowcast_det", - "incremental", - None, - False, - "bps", - True, - 1, - False, - False, - 0, - False, - None, - None, - 5, - ), - ( - 5, - 10, - 5, - 8, - "external_nowcast_ens", - "incremental", - None, - False, - "spn", - True, - 5, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 5, - 10, - 5, - 8, - "external_nowcast_ens", - "incremental", - None, - False, - "spn", - True, - 5, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 10, - 5, - 8, - "external_nowcast_ens", - "incremental", - None, - False, - "spn", - True, - 5, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 10, - 1, - 8, - "external_nowcast_ens", - "incremental", - "cdf", - False, - "bps", - True, - 5, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 5, - 10, - 1, - 8, - "external_nowcast_ens", - "incremental", - "obs", - False, - "spn", - True, - 5, - False, - False, - 0, - False, - None, - None, - None, - ), - ( - 1, - 10, - 5, - 8, - "external_nowcast_ens", - "incremental", - "cdf", - False, - "bps", - True, - 5, - False, - False, - 0, - False, - None, - None, - 5, - ), + (1, 3, 6, 8,'steps', None, None, False, "bps", True, 6, False, False, 80, False, None, 40, None), + (5, 3, 5, 6,'steps', "obs", "mean", False, "bps", False, 5, False, False, 80, False, None, 40, None), + (5, 3, 5, 6,'steps', "incremental", "cdf", False, "bps", False, 5, False, False, 80, False, None, 25, None), + (5, 3, 5, 6,'steps', "incremental", "cdf", False, "bps", False, 5, False, False, 80, False, None, 40, None), + (5, 3, 5, 6,'steps', "incremental", "cdf", False, "bps", False, 5, False, False, 80, False, None, 60, None), + #Test the externally provided nowcast + (1, 10, 1, 8,'external_nowcast_det', None, None, False, "spn", True, 1, False, False, 0, False, None, None, None), + (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "bps", True, 1, False, False, 0, False, None, None, None), + (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "spn", True, 1, False, False, 80, False, None, None, None), + (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "bps", True, 1, True, False, 0, False, None, None, None), + (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "spn", True, 1, False, True, 0, False, None, None, None), + (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "bps", True, 1, True, True, 0, False, None, None, None), + (1, 10, 1, 8,'external_nowcast_det', "incremental", "cdf", False, "spn", True, 1, False, False, 0, True, None, None, None), + (1, 10, 1, 8,'external_nowcast_det', "incremental", "obs", False, "bps", True, 1, False, False, 0, False, None, None, None), + (1, 10, 1, 8,'external_nowcast_det', "incremental", None, False, "bps", True, 1, False, False, 0, False, None, None, 5), + (5, 10, 5, 8,'external_nowcast_ens', "incremental", None, False, "spn", True, 5, False, False, 0, False, None, None, None), + (5, 10, 5, 8,'external_nowcast_ens', "incremental", None, False, "spn", True, 5, False, False, 0, False, None, None, None), + (1, 10, 5, 8,'external_nowcast_ens', "incremental", None, False, "spn", True, 5, False, False, 0, False, None, None, None), + (1, 10, 1, 8,'external_nowcast_ens', "incremental", "cdf", False, "bps", True, 5, False, False, 0, False, None, None, None), + (5, 10, 1, 8,'external_nowcast_ens', "incremental", "obs", False, "spn", True, 5, False, False, 0, False, None, None, None), + (1, 10, 5, 8,'external_nowcast_ens', "incremental", "cdf", False, "bps", True, 5, False, False, 0, False, None, None, 5) ] +# fmt:on def run_and_assert_forecast( From ae6d405690ea702cf8bb6fca87e524acb671c136 Mon Sep 17 00:00:00 2001 From: Lesley De Cruz Date: Thu, 28 May 2026 00:21:28 +0200 Subject: [PATCH 15/16] Silence codacity with correct position of doc summary (on the second line) --- pysteps/tests/test_blending_steps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysteps/tests/test_blending_steps.py b/pysteps/tests/test_blending_steps.py index 78d3824b..b98aaae5 100644 --- a/pysteps/tests/test_blending_steps.py +++ b/pysteps/tests/test_blending_steps.py @@ -565,7 +565,8 @@ def test_steps_blending_partial_zero_radar(ar_order): ], ) def test_steps_blending_single_process(write_skill): - """Simulate one process-pool worker holding a single NWP model slice. + """ + Simulate one process-pool worker holding a single NWP model slice. This mirrors the one-per-proc mode in multiprocessing.py: each worker receives one model's precip/velocity slice, its own outdir_path_skill From f52c8a8709dea9e8a82fddebe710de73eebbf272 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 22:33:55 +0000 Subject: [PATCH 16/16] docs: restore forecast parameter doc formatting --- pysteps/blending/steps.py | 214 +++++++++++++++++++++++++++++++++++--- 1 file changed, 197 insertions(+), 17 deletions(-) diff --git a/pysteps/blending/steps.py b/pysteps/blending/steps.py index e8871417..d8d371e4 100644 --- a/pysteps/blending/steps.py +++ b/pysteps/blending/steps.py @@ -3466,23 +3466,203 @@ def forecast( are skipped for the extrapolation cascade and this field is used instead. Requires ``nowcasting_method='external_nowcast'``. Defaults to None. - Configuration parameters - ------------------------ - All remaining keyword arguments correspond directly to fields in - :class:`StepsBlendingConfig`, with the following name differences: - - ======================== ================================ - ``forecast()`` argument ``StepsBlendingConfig`` field - ======================== ================================ - ``precip_thr`` ``precip_threshold`` - ``norain_thr`` ``norain_threshold`` - ``extrap_method`` ``extrapolation_method`` - ``decomp_method`` ``decomposition_method`` - ``vel_pert_method`` ``velocity_perturbation_method`` - ``extrap_kwargs`` ``extrapolation_kwargs`` - ``vel_pert_kwargs`` ``velocity_perturbation_kwargs`` - ``clim_kwargs`` ``climatology_kwargs`` - ======================== ================================ + n_cascade_levels: int, optional + The number of cascade levels to use. Defaults to 6, + see issue #385 on GitHub. + blend_nwp_members: bool + Check if NWP models/members should be used individually, or if all of + them are blended together per nowcast ensemble member. Standard set to + false. + precip_thr: float, optional + Specifies the threshold value for minimum observable precipitation + intensity. Required if mask_method is not None or conditional is True. + norain_thr: float + Specifies the threshold value for the fraction of rainy (see above) pixels + in the radar rainfall field below which we consider there to be no rain. + Depends on the amount of clutter typically present. + Standard set to 0.0 + kmperpixel: float, optional + Spatial resolution of the input data (kilometers/pixel). Required if + vel_pert_method is not None or mask_method is 'incremental'. + extrap_method: str, optional + Name of the extrapolation method to use. See the documentation of + :py:mod:`pysteps.extrapolation.interface`. + decomp_method: {'fft'}, optional + Name of the cascade decomposition method to use. See the documentation + of :py:mod:`pysteps.cascade.interface`. + bandpass_filter_method: {'gaussian', 'uniform'}, optional + Name of the bandpass filter method to use with the cascade decomposition. + See the documentation of :py:mod:`pysteps.cascade.interface`. + nowcasting_method: {'steps', 'external_nowcast'}, + Name of the nowcasting method used to generate the nowcasts. If an external + nowcast is provided, the script will use this as input and bypass the + autoregression and advection of the extrapolation cascade. Defaults to 'steps', + which follows the method described in :cite:`Imhoff2023`. Note, if + nowcasting_method is 'external_nowcast', precip_nowcast cannot be None. + noise_method: {'parametric','nonparametric','ssft','nested',None}, optional + Name of the noise generator to use for perturbating the precipitation + field. See the documentation of :py:mod:`pysteps.noise.interface`. If set to None, + no noise is generated. + noise_stddev_adj: {'auto','fixed',None}, optional + Optional adjustment for the standard deviations of the noise fields added + to each cascade level. This is done to compensate incorrect std. dev. + estimates of casace levels due to presence of no-rain areas. 'auto'=use + the method implemented in :py:func:`pysteps.noise.utils.compute_noise_stddev_adjs`. + 'fixed'= use the formula given in :cite:`BPS2006` (eq. 6), None=disable + noise std. dev adjustment. + ar_order: int, optional + The order of the autoregressive model to use. Must be >= 1. + vel_pert_method: {'bps',None}, optional + Name of the noise generator to use for perturbing the advection field. See + the documentation of :py:mod:`pysteps.noise.interface`. If set to None, the advection + field is not perturbed. + weights_method: {'bps','spn'}, optional + The calculation method of the blending weights. Options are the method + by :cite:`BPS2006` and the covariance-based method by :cite:`SPN2013`. + Defaults to bps. + timestep_start_full_nwp_weight: int, optional. + The timestep, which should be smaller than timesteps, at which a linear + transition takes place from the calculated weights to full (1.0) NWP weight + (and zero extrapolation and noise weight) to ensure the blending + procedure becomes equal to the NWP forecast(s) at the last timestep + of the blending procedure. If not provided, the blending stick to the + theoretical weights provided by the chosen weights_method for a given + lead time and skill of each blending component. + conditional: bool, optional + If set to True, compute the statistics of the precipitation field + conditionally by excluding pixels where the values are below the threshold + precip_thr. + probmatching_method: {'cdf','mean',None}, optional + Method for matching the statistics of the forecast field with those of + the most recently observed one. 'cdf'=map the forecast CDF to the observed + one, 'mean'=adjust only the conditional mean value of the forecast field + in precipitation areas, None=no matching applied. Using 'mean' requires + that mask_method is not None. + mask_method: {'obs','incremental',None}, optional + The method to use for masking no precipitation areas in the forecast field. + The masked pixels are set to the minimum value of the observations. + 'obs' = apply precip_thr to the most recently observed precipitation intensity + field, 'incremental' = iteratively buffer the mask with a certain rate + (currently it is 1 km/min), None=no masking. + resample_distribution: bool, optional + Method to resample the distribution from the extrapolation and NWP cascade as input + for the probability matching. Not resampling these distributions may lead to losing + some extremes when the weight of both the extrapolation and NWP cascade is similar. + Defaults to True. + smooth_radar_mask_range: int, Default is 0. + Method to smooth the transition between the radar-NWP-noise blend and the NWP-noise + blend near the edge of the radar domain (radar mask), where the radar data is either + not present anymore or is not reliable. If set to 0 (grid cells), this generates a + normal forecast without smoothing. To create a smooth mask, this range should be a + positive value, representing a buffer band of a number of pixels by which the mask + is cropped and smoothed. The smooth radar mask removes the hard edges between NWP + and radar in the final blended product. Typically, a value between 50 and 100 km + can be used. 80 km generally gives good results. + callback: function, optional + Optional function that is called after computation of each time step of + the nowcast. The function takes one argument: a three-dimensional array + of shape (n_ens_members,h,w), where h and w are the height and width + of the input field precip, respectively. This can be used, for instance, + writing the outputs into files. + return_output: bool, optional + Set to False to disable returning the outputs as numpy arrays. This can + save memory if the intermediate results are written to output files using + the callback function. + seed: int, optional + Optional seed number for the random generators. + num_workers: int, optional + The number of workers to use for parallel computation. Applicable if dask + is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it + is advisable to disable OpenMP by setting the environment variable + OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous + threads. + fft_method: str, optional + A string defining the FFT method to use (see FFT methods in + :py:func:`pysteps.utils.interface.get_method`). + Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed, + the recommended method is 'pyfftw'. + domain: {"spatial", "spectral"} + If "spatial", all computations are done in the spatial domain (the + classical STEPS model). If "spectral", the AR(2) models and stochastic + perturbations are applied directly in the spectral domain to reduce + memory footprint and improve performance :cite:`PCH2019b`. + outdir_path_skill: string, optional + Path to folder where the historical skill are stored. Defaults to + path_workdir from rcparams. If no path is given, './tmp' will be used. + extrap_kwargs: dict, optional + Optional dictionary containing keyword arguments for the extrapolation + method. See the documentation of :py:func:`pysteps.extrapolation.interface`. + filter_kwargs: dict, optional + Optional dictionary containing keyword arguments for the filter method. + See the documentation of :py:mod:`pysteps.cascade.bandpass_filters`. + noise_kwargs: dict, optional + Optional dictionary containing keyword arguments for the initializer of + the noise generator. See the documentation of :py:mod:`pysteps.noise.fftgenerators`. + vel_pert_kwargs: dict, optional + Optional dictionary containing keyword arguments 'p_par' and 'p_perp' for + the initializer of the velocity perturbator. The choice of the optimal + parameters depends on the domain and the used optical flow method. + + Default parameters from :cite:`BPS2006`: + p_par = [10.88, 0.23, -7.68] + p_perp = [5.76, 0.31, -2.72] + + Parameters fitted to the data (optical flow/domain): + + darts/fmi: + p_par = [13.71259667, 0.15658963, -16.24368207] + p_perp = [8.26550355, 0.17820458, -9.54107834] + + darts/mch: + p_par = [24.27562298, 0.11297186, -27.30087471] + p_perp = [-7.80797846e+01, -3.38641048e-02, 7.56715304e+01] + + darts/fmi+mch: + p_par = [16.55447057, 0.14160448, -19.24613059] + p_perp = [14.75343395, 0.11785398, -16.26151612] + + lucaskanade/fmi: + p_par = [2.20837526, 0.33887032, -2.48995355] + p_perp = [2.21722634, 0.32359621, -2.57402761] + + lucaskanade/mch: + p_par = [2.56338484, 0.3330941, -2.99714349] + p_perp = [1.31204508, 0.3578426, -1.02499891] + + lucaskanade/fmi+mch: + p_par = [2.31970635, 0.33734287, -2.64972861] + p_perp = [1.90769947, 0.33446594, -2.06603662] + + vet/fmi: + p_par = [0.25337388, 0.67542291, 11.04895538] + p_perp = [0.02432118, 0.99613295, 7.40146505] + + vet/mch: + p_par = [0.5075159, 0.53895212, 7.90331791] + p_perp = [0.68025501, 0.41761289, 4.73793581] + + vet/fmi+mch: + p_par = [0.29495222, 0.62429207, 8.6804131 ] + p_perp = [0.23127377, 0.59010281, 5.98180004] + + fmi=Finland, mch=Switzerland, fmi+mch=both pooled into the same data set + + The above parameters have been fitted by using run_vel_pert_analysis.py + and fit_vel_pert_params.py located in the scripts directory. + + See :py:mod:`pysteps.noise.motion` for additional documentation. + clim_kwargs: dict, optional + Optional dictionary containing keyword arguments for the climatological + skill file. Arguments can consist of: 'outdir_path', 'n_models' + (the number of NWP models) and 'window_length' (the minimum number of + days the clim file should have, otherwise the default is used). + mask_kwargs: dict + Optional dictionary containing mask keyword arguments 'mask_f', + 'mask_rim' and 'max_mask_rim', the factor defining the the mask + increment and the (maximum) rim size, respectively. + The mask increment is defined as mask_f*timestep/kmperpixel. + measure_time: bool + If set to True, measure, print and return the computation time. Returns