diff --git a/CHANGELOG.md b/CHANGELOG.md index a38baf1d..de394e74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Changed + +- Standardized eta handling across all damage reduction recipes. Previously, the `adding_up` recipe did not use eta parameters while `risk_aversion` did. Now all recipes require and use eta values consistently. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) + - File naming convention updated to include eta value for all reduced damage outputs (e.g., `adding_up_cc_eta1.0.zarr`) + - Simplified `reduce_damages()` and `subset_USA_reduced_damages()` functions by removing conditional logic for eta-less processing + - Updated `StackedDamages.adding_up_damages` property to reference eta-specific file paths +- Refactored mortality damage preprocessing to support eta-aware damage loading. The `prep_mortality_damages()` function now accepts multiple eta values and merges damage data across the eta dimension. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) +- Added support for mortality version 9 with VLY (Value of a Life Year) valuation using EPA population-averaged scaling. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) +- Updated chunking strategy in `reduce_damages()` to handle eta as a mapped dimension alongside batch operations, improving consistency in batch dimension handling for both standard and quantile regression workflows. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) + +### Fixed + +- Pinned `scipy=1.15.3` to resolve `statsmodels==0.14.4` import issues. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) +- Updated all unit tests to work with eta-aware damage processing, including test fixtures for mortality damages and adding-up calculations. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) + +### Removed + +- Removed assertion preventing `adding_up` recipe from accepting eta arguments, as this recipe now processes damages with eta values like all other recipes. ([PR #417](https://github.com/ClimateImpactLab/dscim/pull/417), [@JMGilbert](https://github.com/JMGilbert)) ## [0.7.0] - 2025-08-15 diff --git a/requirements.txt b/requirements.txt index 61776210..359799fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ impactlab-tools==0.6.0 p_tqdm==1.4.2 pyarrow==21.0.0 numcodecs==0.15.1 +scipy==1.15.3 diff --git a/src/dscim/menu/simple_storage.py b/src/dscim/menu/simple_storage.py index 37dd7ce3..9121e2fd 100644 --- a/src/dscim/menu/simple_storage.py +++ b/src/dscim/menu/simple_storage.py @@ -384,8 +384,8 @@ def gdppc(self): def adding_up_damages(self): """This property calls pre-calculated adding-up IR-level 'mean' over batches.""" - mean_cc = f"{self.ce_path}/adding_up_cc.zarr" - mean_no_cc = f"{self.ce_path}/adding_up_no_cc.zarr" + mean_cc = f"{self.ce_path}/adding_up_cc_eta{self.eta}.zarr" + mean_no_cc = f"{self.ce_path}/adding_up_no_cc_eta{self.eta}.zarr" if os.path.exists(mean_cc) and os.path.exists(mean_no_cc): self.logger.info( diff --git a/src/dscim/preprocessing/input_damages.py b/src/dscim/preprocessing/input_damages.py index c6a32b0c..d576fd66 100644 --- a/src/dscim/preprocessing/input_damages.py +++ b/src/dscim/preprocessing/input_damages.py @@ -721,6 +721,7 @@ def prep_mortality_damages( outpath, mortality_version, path_econ, + etas, ): ec = EconVars(path_econ=path_econ) @@ -744,6 +745,10 @@ def prep_mortality_damages( scaling_deaths = "epa_row" scaling_costs = "epa_scaled" valuation = "vsl" + elif mortality_version == 9: + scaling_deaths = "epa_popavg" + scaling_costs = "epa_scaled" + valuation = "vly" else: raise ValueError("Mortality version not valid: ", str(mortality_version)) @@ -776,7 +781,19 @@ def prep( valuation=valuation, ).drop(["gcm", "valuation"]) - data = xr.open_mfdataset(paths, preprocess=prep, parallel=True, engine="zarr") + d_ls = [] + for eta in etas: + paths_ls = [paths.format(i, eta) for i in range(15)] + data = ( + xr.open_mfdataset( + paths_ls, preprocess=prep, parallel=True, engine="zarr" + ) + .assign_coords({"eta": eta}) + .expand_dims("eta") + ) + d_ls.append(data) + + data = xr.merge(d_ls) damages = xr.Dataset( { @@ -798,6 +815,7 @@ def prep( damages = damages.chunk( { "batch": 15, + "eta": 1, "ssp": 1, "model": 1, "rcp": 1, diff --git a/src/dscim/preprocessing/preprocessing.py b/src/dscim/preprocessing/preprocessing.py index 53a56b3b..92c47c20 100644 --- a/src/dscim/preprocessing/preprocessing.py +++ b/src/dscim/preprocessing/preprocessing.py @@ -82,15 +82,6 @@ def reduce_damages( zero=False, quantreg=False, ): - if recipe == "adding_up": - assert ( - eta is None - ), "Adding up does not take an eta argument. Please set to None." - # client = Client(n_workers=35, memory_limit="9G", threads_per_worker=1) - - if recipe == "risk_aversion": - assert not quantreg, "Quantile regression is not compatible with risk aversion. Please set quantreg to False." - with open(config) as stream: c = yaml.safe_load(stream) params = c["sectors"][sector] @@ -123,15 +114,16 @@ def reduce_damages( "model": 1, "ssp": 1, } + map_dims = ["eta"] if quantreg: chunkies["batch"] = 1 - ce_batch_dims = [i for i in gdppc.dims] + [ - i for i in ds.dims if i not in gdppc.dims - ] else: - ce_batch_dims = [i for i in gdppc.dims] + [ - i for i in ds.dims if i not in gdppc.dims and i != "batch" - ] + map_dims.append("batch") + + ce_batch_dims = [i for i in gdppc.dims] + [ + i for i in ds.dims if i not in gdppc.dims and i not in map_dims + ] + ce_batch_coords = {c: ds[c].values for c in ce_batch_dims} ce_batch_coords["region"] = [ i for i in gdppc.region.values if i in ce_batch_coords["region"] @@ -145,6 +137,8 @@ def reduce_damages( ).chunk(chunkies) other = xr.open_zarr(damages).chunk(chunkies) + if "eta" in other.coords: + other = other.sel(eta=eta, drop=True) out = other.map_blocks( ce_from_chunk, @@ -172,7 +166,7 @@ def reduce_damages( if recipe == "adding_up": out.to_zarr( - f"{outpath}/{recipe}_{reduction}.zarr", + f"{outpath}/{recipe}_{reduction}_eta{eta}.zarr", consolidated=True, mode="w", ) @@ -289,14 +283,9 @@ def subset_USA_reduced_damages( eta, input_path, ): - if recipe == "adding_up": - ds = xr.open_zarr( - f"{input_path}/{sector}/{recipe}_{reduction}.zarr", - ) - elif recipe == "risk_aversion": - ds = xr.open_zarr( - f"{input_path}/{sector}/{recipe}_{reduction}_eta{eta}.zarr", - ) + ds = xr.open_zarr( + f"{input_path}/{sector}/{recipe}_{reduction}_eta{eta}.zarr", + ) US_territories = [ "USA", @@ -321,18 +310,11 @@ def subset_USA_reduced_damages( for var in subset.variables: subset[var].encoding.clear() - if recipe == "adding_up": - subset.to_zarr( - f"{input_path}/{sector}_USA/{recipe}_{reduction}.zarr", - consolidated=True, - mode="w", - ) - elif recipe == "risk_aversion": - subset.to_zarr( - f"{input_path}/{sector}_USA/{recipe}_{reduction}_eta{eta}.zarr", - consolidated=True, - mode="w", - ) + subset.to_zarr( + f"{input_path}/{sector}_USA/{recipe}_{reduction}_eta{eta}.zarr", + consolidated=True, + mode="w", + ) def subset_USA_ssp_econ( diff --git a/tests/data/CEs/adding_up_cc.zarr/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/.zgroup b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zgroup similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/.zgroup rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zgroup diff --git a/tests/data/CEs/adding_up_cc.zarr/.zmetadata b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zmetadata similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/.zmetadata rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/.zmetadata diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/0.0.0.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.0.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/0.0.0.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.0.0.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/0.0.1.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.0.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/0.0.1.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.0.1.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/0.1.0.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.1.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/0.1.0.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.1.0.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/0.1.1.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.1.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/0.1.1.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/0.1.1.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/1.0.0.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.0.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/1.0.0.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.0.0.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/1.0.1.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.0.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/1.0.1.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.0.1.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/1.1.0.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.1.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/1.1.0.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.1.0.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/cc/1.1.1.0.0.0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.1.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/cc/1.1.1.0.0.0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/cc/1.1.1.0.0.0 diff --git a/tests/data/CEs/adding_up_cc.zarr/gcm/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/gcm/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/gcm/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/gcm/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/gcm/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/gcm/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/gcm/0 diff --git a/tests/data/CEs/adding_up_cc.zarr/model/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/model/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/model/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/model/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/model/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/model/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/model/0 diff --git a/tests/data/CEs/adding_up_cc.zarr/rcp/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/rcp/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/rcp/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/rcp/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/rcp/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/rcp/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/rcp/0 diff --git a/tests/data/CEs/adding_up_cc.zarr/region/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/region/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/region/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/region/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/region/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/region/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/region/0 diff --git a/tests/data/CEs/adding_up_cc.zarr/ssp/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/ssp/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/ssp/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/ssp/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/ssp/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/ssp/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/ssp/0 diff --git a/tests/data/CEs/adding_up_cc.zarr/year/.zarray b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/.zarray similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/year/.zarray rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/.zarray diff --git a/tests/data/CEs/adding_up_cc.zarr/year/.zattrs b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/year/.zattrs rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/.zattrs diff --git a/tests/data/CEs/adding_up_cc.zarr/year/0 b/tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/0 similarity index 100% rename from tests/data/CEs/adding_up_cc.zarr/year/0 rename to tests/data/CEs/adding_up_cc_eta1.421158116.zarr/year/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/.zgroup b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zgroup similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/.zgroup rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zgroup diff --git a/tests/data/CEs/adding_up_no_cc.zarr/.zmetadata b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zmetadata similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/.zmetadata rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/.zmetadata diff --git a/tests/data/CEs/adding_up_no_cc.zarr/gcm/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/gcm/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/gcm/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/gcm/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/gcm/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/gcm/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/gcm/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/model/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/model/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/model/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/model/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/model/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/model/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/model/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.0.0.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.0.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.0.0.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.0.0.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.0.1.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.0.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.0.1.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.0.1.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.1.0.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.1.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.1.0.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.1.0.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.1.1.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.1.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/0.1.1.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/0.1.1.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.0.0.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.0.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.0.0.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.0.0.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.0.1.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.0.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.0.1.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.0.1.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.1.0.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.1.0.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.1.0.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.1.0.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.1.1.0.0.0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.1.1.0.0.0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/no_cc/1.1.1.0.0.0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/no_cc/1.1.1.0.0.0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/rcp/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/rcp/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/rcp/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/rcp/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/rcp/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/rcp/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/rcp/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/region/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/region/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/region/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/region/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/region/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/region/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/region/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/ssp/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/ssp/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/ssp/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/ssp/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/ssp/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/ssp/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/ssp/0 diff --git a/tests/data/CEs/adding_up_no_cc.zarr/year/.zarray b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/.zarray similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/year/.zarray rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/.zarray diff --git a/tests/data/CEs/adding_up_no_cc.zarr/year/.zattrs b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/.zattrs similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/year/.zattrs rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/.zattrs diff --git a/tests/data/CEs/adding_up_no_cc.zarr/year/0 b/tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/0 similarity index 100% rename from tests/data/CEs/adding_up_no_cc.zarr/year/0 rename to tests/data/CEs/adding_up_no_cc_eta1.421158116.zarr/year/0 diff --git a/tests/test_input_damages.py b/tests/test_input_damages.py index 75d1b0a0..48ddbd9e 100644 --- a/tests/test_input_damages.py +++ b/tests/test_input_damages.py @@ -993,7 +993,7 @@ def test_calculate_energy_damages( xr.testing.assert_equal(ds_out_expected, ds_out_actual) -@pytest.mark.parametrize("version_test", [0, 1, 4, 5]) +@pytest.mark.parametrize("version_test", [0, 1, 4, 5, 9]) def test_prep_mortality_damages( tmp_path, version_test, @@ -1002,81 +1002,83 @@ def test_prep_mortality_damages( """ Test that prep_mortality_damages correctly reshapes different versions of mortality estimate runs for use in integration system and saves to zarr file """ - for b in ["6", "9"]: - ds_in = xr.Dataset( - { - "monetized_costs": ( - [ - "gcm", - "batch", - "ssp", - "rcp", - "model", - "year", - "region", - "scaling", - "valuation", - ], - np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 0), - ), - "monetized_deaths": ( - [ - "gcm", - "batch", - "ssp", - "rcp", - "model", - "year", - "region", - "scaling", - "valuation", - ], - np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 1), - ), - "monetized_histclim_deaths": ( - [ - "gcm", - "batch", - "ssp", - "rcp", - "model", - "year", - "region", - "scaling", - "valuation", - ], - np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 2), - ), - }, - coords={ - "batch": (["batch"], [b]), - "gcm": (["gcm"], ["ACCESS1-0", "GFDL-CM3"]), - "model": (["model"], ["IIASA GDP", "OECD Env-Growth"]), - "rcp": (["rcp"], ["rcp45", "rcp85"]), - "region": (["region"], ["USA.test_region", "ZWE.test_region"]), - "scaling": ( - ["scaling"], - ["epa_scaled", "epa_iso_scaled", "epa_popavg", "epa_row"], - ), - "ssp": (["ssp"], ["SSP2", "SSP3"]), - "valuation": (["valuation"], ["vsl", "vly"]), - "year": (["year"], [2010, 2099]), - }, - ) + for b in range(15): + for e in ["1.0", "1.34"]: + ds_in = xr.Dataset( + { + "monetized_costs": ( + [ + "gcm", + "batch", + "ssp", + "rcp", + "model", + "year", + "region", + "scaling", + "valuation", + ], + np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 0), + ), + "monetized_deaths": ( + [ + "gcm", + "batch", + "ssp", + "rcp", + "model", + "year", + "region", + "scaling", + "valuation", + ], + np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 1), + ), + "monetized_histclim_deaths": ( + [ + "gcm", + "batch", + "ssp", + "rcp", + "model", + "year", + "region", + "scaling", + "valuation", + ], + np.full((2, 1, 2, 2, 2, 2, 2, 4, 2), 2), + ), + }, + coords={ + "batch": (["batch"], [f"{b}"]), + "gcm": (["gcm"], ["ACCESS1-0", "GFDL-CM3"]), + "model": (["model"], ["IIASA GDP", "OECD Env-Growth"]), + "rcp": (["rcp"], ["rcp45", "rcp85"]), + "region": (["region"], ["USA.test_region", "ZWE.test_region"]), + "scaling": ( + ["scaling"], + ["epa_scaled", "epa_iso_scaled", "epa_popavg", "epa_row"], + ), + "ssp": (["ssp"], ["SSP2", "SSP3"]), + "valuation": (["valuation"], ["vsl", "vly"]), + "year": (["year"], [2010, 2099]), + }, + ) - d = os.path.join(tmp_path, "mortality_in") - if not os.path.exists(d): - os.makedirs(d) - infile = os.path.join(d, f"mortality_damages_batch{b}.zarr") + d = os.path.join(tmp_path, "mortality_in") + if not os.path.exists(d): + os.makedirs(d) + infile = os.path.join(d, f"mortality_damages_batch{b}_eta{e}.zarr") - ds_in.to_zarr(infile) + ds_in.to_zarr(infile) prep_mortality_damages( gcms=["ACCESS1-0", "GFDL-CM3"], - paths=[ - os.path.join(tmp_path, f"mortality_in/mortality_damages_batch{b}.zarr") - for b in [6, 9] - ], + paths=str( + os.path.join( + tmp_path, "mortality_in/mortality_damages_batch{0}_eta{1}.zarr" + ) + ), vars={ "delta_costs": "monetized_costs", "delta_deaths": "monetized_deaths", @@ -1085,6 +1087,7 @@ def test_prep_mortality_damages( outpath=os.path.join(tmp_path, "mortality_out"), mortality_version=version_test, path_econ=os.path.join(tmp_path, "econvars_for_test", "econvars_for_test.zarr"), + etas=[1.0, 1.34], ) ds_out_actual = xr.open_zarr( @@ -1098,26 +1101,32 @@ def test_prep_mortality_damages( ds_out_expected = xr.Dataset( { "delta": ( - ["gcm", "batch", "ssp", "rcp", "model", "year", "region"], - np.float32(np.full((2, 2, 2, 2, 2, 2, 2), -0.90681089)), + ["gcm", "batch", "ssp", "rcp", "model", "year", "region", "eta"], + np.float32(np.full((2, 15, 2, 2, 2, 2, 2, 2), -0.90681089)), ), "histclim": ( - ["gcm", "batch", "ssp", "rcp", "model", "year", "region"], - np.float32(np.full((2, 2, 2, 2, 2, 2, 2), 2 * 0.90681089)), + ["gcm", "batch", "ssp", "rcp", "model", "year", "region", "eta"], + np.float32(np.full((2, 15, 2, 2, 2, 2, 2, 2), 2 * 0.90681089)), ), }, coords={ - "batch": (["batch"], ["batch6", "batch9"]), + "batch": (["batch"], ["batch" + str(i) for i in range(15)]), "gcm": (["gcm"], ["ACCESS1-0", "GFDL-CM3"]), "model": (["model"], ["IIASA GDP", "OECD Env-Growth"]), "rcp": (["rcp"], ["rcp45", "rcp85"]), "region": (["region"], ["USA.test_region", "ZWE.test_region"]), "ssp": (["ssp"], ["SSP2", "SSP3"]), "year": (["year"], [2010, 2099]), + "eta": (["eta"], [1.0, 1.34]), }, ) - xr.testing.assert_equal(ds_out_expected, ds_out_actual) + xr.testing.assert_equal( + ds_out_expected, + ds_out_actual.sel(ds_out_expected.coords).transpose( + *list(ds_out_expected.dims) + ), + ) def test_error_prep_mortality_damages(tmp_path): @@ -1127,10 +1136,12 @@ def test_error_prep_mortality_damages(tmp_path): with pytest.raises(ValueError) as excinfo: prep_mortality_damages( gcms=["ACCESS1-0", "GFDL-CM3"], - paths=[ - os.path.join(tmp_path, f"mortality_in/mortality_damages_batch{b}.zarr") - for b in [6, 9] - ], + paths=str( + os.path.join( + tmp_path, + f"mortality_in/mortality_damages_batch{0}_eta{1}.zarr.zarr", + ) + ), vars={ "delta_costs": "monetized_costs", "delta_deaths": "monetized_deaths", @@ -1141,6 +1152,7 @@ def test_error_prep_mortality_damages(tmp_path): path_econ=os.path.join( tmp_path, "econvars_for_test", "econvars_for_test.zarr" ), + etas=[1.0, 1.34], ) assert "Mortality version not valid: " in str(excinfo.value) diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index 9bdfb47b..2597d5ae 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -69,12 +69,8 @@ def test_subset_USA_reduced_damages(tmp_path, recipe): d = tmp_path / "USA_econ" d.mkdir() - if recipe == "adding_up": - infile = d / f"{sector}/{recipe}_{reduction}.zarr" - outfile = d / f"{sector}_USA/{recipe}_{reduction}.zarr" - else: - infile = d / f"{sector}/{recipe}_{reduction}_eta{eta}.zarr" - outfile = d / f"{sector}_USA/{recipe}_{reduction}_eta{eta}.zarr" + infile = d / f"{sector}/{recipe}_{reduction}_eta{eta}.zarr" + outfile = d / f"{sector}_USA/{recipe}_{reduction}_eta{eta}.zarr" ds_in = xr.Dataset( { @@ -257,25 +253,6 @@ def test_sum_AMEL(tmp_path): ) -def test_reduce_damages_error_eta(): - """ - Test that reduce_damages complains when adding up is passed an eta argument - """ - with pytest.raises(AssertionError) as excinfo: - reduce_damages( - "adding_up", - "cc", - 10, - "dummy_sector1", - "/configdirectory/config.yml", - "/reductiondirectory/reduction.zarr", - ) - assert ( - str(excinfo.value) - == "Adding up does not take an eta argument. Please set to None." - ) - - @pytest.mark.parametrize( "recipe, eta", [ @@ -404,14 +381,9 @@ def test_reduce_damages(tmp_path, recipe, eta): + 38.39265060424805 # Since the dummy data gets set to less than the bottom code, set the expected output equal to the bottom code ) - if recipe == "adding_up": - damages_reduced_actual_path = ( - f"{reduced_damages_out}/dummy_sector1/{recipe}_cc.zarr" - ) - else: - damages_reduced_actual_path = ( - f"{reduced_damages_out}/dummy_sector1/{recipe}_cc_eta{eta}.zarr" - ) + damages_reduced_actual_path = ( + f"{reduced_damages_out}/dummy_sector1/{recipe}_cc_eta{eta}.zarr" + ) xr.testing.assert_equal( xr.open_zarr(damages_reduced_actual_path), damages_reduced_out_expected diff --git a/tests/test_simple_storage.py b/tests/test_simple_storage.py index b12d2424..5dd4b9f9 100644 --- a/tests/test_simple_storage.py +++ b/tests/test_simple_storage.py @@ -45,8 +45,12 @@ def test_adding_up_damages(stacked_damages): stacked_damages.adding_up_damages, ( ( - xr.open_zarr(f"{stacked_damages.ce_path}/adding_up_no_cc.zarr").no_cc - - xr.open_zarr(f"{stacked_damages.ce_path}/adding_up_cc.zarr").cc + xr.open_zarr( + f"{stacked_damages.ce_path}/adding_up_no_cc_eta1.421158116.zarr" + ).no_cc + - xr.open_zarr( + f"{stacked_damages.ce_path}/adding_up_cc_eta1.421158116.zarr" + ).cc ) * stacked_damages.pop ).sum("region"), @@ -818,8 +822,8 @@ def test_stackeddamages_adding_up_damages(tmp_path): # data sets to a temporary Zarr Store on disk. d = tmp_path / "stackeddamages" d.mkdir() - adding_up_cc_path = d / "adding_up_cc.zarr" - adding_up_no_cc_path = d / "adding_up_no_cc.zarr" + adding_up_cc_path = d / "adding_up_cc_eta1.421158116.zarr" + adding_up_no_cc_path = d / "adding_up_no_cc_eta1.421158116.zarr" # Fake damage data without climate change -- dump to zarr. input_no_cc = xr.Dataset( { @@ -864,7 +868,7 @@ class MockEconVars: save_path="", econ_vars=fake_econvars, climate_vars=fake_climate, - eta=0, + eta=1.421158116, gdppc_bottom_code=0, subset_dict={}, ce_path=str(d), # This MUST be set.