From 75e3f806a9abeb49331a2154fceebe06d84641df Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 2 Feb 2026 12:25:28 -0700 Subject: [PATCH 1/3] replace fill values --- imap_processing/ialirt/l0/process_codice.py | 14 +++--- imap_processing/ialirt/l0/process_swapi.py | 55 +++++++++++---------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/imap_processing/ialirt/l0/process_codice.py b/imap_processing/ialirt/l0/process_codice.py index 8ca31eddc..8280802e1 100644 --- a/imap_processing/ialirt/l0/process_codice.py +++ b/imap_processing/ialirt/l0/process_codice.py @@ -34,8 +34,6 @@ logger = logging.getLogger(__name__) -FILLVAL_UINT8 = 255 -FILLVAL_FLOAT32 = Decimal(str(-1.0e31)) COD_LO_COUNTER = 232 COD_HI_COUNTER = 197 COD_LO_RANGE = range(0, 15) @@ -357,9 +355,9 @@ def calculate_ratios( fe_over_o_abundance = Decimal(f"{float(fe_over_o_abundance):.3f}") else: c_over_o_abundance, mg_over_o_abundance, fe_over_o_abundance = ( - FILLVAL_FLOAT32, - FILLVAL_FLOAT32, - FILLVAL_FLOAT32, + None, + None, + None, ) if float(pseudo_density_dict["cplus5"]) != 0: @@ -369,7 +367,7 @@ def calculate_ratios( c_plus_6_over_c_plus_5 = Decimal(f"{float(c_plus_6_over_c_plus_5):.3f}") else: - c_plus_6_over_c_plus_5 = FILLVAL_FLOAT32 + c_plus_6_over_c_plus_5 = None if float(pseudo_density_dict["oplus6"]) != 0: o_plus_7_over_o_plus_6 = ( @@ -377,7 +375,7 @@ def calculate_ratios( ) o_plus_7_over_o_plus_6 = Decimal(f"{float(o_plus_7_over_o_plus_6):.3f}") else: - o_plus_7_over_o_plus_6 = FILLVAL_FLOAT32 + o_plus_7_over_o_plus_6 = None if float(pseudo_density_dict["fe_hiq"]) != 0: fe_low_over_fe_high = ( @@ -385,7 +383,7 @@ def calculate_ratios( ) fe_low_over_fe_high = Decimal(f"{float(fe_low_over_fe_high):.3f}") else: - fe_low_over_fe_high = FILLVAL_FLOAT32 + fe_low_over_fe_high = None return COD_LO_L2( c_over_o_abundance=c_over_o_abundance, diff --git a/imap_processing/ialirt/l0/process_swapi.py b/imap_processing/ialirt/l0/process_swapi.py index 410cc6361..c94ed2e80 100644 --- a/imap_processing/ialirt/l0/process_swapi.py +++ b/imap_processing/ialirt/l0/process_swapi.py @@ -22,7 +22,6 @@ logger = logging.getLogger(__name__) NUM_IALIRT_ENERGY_STEPS = 63 -FILLVAL_FLOAT32 = -1.0e31 def count_rate( @@ -142,7 +141,7 @@ def optimize_pseudo_parameters( # report speed only if fit fails if sol is None: sol = initial_param_guess.copy() - sol[1:] = FILLVAL_FLOAT32 + sol[1:] = np.nan return sol @@ -187,6 +186,10 @@ def geometric_mean( & ~np.isnan(pseudo_proton_temperature_list) ) + if not np.any(valid): + avg_swapi_met = np.mean(met_arr) + return avg_swapi_met, np.nan, np.nan, np.nan + pseudo_speed_arr = np.asarray(pseudo_speed_list)[valid] avg_pseudo_speed = np.exp(np.mean(np.log(pseudo_speed_arr))) @@ -201,7 +204,7 @@ def geometric_mean( return avg_swapi_met, avg_proton_density, avg_pseudo_speed, avg_proton_temperature -def process_swapi_ialirt( +def process_swapi_ialirt( # noqa: PLR0912 unpacked_data: xr.Dataset, calibration_lut_table: pd.DataFrame ) -> list[dict]: """ @@ -324,35 +327,35 @@ def process_swapi_ialirt( pseudo_proton_temperature_list[-5:], ) - # replace nans (resulting from geometric means that - # include fill values) with fill values - ( - avg_pseudo_proton_speed, - avg_pseudo_proton_density, - avg_pseudo_proton_temperature, - ) = np.nan_to_num( - ( - avg_pseudo_proton_speed, - avg_pseudo_proton_density, - avg_pseudo_proton_temperature, - ), - nan=FILLVAL_FLOAT32, - ) + if avg_pseudo_proton_speed is not None and np.isfinite( + avg_pseudo_proton_speed + ): + avg_pseudo_proton_speed = Decimal(f"{avg_pseudo_proton_speed:.3f}") + else: + avg_pseudo_proton_speed = None + if avg_pseudo_proton_density is not None and np.isfinite( + avg_pseudo_proton_density + ): + avg_pseudo_proton_density = Decimal(f"{avg_pseudo_proton_density:.3f}") + else: + avg_pseudo_proton_density = None + if avg_pseudo_proton_temperature is not None and np.isfinite( + avg_pseudo_proton_temperature + ): + avg_pseudo_proton_temperature = Decimal( + f"{avg_pseudo_proton_temperature:.3f}" + ) + else: + avg_pseudo_proton_temperature = None swapi_data.append( _populate_instrument_header_items(met) | { "instrument": "swapi", "swapi_epoch": int(met_to_ttj2000ns(avg_swapi_met)), - "swapi_pseudo_proton_speed": Decimal( - f"{avg_pseudo_proton_speed:.3f}" - ), - "swapi_pseudo_proton_density": Decimal( - f"{avg_pseudo_proton_density:.3f}" - ), - "swapi_pseudo_proton_temperature": Decimal( - f"{avg_pseudo_proton_temperature:.3f}" - ), + "swapi_pseudo_proton_speed": avg_pseudo_proton_speed, + "swapi_pseudo_proton_density": avg_pseudo_proton_density, + "swapi_pseudo_proton_temperature": avg_pseudo_proton_temperature, } ) if incomplete_groups: From 045541208126767c158dbfad92b2e588bf545953 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 2 Feb 2026 15:58:57 -0700 Subject: [PATCH 2/3] fix tests --- .../tests/ialirt/unit/test_process_codice.py | 8 ++--- .../tests/ialirt/unit/test_process_swapi.py | 30 ++++++------------- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/imap_processing/tests/ialirt/unit/test_process_codice.py b/imap_processing/tests/ialirt/unit/test_process_codice.py index 2d13fc803..2f76436cd 100644 --- a/imap_processing/tests/ialirt/unit/test_process_codice.py +++ b/imap_processing/tests/ialirt/unit/test_process_codice.py @@ -29,8 +29,6 @@ from imap_processing.ialirt.l0.process_codice import ( COD_HI_COUNTER, COD_LO_COUNTER, - FILLVAL_FLOAT32, - FILLVAL_UINT8, concatenate_bytes, convert_to_intensities, create_xarray_dataset, @@ -434,7 +432,7 @@ def test_group_and_decompress_ialirt_cod_lo( # Verify that we grouped the values properly. counter_values = cod_lo_test_dataset["cod_lo_counter"].data - valid_values = counter_values[counter_values != FILLVAL_UINT8] + valid_values = counter_values[counter_values != 255] resets = np.where(valid_values == COD_LO_COUNTER) count = increment = 0 @@ -520,7 +518,7 @@ def test_group_and_decompress_ialirt_cod_hi( # Verify that we grouped the values properly. counter_values = cod_hi_test_dataset["cod_hi_counter"].data - valid_values = counter_values[counter_values != FILLVAL_UINT8] + valid_values = counter_values[counter_values != 255] resets = np.where(valid_values == COD_HI_COUNTER) count = increment = 0 @@ -774,7 +772,7 @@ def test_process_codice_lo( assert len(cod_lo_data) == 9 for product in l2_products: - assert cod_lo_data[0][product] == FILLVAL_FLOAT32 + assert cod_lo_data[0][product] is None @pytest.mark.external_test_data diff --git a/imap_processing/tests/ialirt/unit/test_process_swapi.py b/imap_processing/tests/ialirt/unit/test_process_swapi.py index 3d7b6b9eb..e1f0a04b2 100644 --- a/imap_processing/tests/ialirt/unit/test_process_swapi.py +++ b/imap_processing/tests/ialirt/unit/test_process_swapi.py @@ -7,7 +7,6 @@ from imap_processing import imap_module_directory from imap_processing.ialirt.l0.process_swapi import ( - FILLVAL_FLOAT32, Consts, count_rate, geometric_mean, @@ -166,20 +165,9 @@ def test_process_swapi_ialirt( swapi_result = process_swapi_ialirt(xarray_data, esa_unit_conversion_table) - key_names = [ - "apid", - "met", - "met_in_utc", - "ttj2000ns", - "swapi_pseudo_proton_density", - "swapi_pseudo_proton_speed", - "swapi_pseudo_proton_temperature", - ] - - for key in key_names: - assert swapi_result[0][key] is not None, ( - f"The expected attribute {key} was not filled in the result dict." - ) + assert swapi_result[0]["swapi_pseudo_proton_speed"] is None + assert swapi_result[0]["swapi_pseudo_proton_density"] is None + assert swapi_result[0]["swapi_pseudo_proton_temperature"] is None def test_count_rate(): @@ -300,8 +288,8 @@ def test_optimize_parameters_exception_handling(): ) np.testing.assert_allclose(speed, expected_speed, rtol=1e-6) - np.testing.assert_allclose(density, FILLVAL_FLOAT32) - np.testing.assert_allclose(temperature, FILLVAL_FLOAT32) + np.testing.assert_allclose(density, np.nan) + np.testing.assert_allclose(temperature, np.nan) def test_optimize_parameters_bad_fit_handling(): @@ -337,8 +325,8 @@ def test_optimize_parameters_bad_fit_handling(): ) np.testing.assert_allclose(speed, expected_speed, rtol=1e-6) - np.testing.assert_allclose(density, FILLVAL_FLOAT32) - np.testing.assert_allclose(temperature, FILLVAL_FLOAT32) + np.testing.assert_allclose(density, np.nan) + np.testing.assert_allclose(temperature, np.nan) def test_optimize_parameters_bad_covariance_handling(): @@ -371,8 +359,8 @@ def test_optimize_parameters_bad_covariance_handling(): ) np.testing.assert_allclose(speed, expected_speed, rtol=1e-6) - np.testing.assert_allclose(density, FILLVAL_FLOAT32) - np.testing.assert_allclose(temperature, FILLVAL_FLOAT32) + np.testing.assert_allclose(density, np.nan) + np.testing.assert_allclose(temperature, np.nan) def test_geometric_mean(): From 790d95fe66412e37e3d5e4240e3dbb85857feb00 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 3 Feb 2026 10:30:00 -0700 Subject: [PATCH 3/3] formatting --- imap_processing/ialirt/l0/process_swapi.py | 42 +++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/imap_processing/ialirt/l0/process_swapi.py b/imap_processing/ialirt/l0/process_swapi.py index c94ed2e80..79812ed1d 100644 --- a/imap_processing/ialirt/l0/process_swapi.py +++ b/imap_processing/ialirt/l0/process_swapi.py @@ -204,7 +204,7 @@ def geometric_mean( return avg_swapi_met, avg_proton_density, avg_pseudo_speed, avg_proton_temperature -def process_swapi_ialirt( # noqa: PLR0912 +def process_swapi_ialirt( unpacked_data: xr.Dataset, calibration_lut_table: pd.DataFrame ) -> list[dict]: """ @@ -327,26 +327,26 @@ def process_swapi_ialirt( # noqa: PLR0912 pseudo_proton_temperature_list[-5:], ) - if avg_pseudo_proton_speed is not None and np.isfinite( - avg_pseudo_proton_speed - ): - avg_pseudo_proton_speed = Decimal(f"{avg_pseudo_proton_speed:.3f}") - else: - avg_pseudo_proton_speed = None - if avg_pseudo_proton_density is not None and np.isfinite( - avg_pseudo_proton_density - ): - avg_pseudo_proton_density = Decimal(f"{avg_pseudo_proton_density:.3f}") - else: - avg_pseudo_proton_density = None - if avg_pseudo_proton_temperature is not None and np.isfinite( - avg_pseudo_proton_temperature - ): - avg_pseudo_proton_temperature = Decimal( - f"{avg_pseudo_proton_temperature:.3f}" - ) - else: - avg_pseudo_proton_temperature = None + avg_pseudo_proton_speed = ( + Decimal(f"{avg_pseudo_proton_speed:.3f}") + if avg_pseudo_proton_speed is not None + and np.isfinite(avg_pseudo_proton_speed) + else None + ) + + avg_pseudo_proton_density = ( + Decimal(f"{avg_pseudo_proton_density:.3f}") + if avg_pseudo_proton_density is not None + and np.isfinite(avg_pseudo_proton_density) + else None + ) + + avg_pseudo_proton_temperature = ( + Decimal(f"{avg_pseudo_proton_temperature:.3f}") + if avg_pseudo_proton_temperature is not None + and np.isfinite(avg_pseudo_proton_temperature) + else None + ) swapi_data.append( _populate_instrument_header_items(met)