Skip to content

I-ALiRT - replace fill values #2660

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
laspsandoval merged 3 commits into from
Feb 4, 2026
Merged

I-ALiRT - replace fill values #2660

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
75e3f80
replace fill values
Feb 2, 2026
0455412
fix tests
Feb 2, 2026
790d95f
formatting
Feb 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions imap_processing/ialirt/l0/process_codice.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@

logger = logging.getLogger(__name__)

FILLVAL_UINT8 = 255
FILLVAL_FLOAT32 = Decimal(str(-1.0e31))
COD_LO_COUNTER = 232
COD_HI_COUNTER = 197
COD_LO_RANGE = range(0, 15)
Expand Down Expand Up @@ -357,9 +355,9 @@ def calculate_ratios(
fe_over_o_abundance = Decimal(f"{float(fe_over_o_abundance):.3f}")
else:
c_over_o_abundance, mg_over_o_abundance, fe_over_o_abundance = (
FILLVAL_FLOAT32,
FILLVAL_FLOAT32,
FILLVAL_FLOAT32,
None,
None,
None,
)

if float(pseudo_density_dict["cplus5"]) != 0:
Expand All @@ -369,23 +367,23 @@ def calculate_ratios(

c_plus_6_over_c_plus_5 = Decimal(f"{float(c_plus_6_over_c_plus_5):.3f}")
else:
c_plus_6_over_c_plus_5 = FILLVAL_FLOAT32
c_plus_6_over_c_plus_5 = None

if float(pseudo_density_dict["oplus6"]) != 0:
o_plus_7_over_o_plus_6 = (
pseudo_density_dict["oplus7"] / pseudo_density_dict["oplus6"]
)
o_plus_7_over_o_plus_6 = Decimal(f"{float(o_plus_7_over_o_plus_6):.3f}")
else:
o_plus_7_over_o_plus_6 = FILLVAL_FLOAT32
o_plus_7_over_o_plus_6 = None

if float(pseudo_density_dict["fe_hiq"]) != 0:
fe_low_over_fe_high = (
pseudo_density_dict["fe_loq"] / pseudo_density_dict["fe_hiq"]
)
fe_low_over_fe_high = Decimal(f"{float(fe_low_over_fe_high):.3f}")
else:
fe_low_over_fe_high = FILLVAL_FLOAT32
fe_low_over_fe_high = None

return COD_LO_L2(
c_over_o_abundance=c_over_o_abundance,
Expand Down
51 changes: 27 additions & 24 deletions imap_processing/ialirt/l0/process_swapi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
logger = logging.getLogger(__name__)

NUM_IALIRT_ENERGY_STEPS = 63
FILLVAL_FLOAT32 = -1.0e31


def count_rate(
Expand Down Expand Up @@ -142,7 +141,7 @@ def optimize_pseudo_parameters(
# report speed only if fit fails
if sol is None:
sol = initial_param_guess.copy()
sol[1:] = FILLVAL_FLOAT32
sol[1:] = np.nan

return sol

Expand Down Expand Up @@ -187,6 +186,10 @@ def geometric_mean(
& ~np.isnan(pseudo_proton_temperature_list)
)

if not np.any(valid):
avg_swapi_met = np.mean(met_arr)
return avg_swapi_met, np.nan, np.nan, np.nan

pseudo_speed_arr = np.asarray(pseudo_speed_list)[valid]
avg_pseudo_speed = np.exp(np.mean(np.log(pseudo_speed_arr)))

Expand Down Expand Up @@ -324,35 +327,35 @@ def process_swapi_ialirt(
pseudo_proton_temperature_list[-5:],
)

# replace nans (resulting from geometric means that
# include fill values) with fill values
(
avg_pseudo_proton_speed,
avg_pseudo_proton_density,
avg_pseudo_proton_temperature,
) = np.nan_to_num(
(
avg_pseudo_proton_speed,
avg_pseudo_proton_density,
avg_pseudo_proton_temperature,
),
nan=FILLVAL_FLOAT32,
avg_pseudo_proton_speed = (
Decimal(f"{avg_pseudo_proton_speed:.3f}")
if avg_pseudo_proton_speed is not None
and np.isfinite(avg_pseudo_proton_speed)
else None
)

avg_pseudo_proton_density = (
Decimal(f"{avg_pseudo_proton_density:.3f}")
if avg_pseudo_proton_density is not None
and np.isfinite(avg_pseudo_proton_density)
else None
)

avg_pseudo_proton_temperature = (
Decimal(f"{avg_pseudo_proton_temperature:.3f}")
if avg_pseudo_proton_temperature is not None
and np.isfinite(avg_pseudo_proton_temperature)
else None
)

swapi_data.append(
_populate_instrument_header_items(met)
| {
"instrument": "swapi",
"swapi_epoch": int(met_to_ttj2000ns(avg_swapi_met)),
"swapi_pseudo_proton_speed": Decimal(
f"{avg_pseudo_proton_speed:.3f}"
),
"swapi_pseudo_proton_density": Decimal(
f"{avg_pseudo_proton_density:.3f}"
),
"swapi_pseudo_proton_temperature": Decimal(
f"{avg_pseudo_proton_temperature:.3f}"
),
"swapi_pseudo_proton_speed": avg_pseudo_proton_speed,
"swapi_pseudo_proton_density": avg_pseudo_proton_density,
"swapi_pseudo_proton_temperature": avg_pseudo_proton_temperature,
}
)
if incomplete_groups:
Expand Down
8 changes: 3 additions & 5 deletions imap_processing/tests/ialirt/unit/test_process_codice.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,6 @@
from imap_processing.ialirt.l0.process_codice import (
COD_HI_COUNTER,
COD_LO_COUNTER,
FILLVAL_FLOAT32,
FILLVAL_UINT8,
concatenate_bytes,
convert_to_intensities,
create_xarray_dataset,
Expand Down Expand Up @@ -434,7 +432,7 @@ def test_group_and_decompress_ialirt_cod_lo(

# Verify that we grouped the values properly.
counter_values = cod_lo_test_dataset["cod_lo_counter"].data
valid_values = counter_values[counter_values != FILLVAL_UINT8]
valid_values = counter_values[counter_values != 255]
resets = np.where(valid_values == COD_LO_COUNTER)

count = increment = 0
Expand Down Expand Up @@ -520,7 +518,7 @@ def test_group_and_decompress_ialirt_cod_hi(

# Verify that we grouped the values properly.
counter_values = cod_hi_test_dataset["cod_hi_counter"].data
valid_values = counter_values[counter_values != FILLVAL_UINT8]
valid_values = counter_values[counter_values != 255]
resets = np.where(valid_values == COD_HI_COUNTER)

count = increment = 0
Expand Down Expand Up @@ -774,7 +772,7 @@ def test_process_codice_lo(
assert len(cod_lo_data) == 9

for product in l2_products:
assert cod_lo_data[0][product] == FILLVAL_FLOAT32
assert cod_lo_data[0][product] is None


@pytest.mark.external_test_data
Expand Down
30 changes: 9 additions & 21 deletions imap_processing/tests/ialirt/unit/test_process_swapi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@

from imap_processing import imap_module_directory
from imap_processing.ialirt.l0.process_swapi import (
FILLVAL_FLOAT32,
Consts,
count_rate,
geometric_mean,
Expand Down Expand Up @@ -166,20 +165,9 @@ def test_process_swapi_ialirt(

swapi_result = process_swapi_ialirt(xarray_data, esa_unit_conversion_table)

key_names = [
"apid",
"met",
"met_in_utc",
"ttj2000ns",
"swapi_pseudo_proton_density",
"swapi_pseudo_proton_speed",
"swapi_pseudo_proton_temperature",
]

for key in key_names:
assert swapi_result[0][key] is not None, (
f"The expected attribute {key} was not filled in the result dict."
)
assert swapi_result[0]["swapi_pseudo_proton_speed"] is None
assert swapi_result[0]["swapi_pseudo_proton_density"] is None
assert swapi_result[0]["swapi_pseudo_proton_temperature"] is None


def test_count_rate():
Expand Down Expand Up @@ -300,8 +288,8 @@ def test_optimize_parameters_exception_handling():
)

np.testing.assert_allclose(speed, expected_speed, rtol=1e-6)
np.testing.assert_allclose(density, FILLVAL_FLOAT32)
np.testing.assert_allclose(temperature, FILLVAL_FLOAT32)
np.testing.assert_allclose(density, np.nan)
np.testing.assert_allclose(temperature, np.nan)


def test_optimize_parameters_bad_fit_handling():
Expand Down Expand Up @@ -337,8 +325,8 @@ def test_optimize_parameters_bad_fit_handling():
)

np.testing.assert_allclose(speed, expected_speed, rtol=1e-6)
np.testing.assert_allclose(density, FILLVAL_FLOAT32)
np.testing.assert_allclose(temperature, FILLVAL_FLOAT32)
np.testing.assert_allclose(density, np.nan)
np.testing.assert_allclose(temperature, np.nan)


def test_optimize_parameters_bad_covariance_handling():
Expand Down Expand Up @@ -371,8 +359,8 @@ def test_optimize_parameters_bad_covariance_handling():
)

np.testing.assert_allclose(speed, expected_speed, rtol=1e-6)
np.testing.assert_allclose(density, FILLVAL_FLOAT32)
np.testing.assert_allclose(temperature, FILLVAL_FLOAT32)
np.testing.assert_allclose(density, np.nan)
np.testing.assert_allclose(temperature, np.nan)


def test_geometric_mean():
Expand Down