Skip to content

Commit ca0002b

Browse files
author
Andrzej Pijanowski
committed
feat: enhance custom mappings support with detailed merge behavior and examples
1 parent 59b36b8 commit ca0002b

File tree

3 files changed

+142
-27
lines changed

3 files changed

+142
-27
lines changed

‎README.md‎

Lines changed: 75 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -713,18 +713,69 @@ SFEOS provides environment variables to customize Elasticsearch/OpenSearch index
713713

714714
### Custom Mappings (`STAC_FASTAPI_ES_CUSTOM_MAPPINGS`)
715715

716-
Accepts a JSON string representing a properties dictionary that will be merged into the default item mappings. Custom mappings will overwrite defaults if keys collide.
716+
Accepts a JSON string with the same structure as the default ES mappings. The custom mappings are **recursively merged** with the defaults at the root level.
717+
718+
#### Merge Behavior
719+
720+
The merge follows these rules:
721+
722+
| Scenario | Result |
723+
|----------|--------|
724+
| Key only in defaults | Preserved |
725+
| Key only in custom | Added |
726+
| Key in both, both are dicts | Recursively merged |
727+
| Key in both, values are not both dicts | **Custom overwrites default** |
728+
729+
**Example - Adding new properties (merged):**
730+
731+
```json
732+
// Default has: {"geometry": {"type": "geo_shape"}}
733+
// Custom has: {"geometry": {"ignore_malformed": true}}
734+
// Result: {"geometry": {"type": "geo_shape", "ignore_malformed": true}}
735+
```
736+
737+
**Example - Overriding a value (replaced):**
738+
739+
```json
740+
// Default has: {"properties": {"datetime": {"type": "date_nanos"}}}
741+
// Custom has: {"properties": {"datetime": {"type": "date"}}}
742+
// Result: {"properties": {"datetime": {"type": "date"}}}
743+
```
744+
745+
#### JSON Structure
746+
747+
The custom JSON should mirror the structure of the default mappings. For STAC item properties, the path is `properties.properties.properties`:
748+
749+
```
750+
{
751+
"numeric_detection": false,
752+
"dynamic_templates": [...],
753+
"properties": { # Top-level ES mapping properties
754+
"id": {...},
755+
"geometry": {...},
756+
"properties": { # STAC item "properties" field
757+
"type": "object",
758+
"properties": { # Nested properties within STAC properties
759+
"datetime": {...},
760+
"sar:frequency_band": {...} # <-- Custom extension fields go here
761+
}
762+
}
763+
}
764+
}
765+
```
717766

718767
**Example - Adding SAR Extension Fields:**
719768

720769
```bash
721770
export STAC_FASTAPI_ES_CUSTOM_MAPPINGS='{
722771
"properties": {
723772
"properties": {
724-
"sar:frequency_band": {"type": "keyword"},
725-
"sar:center_frequency": {"type": "float"},
726-
"sar:polarizations": {"type": "keyword"},
727-
"sar:product_type": {"type": "keyword"}
773+
"properties": {
774+
"sar:frequency_band": {"type": "keyword"},
775+
"sar:center_frequency": {"type": "float"},
776+
"sar:polarizations": {"type": "keyword"},
777+
"sar:product_type": {"type": "keyword"}
778+
}
728779
}
729780
}
730781
}'
@@ -736,13 +787,25 @@ export STAC_FASTAPI_ES_CUSTOM_MAPPINGS='{
736787
export STAC_FASTAPI_ES_CUSTOM_MAPPINGS='{
737788
"properties": {
738789
"properties": {
739-
"cube:dimensions": {"type": "object", "enabled": false},
740-
"cube:variables": {"type": "object", "enabled": false}
790+
"properties": {
791+
"cube:dimensions": {"type": "object", "enabled": false},
792+
"cube:variables": {"type": "object", "enabled": false}
793+
}
741794
}
742795
}
743796
}'
744797
```
745798

799+
**Example - Adding geometry options:**
800+
801+
```bash
802+
export STAC_FASTAPI_ES_CUSTOM_MAPPINGS='{
803+
"properties": {
804+
"geometry": {"ignore_malformed": true}
805+
}
806+
}'
807+
```
808+
746809
### Dynamic Mapping Control (`STAC_FASTAPI_ES_DYNAMIC_MAPPING`)
747810

748811
Controls how Elasticsearch/OpenSearch handles fields not defined in the mapping:
@@ -765,9 +828,11 @@ export STAC_FASTAPI_ES_DYNAMIC_MAPPING=false
765828
export STAC_FASTAPI_ES_CUSTOM_MAPPINGS='{
766829
"properties": {
767830
"properties": {
768-
"platform": {"type": "keyword"},
769-
"eo:cloud_cover": {"type": "float"},
770-
"view:sun_elevation": {"type": "float"}
831+
"properties": {
832+
"platform": {"type": "keyword"},
833+
"eo:cloud_cover": {"type": "float"},
834+
"view:sun_elevation": {"type": "float"}
835+
}
771836
}
772837
}
773838
}'

‎stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/mappings.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,13 @@ def apply_custom_mappings(
8282
) -> None:
8383
"""Apply custom mappings from a JSON string to the mappings dictionary.
8484
85+
The custom mappings JSON should have the same structure as ES_ITEMS_MAPPINGS.
86+
It will be recursively merged at the root level, allowing users to override
87+
any part of the mapping including properties, dynamic_templates, etc.
88+
8589
Args:
8690
mappings: The mappings dictionary to modify (modified in place).
87-
custom_mappings_json: JSON string containing custom property mappings.
91+
custom_mappings_json: JSON string containing custom mappings.
8892
8993
Raises:
9094
Logs error if JSON parsing or merging fails.
@@ -94,7 +98,7 @@ def apply_custom_mappings(
9498

9599
try:
96100
custom_mappings = json.loads(custom_mappings_json)
97-
merge_mappings(mappings["properties"], custom_mappings)
101+
merge_mappings(mappings, custom_mappings)
98102
except json.JSONDecodeError as e:
99103
logger.error(f"Failed to parse STAC_FASTAPI_ES_CUSTOM_MAPPINGS JSON: {e}")
100104
except Exception as e:

‎stac_fastapi/tests/sfeos_helpers/test_mappings.py‎

Lines changed: 61 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,13 @@ def test_custom_overwrites_on_key_collision(self):
4545
merge_mappings(base, custom)
4646
assert base["level1"]["a"] == {"type": "date"}
4747

48+
def test_merge_adds_properties_to_existing_nested_dict(self):
49+
"""Test that merging adds new properties to existing nested dicts."""
50+
base = {"geometry": {"type": "geo_shape"}}
51+
custom = {"geometry": {"ignore_malformed": True}}
52+
merge_mappings(base, custom)
53+
assert base == {"geometry": {"type": "geo_shape", "ignore_malformed": True}}
54+
4855
@pytest.mark.parametrize(
4956
"base,custom,expected",
5057
[
@@ -111,24 +118,49 @@ def test_no_op_for_empty_input(self, custom_json):
111118
apply_custom_mappings(mappings, custom_json)
112119
assert mappings == original
113120

114-
def test_merges_valid_json(self):
115-
"""Test that valid JSON custom mappings are merged into properties."""
121+
def test_merges_at_root_level(self):
122+
"""Test that custom mappings are merged at the root level."""
116123
mappings = {
124+
"numeric_detection": False,
117125
"properties": {
118-
"properties": {"properties": {"datetime": {"type": "date_nanos"}}}
119-
}
126+
"id": {"type": "keyword"},
127+
"properties": {"properties": {"datetime": {"type": "date_nanos"}}},
128+
},
120129
}
121130
custom_json = json.dumps(
122-
{"properties": {"properties": {"sar:frequency_band": {"type": "keyword"}}}}
131+
{
132+
"properties": {
133+
"properties": {
134+
"properties": {"sar:frequency_band": {"type": "keyword"}}
135+
},
136+
"bbox": {"type": "object", "enabled": False},
137+
}
138+
}
123139
)
124140
apply_custom_mappings(mappings, custom_json)
125141

142+
# Existing fields preserved
143+
assert mappings["properties"]["id"] == {"type": "keyword"}
126144
assert mappings["properties"]["properties"]["properties"]["datetime"] == {
127145
"type": "date_nanos"
128146
}
147+
# New fields added
129148
assert mappings["properties"]["properties"]["properties"][
130149
"sar:frequency_band"
131150
] == {"type": "keyword"}
151+
assert mappings["properties"]["bbox"] == {"type": "object", "enabled": False}
152+
153+
def test_can_override_dynamic_templates(self):
154+
"""Test that dynamic_templates can be overridden via custom mappings."""
155+
mappings = {
156+
"dynamic_templates": [{"old": "template"}],
157+
"properties": {"id": {"type": "keyword"}},
158+
}
159+
custom_json = json.dumps({"dynamic_templates": [{"new": "template"}]})
160+
apply_custom_mappings(mappings, custom_json)
161+
162+
assert mappings["dynamic_templates"] == [{"new": "template"}]
163+
assert mappings["properties"]["id"] == {"type": "keyword"}
132164

133165
def test_invalid_json_logs_error_and_preserves_mappings(self, caplog):
134166
"""Test that invalid JSON logs an error and doesn't modify mappings."""
@@ -159,7 +191,11 @@ def test_dynamic_mapping_values(self, dynamic_mapping, expected):
159191
def test_custom_mappings_merged_preserving_defaults(self):
160192
"""Test that custom mappings are merged while preserving default fields."""
161193
custom = json.dumps(
162-
{"properties": {"properties": {"custom:field": {"type": "keyword"}}}}
194+
{
195+
"properties": {
196+
"properties": {"properties": {"custom:field": {"type": "keyword"}}}
197+
}
198+
}
163199
)
164200
mappings = get_items_mappings(custom_mappings=custom)
165201

@@ -177,7 +213,11 @@ def test_custom_mappings_merged_preserving_defaults(self):
177213
def test_custom_can_override_defaults(self):
178214
"""Test that custom mappings can override default field types."""
179215
custom = json.dumps(
180-
{"properties": {"properties": {"datetime": {"type": "date"}}}}
216+
{
217+
"properties": {
218+
"properties": {"properties": {"datetime": {"type": "date"}}}
219+
}
220+
}
181221
)
182222
mappings = get_items_mappings(custom_mappings=custom)
183223
assert mappings["properties"]["properties"]["properties"]["datetime"] == {
@@ -212,9 +252,11 @@ class TestSTACExtensionUseCases:
212252
{
213253
"properties": {
214254
"properties": {
215-
"sar:frequency_band": {"type": "keyword"},
216-
"sar:center_frequency": {"type": "float"},
217-
"sar:polarizations": {"type": "keyword"},
255+
"properties": {
256+
"sar:frequency_band": {"type": "keyword"},
257+
"sar:center_frequency": {"type": "float"},
258+
"sar:polarizations": {"type": "keyword"},
259+
}
218260
}
219261
}
220262
},
@@ -224,8 +266,10 @@ class TestSTACExtensionUseCases:
224266
{
225267
"properties": {
226268
"properties": {
227-
"cube:dimensions": {"type": "object", "enabled": False},
228-
"cube:variables": {"type": "object", "enabled": False},
269+
"properties": {
270+
"cube:dimensions": {"type": "object", "enabled": False},
271+
"cube:variables": {"type": "object", "enabled": False},
272+
}
229273
}
230274
}
231275
},
@@ -238,7 +282,7 @@ def test_add_extension_fields(self, extension_name, custom_fields):
238282
mappings = get_items_mappings(custom_mappings=json.dumps(custom_fields))
239283

240284
props = mappings["properties"]["properties"]["properties"]
241-
for field_name, field_config in custom_fields["properties"][
285+
for field_name, field_config in custom_fields["properties"]["properties"][
242286
"properties"
243287
].items():
244288
assert props[field_name] == field_config
@@ -250,8 +294,10 @@ def test_performance_optimization_with_disabled_dynamic_mapping(self):
250294
query_fields = {
251295
"properties": {
252296
"properties": {
253-
"platform": {"type": "keyword"},
254-
"eo:cloud_cover": {"type": "float"},
297+
"properties": {
298+
"platform": {"type": "keyword"},
299+
"eo:cloud_cover": {"type": "float"},
300+
}
255301
}
256302
}
257303
}

0 commit comments

Comments
 (0)