Add datetime range aliases for optimized index filtering

stac_fastapi/core/stac_fastapi/core/core.py

@@ -510,18 +510,20 @@ async def post_all_collections(
         return await self.all_collections(
             limit=search_request.limit if hasattr(search_request, "limit") else None,
             bbox=search_request.bbox if hasattr(search_request, "bbox") else None,
-            datetime=search_request.datetime
-            if hasattr(search_request, "datetime")
-            else None,
+            datetime=(
+                search_request.datetime if hasattr(search_request, "datetime") else None
+            ),
             token=search_request.token if hasattr(search_request, "token") else None,
             fields=fields,
             sortby=sortby,
-            filter_expr=search_request.filter
-            if hasattr(search_request, "filter")
-            else None,
-            filter_lang=search_request.filter_lang
-            if hasattr(search_request, "filter_lang")
-            else None,
+            filter_expr=(
+                search_request.filter if hasattr(search_request, "filter") else None
+            ),
+            filter_lang=(
+                search_request.filter_lang
+                if hasattr(search_request, "filter_lang")
+                else None
+            ),
             query=search_request.query if hasattr(search_request, "query") else None,
             q=search_request.q if hasattr(search_request, "q") else None,
             request=request,
@@ -792,7 +794,7 @@ async def post_search(
 
         datetime_parsed = format_datetime_range(date_str=search_request.datetime)
         try:
-            search, datetime_search = self.database.apply_datetime_filter(
+            search = self.database.apply_datetime_filter(
                 search=search, datetime=datetime_parsed
             )
         except (ValueError, TypeError) as e:
@@ -866,7 +868,7 @@ async def post_search(
             token=token_param,
             sort=sort,
             collection_ids=getattr(search_request, "collections", None),
-            datetime_search=datetime_search,
+            datetime_search=datetime_parsed,
         )
 
         fields = getattr(search_request, "fields", None)

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py

@@ -434,9 +434,7 @@ def apply_collections_filter(search: Search, collection_ids: List[str]):
         return search.filter("terms", collection=collection_ids)
 
     @staticmethod
-    def apply_datetime_filter(
-        search: Search, datetime: Optional[str]
-    ) -> Tuple[Search, Dict[str, Optional[str]]]:
+    def apply_datetime_filter(search: Search, datetime: Optional[str]) -> Search:
         """Apply a filter to search on datetime, start_datetime, and end_datetime fields.
 
         Args:
@@ -446,15 +444,15 @@ def apply_datetime_filter(
         Returns:
             The filtered search object.
         """
+        datetime_search = return_date(datetime)
+
         # USE_DATETIME env var
         # True: Search by datetime, if null search by start/end datetime
         # False: Always search only by start/end datetime
         USE_DATETIME = get_bool_env("USE_DATETIME", default=True)
 
-        datetime_search = return_date(datetime)
-
         if not datetime_search:
-            return search, datetime_search
+            return search
 
         if USE_DATETIME:
             if "eq" in datetime_search:
@@ -531,10 +529,7 @@ def apply_datetime_filter(
                     ),
                 ]
 
-            return (
-                search.query(Q("bool", should=should, minimum_should_match=1)),
-                datetime_search,
-            )
+            return search.query(Q("bool", should=should, minimum_should_match=1))
         else:
             if "eq" in datetime_search:
                 filter_query = Q(
@@ -568,7 +563,7 @@ def apply_datetime_filter(
                         ),
                     ],
                 )
-            return search.query(filter_query), datetime_search
+        return search.query(filter_query)
 
     @staticmethod
     def apply_bbox_filter(search: Search, bbox: List):
@@ -723,7 +718,7 @@ async def execute_search(
         token: Optional[str],
         sort: Optional[Dict[str, Dict[str, str]]],
         collection_ids: Optional[List[str]],
-        datetime_search: Dict[str, Optional[str]],
+        datetime_search: str,
         ignore_unavailable: bool = True,
     ) -> Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]:
         """Execute a search query with limit and other optional parameters.
@@ -734,7 +729,7 @@ async def execute_search(
             token (Optional[str]): The token used to return the next set of results.
             sort (Optional[Dict[str, Dict[str, str]]]): Specifies how the results should be sorted.
             collection_ids (Optional[List[str]]): The collection ids to search.
-            datetime_search (Dict[str, Optional[str]]): Datetime range used for index selection.
+            datetime_search (str): Datetime used for index selection.
             ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True.
 
         Returns:
@@ -824,7 +819,7 @@ async def aggregate(
         geometry_geohash_grid_precision: int,
         geometry_geotile_grid_precision: int,
         datetime_frequency_interval: str,
-        datetime_search,
+        datetime_search: str,
         ignore_unavailable: Optional[bool] = True,
     ):
         """Return aggregations of STAC Items."""

stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py

@@ -457,9 +457,7 @@ def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]
         )
 
     @staticmethod
-    def apply_datetime_filter(
-        search: Search, datetime: Optional[str]
-    ) -> Tuple[Search, Dict[str, Optional[str]]]:
+    def apply_datetime_filter(search: Search, datetime: Optional[str]) -> Search:
         """Apply a filter to search on datetime, start_datetime, and end_datetime fields.
 
         Args:
@@ -471,14 +469,14 @@ def apply_datetime_filter(
         """
         datetime_search = return_date(datetime)
 
-        if not datetime_search:
-            return search, datetime_search
-
         # USE_DATETIME env var
         # True: Search by datetime, if null search by start/end datetime
         # False: Always search only by start/end datetime
         USE_DATETIME = get_bool_env("USE_DATETIME", default=True)
 
+        if not datetime_search:
+            return search
+
         if USE_DATETIME:
             if "eq" in datetime_search:
                 # For exact matches, include:
@@ -554,10 +552,7 @@ def apply_datetime_filter(
                     ),
                 ]
 
-            return (
-                search.query(Q("bool", should=should, minimum_should_match=1)),
-                datetime_search,
-            )
+            return search.query(Q("bool", should=should, minimum_should_match=1))
         else:
             if "eq" in datetime_search:
                 filter_query = Q(
@@ -591,7 +586,7 @@ def apply_datetime_filter(
                         ),
                     ],
                 )
-        return search.query(filter_query), datetime_search
+        return search.query(filter_query)
 
     @staticmethod
     def apply_bbox_filter(search: Search, bbox: List):
@@ -728,7 +723,7 @@ async def execute_search(
         token: Optional[str],
         sort: Optional[Dict[str, Dict[str, str]]],
         collection_ids: Optional[List[str]],
-        datetime_search: Dict[str, Optional[str]],
+        datetime_search: str,
         ignore_unavailable: bool = True,
     ) -> Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]:
         """Execute a search query with limit and other optional parameters.
@@ -739,7 +734,7 @@ async def execute_search(
             token (Optional[str]): The token used to return the next set of results.
             sort (Optional[Dict[str, Dict[str, str]]]): Specifies how the results should be sorted.
             collection_ids (Optional[List[str]]): The collection ids to search.
-            datetime_search (Dict[str, Optional[str]]): Datetime range used for index selection.
+            datetime_search (str): Datetime used for index selection.
             ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True.
 
         Returns:
@@ -835,7 +830,7 @@ async def aggregate(
         geometry_geohash_grid_precision: int,
         geometry_geotile_grid_precision: int,
         datetime_frequency_interval: str,
-        datetime_search,
+        datetime_search: str,
         ignore_unavailable: Optional[bool] = True,
     ):
         """Return aggregations of STAC Items."""

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py

@@ -313,11 +313,9 @@ async def aggregate(
             )
 
         if aggregate_request.datetime:
-            search, datetime_search = self.database.apply_datetime_filter(
+            search = self.database.apply_datetime_filter(
                 search=search, datetime=aggregate_request.datetime
             )
-        else:
-            datetime_search = {"gte": None, "lte": None}
 
         if aggregate_request.bbox:
             bbox = aggregate_request.bbox
@@ -416,7 +414,7 @@ async def aggregate(
                 geometry_geohash_grid_precision,
                 geometry_geotile_grid_precision,
                 datetime_frequency_interval,
-                datetime_search,
+                aggregate_request.datetime,
             )
         except Exception as error:
             if not isinstance(error, IndexError):

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py

@@ -145,7 +145,7 @@ def extract_date(date_str: str) -> date:
         date_str: ISO format date string
 
     Returns:
-        A date object extracted from the input string.
+        A date object extracted from the input string or None.
     """
     date_str = date_str.replace("Z", "+00:00")
     return datetime_type.fromisoformat(date_str).date()

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py

@@ -4,11 +4,9 @@
 """
 
 import re
-from datetime import datetime
+from datetime import date, datetime
 from functools import lru_cache
-from typing import Any, List, Optional
-
-from dateutil.parser import parse  # type: ignore[import]
+from typing import Any, Dict, List, Optional, Tuple
 
 from stac_fastapi.sfeos_helpers.mappings import (
     _ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE,
@@ -71,54 +69,103 @@ def indices(collection_ids: Optional[List[str]]) -> str:
 
 
 def filter_indexes_by_datetime(
-    indexes: List[str], gte: Optional[str], lte: Optional[str]
+    collection_indexes: List[Tuple[Dict[str, str], ...]],
+    datetime_search: Dict[str, Dict[str, Optional[str]]],
+    use_datetime: bool,
 ) -> List[str]:
-    """Filter indexes based on datetime range extracted from index names.
+    """
+    Filter Elasticsearch index aliases based on datetime search criteria.
+
+    Filters a list of collection indexes by matching their datetime, start_datetime, and end_datetime
+    aliases against the provided search criteria. Each criterion can have optional 'gte' (greater than
+    or equal) and 'lte' (less than or equal) bounds.
 
     Args:
-        indexes: List of index names containing dates
-        gte: Greater than or equal date filter (ISO format, optional 'Z' suffix)
-        lte: Less than or equal date filter (ISO format, optional 'Z' suffix)
+        collection_indexes (List[Tuple[Dict[str, str], ...]]): A list of tuples containing dictionaries
+            with 'datetime', 'start_datetime', and 'end_datetime' aliases.
+        datetime_search (Dict[str, Dict[str, Optional[str]]]): A dictionary with keys 'datetime',
+            'start_datetime', and 'end_datetime', each containing 'gte' and 'lte' criteria as ISO format
+            datetime strings or None.
+        use_datetime (bool): Flag determining which datetime field to filter on:
+            - True: Filters using 'datetime' alias.
+            - False: Filters using 'start_datetime' and 'end_datetime' aliases.
 
     Returns:
-        List of filtered index names
+        List[str]: A list of start_datetime aliases that match all provided search criteria.
     """
 
-    def parse_datetime(dt_str: str) -> datetime:
-        """Parse datetime string, handling both with and without 'Z' suffix."""
-        return parse(dt_str).replace(tzinfo=None)
-
-    def extract_date_range_from_index(index_name: str) -> tuple:
-        """Extract start and end dates from index name."""
-        date_pattern = r"(\d{4}-\d{2}-\d{2})"
-        dates = re.findall(date_pattern, index_name)
-
-        if len(dates) == 1:
-            start_date = datetime.strptime(dates[0], "%Y-%m-%d")
-            max_date = datetime.max.replace(microsecond=0)
-            return start_date, max_date
-        else:
-            start_date = datetime.strptime(dates[0], "%Y-%m-%d")
-            end_date = datetime.strptime(dates[1], "%Y-%m-%d")
-            return start_date, end_date
-
-    def is_index_in_range(
-        start_date: datetime, end_date: datetime, gte_dt: datetime, lte_dt: datetime
+    def extract_date_from_alias(alias: str) -> Optional[tuple[datetime, datetime]]:
+        date_pattern = re.compile(r"\d{4}-\d{2}-\d{2}")
+        try:
+            dates = date_pattern.findall(alias)
+
+            if not dates:
+                return None
+
+            if len(dates) >= 2:
+                return datetime.strptime(dates[-2], "%Y-%m-%d"), datetime.strptime(
+                    dates[-1], "%Y-%m-%d"
+                )
+            else:
+                date = datetime.strptime(dates[-1], "%Y-%m-%d")
+                return date, date
+        except (ValueError, IndexError):
+            return None
+
+    def parse_search_date(date_str: Optional[str]) -> Optional[date]:
+        if not date_str:
+            return None
+        date_str = date_str.rstrip("Z")
+        return datetime.fromisoformat(date_str).date()
+
+    def check_criteria(
+        value_begin: datetime, value_end: datetime, criteria: Dict
     ) -> bool:
-        """Check if index date range overlaps with filter range."""
-        return not (
-            end_date.date() < gte_dt.date() or start_date.date() > lte_dt.date()
-        )
+        gte = parse_search_date(criteria.get("gte"))
+        lte = parse_search_date(criteria.get("lte"))
 
-    gte_dt = parse_datetime(gte) if gte else datetime.min.replace(microsecond=0)
-    lte_dt = parse_datetime(lte) if lte else datetime.max.replace(microsecond=0)
+        if gte and value_end.date() < gte:
+            return False
+        if lte and value_begin.date() > lte:
+            return False
+
+        return True
 
     filtered_indexes = []
 
-    for index in indexes:
-        start_date, end_date = extract_date_range_from_index(index)
-        if is_index_in_range(start_date, end_date, gte_dt, lte_dt):
-            filtered_indexes.append(index)
+    for index_tuple in collection_indexes:
+        if not index_tuple:
+            continue
+
+        index_dict = index_tuple[0]
+        start_datetime_alias = index_dict.get("start_datetime")
+        end_datetime_alias = index_dict.get("end_datetime")
+        datetime_alias = index_dict.get("datetime")
+
+        if start_datetime_alias:
+            start_date = extract_date_from_alias(start_datetime_alias)
+            if not check_criteria(
+                start_date[0], start_date[1], datetime_search.get("start_datetime", {})
+            ):
+                continue
+        if end_datetime_alias:
+            end_date = extract_date_from_alias(end_datetime_alias)
+            if not check_criteria(
+                end_date[0], end_date[1], datetime_search.get("end_datetime", {})
+            ):
+                continue
+        if datetime_alias:
+            datetime_date = extract_date_from_alias(datetime_alias)
+            if not check_criteria(
+                datetime_date[0], datetime_date[1], datetime_search.get("datetime", {})
+            ):
+                continue
+
+        primary_datetime_alias = (
+            datetime_alias if use_datetime else start_datetime_alias
+        )
+
+        filtered_indexes.append(primary_datetime_alias)
 
     return filtered_indexes
 

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py

@@ -130,7 +130,7 @@ def apply_collections_datetime_filter_shared(
 
 
 def apply_collections_bbox_filter_shared(
-    bbox: Union[str, List[float], None]
+    bbox: Union[str, List[float], None],
 ) -> Optional[Dict[str, Dict]]:
     """Create a geo_shape filter for collections bbox search.