diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 3334a4db3..305ff8ff2 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -510,18 +510,20 @@ async def post_all_collections( return await self.all_collections( limit=search_request.limit if hasattr(search_request, "limit") else None, bbox=search_request.bbox if hasattr(search_request, "bbox") else None, - datetime=search_request.datetime - if hasattr(search_request, "datetime") - else None, + datetime=( + search_request.datetime if hasattr(search_request, "datetime") else None + ), token=search_request.token if hasattr(search_request, "token") else None, fields=fields, sortby=sortby, - filter_expr=search_request.filter - if hasattr(search_request, "filter") - else None, - filter_lang=search_request.filter_lang - if hasattr(search_request, "filter_lang") - else None, + filter_expr=( + search_request.filter if hasattr(search_request, "filter") else None + ), + filter_lang=( + search_request.filter_lang + if hasattr(search_request, "filter_lang") + else None + ), query=search_request.query if hasattr(search_request, "query") else None, q=search_request.q if hasattr(search_request, "q") else None, request=request, @@ -792,7 +794,7 @@ async def post_search( datetime_parsed = format_datetime_range(date_str=search_request.datetime) try: - search, datetime_search = self.database.apply_datetime_filter( + search = self.database.apply_datetime_filter( search=search, datetime=datetime_parsed ) except (ValueError, TypeError) as e: @@ -866,7 +868,7 @@ async def post_search( token=token_param, sort=sort, collection_ids=getattr(search_request, "collections", None), - datetime_search=datetime_search, + datetime_search=datetime_parsed, ) fields = getattr(search_request, "fields", None) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 35b6ae31a..aaa3b1af2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -434,9 +434,7 @@ def apply_collections_filter(search: Search, collection_ids: List[str]): return search.filter("terms", collection=collection_ids) @staticmethod - def apply_datetime_filter( - search: Search, datetime: Optional[str] - ) -> Tuple[Search, Dict[str, Optional[str]]]: + def apply_datetime_filter(search: Search, datetime: Optional[str]) -> Search: """Apply a filter to search on datetime, start_datetime, and end_datetime fields. Args: @@ -446,15 +444,15 @@ def apply_datetime_filter( Returns: The filtered search object. """ + datetime_search = return_date(datetime) + # USE_DATETIME env var # True: Search by datetime, if null search by start/end datetime # False: Always search only by start/end datetime USE_DATETIME = get_bool_env("USE_DATETIME", default=True) - datetime_search = return_date(datetime) - if not datetime_search: - return search, datetime_search + return search if USE_DATETIME: if "eq" in datetime_search: @@ -531,10 +529,7 @@ def apply_datetime_filter( ), ] - return ( - search.query(Q("bool", should=should, minimum_should_match=1)), - datetime_search, - ) + return search.query(Q("bool", should=should, minimum_should_match=1)) else: if "eq" in datetime_search: filter_query = Q( @@ -568,7 +563,7 @@ def apply_datetime_filter( ), ], ) - return search.query(filter_query), datetime_search + return search.query(filter_query) @staticmethod def apply_bbox_filter(search: Search, bbox: List): @@ -723,7 +718,7 @@ async def execute_search( token: Optional[str], sort: Optional[Dict[str, Dict[str, str]]], collection_ids: Optional[List[str]], - datetime_search: Dict[str, Optional[str]], + datetime_search: str, ignore_unavailable: bool = True, ) -> Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]: """Execute a search query with limit and other optional parameters. @@ -734,7 +729,7 @@ async def execute_search( token (Optional[str]): The token used to return the next set of results. sort (Optional[Dict[str, Dict[str, str]]]): Specifies how the results should be sorted. collection_ids (Optional[List[str]]): The collection ids to search. - datetime_search (Dict[str, Optional[str]]): Datetime range used for index selection. + datetime_search (str): Datetime used for index selection. ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True. Returns: @@ -824,7 +819,7 @@ async def aggregate( geometry_geohash_grid_precision: int, geometry_geotile_grid_precision: int, datetime_frequency_interval: str, - datetime_search, + datetime_search: str, ignore_unavailable: Optional[bool] = True, ): """Return aggregations of STAC Items.""" diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 05aac1763..15406b93c 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -457,9 +457,7 @@ def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str] ) @staticmethod - def apply_datetime_filter( - search: Search, datetime: Optional[str] - ) -> Tuple[Search, Dict[str, Optional[str]]]: + def apply_datetime_filter(search: Search, datetime: Optional[str]) -> Search: """Apply a filter to search on datetime, start_datetime, and end_datetime fields. Args: @@ -471,14 +469,14 @@ def apply_datetime_filter( """ datetime_search = return_date(datetime) - if not datetime_search: - return search, datetime_search - # USE_DATETIME env var # True: Search by datetime, if null search by start/end datetime # False: Always search only by start/end datetime USE_DATETIME = get_bool_env("USE_DATETIME", default=True) + if not datetime_search: + return search + if USE_DATETIME: if "eq" in datetime_search: # For exact matches, include: @@ -554,10 +552,7 @@ def apply_datetime_filter( ), ] - return ( - search.query(Q("bool", should=should, minimum_should_match=1)), - datetime_search, - ) + return search.query(Q("bool", should=should, minimum_should_match=1)) else: if "eq" in datetime_search: filter_query = Q( @@ -591,7 +586,7 @@ def apply_datetime_filter( ), ], ) - return search.query(filter_query), datetime_search + return search.query(filter_query) @staticmethod def apply_bbox_filter(search: Search, bbox: List): @@ -728,7 +723,7 @@ async def execute_search( token: Optional[str], sort: Optional[Dict[str, Dict[str, str]]], collection_ids: Optional[List[str]], - datetime_search: Dict[str, Optional[str]], + datetime_search: str, ignore_unavailable: bool = True, ) -> Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]: """Execute a search query with limit and other optional parameters. @@ -739,7 +734,7 @@ async def execute_search( token (Optional[str]): The token used to return the next set of results. sort (Optional[Dict[str, Dict[str, str]]]): Specifies how the results should be sorted. collection_ids (Optional[List[str]]): The collection ids to search. - datetime_search (Dict[str, Optional[str]]): Datetime range used for index selection. + datetime_search (str): Datetime used for index selection. ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True. Returns: @@ -835,7 +830,7 @@ async def aggregate( geometry_geohash_grid_precision: int, geometry_geotile_grid_precision: int, datetime_frequency_interval: str, - datetime_search, + datetime_search: str, ignore_unavailable: Optional[bool] = True, ): """Return aggregations of STAC Items.""" diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py index 1f77cd9ee..a75d575d7 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py @@ -313,11 +313,9 @@ async def aggregate( ) if aggregate_request.datetime: - search, datetime_search = self.database.apply_datetime_filter( + search = self.database.apply_datetime_filter( search=search, datetime=aggregate_request.datetime ) - else: - datetime_search = {"gte": None, "lte": None} if aggregate_request.bbox: bbox = aggregate_request.bbox @@ -416,7 +414,7 @@ async def aggregate( geometry_geohash_grid_precision, geometry_geotile_grid_precision, datetime_frequency_interval, - datetime_search, + aggregate_request.datetime, ) except Exception as error: if not isinstance(error, IndexError): diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py index 11360fc1e..041cdd1bc 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py @@ -145,7 +145,7 @@ def extract_date(date_str: str) -> date: date_str: ISO format date string Returns: - A date object extracted from the input string. + A date object extracted from the input string or None. """ date_str = date_str.replace("Z", "+00:00") return datetime_type.fromisoformat(date_str).date() diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py index c36a36fa0..19a995c5e 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py @@ -4,11 +4,9 @@ """ import re -from datetime import datetime +from datetime import date, datetime from functools import lru_cache -from typing import Any, List, Optional - -from dateutil.parser import parse # type: ignore[import] +from typing import Any, Dict, List, Optional, Tuple from stac_fastapi.sfeos_helpers.mappings import ( _ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE, @@ -71,54 +69,103 @@ def indices(collection_ids: Optional[List[str]]) -> str: def filter_indexes_by_datetime( - indexes: List[str], gte: Optional[str], lte: Optional[str] + collection_indexes: List[Tuple[Dict[str, str], ...]], + datetime_search: Dict[str, Dict[str, Optional[str]]], + use_datetime: bool, ) -> List[str]: - """Filter indexes based on datetime range extracted from index names. + """ + Filter Elasticsearch index aliases based on datetime search criteria. + + Filters a list of collection indexes by matching their datetime, start_datetime, and end_datetime + aliases against the provided search criteria. Each criterion can have optional 'gte' (greater than + or equal) and 'lte' (less than or equal) bounds. Args: - indexes: List of index names containing dates - gte: Greater than or equal date filter (ISO format, optional 'Z' suffix) - lte: Less than or equal date filter (ISO format, optional 'Z' suffix) + collection_indexes (List[Tuple[Dict[str, str], ...]]): A list of tuples containing dictionaries + with 'datetime', 'start_datetime', and 'end_datetime' aliases. + datetime_search (Dict[str, Dict[str, Optional[str]]]): A dictionary with keys 'datetime', + 'start_datetime', and 'end_datetime', each containing 'gte' and 'lte' criteria as ISO format + datetime strings or None. + use_datetime (bool): Flag determining which datetime field to filter on: + - True: Filters using 'datetime' alias. + - False: Filters using 'start_datetime' and 'end_datetime' aliases. Returns: - List of filtered index names + List[str]: A list of start_datetime aliases that match all provided search criteria. """ - def parse_datetime(dt_str: str) -> datetime: - """Parse datetime string, handling both with and without 'Z' suffix.""" - return parse(dt_str).replace(tzinfo=None) - - def extract_date_range_from_index(index_name: str) -> tuple: - """Extract start and end dates from index name.""" - date_pattern = r"(\d{4}-\d{2}-\d{2})" - dates = re.findall(date_pattern, index_name) - - if len(dates) == 1: - start_date = datetime.strptime(dates[0], "%Y-%m-%d") - max_date = datetime.max.replace(microsecond=0) - return start_date, max_date - else: - start_date = datetime.strptime(dates[0], "%Y-%m-%d") - end_date = datetime.strptime(dates[1], "%Y-%m-%d") - return start_date, end_date - - def is_index_in_range( - start_date: datetime, end_date: datetime, gte_dt: datetime, lte_dt: datetime + def extract_date_from_alias(alias: str) -> Optional[tuple[datetime, datetime]]: + date_pattern = re.compile(r"\d{4}-\d{2}-\d{2}") + try: + dates = date_pattern.findall(alias) + + if not dates: + return None + + if len(dates) >= 2: + return datetime.strptime(dates[-2], "%Y-%m-%d"), datetime.strptime( + dates[-1], "%Y-%m-%d" + ) + else: + date = datetime.strptime(dates[-1], "%Y-%m-%d") + return date, date + except (ValueError, IndexError): + return None + + def parse_search_date(date_str: Optional[str]) -> Optional[date]: + if not date_str: + return None + date_str = date_str.rstrip("Z") + return datetime.fromisoformat(date_str).date() + + def check_criteria( + value_begin: datetime, value_end: datetime, criteria: Dict ) -> bool: - """Check if index date range overlaps with filter range.""" - return not ( - end_date.date() < gte_dt.date() or start_date.date() > lte_dt.date() - ) + gte = parse_search_date(criteria.get("gte")) + lte = parse_search_date(criteria.get("lte")) - gte_dt = parse_datetime(gte) if gte else datetime.min.replace(microsecond=0) - lte_dt = parse_datetime(lte) if lte else datetime.max.replace(microsecond=0) + if gte and value_end.date() < gte: + return False + if lte and value_begin.date() > lte: + return False + + return True filtered_indexes = [] - for index in indexes: - start_date, end_date = extract_date_range_from_index(index) - if is_index_in_range(start_date, end_date, gte_dt, lte_dt): - filtered_indexes.append(index) + for index_tuple in collection_indexes: + if not index_tuple: + continue + + index_dict = index_tuple[0] + start_datetime_alias = index_dict.get("start_datetime") + end_datetime_alias = index_dict.get("end_datetime") + datetime_alias = index_dict.get("datetime") + + if start_datetime_alias: + start_date = extract_date_from_alias(start_datetime_alias) + if not check_criteria( + start_date[0], start_date[1], datetime_search.get("start_datetime", {}) + ): + continue + if end_datetime_alias: + end_date = extract_date_from_alias(end_datetime_alias) + if not check_criteria( + end_date[0], end_date[1], datetime_search.get("end_datetime", {}) + ): + continue + if datetime_alias: + datetime_date = extract_date_from_alias(datetime_alias) + if not check_criteria( + datetime_date[0], datetime_date[1], datetime_search.get("datetime", {}) + ): + continue + + primary_datetime_alias = ( + datetime_alias if use_datetime else start_datetime_alias + ) + + filtered_indexes.append(primary_datetime_alias) return filtered_indexes diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py index 72285a56f..abc8b7e6f 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py @@ -130,7 +130,7 @@ def apply_collections_datetime_filter_shared( def apply_collections_bbox_filter_shared( - bbox: Union[str, List[float], None] + bbox: Union[str, List[float], None], ) -> Optional[Dict[str, Dict]]: """Create a geo_shape filter for collections bbox search. diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py index 42028a7a3..541c883cf 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py @@ -1,8 +1,9 @@ """Search engine adapters for different implementations.""" import uuid -from typing import Any, Dict +from typing import Any, Dict, List, Literal +from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.sfeos_helpers.database import ( index_alias_by_collection_id, index_by_collection_id, @@ -18,6 +19,16 @@ class IndexOperations: """Base class for search engine adapters with common implementations.""" + @property + def use_datetime(self) -> bool: + """Get USE_DATETIME setting dynamically.""" + return get_bool_env("USE_DATETIME", default=True) + + @property + def primary_datetime_name(self) -> str: + """Get primary datetime field name based on current USE_DATETIME setting.""" + return "datetime" if self.use_datetime else "start_datetime" + async def create_simple_index(self, client: Any, collection_id: str) -> str: """Create a simple index for the given collection. @@ -39,26 +50,51 @@ async def create_simple_index(self, client: Any, collection_id: str) -> str: return index_name async def create_datetime_index( - self, client: Any, collection_id: str, start_date: str + self, + client: Any, + collection_id: str, + start_datetime: str | None, + datetime: str | None, + end_datetime: str | None, ) -> str: """Create a datetime-based index for the given collection. Args: client: Search engine client instance. collection_id (str): Collection identifier. - start_date (str): Start date for the alias. + start_datetime (str | None): Start datetime for the index alias. + datetime (str | None): Datetime for the datetime alias. + end_datetime (str | None): End datetime for the index alias. Returns: - str: Created index alias name. + str: Created datetime alias name. """ index_name = self.create_index_name(collection_id) - alias_name = self.create_alias_name(collection_id, start_date) collection_alias = index_alias_by_collection_id(collection_id) + + aliases: Dict[str, Any] = { + collection_alias: {}, + } + + if start_datetime: + alias_start_date = self.create_alias_name( + collection_id, "start_datetime", start_datetime + ) + alias_end_date = self.create_alias_name( + collection_id, "end_datetime", end_datetime + ) + aliases[alias_start_date] = {} + aliases[alias_end_date] = {} + created_alias = alias_start_date + else: + created_alias = self.create_alias_name(collection_id, "datetime", datetime) + aliases[created_alias] = {} + await client.indices.create( index=index_name, - body=self._create_index_body({collection_alias: {}, alias_name: {}}), + body=self._create_index_body(aliases), ) - return alias_name + return created_alias @staticmethod async def update_index_alias(client: Any, end_date: str, old_alias: str) -> str: @@ -84,23 +120,33 @@ async def update_index_alias(client: Any, end_date: str, old_alias: str) -> str: return new_alias @staticmethod - async def change_alias_name(client: Any, old_alias: str, new_alias: str) -> None: - """Change alias name from old to new. + async def change_alias_name( + client: Any, + old_start_datetime_alias: str, + aliases_to_change: List[str], + aliases_to_create: List[str], + ) -> None: + """Change alias names by removing old aliases and adding new ones. Args: client: Search engine client instance. - old_alias (str): Current alias name. - new_alias (str): New alias name. + old_start_datetime_alias (str): Current start_datetime alias name to identify the index. + aliases_to_change (List[str]): List of old alias names to remove. + aliases_to_create (List[str]): List of new alias names to add. Returns: None """ - aliases_info = await client.indices.get_alias(name=old_alias) + aliases_info = await client.indices.get_alias(name=old_start_datetime_alias) + index_name = list(aliases_info.keys())[0] actions = [] - for index_name in aliases_info.keys(): - actions.append({"remove": {"index": index_name, "alias": old_alias}}) + for new_alias in aliases_to_create: actions.append({"add": {"index": index_name, "alias": new_alias}}) + + for old_alias in aliases_to_change: + actions.append({"remove": {"index": index_name, "alias": old_alias}}) + await client.indices.update_aliases(body={"actions": actions}) @staticmethod @@ -117,18 +163,23 @@ def create_index_name(collection_id: str) -> str: return f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{uuid.uuid4()}" @staticmethod - def create_alias_name(collection_id: str, start_date: str) -> str: - """Create index name from collection ID and uuid4. + def create_alias_name( + collection_id: str, + name: Literal["start_datetime", "datetime", "end_datetime"], + start_date: str, + ) -> str: + """Create alias name from collection ID and date. Args: collection_id (str): Collection identifier. - start_date (str): Start date for the alias. + name (Literal["start_datetime", "datetime", "end_datetime"]): Type of alias to create. + start_date (str): Date value for the alias. Returns: - str: Alias name with initial date. + str: Formatted alias name with prefix, type, collection ID, and date. """ cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE) - return f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{start_date}" + return f"{ITEMS_INDEX_PREFIX}{name}_{cleaned.lower()}_{start_date}" @staticmethod def _create_index_body(aliases: Dict[str, Dict]) -> Dict[str, Any]: @@ -146,21 +197,26 @@ def _create_index_body(aliases: Dict[str, Dict]) -> Dict[str, Any]: "settings": ES_ITEMS_SETTINGS, } - @staticmethod - async def find_latest_item_in_index(client: Any, index_name: str) -> dict[str, Any]: - """Find the latest item date in the specified index. + async def find_latest_item_in_index( + self, client: Any, index_name: str + ) -> dict[str, Any]: + """Find the latest item in the specified index. Args: client: Search engine client instance. index_name (str): Name of the index to query. Returns: - datetime: Date of the latest item in the index. + dict[str, Any]: Latest item document from the index with metadata. """ query = { "size": 1, - "sort": [{"properties.datetime": {"order": "desc"}}], - "_source": ["properties.datetime"], + "sort": [{f"properties.{self.primary_datetime_name}": {"order": "desc"}}], + "_source": [ + "properties.start_datetime", + "properties.datetime", + "properties.end_datetime", + ], } response = await client.search(index=index_name, body=query) diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py index 06e9c7298..14e831910 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py @@ -1,10 +1,11 @@ """Async index insertion strategies.""" + import logging -from datetime import timedelta -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from fastapi import HTTPException, status +from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.sfeos_helpers.database import ( extract_date, extract_first_date_from_index, @@ -14,7 +15,7 @@ from .base import BaseIndexInserter from .index_operations import IndexOperations -from .managers import DatetimeIndexManager +from .managers import DatetimeIndexManager, ProductDatetimes from .selection import DatetimeBasedIndexSelector logger = logging.getLogger(__name__) @@ -34,6 +35,24 @@ def __init__(self, client: Any, index_operations: IndexOperations): self.index_operations = index_operations self.datetime_manager = DatetimeIndexManager(client, index_operations) + @property + def use_datetime(self) -> bool: + """Get USE_DATETIME setting dynamically. + + Returns: + bool: Current value of USE_DATETIME environment variable. + """ + return get_bool_env("USE_DATETIME", default=True) + + @property + def primary_datetime_name(self) -> str: + """Get primary datetime field name based on current USE_DATETIME setting. + + Returns: + str: "datetime" if USE_DATETIME is True, else "start_datetime". + """ + return "datetime" if self.use_datetime else "start_datetime" + @staticmethod def should_create_collection_index() -> bool: """Whether this strategy requires collection index creation. @@ -89,7 +108,7 @@ async def prepare_bulk_actions( logger.error(msg) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=msg) - items.sort(key=lambda item: item["properties"]["datetime"]) + items.sort(key=lambda item: item["properties"][self.primary_datetime_name]) index_selector = DatetimeBasedIndexSelector(self.client) await self._ensure_indexes_exist(index_selector, collection_id, items) @@ -118,7 +137,7 @@ async def _get_target_index_internal( collection_id: str, product: Dict[str, Any], check_size: bool = True, - ) -> str: + ) -> Optional[str]: """Get target index with size checking internally. Args: @@ -130,44 +149,84 @@ async def _get_target_index_internal( Returns: str: Target index name. """ - product_datetime = self.datetime_manager.validate_product_datetime(product) - datetime_range = {"gte": product_datetime, "lte": product_datetime} + product_datetimes = self.datetime_manager.validate_product_datetimes( + product, self.use_datetime + ) + + primary_datetime_value = ( + product_datetimes.datetime + if self.use_datetime + else product_datetimes.start_datetime + ) + target_index = await index_selector.select_indexes( - [collection_id], datetime_range + [collection_id], primary_datetime_value, for_insertion=True ) all_indexes = await index_selector.get_collection_indexes(collection_id) if not all_indexes: target_index = await self.datetime_manager.handle_new_collection( - collection_id, product_datetime + collection_id, self.primary_datetime_name, product_datetimes ) await index_selector.refresh_cache() return target_index - all_indexes.sort() - start_date = extract_date(product_datetime) - end_date = extract_first_date_from_index(all_indexes[0]) + all_indexes = sorted( + all_indexes, key=lambda x: x[0][self.primary_datetime_name] + ) + start_date = extract_date(primary_datetime_value) + end_date = extract_first_date_from_index( + all_indexes[0][0][self.primary_datetime_name] + ) if start_date < end_date: alias = await self.datetime_manager.handle_early_date( - collection_id, start_date, end_date + collection_id, + self.primary_datetime_name, + product_datetimes, + all_indexes[0][0], ) await index_selector.refresh_cache() - return alias - if target_index != all_indexes[-1]: - return target_index + if target_index != all_indexes[-1][0][self.primary_datetime_name]: + for item in all_indexes: + aliases_dict = item[0] + if target_index in aliases_dict.values(): + await self.datetime_manager.handle_early_date( + collection_id, + self.primary_datetime_name, + product_datetimes, + aliases_dict, + ) + return target_index if check_size and await self.datetime_manager.size_manager.is_index_oversized( target_index ): - target_index = await self.datetime_manager.handle_oversized_index( - collection_id, target_index, product_datetime - ) - await index_selector.refresh_cache() - - return target_index + for item in all_indexes: + aliases_dict = item[0] + if target_index in aliases_dict.values(): + target_index = await self.datetime_manager.handle_oversized_index( + collection_id, + self.primary_datetime_name, + product_datetimes, + aliases_dict, + ) + await index_selector.refresh_cache() + return target_index + + for item in all_indexes: + aliases_dict = item[0] + if target_index in aliases_dict.values(): + await self.datetime_manager.handle_early_date( + collection_id, + self.primary_datetime_name, + product_datetimes, + aliases_dict, + ) + return target_index + return None async def _ensure_indexes_exist( self, index_selector, collection_id: str, items: List[Dict[str, Any]] @@ -183,10 +242,23 @@ async def _ensure_indexes_exist( if not all_indexes: first_item = items[0] + properties = first_item["properties"] + index_params = { + "start_datetime": str(extract_date(properties["start_datetime"])) + if self.primary_datetime_name == "start_datetime" + else None, + "datetime": str(extract_date(properties["datetime"])) + if self.primary_datetime_name == "datetime" + else None, + "end_datetime": str(extract_date(properties["end_datetime"])) + if self.primary_datetime_name == "start_datetime" + else None, + } + await self.index_operations.create_datetime_index( self.client, collection_id, - extract_date(first_item["properties"]["datetime"]), + **index_params, ) await index_selector.refresh_cache() @@ -212,8 +284,11 @@ async def _check_and_handle_oversized_index( ) all_indexes = await index_selector.get_collection_indexes(collection_id) - all_indexes.sort() - latest_index = all_indexes[-1] + all_indexes = sorted( + all_indexes, key=lambda x: x[0][self.primary_datetime_name] + ) + + latest_index = all_indexes[-1][0][self.primary_datetime_name] if first_item_index != latest_index: return None @@ -226,14 +301,17 @@ async def _check_and_handle_oversized_index( latest_item = await self.index_operations.find_latest_item_in_index( self.client, latest_index ) - product_datetime = latest_item["_source"]["properties"]["datetime"] - end_date = extract_date(product_datetime) - await self.index_operations.update_index_alias( - self.client, str(end_date), latest_index + product_datetimes = ProductDatetimes( + start_datetime=latest_item["_source"]["properties"]["start_datetime"], + datetime=latest_item["_source"]["properties"]["datetime"], + end_datetime=latest_item["_source"]["properties"]["end_datetime"], ) - next_day_start = end_date + timedelta(days=1) - await self.index_operations.create_datetime_index( - self.client, collection_id, str(next_day_start) + + await self.datetime_manager.handle_oversized_index( + collection_id, + self.primary_datetime_name, + product_datetimes, + all_indexes[-1][0], ) await index_selector.refresh_cache() diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py index 1194e6345..bb6d4d9c6 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py @@ -2,9 +2,10 @@ import logging import os -from datetime import datetime, timedelta -from typing import Any, Dict +from datetime import timedelta +from typing import Any, Dict, NamedTuple +from dateutil import parser # type: ignore from fastapi import HTTPException, status from stac_fastapi.sfeos_helpers.database import ( @@ -17,6 +18,20 @@ logger = logging.getLogger(__name__) +class ProductDatetimes(NamedTuple): + """Named tuple representing product datetime fields. + + Attributes: + start_datetime (str | None): ISO format start datetime string or None. + datetime (str | None): ISO format datetime string or None. + end_datetime (str | None): ISO format end datetime string or None. + """ + + start_datetime: str | None + datetime: str | None + end_datetime: str | None + + class IndexSizeManager: """Manages index size limits and operations.""" @@ -108,91 +123,248 @@ def __init__(self, client: Any, index_operations: IndexOperations): self.size_manager = IndexSizeManager(client) @staticmethod - def validate_product_datetime(product: Dict[str, Any]) -> str: - """Validate and extract datetime from product. + def validate_product_datetimes( + product: Dict[str, Any], use_datetime + ) -> ProductDatetimes: + """Validate and extract datetime fields from product. + + Validation rules depend on USE_DATETIME: + - USE_DATETIME=True: 'datetime' is required, optional start/end + - USE_DATETIME=False: both 'start_datetime' and 'end_datetime' required, start <= end Args: product (Dict[str, Any]): Product data containing datetime information. + use_datetime (bool): Flag determining validation mode. + - True: validates against 'datetime' field. + - False: validates against 'start_datetime' and 'end_datetime' fields. Returns: - str: Validated product datetime. + ProductDatetimes: Named tuple containing parsed datetime values: + - start_datetime (str | None): ISO 8601 start datetime string or None. + - datetime (str | None): ISO 8601 datetime string or None. + - end_datetime (str | None): ISO 8601 end datetime string or None. Raises: - HTTPException: If product datetime is missing or invalid. + HTTPException: If validation fails based on USE_DATETIME configuration. """ - product_datetime = product["properties"]["datetime"] - if not product_datetime: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Product datetime is required for indexing", - ) - return product_datetime + properties = product.get("properties", {}) + start_str = properties.get("start_datetime") + dt_str = properties.get("datetime") + end_str = properties.get("end_datetime") + + start = parser.isoparse(start_str) if start_str else None + dt = parser.isoparse(dt_str) if dt_str else None + end = parser.isoparse(end_str) if end_str else None + + if use_datetime: + if not dt: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="'datetime' field is required", + ) + else: + if not start or not end: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Both 'start_datetime' and 'end_datetime' fields are required", + ) + if not (start <= end): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="'start_datetime' must be <= 'end_datetime'", + ) + if dt and not (start <= dt <= end): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="'start_datetime' <= 'datetime' <= 'end_datetime' is required", + ) + + return ProductDatetimes( + start_datetime=start_str, + datetime=dt_str, + end_datetime=end_str, + ) async def handle_new_collection( - self, collection_id: str, product_datetime: str + self, + collection_id: str, + primary_datetime_name: str, + product_datetimes: ProductDatetimes, ) -> str: """Handle index creation for new collection asynchronously. Args: collection_id (str): Collection identifier. - product_datetime (str): Product datetime for index naming. - + primary_datetime_name (str): Name of the primary datetime field. + If "start_datetime", indexes are created on start_datetime and end_datetime fields. + If "datetime", indexes are created on the datetime field. + product_datetimes (ProductDatetimes): Object containing start_datetime, datetime, and end_datetime. Returns: - str: Created index name. + str: Created datetime index name. """ + index_params = { + "start_datetime": str(extract_date(product_datetimes.start_datetime)) + if primary_datetime_name == "start_datetime" + else None, + "datetime": str(extract_date(product_datetimes.datetime)) + if primary_datetime_name == "datetime" + else None, + "end_datetime": str(extract_date(product_datetimes.end_datetime)) + if primary_datetime_name == "start_datetime" + else None, + } + target_index = await self.index_operations.create_datetime_index( - self.client, collection_id, extract_date(product_datetime) + self.client, collection_id, **index_params ) + logger.info( f"Successfully created index '{target_index}' for collection '{collection_id}'" ) return target_index async def handle_early_date( - self, collection_id: str, start_date: datetime, end_date: datetime + self, + collection_id: str, + primary_datetime_name: str, + product_datetimes: ProductDatetimes, + old_aliases: Dict[str, str], ) -> str: """Handle product with date earlier than existing indexes asynchronously. Args: collection_id (str): Collection identifier. - start_date (datetime): Start date for the new index. - end_date (datetime): End date for alias update. + primary_datetime_name (str): Name of the primary datetime field. + If "start_datetime", handles start_datetime and end_datetime fields. + If "datetime", handles the datetime field. + product_datetimes (ProductDatetimes): Object containing start_datetime, datetime, and end_datetime. + old_aliases (Dict[str, str]): Dictionary mapping alias types to their current names. Returns: - str: Updated alias name. + str: Updated datetime alias name. """ - old_alias = self.index_operations.create_alias_name( - collection_id, str(end_date) - ) - new_alias = self.index_operations.create_alias_name( - collection_id, str(start_date) - ) - await self.index_operations.change_alias_name(self.client, old_alias, new_alias) - return new_alias + new_aliases = [] + old_alias_names = [] + + if primary_datetime_name == "start_datetime": + new_start_alias = self.index_operations.create_alias_name( + collection_id, + "start_datetime", + str(extract_date(product_datetimes.start_datetime)), + ) + + if extract_date( + product_datetimes.start_datetime + ) < extract_first_date_from_index(old_aliases["start_datetime"]): + new_aliases.append(new_start_alias) + old_alias_names.append(old_aliases["start_datetime"]) + + new_end_alias = self.index_operations.create_alias_name( + collection_id, + "end_datetime", + str(extract_date(product_datetimes.end_datetime)), + ) + + if extract_date( + product_datetimes.end_datetime + ) > extract_first_date_from_index(old_aliases["end_datetime"]): + new_aliases.append(new_end_alias) + old_alias_names.append(old_aliases["end_datetime"]) + + new_primary_alias = new_start_alias + else: + + new_primary_alias = self.index_operations.create_alias_name( + collection_id, "datetime", str(extract_date(product_datetimes.datetime)) + ) + + if extract_date(product_datetimes.datetime) < extract_first_date_from_index( + old_aliases["datetime"] + ): + new_aliases.append(new_primary_alias) + old_alias_names.append(old_aliases["datetime"]) + + if old_alias_names: + await self.index_operations.change_alias_name( + self.client, + old_aliases[primary_datetime_name], + old_alias_names, + new_aliases, + ) + + return new_primary_alias async def handle_oversized_index( - self, collection_id: str, target_index: str, product_datetime: str + self, + collection_id: str, + primary_datetime_name: str, + product_datetimes: ProductDatetimes, + old_aliases: Dict[str, str], ) -> str: """Handle index that exceeds size limit asynchronously. Args: collection_id (str): Collection identifier. - target_index (str): Current target index name. - product_datetime (str): Product datetime for new index. + primary_datetime_name (str): Name of the primary datetime field. + If "start_datetime", handles start_datetime and end_datetime fields. + If "datetime", handles the datetime field. + product_datetimes (ProductDatetimes): Object containing start_datetime, datetime, and end_datetime. + old_aliases (Dict[str, str]): Dictionary mapping alias types to their current names. Returns: - str: New or updated index name. + str: Updated or newly created datetime alias name. """ - end_date = extract_date(product_datetime) - latest_index_start = extract_first_date_from_index(target_index) + current_alias = old_aliases[primary_datetime_name] + new_aliases = [] + old_alias_names = [] + + if primary_datetime_name == "start_datetime": + start_dt = extract_date(product_datetimes.start_datetime) + end_dt = extract_date(product_datetimes.end_datetime) + old_start_dt = extract_first_date_from_index(current_alias) + old_end_dt = extract_first_date_from_index(old_aliases["end_datetime"]) + + if start_dt != old_start_dt: + new_start_alias = f"{current_alias}-{str(start_dt)}" + new_aliases.append(new_start_alias) + old_alias_names.append(current_alias) + + if end_dt > old_end_dt: + new_end_alias = self.index_operations.create_alias_name( + collection_id, "end_datetime", str(end_dt) + ) + new_aliases.append(new_end_alias) + old_alias_names.append(old_aliases["end_datetime"]) - if end_date != latest_index_start: - await self.index_operations.update_index_alias( - self.client, str(end_date), target_index - ) - target_index = await self.index_operations.create_datetime_index( - self.client, collection_id, str(end_date + timedelta(days=1)) - ) + if old_alias_names: + await self.index_operations.change_alias_name( + self.client, current_alias, old_alias_names, new_aliases + ) - return target_index + if start_dt != old_start_dt: + return await self.index_operations.create_datetime_index( + self.client, + collection_id, + start_datetime=str(start_dt + timedelta(days=1)), + datetime=None, + end_datetime=str(end_dt), + ) + else: + dt = extract_date(product_datetimes.datetime) + old_dt = extract_first_date_from_index(current_alias) + + if dt != old_dt: + new_datetime_alias = f"{current_alias}-{str(dt)}" + await self.index_operations.change_alias_name( + self.client, current_alias, [current_alias], [new_datetime_alias] + ) + return await self.index_operations.create_datetime_index( + self.client, + collection_id, + start_datetime=None, + datetime=str(dt + timedelta(days=1)), + end_datetime=None, + ) + + return current_alias diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py index 95f406728..6bc0cba93 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py @@ -1,7 +1,7 @@ """Base classes for index selection strategies.""" from abc import ABC, abstractmethod -from typing import Dict, List, Optional +from typing import List, Optional class BaseIndexSelector(ABC): @@ -11,13 +11,17 @@ class BaseIndexSelector(ABC): async def select_indexes( self, collection_ids: Optional[List[str]], - datetime_search: Dict[str, Optional[str]], + datetime_search: str, + for_insertion: bool = False, ) -> str: """Select appropriate indexes asynchronously. Args: collection_ids (Optional[List[str]]): List of collection IDs to filter by. - datetime_search (Dict[str, Optional[str]]): Datetime search criteria. + datetime_search (str): Datetime search criteria. + for_insertion (bool): If True, selects indexes for inserting items into + the database. If False, selects indexes for searching/querying items. + Defaults to False (search mode). Returns: str: Comma-separated string of selected index names. diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py index 3b65244d4..60022eb98 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py @@ -1,9 +1,9 @@ """Cache management for index selection strategies.""" +import copy import threading import time -from collections import defaultdict -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from stac_fastapi.sfeos_helpers.database import index_alias_by_collection_id from stac_fastapi.sfeos_helpers.mappings import ITEMS_INDEX_PREFIX @@ -18,7 +18,7 @@ def __init__(self, cache_ttl_seconds: int = 3600): Args: cache_ttl_seconds (int): Time-to-live for cache entries in seconds. """ - self._cache: Optional[Dict[str, List[str]]] = None + self._cache: Optional[Dict[str, List[Tuple[Dict[str, str]]]]] = None self._timestamp: float = 0 self._ttl = cache_ttl_seconds self._lock = threading.Lock() @@ -32,30 +32,32 @@ def is_expired(self) -> bool: """ return time.time() - self._timestamp > self._ttl - def get_cache(self) -> Optional[Dict[str, List[str]]]: + def get_cache(self) -> Optional[Dict[str, List[Tuple[Dict[str, str]]]]]: """Get the current cache if not expired. Returns: - Optional[Dict[str, List[str]]]: Cache data if valid, None if expired. + Optional[Dict[str, List[Tuple[Dict[str, str]]]]]: Cache data if valid, None if expired. """ with self._lock: if self.is_expired: return None - return {k: v.copy() for k, v in self._cache.items()} + return copy.deepcopy(self._cache) if self._cache else None - def set_cache(self, data: Dict[str, List[str]]) -> None: + def set_cache(self, data: Dict[str, List[Tuple[Dict[str, str]]]]) -> None: """Set cache data and update timestamp. Args: - data (Dict[str, List[str]]): Cache data to store. + data (Dict[str, List[Tuple[Dict[str, str]]]]): Cache data to store. """ - self._cache = data - self._timestamp = time.time() + with self._lock: + self._cache = data + self._timestamp = time.time() def clear_cache(self) -> None: """Clear the cache and reset timestamp.""" - self._cache = None - self._timestamp = 0 + with self._lock: + self._cache = None + self._timestamp = 0 class IndexAliasLoader: @@ -71,15 +73,16 @@ def __init__(self, client: Any, cache_manager: IndexCacheManager): self.client = client self.cache_manager = cache_manager - async def load_aliases(self) -> Dict[str, List[str]]: + async def load_aliases(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: """Load index aliases from search engine. Returns: - Dict[str, List[str]]: Mapping of base aliases to item aliases. + Dict[str, List[Tuple[Dict[str, str]]]]: Mapping of main collection aliases to their data. """ response = await self.client.indices.get_alias(index=f"{ITEMS_INDEX_PREFIX}*") - result = defaultdict(list) - for index_info in response.values(): + result: Dict[str, List[Tuple[Dict[str, str]]]] = {} + + for index_name, index_info in response.items(): aliases = index_info.get("aliases", {}) items_aliases = sorted( [ @@ -90,38 +93,89 @@ async def load_aliases(self) -> Dict[str, List[str]]: ) if items_aliases: - result[items_aliases[0]].extend(items_aliases[1:]) + main_alias = self._find_main_alias(items_aliases) + aliases_dict = self._organize_aliases(items_aliases, main_alias) + + if aliases_dict and main_alias not in result: + result[main_alias] = [(aliases_dict,)] self.cache_manager.set_cache(result) return result - async def get_aliases(self) -> Dict[str, List[str]]: + @staticmethod + def _find_main_alias(aliases: List[str]) -> str: + """Find the main collection alias (without temporal suffixes). + + Args: + aliases (List[str]): List of all aliases for an index. + + Returns: + str: The main collection alias. + """ + temporal_keywords = ["datetime", "start_datetime", "end_datetime"] + + for alias in aliases: + if not any(keyword in alias for keyword in temporal_keywords): + return alias + + return aliases[0] + + @staticmethod + def _organize_aliases(aliases: List[str], main_alias: str) -> Dict[str, str]: + """Organize temporal aliases into a dictionary with type as key. + + Args: + aliases (List[str]): All aliases for the index. + main_alias (str): The main collection alias. + + Returns: + Dict[str, str]: Dictionary with datetime types as keys and alias names as values. + """ + aliases_dict = {} + + for alias in aliases: + if alias == main_alias: + continue + + if "start_datetime" in alias: + aliases_dict["start_datetime"] = alias + elif "end_datetime" in alias: + aliases_dict["end_datetime"] = alias + elif "datetime" in alias: + aliases_dict["datetime"] = alias + + return aliases_dict + + async def get_aliases(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: """Get aliases from cache or load if expired. Returns: - Dict[str, List[str]]: Alias mapping data. + Dict[str, List[Tuple[Dict[str, str]]]]: Alias mapping data. """ cached = self.cache_manager.get_cache() if cached is not None: return cached return await self.load_aliases() - async def refresh_aliases(self) -> Dict[str, List[str]]: + async def refresh_aliases(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: """Force refresh aliases from search engine. Returns: - Dict[str, List[str]]: Fresh alias mapping data. + Dict[str, List[Tuple[Dict[str, str]]]]: Fresh alias mapping data. """ return await self.load_aliases() - async def get_collection_indexes(self, collection_id: str) -> List[str]: - """Get all index aliases for a specific collection. + async def get_collection_indexes( + self, collection_id: str + ) -> List[Tuple[Dict[str, str]]]: + """Get index information for a specific collection. Args: collection_id (str): Collection identifier. Returns: - List[str]: List of index aliases for the collection. + List[Tuple[Dict[str, str]]]: List of tuples with alias dictionaries. """ aliases = await self.get_aliases() - return aliases.get(index_alias_by_collection_id(collection_id), []) + main_alias = index_alias_by_collection_id(collection_id) + return aliases.get(main_alias, []) diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py index 20f919ab9..b53d8aa6d 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py @@ -1,8 +1,9 @@ """Async index selectors with datetime-based filtering.""" -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple -from stac_fastapi.sfeos_helpers.database import filter_indexes_by_datetime +from stac_fastapi.core.utilities import get_bool_env +from stac_fastapi.sfeos_helpers.database import filter_indexes_by_datetime, return_date from stac_fastapi.sfeos_helpers.mappings import ITEM_INDICES from ...database import indices @@ -40,23 +41,30 @@ def __init__(self, client: Any): self.alias_loader = IndexAliasLoader(client, self.cache_manager) self._initialized = True - async def refresh_cache(self) -> Dict[str, List[str]]: + @property + def use_datetime(self) -> bool: + """Get USE_DATETIME setting dynamically.""" + return get_bool_env("USE_DATETIME", default=True) + + async def refresh_cache(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: """Force refresh of the aliases cache. Returns: - Dict[str, List[str]]: Refreshed dictionary mapping base collection aliases + Dict[str, List[Tuple[Dict[str, str]]]]: Refreshed dictionary mapping base collection aliases to lists of their corresponding item index aliases. """ return await self.alias_loader.refresh_aliases() - async def get_collection_indexes(self, collection_id: str) -> List[str]: + async def get_collection_indexes( + self, collection_id: str + ) -> List[tuple[dict[str, str]]]: """Get all index aliases for a specific collection. Args: collection_id (str): The ID of the collection to retrieve indexes for. Returns: - List[str]: List of index aliases associated with the collection. + List[tuple[dict[str, str]]]: List of index aliases associated with the collection. Returns empty list if collection is not found in cache. """ return await self.alias_loader.get_collection_indexes(collection_id) @@ -64,7 +72,8 @@ async def get_collection_indexes(self, collection_id: str) -> List[str]: async def select_indexes( self, collection_ids: Optional[List[str]], - datetime_search: Dict[str, Optional[str]], + datetime_search: str, + for_insertion: bool = False, ) -> str: """Select indexes filtered by collection IDs and datetime criteria. @@ -75,22 +84,23 @@ async def select_indexes( Args: collection_ids (Optional[List[str]]): List of collection IDs to filter by. If None or empty, returns all item indices. - datetime_search (Dict[str, Optional[str]]): Dictionary containing datetime - search criteria with 'gte' and 'lte' keys for range filtering. + datetime_search (str): Datetime search criteria. + for_insertion (bool): If True, selects indexes for inserting items into + the database. If False, selects indexes for searching/querying items. + Defaults to False (search mode). Returns: str: Comma-separated string of selected index names that match the collection and datetime criteria. Returns empty string if no indexes match the criteria. """ + datetime_filters = self.parse_datetime_filters(datetime_search, for_insertion) if collection_ids: selected_indexes = [] for collection_id in collection_ids: collection_indexes = await self.get_collection_indexes(collection_id) filtered_indexes = filter_indexes_by_datetime( - collection_indexes, - datetime_search.get("gte"), - datetime_search.get("lte"), + collection_indexes, datetime_filters, self.use_datetime ) selected_indexes.extend(filtered_indexes) @@ -98,6 +108,49 @@ async def select_indexes( return ITEM_INDICES + def parse_datetime_filters( + self, datetime: str, for_insertion: bool + ) -> Dict[str, Dict[str, Optional[str]]]: + """Parse datetime string into structured filter criteria. + + Args: + datetime: Datetime search criteria string + for_insertion (bool): If True, generates filters for inserting items. + If False, generates filters for searching items. Defaults to False. + + Returns: + Dictionary with datetime, start_datetime, and end_datetime filters + """ + parsed_datetime = return_date(datetime) + + if for_insertion: + return { + "datetime": { + "gte": None, + "lte": datetime if self.use_datetime else None, + }, + "start_datetime": { + "gte": None, + "lte": datetime if not self.use_datetime else None, + }, + "end_datetime": {"gte": None, "lte": None}, + } + + return { + "datetime": { + "gte": parsed_datetime.get("gte") if self.use_datetime else None, + "lte": parsed_datetime.get("lte") if self.use_datetime else None, + }, + "start_datetime": { + "gte": parsed_datetime.get("gte") if not self.use_datetime else None, + "lte": None, + }, + "end_datetime": { + "gte": None, + "lte": parsed_datetime.get("lte") if not self.use_datetime else None, + }, + } + class UnfilteredIndexSelector(BaseIndexSelector): """Index selector that returns all available indices without filtering.""" @@ -105,15 +158,19 @@ class UnfilteredIndexSelector(BaseIndexSelector): async def select_indexes( self, collection_ids: Optional[List[str]], - datetime_search: Dict[str, Optional[str]], + datetime_search: str, + for_insertion: bool = False, ) -> str: """Select all indices for given collections without datetime filtering. Args: collection_ids (Optional[List[str]]): List of collection IDs to filter by. If None, all collections are considered. - datetime_search (Dict[str, Optional[str]]): Datetime search criteria + datetime_search (str): Datetime search criteria (ignored by this implementation). + for_insertion (bool): If True, selects indexes for inserting items into + the database. If False, selects indexes for searching/querying items. + Defaults to False (search mode). Returns: str: Comma-separated string of all available index names for the collections. diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 38d7e5978..c40cf0b90 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -3,7 +3,6 @@ import uuid from copy import deepcopy from datetime import datetime, timedelta -from unittest.mock import patch import pytest @@ -845,635 +844,6 @@ async def test_big_int_eo_search( assert results == {value} -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_item_in_past_date_change_alias_name_for_datetime_index( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - item = load_test_data("test_item.json") - item["id"] = str(uuid.uuid4()) - item["properties"]["datetime"] = "2012-02-12T12:30:22Z" - - response = await app_client.post( - f"/collections/{item['collection']}/items", json=item - ) - assert response.status_code == 201 - indices = await txn_client.database.client.indices.get_alias( - index="items_test-collection" - ) - expected_aliases = [ - "items_test-collection_2012-02-12", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_item_uses_existing_datetime_index_for_datetime_index( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - item = load_test_data("test_item.json") - item["id"] = str(uuid.uuid4()) - - response = await app_client.post( - f"/collections/{item['collection']}/items", json=item - ) - - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias( - index="items_test-collection" - ) - expected_aliases = [ - "items_test-collection_2020-02-12", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_item_with_different_date_same_index_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - item = load_test_data("test_item.json") - item["id"] = str(uuid.uuid4()) - item["properties"]["datetime"] = "2022-02-12T12:30:22Z" - - response = await app_client.post( - f"/collections/{item['collection']}/items", json=item - ) - - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias( - index="items_test-collection" - ) - expected_aliases = [ - "items_test-collection_2020-02-12", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_new_index_when_size_limit_exceeded_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - item = load_test_data("test_item.json") - item["id"] = str(uuid.uuid4()) - item["properties"]["datetime"] = "2024-02-12T12:30:22Z" - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.return_value = 26.0 - response = await app_client.post( - f"/collections/{item['collection']}/items", json=item - ) - - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias(index="*") - expected_aliases = [ - "items_test-collection_2020-02-12-2024-02-12", - "items_test-collection_2024-02-13", - ] - all_aliases = set() - - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - item_2 = deepcopy(item) - item_2["id"] = str(uuid.uuid4()) - item_2["properties"]["datetime"] = "2023-02-12T12:30:22Z" - response_2 = await app_client.post( - f"/collections/{item_2['collection']}/items", json=item_2 - ) - assert response_2.status_code == 201 - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_item_fails_without_datetime_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - item = load_test_data("test_item.json") - item["id"] = str(uuid.uuid4()) - item["properties"]["datetime"] = None - response = await app_client.post( - f"/collections/{item['collection']}/items", json=item - ) - assert response.status_code == 400 - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_bulk_create_items_with_same_date_range_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - base_item = load_test_data("test_item.json") - items_dict = {} - - for i in range(10): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" - items_dict[item["id"]] = item - - payload = {"type": "FeatureCollection", "features": list(items_dict.values())} - response = await app_client.post( - f"/collections/{base_item['collection']}/items", json=payload - ) - - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias(index="*") - expected_aliases = [ - "items_test-collection_2020-02-12", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - return all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_bulk_create_items_with_different_date_ranges_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - base_item = load_test_data("test_item.json") - items_dict = {} - - for i in range(3): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" - items_dict[item["id"]] = item - - for i in range(2): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"]["datetime"] = f"2010-02-{10 + i}T12:30:22Z" - items_dict[item["id"]] = item - - payload = {"type": "FeatureCollection", "features": list(items_dict.values())} - - response = await app_client.post( - f"/collections/{base_item['collection']}/items", json=payload - ) - - assert response.status_code == 201 - indices = await txn_client.database.client.indices.get_alias(index="*") - - expected_aliases = ["items_test-collection_2010-02-10"] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_bulk_create_items_with_size_limit_exceeded_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2019-02", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias(index="*") - expected_aliases = [ - "items_test-collection_2010-02-10-2020-02-12", - "items_test-collection_2020-02-13", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_bulk_create_items_with_early_date_in_second_batch_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias(index="*") - expected_aliases = [ - "items_test-collection_2008-01-15-2020-02-12", - "items_test-collection_2020-02-13", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_bulk_create_items_and_retrieve_by_id_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - response = await app_client.get( - f"/collections/{collection_id}/items/{base_item['id']}" - ) - assert response.status_code == 200 - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_patch_collection_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - patch_data = { - "description": "Updated description via PATCH", - } - response = await app_client.patch( - f"/collections/{collection_id}?refresh=true", json=patch_data - ) - assert response.status_code == 200 - assert response.json()["description"] == "Updated description via PATCH" - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_put_collection_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - collection_response = await app_client.get(f"/collections/{collection_id}") - assert collection_response.status_code == 200 - collection_data = collection_response.json() - - collection_data["description"] = "Updated description via PUT" - collection_data["title"] = "Updated title via PUT" - response = await app_client.put( - f"/collections/{collection_id}?refresh=true", json=collection_data - ) - assert response.json()["description"] == "Updated description via PUT" - assert response.json()["title"] == "Updated title via PUT" - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_patch_item_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - patch_data = {"properties": {"description": "Updated description via PATCH"}} - - response = await app_client.patch( - f"/collections/{collection_id}/items/{base_item['id']}", json=patch_data - ) - assert response.status_code == 200 - assert ( - response.json()["properties"]["description"] - == "Updated description via PATCH" - ) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_put_item_for_datetime_index(app_client, load_test_data, txn_client, ctx): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - item_response = await app_client.get( - f"/collections/{collection_id}/items/{base_item['id']}" - ) - assert item_response.status_code == 200 - item_data = item_response.json() - - item_data["properties"]["platform"] = "Updated platform via PUT" - response = await app_client.put( - f"/collections/{collection_id}/items/{base_item['id']}", json=item_data - ) - assert response.json()["properties"]["platform"] == "Updated platform via PUT" - - @pytest.mark.asyncio async def test_global_collection_max_limit_set(app_client, txn_client, load_test_data): """Test with global collection max limit set, expect cap the limit""" diff --git a/stac_fastapi/tests/api/test_api_datetime_filtering.py b/stac_fastapi/tests/api/test_api_datetime_filtering.py new file mode 100644 index 000000000..eb41ec454 --- /dev/null +++ b/stac_fastapi/tests/api/test_api_datetime_filtering.py @@ -0,0 +1,1633 @@ +import os +import uuid +from copy import deepcopy +from unittest.mock import patch + +import pytest + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_in_past_date_change_alias_name_for_datetime_index( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = "2012-02-12T12:30:22Z" + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 201 + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_start_datetime_test-collection_2012-02-12", + "items_end_datetime_test-collection_2020-02-16", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_uses_existing_datetime_index_for_datetime_index( + mock_datetime_env, app_client, ctx, load_test_data, txn_client, monkeypatch +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_start_datetime_test-collection_2020-02-08", + "items_end_datetime_test-collection_2020-02-16", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_with_different_date_same_index_for_datetime_index( + mock_datetime_env, + app_client, + load_test_data, + txn_client, + ctx, +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = "2020-02-11T12:30:22Z" + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_start_datetime_test-collection_2020-02-08", + "items_end_datetime_test-collection_2020-02-16", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_new_index_when_size_limit_exceeded_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = "2020-02-11T12:30:22Z" + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.return_value = 26.0 + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_start_datetime_test-collection_2020-02-08-2020-02-11", + ] + all_aliases = set() + + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + item_2 = deepcopy(item) + item_2["id"] = str(uuid.uuid4()) + item_2["properties"]["start_datetime"] = "2020-02-10T12:30:22Z" + response_2 = await app_client.post( + f"/collections/{item_2['collection']}/items", json=item_2 + ) + assert response_2.status_code == 201 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_fails_without_datetime_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = None + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 400 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_bulk_create_items_with_same_date_range_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + base_item = load_test_data("test_item.json") + items_dict = {} + + for i in range(10): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = f"2020-02-{12 + i}T12:30:22Z" + item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" + item["properties"]["end_datetime"] = f"2020-02-{12 + i}T12:30:22Z" + items_dict[item["id"]] = item + + payload = {"type": "FeatureCollection", "features": list(items_dict.values())} + response = await app_client.post( + f"/collections/{base_item['collection']}/items", json=payload + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_start_datetime_test-collection_2020-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + return all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_bulk_create_items_with_different_date_ranges_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + base_item = load_test_data("test_item.json") + items_dict = {} + + for i in range(3): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = f"2020-02-{12 + i}T12:30:22Z" + item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" + item["properties"]["end_datetime"] = f"2020-02-{12 + i}T12:30:22Z" + items_dict[item["id"]] = item + + for i in range(2): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = f"2010-02-{10 + i}T12:30:22Z" + item["properties"]["datetime"] = f"2010-02-{10 + i}T12:30:22Z" + item["properties"]["end_datetime"] = f"2010-02-{10 + i}T12:30:22Z" + items_dict[item["id"]] = item + + payload = {"type": "FeatureCollection", "features": list(items_dict.values())} + + response = await app_client.post( + f"/collections/{base_item['collection']}/items", json=payload + ) + + assert response.status_code == 201 + indices = await txn_client.database.client.indices.get_alias(index="*") + + expected_aliases = ["items_start_datetime_test-collection_2010-02-10"] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_bulk_create_items_with_size_limit_exceeded_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2019-02", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_start_datetime_test-collection_2010-02-10-2020-02-08", + "items_start_datetime_test-collection_2020-02-09", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_bulk_create_items_with_early_date_in_second_batch_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_start_datetime_test-collection_2008-01-15-2020-02-08", + "items_start_datetime_test-collection_2020-02-09", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_bulk_create_items_and_retrieve_by_id_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + response = await app_client.get( + f"/collections/{collection_id}/items/{base_item['id']}" + ) + assert response.status_code == 200 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_patch_collection_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + patch_data = { + "description": "Updated description via PATCH", + } + response = await app_client.patch( + f"/collections/{collection_id}?refresh=true", json=patch_data + ) + assert response.status_code == 200 + assert response.json()["description"] == "Updated description via PATCH" + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_put_collection_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + collection_response = await app_client.get(f"/collections/{collection_id}") + assert collection_response.status_code == 200 + collection_data = collection_response.json() + + collection_data["description"] = "Updated description via PUT" + collection_data["title"] = "Updated title via PUT" + response = await app_client.put( + f"/collections/{collection_id}?refresh=true", json=collection_data + ) + assert response.json()["description"] == "Updated description via PUT" + assert response.json()["title"] == "Updated title via PUT" + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_patch_item_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + patch_data = {"properties": {"description": "Updated description via PATCH"}} + + response = await app_client.patch( + f"/collections/{collection_id}/items/{base_item['id']}", json=patch_data + ) + assert response.status_code == 200 + assert ( + response.json()["properties"]["description"] + == "Updated description via PATCH" + ) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_put_item_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + item_response = await app_client.get( + f"/collections/{collection_id}/items/{base_item['id']}" + ) + assert item_response.status_code == 200 + item_data = item_response.json() + + item_data["properties"]["platform"] = "Updated platform via PUT" + response = await app_client.put( + f"/collections/{collection_id}/items/{base_item['id']}", json=item_data + ) + assert response.json()["properties"]["platform"] == "Updated platform via PUT" + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_new_item_in_new_collection_for_datetime_index( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + response = await app_client.post("/collections/new-collection/items", json=item) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_new-collection" + ) + expected_aliases = [ + "items_end_datetime_new-collection_2020-02-16", + "items_start_datetime_new-collection_2020-02-08", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_with_invalid_datetime_ordering_should_fail( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + item["properties"]["start_datetime"] = "2024-02-12T12:30:22Z" + item["properties"]["end_datetime"] = "2022-02-12T12:30:22Z" + + await app_client.post("/collections", json=new_collection) + + response = await app_client.post("/collections/new-collection/items", json=item) + assert response.status_code == 400 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_update_item_with_changed_end_datetime( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + updated_item = item.copy() + updated_item["properties"]["end_datetime"] = "2020-02-19T12:30:22Z" + + response = await app_client.put( + f"/collections/new-collection/items/{item['id']}", json=updated_item + ) + + assert response.status_code == 200 + + indices = await txn_client.database.client.indices.get_alias( + index="items_new-collection" + ) + expected_aliases = [ + "items_end_datetime_new-collection_2020-02-19", + "items_start_datetime_new-collection_2020-02-08", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_update_item_with_changed_datetime( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + updated_item = item.copy() + updated_item["properties"]["datetime"] = "2020-02-14T12:30:22Z" + + response = await app_client.put( + f"/collections/new-collection/items/{item['id']}", json=updated_item + ) + + assert response.status_code == 200 + + indices = await txn_client.database.client.indices.get_alias( + index="items_new-collection" + ) + expected_aliases = [ + "items_end_datetime_new-collection_2020-02-16", + "items_start_datetime_new-collection_2020-02-08", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_by_datetime_range_with_stac_query( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2020-02-01T00:00:00Z/2020-02-28T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] > 0 + assert len(result["features"]) > 0 + assert any(feature["id"] == item["id"] for feature in result["features"]) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_by_start_datetime_with_stac_query( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2020-02-08T00:00:00Z/.." + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] > 0 + assert any(feature["id"] == item["id"] for feature in result["features"]) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_not_found_outside_datetime_range( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2021-01-01T00:00:00Z/2021-12-31T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] == 0 + assert len(result["features"]) == 0 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_after_datetime_update_with_stac_query( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + updated_item = item.copy() + updated_item["properties"]["datetime"] = "2020-02-14T12:30:22Z" + + await app_client.put( + f"/collections/new-collection/items/{item['id']}", json=updated_item + ) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2020-02-14T00:00:00Z/2020-02-14T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] > 0 + assert any(feature["id"] == item["id"] for feature in result["features"]) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_by_multiple_collections_with_stac_query( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + collection1 = load_test_data("test_collection.json") + collection1["id"] = "collection-1" + + collection2 = load_test_data("test_collection.json") + collection2["id"] = "collection-2" + + item1 = load_test_data("test_item.json") + item1["collection"] = "collection-1" + item1["id"] = "item-1" + + item2 = load_test_data("test_item.json") + item2["collection"] = "collection-2" + item2["id"] = "item-2" + + await app_client.post("/collections", json=collection1) + await app_client.post("/collections", json=collection2) + await app_client.post("/collections/collection-1/items", json=item1) + await app_client.post("/collections/collection-2/items", json=item2) + + response = await app_client.get( + "/search?collections=collection-1,collection-2&datetime=2020-02-01T00:00:00Z/2020-02-28T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] >= 2 + feature_ids = {feature["id"] for feature in result["features"]} + assert "item-1" in feature_ids + assert "item-2" in feature_ids + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_with_the_same_date_change_alias_name_for_datetime_index( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 201 + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_start_datetime_test-collection_2020-02-08", + "items_end_datetime_test-collection_2020-02-16", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_with_datetime_field_creates_single_alias( + app_client, + ctx, + load_test_data, + txn_client, +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = None + item["properties"]["end_datetime"] = None + item["properties"]["datetime"] = "2024-06-15T12:30:22Z" + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_datetime_test-collection_2020-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) + assert not any("start_datetime" in alias for alias in all_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_alias_created_for_past_date( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = "2012-02-12T12:30:22Z" + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 201 + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_datetime_test-collection_2012-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_reuses_existing_index_for_default_date( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_datetime_test-collection_2020-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_groups_same_year_dates_in_single_index( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = "2022-02-12T12:30:22Z" + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_datetime_test-collection_2020-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_creates_new_index_when_size_limit_exceeded( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = "2024-02-12T12:30:22Z" + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.return_value = 26.0 + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_datetime_test-collection_2020-02-12-2024-02-12", + "items_datetime_test-collection_2024-02-13", + ] + all_aliases = set() + + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + item_2 = deepcopy(item) + item_2["id"] = str(uuid.uuid4()) + item_2["properties"]["datetime"] = "2023-02-12T12:30:22Z" + response_2 = await app_client.post( + f"/collections/{item_2['collection']}/items", json=item_2 + ) + assert response_2.status_code == 201 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_rejects_item_without_datetime_field( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = None + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 400 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_bulk_insert_with_same_date_range( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + base_item = load_test_data("test_item.json") + items_dict = {} + + for i in range(10): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" + items_dict[item["id"]] = item + + payload = {"type": "FeatureCollection", "features": list(items_dict.values())} + response = await app_client.post( + f"/collections/{base_item['collection']}/items", json=payload + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_datetime_test-collection_2020-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + return all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_bulk_insert_with_different_date_ranges( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + base_item = load_test_data("test_item.json") + items_dict = {} + + for i in range(3): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" + items_dict[item["id"]] = item + + for i in range(2): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = f"2010-02-{10 + i}T12:30:22Z" + items_dict[item["id"]] = item + + payload = {"type": "FeatureCollection", "features": list(items_dict.values())} + + response = await app_client.post( + f"/collections/{base_item['collection']}/items", json=payload + ) + + assert response.status_code == 201 + indices = await txn_client.database.client.indices.get_alias(index="*") + + expected_aliases = ["items_datetime_test-collection_2010-02-10"] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_bulk_insert_handles_size_limit_correctly( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2019-02", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_datetime_test-collection_2010-02-10-2020-02-12", + "items_datetime_test-collection_2020-02-13", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_splits_index_when_earlier_date_added_after_limit( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_datetime_test-collection_2008-01-15-2020-02-12", + "items_datetime_test-collection_2020-02-13", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_bulk_insert_allows_item_retrieval( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + response = await app_client.get( + f"/collections/{collection_id}/items/{base_item['id']}" + ) + assert response.status_code == 200 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_collection_patch_operation( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + patch_data = { + "description": "Updated description via PATCH", + } + response = await app_client.patch( + f"/collections/{collection_id}?refresh=true", json=patch_data + ) + assert response.status_code == 200 + assert response.json()["description"] == "Updated description via PATCH" + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_collection_put_operation( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + collection_response = await app_client.get(f"/collections/{collection_id}") + assert collection_response.status_code == 200 + collection_data = collection_response.json() + + collection_data["description"] = "Updated description via PUT" + collection_data["title"] = "Updated title via PUT" + response = await app_client.put( + f"/collections/{collection_id}?refresh=true", json=collection_data + ) + assert response.json()["description"] == "Updated description via PUT" + assert response.json()["title"] == "Updated title via PUT" + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_item_patch_operation( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + patch_data = {"properties": {"description": "Updated description via PATCH"}} + + response = await app_client.patch( + f"/collections/{collection_id}/items/{base_item['id']}", json=patch_data + ) + assert response.status_code == 200 + assert ( + response.json()["properties"]["description"] + == "Updated description via PATCH" + ) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_item_put_operation( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + item_response = await app_client.get( + f"/collections/{collection_id}/items/{base_item['id']}" + ) + assert item_response.status_code == 200 + item_data = item_response.json() + + item_data["properties"]["platform"] = "Updated platform via PUT" + response = await app_client.put( + f"/collections/{collection_id}/items/{base_item['id']}", json=item_data + ) + assert response.json()["properties"]["platform"] == "Updated platform via PUT" diff --git a/stac_fastapi/tests/conftest.py b/stac_fastapi/tests/conftest.py index b461e7221..d6a48c41d 100644 --- a/stac_fastapi/tests/conftest.py +++ b/stac_fastapi/tests/conftest.py @@ -396,3 +396,13 @@ def build_test_app(): # Create and return the app api = StacApi(**test_config) return api.app + + +@pytest.fixture +def mock_datetime_env(txn_client, monkeypatch): + """Set USE_DATETIME environment variable to False for testing.""" + monkeypatch.setenv("USE_DATETIME", "false") + if hasattr(txn_client.database.async_index_selector, "cache_manager"): + txn_client.database.async_index_selector.cache_manager.clear_cache() + yield + monkeypatch.setenv("USE_DATETIME", "true") diff --git a/stac_fastapi/tests/database/test_database.py b/stac_fastapi/tests/database/test_database.py index 67897c153..e02d2165c 100644 --- a/stac_fastapi/tests/database/test_database.py +++ b/stac_fastapi/tests/database/test_database.py @@ -4,7 +4,10 @@ import pytest from stac_pydantic import api -from stac_fastapi.sfeos_helpers.database import index_alias_by_collection_id +from stac_fastapi.sfeos_helpers.database import ( + filter_indexes_by_datetime, + index_alias_by_collection_id, +) from stac_fastapi.sfeos_helpers.mappings import ( COLLECTIONS_INDEX, ES_COLLECTIONS_MAPPINGS, @@ -46,3 +49,320 @@ async def test_index_mapping_items(txn_client, load_test_data): actual_mappings["dynamic_templates"] == ES_ITEMS_MAPPINGS["dynamic_templates"] ) await txn_client.delete_collection(collection["id"]) + + +@pytest.mark.datetime_filtering +def test_filter_datetime_field_outside_range(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": "2021-01-01T00:00:00Z", "lte": "2021-12-31T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_start_datetime_field_with_gte(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": "2020-02-01T00:00:00Z", "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 1 + + +@pytest.mark.datetime_filtering +def test_filter_end_datetime_field_with_lte(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": "2020-02-28T23:59:59Z"}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 1 + + +@pytest.mark.datetime_filtering +def test_filter_all_criteria_matching(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": "2020-02-01T00:00:00Z", "lte": "2020-02-28T23:59:59Z"}, + "start_datetime": {"gte": "2020-02-01T00:00:00Z", "lte": None}, + "end_datetime": {"gte": None, "lte": "2020-02-28T23:59:59Z"}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 1 + + +@pytest.mark.datetime_filtering +def test_filter_datetime_field_fails_gte(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": "2020-02-15T00:00:00Z", "lte": "2020-02-28T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_datetime_field_fails_lte(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": "2020-01-01T00:00:00Z", "lte": "2020-02-10T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_start_datetime_range_format(mock_datetime_env): + collection_indexes = [ + ( + { + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08-2022-04-05", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": "2020-02-01T00:00:00Z", "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 1 + assert result[0] == "items_start_datetime_new-collection_2020-02-08-2022-04-05" + + +@pytest.mark.datetime_filtering +def test_filter_start_datetime_range_fails_gte(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08-2022-04-05", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": "2022-05-01T00:00:00Z", "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_multiple_indexes_mixed_results(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + }, + ), + ( + { + "datetime": "items_datetime_new-collection_2020-02-15", + }, + ), + ( + { + "datetime": "items_datetime_new-collection_2021-03-15", + }, + ), + ] + datetime_search = { + "datetime": {"gte": "2020-02-01T00:00:00Z", "lte": "2020-02-28T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, True) + + assert len(result) == 2 + assert "items_datetime_new-collection_2020-02-12" in result + assert "items_datetime_new-collection_2020-02-15" in result + + +@pytest.mark.datetime_filtering +def test_filter_empty_collection(): + collection_indexes = [] + datetime_search = { + "datetime": {"gte": "2020-02-01T00:00:00Z", "lte": "2020-02-28T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_all_criteria_none(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 1 + + +@pytest.mark.datetime_filtering +def test_filter_end_datetime_outside_range(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": "2020-02-10T23:59:59Z"}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_complex_mixed_criteria(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ), + ( + { + "datetime": "items_datetime_new-collection_2020-02-14", + "end_datetime": "items_end_datetime_new-collection_2020-02-18", + "start_datetime": "items_start_datetime_new-collection_2020-02-10", + }, + ), + ] + datetime_search = { + "datetime": {"gte": "2020-02-12T00:00:00Z", "lte": "2020-02-28T23:59:59Z"}, + "start_datetime": {"gte": "2020-02-01T00:00:00Z", "lte": None}, + "end_datetime": {"gte": None, "lte": "2020-02-20T23:59:59Z"}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 2 + + +@pytest.mark.datetime_filtering +def test_filter_with_single_date_range(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-12", + "start_datetime": "items_start_datetime_new-collection_2020-02-12", + }, + ) + ] + datetime_search = { + "datetime": {"gte": "2020-02-12T00:00:00Z", "lte": "2020-02-12T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) + + assert len(result) == 1