From 7b7f31e917a7f4725bb059a32df4e5db3bca2c1f Mon Sep 17 00:00:00 2001 From: Grzegorz Pustulka Date: Fri, 21 Nov 2025 15:11:40 +0100 Subject: [PATCH 1/8] before tests --- .../elasticsearch/database_logic.py | 25 ++- .../stac_fastapi/opensearch/database_logic.py | 28 +++- .../sfeos_helpers/database/datetime.py | 7 +- .../sfeos_helpers/database/index.py | 101 ++++++----- .../search_engine/index_operations.py | 64 ++++--- .../sfeos_helpers/search_engine/inserters.py | 68 +++++--- .../sfeos_helpers/search_engine/managers.py | 158 ++++++++++++++---- .../search_engine/selection/cache_manager.py | 116 ++++++++++--- .../search_engine/selection/selectors.py | 17 +- 9 files changed, 408 insertions(+), 176 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 35b6ae31a..81c575bd8 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -436,7 +436,7 @@ def apply_collections_filter(search: Search, collection_ids: List[str]): @staticmethod def apply_datetime_filter( search: Search, datetime: Optional[str] - ) -> Tuple[Search, Dict[str, Optional[str]]]: + ) -> Tuple[Search, Dict[str, Dict[str, Optional[str]]]]: """Apply a filter to search on datetime, start_datetime, and end_datetime fields. Args: @@ -446,15 +446,30 @@ def apply_datetime_filter( Returns: The filtered search object. """ + datetime_search = return_date(datetime) + # USE_DATETIME env var # True: Search by datetime, if null search by start/end datetime # False: Always search only by start/end datetime USE_DATETIME = get_bool_env("USE_DATETIME", default=True) - datetime_search = return_date(datetime) + result_metadata = { + "datetime": { + "gte": datetime_search.get("gte") if USE_DATETIME else None, + "lte": datetime_search.get("lte") if USE_DATETIME else None, + }, + "start_datetime": { + "gte": datetime_search.get("gte") if not USE_DATETIME else None, + "lte": None, + }, + "end_datetime": { + "gte": None, + "lte": datetime_search.get("lte") if not USE_DATETIME else None, + }, + } if not datetime_search: - return search, datetime_search + return search, result_metadata if USE_DATETIME: if "eq" in datetime_search: @@ -533,7 +548,7 @@ def apply_datetime_filter( return ( search.query(Q("bool", should=should, minimum_should_match=1)), - datetime_search, + result_metadata, ) else: if "eq" in datetime_search: @@ -568,7 +583,7 @@ def apply_datetime_filter( ), ], ) - return search.query(filter_query), datetime_search + return search.query(filter_query), result_metadata @staticmethod def apply_bbox_filter(search: Search, bbox: List): diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 05aac1763..993876475 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -459,7 +459,7 @@ def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str] @staticmethod def apply_datetime_filter( search: Search, datetime: Optional[str] - ) -> Tuple[Search, Dict[str, Optional[str]]]: + ) -> Tuple[Search, Dict[str, Dict[str, Optional[str]]]]: """Apply a filter to search on datetime, start_datetime, and end_datetime fields. Args: @@ -471,14 +471,30 @@ def apply_datetime_filter( """ datetime_search = return_date(datetime) - if not datetime_search: - return search, datetime_search - # USE_DATETIME env var # True: Search by datetime, if null search by start/end datetime # False: Always search only by start/end datetime USE_DATETIME = get_bool_env("USE_DATETIME", default=True) + result_metadata = { + "datetime": { + "gte": datetime_search.get("gte") if USE_DATETIME else None, + "lte": datetime_search.get("lte") if USE_DATETIME else None, + }, + "start_datetime": { + "gte": datetime_search.get("gte") if not USE_DATETIME else None, + "lte": None, + }, + "end_datetime": { + "gte": None, + "lte": datetime_search.get("lte") if not USE_DATETIME else None, + }, + } + + if not datetime_search: + return search, result_metadata + + if USE_DATETIME: if "eq" in datetime_search: # For exact matches, include: @@ -556,7 +572,7 @@ def apply_datetime_filter( return ( search.query(Q("bool", should=should, minimum_should_match=1)), - datetime_search, + result_metadata, ) else: if "eq" in datetime_search: @@ -591,7 +607,7 @@ def apply_datetime_filter( ), ], ) - return search.query(filter_query), datetime_search + return search.query(filter_query), result_metadata @staticmethod def apply_bbox_filter(search: Search, bbox: List): diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py index 11360fc1e..a8b93edb1 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py @@ -138,15 +138,18 @@ def return_date( return result -def extract_date(date_str: str) -> date: +def extract_date(date_str: str | None) -> date | None: """Extract date from ISO format string. Args: date_str: ISO format date string Returns: - A date object extracted from the input string. + A date object extracted from the input string or None. """ + if not date_str: + return None + date_str = date_str.replace("Z", "+00:00") return datetime_type.fromisoformat(date_str).date() diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py index c36a36fa0..ba360a1e7 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py @@ -6,7 +6,7 @@ import re from datetime import datetime from functools import lru_cache -from typing import Any, List, Optional +from typing import Any, List, Optional, Tuple, Dict from dateutil.parser import parse # type: ignore[import] @@ -71,54 +71,69 @@ def indices(collection_ids: Optional[List[str]]) -> str: def filter_indexes_by_datetime( - indexes: List[str], gte: Optional[str], lte: Optional[str] + collection_indexes: List[Tuple[Dict[str, str], ...]], + datetime_search: Dict[str, Dict[str, str | None]] ) -> List[str]: - """Filter indexes based on datetime range extracted from index names. + def extract_date_from_alias(alias: str) -> tuple[datetime, datetime] | None: + date_pattern = re.compile(r'\d{4}-\d{2}-\d{2}') + try: + dates = date_pattern.findall(alias) + + if not dates: + return None + + if len(dates) >= 2: + return datetime.strptime(dates[-2], '%Y-%m-%d'), datetime.strptime(dates[-1], '%Y-%m-%d') + else: + date = datetime.strptime(dates[-1], '%Y-%m-%d') + return date, date + except (ValueError, IndexError): + return None + + def parse_search_date(date_str: str | None) -> Optional[datetime.date]: + if not date_str: + return None + return datetime.fromisoformat(date_str.replace('Z', '+00:00')).date() + + def check_criteria(value_begin: datetime.date, value_end: datetime.date, criteria: Dict) -> bool: + gte = parse_search_date(criteria.get('gte')) + lte = parse_search_date(criteria.get('lte')) + + if gte and value_begin.date() < gte: + return False + if lte and value_end.date() > lte: + return False + return True - Args: - indexes: List of index names containing dates - gte: Greater than or equal date filter (ISO format, optional 'Z' suffix) - lte: Less than or equal date filter (ISO format, optional 'Z' suffix) + filtered_indexes = [] - Returns: - List of filtered index names - """ + for index_tuple in collection_indexes: + if not index_tuple: + continue - def parse_datetime(dt_str: str) -> datetime: - """Parse datetime string, handling both with and without 'Z' suffix.""" - return parse(dt_str).replace(tzinfo=None) - - def extract_date_range_from_index(index_name: str) -> tuple: - """Extract start and end dates from index name.""" - date_pattern = r"(\d{4}-\d{2}-\d{2})" - dates = re.findall(date_pattern, index_name) - - if len(dates) == 1: - start_date = datetime.strptime(dates[0], "%Y-%m-%d") - max_date = datetime.max.replace(microsecond=0) - return start_date, max_date - else: - start_date = datetime.strptime(dates[0], "%Y-%m-%d") - end_date = datetime.strptime(dates[1], "%Y-%m-%d") - return start_date, end_date - - def is_index_in_range( - start_date: datetime, end_date: datetime, gte_dt: datetime, lte_dt: datetime - ) -> bool: - """Check if index date range overlaps with filter range.""" - return not ( - end_date.date() < gte_dt.date() or start_date.date() > lte_dt.date() - ) - - gte_dt = parse_datetime(gte) if gte else datetime.min.replace(microsecond=0) - lte_dt = parse_datetime(lte) if lte else datetime.max.replace(microsecond=0) + index_dict = index_tuple[0] + start_datetime_alias = index_dict.get('start_datetime') + end_datetime_alias = index_dict.get('end_datetime') + datetime_alias = index_dict.get('datetime') - filtered_indexes = [] + if not start_datetime_alias: + continue + + start_range = extract_date_from_alias(start_datetime_alias) + end_date = extract_date_from_alias(end_datetime_alias) + datetime_date = extract_date_from_alias(datetime_alias) + + if not start_range or not end_date or not datetime_date: + continue + + if not check_criteria(start_range[0], start_range[1], datetime_search.get('start_datetime', {})): + continue + if not check_criteria(end_date[0], end_date[1], datetime_search.get('end_datetime', {})): + continue + if not check_criteria(datetime_date[0], datetime_date[1], datetime_search.get('datetime', {})): + continue - for index in indexes: - start_date, end_date = extract_date_range_from_index(index) - if is_index_in_range(start_date, end_date, gte_dt, lte_dt): - filtered_indexes.append(index) + filtered_indexes.append(start_datetime_alias) return filtered_indexes diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py index 42028a7a3..0b26f1657 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py @@ -1,7 +1,7 @@ """Search engine adapters for different implementations.""" import uuid -from typing import Any, Dict +from typing import Any, Dict, Literal, List from stac_fastapi.sfeos_helpers.database import ( index_alias_by_collection_id, @@ -39,26 +39,38 @@ async def create_simple_index(self, client: Any, collection_id: str) -> str: return index_name async def create_datetime_index( - self, client: Any, collection_id: str, start_date: str + self, client: Any, collection_id: str, start_datetime: str, datetime: str | None, end_datetime: str ) -> str: """Create a datetime-based index for the given collection. Args: client: Search engine client instance. collection_id (str): Collection identifier. - start_date (str): Start date for the alias. + start_datetime (str): Start datetime for the index alias. + datetime (str | None): Datetime for the datetime alias (can be None). + end_datetime (str): End datetime for the index alias. Returns: - str: Created index alias name. + str: Created start_datetime alias name. """ index_name = self.create_index_name(collection_id) - alias_name = self.create_alias_name(collection_id, start_date) + alias_start_date = self.create_alias_name(collection_id, "start_datetime", start_datetime) + alias_date = self.create_alias_name(collection_id, "datetime", datetime) + alias_end_date = self.create_alias_name(collection_id, "end_datetime", end_datetime) collection_alias = index_alias_by_collection_id(collection_id) + + aliases = { + collection_alias: {}, + alias_start_date: {}, + alias_date: {}, + alias_end_date: {}, + } + await client.indices.create( index=index_name, - body=self._create_index_body({collection_alias: {}, alias_name: {}}), + body=self._create_index_body(aliases), ) - return alias_name + return alias_start_date @staticmethod async def update_index_alias(client: Any, end_date: str, old_alias: str) -> str: @@ -84,23 +96,28 @@ async def update_index_alias(client: Any, end_date: str, old_alias: str) -> str: return new_alias @staticmethod - async def change_alias_name(client: Any, old_alias: str, new_alias: str) -> None: - """Change alias name from old to new. + async def change_alias_name(client: Any, old_start_datetime_alias: str, aliases_to_change: List[str], aliases_to_create: List[str]) -> None: + """Change alias names by removing old aliases and adding new ones. Args: client: Search engine client instance. - old_alias (str): Current alias name. - new_alias (str): New alias name. + old_start_datetime_alias (str): Current start_datetime alias name to identify the index. + aliases_to_change (List[str]): List of old alias names to remove. + aliases_to_create (List[str]): List of new alias names to add. Returns: None """ - aliases_info = await client.indices.get_alias(name=old_alias) + aliases_info = await client.indices.get_alias(name=old_start_datetime_alias) + index_name = list(aliases_info.keys())[0] actions = [] - for index_name in aliases_info.keys(): + for old_alias in aliases_to_change: actions.append({"remove": {"index": index_name, "alias": old_alias}}) + + for new_alias in aliases_to_create: actions.append({"add": {"index": index_name, "alias": new_alias}}) + await client.indices.update_aliases(body={"actions": actions}) @staticmethod @@ -117,18 +134,21 @@ def create_index_name(collection_id: str) -> str: return f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{uuid.uuid4()}" @staticmethod - def create_alias_name(collection_id: str, start_date: str) -> str: - """Create index name from collection ID and uuid4. + def create_alias_name( + collection_id: str, name: Literal["start_datetime", "datetime", "end_datetime"], start_date: str + ) -> str: + """Create alias name from collection ID and date. Args: collection_id (str): Collection identifier. - start_date (str): Start date for the alias. + name (Literal["start_datetime", "datetime", "end_datetime"]): Type of alias to create. + start_date (str): Date value for the alias. Returns: - str: Alias name with initial date. + str: Formatted alias name with prefix, type, collection ID, and date. """ cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE) - return f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{start_date}" + return f"{ITEMS_INDEX_PREFIX}{name}_{cleaned.lower()}_{start_date}" @staticmethod def _create_index_body(aliases: Dict[str, Dict]) -> Dict[str, Any]: @@ -148,19 +168,19 @@ def _create_index_body(aliases: Dict[str, Dict]) -> Dict[str, Any]: @staticmethod async def find_latest_item_in_index(client: Any, index_name: str) -> dict[str, Any]: - """Find the latest item date in the specified index. + """Find the latest item in the specified index. Args: client: Search engine client instance. index_name (str): Name of the index to query. Returns: - datetime: Date of the latest item in the index. + dict[str, Any]: Latest item document from the index with metadata. """ query = { "size": 1, - "sort": [{"properties.datetime": {"order": "desc"}}], - "_source": ["properties.datetime"], + "sort": [{"properties.start_datetime": {"order": "desc"}}], + "_source": ["properties.start_datetime", "properties.datetime", "properties.end_datetime"], } response = await client.search(index=index_name, body=query) diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py index 06e9c7298..c89509ec3 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py @@ -1,6 +1,5 @@ """Async index insertion strategies.""" import logging -from datetime import timedelta from typing import Any, Dict, List from fastapi import HTTPException, status @@ -14,7 +13,7 @@ from .base import BaseIndexInserter from .index_operations import IndexOperations -from .managers import DatetimeIndexManager +from .managers import DatetimeIndexManager, ProductDatetimes from .selection import DatetimeBasedIndexSelector logger = logging.getLogger(__name__) @@ -89,7 +88,7 @@ async def prepare_bulk_actions( logger.error(msg) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=msg) - items.sort(key=lambda item: item["properties"]["datetime"]) + items.sort(key=lambda item: item["properties"]["start_datetime"]) index_selector = DatetimeBasedIndexSelector(self.client) await self._ensure_indexes_exist(index_selector, collection_id, items) @@ -130,42 +129,56 @@ async def _get_target_index_internal( Returns: str: Target index name. """ - product_datetime = self.datetime_manager.validate_product_datetime(product) - datetime_range = {"gte": product_datetime, "lte": product_datetime} + product_datetimes = self.datetime_manager.validate_product_datetimes(product) + + datetime_ranges = { + "start_datetime": {"gte": product_datetimes.start_datetime, "lte": product_datetimes.start_datetime}, + "end_datetime": {"gte": product_datetimes.end_datetime, "lte": product_datetimes.end_datetime}, + "datetime": {"gte": product_datetimes.datetime, "lte": product_datetimes.datetime} + } + target_index = await index_selector.select_indexes( - [collection_id], datetime_range + [collection_id], datetime_ranges ) all_indexes = await index_selector.get_collection_indexes(collection_id) if not all_indexes: target_index = await self.datetime_manager.handle_new_collection( - collection_id, product_datetime + collection_id, product_datetimes ) await index_selector.refresh_cache() return target_index - all_indexes.sort() - start_date = extract_date(product_datetime) - end_date = extract_first_date_from_index(all_indexes[0]) + all_indexes = sorted(all_indexes, key=lambda x: x[0]['start_datetime']) + start_date = extract_date(product_datetimes.start_datetime) + end_date = extract_first_date_from_index(all_indexes[0][0]["start_datetime"]) if start_date < end_date: alias = await self.datetime_manager.handle_early_date( - collection_id, start_date, end_date + collection_id, product_datetimes, all_indexes[0][0] ) await index_selector.refresh_cache() - return alias if target_index != all_indexes[-1]: - return target_index + for item in all_indexes: + aliases_dict = item[0] + if target_index in aliases_dict.values(): + await self.datetime_manager.handle_early_date( + collection_id, product_datetimes, aliases_dict + ) + return target_index if check_size and await self.datetime_manager.size_manager.is_index_oversized( target_index ): - target_index = await self.datetime_manager.handle_oversized_index( - collection_id, target_index, product_datetime - ) - await index_selector.refresh_cache() + for item in all_indexes: + aliases_dict = item[0] + if target_index in aliases_dict.values(): + target_index = await self.datetime_manager.handle_oversized_index( + collection_id, product_datetimes, aliases_dict + ) + await index_selector.refresh_cache() return target_index @@ -186,7 +199,9 @@ async def _ensure_indexes_exist( await self.index_operations.create_datetime_index( self.client, collection_id, + extract_date(first_item["properties"]["start_datetime"]), extract_date(first_item["properties"]["datetime"]), + extract_date(first_item["properties"]["end_datetime"]), ) await index_selector.refresh_cache() @@ -212,8 +227,9 @@ async def _check_and_handle_oversized_index( ) all_indexes = await index_selector.get_collection_indexes(collection_id) - all_indexes.sort() - latest_index = all_indexes[-1] + all_indexes = sorted(all_indexes, key=lambda x: x[0]['start_datetime']) + + latest_index = all_indexes[-1][0] if first_item_index != latest_index: return None @@ -226,15 +242,13 @@ async def _check_and_handle_oversized_index( latest_item = await self.index_operations.find_latest_item_in_index( self.client, latest_index ) - product_datetime = latest_item["_source"]["properties"]["datetime"] - end_date = extract_date(product_datetime) - await self.index_operations.update_index_alias( - self.client, str(end_date), latest_index - ) - next_day_start = end_date + timedelta(days=1) - await self.index_operations.create_datetime_index( - self.client, collection_id, str(next_day_start) + product_datetimes = ProductDatetimes( + start_datetime=latest_item["_source"]["properties"]["start_datetime"], + datetime=latest_item["_source"]["properties"]["datetime"], + end_datetime=latest_item["_source"]["properties"]["end_datetime"], ) + + await self.datetime_manager.handle_oversized_index(collection_id, product_datetimes, latest_index) await index_selector.refresh_cache() diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py index 1194e6345..d51d3fbf9 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py @@ -2,8 +2,8 @@ import logging import os -from datetime import datetime, timedelta -from typing import Any, Dict +from datetime import timedelta +from typing import Any, Dict, NamedTuple from fastapi import HTTPException, status @@ -17,6 +17,12 @@ logger = logging.getLogger(__name__) +class ProductDatetimes(NamedTuple): + start_datetime: str + datetime: str | None + end_datetime: str + + class IndexSizeManager: """Manages index size limits and operations.""" @@ -108,41 +114,63 @@ def __init__(self, client: Any, index_operations: IndexOperations): self.size_manager = IndexSizeManager(client) @staticmethod - def validate_product_datetime(product: Dict[str, Any]) -> str: - """Validate and extract datetime from product. + def validate_product_datetimes(product: Dict[str, Any]) -> ProductDatetimes: + """Validate and extract datetime fields from product. Args: product (Dict[str, Any]): Product data containing datetime information. Returns: - str: Validated product datetime. + ProductDatetimes: Named tuple containing: + - start_datetime (str): Start datetime value + - datetime (str | None): datetime value + - end_datetime (str): End datetime value Raises: - HTTPException: If product datetime is missing or invalid. + HTTPException: If product start_datetime is missing or invalid. """ - product_datetime = product["properties"]["datetime"] - if not product_datetime: + + properties = product.get("properties", {}) + start_datetime_value = properties.get("start_datetime") + datetime_value = properties.get("datetime") + end_datetime_value = properties.get("end_datetime") + + if not start_datetime_value: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Product 'start_datetime' is required for indexing", + ) + + if not datetime_value and not end_datetime_value: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail="Product datetime is required for indexing", + detail="'end_datetime' is required when 'datetime' is not provided." ) - return product_datetime + + return ProductDatetimes( + start_datetime=start_datetime_value, + datetime=datetime_value, + end_datetime=end_datetime_value, + ) async def handle_new_collection( - self, collection_id: str, product_datetime: str + self, collection_id: str, product_datetimes: ProductDatetimes ) -> str: """Handle index creation for new collection asynchronously. Args: collection_id (str): Collection identifier. - product_datetime (str): Product datetime for index naming. - + product_datetimes (ProductDatetimes): Object containing start_datetime, datetime, and end_datetime. Returns: - str: Created index name. + str: Created start_datetime alias name. """ + start_datetime = extract_date(product_datetimes.start_datetime) + datetime = extract_date(product_datetimes.datetime) + end_datetime = extract_date(product_datetimes.end_datetime) + target_index = await self.index_operations.create_datetime_index( - self.client, collection_id, extract_date(product_datetime) + self.client, collection_id, start_datetime, datetime, end_datetime ) logger.info( f"Successfully created index '{target_index}' for collection '{collection_id}'" @@ -150,49 +178,105 @@ async def handle_new_collection( return target_index async def handle_early_date( - self, collection_id: str, start_date: datetime, end_date: datetime + self, collection_id: str, product_datetimes: ProductDatetimes, old_aliases: Dict[str, str] ) -> str: """Handle product with date earlier than existing indexes asynchronously. Args: collection_id (str): Collection identifier. - start_date (datetime): Start date for the new index. - end_date (datetime): End date for alias update. + product_datetimes (ProductDatetimes): Object containing start_datetime, datetime, and end_datetime. + old_aliases (Dict[str, str]): Dictionary mapping alias types to their current names. Returns: - str: Updated alias name. + str: Updated start_datetime alias name. """ - old_alias = self.index_operations.create_alias_name( - collection_id, str(end_date) - ) - new_alias = self.index_operations.create_alias_name( - collection_id, str(start_date) + start_dt = extract_date(product_datetimes.start_datetime) + dt = extract_date(product_datetimes.datetime) + end_dt = extract_date(product_datetimes.end_datetime) + + old_start_datetime_alias = extract_first_date_from_index(old_aliases["start_datetime"]) + old_datetime_alias = extract_first_date_from_index(old_aliases["datetime"]) + old_end_datetime_alias = extract_first_date_from_index(old_aliases["end_datetime"]) + + new_start_datetime_alias = self.index_operations.create_alias_name( + collection_id, "start_datetime", str(start_dt) ) - await self.index_operations.change_alias_name(self.client, old_alias, new_alias) - return new_alias + + aliases_to_change = [] + aliases_to_create = [] + + if start_dt > old_start_datetime_alias: + aliases_to_create.append(new_start_datetime_alias) + aliases_to_change.append(old_aliases["start_datetime"]) + + + if dt > old_datetime_alias: + new_datetime_alias = self.index_operations.create_alias_name( + collection_id, "datetime", str(dt) + ) + aliases_to_create.append(new_datetime_alias) + aliases_to_change.append(old_aliases["datetime"]) + + if end_dt > old_end_datetime_alias: + new_end_datetime_alias = self.index_operations.create_alias_name( + collection_id, "end_datetime", str(end_dt) + ) + aliases_to_create.append(new_end_datetime_alias) + aliases_to_change.append(old_aliases["end_datetime"]) + + if aliases_to_change: + await self.index_operations.change_alias_name( + self.client, old_aliases["start_datetime"], aliases_to_change, aliases_to_create + ) + return new_start_datetime_alias async def handle_oversized_index( - self, collection_id: str, target_index: str, product_datetime: str + self, collection_id: str, product_datetimes: ProductDatetimes, old_aliases: Dict[str, str] ) -> str: """Handle index that exceeds size limit asynchronously. Args: collection_id (str): Collection identifier. - target_index (str): Current target index name. - product_datetime (str): Product datetime for new index. + product_datetimes (ProductDatetimes): Object containing start_datetime, datetime, and end_datetime. + old_aliases (Dict[str, str]): Dictionary mapping alias types to their current names. Returns: - str: New or updated index name. + str: Updated or newly created start_datetime alias name. """ - end_date = extract_date(product_datetime) - latest_index_start = extract_first_date_from_index(target_index) + target_index = old_aliases["start_datetime"] + start_dt = extract_date(product_datetimes.start_datetime) + dt = extract_date(product_datetimes.datetime) + end_dt = extract_date(product_datetimes.end_datetime) + + old_start_datetime_alias = extract_first_date_from_index(target_index) + old_datetime_alias = extract_first_date_from_index(old_aliases["datetime"]) + old_end_datetime_alias = extract_first_date_from_index(old_aliases["end_datetime"]) + + if start_dt != old_start_datetime_alias: + aliases_to_change = [] + aliases_to_create = [] + + new_start_datetime_alias = f"{target_index}-{str(start_dt)}" + aliases_to_create.append(new_start_datetime_alias) + aliases_to_change.append(target_index) + + if dt > old_datetime_alias: + new_datetime_alias = self.index_operations.create_alias_name( + collection_id, "datetime", str(dt) + ) + aliases_to_create.append(new_datetime_alias) + aliases_to_change.append(old_aliases["datetime"]) - if end_date != latest_index_start: - await self.index_operations.update_index_alias( - self.client, str(end_date), target_index - ) + if end_dt > old_end_datetime_alias: + new_end_datetime_alias = self.index_operations.create_alias_name( + collection_id, "end_datetime", str(end_dt) + ) + aliases_to_create.append(new_end_datetime_alias) + aliases_to_change.append(old_aliases["end_datetime"]) + + await self.index_operations.change_alias_name(self.client, target_index, aliases_to_change, aliases_to_create) target_index = await self.index_operations.create_datetime_index( - self.client, collection_id, str(end_date + timedelta(days=1)) + self.client, collection_id, str(start_dt + timedelta(days=1)), str(dt), str(end_dt) ) return target_index diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py index 3b65244d4..40d4b08fc 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py @@ -1,9 +1,9 @@ """Cache management for index selection strategies.""" +import copy import threading import time -from collections import defaultdict -from typing import Any, Dict, List, Optional +from typing import Dict, List, Optional, Tuple, Any from stac_fastapi.sfeos_helpers.database import index_alias_by_collection_id from stac_fastapi.sfeos_helpers.mappings import ITEMS_INDEX_PREFIX @@ -18,7 +18,7 @@ def __init__(self, cache_ttl_seconds: int = 3600): Args: cache_ttl_seconds (int): Time-to-live for cache entries in seconds. """ - self._cache: Optional[Dict[str, List[str]]] = None + self._cache: Optional[Dict[str, List[Tuple[Dict[str, str]]]]] = None self._timestamp: float = 0 self._ttl = cache_ttl_seconds self._lock = threading.Lock() @@ -32,30 +32,32 @@ def is_expired(self) -> bool: """ return time.time() - self._timestamp > self._ttl - def get_cache(self) -> Optional[Dict[str, List[str]]]: + def get_cache(self) -> Optional[Dict[str, List[Tuple[Dict[str, str]]]]]: """Get the current cache if not expired. Returns: - Optional[Dict[str, List[str]]]: Cache data if valid, None if expired. + Optional[Dict[str, List[Tuple[Dict[str, str]]]]]: Cache data if valid, None if expired. """ with self._lock: if self.is_expired: return None - return {k: v.copy() for k, v in self._cache.items()} + return copy.deepcopy(self._cache) if self._cache else None - def set_cache(self, data: Dict[str, List[str]]) -> None: + def set_cache(self, data: Dict[str, List[Tuple[Dict[str, str]]]]) -> None: """Set cache data and update timestamp. Args: - data (Dict[str, List[str]]): Cache data to store. + data (Dict[str, List[Tuple[Dict[str, str]]]]): Cache data to store. """ - self._cache = data - self._timestamp = time.time() + with self._lock: + self._cache = data + self._timestamp = time.time() def clear_cache(self) -> None: """Clear the cache and reset timestamp.""" - self._cache = None - self._timestamp = 0 + with self._lock: + self._cache = None + self._timestamp = 0 class IndexAliasLoader: @@ -71,15 +73,27 @@ def __init__(self, client: Any, cache_manager: IndexCacheManager): self.client = client self.cache_manager = cache_manager - async def load_aliases(self) -> Dict[str, List[str]]: + async def load_aliases(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: """Load index aliases from search engine. Returns: - Dict[str, List[str]]: Mapping of base aliases to item aliases. + Dict[str, List[Tuple[Dict[str, str]]]]: Mapping of main collection aliases to their data. + + Example: + { + 'items_sentinel-2-l1c': [ + ({ + 'datetime': 'items_datetime_sentinel-2-l1c_2025-09-27', + 'start_datetime': 'items_start_datetime_sentinel-2-l1c_2025-09-27', + 'end_datetime': 'items_end_datetime_sentinel-2-l1c_2025-09-28' + },) + ] + } """ response = await self.client.indices.get_alias(index=f"{ITEMS_INDEX_PREFIX}*") - result = defaultdict(list) - for index_info in response.values(): + result = {} + + for index_name, index_info in response.items(): aliases = index_info.get("aliases", {}) items_aliases = sorted( [ @@ -90,38 +104,90 @@ async def load_aliases(self) -> Dict[str, List[str]]: ) if items_aliases: - result[items_aliases[0]].extend(items_aliases[1:]) + main_alias = self._find_main_alias(items_aliases) + aliases_dict = self._organize_aliases(items_aliases, main_alias) + + if aliases_dict: + if main_alias not in result: + result[main_alias] = [] + + result[main_alias].append((aliases_dict,)) self.cache_manager.set_cache(result) return result - async def get_aliases(self) -> Dict[str, List[str]]: + @staticmethod + def _find_main_alias(aliases: List[str]) -> str: + """Find the main collection alias (without temporal suffixes). + + Args: + aliases (List[str]): List of all aliases for an index. + + Returns: + str: The main collection alias. + """ + temporal_keywords = ['datetime', 'start_datetime', 'end_datetime'] + + for alias in aliases: + if not any(keyword in alias for keyword in temporal_keywords): + return alias + + return aliases[0] + + @staticmethod + def _organize_aliases(aliases: List[str], main_alias: str) -> Dict[str, str]: + """Organize temporal aliases into a dictionary with type as key. + + Args: + aliases (List[str]): All aliases for the index. + main_alias (str): The main collection alias. + + Returns: + Dict[str, str]: Dictionary with datetime types as keys and alias names as values. + """ + aliases_dict = {} + + for alias in aliases: + if alias == main_alias: + continue + + if 'start_datetime' in alias: + aliases_dict['start_datetime'] = alias + elif 'end_datetime' in alias: + aliases_dict['end_datetime'] = alias + elif 'datetime' in alias: + aliases_dict['datetime'] = alias + + return aliases_dict + + async def get_aliases(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: """Get aliases from cache or load if expired. Returns: - Dict[str, List[str]]: Alias mapping data. + Dict[str, List[Tuple[Dict[str, str]]]]: Alias mapping data. """ cached = self.cache_manager.get_cache() if cached is not None: return cached return await self.load_aliases() - async def refresh_aliases(self) -> Dict[str, List[str]]: + async def refresh_aliases(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: """Force refresh aliases from search engine. Returns: - Dict[str, List[str]]: Fresh alias mapping data. + Dict[str, List[Tuple[Dict[str, str]]]]: Fresh alias mapping data. """ return await self.load_aliases() - async def get_collection_indexes(self, collection_id: str) -> List[str]: - """Get all index aliases for a specific collection. + async def get_collection_indexes(self, collection_id: str) -> List[Tuple[Dict[str, str]]]: + """Get index information for a specific collection. Args: collection_id (str): Collection identifier. Returns: - List[str]: List of index aliases for the collection. + List[Tuple[Dict[str, str]]]: List of tuples with alias dictionaries. """ aliases = await self.get_aliases() - return aliases.get(index_alias_by_collection_id(collection_id), []) + main_alias = index_alias_by_collection_id(collection_id) + return aliases.get(main_alias, []) diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py index 20f919ab9..8d2555957 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py @@ -1,6 +1,6 @@ """Async index selectors with datetime-based filtering.""" -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from stac_fastapi.sfeos_helpers.database import filter_indexes_by_datetime from stac_fastapi.sfeos_helpers.mappings import ITEM_INDICES @@ -40,23 +40,23 @@ def __init__(self, client: Any): self.alias_loader = IndexAliasLoader(client, self.cache_manager) self._initialized = True - async def refresh_cache(self) -> Dict[str, List[str]]: + async def refresh_cache(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: """Force refresh of the aliases cache. Returns: - Dict[str, List[str]]: Refreshed dictionary mapping base collection aliases + Dict[str, List[Tuple[Dict[str, str]]]]: Refreshed dictionary mapping base collection aliases to lists of their corresponding item index aliases. """ return await self.alias_loader.refresh_aliases() - async def get_collection_indexes(self, collection_id: str) -> List[str]: + async def get_collection_indexes(self, collection_id: str) -> List[tuple[dict[str, str]]]: """Get all index aliases for a specific collection. Args: collection_id (str): The ID of the collection to retrieve indexes for. Returns: - List[str]: List of index aliases associated with the collection. + List[tuple[dict[str, str]]]: List of index aliases associated with the collection. Returns empty list if collection is not found in cache. """ return await self.alias_loader.get_collection_indexes(collection_id) @@ -64,7 +64,7 @@ async def get_collection_indexes(self, collection_id: str) -> List[str]: async def select_indexes( self, collection_ids: Optional[List[str]], - datetime_search: Dict[str, Optional[str]], + datetime_search: Dict[str, dict[str, str]], ) -> str: """Select indexes filtered by collection IDs and datetime criteria. @@ -75,7 +75,7 @@ async def select_indexes( Args: collection_ids (Optional[List[str]]): List of collection IDs to filter by. If None or empty, returns all item indices. - datetime_search (Dict[str, Optional[str]]): Dictionary containing datetime + datetime_search (Dict[str, dict[str, str]]): Dictionary containing datetime search criteria with 'gte' and 'lte' keys for range filtering. Returns: @@ -89,8 +89,7 @@ async def select_indexes( collection_indexes = await self.get_collection_indexes(collection_id) filtered_indexes = filter_indexes_by_datetime( collection_indexes, - datetime_search.get("gte"), - datetime_search.get("lte"), + datetime_search ) selected_indexes.extend(filtered_indexes) From d2b4f343576f36eaafe2480e625d9ce05571eeb2 Mon Sep 17 00:00:00 2001 From: Grzegorz Pustulka Date: Sun, 23 Nov 2025 04:59:27 +0100 Subject: [PATCH 2/8] Add additional temporal aliases --- .idea/workspace.xml | 63 ++ stac_fastapi/core/stac_fastapi/core/core.py | 20 +- .../stac_fastapi/elasticsearch/version.py | 5 + .../stac_fastapi/opensearch/database_logic.py | 1 - .../sfeos_helpers/database/datetime.py | 5 +- .../sfeos_helpers/database/index.py | 77 ++- .../sfeos_helpers/database/query.py | 2 +- .../search_engine/index_operations.py | 38 +- .../sfeos_helpers/search_engine/inserters.py | 39 +- .../sfeos_helpers/search_engine/managers.py | 67 +- .../search_engine/selection/base.py | 2 +- .../search_engine/selection/cache_manager.py | 22 +- .../search_engine/selection/selectors.py | 9 +- stac_fastapi/tests/api/test_api.py | 583 ++++++++++++++---- stac_fastapi/tests/database/test_database.py | 329 +++++++++- 15 files changed, 1060 insertions(+), 202 deletions(-) create mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 000000000..46ed682a1 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,63 @@ + + + + + + + + + + + + + { + "customColor": "", + "associatedIndex": 0 +} + + + + { + "keyToString": { + "Python.core.executor": "Run", + "RunOnceActivity.ShowReadmeOnStart": "true", + "RunOnceActivity.git.unshallow": "true", + "git-widget-placeholder": "main", + "last_opened_file_path": "/Users/gpusulka/Projects/stac-fastapi-elasticsearch-opensearch" + } +} + + + + + + + + + + + 1758823057239 + + + + + + + + + + \ No newline at end of file diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 3334a4db3..b49121b2c 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -510,18 +510,20 @@ async def post_all_collections( return await self.all_collections( limit=search_request.limit if hasattr(search_request, "limit") else None, bbox=search_request.bbox if hasattr(search_request, "bbox") else None, - datetime=search_request.datetime - if hasattr(search_request, "datetime") - else None, + datetime=( + search_request.datetime if hasattr(search_request, "datetime") else None + ), token=search_request.token if hasattr(search_request, "token") else None, fields=fields, sortby=sortby, - filter_expr=search_request.filter - if hasattr(search_request, "filter") - else None, - filter_lang=search_request.filter_lang - if hasattr(search_request, "filter_lang") - else None, + filter_expr=( + search_request.filter if hasattr(search_request, "filter") else None + ), + filter_lang=( + search_request.filter_lang + if hasattr(search_request, "filter_lang") + else None + ), query=search_request.query if hasattr(search_request, "query") else None, q=search_request.q if hasattr(search_request, "q") else None, request=request, diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index 4a912c90b..48c7bd93d 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,7 @@ """library version.""" +<<<<<<< HEAD __version__ = "6.7.5" +======= + +__version__ = "6.7.4" +>>>>>>> 798a2c5 (Add additional temporal aliases) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 993876475..a3e6a061f 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -494,7 +494,6 @@ def apply_datetime_filter( if not datetime_search: return search, result_metadata - if USE_DATETIME: if "eq" in datetime_search: # For exact matches, include: diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py index a8b93edb1..041cdd1bc 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/datetime.py @@ -138,7 +138,7 @@ def return_date( return result -def extract_date(date_str: str | None) -> date | None: +def extract_date(date_str: str) -> date: """Extract date from ISO format string. Args: @@ -147,9 +147,6 @@ def extract_date(date_str: str | None) -> date | None: Returns: A date object extracted from the input string or None. """ - if not date_str: - return None - date_str = date_str.replace("Z", "+00:00") return datetime_type.fromisoformat(date_str).date() diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py index ba360a1e7..20dcbaf32 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py @@ -4,11 +4,9 @@ """ import re -from datetime import datetime +from datetime import date, datetime from functools import lru_cache -from typing import Any, List, Optional, Tuple, Dict - -from dateutil.parser import parse # type: ignore[import] +from typing import Any, Dict, List, Optional, Tuple from stac_fastapi.sfeos_helpers.mappings import ( _ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE, @@ -72,10 +70,28 @@ def indices(collection_ids: Optional[List[str]]) -> str: def filter_indexes_by_datetime( collection_indexes: List[Tuple[Dict[str, str], ...]], - datetime_search: Dict[str, Dict[str, str | None]] + datetime_search: Dict[str, Dict[str, str | None]], ) -> List[str]: + """ + Filter Elasticsearch index aliases based on datetime search criteria. + + Filters a list of collection indexes by matching their datetime, start_datetime, and end_datetime + aliases against the provided search criteria. Each criterion can have optional 'gte' (greater than + or equal) and 'lte' (less than or equal) bounds. + + Args: + collection_indexes (List[Tuple[Dict[str, str], ...]]): A list of tuples containing dictionaries + with 'datetime', 'start_datetime', and 'end_datetime' aliases. + datetime_search (Dict[str, Dict[str, str | None]]): A dictionary with keys 'datetime', + 'start_datetime', and 'end_datetime', each containing 'gte' and 'lte' criteria as ISO format + datetime strings or None. + + Returns: + List[str]: A list of start_datetime aliases that match all provided search criteria. + """ + def extract_date_from_alias(alias: str) -> tuple[datetime, datetime] | None: - date_pattern = re.compile(r'\d{4}-\d{2}-\d{2}') + date_pattern = re.compile(r"\d{4}-\d{2}-\d{2}") try: dates = date_pattern.findall(alias) @@ -83,26 +99,30 @@ def extract_date_from_alias(alias: str) -> tuple[datetime, datetime] | None: return None if len(dates) >= 2: - return datetime.strptime(dates[-2], '%Y-%m-%d'), datetime.strptime(dates[-1], '%Y-%m-%d') + return datetime.strptime(dates[-2], "%Y-%m-%d"), datetime.strptime( + dates[-1], "%Y-%m-%d" + ) else: - date = datetime.strptime(dates[-1], '%Y-%m-%d') + date = datetime.strptime(dates[-1], "%Y-%m-%d") return date, date except (ValueError, IndexError): return None - def parse_search_date(date_str: str | None) -> Optional[datetime.date]: + def parse_search_date(date_str: str | None) -> Optional[date]: if not date_str: return None - return datetime.fromisoformat(date_str.replace('Z', '+00:00')).date() + date_str = date_str.rstrip("Z") + return datetime.fromisoformat(date_str).date() - def check_criteria(value_begin: datetime.date, value_end: datetime.date, criteria: Dict) -> bool: - gte = parse_search_date(criteria.get('gte')) - lte = parse_search_date(criteria.get('lte')) + def check_criteria(value_begin: date, value_end: date, criteria: Dict) -> bool: + gte = parse_search_date(criteria.get("gte")) + lte = parse_search_date(criteria.get("lte")) - if gte and value_begin.date() < gte: + if gte and value_end < gte: return False - if lte and value_end.date() > lte: + if lte and value_begin > lte: return False + return True filtered_indexes = [] @@ -112,9 +132,9 @@ def check_criteria(value_begin: datetime.date, value_end: datetime.date, criteri continue index_dict = index_tuple[0] - start_datetime_alias = index_dict.get('start_datetime') - end_datetime_alias = index_dict.get('end_datetime') - datetime_alias = index_dict.get('datetime') + start_datetime_alias = index_dict.get("start_datetime") + end_datetime_alias = index_dict.get("end_datetime") + datetime_alias = index_dict.get("datetime") if not start_datetime_alias: continue @@ -123,15 +143,20 @@ def check_criteria(value_begin: datetime.date, value_end: datetime.date, criteri end_date = extract_date_from_alias(end_datetime_alias) datetime_date = extract_date_from_alias(datetime_alias) - if not start_range or not end_date or not datetime_date: - continue - - if not check_criteria(start_range[0], start_range[1], datetime_search.get('start_datetime', {})): - continue - if not check_criteria(end_date[0], end_date[1], datetime_search.get('end_datetime', {})): - continue - if not check_criteria(datetime_date[0], datetime_date[1], datetime_search.get('datetime', {})): + if not check_criteria( + start_range[0], start_range[1], datetime_search.get("start_datetime", {}) + ): continue + if end_date: + if not check_criteria( + end_date[0], end_date[1], datetime_search.get("end_datetime", {}) + ): + continue + if datetime_date: + if not check_criteria( + datetime_date[0], datetime_date[1], datetime_search.get("datetime", {}) + ): + continue filtered_indexes.append(start_datetime_alias) diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py index 72285a56f..abc8b7e6f 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/query.py @@ -130,7 +130,7 @@ def apply_collections_datetime_filter_shared( def apply_collections_bbox_filter_shared( - bbox: Union[str, List[float], None] + bbox: Union[str, List[float], None], ) -> Optional[Dict[str, Dict]]: """Create a geo_shape filter for collections bbox search. diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py index 0b26f1657..f3e033fdd 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py @@ -1,7 +1,7 @@ """Search engine adapters for different implementations.""" import uuid -from typing import Any, Dict, Literal, List +from typing import Any, Dict, List, Literal from stac_fastapi.sfeos_helpers.database import ( index_alias_by_collection_id, @@ -39,7 +39,12 @@ async def create_simple_index(self, client: Any, collection_id: str) -> str: return index_name async def create_datetime_index( - self, client: Any, collection_id: str, start_datetime: str, datetime: str | None, end_datetime: str + self, + client: Any, + collection_id: str, + start_datetime: str, + datetime: str, + end_datetime: str, ) -> str: """Create a datetime-based index for the given collection. @@ -47,19 +52,23 @@ async def create_datetime_index( client: Search engine client instance. collection_id (str): Collection identifier. start_datetime (str): Start datetime for the index alias. - datetime (str | None): Datetime for the datetime alias (can be None). + datetime (str): Datetime for the datetime alias (can be None). end_datetime (str): End datetime for the index alias. Returns: str: Created start_datetime alias name. """ index_name = self.create_index_name(collection_id) - alias_start_date = self.create_alias_name(collection_id, "start_datetime", start_datetime) + alias_start_date = self.create_alias_name( + collection_id, "start_datetime", start_datetime + ) alias_date = self.create_alias_name(collection_id, "datetime", datetime) - alias_end_date = self.create_alias_name(collection_id, "end_datetime", end_datetime) + alias_end_date = self.create_alias_name( + collection_id, "end_datetime", end_datetime + ) collection_alias = index_alias_by_collection_id(collection_id) - aliases = { + aliases: Dict[str, Any] = { collection_alias: {}, alias_start_date: {}, alias_date: {}, @@ -96,7 +105,12 @@ async def update_index_alias(client: Any, end_date: str, old_alias: str) -> str: return new_alias @staticmethod - async def change_alias_name(client: Any, old_start_datetime_alias: str, aliases_to_change: List[str], aliases_to_create: List[str]) -> None: + async def change_alias_name( + client: Any, + old_start_datetime_alias: str, + aliases_to_change: List[str], + aliases_to_create: List[str], + ) -> None: """Change alias names by removing old aliases and adding new ones. Args: @@ -135,7 +149,9 @@ def create_index_name(collection_id: str) -> str: @staticmethod def create_alias_name( - collection_id: str, name: Literal["start_datetime", "datetime", "end_datetime"], start_date: str + collection_id: str, + name: Literal["start_datetime", "datetime", "end_datetime"], + start_date: str, ) -> str: """Create alias name from collection ID and date. @@ -180,7 +196,11 @@ async def find_latest_item_in_index(client: Any, index_name: str) -> dict[str, A query = { "size": 1, "sort": [{"properties.start_datetime": {"order": "desc"}}], - "_source": ["properties.start_datetime", "properties.datetime", "properties.end_datetime"], + "_source": [ + "properties.start_datetime", + "properties.datetime", + "properties.end_datetime", + ], } response = await client.search(index=index_name, body=query) diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py index c89509ec3..310f89516 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py @@ -1,6 +1,7 @@ """Async index insertion strategies.""" + import logging -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from fastapi import HTTPException, status @@ -117,7 +118,7 @@ async def _get_target_index_internal( collection_id: str, product: Dict[str, Any], check_size: bool = True, - ) -> str: + ) -> Optional[str]: """Get target index with size checking internally. Args: @@ -132,9 +133,9 @@ async def _get_target_index_internal( product_datetimes = self.datetime_manager.validate_product_datetimes(product) datetime_ranges = { - "start_datetime": {"gte": product_datetimes.start_datetime, "lte": product_datetimes.start_datetime}, - "end_datetime": {"gte": product_datetimes.end_datetime, "lte": product_datetimes.end_datetime}, - "datetime": {"gte": product_datetimes.datetime, "lte": product_datetimes.datetime} + "start_datetime": {"gte": None, "lte": product_datetimes.start_datetime}, + "end_datetime": {"gte": None, "lte": None}, + "datetime": {"gte": None, "lte": None}, } target_index = await index_selector.select_indexes( @@ -149,7 +150,7 @@ async def _get_target_index_internal( await index_selector.refresh_cache() return target_index - all_indexes = sorted(all_indexes, key=lambda x: x[0]['start_datetime']) + all_indexes = sorted(all_indexes, key=lambda x: x[0]["start_datetime"]) start_date = extract_date(product_datetimes.start_datetime) end_date = extract_first_date_from_index(all_indexes[0][0]["start_datetime"]) @@ -160,7 +161,7 @@ async def _get_target_index_internal( await index_selector.refresh_cache() return alias - if target_index != all_indexes[-1]: + if target_index != all_indexes[-1][0]["start_datetime"]: for item in all_indexes: aliases_dict = item[0] if target_index in aliases_dict.values(): @@ -179,8 +180,16 @@ async def _get_target_index_internal( collection_id, product_datetimes, aliases_dict ) await index_selector.refresh_cache() + return target_index - return target_index + for item in all_indexes: + aliases_dict = item[0] + if target_index in aliases_dict.values(): + await self.datetime_manager.handle_early_date( + collection_id, product_datetimes, aliases_dict + ) + return target_index + return None async def _ensure_indexes_exist( self, index_selector, collection_id: str, items: List[Dict[str, Any]] @@ -199,9 +208,9 @@ async def _ensure_indexes_exist( await self.index_operations.create_datetime_index( self.client, collection_id, - extract_date(first_item["properties"]["start_datetime"]), - extract_date(first_item["properties"]["datetime"]), - extract_date(first_item["properties"]["end_datetime"]), + str(extract_date(first_item["properties"]["start_datetime"])), + str(extract_date(first_item["properties"]["datetime"])), + str(extract_date(first_item["properties"]["end_datetime"])), ) await index_selector.refresh_cache() @@ -227,9 +236,9 @@ async def _check_and_handle_oversized_index( ) all_indexes = await index_selector.get_collection_indexes(collection_id) - all_indexes = sorted(all_indexes, key=lambda x: x[0]['start_datetime']) + all_indexes = sorted(all_indexes, key=lambda x: x[0]["start_datetime"]) - latest_index = all_indexes[-1][0] + latest_index = all_indexes[-1][0]["start_datetime"] if first_item_index != latest_index: return None @@ -248,7 +257,9 @@ async def _check_and_handle_oversized_index( end_datetime=latest_item["_source"]["properties"]["end_datetime"], ) - await self.datetime_manager.handle_oversized_index(collection_id, product_datetimes, latest_index) + await self.datetime_manager.handle_oversized_index( + collection_id, product_datetimes, all_indexes[-1][0] + ) await index_selector.refresh_cache() diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py index d51d3fbf9..d0b1b0ab6 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py @@ -18,8 +18,16 @@ class ProductDatetimes(NamedTuple): + """Named tuple representing product datetime fields. + + Attributes: + start_datetime (str): ISO format start datetime string. + datetime (str): ISO format datetime string. + end_datetime (str): ISO format end datetime string. + """ + start_datetime: str - datetime: str | None + datetime: str end_datetime: str @@ -123,28 +131,27 @@ def validate_product_datetimes(product: Dict[str, Any]) -> ProductDatetimes: Returns: ProductDatetimes: Named tuple containing: - start_datetime (str): Start datetime value - - datetime (str | None): datetime value + - datetime (str): datetime value - end_datetime (str): End datetime value Raises: HTTPException: If product start_datetime is missing or invalid. """ - properties = product.get("properties", {}) start_datetime_value = properties.get("start_datetime") datetime_value = properties.get("datetime") end_datetime_value = properties.get("end_datetime") - if not start_datetime_value: + if not start_datetime_value or not datetime_value or not end_datetime_value: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Product 'start_datetime' is required for indexing", + detail="Product 'start_datetime', 'datetime' and 'end_datetime' is required for indexing", ) - if not datetime_value and not end_datetime_value: + if not (start_datetime_value <= datetime_value <= end_datetime_value): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail="'end_datetime' is required when 'datetime' is not provided." + detail="'start_datetime' <= 'datetime' <= 'end_datetime' is required", ) return ProductDatetimes( @@ -170,7 +177,11 @@ async def handle_new_collection( end_datetime = extract_date(product_datetimes.end_datetime) target_index = await self.index_operations.create_datetime_index( - self.client, collection_id, start_datetime, datetime, end_datetime + self.client, + collection_id, + str(start_datetime), + str(datetime), + str(end_datetime), ) logger.info( f"Successfully created index '{target_index}' for collection '{collection_id}'" @@ -178,7 +189,10 @@ async def handle_new_collection( return target_index async def handle_early_date( - self, collection_id: str, product_datetimes: ProductDatetimes, old_aliases: Dict[str, str] + self, + collection_id: str, + product_datetimes: ProductDatetimes, + old_aliases: Dict[str, str], ) -> str: """Handle product with date earlier than existing indexes asynchronously. @@ -194,9 +208,13 @@ async def handle_early_date( dt = extract_date(product_datetimes.datetime) end_dt = extract_date(product_datetimes.end_datetime) - old_start_datetime_alias = extract_first_date_from_index(old_aliases["start_datetime"]) + old_start_datetime_alias = extract_first_date_from_index( + old_aliases["start_datetime"] + ) old_datetime_alias = extract_first_date_from_index(old_aliases["datetime"]) - old_end_datetime_alias = extract_first_date_from_index(old_aliases["end_datetime"]) + old_end_datetime_alias = extract_first_date_from_index( + old_aliases["end_datetime"] + ) new_start_datetime_alias = self.index_operations.create_alias_name( collection_id, "start_datetime", str(start_dt) @@ -205,11 +223,10 @@ async def handle_early_date( aliases_to_change = [] aliases_to_create = [] - if start_dt > old_start_datetime_alias: + if start_dt < old_start_datetime_alias: aliases_to_create.append(new_start_datetime_alias) aliases_to_change.append(old_aliases["start_datetime"]) - if dt > old_datetime_alias: new_datetime_alias = self.index_operations.create_alias_name( collection_id, "datetime", str(dt) @@ -226,12 +243,18 @@ async def handle_early_date( if aliases_to_change: await self.index_operations.change_alias_name( - self.client, old_aliases["start_datetime"], aliases_to_change, aliases_to_create + self.client, + old_aliases["start_datetime"], + aliases_to_change, + aliases_to_create, ) return new_start_datetime_alias async def handle_oversized_index( - self, collection_id: str, product_datetimes: ProductDatetimes, old_aliases: Dict[str, str] + self, + collection_id: str, + product_datetimes: ProductDatetimes, + old_aliases: Dict[str, str], ) -> str: """Handle index that exceeds size limit asynchronously. @@ -250,7 +273,9 @@ async def handle_oversized_index( old_start_datetime_alias = extract_first_date_from_index(target_index) old_datetime_alias = extract_first_date_from_index(old_aliases["datetime"]) - old_end_datetime_alias = extract_first_date_from_index(old_aliases["end_datetime"]) + old_end_datetime_alias = extract_first_date_from_index( + old_aliases["end_datetime"] + ) if start_dt != old_start_datetime_alias: aliases_to_change = [] @@ -274,9 +299,15 @@ async def handle_oversized_index( aliases_to_create.append(new_end_datetime_alias) aliases_to_change.append(old_aliases["end_datetime"]) - await self.index_operations.change_alias_name(self.client, target_index, aliases_to_change, aliases_to_create) + await self.index_operations.change_alias_name( + self.client, target_index, aliases_to_change, aliases_to_create + ) target_index = await self.index_operations.create_datetime_index( - self.client, collection_id, str(start_dt + timedelta(days=1)), str(dt), str(end_dt) + self.client, + collection_id, + str(start_dt + timedelta(days=1)), + str(dt), + str(end_dt), ) return target_index diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py index 95f406728..aa9fc7edb 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py @@ -11,7 +11,7 @@ class BaseIndexSelector(ABC): async def select_indexes( self, collection_ids: Optional[List[str]], - datetime_search: Dict[str, Optional[str]], + datetime_search: Dict[str, dict[str, str]], ) -> str: """Select appropriate indexes asynchronously. diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py index 40d4b08fc..135cd5b0e 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py @@ -3,7 +3,7 @@ import copy import threading import time -from typing import Dict, List, Optional, Tuple, Any +from typing import Any, Dict, List, Optional, Tuple from stac_fastapi.sfeos_helpers.database import index_alias_by_collection_id from stac_fastapi.sfeos_helpers.mappings import ITEMS_INDEX_PREFIX @@ -91,7 +91,7 @@ async def load_aliases(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: } """ response = await self.client.indices.get_alias(index=f"{ITEMS_INDEX_PREFIX}*") - result = {} + result: Dict[str, List[Tuple[Dict[str, str]]]] = {} for index_name, index_info in response.items(): aliases = index_info.get("aliases", {}) @@ -126,7 +126,7 @@ def _find_main_alias(aliases: List[str]) -> str: Returns: str: The main collection alias. """ - temporal_keywords = ['datetime', 'start_datetime', 'end_datetime'] + temporal_keywords = ["datetime", "start_datetime", "end_datetime"] for alias in aliases: if not any(keyword in alias for keyword in temporal_keywords): @@ -151,12 +151,12 @@ def _organize_aliases(aliases: List[str], main_alias: str) -> Dict[str, str]: if alias == main_alias: continue - if 'start_datetime' in alias: - aliases_dict['start_datetime'] = alias - elif 'end_datetime' in alias: - aliases_dict['end_datetime'] = alias - elif 'datetime' in alias: - aliases_dict['datetime'] = alias + if "start_datetime" in alias: + aliases_dict["start_datetime"] = alias + elif "end_datetime" in alias: + aliases_dict["end_datetime"] = alias + elif "datetime" in alias: + aliases_dict["datetime"] = alias return aliases_dict @@ -179,7 +179,9 @@ async def refresh_aliases(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: """ return await self.load_aliases() - async def get_collection_indexes(self, collection_id: str) -> List[Tuple[Dict[str, str]]]: + async def get_collection_indexes( + self, collection_id: str + ) -> List[Tuple[Dict[str, str]]]: """Get index information for a specific collection. Args: diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py index 8d2555957..b3a2d14ce 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py @@ -49,7 +49,9 @@ async def refresh_cache(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: """ return await self.alias_loader.refresh_aliases() - async def get_collection_indexes(self, collection_id: str) -> List[tuple[dict[str, str]]]: + async def get_collection_indexes( + self, collection_id: str + ) -> List[tuple[dict[str, str]]]: """Get all index aliases for a specific collection. Args: @@ -88,8 +90,7 @@ async def select_indexes( for collection_id in collection_ids: collection_indexes = await self.get_collection_indexes(collection_id) filtered_indexes = filter_indexes_by_datetime( - collection_indexes, - datetime_search + collection_indexes, datetime_search ) selected_indexes.extend(filtered_indexes) @@ -104,7 +105,7 @@ class UnfilteredIndexSelector(BaseIndexSelector): async def select_indexes( self, collection_ids: Optional[List[str]], - datetime_search: Dict[str, Optional[str]], + datetime_search: Dict[str, dict[str, str]], ) -> str: """Select all indices for given collections without datetime filtering. diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 38d7e5978..1c0f4c402 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -845,92 +845,98 @@ async def test_big_int_eo_search( assert results == {value} -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_item_in_past_date_change_alias_name_for_datetime_index( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - item = load_test_data("test_item.json") - item["id"] = str(uuid.uuid4()) - item["properties"]["datetime"] = "2012-02-12T12:30:22Z" - - response = await app_client.post( - f"/collections/{item['collection']}/items", json=item - ) - assert response.status_code == 201 - indices = await txn_client.database.client.indices.get_alias( - index="items_test-collection" - ) - expected_aliases = [ - "items_test-collection_2012-02-12", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_item_uses_existing_datetime_index_for_datetime_index( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - item = load_test_data("test_item.json") - item["id"] = str(uuid.uuid4()) - - response = await app_client.post( - f"/collections/{item['collection']}/items", json=item - ) - - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias( - index="items_test-collection" - ) - expected_aliases = [ - "items_test-collection_2020-02-12", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_item_with_different_date_same_index_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - item = load_test_data("test_item.json") - item["id"] = str(uuid.uuid4()) - item["properties"]["datetime"] = "2022-02-12T12:30:22Z" - - response = await app_client.post( - f"/collections/{item['collection']}/items", json=item - ) - - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias( - index="items_test-collection" - ) - expected_aliases = [ - "items_test-collection_2020-02-12", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) +# @pytest.mark.datetime_filtering +# @pytest.mark.asyncio +# async def test_create_item_in_past_date_change_alias_name_for_datetime_index( +# app_client, ctx, load_test_data, txn_client +# ): +# if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): +# pytest.skip() +# +# item = load_test_data("test_item.json") +# item["id"] = str(uuid.uuid4()) +# item["properties"]["start_datetime"] = "2012-02-12T12:30:22Z" +# +# response = await app_client.post( +# f"/collections/{item['collection']}/items", json=item +# ) +# assert response.status_code == 201 +# indices = await txn_client.database.client.indices.get_alias( +# index="items_test-collection" +# ) +# expected_aliases = [ +# "items_start_datetime_test-collection_2012-02-12", +# "items_datetime_test-collection_2020-02-12", +# "items_end_datetime_test-collection_2020-02-16", +# ] +# all_aliases = set() +# for index_info in indices.values(): +# all_aliases.update(index_info.get("aliases", {}).keys()) +# +# assert all(alias in all_aliases for alias in expected_aliases) +# +# +# @pytest.mark.datetime_filtering +# @pytest.mark.asyncio +# async def test_create_item_uses_existing_datetime_index_for_datetime_index( +# app_client, ctx, load_test_data, txn_client +# ): +# if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): +# pytest.skip() +# +# item = load_test_data("test_item.json") +# item["id"] = str(uuid.uuid4()) +# +# response = await app_client.post( +# f"/collections/{item['collection']}/items", json=item +# ) +# +# assert response.status_code == 201 +# +# indices = await txn_client.database.client.indices.get_alias( +# index="items_test-collection" +# ) +# expected_aliases = [ +# "items_start_datetime_test-collection_2020-02-08", +# "items_datetime_test-collection_2020-02-12", +# "items_end_datetime_test-collection_2020-02-16", +# ] +# all_aliases = set() +# for index_info in indices.values(): +# all_aliases.update(index_info.get("aliases", {}).keys()) +# assert all(alias in all_aliases for alias in expected_aliases) +# +# +# @pytest.mark.datetime_filtering +# @pytest.mark.asyncio +# async def test_create_item_with_different_date_same_index_for_datetime_index( +# app_client, load_test_data, txn_client, ctx +# ): +# if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): +# pytest.skip() +# +# item = load_test_data("test_item.json") +# item["id"] = str(uuid.uuid4()) +# item["properties"]["start_datetime"] = "2020-02-11T12:30:22Z" +# +# response = await app_client.post( +# f"/collections/{item['collection']}/items", json=item +# ) +# +# assert response.status_code == 201 +# +# indices = await txn_client.database.client.indices.get_alias( +# index="items_test-collection" +# ) +# expected_aliases = [ +# "items_start_datetime_test-collection_2020-02-08", +# "items_datetime_test-collection_2020-02-12", +# "items_end_datetime_test-collection_2020-02-16" +# ] +# all_aliases = set() +# for index_info in indices.values(): +# all_aliases.update(index_info.get("aliases", {}).keys()) +# assert all(alias in all_aliases for alias in expected_aliases) @pytest.mark.datetime_filtering @@ -943,7 +949,7 @@ async def test_create_new_index_when_size_limit_exceeded_for_datetime_index( item = load_test_data("test_item.json") item["id"] = str(uuid.uuid4()) - item["properties"]["datetime"] = "2024-02-12T12:30:22Z" + item["properties"]["start_datetime"] = "2020-02-11T12:30:22Z" with patch( "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" @@ -957,8 +963,8 @@ async def test_create_new_index_when_size_limit_exceeded_for_datetime_index( indices = await txn_client.database.client.indices.get_alias(index="*") expected_aliases = [ - "items_test-collection_2020-02-12-2024-02-12", - "items_test-collection_2024-02-13", + "items_start_datetime_test-collection_2020-02-08-2020-02-11", + "items_start_datetime_test-collection_2020-02-12", ] all_aliases = set() @@ -968,7 +974,7 @@ async def test_create_new_index_when_size_limit_exceeded_for_datetime_index( item_2 = deepcopy(item) item_2["id"] = str(uuid.uuid4()) - item_2["properties"]["datetime"] = "2023-02-12T12:30:22Z" + item_2["properties"]["start_datetime"] = "2020-02-10T12:30:22Z" response_2 = await app_client.post( f"/collections/{item_2['collection']}/items", json=item_2 ) @@ -985,7 +991,7 @@ async def test_create_item_fails_without_datetime_for_datetime_index( item = load_test_data("test_item.json") item["id"] = str(uuid.uuid4()) - item["properties"]["datetime"] = None + item["properties"]["start_datetime"] = None response = await app_client.post( f"/collections/{item['collection']}/items", json=item ) @@ -1006,7 +1012,9 @@ async def test_bulk_create_items_with_same_date_range_for_datetime_index( for i in range(10): item = deepcopy(base_item) item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = f"2020-02-{12 + i}T12:30:22Z" item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" + item["properties"]["end_datetime"] = f"2020-02-{12 + i}T12:30:22Z" items_dict[item["id"]] = item payload = {"type": "FeatureCollection", "features": list(items_dict.values())} @@ -1018,7 +1026,7 @@ async def test_bulk_create_items_with_same_date_range_for_datetime_index( indices = await txn_client.database.client.indices.get_alias(index="*") expected_aliases = [ - "items_test-collection_2020-02-12", + "items_start_datetime_test-collection_2020-02-12", ] all_aliases = set() for index_info in indices.values(): @@ -1040,13 +1048,17 @@ async def test_bulk_create_items_with_different_date_ranges_for_datetime_index( for i in range(3): item = deepcopy(base_item) item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = f"2020-02-{12 + i}T12:30:22Z" item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" + item["properties"]["end_datetime"] = f"2020-02-{12 + i}T12:30:22Z" items_dict[item["id"]] = item for i in range(2): item = deepcopy(base_item) item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = f"2010-02-{10 + i}T12:30:22Z" item["properties"]["datetime"] = f"2010-02-{10 + i}T12:30:22Z" + item["properties"]["end_datetime"] = f"2010-02-{10 + i}T12:30:22Z" items_dict[item["id"]] = item payload = {"type": "FeatureCollection", "features": list(items_dict.values())} @@ -1058,7 +1070,7 @@ async def test_bulk_create_items_with_different_date_ranges_for_datetime_index( assert response.status_code == 201 indices = await txn_client.database.client.indices.get_alias(index="*") - expected_aliases = ["items_test-collection_2010-02-10"] + expected_aliases = ["items_start_datetime_test-collection_2010-02-10"] all_aliases = set() for index_info in indices.values(): all_aliases.update(index_info.get("aliases", {}).keys()) @@ -1082,7 +1094,7 @@ def create_items(date_prefix: str, start_day: int, count: int) -> dict: item = deepcopy(base_item) item["id"] = str(uuid.uuid4()) item["properties"][ - "datetime" + "start_datetime" ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" items[item["id"]] = item return items @@ -1116,8 +1128,8 @@ def create_items(date_prefix: str, start_day: int, count: int) -> dict: indices = await txn_client.database.client.indices.get_alias(index="*") expected_aliases = [ - "items_test-collection_2010-02-10-2020-02-12", - "items_test-collection_2020-02-13", + "items_start_datetime_test-collection_2010-02-10-2020-02-08", + "items_start_datetime_test-collection_2020-02-09", ] all_aliases = set() for index_info in indices.values(): @@ -1142,7 +1154,7 @@ def create_items(date_prefix: str, start_day: int, count: int) -> dict: item = deepcopy(base_item) item["id"] = str(uuid.uuid4()) item["properties"][ - "datetime" + "start_datetime" ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" items[item["id"]] = item return items @@ -1176,8 +1188,8 @@ def create_items(date_prefix: str, start_day: int, count: int) -> dict: indices = await txn_client.database.client.indices.get_alias(index="*") expected_aliases = [ - "items_test-collection_2008-01-15-2020-02-12", - "items_test-collection_2020-02-13", + "items_start_datetime_test-collection_2008-01-15-2020-02-08", + "items_start_datetime_test-collection_2020-02-09", ] all_aliases = set() for index_info in indices.values(): @@ -1202,7 +1214,7 @@ def create_items(date_prefix: str, start_day: int, count: int) -> dict: item = deepcopy(base_item) item["id"] = str(uuid.uuid4()) item["properties"][ - "datetime" + "start_datetime" ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" items[item["id"]] = item return items @@ -1257,7 +1269,7 @@ def create_items(date_prefix: str, start_day: int, count: int) -> dict: item = deepcopy(base_item) item["id"] = str(uuid.uuid4()) item["properties"][ - "datetime" + "start_datetime" ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" items[item["id"]] = item return items @@ -1314,7 +1326,7 @@ def create_items(date_prefix: str, start_day: int, count: int) -> dict: item = deepcopy(base_item) item["id"] = str(uuid.uuid4()) item["properties"][ - "datetime" + "start_datetime" ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" items[item["id"]] = item return items @@ -1374,7 +1386,7 @@ def create_items(date_prefix: str, start_day: int, count: int) -> dict: item = deepcopy(base_item) item["id"] = str(uuid.uuid4()) item["properties"][ - "datetime" + "start_datetime" ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" items[item["id"]] = item return items @@ -1431,7 +1443,7 @@ def create_items(date_prefix: str, start_day: int, count: int) -> dict: item = deepcopy(base_item) item["id"] = str(uuid.uuid4()) item["properties"][ - "datetime" + "start_datetime" ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" items[item["id"]] = item return items @@ -1710,3 +1722,366 @@ async def test_hide_private_data_from_item(app_client, txn_client, load_test_dat assert "private_data" not in item["properties"] del os.environ["EXCLUDED_FROM_ITEMS"] + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_new_item_in_new_collection_for_datetime_index( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + response = await app_client.post("/collections/new-collection/items", json=item) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_new-collection" + ) + expected_aliases = [ + "items_datetime_new-collection_2020-02-12", + "items_end_datetime_new-collection_2020-02-16", + "items_start_datetime_new-collection_2020-02-08", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_with_invalid_datetime_ordering_should_fail( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + item["properties"]["start_datetime"] = "2024-02-12T12:30:22Z" + item["properties"]["datetime"] = "2023-02-12T12:30:22Z" + item["properties"]["end_datetime"] = "2022-02-12T12:30:22Z" + + await app_client.post("/collections", json=new_collection) + + response = await app_client.post("/collections/new-collection/items", json=item) + assert response.status_code == 400 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_update_item_with_changed_end_datetime( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + updated_item = item.copy() + updated_item["properties"]["end_datetime"] = "2020-02-19T12:30:22Z" + + response = await app_client.put( + f"/collections/new-collection/items/{item['id']}", json=updated_item + ) + + assert response.status_code == 200 + + indices = await txn_client.database.client.indices.get_alias( + index="items_new-collection" + ) + expected_aliases = [ + "items_datetime_new-collection_2020-02-12", + "items_end_datetime_new-collection_2020-02-19", + "items_start_datetime_new-collection_2020-02-08", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_update_item_with_changed_datetime( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + updated_item = item.copy() + updated_item["properties"]["datetime"] = "2020-02-14T12:30:22Z" + + response = await app_client.put( + f"/collections/new-collection/items/{item['id']}", json=updated_item + ) + + assert response.status_code == 200 + + indices = await txn_client.database.client.indices.get_alias( + index="items_new-collection" + ) + expected_aliases = [ + "items_datetime_new-collection_2020-02-14", + "items_end_datetime_new-collection_2020-02-16", + "items_start_datetime_new-collection_2020-02-08", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_update_item_with_changed_both_datetime_and_end_datetime( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + updated_item = item.copy() + updated_item["properties"]["datetime"] = "2022-02-15T12:30:22Z" + updated_item["properties"]["end_datetime"] = "2023-02-18T12:30:22Z" + + response = await app_client.put( + f"/collections/new-collection/items/{item['id']}", json=updated_item + ) + + assert response.status_code == 200 + + indices = await txn_client.database.client.indices.get_alias( + index="items_new-collection" + ) + expected_aliases = [ + "items_datetime_new-collection_2022-02-15", + "items_end_datetime_new-collection_2023-02-18", + "items_start_datetime_new-collection_2020-02-08", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_by_datetime_range_with_stac_query( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2020-02-01T00:00:00Z/2020-02-28T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] > 0 + assert len(result["features"]) > 0 + assert any(feature["id"] == item["id"] for feature in result["features"]) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_by_start_datetime_with_stac_query( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2020-02-08T00:00:00Z/.." + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] > 0 + assert any(feature["id"] == item["id"] for feature in result["features"]) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_not_found_outside_datetime_range( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2021-01-01T00:00:00Z/2021-12-31T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] == 0 + assert len(result["features"]) == 0 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_after_datetime_update_with_stac_query( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + updated_item = item.copy() + updated_item["properties"]["datetime"] = "2020-02-14T12:30:22Z" + + await app_client.put( + f"/collections/new-collection/items/{item['id']}", json=updated_item + ) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2020-02-14T00:00:00Z/2020-02-14T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] > 0 + assert any(feature["id"] == item["id"] for feature in result["features"]) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_by_multiple_collections_with_stac_query( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + collection1 = load_test_data("test_collection.json") + collection1["id"] = "collection-1" + + collection2 = load_test_data("test_collection.json") + collection2["id"] = "collection-2" + + item1 = load_test_data("test_item.json") + item1["collection"] = "collection-1" + item1["id"] = "item-1" + + item2 = load_test_data("test_item.json") + item2["collection"] = "collection-2" + item2["id"] = "item-2" + + await app_client.post("/collections", json=collection1) + await app_client.post("/collections", json=collection2) + await app_client.post("/collections/collection-1/items", json=item1) + await app_client.post("/collections/collection-2/items", json=item2) + + response = await app_client.get( + "/search?collections=collection-1,collection-2&datetime=2020-02-01T00:00:00Z/2020-02-28T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] >= 2 + feature_ids = {feature["id"] for feature in result["features"]} + assert "item-1" in feature_ids + assert "item-2" in feature_ids + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_with_the_same_date_change_alias_name_for_datetime_index( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 201 + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_start_datetime_test-collection_2020-02-08", + "items_datetime_test-collection_2020-02-12", + "items_end_datetime_test-collection_2020-02-16", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) diff --git a/stac_fastapi/tests/database/test_database.py b/stac_fastapi/tests/database/test_database.py index 67897c153..aea84e560 100644 --- a/stac_fastapi/tests/database/test_database.py +++ b/stac_fastapi/tests/database/test_database.py @@ -4,7 +4,10 @@ import pytest from stac_pydantic import api -from stac_fastapi.sfeos_helpers.database import index_alias_by_collection_id +from stac_fastapi.sfeos_helpers.database import ( + filter_indexes_by_datetime, + index_alias_by_collection_id, +) from stac_fastapi.sfeos_helpers.mappings import ( COLLECTIONS_INDEX, ES_COLLECTIONS_MAPPINGS, @@ -46,3 +49,327 @@ async def test_index_mapping_items(txn_client, load_test_data): actual_mappings["dynamic_templates"] == ES_ITEMS_MAPPINGS["dynamic_templates"] ) await txn_client.delete_collection(collection["id"]) + + +@pytest.mark.datetime_filtering +def test_filter_datetime_field_outside_range(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": "2021-01-01T00:00:00Z", "lte": "2021-12-31T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_start_datetime_field_with_gte(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": "2020-02-01T00:00:00Z", "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 1 + + +@pytest.mark.datetime_filtering +def test_filter_end_datetime_field_with_lte(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": "2020-02-28T23:59:59Z"}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 1 + + +@pytest.mark.datetime_filtering +def test_filter_all_criteria_matching(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": "2020-02-01T00:00:00Z", "lte": "2020-02-28T23:59:59Z"}, + "start_datetime": {"gte": "2020-02-01T00:00:00Z", "lte": None}, + "end_datetime": {"gte": None, "lte": "2020-02-28T23:59:59Z"}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 1 + + +@pytest.mark.datetime_filtering +def test_filter_datetime_field_fails_gte(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": "2020-02-15T00:00:00Z", "lte": "2020-02-28T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_datetime_field_fails_lte(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": "2020-01-01T00:00:00Z", "lte": "2020-02-10T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_start_datetime_range_format(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08-2022-04-05", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": "2020-02-01T00:00:00Z", "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 1 + assert result[0] == "items_start_datetime_new-collection_2020-02-08-2022-04-05" + + +@pytest.mark.datetime_filtering +def test_filter_start_datetime_range_fails_gte(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08-2022-04-05", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": "2022-05-01T00:00:00Z", "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_multiple_indexes_mixed_results(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ), + ( + { + "datetime": "items_datetime_new-collection_2020-02-15", + "end_datetime": "items_end_datetime_new-collection_2020-02-18", + "start_datetime": "items_start_datetime_new-collection_2020-02-10", + }, + ), + ( + { + "datetime": "items_datetime_new-collection_2021-03-15", + "end_datetime": "items_end_datetime_new-collection_2021-03-20", + "start_datetime": "items_start_datetime_new-collection_2021-03-10", + }, + ), + ] + datetime_search = { + "datetime": {"gte": "2020-02-01T00:00:00Z", "lte": "2020-02-28T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 2 + assert "items_start_datetime_new-collection_2020-02-08" in result + assert "items_start_datetime_new-collection_2020-02-10" in result + + +@pytest.mark.datetime_filtering +def test_filter_empty_collection(): + collection_indexes = [] + datetime_search = { + "datetime": {"gte": "2020-02-01T00:00:00Z", "lte": "2020-02-28T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_all_criteria_none(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 1 + + +@pytest.mark.datetime_filtering +def test_filter_end_datetime_outside_range(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ) + ] + datetime_search = { + "datetime": {"gte": None, "lte": None}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": "2020-02-10T23:59:59Z"}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 0 + + +@pytest.mark.datetime_filtering +def test_filter_complex_mixed_criteria(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-16", + "start_datetime": "items_start_datetime_new-collection_2020-02-08", + }, + ), + ( + { + "datetime": "items_datetime_new-collection_2020-02-14", + "end_datetime": "items_end_datetime_new-collection_2020-02-18", + "start_datetime": "items_start_datetime_new-collection_2020-02-10", + }, + ), + ] + datetime_search = { + "datetime": {"gte": "2020-02-12T00:00:00Z", "lte": "2020-02-28T23:59:59Z"}, + "start_datetime": {"gte": "2020-02-01T00:00:00Z", "lte": None}, + "end_datetime": {"gte": None, "lte": "2020-02-20T23:59:59Z"}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 2 + + +@pytest.mark.datetime_filtering +def test_filter_with_single_date_range(): + collection_indexes = [ + ( + { + "datetime": "items_datetime_new-collection_2020-02-12", + "end_datetime": "items_end_datetime_new-collection_2020-02-12", + "start_datetime": "items_start_datetime_new-collection_2020-02-12", + }, + ) + ] + datetime_search = { + "datetime": {"gte": "2020-02-12T00:00:00Z", "lte": "2020-02-12T23:59:59Z"}, + "start_datetime": {"gte": None, "lte": None}, + "end_datetime": {"gte": None, "lte": None}, + } + + result = filter_indexes_by_datetime(collection_indexes, datetime_search) + + assert len(result) == 1 From abf62c2fdf27f1419b1e69895767b69a4eb0a0d7 Mon Sep 17 00:00:00 2001 From: Grzegorz Pustulka Date: Sun, 23 Nov 2025 05:24:23 +0100 Subject: [PATCH 3/8] fix --- .../stac_fastapi/elasticsearch/database_logic.py | 2 +- .../stac_fastapi/sfeos_helpers/database/index.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 81c575bd8..10b956412 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -583,7 +583,7 @@ def apply_datetime_filter( ), ], ) - return search.query(filter_query), result_metadata + return search.query(filter_query), result_metadata @staticmethod def apply_bbox_filter(search: Search, bbox: List): diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py index 20dcbaf32..8f87e6c63 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py @@ -70,7 +70,7 @@ def indices(collection_ids: Optional[List[str]]) -> str: def filter_indexes_by_datetime( collection_indexes: List[Tuple[Dict[str, str], ...]], - datetime_search: Dict[str, Dict[str, str | None]], + datetime_search: Dict[str, Dict[str, Optional[str]]], ) -> List[str]: """ Filter Elasticsearch index aliases based on datetime search criteria. @@ -82,7 +82,7 @@ def filter_indexes_by_datetime( Args: collection_indexes (List[Tuple[Dict[str, str], ...]]): A list of tuples containing dictionaries with 'datetime', 'start_datetime', and 'end_datetime' aliases. - datetime_search (Dict[str, Dict[str, str | None]]): A dictionary with keys 'datetime', + datetime_search (Dict[str, Dict[str, Optional[str]]]): A dictionary with keys 'datetime', 'start_datetime', and 'end_datetime', each containing 'gte' and 'lte' criteria as ISO format datetime strings or None. @@ -90,7 +90,7 @@ def filter_indexes_by_datetime( List[str]: A list of start_datetime aliases that match all provided search criteria. """ - def extract_date_from_alias(alias: str) -> tuple[datetime, datetime] | None: + def extract_date_from_alias(alias: str) -> Optional[tuple[datetime, datetime]]: date_pattern = re.compile(r"\d{4}-\d{2}-\d{2}") try: dates = date_pattern.findall(alias) @@ -108,7 +108,7 @@ def extract_date_from_alias(alias: str) -> tuple[datetime, datetime] | None: except (ValueError, IndexError): return None - def parse_search_date(date_str: str | None) -> Optional[date]: + def parse_search_date(date_str: Optional[str]) -> Optional[date]: if not date_str: return None date_str = date_str.rstrip("Z") From 925a14a4d3407abe29a9a3ef78a2bbb7ea60ca44 Mon Sep 17 00:00:00 2001 From: Grzegorz Pustulka Date: Sun, 23 Nov 2025 05:34:54 +0100 Subject: [PATCH 4/8] fix --- stac_fastapi/elasticsearch/pyproject.toml | 1 + stac_fastapi/opensearch/pyproject.toml | 1 + .../stac_fastapi/sfeos_helpers/database/index.py | 6 +++--- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/elasticsearch/pyproject.toml b/stac_fastapi/elasticsearch/pyproject.toml index 26429e0e6..ce2496b13 100644 --- a/stac_fastapi/elasticsearch/pyproject.toml +++ b/stac_fastapi/elasticsearch/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "elasticsearch[async]~=8.19.1", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", + "redis==6.4.0", ] [project.urls] diff --git a/stac_fastapi/opensearch/pyproject.toml b/stac_fastapi/opensearch/pyproject.toml index c35537e03..03d7d3161 100644 --- a/stac_fastapi/opensearch/pyproject.toml +++ b/stac_fastapi/opensearch/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "opensearch-py[async]~=2.8.0", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", + "redis==6.4.0", ] [project.urls] diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py index 8f87e6c63..11433d41b 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py @@ -114,13 +114,13 @@ def parse_search_date(date_str: Optional[str]) -> Optional[date]: date_str = date_str.rstrip("Z") return datetime.fromisoformat(date_str).date() - def check_criteria(value_begin: date, value_end: date, criteria: Dict) -> bool: + def check_criteria(value_begin: datetime, value_end: datetime, criteria: Dict) -> bool: gte = parse_search_date(criteria.get("gte")) lte = parse_search_date(criteria.get("lte")) - if gte and value_end < gte: + if gte and value_end.date() < gte: return False - if lte and value_begin > lte: + if lte and value_begin.date() > lte: return False return True From 3de0aa3fc983acc254e1a5706dc401171afb2560 Mon Sep 17 00:00:00 2001 From: Grzegorz Pustulka Date: Sun, 23 Nov 2025 05:46:26 +0100 Subject: [PATCH 5/8] black --- .../stac_fastapi/sfeos_helpers/database/index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py index 11433d41b..347ff6ad7 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py @@ -114,7 +114,9 @@ def parse_search_date(date_str: Optional[str]) -> Optional[date]: date_str = date_str.rstrip("Z") return datetime.fromisoformat(date_str).date() - def check_criteria(value_begin: datetime, value_end: datetime, criteria: Dict) -> bool: + def check_criteria( + value_begin: datetime, value_end: datetime, criteria: Dict + ) -> bool: gte = parse_search_date(criteria.get("gte")) lte = parse_search_date(criteria.get("lte")) From eab070b8b69e9f8fa42cd5ee5782ab2b176d045d Mon Sep 17 00:00:00 2001 From: Grzegorz Pustulka Date: Fri, 28 Nov 2025 13:12:22 +0100 Subject: [PATCH 6/8] pre-commit --- .idea/workspace.xml | 63 ------------------- .../stac_fastapi/elasticsearch/version.py | 5 -- 2 files changed, 68 deletions(-) delete mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 46ed682a1..000000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,63 +0,0 @@ - - - - - - - - - - - - - { - "customColor": "", - "associatedIndex": 0 -} - - - - { - "keyToString": { - "Python.core.executor": "Run", - "RunOnceActivity.ShowReadmeOnStart": "true", - "RunOnceActivity.git.unshallow": "true", - "git-widget-placeholder": "main", - "last_opened_file_path": "/Users/gpusulka/Projects/stac-fastapi-elasticsearch-opensearch" - } -} - - - - - - - - - - - 1758823057239 - - - - - - - - - - \ No newline at end of file diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index 48c7bd93d..4a912c90b 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,7 +1,2 @@ """library version.""" -<<<<<<< HEAD __version__ = "6.7.5" -======= - -__version__ = "6.7.4" ->>>>>>> 798a2c5 (Add additional temporal aliases) From 39c38fe2a4d5862c0cfec3de8b649769c1344493 Mon Sep 17 00:00:00 2001 From: Grzegorz Pustulka Date: Tue, 2 Dec 2025 14:36:31 +0100 Subject: [PATCH 7/8] cr --- stac_fastapi/core/stac_fastapi/core/core.py | 4 +- stac_fastapi/elasticsearch/pyproject.toml | 1 - .../elasticsearch/database_logic.py | 34 +- stac_fastapi/opensearch/pyproject.toml | 1 - .../stac_fastapi/opensearch/database_logic.py | 34 +- .../sfeos_helpers/aggregation/client.py | 6 +- .../sfeos_helpers/database/index.py | 33 +- .../search_engine/index_operations.py | 64 +- .../sfeos_helpers/search_engine/inserters.py | 99 +- .../sfeos_helpers/search_engine/managers.py | 289 +-- .../search_engine/selection/base.py | 10 +- .../search_engine/selection/cache_manager.py | 11 - .../search_engine/selection/selectors.py | 71 +- stac_fastapi/tests/api/test_api.py | 1005 ---------- .../tests/api/test_api_datetime_filtering.py | 1633 +++++++++++++++++ stac_fastapi/tests/conftest.py | 10 + stac_fastapi/tests/database/test_database.py | 41 +- 17 files changed, 2057 insertions(+), 1289 deletions(-) create mode 100644 stac_fastapi/tests/api/test_api_datetime_filtering.py diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index b49121b2c..305ff8ff2 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -794,7 +794,7 @@ async def post_search( datetime_parsed = format_datetime_range(date_str=search_request.datetime) try: - search, datetime_search = self.database.apply_datetime_filter( + search = self.database.apply_datetime_filter( search=search, datetime=datetime_parsed ) except (ValueError, TypeError) as e: @@ -868,7 +868,7 @@ async def post_search( token=token_param, sort=sort, collection_ids=getattr(search_request, "collections", None), - datetime_search=datetime_search, + datetime_search=datetime_parsed, ) fields = getattr(search_request, "fields", None) diff --git a/stac_fastapi/elasticsearch/pyproject.toml b/stac_fastapi/elasticsearch/pyproject.toml index ce2496b13..26429e0e6 100644 --- a/stac_fastapi/elasticsearch/pyproject.toml +++ b/stac_fastapi/elasticsearch/pyproject.toml @@ -33,7 +33,6 @@ dependencies = [ "elasticsearch[async]~=8.19.1", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", - "redis==6.4.0", ] [project.urls] diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 10b956412..aaa3b1af2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -434,9 +434,7 @@ def apply_collections_filter(search: Search, collection_ids: List[str]): return search.filter("terms", collection=collection_ids) @staticmethod - def apply_datetime_filter( - search: Search, datetime: Optional[str] - ) -> Tuple[Search, Dict[str, Dict[str, Optional[str]]]]: + def apply_datetime_filter(search: Search, datetime: Optional[str]) -> Search: """Apply a filter to search on datetime, start_datetime, and end_datetime fields. Args: @@ -453,23 +451,8 @@ def apply_datetime_filter( # False: Always search only by start/end datetime USE_DATETIME = get_bool_env("USE_DATETIME", default=True) - result_metadata = { - "datetime": { - "gte": datetime_search.get("gte") if USE_DATETIME else None, - "lte": datetime_search.get("lte") if USE_DATETIME else None, - }, - "start_datetime": { - "gte": datetime_search.get("gte") if not USE_DATETIME else None, - "lte": None, - }, - "end_datetime": { - "gte": None, - "lte": datetime_search.get("lte") if not USE_DATETIME else None, - }, - } - if not datetime_search: - return search, result_metadata + return search if USE_DATETIME: if "eq" in datetime_search: @@ -546,10 +529,7 @@ def apply_datetime_filter( ), ] - return ( - search.query(Q("bool", should=should, minimum_should_match=1)), - result_metadata, - ) + return search.query(Q("bool", should=should, minimum_should_match=1)) else: if "eq" in datetime_search: filter_query = Q( @@ -583,7 +563,7 @@ def apply_datetime_filter( ), ], ) - return search.query(filter_query), result_metadata + return search.query(filter_query) @staticmethod def apply_bbox_filter(search: Search, bbox: List): @@ -738,7 +718,7 @@ async def execute_search( token: Optional[str], sort: Optional[Dict[str, Dict[str, str]]], collection_ids: Optional[List[str]], - datetime_search: Dict[str, Optional[str]], + datetime_search: str, ignore_unavailable: bool = True, ) -> Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]: """Execute a search query with limit and other optional parameters. @@ -749,7 +729,7 @@ async def execute_search( token (Optional[str]): The token used to return the next set of results. sort (Optional[Dict[str, Dict[str, str]]]): Specifies how the results should be sorted. collection_ids (Optional[List[str]]): The collection ids to search. - datetime_search (Dict[str, Optional[str]]): Datetime range used for index selection. + datetime_search (str): Datetime used for index selection. ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True. Returns: @@ -839,7 +819,7 @@ async def aggregate( geometry_geohash_grid_precision: int, geometry_geotile_grid_precision: int, datetime_frequency_interval: str, - datetime_search, + datetime_search: str, ignore_unavailable: Optional[bool] = True, ): """Return aggregations of STAC Items.""" diff --git a/stac_fastapi/opensearch/pyproject.toml b/stac_fastapi/opensearch/pyproject.toml index 03d7d3161..c35537e03 100644 --- a/stac_fastapi/opensearch/pyproject.toml +++ b/stac_fastapi/opensearch/pyproject.toml @@ -34,7 +34,6 @@ dependencies = [ "opensearch-py[async]~=2.8.0", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", - "redis==6.4.0", ] [project.urls] diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index a3e6a061f..3ffebcff9 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -457,9 +457,7 @@ def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str] ) @staticmethod - def apply_datetime_filter( - search: Search, datetime: Optional[str] - ) -> Tuple[Search, Dict[str, Dict[str, Optional[str]]]]: + def apply_datetime_filter(search: Search, datetime: Optional[str]) -> Search: """Apply a filter to search on datetime, start_datetime, and end_datetime fields. Args: @@ -476,23 +474,8 @@ def apply_datetime_filter( # False: Always search only by start/end datetime USE_DATETIME = get_bool_env("USE_DATETIME", default=True) - result_metadata = { - "datetime": { - "gte": datetime_search.get("gte") if USE_DATETIME else None, - "lte": datetime_search.get("lte") if USE_DATETIME else None, - }, - "start_datetime": { - "gte": datetime_search.get("gte") if not USE_DATETIME else None, - "lte": None, - }, - "end_datetime": { - "gte": None, - "lte": datetime_search.get("lte") if not USE_DATETIME else None, - }, - } - if not datetime_search: - return search, result_metadata + return search if USE_DATETIME: if "eq" in datetime_search: @@ -569,10 +552,7 @@ def apply_datetime_filter( ), ] - return ( - search.query(Q("bool", should=should, minimum_should_match=1)), - result_metadata, - ) + return search.query(Q("bool", should=should, minimum_should_match=1)) else: if "eq" in datetime_search: filter_query = Q( @@ -606,7 +586,7 @@ def apply_datetime_filter( ), ], ) - return search.query(filter_query), result_metadata + return search.query(filter_query) @staticmethod def apply_bbox_filter(search: Search, bbox: List): @@ -743,7 +723,7 @@ async def execute_search( token: Optional[str], sort: Optional[Dict[str, Dict[str, str]]], collection_ids: Optional[List[str]], - datetime_search: Dict[str, Optional[str]], + datetime_search: str, ignore_unavailable: bool = True, ) -> Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]: """Execute a search query with limit and other optional parameters. @@ -754,7 +734,7 @@ async def execute_search( token (Optional[str]): The token used to return the next set of results. sort (Optional[Dict[str, Dict[str, str]]]): Specifies how the results should be sorted. collection_ids (Optional[List[str]]): The collection ids to search. - datetime_search (Dict[str, Optional[str]]): Datetime range used for index selection. + datetime_search (str): Datetime used for index selection. ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True. Returns: @@ -850,7 +830,7 @@ async def aggregate( geometry_geohash_grid_precision: int, geometry_geotile_grid_precision: int, datetime_frequency_interval: str, - datetime_search, + datetime_search: str, ignore_unavailable: Optional[bool] = True, ): """Return aggregations of STAC Items.""" diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py index 1f77cd9ee..a75d575d7 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/aggregation/client.py @@ -313,11 +313,9 @@ async def aggregate( ) if aggregate_request.datetime: - search, datetime_search = self.database.apply_datetime_filter( + search = self.database.apply_datetime_filter( search=search, datetime=aggregate_request.datetime ) - else: - datetime_search = {"gte": None, "lte": None} if aggregate_request.bbox: bbox = aggregate_request.bbox @@ -416,7 +414,7 @@ async def aggregate( geometry_geohash_grid_precision, geometry_geotile_grid_precision, datetime_frequency_interval, - datetime_search, + aggregate_request.datetime, ) except Exception as error: if not isinstance(error, IndexError): diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py index 347ff6ad7..19a995c5e 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/index.py @@ -71,6 +71,7 @@ def indices(collection_ids: Optional[List[str]]) -> str: def filter_indexes_by_datetime( collection_indexes: List[Tuple[Dict[str, str], ...]], datetime_search: Dict[str, Dict[str, Optional[str]]], + use_datetime: bool, ) -> List[str]: """ Filter Elasticsearch index aliases based on datetime search criteria. @@ -85,6 +86,9 @@ def filter_indexes_by_datetime( datetime_search (Dict[str, Dict[str, Optional[str]]]): A dictionary with keys 'datetime', 'start_datetime', and 'end_datetime', each containing 'gte' and 'lte' criteria as ISO format datetime strings or None. + use_datetime (bool): Flag determining which datetime field to filter on: + - True: Filters using 'datetime' alias. + - False: Filters using 'start_datetime' and 'end_datetime' aliases. Returns: List[str]: A list of start_datetime aliases that match all provided search criteria. @@ -138,29 +142,30 @@ def check_criteria( end_datetime_alias = index_dict.get("end_datetime") datetime_alias = index_dict.get("datetime") - if not start_datetime_alias: - continue - - start_range = extract_date_from_alias(start_datetime_alias) - end_date = extract_date_from_alias(end_datetime_alias) - datetime_date = extract_date_from_alias(datetime_alias) - - if not check_criteria( - start_range[0], start_range[1], datetime_search.get("start_datetime", {}) - ): - continue - if end_date: + if start_datetime_alias: + start_date = extract_date_from_alias(start_datetime_alias) + if not check_criteria( + start_date[0], start_date[1], datetime_search.get("start_datetime", {}) + ): + continue + if end_datetime_alias: + end_date = extract_date_from_alias(end_datetime_alias) if not check_criteria( end_date[0], end_date[1], datetime_search.get("end_datetime", {}) ): continue - if datetime_date: + if datetime_alias: + datetime_date = extract_date_from_alias(datetime_alias) if not check_criteria( datetime_date[0], datetime_date[1], datetime_search.get("datetime", {}) ): continue - filtered_indexes.append(start_datetime_alias) + primary_datetime_alias = ( + datetime_alias if use_datetime else start_datetime_alias + ) + + filtered_indexes.append(primary_datetime_alias) return filtered_indexes diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py index f3e033fdd..541c883cf 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/index_operations.py @@ -3,6 +3,7 @@ import uuid from typing import Any, Dict, List, Literal +from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.sfeos_helpers.database import ( index_alias_by_collection_id, index_by_collection_id, @@ -18,6 +19,16 @@ class IndexOperations: """Base class for search engine adapters with common implementations.""" + @property + def use_datetime(self) -> bool: + """Get USE_DATETIME setting dynamically.""" + return get_bool_env("USE_DATETIME", default=True) + + @property + def primary_datetime_name(self) -> str: + """Get primary datetime field name based on current USE_DATETIME setting.""" + return "datetime" if self.use_datetime else "start_datetime" + async def create_simple_index(self, client: Any, collection_id: str) -> str: """Create a simple index for the given collection. @@ -42,44 +53,48 @@ async def create_datetime_index( self, client: Any, collection_id: str, - start_datetime: str, - datetime: str, - end_datetime: str, + start_datetime: str | None, + datetime: str | None, + end_datetime: str | None, ) -> str: """Create a datetime-based index for the given collection. Args: client: Search engine client instance. collection_id (str): Collection identifier. - start_datetime (str): Start datetime for the index alias. - datetime (str): Datetime for the datetime alias (can be None). - end_datetime (str): End datetime for the index alias. + start_datetime (str | None): Start datetime for the index alias. + datetime (str | None): Datetime for the datetime alias. + end_datetime (str | None): End datetime for the index alias. Returns: - str: Created start_datetime alias name. + str: Created datetime alias name. """ index_name = self.create_index_name(collection_id) - alias_start_date = self.create_alias_name( - collection_id, "start_datetime", start_datetime - ) - alias_date = self.create_alias_name(collection_id, "datetime", datetime) - alias_end_date = self.create_alias_name( - collection_id, "end_datetime", end_datetime - ) collection_alias = index_alias_by_collection_id(collection_id) aliases: Dict[str, Any] = { collection_alias: {}, - alias_start_date: {}, - alias_date: {}, - alias_end_date: {}, } + if start_datetime: + alias_start_date = self.create_alias_name( + collection_id, "start_datetime", start_datetime + ) + alias_end_date = self.create_alias_name( + collection_id, "end_datetime", end_datetime + ) + aliases[alias_start_date] = {} + aliases[alias_end_date] = {} + created_alias = alias_start_date + else: + created_alias = self.create_alias_name(collection_id, "datetime", datetime) + aliases[created_alias] = {} + await client.indices.create( index=index_name, body=self._create_index_body(aliases), ) - return alias_start_date + return created_alias @staticmethod async def update_index_alias(client: Any, end_date: str, old_alias: str) -> str: @@ -126,12 +141,12 @@ async def change_alias_name( index_name = list(aliases_info.keys())[0] actions = [] - for old_alias in aliases_to_change: - actions.append({"remove": {"index": index_name, "alias": old_alias}}) - for new_alias in aliases_to_create: actions.append({"add": {"index": index_name, "alias": new_alias}}) + for old_alias in aliases_to_change: + actions.append({"remove": {"index": index_name, "alias": old_alias}}) + await client.indices.update_aliases(body={"actions": actions}) @staticmethod @@ -182,8 +197,9 @@ def _create_index_body(aliases: Dict[str, Dict]) -> Dict[str, Any]: "settings": ES_ITEMS_SETTINGS, } - @staticmethod - async def find_latest_item_in_index(client: Any, index_name: str) -> dict[str, Any]: + async def find_latest_item_in_index( + self, client: Any, index_name: str + ) -> dict[str, Any]: """Find the latest item in the specified index. Args: @@ -195,7 +211,7 @@ async def find_latest_item_in_index(client: Any, index_name: str) -> dict[str, A """ query = { "size": 1, - "sort": [{"properties.start_datetime": {"order": "desc"}}], + "sort": [{f"properties.{self.primary_datetime_name}": {"order": "desc"}}], "_source": [ "properties.start_datetime", "properties.datetime", diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py index 310f89516..14e831910 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/inserters.py @@ -5,6 +5,7 @@ from fastapi import HTTPException, status +from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.sfeos_helpers.database import ( extract_date, extract_first_date_from_index, @@ -34,6 +35,24 @@ def __init__(self, client: Any, index_operations: IndexOperations): self.index_operations = index_operations self.datetime_manager = DatetimeIndexManager(client, index_operations) + @property + def use_datetime(self) -> bool: + """Get USE_DATETIME setting dynamically. + + Returns: + bool: Current value of USE_DATETIME environment variable. + """ + return get_bool_env("USE_DATETIME", default=True) + + @property + def primary_datetime_name(self) -> str: + """Get primary datetime field name based on current USE_DATETIME setting. + + Returns: + str: "datetime" if USE_DATETIME is True, else "start_datetime". + """ + return "datetime" if self.use_datetime else "start_datetime" + @staticmethod def should_create_collection_index() -> bool: """Whether this strategy requires collection index creation. @@ -89,7 +108,7 @@ async def prepare_bulk_actions( logger.error(msg) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=msg) - items.sort(key=lambda item: item["properties"]["start_datetime"]) + items.sort(key=lambda item: item["properties"][self.primary_datetime_name]) index_selector = DatetimeBasedIndexSelector(self.client) await self._ensure_indexes_exist(index_selector, collection_id, items) @@ -130,43 +149,55 @@ async def _get_target_index_internal( Returns: str: Target index name. """ - product_datetimes = self.datetime_manager.validate_product_datetimes(product) + product_datetimes = self.datetime_manager.validate_product_datetimes( + product, self.use_datetime + ) - datetime_ranges = { - "start_datetime": {"gte": None, "lte": product_datetimes.start_datetime}, - "end_datetime": {"gte": None, "lte": None}, - "datetime": {"gte": None, "lte": None}, - } + primary_datetime_value = ( + product_datetimes.datetime + if self.use_datetime + else product_datetimes.start_datetime + ) target_index = await index_selector.select_indexes( - [collection_id], datetime_ranges + [collection_id], primary_datetime_value, for_insertion=True ) all_indexes = await index_selector.get_collection_indexes(collection_id) if not all_indexes: target_index = await self.datetime_manager.handle_new_collection( - collection_id, product_datetimes + collection_id, self.primary_datetime_name, product_datetimes ) await index_selector.refresh_cache() return target_index - all_indexes = sorted(all_indexes, key=lambda x: x[0]["start_datetime"]) - start_date = extract_date(product_datetimes.start_datetime) - end_date = extract_first_date_from_index(all_indexes[0][0]["start_datetime"]) + all_indexes = sorted( + all_indexes, key=lambda x: x[0][self.primary_datetime_name] + ) + start_date = extract_date(primary_datetime_value) + end_date = extract_first_date_from_index( + all_indexes[0][0][self.primary_datetime_name] + ) if start_date < end_date: alias = await self.datetime_manager.handle_early_date( - collection_id, product_datetimes, all_indexes[0][0] + collection_id, + self.primary_datetime_name, + product_datetimes, + all_indexes[0][0], ) await index_selector.refresh_cache() return alias - if target_index != all_indexes[-1][0]["start_datetime"]: + if target_index != all_indexes[-1][0][self.primary_datetime_name]: for item in all_indexes: aliases_dict = item[0] if target_index in aliases_dict.values(): await self.datetime_manager.handle_early_date( - collection_id, product_datetimes, aliases_dict + collection_id, + self.primary_datetime_name, + product_datetimes, + aliases_dict, ) return target_index @@ -177,7 +208,10 @@ async def _get_target_index_internal( aliases_dict = item[0] if target_index in aliases_dict.values(): target_index = await self.datetime_manager.handle_oversized_index( - collection_id, product_datetimes, aliases_dict + collection_id, + self.primary_datetime_name, + product_datetimes, + aliases_dict, ) await index_selector.refresh_cache() return target_index @@ -186,7 +220,10 @@ async def _get_target_index_internal( aliases_dict = item[0] if target_index in aliases_dict.values(): await self.datetime_manager.handle_early_date( - collection_id, product_datetimes, aliases_dict + collection_id, + self.primary_datetime_name, + product_datetimes, + aliases_dict, ) return target_index return None @@ -205,12 +242,23 @@ async def _ensure_indexes_exist( if not all_indexes: first_item = items[0] + properties = first_item["properties"] + index_params = { + "start_datetime": str(extract_date(properties["start_datetime"])) + if self.primary_datetime_name == "start_datetime" + else None, + "datetime": str(extract_date(properties["datetime"])) + if self.primary_datetime_name == "datetime" + else None, + "end_datetime": str(extract_date(properties["end_datetime"])) + if self.primary_datetime_name == "start_datetime" + else None, + } + await self.index_operations.create_datetime_index( self.client, collection_id, - str(extract_date(first_item["properties"]["start_datetime"])), - str(extract_date(first_item["properties"]["datetime"])), - str(extract_date(first_item["properties"]["end_datetime"])), + **index_params, ) await index_selector.refresh_cache() @@ -236,9 +284,11 @@ async def _check_and_handle_oversized_index( ) all_indexes = await index_selector.get_collection_indexes(collection_id) - all_indexes = sorted(all_indexes, key=lambda x: x[0]["start_datetime"]) + all_indexes = sorted( + all_indexes, key=lambda x: x[0][self.primary_datetime_name] + ) - latest_index = all_indexes[-1][0]["start_datetime"] + latest_index = all_indexes[-1][0][self.primary_datetime_name] if first_item_index != latest_index: return None @@ -258,7 +308,10 @@ async def _check_and_handle_oversized_index( ) await self.datetime_manager.handle_oversized_index( - collection_id, product_datetimes, all_indexes[-1][0] + collection_id, + self.primary_datetime_name, + product_datetimes, + all_indexes[-1][0], ) await index_selector.refresh_cache() diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py index d0b1b0ab6..bb6d4d9c6 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/managers.py @@ -5,6 +5,7 @@ from datetime import timedelta from typing import Any, Dict, NamedTuple +from dateutil import parser # type: ignore from fastapi import HTTPException, status from stac_fastapi.sfeos_helpers.database import ( @@ -21,14 +22,14 @@ class ProductDatetimes(NamedTuple): """Named tuple representing product datetime fields. Attributes: - start_datetime (str): ISO format start datetime string. - datetime (str): ISO format datetime string. - end_datetime (str): ISO format end datetime string. + start_datetime (str | None): ISO format start datetime string or None. + datetime (str | None): ISO format datetime string or None. + end_datetime (str | None): ISO format end datetime string or None. """ - start_datetime: str - datetime: str - end_datetime: str + start_datetime: str | None + datetime: str | None + end_datetime: str | None class IndexSizeManager: @@ -122,67 +123,102 @@ def __init__(self, client: Any, index_operations: IndexOperations): self.size_manager = IndexSizeManager(client) @staticmethod - def validate_product_datetimes(product: Dict[str, Any]) -> ProductDatetimes: + def validate_product_datetimes( + product: Dict[str, Any], use_datetime + ) -> ProductDatetimes: """Validate and extract datetime fields from product. + Validation rules depend on USE_DATETIME: + - USE_DATETIME=True: 'datetime' is required, optional start/end + - USE_DATETIME=False: both 'start_datetime' and 'end_datetime' required, start <= end + Args: product (Dict[str, Any]): Product data containing datetime information. + use_datetime (bool): Flag determining validation mode. + - True: validates against 'datetime' field. + - False: validates against 'start_datetime' and 'end_datetime' fields. Returns: - ProductDatetimes: Named tuple containing: - - start_datetime (str): Start datetime value - - datetime (str): datetime value - - end_datetime (str): End datetime value + ProductDatetimes: Named tuple containing parsed datetime values: + - start_datetime (str | None): ISO 8601 start datetime string or None. + - datetime (str | None): ISO 8601 datetime string or None. + - end_datetime (str | None): ISO 8601 end datetime string or None. Raises: - HTTPException: If product start_datetime is missing or invalid. + HTTPException: If validation fails based on USE_DATETIME configuration. """ properties = product.get("properties", {}) - start_datetime_value = properties.get("start_datetime") - datetime_value = properties.get("datetime") - end_datetime_value = properties.get("end_datetime") - - if not start_datetime_value or not datetime_value or not end_datetime_value: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Product 'start_datetime', 'datetime' and 'end_datetime' is required for indexing", - ) - - if not (start_datetime_value <= datetime_value <= end_datetime_value): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="'start_datetime' <= 'datetime' <= 'end_datetime' is required", - ) + start_str = properties.get("start_datetime") + dt_str = properties.get("datetime") + end_str = properties.get("end_datetime") + + start = parser.isoparse(start_str) if start_str else None + dt = parser.isoparse(dt_str) if dt_str else None + end = parser.isoparse(end_str) if end_str else None + + if use_datetime: + if not dt: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="'datetime' field is required", + ) + else: + if not start or not end: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Both 'start_datetime' and 'end_datetime' fields are required", + ) + if not (start <= end): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="'start_datetime' must be <= 'end_datetime'", + ) + if dt and not (start <= dt <= end): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="'start_datetime' <= 'datetime' <= 'end_datetime' is required", + ) return ProductDatetimes( - start_datetime=start_datetime_value, - datetime=datetime_value, - end_datetime=end_datetime_value, + start_datetime=start_str, + datetime=dt_str, + end_datetime=end_str, ) async def handle_new_collection( - self, collection_id: str, product_datetimes: ProductDatetimes + self, + collection_id: str, + primary_datetime_name: str, + product_datetimes: ProductDatetimes, ) -> str: """Handle index creation for new collection asynchronously. Args: collection_id (str): Collection identifier. + primary_datetime_name (str): Name of the primary datetime field. + If "start_datetime", indexes are created on start_datetime and end_datetime fields. + If "datetime", indexes are created on the datetime field. product_datetimes (ProductDatetimes): Object containing start_datetime, datetime, and end_datetime. Returns: - str: Created start_datetime alias name. + str: Created datetime index name. """ - start_datetime = extract_date(product_datetimes.start_datetime) - datetime = extract_date(product_datetimes.datetime) - end_datetime = extract_date(product_datetimes.end_datetime) + index_params = { + "start_datetime": str(extract_date(product_datetimes.start_datetime)) + if primary_datetime_name == "start_datetime" + else None, + "datetime": str(extract_date(product_datetimes.datetime)) + if primary_datetime_name == "datetime" + else None, + "end_datetime": str(extract_date(product_datetimes.end_datetime)) + if primary_datetime_name == "start_datetime" + else None, + } target_index = await self.index_operations.create_datetime_index( - self.client, - collection_id, - str(start_datetime), - str(datetime), - str(end_datetime), + self.client, collection_id, **index_params ) + logger.info( f"Successfully created index '{target_index}' for collection '{collection_id}'" ) @@ -191,6 +227,7 @@ async def handle_new_collection( async def handle_early_date( self, collection_id: str, + primary_datetime_name: str, product_datetimes: ProductDatetimes, old_aliases: Dict[str, str], ) -> str: @@ -198,61 +235,70 @@ async def handle_early_date( Args: collection_id (str): Collection identifier. + primary_datetime_name (str): Name of the primary datetime field. + If "start_datetime", handles start_datetime and end_datetime fields. + If "datetime", handles the datetime field. product_datetimes (ProductDatetimes): Object containing start_datetime, datetime, and end_datetime. old_aliases (Dict[str, str]): Dictionary mapping alias types to their current names. Returns: - str: Updated start_datetime alias name. + str: Updated datetime alias name. """ - start_dt = extract_date(product_datetimes.start_datetime) - dt = extract_date(product_datetimes.datetime) - end_dt = extract_date(product_datetimes.end_datetime) + new_aliases = [] + old_alias_names = [] - old_start_datetime_alias = extract_first_date_from_index( - old_aliases["start_datetime"] - ) - old_datetime_alias = extract_first_date_from_index(old_aliases["datetime"]) - old_end_datetime_alias = extract_first_date_from_index( - old_aliases["end_datetime"] - ) + if primary_datetime_name == "start_datetime": + new_start_alias = self.index_operations.create_alias_name( + collection_id, + "start_datetime", + str(extract_date(product_datetimes.start_datetime)), + ) - new_start_datetime_alias = self.index_operations.create_alias_name( - collection_id, "start_datetime", str(start_dt) - ) + if extract_date( + product_datetimes.start_datetime + ) < extract_first_date_from_index(old_aliases["start_datetime"]): + new_aliases.append(new_start_alias) + old_alias_names.append(old_aliases["start_datetime"]) - aliases_to_change = [] - aliases_to_create = [] + new_end_alias = self.index_operations.create_alias_name( + collection_id, + "end_datetime", + str(extract_date(product_datetimes.end_datetime)), + ) - if start_dt < old_start_datetime_alias: - aliases_to_create.append(new_start_datetime_alias) - aliases_to_change.append(old_aliases["start_datetime"]) + if extract_date( + product_datetimes.end_datetime + ) > extract_first_date_from_index(old_aliases["end_datetime"]): + new_aliases.append(new_end_alias) + old_alias_names.append(old_aliases["end_datetime"]) - if dt > old_datetime_alias: - new_datetime_alias = self.index_operations.create_alias_name( - collection_id, "datetime", str(dt) - ) - aliases_to_create.append(new_datetime_alias) - aliases_to_change.append(old_aliases["datetime"]) + new_primary_alias = new_start_alias + else: - if end_dt > old_end_datetime_alias: - new_end_datetime_alias = self.index_operations.create_alias_name( - collection_id, "end_datetime", str(end_dt) + new_primary_alias = self.index_operations.create_alias_name( + collection_id, "datetime", str(extract_date(product_datetimes.datetime)) ) - aliases_to_create.append(new_end_datetime_alias) - aliases_to_change.append(old_aliases["end_datetime"]) - if aliases_to_change: + if extract_date(product_datetimes.datetime) < extract_first_date_from_index( + old_aliases["datetime"] + ): + new_aliases.append(new_primary_alias) + old_alias_names.append(old_aliases["datetime"]) + + if old_alias_names: await self.index_operations.change_alias_name( self.client, - old_aliases["start_datetime"], - aliases_to_change, - aliases_to_create, + old_aliases[primary_datetime_name], + old_alias_names, + new_aliases, ) - return new_start_datetime_alias + + return new_primary_alias async def handle_oversized_index( self, collection_id: str, + primary_datetime_name: str, product_datetimes: ProductDatetimes, old_aliases: Dict[str, str], ) -> str: @@ -260,54 +306,65 @@ async def handle_oversized_index( Args: collection_id (str): Collection identifier. + primary_datetime_name (str): Name of the primary datetime field. + If "start_datetime", handles start_datetime and end_datetime fields. + If "datetime", handles the datetime field. product_datetimes (ProductDatetimes): Object containing start_datetime, datetime, and end_datetime. old_aliases (Dict[str, str]): Dictionary mapping alias types to their current names. Returns: - str: Updated or newly created start_datetime alias name. + str: Updated or newly created datetime alias name. """ - target_index = old_aliases["start_datetime"] - start_dt = extract_date(product_datetimes.start_datetime) - dt = extract_date(product_datetimes.datetime) - end_dt = extract_date(product_datetimes.end_datetime) - - old_start_datetime_alias = extract_first_date_from_index(target_index) - old_datetime_alias = extract_first_date_from_index(old_aliases["datetime"]) - old_end_datetime_alias = extract_first_date_from_index( - old_aliases["end_datetime"] - ) - - if start_dt != old_start_datetime_alias: - aliases_to_change = [] - aliases_to_create = [] - - new_start_datetime_alias = f"{target_index}-{str(start_dt)}" - aliases_to_create.append(new_start_datetime_alias) - aliases_to_change.append(target_index) - - if dt > old_datetime_alias: - new_datetime_alias = self.index_operations.create_alias_name( - collection_id, "datetime", str(dt) + current_alias = old_aliases[primary_datetime_name] + new_aliases = [] + old_alias_names = [] + + if primary_datetime_name == "start_datetime": + start_dt = extract_date(product_datetimes.start_datetime) + end_dt = extract_date(product_datetimes.end_datetime) + old_start_dt = extract_first_date_from_index(current_alias) + old_end_dt = extract_first_date_from_index(old_aliases["end_datetime"]) + + if start_dt != old_start_dt: + new_start_alias = f"{current_alias}-{str(start_dt)}" + new_aliases.append(new_start_alias) + old_alias_names.append(current_alias) + + if end_dt > old_end_dt: + new_end_alias = self.index_operations.create_alias_name( + collection_id, "end_datetime", str(end_dt) ) - aliases_to_create.append(new_datetime_alias) - aliases_to_change.append(old_aliases["datetime"]) + new_aliases.append(new_end_alias) + old_alias_names.append(old_aliases["end_datetime"]) - if end_dt > old_end_datetime_alias: - new_end_datetime_alias = self.index_operations.create_alias_name( - collection_id, "end_datetime", str(end_dt) + if old_alias_names: + await self.index_operations.change_alias_name( + self.client, current_alias, old_alias_names, new_aliases ) - aliases_to_create.append(new_end_datetime_alias) - aliases_to_change.append(old_aliases["end_datetime"]) - await self.index_operations.change_alias_name( - self.client, target_index, aliases_to_change, aliases_to_create - ) - target_index = await self.index_operations.create_datetime_index( - self.client, - collection_id, - str(start_dt + timedelta(days=1)), - str(dt), - str(end_dt), - ) + if start_dt != old_start_dt: + return await self.index_operations.create_datetime_index( + self.client, + collection_id, + start_datetime=str(start_dt + timedelta(days=1)), + datetime=None, + end_datetime=str(end_dt), + ) + else: + dt = extract_date(product_datetimes.datetime) + old_dt = extract_first_date_from_index(current_alias) + + if dt != old_dt: + new_datetime_alias = f"{current_alias}-{str(dt)}" + await self.index_operations.change_alias_name( + self.client, current_alias, [current_alias], [new_datetime_alias] + ) + return await self.index_operations.create_datetime_index( + self.client, + collection_id, + start_datetime=None, + datetime=str(dt + timedelta(days=1)), + end_datetime=None, + ) - return target_index + return current_alias diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py index aa9fc7edb..6bc0cba93 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/base.py @@ -1,7 +1,7 @@ """Base classes for index selection strategies.""" from abc import ABC, abstractmethod -from typing import Dict, List, Optional +from typing import List, Optional class BaseIndexSelector(ABC): @@ -11,13 +11,17 @@ class BaseIndexSelector(ABC): async def select_indexes( self, collection_ids: Optional[List[str]], - datetime_search: Dict[str, dict[str, str]], + datetime_search: str, + for_insertion: bool = False, ) -> str: """Select appropriate indexes asynchronously. Args: collection_ids (Optional[List[str]]): List of collection IDs to filter by. - datetime_search (Dict[str, Optional[str]]): Datetime search criteria. + datetime_search (str): Datetime search criteria. + for_insertion (bool): If True, selects indexes for inserting items into + the database. If False, selects indexes for searching/querying items. + Defaults to False (search mode). Returns: str: Comma-separated string of selected index names. diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py index 135cd5b0e..63146dcd8 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py @@ -78,17 +78,6 @@ async def load_aliases(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: Returns: Dict[str, List[Tuple[Dict[str, str]]]]: Mapping of main collection aliases to their data. - - Example: - { - 'items_sentinel-2-l1c': [ - ({ - 'datetime': 'items_datetime_sentinel-2-l1c_2025-09-27', - 'start_datetime': 'items_start_datetime_sentinel-2-l1c_2025-09-27', - 'end_datetime': 'items_end_datetime_sentinel-2-l1c_2025-09-28' - },) - ] - } """ response = await self.client.indices.get_alias(index=f"{ITEMS_INDEX_PREFIX}*") result: Dict[str, List[Tuple[Dict[str, str]]]] = {} diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py index b3a2d14ce..b53d8aa6d 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py @@ -2,7 +2,8 @@ from typing import Any, Dict, List, Optional, Tuple -from stac_fastapi.sfeos_helpers.database import filter_indexes_by_datetime +from stac_fastapi.core.utilities import get_bool_env +from stac_fastapi.sfeos_helpers.database import filter_indexes_by_datetime, return_date from stac_fastapi.sfeos_helpers.mappings import ITEM_INDICES from ...database import indices @@ -40,6 +41,11 @@ def __init__(self, client: Any): self.alias_loader = IndexAliasLoader(client, self.cache_manager) self._initialized = True + @property + def use_datetime(self) -> bool: + """Get USE_DATETIME setting dynamically.""" + return get_bool_env("USE_DATETIME", default=True) + async def refresh_cache(self) -> Dict[str, List[Tuple[Dict[str, str]]]]: """Force refresh of the aliases cache. @@ -66,7 +72,8 @@ async def get_collection_indexes( async def select_indexes( self, collection_ids: Optional[List[str]], - datetime_search: Dict[str, dict[str, str]], + datetime_search: str, + for_insertion: bool = False, ) -> str: """Select indexes filtered by collection IDs and datetime criteria. @@ -77,20 +84,23 @@ async def select_indexes( Args: collection_ids (Optional[List[str]]): List of collection IDs to filter by. If None or empty, returns all item indices. - datetime_search (Dict[str, dict[str, str]]): Dictionary containing datetime - search criteria with 'gte' and 'lte' keys for range filtering. + datetime_search (str): Datetime search criteria. + for_insertion (bool): If True, selects indexes for inserting items into + the database. If False, selects indexes for searching/querying items. + Defaults to False (search mode). Returns: str: Comma-separated string of selected index names that match the collection and datetime criteria. Returns empty string if no indexes match the criteria. """ + datetime_filters = self.parse_datetime_filters(datetime_search, for_insertion) if collection_ids: selected_indexes = [] for collection_id in collection_ids: collection_indexes = await self.get_collection_indexes(collection_id) filtered_indexes = filter_indexes_by_datetime( - collection_indexes, datetime_search + collection_indexes, datetime_filters, self.use_datetime ) selected_indexes.extend(filtered_indexes) @@ -98,6 +108,49 @@ async def select_indexes( return ITEM_INDICES + def parse_datetime_filters( + self, datetime: str, for_insertion: bool + ) -> Dict[str, Dict[str, Optional[str]]]: + """Parse datetime string into structured filter criteria. + + Args: + datetime: Datetime search criteria string + for_insertion (bool): If True, generates filters for inserting items. + If False, generates filters for searching items. Defaults to False. + + Returns: + Dictionary with datetime, start_datetime, and end_datetime filters + """ + parsed_datetime = return_date(datetime) + + if for_insertion: + return { + "datetime": { + "gte": None, + "lte": datetime if self.use_datetime else None, + }, + "start_datetime": { + "gte": None, + "lte": datetime if not self.use_datetime else None, + }, + "end_datetime": {"gte": None, "lte": None}, + } + + return { + "datetime": { + "gte": parsed_datetime.get("gte") if self.use_datetime else None, + "lte": parsed_datetime.get("lte") if self.use_datetime else None, + }, + "start_datetime": { + "gte": parsed_datetime.get("gte") if not self.use_datetime else None, + "lte": None, + }, + "end_datetime": { + "gte": None, + "lte": parsed_datetime.get("lte") if not self.use_datetime else None, + }, + } + class UnfilteredIndexSelector(BaseIndexSelector): """Index selector that returns all available indices without filtering.""" @@ -105,15 +158,19 @@ class UnfilteredIndexSelector(BaseIndexSelector): async def select_indexes( self, collection_ids: Optional[List[str]], - datetime_search: Dict[str, dict[str, str]], + datetime_search: str, + for_insertion: bool = False, ) -> str: """Select all indices for given collections without datetime filtering. Args: collection_ids (Optional[List[str]]): List of collection IDs to filter by. If None, all collections are considered. - datetime_search (Dict[str, Optional[str]]): Datetime search criteria + datetime_search (str): Datetime search criteria (ignored by this implementation). + for_insertion (bool): If True, selects indexes for inserting items into + the database. If False, selects indexes for searching/querying items. + Defaults to False (search mode). Returns: str: Comma-separated string of all available index names for the collections. diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 1c0f4c402..c40cf0b90 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -3,7 +3,6 @@ import uuid from copy import deepcopy from datetime import datetime, timedelta -from unittest.mock import patch import pytest @@ -845,647 +844,6 @@ async def test_big_int_eo_search( assert results == {value} -# @pytest.mark.datetime_filtering -# @pytest.mark.asyncio -# async def test_create_item_in_past_date_change_alias_name_for_datetime_index( -# app_client, ctx, load_test_data, txn_client -# ): -# if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): -# pytest.skip() -# -# item = load_test_data("test_item.json") -# item["id"] = str(uuid.uuid4()) -# item["properties"]["start_datetime"] = "2012-02-12T12:30:22Z" -# -# response = await app_client.post( -# f"/collections/{item['collection']}/items", json=item -# ) -# assert response.status_code == 201 -# indices = await txn_client.database.client.indices.get_alias( -# index="items_test-collection" -# ) -# expected_aliases = [ -# "items_start_datetime_test-collection_2012-02-12", -# "items_datetime_test-collection_2020-02-12", -# "items_end_datetime_test-collection_2020-02-16", -# ] -# all_aliases = set() -# for index_info in indices.values(): -# all_aliases.update(index_info.get("aliases", {}).keys()) -# -# assert all(alias in all_aliases for alias in expected_aliases) -# -# -# @pytest.mark.datetime_filtering -# @pytest.mark.asyncio -# async def test_create_item_uses_existing_datetime_index_for_datetime_index( -# app_client, ctx, load_test_data, txn_client -# ): -# if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): -# pytest.skip() -# -# item = load_test_data("test_item.json") -# item["id"] = str(uuid.uuid4()) -# -# response = await app_client.post( -# f"/collections/{item['collection']}/items", json=item -# ) -# -# assert response.status_code == 201 -# -# indices = await txn_client.database.client.indices.get_alias( -# index="items_test-collection" -# ) -# expected_aliases = [ -# "items_start_datetime_test-collection_2020-02-08", -# "items_datetime_test-collection_2020-02-12", -# "items_end_datetime_test-collection_2020-02-16", -# ] -# all_aliases = set() -# for index_info in indices.values(): -# all_aliases.update(index_info.get("aliases", {}).keys()) -# assert all(alias in all_aliases for alias in expected_aliases) -# -# -# @pytest.mark.datetime_filtering -# @pytest.mark.asyncio -# async def test_create_item_with_different_date_same_index_for_datetime_index( -# app_client, load_test_data, txn_client, ctx -# ): -# if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): -# pytest.skip() -# -# item = load_test_data("test_item.json") -# item["id"] = str(uuid.uuid4()) -# item["properties"]["start_datetime"] = "2020-02-11T12:30:22Z" -# -# response = await app_client.post( -# f"/collections/{item['collection']}/items", json=item -# ) -# -# assert response.status_code == 201 -# -# indices = await txn_client.database.client.indices.get_alias( -# index="items_test-collection" -# ) -# expected_aliases = [ -# "items_start_datetime_test-collection_2020-02-08", -# "items_datetime_test-collection_2020-02-12", -# "items_end_datetime_test-collection_2020-02-16" -# ] -# all_aliases = set() -# for index_info in indices.values(): -# all_aliases.update(index_info.get("aliases", {}).keys()) -# assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_new_index_when_size_limit_exceeded_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - item = load_test_data("test_item.json") - item["id"] = str(uuid.uuid4()) - item["properties"]["start_datetime"] = "2020-02-11T12:30:22Z" - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.return_value = 26.0 - response = await app_client.post( - f"/collections/{item['collection']}/items", json=item - ) - - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias(index="*") - expected_aliases = [ - "items_start_datetime_test-collection_2020-02-08-2020-02-11", - "items_start_datetime_test-collection_2020-02-12", - ] - all_aliases = set() - - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - item_2 = deepcopy(item) - item_2["id"] = str(uuid.uuid4()) - item_2["properties"]["start_datetime"] = "2020-02-10T12:30:22Z" - response_2 = await app_client.post( - f"/collections/{item_2['collection']}/items", json=item_2 - ) - assert response_2.status_code == 201 - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_item_fails_without_datetime_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - item = load_test_data("test_item.json") - item["id"] = str(uuid.uuid4()) - item["properties"]["start_datetime"] = None - response = await app_client.post( - f"/collections/{item['collection']}/items", json=item - ) - assert response.status_code == 400 - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_bulk_create_items_with_same_date_range_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - base_item = load_test_data("test_item.json") - items_dict = {} - - for i in range(10): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"]["start_datetime"] = f"2020-02-{12 + i}T12:30:22Z" - item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" - item["properties"]["end_datetime"] = f"2020-02-{12 + i}T12:30:22Z" - items_dict[item["id"]] = item - - payload = {"type": "FeatureCollection", "features": list(items_dict.values())} - response = await app_client.post( - f"/collections/{base_item['collection']}/items", json=payload - ) - - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias(index="*") - expected_aliases = [ - "items_start_datetime_test-collection_2020-02-12", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - return all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_bulk_create_items_with_different_date_ranges_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - base_item = load_test_data("test_item.json") - items_dict = {} - - for i in range(3): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"]["start_datetime"] = f"2020-02-{12 + i}T12:30:22Z" - item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" - item["properties"]["end_datetime"] = f"2020-02-{12 + i}T12:30:22Z" - items_dict[item["id"]] = item - - for i in range(2): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"]["start_datetime"] = f"2010-02-{10 + i}T12:30:22Z" - item["properties"]["datetime"] = f"2010-02-{10 + i}T12:30:22Z" - item["properties"]["end_datetime"] = f"2010-02-{10 + i}T12:30:22Z" - items_dict[item["id"]] = item - - payload = {"type": "FeatureCollection", "features": list(items_dict.values())} - - response = await app_client.post( - f"/collections/{base_item['collection']}/items", json=payload - ) - - assert response.status_code == 201 - indices = await txn_client.database.client.indices.get_alias(index="*") - - expected_aliases = ["items_start_datetime_test-collection_2010-02-10"] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_bulk_create_items_with_size_limit_exceeded_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "start_datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2019-02", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias(index="*") - expected_aliases = [ - "items_start_datetime_test-collection_2010-02-10-2020-02-08", - "items_start_datetime_test-collection_2020-02-09", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_bulk_create_items_with_early_date_in_second_batch_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "start_datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias(index="*") - expected_aliases = [ - "items_start_datetime_test-collection_2008-01-15-2020-02-08", - "items_start_datetime_test-collection_2020-02-09", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_bulk_create_items_and_retrieve_by_id_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "start_datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - response = await app_client.get( - f"/collections/{collection_id}/items/{base_item['id']}" - ) - assert response.status_code == 200 - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_patch_collection_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "start_datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - patch_data = { - "description": "Updated description via PATCH", - } - response = await app_client.patch( - f"/collections/{collection_id}?refresh=true", json=patch_data - ) - assert response.status_code == 200 - assert response.json()["description"] == "Updated description via PATCH" - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_put_collection_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "start_datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - collection_response = await app_client.get(f"/collections/{collection_id}") - assert collection_response.status_code == 200 - collection_data = collection_response.json() - - collection_data["description"] = "Updated description via PUT" - collection_data["title"] = "Updated title via PUT" - response = await app_client.put( - f"/collections/{collection_id}?refresh=true", json=collection_data - ) - assert response.json()["description"] == "Updated description via PUT" - assert response.json()["title"] == "Updated title via PUT" - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_patch_item_for_datetime_index( - app_client, load_test_data, txn_client, ctx -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "start_datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - patch_data = {"properties": {"description": "Updated description via PATCH"}} - - response = await app_client.patch( - f"/collections/{collection_id}/items/{base_item['id']}", json=patch_data - ) - assert response.status_code == 200 - assert ( - response.json()["properties"]["description"] - == "Updated description via PATCH" - ) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_put_item_for_datetime_index(app_client, load_test_data, txn_client, ctx): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip("Datetime index filtering not enabled") - - base_item = load_test_data("test_item.json") - collection_id = base_item["collection"] - - def create_items(date_prefix: str, start_day: int, count: int) -> dict: - items = {} - for i in range(count): - item = deepcopy(base_item) - item["id"] = str(uuid.uuid4()) - item["properties"][ - "start_datetime" - ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" - items[item["id"]] = item - return items - - with patch( - "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" - ) as mock_get_size: - mock_get_size.side_effect = [10, 26] - - first_items = create_items("2010-02", start_day=10, count=2) - first_payload = { - "type": "FeatureCollection", - "features": list(first_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=first_payload - ) - assert response.status_code == 201 - - second_items = create_items("2008-01", start_day=15, count=3) - second_payload = { - "type": "FeatureCollection", - "features": list(second_items.values()), - } - response = await app_client.post( - f"/collections/{collection_id}/items", json=second_payload - ) - assert response.status_code == 201 - - item_response = await app_client.get( - f"/collections/{collection_id}/items/{base_item['id']}" - ) - assert item_response.status_code == 200 - item_data = item_response.json() - - item_data["properties"]["platform"] = "Updated platform via PUT" - response = await app_client.put( - f"/collections/{collection_id}/items/{base_item['id']}", json=item_data - ) - assert response.json()["properties"]["platform"] == "Updated platform via PUT" - - @pytest.mark.asyncio async def test_global_collection_max_limit_set(app_client, txn_client, load_test_data): """Test with global collection max limit set, expect cap the limit""" @@ -1722,366 +1080,3 @@ async def test_hide_private_data_from_item(app_client, txn_client, load_test_dat assert "private_data" not in item["properties"] del os.environ["EXCLUDED_FROM_ITEMS"] - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_new_item_in_new_collection_for_datetime_index( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - new_collection = load_test_data("test_collection.json") - new_collection["id"] = "new-collection" - - item = load_test_data("test_item.json") - item["collection"] = "new-collection" - - await app_client.post("/collections", json=new_collection) - response = await app_client.post("/collections/new-collection/items", json=item) - - assert response.status_code == 201 - - indices = await txn_client.database.client.indices.get_alias( - index="items_new-collection" - ) - expected_aliases = [ - "items_datetime_new-collection_2020-02-12", - "items_end_datetime_new-collection_2020-02-16", - "items_start_datetime_new-collection_2020-02-08", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_item_with_invalid_datetime_ordering_should_fail( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - new_collection = load_test_data("test_collection.json") - new_collection["id"] = "new-collection" - - item = load_test_data("test_item.json") - item["collection"] = "new-collection" - item["properties"]["start_datetime"] = "2024-02-12T12:30:22Z" - item["properties"]["datetime"] = "2023-02-12T12:30:22Z" - item["properties"]["end_datetime"] = "2022-02-12T12:30:22Z" - - await app_client.post("/collections", json=new_collection) - - response = await app_client.post("/collections/new-collection/items", json=item) - assert response.status_code == 400 - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_update_item_with_changed_end_datetime( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - new_collection = load_test_data("test_collection.json") - new_collection["id"] = "new-collection" - - item = load_test_data("test_item.json") - item["collection"] = "new-collection" - - await app_client.post("/collections", json=new_collection) - await app_client.post("/collections/new-collection/items", json=item) - - updated_item = item.copy() - updated_item["properties"]["end_datetime"] = "2020-02-19T12:30:22Z" - - response = await app_client.put( - f"/collections/new-collection/items/{item['id']}", json=updated_item - ) - - assert response.status_code == 200 - - indices = await txn_client.database.client.indices.get_alias( - index="items_new-collection" - ) - expected_aliases = [ - "items_datetime_new-collection_2020-02-12", - "items_end_datetime_new-collection_2020-02-19", - "items_start_datetime_new-collection_2020-02-08", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_update_item_with_changed_datetime( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - new_collection = load_test_data("test_collection.json") - new_collection["id"] = "new-collection" - - item = load_test_data("test_item.json") - item["collection"] = "new-collection" - - await app_client.post("/collections", json=new_collection) - await app_client.post("/collections/new-collection/items", json=item) - - updated_item = item.copy() - updated_item["properties"]["datetime"] = "2020-02-14T12:30:22Z" - - response = await app_client.put( - f"/collections/new-collection/items/{item['id']}", json=updated_item - ) - - assert response.status_code == 200 - - indices = await txn_client.database.client.indices.get_alias( - index="items_new-collection" - ) - expected_aliases = [ - "items_datetime_new-collection_2020-02-14", - "items_end_datetime_new-collection_2020-02-16", - "items_start_datetime_new-collection_2020-02-08", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_update_item_with_changed_both_datetime_and_end_datetime( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - new_collection = load_test_data("test_collection.json") - new_collection["id"] = "new-collection" - - item = load_test_data("test_item.json") - item["collection"] = "new-collection" - - await app_client.post("/collections", json=new_collection) - await app_client.post("/collections/new-collection/items", json=item) - - updated_item = item.copy() - updated_item["properties"]["datetime"] = "2022-02-15T12:30:22Z" - updated_item["properties"]["end_datetime"] = "2023-02-18T12:30:22Z" - - response = await app_client.put( - f"/collections/new-collection/items/{item['id']}", json=updated_item - ) - - assert response.status_code == 200 - - indices = await txn_client.database.client.indices.get_alias( - index="items_new-collection" - ) - expected_aliases = [ - "items_datetime_new-collection_2022-02-15", - "items_end_datetime_new-collection_2023-02-18", - "items_start_datetime_new-collection_2020-02-08", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - assert all(alias in all_aliases for alias in expected_aliases) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_search_item_by_datetime_range_with_stac_query( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - new_collection = load_test_data("test_collection.json") - new_collection["id"] = "new-collection" - - item = load_test_data("test_item.json") - item["collection"] = "new-collection" - - await app_client.post("/collections", json=new_collection) - await app_client.post("/collections/new-collection/items", json=item) - - response = await app_client.get( - "/search?collections=new-collection&datetime=2020-02-01T00:00:00Z/2020-02-28T23:59:59Z" - ) - assert response.status_code == 200 - - result = response.json() - assert result["numberMatched"] > 0 - assert len(result["features"]) > 0 - assert any(feature["id"] == item["id"] for feature in result["features"]) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_search_item_by_start_datetime_with_stac_query( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - new_collection = load_test_data("test_collection.json") - new_collection["id"] = "new-collection" - - item = load_test_data("test_item.json") - item["collection"] = "new-collection" - - await app_client.post("/collections", json=new_collection) - await app_client.post("/collections/new-collection/items", json=item) - - response = await app_client.get( - "/search?collections=new-collection&datetime=2020-02-08T00:00:00Z/.." - ) - assert response.status_code == 200 - - result = response.json() - assert result["numberMatched"] > 0 - assert any(feature["id"] == item["id"] for feature in result["features"]) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_search_item_not_found_outside_datetime_range( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - new_collection = load_test_data("test_collection.json") - new_collection["id"] = "new-collection" - - item = load_test_data("test_item.json") - item["collection"] = "new-collection" - - await app_client.post("/collections", json=new_collection) - await app_client.post("/collections/new-collection/items", json=item) - - response = await app_client.get( - "/search?collections=new-collection&datetime=2021-01-01T00:00:00Z/2021-12-31T23:59:59Z" - ) - assert response.status_code == 200 - - result = response.json() - assert result["numberMatched"] == 0 - assert len(result["features"]) == 0 - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_search_item_after_datetime_update_with_stac_query( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - new_collection = load_test_data("test_collection.json") - new_collection["id"] = "new-collection" - - item = load_test_data("test_item.json") - item["collection"] = "new-collection" - - await app_client.post("/collections", json=new_collection) - await app_client.post("/collections/new-collection/items", json=item) - - updated_item = item.copy() - updated_item["properties"]["datetime"] = "2020-02-14T12:30:22Z" - - await app_client.put( - f"/collections/new-collection/items/{item['id']}", json=updated_item - ) - - response = await app_client.get( - "/search?collections=new-collection&datetime=2020-02-14T00:00:00Z/2020-02-14T23:59:59Z" - ) - assert response.status_code == 200 - - result = response.json() - assert result["numberMatched"] > 0 - assert any(feature["id"] == item["id"] for feature in result["features"]) - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_search_item_by_multiple_collections_with_stac_query( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - collection1 = load_test_data("test_collection.json") - collection1["id"] = "collection-1" - - collection2 = load_test_data("test_collection.json") - collection2["id"] = "collection-2" - - item1 = load_test_data("test_item.json") - item1["collection"] = "collection-1" - item1["id"] = "item-1" - - item2 = load_test_data("test_item.json") - item2["collection"] = "collection-2" - item2["id"] = "item-2" - - await app_client.post("/collections", json=collection1) - await app_client.post("/collections", json=collection2) - await app_client.post("/collections/collection-1/items", json=item1) - await app_client.post("/collections/collection-2/items", json=item2) - - response = await app_client.get( - "/search?collections=collection-1,collection-2&datetime=2020-02-01T00:00:00Z/2020-02-28T23:59:59Z" - ) - assert response.status_code == 200 - - result = response.json() - assert result["numberMatched"] >= 2 - feature_ids = {feature["id"] for feature in result["features"]} - assert "item-1" in feature_ids - assert "item-2" in feature_ids - - -@pytest.mark.datetime_filtering -@pytest.mark.asyncio -async def test_create_item_with_the_same_date_change_alias_name_for_datetime_index( - app_client, ctx, load_test_data, txn_client -): - if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): - pytest.skip() - - item = load_test_data("test_item.json") - item["id"] = str(uuid.uuid4()) - - response = await app_client.post( - f"/collections/{item['collection']}/items", json=item - ) - assert response.status_code == 201 - indices = await txn_client.database.client.indices.get_alias( - index="items_test-collection" - ) - expected_aliases = [ - "items_start_datetime_test-collection_2020-02-08", - "items_datetime_test-collection_2020-02-12", - "items_end_datetime_test-collection_2020-02-16", - ] - all_aliases = set() - for index_info in indices.values(): - all_aliases.update(index_info.get("aliases", {}).keys()) - - assert all(alias in all_aliases for alias in expected_aliases) diff --git a/stac_fastapi/tests/api/test_api_datetime_filtering.py b/stac_fastapi/tests/api/test_api_datetime_filtering.py new file mode 100644 index 000000000..eb41ec454 --- /dev/null +++ b/stac_fastapi/tests/api/test_api_datetime_filtering.py @@ -0,0 +1,1633 @@ +import os +import uuid +from copy import deepcopy +from unittest.mock import patch + +import pytest + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_in_past_date_change_alias_name_for_datetime_index( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = "2012-02-12T12:30:22Z" + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 201 + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_start_datetime_test-collection_2012-02-12", + "items_end_datetime_test-collection_2020-02-16", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_uses_existing_datetime_index_for_datetime_index( + mock_datetime_env, app_client, ctx, load_test_data, txn_client, monkeypatch +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_start_datetime_test-collection_2020-02-08", + "items_end_datetime_test-collection_2020-02-16", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_with_different_date_same_index_for_datetime_index( + mock_datetime_env, + app_client, + load_test_data, + txn_client, + ctx, +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = "2020-02-11T12:30:22Z" + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_start_datetime_test-collection_2020-02-08", + "items_end_datetime_test-collection_2020-02-16", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_new_index_when_size_limit_exceeded_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = "2020-02-11T12:30:22Z" + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.return_value = 26.0 + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_start_datetime_test-collection_2020-02-08-2020-02-11", + ] + all_aliases = set() + + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + item_2 = deepcopy(item) + item_2["id"] = str(uuid.uuid4()) + item_2["properties"]["start_datetime"] = "2020-02-10T12:30:22Z" + response_2 = await app_client.post( + f"/collections/{item_2['collection']}/items", json=item_2 + ) + assert response_2.status_code == 201 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_fails_without_datetime_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = None + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 400 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_bulk_create_items_with_same_date_range_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + base_item = load_test_data("test_item.json") + items_dict = {} + + for i in range(10): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = f"2020-02-{12 + i}T12:30:22Z" + item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" + item["properties"]["end_datetime"] = f"2020-02-{12 + i}T12:30:22Z" + items_dict[item["id"]] = item + + payload = {"type": "FeatureCollection", "features": list(items_dict.values())} + response = await app_client.post( + f"/collections/{base_item['collection']}/items", json=payload + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_start_datetime_test-collection_2020-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + return all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_bulk_create_items_with_different_date_ranges_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + base_item = load_test_data("test_item.json") + items_dict = {} + + for i in range(3): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = f"2020-02-{12 + i}T12:30:22Z" + item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" + item["properties"]["end_datetime"] = f"2020-02-{12 + i}T12:30:22Z" + items_dict[item["id"]] = item + + for i in range(2): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = f"2010-02-{10 + i}T12:30:22Z" + item["properties"]["datetime"] = f"2010-02-{10 + i}T12:30:22Z" + item["properties"]["end_datetime"] = f"2010-02-{10 + i}T12:30:22Z" + items_dict[item["id"]] = item + + payload = {"type": "FeatureCollection", "features": list(items_dict.values())} + + response = await app_client.post( + f"/collections/{base_item['collection']}/items", json=payload + ) + + assert response.status_code == 201 + indices = await txn_client.database.client.indices.get_alias(index="*") + + expected_aliases = ["items_start_datetime_test-collection_2010-02-10"] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_bulk_create_items_with_size_limit_exceeded_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2019-02", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_start_datetime_test-collection_2010-02-10-2020-02-08", + "items_start_datetime_test-collection_2020-02-09", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_bulk_create_items_with_early_date_in_second_batch_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_start_datetime_test-collection_2008-01-15-2020-02-08", + "items_start_datetime_test-collection_2020-02-09", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_bulk_create_items_and_retrieve_by_id_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + response = await app_client.get( + f"/collections/{collection_id}/items/{base_item['id']}" + ) + assert response.status_code == 200 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_patch_collection_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + patch_data = { + "description": "Updated description via PATCH", + } + response = await app_client.patch( + f"/collections/{collection_id}?refresh=true", json=patch_data + ) + assert response.status_code == 200 + assert response.json()["description"] == "Updated description via PATCH" + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_put_collection_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + collection_response = await app_client.get(f"/collections/{collection_id}") + assert collection_response.status_code == 200 + collection_data = collection_response.json() + + collection_data["description"] = "Updated description via PUT" + collection_data["title"] = "Updated title via PUT" + response = await app_client.put( + f"/collections/{collection_id}?refresh=true", json=collection_data + ) + assert response.json()["description"] == "Updated description via PUT" + assert response.json()["title"] == "Updated title via PUT" + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_patch_item_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + patch_data = {"properties": {"description": "Updated description via PATCH"}} + + response = await app_client.patch( + f"/collections/{collection_id}/items/{base_item['id']}", json=patch_data + ) + assert response.status_code == 200 + assert ( + response.json()["properties"]["description"] + == "Updated description via PATCH" + ) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_put_item_for_datetime_index( + mock_datetime_env, app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "start_datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + item_response = await app_client.get( + f"/collections/{collection_id}/items/{base_item['id']}" + ) + assert item_response.status_code == 200 + item_data = item_response.json() + + item_data["properties"]["platform"] = "Updated platform via PUT" + response = await app_client.put( + f"/collections/{collection_id}/items/{base_item['id']}", json=item_data + ) + assert response.json()["properties"]["platform"] == "Updated platform via PUT" + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_new_item_in_new_collection_for_datetime_index( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + response = await app_client.post("/collections/new-collection/items", json=item) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_new-collection" + ) + expected_aliases = [ + "items_end_datetime_new-collection_2020-02-16", + "items_start_datetime_new-collection_2020-02-08", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_with_invalid_datetime_ordering_should_fail( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + item["properties"]["start_datetime"] = "2024-02-12T12:30:22Z" + item["properties"]["end_datetime"] = "2022-02-12T12:30:22Z" + + await app_client.post("/collections", json=new_collection) + + response = await app_client.post("/collections/new-collection/items", json=item) + assert response.status_code == 400 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_update_item_with_changed_end_datetime( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + updated_item = item.copy() + updated_item["properties"]["end_datetime"] = "2020-02-19T12:30:22Z" + + response = await app_client.put( + f"/collections/new-collection/items/{item['id']}", json=updated_item + ) + + assert response.status_code == 200 + + indices = await txn_client.database.client.indices.get_alias( + index="items_new-collection" + ) + expected_aliases = [ + "items_end_datetime_new-collection_2020-02-19", + "items_start_datetime_new-collection_2020-02-08", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_update_item_with_changed_datetime( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + updated_item = item.copy() + updated_item["properties"]["datetime"] = "2020-02-14T12:30:22Z" + + response = await app_client.put( + f"/collections/new-collection/items/{item['id']}", json=updated_item + ) + + assert response.status_code == 200 + + indices = await txn_client.database.client.indices.get_alias( + index="items_new-collection" + ) + expected_aliases = [ + "items_end_datetime_new-collection_2020-02-16", + "items_start_datetime_new-collection_2020-02-08", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_by_datetime_range_with_stac_query( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2020-02-01T00:00:00Z/2020-02-28T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] > 0 + assert len(result["features"]) > 0 + assert any(feature["id"] == item["id"] for feature in result["features"]) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_by_start_datetime_with_stac_query( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2020-02-08T00:00:00Z/.." + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] > 0 + assert any(feature["id"] == item["id"] for feature in result["features"]) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_not_found_outside_datetime_range( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2021-01-01T00:00:00Z/2021-12-31T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] == 0 + assert len(result["features"]) == 0 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_after_datetime_update_with_stac_query( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + new_collection = load_test_data("test_collection.json") + new_collection["id"] = "new-collection" + + item = load_test_data("test_item.json") + item["collection"] = "new-collection" + + await app_client.post("/collections", json=new_collection) + await app_client.post("/collections/new-collection/items", json=item) + + updated_item = item.copy() + updated_item["properties"]["datetime"] = "2020-02-14T12:30:22Z" + + await app_client.put( + f"/collections/new-collection/items/{item['id']}", json=updated_item + ) + + response = await app_client.get( + "/search?collections=new-collection&datetime=2020-02-14T00:00:00Z/2020-02-14T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] > 0 + assert any(feature["id"] == item["id"] for feature in result["features"]) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_search_item_by_multiple_collections_with_stac_query( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + collection1 = load_test_data("test_collection.json") + collection1["id"] = "collection-1" + + collection2 = load_test_data("test_collection.json") + collection2["id"] = "collection-2" + + item1 = load_test_data("test_item.json") + item1["collection"] = "collection-1" + item1["id"] = "item-1" + + item2 = load_test_data("test_item.json") + item2["collection"] = "collection-2" + item2["id"] = "item-2" + + await app_client.post("/collections", json=collection1) + await app_client.post("/collections", json=collection2) + await app_client.post("/collections/collection-1/items", json=item1) + await app_client.post("/collections/collection-2/items", json=item2) + + response = await app_client.get( + "/search?collections=collection-1,collection-2&datetime=2020-02-01T00:00:00Z/2020-02-28T23:59:59Z" + ) + assert response.status_code == 200 + + result = response.json() + assert result["numberMatched"] >= 2 + feature_ids = {feature["id"] for feature in result["features"]} + assert "item-1" in feature_ids + assert "item-2" in feature_ids + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_with_the_same_date_change_alias_name_for_datetime_index( + mock_datetime_env, app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 201 + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_start_datetime_test-collection_2020-02-08", + "items_end_datetime_test-collection_2020-02-16", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_create_item_with_datetime_field_creates_single_alias( + app_client, + ctx, + load_test_data, + txn_client, +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["start_datetime"] = None + item["properties"]["end_datetime"] = None + item["properties"]["datetime"] = "2024-06-15T12:30:22Z" + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_datetime_test-collection_2020-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) + assert not any("start_datetime" in alias for alias in all_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_alias_created_for_past_date( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = "2012-02-12T12:30:22Z" + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 201 + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_datetime_test-collection_2012-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_reuses_existing_index_for_default_date( + app_client, ctx, load_test_data, txn_client +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_datetime_test-collection_2020-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_groups_same_year_dates_in_single_index( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = "2022-02-12T12:30:22Z" + + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias( + index="items_test-collection" + ) + expected_aliases = [ + "items_datetime_test-collection_2020-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_creates_new_index_when_size_limit_exceeded( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = "2024-02-12T12:30:22Z" + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.return_value = 26.0 + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_datetime_test-collection_2020-02-12-2024-02-12", + "items_datetime_test-collection_2024-02-13", + ] + all_aliases = set() + + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + item_2 = deepcopy(item) + item_2["id"] = str(uuid.uuid4()) + item_2["properties"]["datetime"] = "2023-02-12T12:30:22Z" + response_2 = await app_client.post( + f"/collections/{item_2['collection']}/items", json=item_2 + ) + assert response_2.status_code == 201 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_rejects_item_without_datetime_field( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + item = load_test_data("test_item.json") + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = None + response = await app_client.post( + f"/collections/{item['collection']}/items", json=item + ) + assert response.status_code == 400 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_bulk_insert_with_same_date_range( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + base_item = load_test_data("test_item.json") + items_dict = {} + + for i in range(10): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" + items_dict[item["id"]] = item + + payload = {"type": "FeatureCollection", "features": list(items_dict.values())} + response = await app_client.post( + f"/collections/{base_item['collection']}/items", json=payload + ) + + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_datetime_test-collection_2020-02-12", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + return all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_bulk_insert_with_different_date_ranges( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip() + + base_item = load_test_data("test_item.json") + items_dict = {} + + for i in range(3): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = f"2020-02-{12 + i}T12:30:22Z" + items_dict[item["id"]] = item + + for i in range(2): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"]["datetime"] = f"2010-02-{10 + i}T12:30:22Z" + items_dict[item["id"]] = item + + payload = {"type": "FeatureCollection", "features": list(items_dict.values())} + + response = await app_client.post( + f"/collections/{base_item['collection']}/items", json=payload + ) + + assert response.status_code == 201 + indices = await txn_client.database.client.indices.get_alias(index="*") + + expected_aliases = ["items_datetime_test-collection_2010-02-10"] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_bulk_insert_handles_size_limit_correctly( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2019-02", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_datetime_test-collection_2010-02-10-2020-02-12", + "items_datetime_test-collection_2020-02-13", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_splits_index_when_earlier_date_added_after_limit( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + indices = await txn_client.database.client.indices.get_alias(index="*") + expected_aliases = [ + "items_datetime_test-collection_2008-01-15-2020-02-12", + "items_datetime_test-collection_2020-02-13", + ] + all_aliases = set() + for index_info in indices.values(): + all_aliases.update(index_info.get("aliases", {}).keys()) + assert all(alias in all_aliases for alias in expected_aliases) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_bulk_insert_allows_item_retrieval( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + response = await app_client.get( + f"/collections/{collection_id}/items/{base_item['id']}" + ) + assert response.status_code == 200 + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_collection_patch_operation( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + patch_data = { + "description": "Updated description via PATCH", + } + response = await app_client.patch( + f"/collections/{collection_id}?refresh=true", json=patch_data + ) + assert response.status_code == 200 + assert response.json()["description"] == "Updated description via PATCH" + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_collection_put_operation( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + collection_response = await app_client.get(f"/collections/{collection_id}") + assert collection_response.status_code == 200 + collection_data = collection_response.json() + + collection_data["description"] = "Updated description via PUT" + collection_data["title"] = "Updated title via PUT" + response = await app_client.put( + f"/collections/{collection_id}?refresh=true", json=collection_data + ) + assert response.json()["description"] == "Updated description via PUT" + assert response.json()["title"] == "Updated title via PUT" + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_item_patch_operation( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + patch_data = {"properties": {"description": "Updated description via PATCH"}} + + response = await app_client.patch( + f"/collections/{collection_id}/items/{base_item['id']}", json=patch_data + ) + assert response.status_code == 200 + assert ( + response.json()["properties"]["description"] + == "Updated description via PATCH" + ) + + +@pytest.mark.datetime_filtering +@pytest.mark.asyncio +async def test_datetime_index_item_put_operation( + app_client, load_test_data, txn_client, ctx +): + if not os.getenv("ENABLE_DATETIME_INDEX_FILTERING"): + pytest.skip("Datetime index filtering not enabled") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + def create_items(date_prefix: str, start_day: int, count: int) -> dict: + items = {} + for i in range(count): + item = deepcopy(base_item) + item["id"] = str(uuid.uuid4()) + item["properties"][ + "datetime" + ] = f"{date_prefix}-{start_day + i:02d}T12:30:22Z" + items[item["id"]] = item + return items + + with patch( + "stac_fastapi.sfeos_helpers.search_engine.managers.IndexSizeManager.get_index_size_in_gb" + ) as mock_get_size: + mock_get_size.side_effect = [10, 26] + + first_items = create_items("2010-02", start_day=10, count=2) + first_payload = { + "type": "FeatureCollection", + "features": list(first_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=first_payload + ) + assert response.status_code == 201 + + second_items = create_items("2008-01", start_day=15, count=3) + second_payload = { + "type": "FeatureCollection", + "features": list(second_items.values()), + } + response = await app_client.post( + f"/collections/{collection_id}/items", json=second_payload + ) + assert response.status_code == 201 + + item_response = await app_client.get( + f"/collections/{collection_id}/items/{base_item['id']}" + ) + assert item_response.status_code == 200 + item_data = item_response.json() + + item_data["properties"]["platform"] = "Updated platform via PUT" + response = await app_client.put( + f"/collections/{collection_id}/items/{base_item['id']}", json=item_data + ) + assert response.json()["properties"]["platform"] == "Updated platform via PUT" diff --git a/stac_fastapi/tests/conftest.py b/stac_fastapi/tests/conftest.py index b461e7221..d6a48c41d 100644 --- a/stac_fastapi/tests/conftest.py +++ b/stac_fastapi/tests/conftest.py @@ -396,3 +396,13 @@ def build_test_app(): # Create and return the app api = StacApi(**test_config) return api.app + + +@pytest.fixture +def mock_datetime_env(txn_client, monkeypatch): + """Set USE_DATETIME environment variable to False for testing.""" + monkeypatch.setenv("USE_DATETIME", "false") + if hasattr(txn_client.database.async_index_selector, "cache_manager"): + txn_client.database.async_index_selector.cache_manager.clear_cache() + yield + monkeypatch.setenv("USE_DATETIME", "true") diff --git a/stac_fastapi/tests/database/test_database.py b/stac_fastapi/tests/database/test_database.py index aea84e560..e02d2165c 100644 --- a/stac_fastapi/tests/database/test_database.py +++ b/stac_fastapi/tests/database/test_database.py @@ -68,7 +68,7 @@ def test_filter_datetime_field_outside_range(): "end_datetime": {"gte": None, "lte": None}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 0 @@ -90,7 +90,7 @@ def test_filter_start_datetime_field_with_gte(): "end_datetime": {"gte": None, "lte": None}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 1 @@ -112,7 +112,7 @@ def test_filter_end_datetime_field_with_lte(): "end_datetime": {"gte": None, "lte": "2020-02-28T23:59:59Z"}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 1 @@ -134,7 +134,7 @@ def test_filter_all_criteria_matching(): "end_datetime": {"gte": None, "lte": "2020-02-28T23:59:59Z"}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 1 @@ -156,7 +156,7 @@ def test_filter_datetime_field_fails_gte(): "end_datetime": {"gte": None, "lte": None}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 0 @@ -178,17 +178,16 @@ def test_filter_datetime_field_fails_lte(): "end_datetime": {"gte": None, "lte": None}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 0 @pytest.mark.datetime_filtering -def test_filter_start_datetime_range_format(): +def test_filter_start_datetime_range_format(mock_datetime_env): collection_indexes = [ ( { - "datetime": "items_datetime_new-collection_2020-02-12", "end_datetime": "items_end_datetime_new-collection_2020-02-16", "start_datetime": "items_start_datetime_new-collection_2020-02-08-2022-04-05", }, @@ -200,7 +199,7 @@ def test_filter_start_datetime_range_format(): "end_datetime": {"gte": None, "lte": None}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 1 assert result[0] == "items_start_datetime_new-collection_2020-02-08-2022-04-05" @@ -223,7 +222,7 @@ def test_filter_start_datetime_range_fails_gte(): "end_datetime": {"gte": None, "lte": None}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 0 @@ -234,22 +233,16 @@ def test_filter_multiple_indexes_mixed_results(): ( { "datetime": "items_datetime_new-collection_2020-02-12", - "end_datetime": "items_end_datetime_new-collection_2020-02-16", - "start_datetime": "items_start_datetime_new-collection_2020-02-08", }, ), ( { "datetime": "items_datetime_new-collection_2020-02-15", - "end_datetime": "items_end_datetime_new-collection_2020-02-18", - "start_datetime": "items_start_datetime_new-collection_2020-02-10", }, ), ( { "datetime": "items_datetime_new-collection_2021-03-15", - "end_datetime": "items_end_datetime_new-collection_2021-03-20", - "start_datetime": "items_start_datetime_new-collection_2021-03-10", }, ), ] @@ -259,11 +252,11 @@ def test_filter_multiple_indexes_mixed_results(): "end_datetime": {"gte": None, "lte": None}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, True) assert len(result) == 2 - assert "items_start_datetime_new-collection_2020-02-08" in result - assert "items_start_datetime_new-collection_2020-02-10" in result + assert "items_datetime_new-collection_2020-02-12" in result + assert "items_datetime_new-collection_2020-02-15" in result @pytest.mark.datetime_filtering @@ -275,7 +268,7 @@ def test_filter_empty_collection(): "end_datetime": {"gte": None, "lte": None}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 0 @@ -297,7 +290,7 @@ def test_filter_all_criteria_none(): "end_datetime": {"gte": None, "lte": None}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 1 @@ -319,7 +312,7 @@ def test_filter_end_datetime_outside_range(): "end_datetime": {"gte": None, "lte": "2020-02-10T23:59:59Z"}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 0 @@ -348,7 +341,7 @@ def test_filter_complex_mixed_criteria(): "end_datetime": {"gte": None, "lte": "2020-02-20T23:59:59Z"}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 2 @@ -370,6 +363,6 @@ def test_filter_with_single_date_range(): "end_datetime": {"gte": None, "lte": None}, } - result = filter_indexes_by_datetime(collection_indexes, datetime_search) + result = filter_indexes_by_datetime(collection_indexes, datetime_search, False) assert len(result) == 1 From 14a06ca4833ce822c9f87302278bbd074e465c76 Mon Sep 17 00:00:00 2001 From: Grzegorz Pustulka Date: Tue, 2 Dec 2025 14:55:25 +0100 Subject: [PATCH 8/8] fix --- .../opensearch/stac_fastapi/opensearch/database_logic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 3ffebcff9..15406b93c 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -586,7 +586,7 @@ def apply_datetime_filter(search: Search, datetime: Optional[str]) -> Search: ), ], ) - return search.query(filter_query) + return search.query(filter_query) @staticmethod def apply_bbox_filter(search: Search, bbox: List):