diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 172d57b8..156cfd10 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -75,7 +75,7 @@ ) from ravendb.documents.operations.configuration.definitions import StudioConfiguration, StudioEnvironment -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage # AI Operations from ravendb.documents.ai import ( @@ -115,6 +115,13 @@ GetAiAgentsResponse, AddOrUpdateAiAgentOperation, DeleteAiAgentOperation, + GetConversationMessagesOperation, + GetConversationMessagesOptions, + AiConversationMessagesResult, + AiConversationMessage, + AiToolCallResult, + AiMessageRole, + AiConversationDetailLevel, ) from ravendb.documents.operations.ai import ( ChunkingOptions, @@ -188,6 +195,24 @@ from ravendb.documents.operations.ongoing_tasks import ( OngoingTaskPullReplicationAsSink, OngoingTaskPullReplicationAsHub, + OngoingTaskCdcSink, +) +from ravendb.documents.operations.cdc_sink import ( + CdcSinkConfiguration, + CdcSinkTableConfig, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcColumnMapping, + CdcColumnType, + CdcSinkRelationType, + CdcSinkTaskState, + CdcSinkTableLoadState, + AddCdcSinkOperation, + AddCdcSinkOperationResult, + UpdateCdcSinkOperation, + UpdateCdcSinkOperationResult, ) from ravendb.documents.operations.revisions import ( RevisionsCollectionConfiguration, @@ -356,6 +381,9 @@ GetCertificatesResponse, PutClientCertificateOperation, SecurityClearance, + CertificateUsage, + SsoProvider, + SsoIdentifier, ) from ravendb.serverwide.operations.common import ( BuildNumber, @@ -366,6 +394,17 @@ GetDatabaseRecordOperation, ) +from ravendb.serverwide.operations.connection_strings import ( + ServerWideConnectionString, + ServerWideConnectionStringUsage, + PutServerWideConnectionStringOperation, + PutServerWideConnectionStringResult, + GetServerWideConnectionStringsOperation, + GetServerWideConnectionStringsResult, + RemoveServerWideConnectionStringOperation, + RemoveServerWideConnectionStringResult, +) + from ravendb.documents.identity.hilo import ( HiLoIdGenerator, MultiTypeHiLoGenerator, diff --git a/ravendb/documents/ai/ai_conversation.py b/ravendb/documents/ai/ai_conversation.py index 50ea0fef..17f42916 100644 --- a/ravendb/documents/ai/ai_conversation.py +++ b/ravendb/documents/ai/ai_conversation.py @@ -43,6 +43,7 @@ def __init__( conversation_id: str = None, change_vector: str = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ): self._store = store self._agent_id = agent_id @@ -50,6 +51,7 @@ def __init__( self._conversation_id = conversation_id self._change_vector = change_vector self._debug = debug + self._cancel_pending_action_tools = cancel_pending_action_tools self._prompt_parts: List[ContentPart] = [] self._action_responses: Dict[str, AiAgentActionResponse] = {} @@ -137,17 +139,26 @@ def add_artificial_action_with_response(self, tool_id: str, action_response) -> self._artificial_actions.append(AiAgentArtificialActionResponse(tool_id=tool_id, content=content)) - def run(self) -> AiAnswer: + def run(self, cancellation_event=None) -> AiAnswer: self._dispatched_tool_ids.clear() while True: - r = self._run_internal() + r = self._run_internal(cancellation_event=cancellation_event) if self._handle_server_reply(r): return r - def stream(self, stream_property_path: str = None, on_chunk: Optional[Callable[[str], None]] = None) -> AiAnswer: + def stream( + self, + stream_property_path: str = None, + on_chunk: Optional[Callable[[str], None]] = None, + cancellation_event=None, + ) -> AiAnswer: while True: - r = self._run_internal(stream_property_path=stream_property_path, streamed_chunks_callback=on_chunk) + r = self._run_internal( + stream_property_path=stream_property_path, + streamed_chunks_callback=on_chunk, + cancellation_event=cancellation_event, + ) if self._handle_server_reply(r): return r @@ -155,6 +166,7 @@ def _run_internal( self, stream_property_path: Optional[str] = None, streamed_chunks_callback: Optional[Callable[[str], None]] = None, + cancellation_event=None, ) -> AiAnswer: from ravendb.documents.operations.ai.agents import RunConversationOperation import time @@ -194,6 +206,8 @@ def _run_internal( streamed_chunks_callback=streamed_chunks_callback, attachments_commands=self._attachments_commands, debug=self._debug, + cancel_pending_action_tools=self._cancel_pending_action_tools, + cancellation_event=cancellation_event, ) try: @@ -203,6 +217,7 @@ def _run_internal( self._change_vector = result.change_vector self._conversation_id = result.conversation_id + self._cancel_pending_action_tools = False self._action_requests = result.action_requests or [] return AiAnswer( diff --git a/ravendb/documents/ai/ai_operations.py b/ravendb/documents/ai/ai_operations.py index 77bf1ad8..197d3f0d 100644 --- a/ravendb/documents/ai/ai_operations.py +++ b/ravendb/documents/ai/ai_operations.py @@ -78,6 +78,7 @@ def conversation( creation_options: "AiConversationCreationOptions" = None, change_vector: str = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ) -> AiConversation: """ Creates a new conversation with the specified AI agent. @@ -88,12 +89,22 @@ def conversation( creation_options: Optional creation options for the conversation change_vector: Optional change vector for concurrency control debug: Optional flag enabling server-side conversation debugging + cancel_pending_action_tools: When True, the server auto-answers any open action + tool calls instead of requiring a response Returns: Conversation operations interface for managing the conversation """ - return AiConversation(self._store, agent_id, creation_options, conversation_id, change_vector, debug) + return AiConversation( + self._store, + agent_id, + creation_options, + conversation_id, + change_vector, + debug, + cancel_pending_action_tools, + ) def conversation_with_id(self, conversation_id: str, change_vector: str = None) -> AiConversation: """ @@ -115,3 +126,19 @@ def conversation_with_id(self, conversation_id: str, change_vector: str = None) from ravendb.documents.ai.ai_conversation import AiConversation return AiConversation.with_conversation_id(self._store, conversation_id, change_vector) + + def get_conversation_messages(self, conversation_id_or_options) -> "AiConversationMessagesResult": + """ + Reads messages from an AI agent conversation. + + Args: + conversation_id_or_options: The conversation document ID, or a + GetConversationMessagesOptions instance for paging/filtering control. + + Returns: + The conversation messages result; None when the conversation does not exist. + """ + from ravendb.documents.operations.ai.agents import GetConversationMessagesOperation + + operation = GetConversationMessagesOperation(conversation_id_or_options) + return self._store.maintenance.send(operation) diff --git a/ravendb/documents/operations/ai/agents/__init__.py b/ravendb/documents/operations/ai/agents/__init__.py index d8099d41..df5496f8 100644 --- a/ravendb/documents/operations/ai/agents/__init__.py +++ b/ravendb/documents/operations/ai/agents/__init__.py @@ -39,6 +39,17 @@ AiConversationParameterOptions, ) +from .get_conversation_messages_operation import ( + GetConversationMessagesOperation, + GetConversationMessagesOptions, + GetConversationMessagesCommand, + AiConversationMessagesResult, + AiConversationMessage, + AiToolCallResult, + AiMessageRole, + AiConversationDetailLevel, +) + __all__ = [ "AiAgentConfiguration", "AiAgentConfigurationResult", @@ -68,4 +79,12 @@ "GetAiAgentsResponse", "AddOrUpdateAiAgentOperation", "DeleteAiAgentOperation", + "GetConversationMessagesOperation", + "GetConversationMessagesOptions", + "GetConversationMessagesCommand", + "AiConversationMessagesResult", + "AiConversationMessage", + "AiToolCallResult", + "AiMessageRole", + "AiConversationDetailLevel", ] diff --git a/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py b/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py new file mode 100644 index 00000000..f5968aeb --- /dev/null +++ b/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +import enum +import json +from datetime import datetime +from typing import Any, Dict, List, Optional, TYPE_CHECKING + +import requests + +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.documents.conventions import DocumentConventions +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.tools.utils import Utils + +if TYPE_CHECKING: + from ravendb.documents.operations.ai.agents.run_conversation_operation import AiUsage + + +class AiMessageRole(enum.Enum): + SYSTEM = "System" + USER = "User" + ASSISTANT = "Assistant" + SUMMARY = "Summary" + INTERNAL = "Internal" + + def __str__(self) -> str: + return self.value + + +class AiConversationDetailLevel(enum.Enum): + SIMPLE = "Simple" + DETAILED = "Detailed" + FULL = "Full" + + def __str__(self) -> str: + return self.value + + +class AiToolCallResult: + def __init__( + self, + id: Optional[str] = None, + name: Optional[str] = None, + arguments: Optional[str] = None, + result: Optional[str] = None, + sub_conversation_id: Optional[str] = None, + ): + self.id = id + self.name = name + self.arguments = arguments + self.result = result + self.sub_conversation_id = sub_conversation_id + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiToolCallResult: + return cls( + id=json_dict.get("Id"), + name=json_dict.get("Name"), + arguments=json_dict.get("Arguments"), + result=json_dict.get("Result"), + sub_conversation_id=json_dict.get("SubConversationId"), + ) + + def to_json(self) -> Dict[str, Any]: + return { + "Id": self.id, + "Name": self.name, + "Arguments": self.arguments, + "Result": self.result, + "SubConversationId": self.sub_conversation_id, + } + + +class AiConversationMessage: + def __init__( + self, + role: Optional[AiMessageRole] = None, + content: Optional[str] = None, + attachments: Optional[List[str]] = None, + timestamp: Optional[datetime] = None, + tool_calls: Optional[List[AiToolCallResult]] = None, + usage: Optional["AiUsage"] = None, + sub_conversation_id: Optional[str] = None, + ): + self.role = role + self.content = content + self.attachments = attachments + self.timestamp = timestamp + self.tool_calls = tool_calls + self.usage = usage + self.sub_conversation_id = sub_conversation_id + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiConversationMessage: + from ravendb.documents.operations.ai.agents.run_conversation_operation import AiUsage + + role_value = json_dict.get("Role") + tool_calls = json_dict.get("ToolCalls") + return cls( + role=AiMessageRole(role_value) if role_value else None, + content=json_dict.get("Content"), + attachments=json_dict.get("Attachments"), + timestamp=Utils.string_to_datetime(json_dict.get("Timestamp")), + tool_calls=[AiToolCallResult.from_json(tc) for tc in tool_calls] if tool_calls is not None else None, + usage=AiUsage.from_json(json_dict["Usage"]) if json_dict.get("Usage") else None, + sub_conversation_id=json_dict.get("SubConversationId"), + ) + + def to_json(self) -> Dict[str, Any]: + return { + "Role": self.role.value if self.role else None, + "Content": self.content, + "Attachments": self.attachments, + "Timestamp": Utils.datetime_to_string(self.timestamp), + "ToolCalls": [tc.to_json() for tc in self.tool_calls] if self.tool_calls is not None else None, + "Usage": self.usage.to_json() if self.usage else None, + "SubConversationId": self.sub_conversation_id, + } + + +class AiConversationMessagesResult: + def __init__( + self, + conversation_id: Optional[str] = None, + agent: Optional[str] = None, + parameters: Optional[Dict[str, Any]] = None, + total_usage: Optional["AiUsage"] = None, + last_message_at: Optional[datetime] = None, + messages: Optional[List[AiConversationMessage]] = None, + has_more_messages: bool = False, + sub_conversation_ids: Optional[List[str]] = None, + attachments: Optional[List[str]] = None, + ): + self.conversation_id = conversation_id + self.agent = agent + self.parameters = parameters + self.total_usage = total_usage + self.last_message_at = last_message_at + self.messages = messages + self.has_more_messages = has_more_messages + self.sub_conversation_ids = sub_conversation_ids + self.attachments = attachments + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiConversationMessagesResult: + from ravendb.documents.operations.ai.agents.run_conversation_operation import AiUsage + + messages = json_dict.get("Messages") + return cls( + conversation_id=json_dict.get("ConversationId"), + agent=json_dict.get("Agent"), + parameters=json_dict.get("Parameters"), + total_usage=AiUsage.from_json(json_dict["TotalUsage"]) if json_dict.get("TotalUsage") else None, + last_message_at=Utils.string_to_datetime(json_dict.get("LastMessageAt")), + messages=[AiConversationMessage.from_json(m) for m in messages] if messages is not None else None, + has_more_messages=json_dict.get("HasMoreMessages", False), + sub_conversation_ids=json_dict.get("SubConversationIds"), + attachments=json_dict.get("Attachments"), + ) + + def to_json(self) -> Dict[str, Any]: + return { + "ConversationId": self.conversation_id, + "Agent": self.agent, + "Parameters": self.parameters, + "TotalUsage": self.total_usage.to_json() if self.total_usage else None, + "LastMessageAt": Utils.datetime_to_string(self.last_message_at), + "HasMoreMessages": self.has_more_messages, + "SubConversationIds": self.sub_conversation_ids, + "Attachments": self.attachments, + "Messages": [m.to_json() for m in self.messages] if self.messages is not None else None, + } + + +class GetConversationMessagesOptions: + def __init__( + self, + conversation_id: Optional[str] = None, + before: Optional[datetime] = None, + after: Optional[datetime] = None, + page_size: int = 2147483647, + detail_level: AiConversationDetailLevel = AiConversationDetailLevel.SIMPLE, + ): + self.conversation_id = conversation_id + self.before = before + self.after = after + self.page_size = page_size + self.detail_level = detail_level + + def validate(self) -> None: + if not self.conversation_id: + raise ValueError("conversation_id cannot be None or empty") + + if self.before is not None and self.after is not None: + raise ValueError("before and after cannot both be specified.") + + if self.page_size <= 0: + raise ValueError("PageSize must be greater than 0.") + + +class GetConversationMessagesOperation(MaintenanceOperation[AiConversationMessagesResult]): + def __init__(self, conversation_id_or_options): + if isinstance(conversation_id_or_options, GetConversationMessagesOptions): + parameters = conversation_id_or_options + else: + parameters = GetConversationMessagesOptions(conversation_id=conversation_id_or_options) + parameters.validate() + self._parameters = parameters + + def get_command(self, conventions: DocumentConventions) -> RavenCommand[AiConversationMessagesResult]: + return GetConversationMessagesCommand(self._parameters) + + +class GetConversationMessagesCommand(RavenCommand[AiConversationMessagesResult]): + def __init__(self, parameters: GetConversationMessagesOptions): + super().__init__(AiConversationMessagesResult) + self._parameters = parameters + + def is_read_request(self) -> bool: + return True + + def create_request(self, node: ServerNode) -> requests.Request: + from urllib.parse import quote + + url = ( + f"{node.url}/databases/{node.database}/ai/agent/conversation/messages" + f"?conversationId={quote(self._parameters.conversation_id)}" + ) + if self._parameters.before is not None: + url += f"&before={quote(Utils.datetime_to_string(self._parameters.before))}" + if self._parameters.after is not None: + url += f"&after={quote(Utils.datetime_to_string(self._parameters.after))}" + url += f"&pageSize={self._parameters.page_size}" + detail_level = self._parameters.detail_level or AiConversationDetailLevel.SIMPLE + url += f"&detailLevel={detail_level.value}" + + return requests.Request("GET", url) + + def set_response(self, response: str, from_cache: bool) -> None: + # A null response (404 for a missing conversation) leaves the result None. + if response is None: + return + + response_json = json.loads(response) + result = AiConversationMessagesResult.from_json(response_json) + + # Parameters are parsed natively so heterogeneous values keep their types + # (int stays int, float stays float, lists stay lists). + parameters = response_json.get("Parameters") + if isinstance(parameters, dict): + result.parameters = dict(parameters) + + self.result = result diff --git a/ravendb/documents/operations/ai/agents/run_conversation_operation.py b/ravendb/documents/operations/ai/agents/run_conversation_operation.py index b830284e..f192d643 100644 --- a/ravendb/documents/operations/ai/agents/run_conversation_operation.py +++ b/ravendb/documents/operations/ai/agents/run_conversation_operation.py @@ -295,6 +295,8 @@ def __init__( streamed_chunks_callback: Optional[Callable[[str], None]] = None, attachments_commands: Optional[List[Any]] = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, + cancellation_event=None, ): if not agent_id or (isinstance(agent_id, str) and agent_id.isspace()): raise ValueError("agent_id cannot be None or empty") @@ -314,6 +316,8 @@ def __init__( self._streamed_chunks_callback = streamed_chunks_callback self._attachments_commands = attachments_commands or [] self._debug = debug + self._cancel_pending_action_tools = cancel_pending_action_tools + self._cancellation_event = cancellation_event def get_command(self, conventions: DocumentConventions) -> RavenCommand[ConversationResult[TSchema]]: return RunConversationCommand( @@ -329,6 +333,8 @@ def get_command(self, conventions: DocumentConventions) -> RavenCommand[Conversa conventions=conventions, attachments_commands=self._attachments_commands, debug=self._debug, + cancel_pending_action_tools=self._cancel_pending_action_tools, + cancellation_event=self._cancellation_event, ) @@ -347,6 +353,8 @@ def __init__( conventions: Optional[DocumentConventions] = None, attachments_commands: Optional[List[Any]] = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, + cancellation_event=None, ): from ravendb.util.util import RaftIdGenerator from ravendb.documents.commands.batches import PutAttachmentCommandData @@ -363,6 +371,8 @@ def __init__( self._streamed_chunks_callback = streamed_chunks_callback self._conventions = conventions self._debug = debug + self._cancel_pending_action_tools = cancel_pending_action_tools + self._cancellation_event = cancellation_event self._attachments_commands = attachments_commands or [] # Raft id pinned at construction so retries keep the same id. @@ -409,6 +419,10 @@ def create_request(self, node: ServerNode) -> requests.Request: if self._debug is not None: url += f"&debug={self._debug}" + # Always appended, after the debug parameter (C# order: streaming, changeVector, debug, + # cancelPendingActionTools), so the wire carries it even when False. + url += f"&cancelPendingActionTools={self._cancel_pending_action_tools}" + request_body = ConversationRequestBody( action_responses=self._action_responses, artificial_actions=self._artificial_actions, @@ -449,6 +463,12 @@ def process_response(self, cache, response: requests.Response, url) -> ResponseD return super().process_response(cache, response, url) for line in response.iter_lines(decode_unicode=True): + if self._cancellation_event is not None and self._cancellation_event.is_set(): + # Cancellation is observed between lines, so a mid-stream cancel stops the + # read promptly instead of consuming the remaining stream (RavenDB-26693). + from concurrent.futures import CancelledError + + raise CancelledError("The operation was canceled.") if not line: continue if line.startswith("{"): diff --git a/ravendb/documents/operations/ai/ai_connection_string.py b/ravendb/documents/operations/ai/ai_connection_string.py index 9e39e520..1ed2a11b 100644 --- a/ravendb/documents/operations/ai/ai_connection_string.py +++ b/ravendb/documents/operations/ai/ai_connection_string.py @@ -1,5 +1,5 @@ import enum -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, List from ravendb.serverwide.server_operation_executor import ConnectionStringType from ravendb.documents.operations.ai.azure_open_ai_settings import AzureOpenAiSettings @@ -11,7 +11,7 @@ from ravendb.documents.operations.ai.open_ai_settings import OpenAiSettings from ravendb.documents.operations.ai.vertex_settings import VertexSettings -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage class AiModelType(enum.Enum): @@ -45,8 +45,9 @@ def __init__( mistral_ai_settings: Optional[MistralAiSettings] = None, vertex_settings: Optional[VertexSettings] = None, model_type: AiModelType = None, + used_by: Optional[List[ConnectionStringUsage]] = None, ): - super().__init__(name) + super().__init__(name, used_by) self.identifier = identifier self.openai_settings = openai_settings self.azure_openai_settings = azure_openai_settings @@ -195,4 +196,5 @@ def from_json(cls, json_dict: Dict[str, Any]) -> "AiConnectionString": VertexSettings.from_json(json_dict["VertexSettings"]) if json_dict.get("VertexSettings") else None ), model_type=AiModelType(json_dict["ModelType"]) if json_dict.get("ModelType") else None, + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/attachments/__init__.py b/ravendb/documents/operations/attachments/__init__.py index 1dfdcd40..19241389 100644 --- a/ravendb/documents/operations/attachments/__init__.py +++ b/ravendb/documents/operations/attachments/__init__.py @@ -404,6 +404,7 @@ def __init__( custom_server_url: str = None, force_path_style: bool = None, storage_class: Optional[S3StorageClass] = None, + disable_checksum_validation: bool = False, ): self.aws_access_key = aws_access_key self.aws_secret_key = aws_secret_key @@ -414,6 +415,7 @@ def __init__( self.custom_server_url = custom_server_url self.force_path_style = force_path_style self.storage_class = storage_class + self.disable_checksum_validation = disable_checksum_validation @classmethod def from_json(cls, json_dict: dict) -> RemoteAttachmentsS3Settings: @@ -428,6 +430,7 @@ def from_json(cls, json_dict: dict) -> RemoteAttachmentsS3Settings: json_dict.get("CustomServerUrl"), json_dict.get("ForcePathStyle"), S3StorageClass(storage_class_raw) if storage_class_raw is not None else None, + json_dict.get("DisableChecksumValidation", False), ) def to_json(self) -> dict: @@ -440,11 +443,45 @@ def to_json(self) -> dict: "BucketName": self.bucket_name, "CustomServerUrl": self.custom_server_url, "ForcePathStyle": self.force_path_style, + "DisableChecksumValidation": self.disable_checksum_validation, } if self.storage_class is not None: result["StorageClass"] = self.storage_class.value return result + def __eq__(self, other) -> bool: + if not isinstance(other, RemoteAttachmentsS3Settings): + return False + + return ( + self.aws_region_name == other.aws_region_name + and self.bucket_name == other.bucket_name + and self.remote_folder_name == other.remote_folder_name + and self.custom_server_url == other.custom_server_url + and self.force_path_style == other.force_path_style + and self.disable_checksum_validation == other.disable_checksum_validation + and self.storage_class == other.storage_class + and self.aws_access_key == other.aws_access_key + and self.aws_secret_key == other.aws_secret_key + and self.aws_session_token == other.aws_session_token + ) + + def __hash__(self) -> int: + return hash( + ( + self.aws_region_name, + self.bucket_name, + self.remote_folder_name, + self.custom_server_url, + self.force_path_style, + self.disable_checksum_validation, + self.storage_class, + self.aws_access_key, + self.aws_secret_key, + self.aws_session_token, + ) + ) + class RemoteAttachmentsAzureSettings: def __init__( diff --git a/ravendb/documents/operations/backups/settings.py b/ravendb/documents/operations/backups/settings.py index 565e3df8..10a99f2d 100644 --- a/ravendb/documents/operations/backups/settings.py +++ b/ravendb/documents/operations/backups/settings.py @@ -130,6 +130,7 @@ def __init__( bucket_name: str = None, custom_server_url: str = None, force_path_style: bool = None, + disable_checksum_validation: bool = False, ): super().__init__( disabled, @@ -143,6 +144,7 @@ def __init__( self.bucket_name = bucket_name self.custom_server_url = custom_server_url self.force_path_style = force_path_style + self.disable_checksum_validation = disable_checksum_validation @classmethod def from_json(cls, json_dict: Dict[str, Any]) -> S3Settings: @@ -157,6 +159,7 @@ def from_json(cls, json_dict: Dict[str, Any]) -> S3Settings: json_dict["BucketName"], json_dict["CustomServerUrl"], json_dict["ForcePathStyle"], + json_dict.get("DisableChecksumValidation", False), ) def to_json(self) -> Dict[str, Any]: @@ -171,8 +174,34 @@ def to_json(self) -> Dict[str, Any]: "BucketName": self.bucket_name, "CustomServerUrl": self.custom_server_url, "ForcePathStyle": self.force_path_style, + "DisableChecksumValidation": self.disable_checksum_validation, } + def __eq__(self, other) -> bool: + if not isinstance(other, S3Settings): + return False + + return ( + self.aws_region_name == other.aws_region_name + and self.bucket_name == other.bucket_name + and self.remote_folder_name == other.remote_folder_name + and self.custom_server_url == other.custom_server_url + and self.force_path_style == other.force_path_style + and self.disable_checksum_validation == other.disable_checksum_validation + ) + + def __hash__(self) -> int: + return hash( + ( + self.aws_region_name, + self.bucket_name, + self.remote_folder_name, + self.custom_server_url, + self.force_path_style, + self.disable_checksum_validation, + ) + ) + class GlacierSettings(AmazonSettings): def __init__( diff --git a/ravendb/documents/operations/cdc_sink/__init__.py b/ravendb/documents/operations/cdc_sink/__init__.py new file mode 100644 index 00000000..b65269f5 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/__init__.py @@ -0,0 +1,43 @@ +from ravendb.documents.operations.cdc_sink.add_cdc_sink_operation import ( + AddCdcSinkOperation, + AddCdcSinkOperationResult, +) +from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import ( + CdcColumnMapping, + CdcColumnType, + CdcSinkConfiguration, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcSinkRelationType, + CdcSinkTableConfig, +) +from ravendb.documents.operations.cdc_sink.cdc_sink_task_state import ( + CdcSinkTableLoadState, + CdcSinkTablesDict, + CdcSinkTaskState, +) +from ravendb.documents.operations.cdc_sink.update_cdc_sink_operation import ( + UpdateCdcSinkOperation, + UpdateCdcSinkOperationResult, +) + +__all__ = [ + "AddCdcSinkOperation", + "AddCdcSinkOperationResult", + "UpdateCdcSinkOperation", + "UpdateCdcSinkOperationResult", + "CdcSinkConfiguration", + "CdcSinkTableConfig", + "CdcSinkEmbeddedTableConfig", + "CdcSinkLinkedTableConfig", + "CdcSinkOnDeleteConfig", + "CdcSinkPostgresSettings", + "CdcColumnMapping", + "CdcColumnType", + "CdcSinkRelationType", + "CdcSinkTaskState", + "CdcSinkTableLoadState", + "CdcSinkTablesDict", +] diff --git a/ravendb/documents/operations/cdc_sink/add_cdc_sink_operation.py b/ravendb/documents/operations/cdc_sink/add_cdc_sink_operation.py new file mode 100644 index 00000000..cabe5189 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/add_cdc_sink_operation.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, TYPE_CHECKING + +import requests + +from ravendb.documents.conventions import DocumentConventions +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration + + +class AddCdcSinkOperationResult: + def __init__(self, raft_command_index: int = 0, task_id: int = 0): + self.raft_command_index = raft_command_index + self.task_id = task_id + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "AddCdcSinkOperationResult": + return cls( + raft_command_index=json_dict.get("RaftCommandIndex", 0), + task_id=json_dict.get("TaskId", 0), + ) + + +class AddCdcSinkOperation(MaintenanceOperation[AddCdcSinkOperationResult]): + def __init__(self, configuration: "CdcSinkConfiguration"): + if configuration is None: + raise ValueError("configuration cannot be None") + self._configuration = configuration + + def get_command(self, conventions: DocumentConventions) -> RavenCommand[AddCdcSinkOperationResult]: + return AddCdcSinkCommand(conventions, self._configuration) + + +class AddCdcSinkCommand(RavenCommand[AddCdcSinkOperationResult], RaftCommand): + def __init__(self, conventions: DocumentConventions, configuration: "CdcSinkConfiguration"): + super().__init__(AddCdcSinkOperationResult) + self._conventions = conventions + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink" + + request = requests.Request("PUT", url) + request.headers = {"Content-Type": "application/json"} + request.data = json.dumps(self._configuration.to_json()) + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = AddCdcSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() diff --git a/ravendb/documents/operations/cdc_sink/cdc_sink_configuration.py b/ravendb/documents/operations/cdc_sink/cdc_sink_configuration.py new file mode 100644 index 00000000..eee6e9c9 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/cdc_sink_configuration.py @@ -0,0 +1,294 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, List, Optional + + +class CdcColumnType(Enum): + DEFAULT = "Default" + JSON = "Json" + ATTACHMENT = "Attachment" + + +class CdcSinkRelationType(Enum): + ARRAY = "Array" + MAP = "Map" + VALUE = "Value" + + +class CdcColumnMapping: + def __init__( + self, + column: Optional[str] = None, + name: Optional[str] = None, + type: CdcColumnType = CdcColumnType.DEFAULT, + ): + self.column = column + self.name = name + self.type = type + + def to_json(self) -> Dict[str, Any]: + json_dict = {"Column": self.column, "Name": self.name} + if self.type != CdcColumnType.DEFAULT: + json_dict["Type"] = self.type.value + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcColumnMapping": + type_raw = json_dict.get("Type") + return cls( + column=json_dict.get("Column"), + name=json_dict.get("Name"), + type=CdcColumnType(type_raw) if type_raw is not None else CdcColumnType.DEFAULT, + ) + + +class CdcSinkOnDeleteConfig: + def __init__(self, patch: Optional[str] = None, ignore_deletes: bool = False): + self.patch = patch + self.ignore_deletes = ignore_deletes + + def to_json(self) -> Dict[str, Any]: + return {"Patch": self.patch, "IgnoreDeletes": self.ignore_deletes} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkOnDeleteConfig": + return cls( + patch=json_dict.get("Patch"), + ignore_deletes=json_dict.get("IgnoreDeletes", False), + ) + + +class CdcSinkPostgresSettings: + def __init__(self, publication_name: Optional[str] = None, slot_name: Optional[str] = None): + self.publication_name = publication_name + self.slot_name = slot_name + + def to_json(self) -> Dict[str, Any]: + return {"PublicationName": self.publication_name, "SlotName": self.slot_name} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkPostgresSettings": + return cls( + publication_name=json_dict.get("PublicationName"), + slot_name=json_dict.get("SlotName"), + ) + + +class CdcSinkLinkedTableConfig: + def __init__( + self, + source_table_schema: Optional[str] = None, + source_table_name: Optional[str] = None, + property_name: Optional[str] = None, + join_columns: Optional[List[str]] = None, + linked_collection_name: Optional[str] = None, + ): + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.property_name = property_name + self.join_columns = join_columns if join_columns is not None else [] + self.linked_collection_name = linked_collection_name + + def to_json(self) -> Dict[str, Any]: + return { + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "PropertyName": self.property_name, + "JoinColumns": self.join_columns, + "LinkedCollectionName": self.linked_collection_name, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkLinkedTableConfig": + return cls( + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + property_name=json_dict.get("PropertyName"), + join_columns=json_dict.get("JoinColumns") or [], + linked_collection_name=json_dict.get("LinkedCollectionName"), + ) + + +class CdcSinkEmbeddedTableConfig: + def __init__( + self, + source_table_schema: Optional[str] = None, + source_table_name: Optional[str] = None, + property_name: Optional[str] = None, + columns: Optional[List[CdcColumnMapping]] = None, + primary_key_columns: Optional[List[str]] = None, + join_columns: Optional[List[str]] = None, + type: CdcSinkRelationType = CdcSinkRelationType.ARRAY, + patch: Optional[str] = None, + on_delete: Optional[CdcSinkOnDeleteConfig] = None, + case_sensitive_keys: bool = False, + embedded_tables: Optional[List["CdcSinkEmbeddedTableConfig"]] = None, + linked_tables: Optional[List[CdcSinkLinkedTableConfig]] = None, + ): + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.property_name = property_name + self.columns = columns if columns is not None else [] + self.primary_key_columns = primary_key_columns if primary_key_columns is not None else [] + self.join_columns = join_columns if join_columns is not None else [] + self.type = type + self.patch = patch + self.on_delete = on_delete + self.case_sensitive_keys = case_sensitive_keys + self.embedded_tables = embedded_tables if embedded_tables is not None else [] + self.linked_tables = linked_tables if linked_tables is not None else [] + + def to_json(self) -> Dict[str, Any]: + return { + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "PropertyName": self.property_name, + "Columns": [column.to_json() for column in self.columns], + "PrimaryKeyColumns": self.primary_key_columns, + "JoinColumns": self.join_columns, + "Type": self.type.value, + "Patch": self.patch, + "OnDelete": self.on_delete.to_json() if self.on_delete is not None else None, + "CaseSensitiveKeys": self.case_sensitive_keys, + "EmbeddedTables": [embedded.to_json() for embedded in self.embedded_tables], + "LinkedTables": [linked.to_json() for linked in self.linked_tables], + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkEmbeddedTableConfig": + type_raw = json_dict.get("Type") + on_delete_raw = json_dict.get("OnDelete") + return cls( + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + property_name=json_dict.get("PropertyName"), + columns=[CdcColumnMapping.from_json(column) for column in (json_dict.get("Columns") or [])], + primary_key_columns=json_dict.get("PrimaryKeyColumns") or [], + join_columns=json_dict.get("JoinColumns") or [], + type=CdcSinkRelationType(type_raw) if type_raw is not None else CdcSinkRelationType.ARRAY, + patch=json_dict.get("Patch"), + on_delete=CdcSinkOnDeleteConfig.from_json(on_delete_raw) if on_delete_raw is not None else None, + case_sensitive_keys=json_dict.get("CaseSensitiveKeys", False), + embedded_tables=[ + CdcSinkEmbeddedTableConfig.from_json(embedded) for embedded in (json_dict.get("EmbeddedTables") or []) + ], + linked_tables=[ + CdcSinkLinkedTableConfig.from_json(linked) for linked in (json_dict.get("LinkedTables") or []) + ], + ) + + +class CdcSinkTableConfig: + def __init__( + self, + collection_name: Optional[str] = None, + source_table_schema: Optional[str] = None, + source_table_name: Optional[str] = None, + columns: Optional[List[CdcColumnMapping]] = None, + primary_key_columns: Optional[List[str]] = None, + patch: Optional[str] = None, + on_delete: Optional[CdcSinkOnDeleteConfig] = None, + disabled: bool = False, + embedded_tables: Optional[List[CdcSinkEmbeddedTableConfig]] = None, + linked_tables: Optional[List[CdcSinkLinkedTableConfig]] = None, + ): + self.collection_name = collection_name + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.columns = columns if columns is not None else [] + self.primary_key_columns = primary_key_columns if primary_key_columns is not None else [] + self.patch = patch + self.on_delete = on_delete + self.disabled = disabled + self.embedded_tables = embedded_tables if embedded_tables is not None else [] + self.linked_tables = linked_tables if linked_tables is not None else [] + + def to_json(self) -> Dict[str, Any]: + return { + "CollectionName": self.collection_name, + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "Columns": [column.to_json() for column in self.columns], + "PrimaryKeyColumns": self.primary_key_columns, + "Patch": self.patch, + "OnDelete": self.on_delete.to_json() if self.on_delete is not None else None, + "Disabled": self.disabled, + "EmbeddedTables": [embedded.to_json() for embedded in self.embedded_tables], + "LinkedTables": [linked.to_json() for linked in self.linked_tables], + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkTableConfig": + on_delete_raw = json_dict.get("OnDelete") + return cls( + collection_name=json_dict.get("CollectionName"), + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + columns=[CdcColumnMapping.from_json(column) for column in (json_dict.get("Columns") or [])], + primary_key_columns=json_dict.get("PrimaryKeyColumns") or [], + patch=json_dict.get("Patch"), + on_delete=CdcSinkOnDeleteConfig.from_json(on_delete_raw) if on_delete_raw is not None else None, + disabled=json_dict.get("Disabled", False), + embedded_tables=[ + CdcSinkEmbeddedTableConfig.from_json(embedded) for embedded in (json_dict.get("EmbeddedTables") or []) + ], + linked_tables=[ + CdcSinkLinkedTableConfig.from_json(linked) for linked in (json_dict.get("LinkedTables") or []) + ], + ) + + +class CdcSinkConfiguration: + def __init__( + self, + task_id: int = 0, + disabled: bool = False, + name: Optional[str] = None, + mentor_node: Optional[str] = None, + pin_to_mentor_node: bool = False, + connection_string_name: Optional[str] = None, + tables: Optional[List[CdcSinkTableConfig]] = None, + postgres: Optional[CdcSinkPostgresSettings] = None, + skip_initial_load: bool = False, + ): + self.task_id = task_id + self.disabled = disabled + self.name = name + self.mentor_node = mentor_node + self.pin_to_mentor_node = pin_to_mentor_node + self.connection_string_name = connection_string_name + self.tables = tables if tables is not None else [] + self.postgres = postgres + self.skip_initial_load = skip_initial_load + + def get_default_task_name(self) -> str: + return f"CDC Sink to {self.connection_string_name}" + + def to_json(self) -> Dict[str, Any]: + return { + "Name": self.name, + "TaskId": self.task_id, + "Disabled": self.disabled, + "ConnectionStringName": self.connection_string_name, + "MentorNode": self.mentor_node, + "PinToMentorNode": self.pin_to_mentor_node, + "Tables": [table.to_json() for table in self.tables], + "Postgres": self.postgres.to_json() if self.postgres is not None else None, + "SkipInitialLoad": self.skip_initial_load, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkConfiguration": + postgres_raw = json_dict.get("Postgres") + return cls( + task_id=json_dict.get("TaskId", 0), + disabled=json_dict.get("Disabled", False), + name=json_dict.get("Name"), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode", False), + connection_string_name=json_dict.get("ConnectionStringName"), + tables=[CdcSinkTableConfig.from_json(table) for table in (json_dict.get("Tables") or [])], + postgres=CdcSinkPostgresSettings.from_json(postgres_raw) if postgres_raw is not None else None, + skip_initial_load=json_dict.get("SkipInitialLoad", False), + ) diff --git a/ravendb/documents/operations/cdc_sink/cdc_sink_task_state.py b/ravendb/documents/operations/cdc_sink/cdc_sink_task_state.py new file mode 100644 index 00000000..481512b7 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/cdc_sink_task_state.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +from typing import Any, Dict, Iterator, List, Optional, Tuple + + +class CdcSinkTablesDict(dict): + """A dict whose string-key lookups ignore case while iteration keeps the stored key casing. + + Mirrors the C# Dictionary(StringComparer.OrdinalIgnoreCase): + lookups compare case-insensitively, but enumeration yields the original keys. + """ + + def __init__(self, *args, **kwargs): + super().__init__() + self._original_keys: Dict[str, str] = {} + self.update(*args, **kwargs) + + @staticmethod + def _lower(key): + return key.lower() if isinstance(key, str) else key + + def __setitem__(self, key, value) -> None: + lowered = self._lower(key) + self._original_keys[lowered] = key + super().__setitem__(lowered, value) + + def __getitem__(self, key): + return super().__getitem__(self._lower(key)) + + def __delitem__(self, key) -> None: + lowered = self._lower(key) + self._original_keys.pop(lowered, None) + super().__delitem__(lowered) + + def __contains__(self, key) -> bool: + return super().__contains__(self._lower(key)) + + def get(self, key, default=None): + return super().get(self._lower(key), default) + + def pop(self, key, *args): + lowered = self._lower(key) + self._original_keys.pop(lowered, None) + return super().pop(lowered, *args) + + def setdefault(self, key, default=None): + if key not in self: + self[key] = default + return self[key] + + def update(self, *args, **kwargs) -> None: + for key, value in dict(*args, **kwargs).items(): + self[key] = value + + def keys(self): + return list(self._original_keys.values()) + + def items(self): + return [(self._original_keys[lowered], value) for lowered, value in super().items()] + + def values(self): + return list(super().values()) + + def __iter__(self) -> Iterator: + for key in self._original_keys.values(): + yield key + + def __len__(self) -> int: + return super().__len__() + + +class CdcSinkTableLoadState: + def __init__( + self, + initial_load_completed: bool = False, + last_key_values: Optional[List[str]] = None, + key_columns: Optional[List[str]] = None, + ): + self.initial_load_completed = initial_load_completed + self.last_key_values = last_key_values + self.key_columns = key_columns + + def to_json(self) -> Dict[str, Any]: + return { + "InitialLoadCompleted": self.initial_load_completed, + "LastKeyValues": self.last_key_values if self.last_key_values is not None else None, + "KeyColumns": self.key_columns if self.key_columns is not None else None, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkTableLoadState": + return cls( + initial_load_completed=json_dict.get("InitialLoadCompleted", False), + last_key_values=json_dict.get("LastKeyValues"), + key_columns=json_dict.get("KeyColumns"), + ) + + +class CdcSinkTaskState: + collection_name = "@cdc-states" + + def __init__( + self, + last_lsn: Optional[str] = None, + tables: Optional[CdcSinkTablesDict] = None, + configuration_name: Optional[str] = None, + ): + self.last_lsn = last_lsn + self.tables = tables if tables is not None else CdcSinkTablesDict() + self.configuration_name = configuration_name + + @classmethod + def get_document_id(cls, configuration_name: str) -> str: + return f"{cls.collection_name}/{configuration_name}" + + def to_json(self) -> Dict[str, Any]: + tables_json: Dict[str, Any] = {} + for key in self.tables.keys(): + tables_json[key] = self.tables[key].to_json() + return { + "ConfigurationName": self.configuration_name, + "LastLsn": self.last_lsn, + "Tables": tables_json, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkTaskState": + tables_raw = json_dict.get("Tables") + tables = CdcSinkTablesDict() + if tables_raw: + for key, value in tables_raw.items(): + tables[key] = CdcSinkTableLoadState.from_json(value) + return cls( + last_lsn=json_dict.get("LastLsn"), + tables=tables, + configuration_name=json_dict.get("ConfigurationName"), + ) diff --git a/ravendb/documents/operations/cdc_sink/update_cdc_sink_operation.py b/ravendb/documents/operations/cdc_sink/update_cdc_sink_operation.py new file mode 100644 index 00000000..7c066327 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/update_cdc_sink_operation.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, TYPE_CHECKING + +import requests + +from ravendb.documents.conventions import DocumentConventions +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration + + +class UpdateCdcSinkOperationResult: + def __init__(self, raft_command_index: int = 0, task_id: int = 0): + self.raft_command_index = raft_command_index + self.task_id = task_id + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "UpdateCdcSinkOperationResult": + return cls( + raft_command_index=json_dict.get("RaftCommandIndex", 0), + task_id=json_dict.get("TaskId", 0), + ) + + +class UpdateCdcSinkOperation(MaintenanceOperation[UpdateCdcSinkOperationResult]): + def __init__(self, task_id: int, configuration: "CdcSinkConfiguration"): + if configuration is None: + raise ValueError("configuration cannot be None") + self._task_id = task_id + self._configuration = configuration + + def get_command(self, conventions: DocumentConventions) -> RavenCommand[UpdateCdcSinkOperationResult]: + return UpdateCdcSinkCommand(conventions, self._task_id, self._configuration) + + +class UpdateCdcSinkCommand(RavenCommand[UpdateCdcSinkOperationResult], RaftCommand): + def __init__(self, conventions: DocumentConventions, task_id: int, configuration: "CdcSinkConfiguration"): + super().__init__(UpdateCdcSinkOperationResult) + self._conventions = conventions + self._task_id = task_id + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink?id={self._task_id}" + + request = requests.Request("PUT", url) + request.headers = {"Content-Type": "application/json"} + request.data = json.dumps(self._configuration.to_json()) + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = UpdateCdcSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() diff --git a/ravendb/documents/operations/connection_strings.py b/ravendb/documents/operations/connection_strings.py index be5c9a44..bf69d730 100644 --- a/ravendb/documents/operations/connection_strings.py +++ b/ravendb/documents/operations/connection_strings.py @@ -1,10 +1,36 @@ from abc import abstractmethod -from typing import Dict, Any +from typing import Any, Dict, List, Optional + + +class ConnectionStringUsage: + """A single usage of a connection string, as reported by the server on GET.""" + + def __init__( + self, + kind: Optional[str] = None, + id: Optional[int] = None, + identifier: Optional[str] = None, + name: Optional[str] = None, + ): + self.kind = kind + self.id = id + self.identifier = identifier + self.name = name + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "ConnectionStringUsage": + return cls( + kind=json_dict.get("Kind"), + id=json_dict.get("Id"), + identifier=json_dict.get("Identifier"), + name=json_dict.get("Name"), + ) class ConnectionString: - def __init__(self, name: str): + def __init__(self, name: str, used_by: Optional[List[ConnectionStringUsage]] = None): self.name = name + self.used_by = used_by if used_by is not None else [] @abstractmethod def get_type(self): diff --git a/ravendb/documents/operations/etl/configuration.py b/ravendb/documents/operations/etl/configuration.py index 205a4757..bee0f13a 100644 --- a/ravendb/documents/operations/etl/configuration.py +++ b/ravendb/documents/operations/etl/configuration.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from typing import Optional, Generic, TypeVar, List, Dict, Any -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage from ravendb.documents.operations.etl.etl_type import EtlType from ravendb.documents.operations.etl.transformation import Transformation import ravendb.serverwide.server_operation_executor @@ -11,8 +11,14 @@ class RavenConnectionString(ConnectionString): - def __init__(self, name: str, database: Optional[str] = None, topology_discovery_urls: Optional[List[str]] = None): - super().__init__(name) + def __init__( + self, + name: str, + database: Optional[str] = None, + topology_discovery_urls: Optional[List[str]] = None, + used_by: Optional[List[ConnectionStringUsage]] = None, + ): + super().__init__(name, used_by) self.database = database self.topology_discovery_urls = topology_discovery_urls @@ -34,6 +40,7 @@ def from_json(cls, json_dict: Dict) -> "RavenConnectionString": name=json_dict["Name"], database=json_dict["Database"], topology_discovery_urls=json_dict["TopologyDiscoveryUrls"], + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/etl/elastic_search/connection.py b/ravendb/documents/operations/etl/elastic_search/connection.py index aa464574..6f4ffafa 100644 --- a/ravendb/documents/operations/etl/elastic_search/connection.py +++ b/ravendb/documents/operations/etl/elastic_search/connection.py @@ -1,6 +1,6 @@ -from typing import List, Dict, Any +from typing import List, Dict, Any, Optional -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage from ravendb.serverwide.server_operation_executor import ConnectionStringType @@ -92,8 +92,14 @@ def from_json(cls, data: Dict[str, Any]) -> "Authentication": class ElasticSearchConnectionString(ConnectionString): - def __init__(self, name: str, nodes: List[str] = None, authentication: Authentication = None): - super().__init__(name) + def __init__( + self, + name: str, + nodes: List[str] = None, + authentication: Authentication = None, + used_by: Optional[List[ConnectionStringUsage]] = None, + ): + super().__init__(name, used_by) self.nodes = nodes self.authentication = authentication @@ -117,4 +123,5 @@ def from_json(cls, json_dict: Dict[str, Any]) -> Any: authentication=( Authentication.from_json(json_dict["Authentication"]) if json_dict["Authentication"] else None ), + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/etl/olap/connection.py b/ravendb/documents/operations/etl/olap/connection.py index f6e7ea76..a5654380 100644 --- a/ravendb/documents/operations/etl/olap/connection.py +++ b/ravendb/documents/operations/etl/olap/connection.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, List from ravendb.documents.operations.backups.settings import ( LocalSettings, @@ -8,7 +8,7 @@ GoogleCloudSettings, FtpSettings, ) -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage import ravendb.serverwide.server_operation_executor from ravendb.documents.operations.etl.configuration import EtlConfiguration @@ -23,8 +23,9 @@ def __init__( glacier_settings: Optional[GlacierSettings] = None, google_cloud_settings: Optional[GoogleCloudSettings] = None, ftp_settings: Optional[FtpSettings] = None, + used_by: Optional[List[ConnectionStringUsage]] = None, ): - super().__init__(name) + super().__init__(name, used_by) self.local_settings = local_settings self.s3_settings = s3_settings self.azure_settings = azure_settings @@ -64,6 +65,7 @@ def from_json(cls, json_dict: dict) -> "OlapConnectionString": else None ), ftp_settings=FtpSettings.from_json(json_dict["FtpSettings"]) if json_dict["FtpSettings"] else None, + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py b/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py new file mode 100644 index 00000000..78c863ca --- /dev/null +++ b/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +from typing import Any, Dict, Optional + +from ravendb.exceptions.exceptions import InvalidOperationException + + +class AzureServiceBusEntraId: + def __init__( + self, + namespace: Optional[str] = None, + tenant_id: Optional[str] = None, + client_id: Optional[str] = None, + client_secret: Optional[str] = None, + ): + self.namespace = namespace + self.tenant_id = tenant_id + self.client_id = client_id + self.client_secret = client_secret + + def is_valid(self) -> bool: + return ( + not _is_blank(self.namespace) + and not _is_blank(self.tenant_id) + and not _is_blank(self.client_id) + and not _is_blank(self.client_secret) + ) + + def to_json(self) -> Dict[str, Any]: + return { + "Namespace": self.namespace, + "TenantId": self.tenant_id, + "ClientId": self.client_id, + "ClientSecret": self.client_secret, + } + + def to_audit_json(self) -> Dict[str, Any]: + # ClientSecret is masked in audit output. + return { + "Namespace": self.namespace, + "TenantId": self.tenant_id, + "ClientId": self.client_id, + } + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]) -> AzureServiceBusEntraId: + return cls( + namespace=json_dict.get("Namespace"), + tenant_id=json_dict.get("TenantId"), + client_id=json_dict.get("ClientId"), + client_secret=json_dict.get("ClientSecret"), + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AzureServiceBusEntraId): + return False + return ( + self.namespace == other.namespace + and self.tenant_id == other.tenant_id + and self.client_id == other.client_id + and self.client_secret == other.client_secret + ) + + def __hash__(self) -> int: + return hash((self.namespace, self.tenant_id, self.client_id, self.client_secret)) + + +class AzureServiceBusPasswordless: + """Machine authentication (Managed Identity); only a namespace is required.""" + + def __init__(self, namespace: Optional[str] = None): + self.namespace = namespace + + def is_valid(self) -> bool: + return not _is_blank(self.namespace) + + def to_json(self) -> Dict[str, Any]: + return {"Namespace": self.namespace} + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]) -> AzureServiceBusPasswordless: + return cls(namespace=json_dict.get("Namespace")) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AzureServiceBusPasswordless): + return False + return self.namespace == other.namespace + + def __hash__(self) -> int: + return hash(self.namespace) + + +def _is_blank(value: Optional[str]) -> bool: + return value is None or (isinstance(value, str) and value.strip() == "") + + +class AzureServiceBusConnectionSettings: + def __init__( + self, + connection_string: Optional[str] = None, + entra_id: Optional[AzureServiceBusEntraId] = None, + passwordless: Optional[AzureServiceBusPasswordless] = None, + ): + self.connection_string = connection_string + self.entra_id = entra_id + self.passwordless = passwordless + + def is_valid_connection(self) -> bool: + if not self._is_only_one_connection_provided(): + return False + + if self.entra_id is not None and self.entra_id.is_valid(): + return True + + if self.passwordless is not None and self.passwordless.is_valid(): + return True + + return self._try_extract_endpoint() is not None + + def _is_only_one_connection_provided(self) -> bool: + count = 0 + if self.entra_id is not None: + count += 1 + if not _is_blank(self.connection_string): + count += 1 + if self.passwordless is not None: + count += 1 + return count == 1 + + def get_service_bus_url(self) -> str: + if not _is_blank(self.connection_string): + endpoint = self._try_extract_endpoint() + if endpoint is not None: + return endpoint + raise InvalidOperationException("No endpoint provided") + + return f"sb://{self._get_namespace()}/" + + def _try_extract_endpoint(self) -> Optional[str]: + if not self.connection_string: + return None + + # Case-insensitive search, matching IndexOf(OrdinalIgnoreCase) in the C# client. + lowered = self.connection_string.lower() + start = lowered.find("sb://") + if start < 0: + return None + + end = self.connection_string.find(";", start) + if end < 0: + return self.connection_string[start:] + + return self.connection_string[start:end] + + def _get_namespace(self) -> str: + if self.entra_id is not None: + return self.entra_id.namespace + + if self.passwordless is not None: + return self.passwordless.namespace + + raise InvalidOperationException("No namespace provided") + + def to_json(self) -> Dict[str, Any]: + # Only set auth fields are written. The connection string is written when it is + # not None and not "" (IsNullOrEmpty), so a whitespace-only value IS written even + # though the exactly-one rule treats it as unset. + json_dict = {} + if self.connection_string: + json_dict["ConnectionString"] = self.connection_string + if self.entra_id is not None: + json_dict["EntraId"] = self.entra_id.to_json() + if self.passwordless is not None: + json_dict["Passwordless"] = self.passwordless.to_json() + return json_dict + + def to_audit_json(self) -> Dict[str, Any]: + json_dict = {} + if self.connection_string: + json_dict["ConnectionString"] = "" + if self.entra_id is not None: + json_dict["EntraId"] = self.entra_id.to_audit_json() + if self.passwordless is not None: + json_dict["Passwordless"] = self.passwordless.to_json() + return json_dict + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]) -> AzureServiceBusConnectionSettings: + entra_id = json_dict.get("EntraId") + passwordless = json_dict.get("Passwordless") + return cls( + connection_string=json_dict.get("ConnectionString"), + entra_id=AzureServiceBusEntraId.from_json(entra_id) if entra_id else None, + passwordless=AzureServiceBusPasswordless.from_json(passwordless) if passwordless else None, + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AzureServiceBusConnectionSettings): + return False + return ( + self.connection_string == other.connection_string + and self.entra_id == other.entra_id + and self.passwordless == other.passwordless + ) + + def __hash__(self) -> int: + return hash((self.connection_string, self.entra_id, self.passwordless)) diff --git a/ravendb/documents/operations/etl/queue/azure_service_bus_sink_source.py b/ravendb/documents/operations/etl/queue/azure_service_bus_sink_source.py new file mode 100644 index 00000000..a02508ab --- /dev/null +++ b/ravendb/documents/operations/etl/queue/azure_service_bus_sink_source.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from typing import List, Optional, Tuple + + +class AzureServiceBusSinkSource: + """Helpers for encoding Azure Service Bus sources as strings. + + Encoding convention: a plain queue name is a Service Bus queue, and + "topic;subscription" is a topic subscription. The ';' separator is + collision-safe because Service Bus naming rules forbid it in names. + """ + + SEPARATOR = ";" + + @staticmethod + def queue(queue_name: str) -> str: + if not queue_name or (isinstance(queue_name, str) and queue_name.isspace()): + raise ValueError("Queue name must be non-empty.") + if AzureServiceBusSinkSource.SEPARATOR in queue_name: + raise ValueError("Queue name must not contain the ';' character.") + return queue_name + + @staticmethod + def subscription(topic_name: str, subscription_name: str) -> str: + if not topic_name or (isinstance(topic_name, str) and topic_name.isspace()): + raise ValueError("Topic name must be non-empty.") + if not subscription_name or (isinstance(subscription_name, str) and subscription_name.isspace()): + raise ValueError("Subscription name must be non-empty.") + if AzureServiceBusSinkSource.SEPARATOR in topic_name: + raise ValueError("Topic name must not contain the ';' character.") + if AzureServiceBusSinkSource.SEPARATOR in subscription_name: + raise ValueError("Subscription name must not contain the ';' character.") + return f"{topic_name}{AzureServiceBusSinkSource.SEPARATOR}{subscription_name}" + + @staticmethod + def validate_entry(entry: str) -> Optional[str]: + """Returns None when the entry is a valid queue name or a valid + topic;subscription pair, otherwise the error message.""" + if not entry or (isinstance(entry, str) and entry.isspace()): + return "Azure Service Bus source entry cannot be empty." + + if AzureServiceBusSinkSource.SEPARATOR in entry and not AzureServiceBusSinkSource.try_parse_subscription(entry): + return ( + f"Azure Service Bus subscription source '{entry}' is invalid. " + f"Use '{AzureServiceBusSinkSource.SEPARATOR}' with a single " + f"'{AzureServiceBusSinkSource.SEPARATOR}' separator and both parts non-empty." + ) + + return None + + @staticmethod + def validate_script(script_name: str, queues: Optional[List[str]]) -> List[str]: + errors = [] + if queues is None: + return errors + + for entry in queues: + error = AzureServiceBusSinkSource.validate_entry(entry) + if error is None: + continue + errors.append(f"Script '{script_name}': {error}") + + return errors + + @staticmethod + def try_parse_subscription(entry: str) -> Optional[Tuple[str, str]]: + if entry is None: + return None + + parts = entry.split(AzureServiceBusSinkSource.SEPARATOR) + if len(parts) != 2: + return None + + if not parts[0] or parts[0].isspace() or not parts[1] or parts[1].isspace(): + return None + + return parts[0], parts[1] diff --git a/ravendb/documents/operations/etl/queue/connection.py b/ravendb/documents/operations/etl/queue/connection.py index 985d65d0..6c20441f 100644 --- a/ravendb/documents/operations/etl/queue/connection.py +++ b/ravendb/documents/operations/etl/queue/connection.py @@ -1,11 +1,15 @@ from enum import Enum +from typing import List, Optional -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage import ravendb.serverwide.server_operation_executor from ravendb.documents.operations.etl.queue.amazon_sqs_connection_settings import AmazonSqsConnectionSettings from ravendb.documents.operations.etl.queue.azure_queue_storage_connection_settings import ( AzureQueueStorageConnectionSettings, ) +from ravendb.documents.operations.etl.queue.azure_service_bus_connection_settings import ( + AzureServiceBusConnectionSettings, +) from ravendb.documents.operations.etl.queue.kafka_connection_settings import KafkaConnectionSettings from ravendb.documents.operations.etl.queue.rabbit_mq_connection_settings import RabbitMqConnectionSettings @@ -16,6 +20,7 @@ class QueueBrokerType(Enum): RABBIT_MQ = "RabbitMq" AZURE_QUEUE_STORAGE = "AzureQueueStorage" AMAZON_SQS = "AmazonSqs" + AZURE_SERVICE_BUS = "AzureServiceBus" class QueueConnectionString(ConnectionString): @@ -27,13 +32,16 @@ def __init__( rabbit_mq_settings: RabbitMqConnectionSettings = None, azure_queue_storage_settings: AzureQueueStorageConnectionSettings = None, amazon_sqs_settings: AmazonSqsConnectionSettings = None, + azure_service_bus_settings: AzureServiceBusConnectionSettings = None, + used_by: Optional[List[ConnectionStringUsage]] = None, ): - super().__init__(name) + super().__init__(name, used_by) self.broker_type = broker_type self.kafka_settings = kafka_settings self.rabbit_mq_settings = rabbit_mq_settings self.azure_queue_storage_settings = azure_queue_storage_settings self.amazon_sqs_settings = amazon_sqs_settings + self.azure_service_bus_settings = azure_service_bus_settings @property def get_type(self): @@ -49,6 +57,9 @@ def to_json(self): self.azure_queue_storage_settings.to_json() if self.azure_queue_storage_settings else None ), "AmazonSqsConnectionSettings": self.amazon_sqs_settings.to_json() if self.amazon_sqs_settings else None, + "AzureServiceBusConnectionSettings": ( + self.azure_service_bus_settings.to_json() if self.azure_service_bus_settings else None + ), "Type": ravendb.serverwide.server_operation_executor.ConnectionStringType.QUEUE, } @@ -77,4 +88,10 @@ def from_json(cls, json_dict: dict) -> "QueueConnectionString": if json_dict["AmazonSqsConnectionSettings"] else None ), + azure_service_bus_settings=( + AzureServiceBusConnectionSettings.from_json(json_dict["AzureServiceBusConnectionSettings"]) + if json_dict.get("AzureServiceBusConnectionSettings") + else None + ), + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/etl/snowflake/connection.py b/ravendb/documents/operations/etl/snowflake/connection.py index b2d0e87b..e622d73f 100644 --- a/ravendb/documents/operations/etl/snowflake/connection.py +++ b/ravendb/documents/operations/etl/snowflake/connection.py @@ -1,12 +1,17 @@ -from typing import Optional +from typing import Optional, List -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage import ravendb.serverwide.server_operation_executor class SnowflakeConnectionString(ConnectionString): - def __init__(self, name: str, connection_string: Optional[str] = None): - super().__init__(name) + def __init__( + self, + name: str, + connection_string: Optional[str] = None, + used_by: Optional[List[ConnectionStringUsage]] = None, + ): + super().__init__(name, used_by) self.connection_string = connection_string @property @@ -25,4 +30,5 @@ def from_json(cls, json_dict: dict) -> "SnowflakeConnectionString": return cls( name=json_dict["Name"], connection_string=json_dict["ConnectionString"], + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/etl/sql/__init__.py b/ravendb/documents/operations/etl/sql/__init__.py index bcd5ec39..10f0d200 100644 --- a/ravendb/documents/operations/etl/sql/__init__.py +++ b/ravendb/documents/operations/etl/sql/__init__.py @@ -1,13 +1,19 @@ -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, List -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage import ravendb.serverwide.server_operation_executor from ravendb.documents.operations.etl.configuration import EtlConfiguration class SqlConnectionString(ConnectionString): - def __init__(self, name: str, connection_string: Optional[str] = None, factory_name: Optional[str] = None): - super().__init__(name) + def __init__( + self, + name: str, + connection_string: Optional[str] = None, + factory_name: Optional[str] = None, + used_by: Optional[List[ConnectionStringUsage]] = None, + ): + super().__init__(name, used_by) self.connection_string = connection_string self.factory_name = factory_name @@ -29,6 +35,7 @@ def from_json(cls, json_dict: Dict[str, Any]) -> "SqlConnectionString": name=json_dict["Name"], connection_string=json_dict["ConnectionString"], factory_name=json_dict["FactoryName"], + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/ongoing_tasks.py b/ravendb/documents/operations/ongoing_tasks.py index 62b869c6..8b8c7bc4 100644 --- a/ravendb/documents/operations/ongoing_tasks.py +++ b/ravendb/documents/operations/ongoing_tasks.py @@ -18,6 +18,7 @@ from ravendb.documents.conventions import DocumentConventions from ravendb.documents.operations.ai.gen_ai_configuration import GenAiConfiguration from ravendb.documents.operations.ai.embeddings_generation_configuration import EmbeddingsGenerationConfiguration + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration class OngoingTaskType(Enum): @@ -33,6 +34,7 @@ class OngoingTaskType(Enum): PULL_REPLICATION_AS_HUB = "PullReplicationAsHub" PULL_REPLICATION_AS_SINK = "PullReplicationAsSink" QUEUE_SINK = "QueueSink" + CDC_SINK = "CdcSink" EMBEDDINGS_GENERATION = "EmbeddingsGeneration" GEN_AI = "GenAi" @@ -280,6 +282,97 @@ def from_json(cls, json_dict: dict) -> "OngoingTaskEmbeddingsGeneration": ) +class OngoingTaskCdcSink(OngoingTask): + """Ongoing task information for CDC Sink tasks.""" + + def __init__( + self, + task_id: Optional[int] = None, + responsible_node: Optional[NodeId] = None, + task_state: Optional[OngoingTaskState] = None, + task_connection_status: Optional[OngoingTaskConnectionStatus] = None, + task_name: Optional[str] = None, + error: Optional[str] = None, + mentor_node: Optional[str] = None, + pin_to_mentor_node: Optional[bool] = None, + configuration: Optional["CdcSinkConfiguration"] = None, + connection_string_name: Optional[str] = None, + factory_name: Optional[str] = None, + last_batch_time=None, + last_checkpoint: Optional[str] = None, + seconds_since_last_batch: Optional[float] = None, + last_activity_time=None, + seconds_since_last_activity: Optional[float] = None, + health_issue: Optional[str] = None, + ): + super().__init__( + task_id=task_id, + task_type=OngoingTaskType.CDC_SINK, + responsible_node=responsible_node, + task_state=task_state, + task_connection_status=task_connection_status, + task_name=task_name, + error=error, + mentor_node=mentor_node, + pin_to_mentor_node=pin_to_mentor_node, + ) + self.configuration = configuration + self.connection_string_name = connection_string_name + self.factory_name = factory_name + self.last_batch_time = last_batch_time + self.last_checkpoint = last_checkpoint + self.seconds_since_last_batch = seconds_since_last_batch + self.last_activity_time = last_activity_time + self.seconds_since_last_activity = seconds_since_last_activity + self.health_issue = health_issue + + def to_json(self) -> dict: + result = super().to_json() + result["ConnectionStringName"] = self.connection_string_name + result["FactoryName"] = self.factory_name + result["Configuration"] = self.configuration.to_json() if self.configuration else None + result["LastBatchTime"] = Utils.datetime_to_string(self.last_batch_time) + result["LastCheckpoint"] = self.last_checkpoint + result["SecondsSinceLastBatch"] = self.seconds_since_last_batch + result["LastActivityTime"] = Utils.datetime_to_string(self.last_activity_time) + result["SecondsSinceLastActivity"] = self.seconds_since_last_activity + result["HealthIssue"] = self.health_issue + return result + + @classmethod + def from_json(cls, json_dict: dict) -> "OngoingTaskCdcSink": + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration + + if json_dict is None: + return None + + task_state_str = json_dict.get("TaskState") + task_connection_status_str = json_dict.get("TaskConnectionStatus") + config_dict = json_dict.get("Configuration") + + return cls( + task_id=json_dict.get("TaskId"), + responsible_node=NodeId.from_json(json_dict.get("ResponsibleNode")), + task_state=OngoingTaskState(task_state_str) if task_state_str else None, + task_connection_status=( + OngoingTaskConnectionStatus(task_connection_status_str) if task_connection_status_str else None + ), + task_name=json_dict.get("TaskName"), + error=json_dict.get("Error"), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode"), + configuration=CdcSinkConfiguration.from_json(config_dict) if config_dict else None, + connection_string_name=json_dict.get("ConnectionStringName"), + factory_name=json_dict.get("FactoryName"), + last_batch_time=Utils.string_to_datetime(json_dict.get("LastBatchTime")), + last_checkpoint=json_dict.get("LastCheckpoint"), + seconds_since_last_batch=json_dict.get("SecondsSinceLastBatch"), + last_activity_time=Utils.string_to_datetime(json_dict.get("LastActivityTime")), + seconds_since_last_activity=json_dict.get("SecondsSinceLastActivity"), + health_issue=json_dict.get("HealthIssue"), + ) + + class OngoingTaskPullReplicationAsHub(OngoingTask): """Ongoing task information for a single pull-replication hub connection.""" @@ -392,6 +485,8 @@ def __init__( access_name: Optional[str] = None, allowed_hub_to_sink_paths: Optional[list] = None, allowed_sink_to_hub_paths: Optional[list] = None, + hub_cursor: Optional[str] = None, + sink_cursor: Optional[str] = None, ): super().__init__( task_id=task_id, @@ -414,6 +509,8 @@ def __init__( self.access_name = access_name self.allowed_hub_to_sink_paths = allowed_hub_to_sink_paths self.allowed_sink_to_hub_paths = allowed_sink_to_hub_paths + self.hub_cursor = hub_cursor + self.sink_cursor = sink_cursor def to_json(self) -> dict: result = super().to_json() @@ -427,6 +524,8 @@ def to_json(self) -> dict: result["AccessName"] = self.access_name result["AllowedHubToSinkPaths"] = self.allowed_hub_to_sink_paths result["AllowedSinkToHubPaths"] = self.allowed_sink_to_hub_paths + result["HubCursor"] = self.hub_cursor + result["SinkCursor"] = self.sink_cursor return result @classmethod @@ -457,6 +556,8 @@ def from_json(cls, json_dict: dict) -> Optional["OngoingTaskPullReplicationAsSin access_name=json_dict.get("AccessName"), allowed_hub_to_sink_paths=json_dict.get("AllowedHubToSinkPaths"), allowed_sink_to_hub_paths=json_dict.get("AllowedSinkToHubPaths"), + hub_cursor=json_dict.get("HubCursor"), + sink_cursor=json_dict.get("SinkCursor"), ) @@ -557,7 +658,7 @@ def get_raft_unique_request_id(self) -> str: class GetOngoingTaskInfoOperation( - MaintenanceOperation[Union[OngoingTask, OngoingTaskGenAi, OngoingTaskEmbeddingsGeneration]] + MaintenanceOperation[Union[OngoingTask, OngoingTaskGenAi, OngoingTaskEmbeddingsGeneration, OngoingTaskCdcSink]] ): """ Operation to retrieve detailed information about a specific ongoing task. @@ -593,14 +694,16 @@ def __init__(self, task_id_or_name: Union[int, str], task_type: OngoingTaskType) def get_command( self, conventions: "DocumentConventions" - ) -> RavenCommand[OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration]: + ) -> RavenCommand[OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration | OngoingTaskCdcSink]: if self._task_name is not None: return GetOngoingTaskInfoOperation._GetOngoingTaskInfoCommand( task_name=self._task_name, task_type=self._task_type ) return GetOngoingTaskInfoOperation._GetOngoingTaskInfoCommand(task_id=self._task_id, task_type=self._task_type) - class _GetOngoingTaskInfoCommand(RavenCommand[OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration]): + class _GetOngoingTaskInfoCommand( + RavenCommand[OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration | OngoingTaskCdcSink] + ): def __init__( self, task_type: OngoingTaskType, @@ -629,12 +732,14 @@ def set_response(self, response: Optional[str], from_cache: bool) -> None: def _deserialize_task( self, json_dict: dict - ) -> OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration: + ) -> OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration | OngoingTaskCdcSink: """Deserialize the task based on its type.""" if self._task_type == OngoingTaskType.GEN_AI: return OngoingTaskGenAi.from_json(json_dict) elif self._task_type == OngoingTaskType.EMBEDDINGS_GENERATION: return OngoingTaskEmbeddingsGeneration.from_json(json_dict) + elif self._task_type == OngoingTaskType.CDC_SINK: + return OngoingTaskCdcSink.from_json(json_dict) elif self._task_type == OngoingTaskType.PULL_REPLICATION_AS_SINK: return OngoingTaskPullReplicationAsSink.from_json(json_dict) else: diff --git a/ravendb/documents/session/document_session_operations/in_memory_document_session_operations.py b/ravendb/documents/session/document_session_operations/in_memory_document_session_operations.py index 35f42d95..686fa59a 100644 --- a/ravendb/documents/session/document_session_operations/in_memory_document_session_operations.py +++ b/ravendb/documents/session/document_session_operations/in_memory_document_session_operations.py @@ -2062,6 +2062,23 @@ def remove_document_by_entity(self, entity): self.__documents_by_entity_to_remove.append(entity) def update_entity_document_info(self, document_info: DocumentInfo, document: dict): + session_info = self.__session.session_info + cluster_id = session_info.cluster_transaction_id if session_info else None + if cluster_id is not None and document_info.change_vector is not None: + from ravendb.util.client_change_vector_utils import ClientChangeVectorUtils + + cv = document_info.change_vector.split(ClientChangeVectorUtils.SEPARATOR) + if len(cv) > 2: + raise InvalidOperationException( + f"The document '{document_info.key}' has invalid change vector " + f"'{document_info.change_vector}'" + ) + + cluster_tx_index = ClientChangeVectorUtils.get_etag_by_id(cv[-1], cluster_id) + if cluster_tx_index > 0: + session_info.last_cluster_transaction_index = max( + session_info.last_cluster_transaction_index or 0, cluster_tx_index + ) self.__document_infos_to_update.append((document_info, document)) def clear_session_state_after_successful_save_changes(self): diff --git a/ravendb/documents/session/misc.py b/ravendb/documents/session/misc.py index 84d201e1..07697321 100644 --- a/ravendb/documents/session/misc.py +++ b/ravendb/documents/session/misc.py @@ -74,6 +74,7 @@ def __init__( self.no_caching = options.no_caching self.last_cluster_transaction_index: Union[None, int] = None + self.cluster_transaction_id: Union[None, str] = None @property def can_use_load_balance_behavior(self) -> bool: diff --git a/ravendb/http/request_executor.py b/ravendb/http/request_executor.py index a9402629..b0911175 100644 --- a/ravendb/http/request_executor.py +++ b/ravendb/http/request_executor.py @@ -56,7 +56,7 @@ class RequestExecutor: __INITIAL_TOPOLOGY_ETAG = -2 __GLOBAL_APPLICATION_IDENTIFIER = uuid.uuid4() - CLIENT_VERSION = "7.2.3" + CLIENT_VERSION = "7.2.5" logger = logging.getLogger("request_executor") # todo: initializer should take also cryptography certificates @@ -660,6 +660,10 @@ def execute( self._throw_failed_to_contact_all_nodes(command, request) return # we either handled this already in the unsuccessful response or we are throwing self._on_succeed_request_invoke(self._database_name, url, response, request, attempt_num) + if session_info is not None and constants.Headers.DATABASE_CLUSTER_TRANSACTION_ID in response.headers: + session_info.cluster_transaction_id = response.headers[ + constants.Headers.DATABASE_CLUSTER_TRANSACTION_ID + ] response_dispose = command.process_response(self._cache, response, url) self._last_returned_response = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) finally: diff --git a/ravendb/serverwide/database_record.py b/ravendb/serverwide/database_record.py index 955b1e84..2e5ed29d 100644 --- a/ravendb/serverwide/database_record.py +++ b/ravendb/serverwide/database_record.py @@ -70,6 +70,7 @@ def __init__(self, database_name: Optional[str] = None): self.sql_etls: List[SqlEtlConfiguration] = [] self.olap_etls: List[OlapEtlConfiguration] = [] self.embeddings_generations: List = [] + self.cdc_sinks: List = [] self.client: Optional[ClientConfiguration] = None self.studio: Optional[StudioConfiguration] = None self.truncated_cluster_transaction_commands_count: int = 0 @@ -119,6 +120,7 @@ def to_json(self): "RavenEtls": self.raven_etls, "SqlEtls": self.sql_etls, "OlapEtls": self.olap_etls, + "CdcSinks": [cdc_sink.to_json() for cdc_sink in self.cdc_sinks], "Client": self.client, "Studio": self.studio, "TruncatedClusterTransactionCommand": self.truncated_cluster_transaction_commands_count, @@ -178,6 +180,13 @@ def from_json(cls, json_dict: dict) -> DatabaseRecord: ] else: record.embeddings_generations = [] + cdc_sinks_data = json_dict.get("CdcSinks", []) + if cdc_sinks_data: + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration + + record.cdc_sinks = [CdcSinkConfiguration.from_json(c) for c in cdc_sinks_data] + else: + record.cdc_sinks = [] record.client = json_dict.get("Client", None) record.studio = json_dict.get("Studio", None) record.truncated_cluster_transaction_commands_count = json_dict.get("TruncatedClusterTransactionCommand", None) diff --git a/ravendb/serverwide/operations/certificates.py b/ravendb/serverwide/operations/certificates.py index 29a22cba..9cb4c0c7 100644 --- a/ravendb/serverwide/operations/certificates.py +++ b/ravendb/serverwide/operations/certificates.py @@ -4,7 +4,7 @@ import enum import json from datetime import datetime -from typing import TYPE_CHECKING, Dict, List +from typing import TYPE_CHECKING, Dict, List, Optional import requests @@ -39,6 +39,51 @@ def __str__(self): return self.value +class CertificateUsage(enum.Enum): + RAVEN_SERVER = "RavenServer" + RAVEN_SERVER_FOR_COMMUNICATION = "RavenServerForCommunication" + CLIENT = "Client" + SSO_SERVER = "SsoServer" + SSO_CLIENT = "SsoClient" + WELL_KNOWN_ISSUER = "WellKnownIssuer" + + def __str__(self): + return self.value + + +class SsoProvider(enum.Enum): + GITHUB = "Github" + GOOGLE = "Google" + MICROSOFT = "Microsoft" + WINDOWS = "Windows" + + def __str__(self): + return self.value + + +class SsoIdentifier: + def __init__(self, provider: SsoProvider = None, domain: str = None, identifier: str = None): + self.provider = provider + self.domain = domain + self.identifier = identifier + + def to_json(self) -> dict: + return { + "Provider": self.provider.value if self.provider else None, + "Domain": self.domain, + "Identifier": self.identifier, + } + + @classmethod + def from_json(cls, json_dict: dict) -> SsoIdentifier: + provider_value = json_dict.get("Provider") + return cls( + provider=SsoProvider(provider_value) if provider_value else None, + domain=json_dict.get("Domain"), + identifier=json_dict.get("Identifier"), + ) + + class CertificateRawData: def __init__(self, raw_data: bytes = None): self.raw_data = raw_data @@ -57,6 +102,10 @@ def __init__( public_key_pinning_hash: str = None, not_before: datetime = None, disabled: bool = False, + usage: CertificateUsage = None, + sso_server_public_key_pinning_hashes: List[str] = None, + allow_any_sso_server: bool = False, + sso_identifiers: List[SsoIdentifier] = None, ): self.name = name self.security_clearance = security_clearance @@ -68,6 +117,12 @@ def __init__( self.public_key_pinning_hash = public_key_pinning_hash self.not_before = not_before self.disabled = disabled + self.usage = usage + self.sso_server_public_key_pinning_hashes = ( + sso_server_public_key_pinning_hashes if sso_server_public_key_pinning_hashes is not None else [] + ) + self.allow_any_sso_server = allow_any_sso_server + self.sso_identifiers = sso_identifiers if sso_identifiers is not None else [] @classmethod def from_json(cls, json_dict: dict) -> CertificateMetadata: @@ -82,6 +137,14 @@ def from_json(cls, json_dict: dict) -> CertificateMetadata: json_dict.get("PublicKeyPinningHash", None), Utils.string_to_datetime(json_dict["NotBefore"]) if "NotBefore" in json_dict else None, json_dict.get("Disabled", False), + CertificateUsage(json_dict["Usage"]) if json_dict.get("Usage") else None, + json_dict.get("SsoServerPublicKeyPinningHashes") or [], + json_dict.get("AllowAnySsoServer", False), + ( + [SsoIdentifier.from_json(sso) for sso in json_dict["SsoIdentifiers"]] + if json_dict.get("SsoIdentifiers") + else [] + ), ) @@ -99,6 +162,10 @@ def __init__( collection_primary_key: str = None, public_key_pinning_hash: str = None, disabled: bool = False, + usage: CertificateUsage = None, + sso_server_public_key_pinning_hashes: List[str] = None, + allow_any_sso_server: bool = False, + sso_identifiers: List[SsoIdentifier] = None, ): super().__init__( name, @@ -110,6 +177,10 @@ def __init__( collection_primary_key, public_key_pinning_hash, disabled=disabled, + usage=usage, + sso_server_public_key_pinning_hashes=sso_server_public_key_pinning_hashes, + allow_any_sso_server=allow_any_sso_server, + sso_identifiers=sso_identifiers, ) self.certificate = certificate self.password = password @@ -126,6 +197,10 @@ def to_json(self) -> dict: "Certificate": self.certificate, "Password": self.password, "Disabled": self.disabled, + "Usage": self.usage.value if self.usage else None, + "SsoServerPublicKeyPinningHashes": self.sso_server_public_key_pinning_hashes, + "AllowAnySsoServer": self.allow_any_sso_server, + "SsoIdentifiers": [sso.to_json() for sso in self.sso_identifiers] if self.sso_identifiers else [], } if self.not_after: json_dict.update({"NotAfter": Utils.datetime_to_string(self.not_after)}) @@ -145,6 +220,14 @@ def from_json(cls, json_dict: dict) -> CertificateDefinition: json_dict["CollectionPrimaryKey"], json_dict["PublicKeyPinningHash"], disabled=json_dict.get("Disabled", False), + usage=CertificateUsage(json_dict["Usage"]) if json_dict.get("Usage") else None, + sso_server_public_key_pinning_hashes=json_dict.get("SsoServerPublicKeyPinningHashes") or [], + allow_any_sso_server=json_dict.get("AllowAnySsoServer", False), + sso_identifiers=( + [SsoIdentifier.from_json(sso) for sso in json_dict["SsoIdentifiers"]] + if json_dict.get("SsoIdentifiers") + else [] + ), ) @@ -456,12 +539,18 @@ def __init__( name: str, clearance: SecurityClearance, disabled: bool = False, + sso_server_public_key_pinning_hashes: Optional[List[str]] = None, + allow_any_sso_server: Optional[bool] = None, + sso_identifiers: Optional[List[SsoIdentifier]] = None, ): self.thumbprint = thumbprint self.permissions = permissions self.name = name self.clearance = clearance self.disabled = disabled + self.sso_server_public_key_pinning_hashes = sso_server_public_key_pinning_hashes + self.allow_any_sso_server = allow_any_sso_server + self.sso_identifiers = sso_identifiers def __init__(self, parameters: Parameters): if parameters is None: @@ -481,10 +570,20 @@ def __init__(self, parameters: Parameters): self.__permissions = parameters.permissions self.__clearance = parameters.clearance self.__disabled = parameters.disabled + self.__sso_server_public_key_pinning_hashes = parameters.sso_server_public_key_pinning_hashes + self.__allow_any_sso_server = parameters.allow_any_sso_server + self.__sso_identifiers = parameters.sso_identifiers def get_command(self, conventions: "DocumentConventions") -> "VoidRavenCommand": return self.__EditCertificateClientCommand( - self.__thumbprint, self.__name, self.__permissions, self.__clearance, self.__disabled + self.__thumbprint, + self.__name, + self.__permissions, + self.__clearance, + self.__disabled, + self.__sso_server_public_key_pinning_hashes, + self.__allow_any_sso_server, + self.__sso_identifiers, ) class __EditCertificateClientCommand(VoidRavenCommand, RaftCommand): @@ -495,6 +594,9 @@ def __init__( permissions: Dict[str, DatabaseAccess], clearance: SecurityClearance, disabled: bool, + sso_server_public_key_pinning_hashes: Optional[List[str]] = None, + allow_any_sso_server: Optional[bool] = None, + sso_identifiers: Optional[List[SsoIdentifier]] = None, ): super().__init__() self.__thumbprint = thumbprint @@ -502,6 +604,9 @@ def __init__( self.__permissions = permissions self.__clearance = clearance self.__disabled = disabled + self.__sso_server_public_key_pinning_hashes = sso_server_public_key_pinning_hashes + self.__allow_any_sso_server = allow_any_sso_server + self.__sso_identifiers = sso_identifiers def is_read_request(self) -> bool: return False @@ -509,15 +614,35 @@ def is_read_request(self) -> bool: def create_request(self, node: ServerNode) -> requests.Request: url = f"{node.url}/admin/certificates/edit" - definition = CertificateDefinition() - definition.thumbprint = self.__thumbprint - definition.permissions = self.__permissions - definition.security_clearance = self.__clearance - definition.name = self.__name - definition.disabled = self.__disabled + # The body is written manually, never from CertificateDefinition.to_json: the SSO + # keys must be present only when the caller provided them, because the server's + # edit handler treats a present key as an explicit replacement (even an empty list + # clears the stored value) and an absent key as "leave untouched". + body = { + "Thumbprint": self.__thumbprint, + "Name": self.__name, + "SecurityClearance": self.__clearance.value if self.__clearance else None, + "Disabled": self.__disabled, + "Permissions": {key: str(value) for key, value in self.__permissions.items()}, + } + if self.__sso_server_public_key_pinning_hashes is not None: + body["SsoServerPublicKeyPinningHashes"] = self.__sso_server_public_key_pinning_hashes + if self.__allow_any_sso_server is not None: + body["AllowAnySsoServer"] = self.__allow_any_sso_server + if self.__sso_identifiers is not None: + sso_ids = [] + for sso_id in self.__sso_identifiers: + entry = { + "Provider": sso_id.provider.value if sso_id.provider else None, + "Identifier": sso_id.identifier, + } + if sso_id.domain: + entry["Domain"] = sso_id.domain + sso_ids.append(entry) + body["SsoIdentifiers"] = sso_ids request = requests.Request("POST", url) - request.data = definition.to_json() + request.data = body return request diff --git a/ravendb/serverwide/operations/connection_strings.py b/ravendb/serverwide/operations/connection_strings.py new file mode 100644 index 00000000..cd44a445 --- /dev/null +++ b/ravendb/serverwide/operations/connection_strings.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional, TYPE_CHECKING + +import requests + +from ravendb.documents.operations.ai.ai_connection_string import AiConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage +from ravendb.documents.operations.etl.configuration import RavenConnectionString +from ravendb.documents.operations.etl.elastic_search.connection import ElasticSearchConnectionString +from ravendb.documents.operations.etl.olap.connection import OlapConnectionString +from ravendb.documents.operations.etl.queue.connection import QueueConnectionString +from ravendb.documents.operations.etl.snowflake.connection import SnowflakeConnectionString +from ravendb.documents.operations.etl.sql import SqlConnectionString +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.operations.common import ServerOperation +from ravendb.serverwide.server_operation_executor import ConnectionStringType +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class ServerWideConnectionStringUsage(ConnectionStringUsage): + """A server-wide usage of a connection string; the server aggregates usages across + databases, so each usage also carries the DatabaseName where the task lives.""" + + def __init__( + self, + kind: Optional[str] = None, + id: Optional[int] = None, + identifier: Optional[str] = None, + name: Optional[str] = None, + database_name: Optional[str] = None, + ): + super().__init__(kind, id, identifier, name) + self.database_name = database_name + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> ServerWideConnectionStringUsage: + return cls( + kind=json_dict.get("Kind"), + id=json_dict.get("Id"), + identifier=json_dict.get("Identifier"), + name=json_dict.get("Name"), + database_name=json_dict.get("DatabaseName"), + ) + + +_CONNECTION_STRING_FROM_JSON = { + ConnectionStringType.RAVEN: RavenConnectionString.from_json, + ConnectionStringType.SQL: SqlConnectionString.from_json, + ConnectionStringType.OLAP: OlapConnectionString.from_json, + ConnectionStringType.ELASTIC_SEARCH: ElasticSearchConnectionString.from_json, + ConnectionStringType.QUEUE: QueueConnectionString.from_json, + ConnectionStringType.SNOWFLAKE: SnowflakeConnectionString.from_json, + ConnectionStringType.AI: AiConnectionString.from_json, +} + + +class ServerWideConnectionString: + def __init__( + self, + connection_string: Optional[ConnectionString] = None, + excluded_databases: Optional[List[str]] = None, + used_by: Optional[List[ServerWideConnectionStringUsage]] = None, + ): + self.connection_string = connection_string + self.excluded_databases = excluded_databases + self.used_by = used_by if used_by is not None else [] + + @property + def name(self) -> Optional[str]: + return self.connection_string.name if self.connection_string else None + + @property + def get_type(self) -> str: + return self.connection_string.get_type if self.connection_string else ConnectionStringType.NONE.value + + def to_json(self) -> Dict[str, Any]: + # UsedBy is read-only server metadata and is never written (C# ToJson omits it). + json_dict = self.connection_string.to_json() if self.connection_string else {} + json_dict["Type"] = self.get_type + json_dict["ExcludedDatabases"] = self.excluded_databases + return json_dict + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]) -> Optional[ServerWideConnectionString]: + if json_dict is None: + return None + + # The server rejects a body without Type; the C# FromBlittable returns null there. + type_value = json_dict.get("Type") + if not type_value: + return None + + try: + connection_string_type = ConnectionStringType(type_value) + except ValueError: + return None + + from_json = _CONNECTION_STRING_FROM_JSON.get(connection_string_type) + if from_json is None: + return None + + result = cls( + connection_string=from_json(json_dict), + excluded_databases=json_dict.get("ExcludedDatabases"), + ) + + used_by = json_dict.get("UsedBy") + if used_by: + result.used_by = [ServerWideConnectionStringUsage.from_json(usage) for usage in used_by] + + return result + + +class PutServerWideConnectionStringResult: + def __init__(self, raft_command_index: Optional[int] = None): + self.raft_command_index = raft_command_index + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> PutServerWideConnectionStringResult: + return cls(raft_command_index=json_dict.get("RaftCommandIndex")) + + +class PutServerWideConnectionStringOperation(ServerOperation[PutServerWideConnectionStringResult]): + def __init__(self, connection_string: ServerWideConnectionString): + if connection_string is None: + raise ValueError("connection_string cannot be None") + if connection_string.connection_string is None: + raise ValueError("connection_string.connection_string must not be None") + + self._connection_string = connection_string + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[PutServerWideConnectionStringResult]: + return self.PutServerWideConnectionStringCommand(self._connection_string) + + class PutServerWideConnectionStringCommand(RavenCommand[PutServerWideConnectionStringResult], RaftCommand): + def __init__(self, connection_string: ServerWideConnectionString): + super().__init__(PutServerWideConnectionStringResult) + if connection_string is None: + raise ValueError("connection_string cannot be None") + self._connection_string = connection_string + + def is_read_request(self) -> bool: + return False + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/admin/configuration/server-wide/connection-strings" + request = requests.Request("PUT", url) + request.headers = {"Content-Type": "application/json"} + request.data = json.dumps(self._connection_string.to_json()) + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = PutServerWideConnectionStringResult.from_json(json.loads(response)) + + +class GetServerWideConnectionStringsResult: + def __init__(self, results: Optional[List[ServerWideConnectionString]] = None): + self.results = results if results is not None else [] + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> GetServerWideConnectionStringsResult: + results = [] + for item in json_dict.get("Results") or []: + parsed = ServerWideConnectionString.from_json(item) + if parsed is not None: + results.append(parsed) + return cls(results) + + +class GetServerWideConnectionStringsOperation(ServerOperation[GetServerWideConnectionStringsResult]): + def __init__( + self, + connection_string_name: Optional[str] = None, + type: Optional[ConnectionStringType] = None, + ): + if connection_string_name is not None and (not connection_string_name or connection_string_name.isspace()): + raise ValueError("Connection string name must not be null or empty.") + + self._connection_string_name = connection_string_name + self._type = type + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[GetServerWideConnectionStringsResult]: + return self.GetServerWideConnectionStringsCommand(self._connection_string_name, self._type) + + class GetServerWideConnectionStringsCommand(RavenCommand[GetServerWideConnectionStringsResult]): + def __init__( + self, + connection_string_name: Optional[str] = None, + type: Optional[ConnectionStringType] = None, + ): + super().__init__(GetServerWideConnectionStringsResult) + self._connection_string_name = connection_string_name + self._type = type + + def is_read_request(self) -> bool: + return True + + def create_request(self, node: ServerNode) -> requests.Request: + from urllib.parse import quote + + url = f"{node.url}/admin/configuration/server-wide/connection-strings" + + query_params = [] + if self._connection_string_name is not None: + query_params.append(f"name={quote(self._connection_string_name)}") + if self._type is not None and self._type != ConnectionStringType.NONE: + # The query value is the enum NAME string, never the Python enum repr. + query_params.append(f"type={self._type.value}") + + if query_params: + url += "?" + "&".join(query_params) + + return requests.Request("GET", url) + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = GetServerWideConnectionStringsResult.from_json(json.loads(response)) + + +class RemoveServerWideConnectionStringResult: + def __init__(self, raft_command_index: Optional[int] = None): + self.raft_command_index = raft_command_index + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> RemoveServerWideConnectionStringResult: + return cls(raft_command_index=json_dict.get("RaftCommandIndex")) + + +class RemoveServerWideConnectionStringOperation(ServerOperation[RemoveServerWideConnectionStringResult]): + def __init__(self, connection_string: ConnectionString): + if connection_string is None: + raise ValueError("connection_string cannot be None") + if not connection_string.name or connection_string.name.isspace(): + raise ValueError("Connection string name must not be null or empty.") + + self._connection_string = connection_string + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[RemoveServerWideConnectionStringResult]: + return self.RemoveServerWideConnectionStringCommand(self._connection_string) + + class RemoveServerWideConnectionStringCommand(RavenCommand[RemoveServerWideConnectionStringResult], RaftCommand): + def __init__(self, connection_string: ConnectionString): + super().__init__(RemoveServerWideConnectionStringResult) + self._connection_string = connection_string + + def is_read_request(self) -> bool: + return False + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + def create_request(self, node: ServerNode) -> requests.Request: + from urllib.parse import quote + + url = ( + f"{node.url}/admin/configuration/server-wide/connection-strings" + f"?name={quote(self._connection_string.name)}" + f"&type={self._connection_string.get_type}" + ) + + return requests.Request("DELETE", url) + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = RemoveServerWideConnectionStringResult.from_json(json.loads(response)) diff --git a/ravendb/tests/ai_agent_tests/test_conversation_messages_live.py b/ravendb/tests/ai_agent_tests/test_conversation_messages_live.py new file mode 100644 index 00000000..95de7fb5 --- /dev/null +++ b/ravendb/tests/ai_agent_tests/test_conversation_messages_live.py @@ -0,0 +1,65 @@ +"""Live-server integration tests for AI conversation messages (C1/C6). + +Asserts the live-verified fact: a missing conversation answers HTTP 404 with an +empty body, and the operation surfaces result None (never an exception). +Skipped when no server is reachable at localhost:8081. +""" + +import os +import unittest + +import requests + +from ravendb.documents.store.definition import DocumentStore + +_LIVE_SERVER_URL = os.environ.get("RAVENDB_LIVE_SERVER_URL", "http://localhost:8081") +_DATABASE = "conversation-messages-live-db" + + +def _server_reachable() -> bool: + try: + response = requests.get(f"{_LIVE_SERVER_URL}/build/version", timeout=3) + return response.status_code == 200 and '"7.2.5"' in response.text + except Exception: + return False + + +@unittest.skipUnless(_server_reachable(), f"no live 7.2.5 server at {_LIVE_SERVER_URL}") +class TestConversationMessagesLive(unittest.TestCase): + @classmethod + def setUpClass(cls): + requests.put(f"{_LIVE_SERVER_URL}/admin/databases?name={_DATABASE}", json={"DatabaseName": _DATABASE}) + + @classmethod + def tearDownClass(cls): + requests.delete(f"{_LIVE_SERVER_URL}/admin/databases?name={_DATABASE}&hard-delete=true") + + def setUp(self): + self.store = DocumentStore(urls=[_LIVE_SERVER_URL], database=_DATABASE) + self.store.initialize() + + def tearDown(self): + self.store.close() + + def test_missing_conversation_answers_404_with_empty_body(self): + response = requests.get( + f"{_LIVE_SERVER_URL}/databases/{_DATABASE}/ai/agent/conversation/messages" + "?conversationId=chats/missing&pageSize=10&detailLevel=Simple" + ) + self.assertEqual(404, response.status_code) + self.assertEqual(0, len(response.content)) + + def test_missing_conversation_result_is_none(self): + result = self.store.ai.get_conversation_messages("chats/missing") + self.assertIsNone(result) + + def test_response_carries_database_cluster_tx_id_header(self): + response = requests.get( + f"{_LIVE_SERVER_URL}/databases/{_DATABASE}/ai/agent/conversation/messages" + "?conversationId=chats/missing&pageSize=10&detailLevel=Simple" + ) + self.assertIsNotNone(response.headers.get("Database-Cluster-Tx-Id")) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/ai_agent_tests/test_get_conversation_messages.py b/ravendb/tests/ai_agent_tests/test_get_conversation_messages.py new file mode 100644 index 00000000..49e0e9e3 --- /dev/null +++ b/ravendb/tests/ai_agent_tests/test_get_conversation_messages.py @@ -0,0 +1,324 @@ +"""Tests for the AI conversation-messages surface, cancelPendingActionTools on +the wire, and the streaming read that honors cancellation. +""" + +import json +import threading +import unittest +from datetime import datetime + +from ravendb.documents.ai.ai_operations import AiOperations +from ravendb.documents.operations.ai.agents import ( + AiConversationDetailLevel, + AiConversationMessage, + AiConversationMessagesResult, + AiMessageRole, + AiToolCallResult, + GetConversationMessagesCommand, + GetConversationMessagesOperation, + GetConversationMessagesOptions, + RunConversationOperation, +) +from ravendb.documents.operations.ai.agents.run_conversation_operation import RunConversationCommand +from ravendb.http.server_node import ServerNode +from ravendb.tools.utils import Utils + + +class TestGetConversationMessagesCommandWire(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + + def _command(self, **kwargs): + options = GetConversationMessagesOptions(**kwargs) + operation = GetConversationMessagesOperation(options) + return operation.get_command(None) + + def test_get_to_conversation_messages_endpoint(self): + command = self._command(conversation_id="chats/1") + request = command.create_request(self.node) + self.assertEqual("GET", request.method) + self.assertTrue(request.url.startswith("http://localhost:8080/databases/db1/ai/agent/conversation/messages?")) + + def test_parameter_order_and_always_present_params(self): + before = datetime(2026, 1, 2, 3, 4, 5, 123456) + command = self._command( + conversation_id="chats/1", before=before, page_size=50, detail_level=AiConversationDetailLevel.DETAILED + ) + url = command.create_request(self.node).url + query = url.split("?", 1)[1] + parts = [p.split("=", 1)[0] for p in query.split("&")] + self.assertEqual(["conversationId", "before", "pageSize", "detailLevel"], parts) + self.assertTrue(url.endswith("&pageSize=50&detailLevel=Detailed")) + + def test_after_appended_only_when_set(self): + after = datetime(2026, 1, 2, 3, 4, 5, 123456) + url = self._command(conversation_id="c", after=after).create_request(self.node).url + self.assertIn("&after=", url) + self.assertNotIn("&before=", url) + + def test_before_timestamp_is_the_seven_digit_raven_form(self): + from urllib.parse import unquote + + before = datetime(2026, 1, 2, 3, 4, 5, 123456) + url = self._command(conversation_id="c", before=before).create_request(self.node).url + ts = Utils.datetime_to_string(before) + self.assertEqual(7, len(ts.split(".")[1])) + query = url.split("?", 1)[1] + before_value = [p.split("=", 1)[1] for p in query.split("&") if p.startswith("before=")][0] + self.assertEqual(ts, unquote(before_value)) + + def test_page_size_default_is_int_max_value(self): + url = self._command(conversation_id="c").create_request(self.node).url + self.assertTrue(url.endswith("&pageSize=2147483647&detailLevel=Simple")) + + def test_is_read_request(self): + self.assertTrue(self._command(conversation_id="c").is_read_request()) + + def test_constructor_validates_conversation_id(self): + with self.assertRaises(ValueError): + GetConversationMessagesOperation("") + with self.assertRaises(ValueError): + GetConversationMessagesOperation(GetConversationMessagesOptions()) + + def test_before_and_after_are_mutually_exclusive(self): + when = datetime(2026, 1, 1) + with self.assertRaises(ValueError) as ctx: + GetConversationMessagesOperation( + GetConversationMessagesOptions(conversation_id="c", before=when, after=when) + ) + self.assertIn("cannot both be specified", str(ctx.exception)) + + def test_page_size_must_be_positive(self): + with self.assertRaises(ValueError) as ctx: + GetConversationMessagesOperation(GetConversationMessagesOptions(conversation_id="c", page_size=0)) + self.assertIn("PageSize must be greater than 0", str(ctx.exception)) + + def test_null_response_leaves_result_none(self): + command = GetConversationMessagesCommand(GetConversationMessagesOptions(conversation_id="missing")) + command.set_response(None, False) + self.assertIsNone(command.result) + + def test_parameters_keep_typed_values(self): + payload = { + "ConversationId": "chats/1", + "Agent": "a1", + "Parameters": { + "big": 3000000000, + "frac": 0.5, + "text": "hi", + "flag": True, + "arr": [1, 2, 3], + "mixed": [1, "x"], + }, + "TotalUsage": None, + "Messages": [], + } + command = GetConversationMessagesCommand(GetConversationMessagesOptions(conversation_id="chats/1")) + command.set_response(json.dumps(payload), False) + params = command.result.parameters + self.assertIsInstance(params["big"], int) + self.assertEqual(3000000000, params["big"]) + self.assertIsInstance(params["frac"], float) + self.assertEqual(0.5, params["frac"]) + self.assertEqual("hi", params["text"]) + self.assertIs(True, params["flag"]) + self.assertEqual([1, 2, 3], params["arr"]) + self.assertEqual([1, "x"], params["mixed"]) + + +class TestAiConversationMessage(unittest.TestCase): + def test_from_json_parses_all_fields(self): + payload = { + "Role": "User", + "Content": "hello", + "Attachments": ["a.txt"], + "Timestamp": "2026-01-02T03:04:05.1234560Z", + "ToolCalls": [{"Id": "t1", "Name": "n", "Arguments": "{}", "Result": "ok", "SubConversationId": None}], + "Usage": { + "PromptTokens": 1, + "CompletionTokens": 2, + "TotalTokens": 3, + "CachedTokens": 0, + "ReasoningTokens": 0, + }, + "SubConversationId": None, + } + message = AiConversationMessage.from_json(payload) + self.assertEqual(AiMessageRole.USER, message.role) + self.assertEqual("hello", message.content) + self.assertEqual(["a.txt"], message.attachments) + self.assertEqual(2026, message.timestamp.year) + self.assertEqual("t1", message.tool_calls[0].id) + self.assertEqual(AiUsage_tokens(message), 3) + + def test_to_json_order_and_role_as_name(self): + message = AiConversationMessage( + role=AiMessageRole.SYSTEM, + content="sys", + attachments=[], + timestamp=datetime(2026, 1, 1), + tool_calls=[], + usage=None, + sub_conversation_id=None, + ) + result = message.to_json() + self.assertEqual( + list(result.keys()), + ["Role", "Content", "Attachments", "Timestamp", "ToolCalls", "Usage", "SubConversationId"], + ) + self.assertEqual("System", result["Role"]) + + def test_pending_tool_call_result_stays_none(self): + tool_call = AiToolCallResult.from_json({"Id": "t1", "Name": "n", "Arguments": "{}", "Result": None}) + self.assertIsNone(tool_call.result) + self.assertEqual( + {"Id": "t1", "Name": "n", "Arguments": "{}", "Result": None, "SubConversationId": None}, + tool_call.to_json(), + ) + + def test_missing_keys_use_defaults(self): + message = AiConversationMessage.from_json({"Role": "Assistant"}) + self.assertIsNone(message.content) + self.assertIsNone(message.attachments) + self.assertIsNone(message.timestamp) + self.assertIsNone(message.tool_calls) + + +def AiUsage_tokens(message): + return message.usage.total_tokens + + +class TestAiConversationMessagesResult(unittest.TestCase): + def test_to_json_key_order(self): + result = AiConversationMessagesResult( + conversation_id="c", + agent="a", + parameters={}, + total_usage=None, + last_message_at=None, + has_more_messages=False, + sub_conversation_ids=None, + attachments=None, + messages=None, + ) + keys = list(result.to_json().keys()) + self.assertEqual( + [ + "ConversationId", + "Agent", + "Parameters", + "TotalUsage", + "LastMessageAt", + "HasMoreMessages", + "SubConversationIds", + "Attachments", + "Messages", + ], + keys, + ) + + def test_missing_key_defaults(self): + result = AiConversationMessagesResult.from_json({"ConversationId": "c"}) + self.assertEqual("c", result.conversation_id) + self.assertIsNone(result.agent) + self.assertIsNone(result.parameters) + self.assertIsNone(result.total_usage) + self.assertIsNone(result.last_message_at) + self.assertIsNone(result.messages) + self.assertFalse(result.has_more_messages) + self.assertIsNone(result.sub_conversation_ids) + self.assertIsNone(result.attachments) + + +class TestAiOperationsIntegration(unittest.TestCase): + def test_get_conversation_messages_reachable_from_store(self): + self.assertTrue(hasattr(AiOperations, "get_conversation_messages")) + + def test_ai_operations_exported_from_package(self): + import ravendb + + self.assertTrue(hasattr(ravendb, "AiConversationMessagesResult")) + self.assertTrue(hasattr(ravendb, "GetConversationMessagesOperation")) + self.assertTrue(hasattr(ravendb, "GetConversationMessagesOptions")) + self.assertTrue(hasattr(ravendb, "AiMessageRole")) + self.assertTrue(hasattr(ravendb, "AiConversationDetailLevel")) + + +class TestCancelPendingActionTools(unittest.TestCase): + def _command(self, cancel=False, debug=None): + operation = RunConversationOperation( + agent_id="a", + conversation_id="c", + debug=debug, + cancel_pending_action_tools=cancel, + ) + return operation.get_command(None) + + def test_flag_always_on_wire_after_debug(self): + url = self._command().create_request(ServerNode("http://localhost:8080", "db1")).url + self.assertTrue(url.endswith("&cancelPendingActionTools=False"), url) + + def test_flag_after_debug_parameter(self): + url = self._command(debug=True).create_request(ServerNode("http://localhost:8080", "db1")).url + self.assertIn("&debug=True&cancelPendingActionTools=False", url) + + def test_flag_true_when_set(self): + url = self._command(cancel=True).create_request(ServerNode("http://localhost:8080", "db1")).url + self.assertTrue(url.endswith("&cancelPendingActionTools=True"), url) + + +class TestStreamingCancellation(unittest.TestCase): + def test_cancelled_stream_stops_without_reading_rest(self): + from ravendb.http.misc import ResponseDisposeHandling + from ravendb.http.http_cache import HttpCache + + lines_read = [] + + class FakeResponse: + def iter_lines(self, decode_unicode=True): + for i in range(100): + lines_read.append(i) + # Real streamed chunks are JSON-encoded strings, not bare objects. + yield f'"chunk {i}"' + + def close(self): + pass + + event = threading.Event() + command = RunConversationCommand( + agent_id="a", + conversation_id="c", + stream_property_path="answer", + streamed_chunks_callback=lambda chunk: event.set(), + cancellation_event=event, + ) + response = FakeResponse() + with self.assertRaises(Exception) as ctx: + command.process_response(HttpCache(), response, "http://x") + self.assertIsInstance(ctx.exception, Exception) + self.assertLess(len(lines_read), 100, "the stream must not be fully consumed") + + def test_no_cancellation_event_reads_to_end(self): + from ravendb.http.misc import ResponseDisposeHandling + from ravendb.http.http_cache import HttpCache + + class FakeResponse: + def iter_lines(self, decode_unicode=True): + yield '{"a": 1}' + yield '{"a": 2}' + + def close(self): + pass + + command = RunConversationCommand( + agent_id="a", + conversation_id="c", + stream_property_path="answer", + streamed_chunks_callback=lambda chunk: None, + ) + handling = command.process_response(HttpCache(), FakeResponse(), "http://x") + self.assertEqual(ResponseDisposeHandling.AUTOMATIC, handling) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/cdc_sink_tests/__init__.py b/ravendb/tests/cdc_sink_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ravendb/tests/cdc_sink_tests/test_cdc_sink_configuration.py b/ravendb/tests/cdc_sink_tests/test_cdc_sink_configuration.py new file mode 100644 index 00000000..06189a47 --- /dev/null +++ b/ravendb/tests/cdc_sink_tests/test_cdc_sink_configuration.py @@ -0,0 +1,440 @@ +"""Unit tests for the CDC Sink configuration classes: CdcSinkConfiguration, +its nested config classes, and the CdcSinkTaskState / CdcSinkTableLoadState +serialization shape. +""" + +import unittest + +from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import ( + CdcColumnMapping, + CdcColumnType, + CdcSinkConfiguration, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcSinkRelationType, + CdcSinkTableConfig, +) +from ravendb.documents.operations.cdc_sink.cdc_sink_task_state import ( + CdcSinkTableLoadState, + CdcSinkTablesDict, + CdcSinkTaskState, +) + + +class TestCdcSinkConfiguration(unittest.TestCase): + def test_to_json_key_order_and_values(self): + config = CdcSinkConfiguration( + task_id=42, + disabled=True, + name="cdc-1", + mentor_node="A", + pin_to_mentor_node=True, + connection_string_name="sql-cs", + tables=[CdcSinkTableConfig(collection_name="Orders", source_table_name="orders")], + postgres=CdcSinkPostgresSettings(publication_name="pub", slot_name="slot"), + skip_initial_load=True, + ) + out = config.to_json() + self.assertEqual( + [ + "Name", + "TaskId", + "Disabled", + "ConnectionStringName", + "MentorNode", + "PinToMentorNode", + "Tables", + "Postgres", + "SkipInitialLoad", + ], + list(out.keys()), + ) + self.assertEqual("cdc-1", out["Name"]) + self.assertEqual(42, out["TaskId"]) + self.assertTrue(out["Disabled"]) + self.assertEqual("sql-cs", out["ConnectionStringName"]) + self.assertEqual("A", out["MentorNode"]) + self.assertTrue(out["PinToMentorNode"]) + self.assertTrue(out["SkipInitialLoad"]) + self.assertEqual({"PublicationName": "pub", "SlotName": "slot"}, out["Postgres"]) + + def test_postgres_is_null_when_unset(self): + out = CdcSinkConfiguration(name="c", connection_string_name="sql").to_json() + self.assertIsNone(out["Postgres"]) + + def test_test_mode_is_never_serialized(self): + out = CdcSinkConfiguration(name="c", connection_string_name="sql").to_json() + self.assertNotIn("TestMode", out) + + def test_from_json_round_trip_with_null_postgres(self): + raw = { + "Name": "c", + "TaskId": 0, + "Disabled": False, + "ConnectionStringName": "sql", + "MentorNode": None, + "PinToMentorNode": False, + "Tables": [], + "Postgres": None, + "SkipInitialLoad": False, + } + config = CdcSinkConfiguration.from_json(raw) + self.assertEqual("c", config.name) + self.assertEqual("sql", config.connection_string_name) + self.assertIsNone(config.postgres) + self.assertEqual([], config.tables) + self.assertEqual(raw, config.to_json()) + + def test_from_json_missing_keys_use_defaults(self): + config = CdcSinkConfiguration.from_json({"ConnectionStringName": "sql"}) + self.assertFalse(config.disabled) + self.assertEqual(0, config.task_id) + self.assertIsNone(config.name) + self.assertFalse(config.pin_to_mentor_node) + self.assertEqual([], config.tables) + self.assertIsNone(config.postgres) + self.assertFalse(config.skip_initial_load) + + def test_get_default_task_name(self): + config = CdcSinkConfiguration(connection_string_name="my-sql") + self.assertEqual("CDC Sink to my-sql", config.get_default_task_name()) + + def test_empty_tables_serialize_as_array_not_omitted(self): + out = CdcSinkConfiguration(name="c", connection_string_name="sql").to_json() + self.assertEqual([], out["Tables"]) + + def test_task_id_round_trips_beyond_2_31(self): + config = CdcSinkConfiguration(task_id=5000000000, name="c", connection_string_name="sql") + self.assertEqual(5000000000, CdcSinkConfiguration.from_json(config.to_json()).task_id) + + +class TestCdcSinkTableConfig(unittest.TestCase): + def test_to_json_keys(self): + table = CdcSinkTableConfig( + collection_name="Orders", + source_table_schema="dbo", + source_table_name="orders", + columns=[CdcColumnMapping(column="id", name="Id")], + primary_key_columns=["id"], + patch="this.Total = $row.total;", + on_delete=CdcSinkOnDeleteConfig(patch="this.Archived = true;", ignore_deletes=True), + disabled=False, + ) + out = table.to_json() + self.assertEqual( + [ + "CollectionName", + "SourceTableSchema", + "SourceTableName", + "Columns", + "PrimaryKeyColumns", + "Patch", + "OnDelete", + "Disabled", + "EmbeddedTables", + "LinkedTables", + ], + list(out.keys()), + ) + self.assertEqual("Orders", out["CollectionName"]) + self.assertEqual(["id"], out["PrimaryKeyColumns"]) + self.assertEqual({"Patch": "this.Archived = true;", "IgnoreDeletes": True}, out["OnDelete"]) + self.assertEqual([], out["EmbeddedTables"]) + self.assertEqual([], out["LinkedTables"]) + + def test_empty_lists_are_arrays_not_omitted(self): + out = CdcSinkTableConfig(collection_name="Orders").to_json() + self.assertEqual([], out["Columns"]) + self.assertEqual([], out["PrimaryKeyColumns"]) + self.assertIsNone(out["OnDelete"]) + + def test_from_json_round_trip(self): + table = CdcSinkTableConfig( + collection_name="Orders", + source_table_schema="dbo", + source_table_name="orders", + columns=[CdcColumnMapping(column="id", name="Id")], + primary_key_columns=["id"], + patch="this.Total = $row.total;", + on_delete=CdcSinkOnDeleteConfig(ignore_deletes=True), + disabled=True, + ) + parsed = CdcSinkTableConfig.from_json(table.to_json()) + self.assertEqual("Orders", parsed.collection_name) + self.assertEqual("dbo", parsed.source_table_schema) + self.assertTrue(parsed.disabled) + self.assertTrue(parsed.on_delete.ignore_deletes) + self.assertEqual("this.Total = $row.total;", parsed.patch) + + +class TestCdcColumnMapping(unittest.TestCase): + def test_type_omitted_when_default(self): + out = CdcColumnMapping(column="id", name="Id").to_json() + self.assertEqual(["Column", "Name"], list(out.keys())) + + def test_type_written_as_enum_name_when_not_default(self): + self.assertEqual( + "Json", CdcColumnMapping(column="data", name="Data", type=CdcColumnType.JSON).to_json()["Type"] + ) + self.assertEqual( + "Attachment", CdcColumnMapping(column="file", name="File", type=CdcColumnType.ATTACHMENT).to_json()["Type"] + ) + + def test_enum_wire_values(self): + self.assertEqual("Default", CdcColumnType.DEFAULT.value) + self.assertEqual("Json", CdcColumnType.JSON.value) + self.assertEqual("Attachment", CdcColumnType.ATTACHMENT.value) + + def test_from_json_default_type_when_absent(self): + mapping = CdcColumnMapping.from_json({"Column": "id", "Name": "Id"}) + self.assertEqual(CdcColumnType.DEFAULT, mapping.type) + mapping = CdcColumnMapping.from_json({"Column": "data", "Name": "Data", "Type": "Json"}) + self.assertEqual(CdcColumnType.JSON, mapping.type) + + +class TestCdcSinkEmbeddedTableConfig(unittest.TestCase): + def test_type_always_written_as_enum_name(self): + for relation_type in CdcSinkRelationType: + config = CdcSinkEmbeddedTableConfig(source_table_name="lines", property_name="Lines", type=relation_type) + self.assertEqual(relation_type.value, config.to_json()["Type"]) + + def test_to_json_keys(self): + config = CdcSinkEmbeddedTableConfig( + source_table_schema="dbo", + source_table_name="order_lines", + property_name="Lines", + columns=[CdcColumnMapping(column="id", name="Id")], + primary_key_columns=["id"], + join_columns=["order_id"], + type=CdcSinkRelationType.ARRAY, + patch="this.Total = $row.total;", + on_delete=CdcSinkOnDeleteConfig(ignore_deletes=True), + case_sensitive_keys=True, + embedded_tables=[CdcSinkEmbeddedTableConfig(source_table_name="child", property_name="Child")], + linked_tables=[ + CdcSinkLinkedTableConfig( + source_table_name="customers", + property_name="Customer", + join_columns=["customer_id"], + linked_collection_name="Customers", + ) + ], + ) + out = config.to_json() + self.assertEqual( + [ + "SourceTableSchema", + "SourceTableName", + "PropertyName", + "Columns", + "PrimaryKeyColumns", + "JoinColumns", + "Type", + "Patch", + "OnDelete", + "CaseSensitiveKeys", + "EmbeddedTables", + "LinkedTables", + ], + list(out.keys()), + ) + self.assertEqual("Array", out["Type"]) + self.assertTrue(out["CaseSensitiveKeys"]) + self.assertEqual(1, len(out["EmbeddedTables"])) + self.assertEqual(1, len(out["LinkedTables"])) + + def test_round_trip(self): + config = CdcSinkEmbeddedTableConfig( + source_table_name="order_lines", + property_name="Lines", + type=CdcSinkRelationType.MAP, + case_sensitive_keys=True, + embedded_tables=[CdcSinkEmbeddedTableConfig(source_table_name="child", property_name="Child")], + linked_tables=[ + CdcSinkLinkedTableConfig( + source_table_name="customers", + property_name="Customer", + join_columns=["customer_id"], + linked_collection_name="Customers", + ) + ], + ) + parsed = CdcSinkEmbeddedTableConfig.from_json(config.to_json()) + self.assertEqual(CdcSinkRelationType.MAP, parsed.type) + self.assertTrue(parsed.case_sensitive_keys) + self.assertEqual("child", parsed.embedded_tables[0].source_table_name) + self.assertEqual("Customers", parsed.linked_tables[0].linked_collection_name) + + +class TestCdcSinkOnDeleteConfig(unittest.TestCase): + def test_to_json_keys(self): + config = CdcSinkOnDeleteConfig(patch="this.Archived = true;", ignore_deletes=True) + self.assertEqual({"Patch": "this.Archived = true;", "IgnoreDeletes": True}, config.to_json()) + + def test_round_trip(self): + parsed = CdcSinkOnDeleteConfig.from_json({"Patch": "p", "IgnoreDeletes": False}) + self.assertEqual("p", parsed.patch) + self.assertFalse(parsed.ignore_deletes) + + +class TestCdcSinkPostgresSettings(unittest.TestCase): + def test_to_json_keys(self): + self.assertEqual( + {"PublicationName": "pub", "SlotName": "slot"}, + CdcSinkPostgresSettings(publication_name="pub", slot_name="slot").to_json(), + ) + + def test_round_trip(self): + parsed = CdcSinkPostgresSettings.from_json({"PublicationName": "pub", "SlotName": "slot"}) + self.assertEqual("pub", parsed.publication_name) + self.assertEqual("slot", parsed.slot_name) + + +class TestCdcSinkTaskState(unittest.TestCase): + def test_collection_name_constant(self): + self.assertEqual("@cdc-states", CdcSinkTaskState.collection_name) + + def test_get_document_id_preserves_casing(self): + self.assertEqual("@cdc-states/MyCdc", CdcSinkTaskState.get_document_id("MyCdc")) + + def test_to_json_shape(self): + state = CdcSinkTaskState(last_lsn="0/1", configuration_name="cdc-1") + state.tables["orders"] = CdcSinkTableLoadState( + initial_load_completed=True, last_key_values=["1"], key_columns=["id"] + ) + out = state.to_json() + self.assertEqual(["ConfigurationName", "LastLsn", "Tables"], list(out.keys())) + self.assertIsInstance(out["Tables"], dict) + self.assertIsInstance(out["Tables"]["orders"], dict) + + def test_to_json_preserves_table_key_casing(self): + state = CdcSinkTaskState(configuration_name="cdc-1") + state.tables["Orders"] = CdcSinkTableLoadState() + out = state.to_json() + self.assertIn("Orders", out["Tables"]) + + def test_tables_dict_lookup_is_case_insensitive(self): + state = CdcSinkTaskState(configuration_name="cdc-1") + state.tables["orders"] = CdcSinkTableLoadState(initial_load_completed=True) + self.assertIn("ORDERS", state.tables) + self.assertTrue(state.tables["ORDERS"].initial_load_completed) + + def test_tables_dict_reinsert_with_different_casing_updates_single_entry(self): + state = CdcSinkTaskState(configuration_name="cdc-1") + state.tables["orders"] = CdcSinkTableLoadState(initial_load_completed=False) + state.tables["ORDERS"] = CdcSinkTableLoadState(initial_load_completed=True) + self.assertEqual(1, len(state.tables)) + self.assertTrue(state.tables["orders"].initial_load_completed) + out = state.to_json() + self.assertEqual(1, len(out["Tables"])) + self.assertTrue(out["Tables"]["ORDERS"]["InitialLoadCompleted"]) + + def test_tables_dict_removal_through_any_casing_cleans_up(self): + state = CdcSinkTaskState(configuration_name="cdc-1") + state.tables["orders"] = CdcSinkTableLoadState(initial_load_completed=True) + state.tables["products"] = CdcSinkTableLoadState() + del state.tables["ORDERS"] + self.assertEqual(1, len(state.tables)) + self.assertNotIn("orders", state.tables) + out = state.to_json() + self.assertEqual(["products"], list(out["Tables"].keys())) + self.assertTrue(out["Tables"]["products"]["InitialLoadCompleted"] is False) + + def test_from_json_missing_tables_yields_empty_dict(self): + state = CdcSinkTaskState.from_json({"ConfigurationName": "cdc-1", "LastLsn": "0/1"}) + self.assertEqual({}, state.tables) + self.assertEqual("0/1", state.last_lsn) + + def test_from_json_builds_case_insensitive_tables_at_parse_time(self): + raw = { + "ConfigurationName": "cdc-1", + "LastLsn": "0/1", + "Tables": {"orders": {"InitialLoadCompleted": True, "LastKeyValues": ["1"], "KeyColumns": ["id"]}}, + } + state = CdcSinkTaskState.from_json(raw) + self.assertTrue(state.tables["ORDERS"].initial_load_completed) + self.assertEqual(["1"], state.tables["Orders"].last_key_values) + self.assertEqual(["id"], state.tables["orders"].key_columns) + + def test_table_load_state_null_lists_on_wire(self): + out = CdcSinkTableLoadState().to_json() + self.assertEqual(["InitialLoadCompleted", "LastKeyValues", "KeyColumns"], list(out.keys())) + self.assertIsNone(out["LastKeyValues"]) + self.assertIsNone(out["KeyColumns"]) + + def test_table_load_state_from_json_nulls_stay_none(self): + load = CdcSinkTableLoadState.from_json({"InitialLoadCompleted": False}) + self.assertIsNone(load.last_key_values) + self.assertIsNone(load.key_columns) + load = CdcSinkTableLoadState.from_json( + {"InitialLoadCompleted": True, "LastKeyValues": None, "KeyColumns": None} + ) + self.assertIsNone(load.last_key_values) + self.assertTrue(load.initial_load_completed) + + def test_tables_dict_type(self): + self.assertIsInstance(CdcSinkTablesDict(), dict) + + +class TestNestedConfigurationRoundTrip(unittest.TestCase): + def test_fully_populated_configuration_survives_round_trip(self): + config = CdcSinkConfiguration( + task_id=7, + name="cdc-nested", + connection_string_name="sql-cs", + skip_initial_load=True, + tables=[ + CdcSinkTableConfig( + collection_name="Orders", + source_table_schema="dbo", + source_table_name="orders", + columns=[ + CdcColumnMapping(column="id", name="Id"), + CdcColumnMapping(column="data", name="Data", type=CdcColumnType.JSON), + ], + primary_key_columns=["id"], + patch="this.Total = $row.total;", + on_delete=CdcSinkOnDeleteConfig(patch="this.Archived = true;", ignore_deletes=True), + disabled=False, + embedded_tables=[ + CdcSinkEmbeddedTableConfig( + source_table_name="order_lines", + property_name="Lines", + columns=[CdcColumnMapping(column="id", name="Id")], + primary_key_columns=["id"], + join_columns=["order_id"], + type=CdcSinkRelationType.ARRAY, + case_sensitive_keys=True, + embedded_tables=[ + CdcSinkEmbeddedTableConfig(source_table_name="line_items", property_name="Items") + ], + ) + ], + linked_tables=[ + CdcSinkLinkedTableConfig( + source_table_name="customers", + property_name="Customer", + join_columns=["customer_id"], + linked_collection_name="Customers", + ) + ], + ), + CdcSinkTableConfig(collection_name="Products", source_table_name="products"), + ], + postgres=CdcSinkPostgresSettings(publication_name="rvn_cdc_p_1", slot_name="rvn_cdc_s_1"), + ) + parsed = CdcSinkConfiguration.from_json(config.to_json()) + self.assertEqual(config.to_json(), parsed.to_json()) + self.assertEqual("Orders", parsed.tables[0].collection_name) + self.assertEqual(CdcColumnType.JSON, parsed.tables[0].columns[1].type) + self.assertEqual("Array", parsed.tables[0].embedded_tables[0].type.value) + self.assertEqual("Customers", parsed.tables[0].linked_tables[0].linked_collection_name) + self.assertEqual("rvn_cdc_p_1", parsed.postgres.publication_name) + # table order is preserved + self.assertEqual(["Orders", "Products"], [t.collection_name for t in parsed.tables]) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/cdc_sink_tests/test_cdc_sink_integration.py b/ravendb/tests/cdc_sink_tests/test_cdc_sink_integration.py new file mode 100644 index 00000000..e08c5041 --- /dev/null +++ b/ravendb/tests/cdc_sink_tests/test_cdc_sink_integration.py @@ -0,0 +1,139 @@ +"""Integration tests for the CDC Sink task surface against a live RavenDB server. + +The server gates CDC Sink behind a license, so the tests follow the AI-agent +integration pattern: skipped when RAVENDB_LICENSE is not set. The add request +is attempted through the standard maintenance executor; a server rejection +surfaces as a base RavenException whose message embeds the server error text, +and an accepted task is read back through GetOngoingTaskInfoOperation to +verify the OngoingTaskType.CDC_SINK dispatch. +""" + +import os +import unittest + +from ravendb.exceptions.raven_exceptions import RavenException +from ravendb.documents.operations.cdc_sink import ( + AddCdcSinkOperation, + CdcColumnMapping, + CdcSinkConfiguration, + CdcSinkTableConfig, + UpdateCdcSinkOperation, +) +from ravendb.documents.operations.connection_string.put_connection_string_operation import ( + PutConnectionStringOperation, +) +from ravendb.documents.operations.connection_string.remove_connection_string_operation import ( + RemoveConnectionStringOperation, +) +from ravendb.documents.operations.etl.sql import SqlConnectionString +from ravendb.documents.operations.ongoing_tasks import ( + DeleteOngoingTaskOperation, + GetOngoingTaskInfoOperation, + OngoingTaskCdcSink, + OngoingTaskType, +) +from ravendb.tests.test_base import TestBase + + +@unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") +class TestCdcSinkIntegration(TestBase): + CONNECTION_STRING_NAME = "cdc-sink-test-sql-cs" + + def setUp(self): + super().setUp() + sql_connection_string = SqlConnectionString( + name=self.CONNECTION_STRING_NAME, + connection_string="Data Source=localhost;Initial Catalog=test;Integrated Security=true", + factory_name="System.Data.SqlClient", + ) + self.store.maintenance.send(PutConnectionStringOperation(sql_connection_string)) + self._created_task_ids = [] + + def tearDown(self): + for task_id in self._created_task_ids: + try: + self.store.maintenance.send(DeleteOngoingTaskOperation(task_id, OngoingTaskType.CDC_SINK)) + except Exception: + pass + try: + self.store.maintenance.send( + RemoveConnectionStringOperation( + SqlConnectionString(name=self.CONNECTION_STRING_NAME, connection_string="", factory_name="") + ) + ) + except Exception: + pass + super().tearDown() + + def _valid_config(self, name="cdc-sink-integration-task"): + return CdcSinkConfiguration( + name=name, + connection_string_name=self.CONNECTION_STRING_NAME, + tables=[ + CdcSinkTableConfig( + collection_name="Orders", + source_table_schema="dbo", + source_table_name="orders", + columns=[ + CdcColumnMapping(column="id", name="Id"), + CdcColumnMapping(column="customer", name="Customer"), + ], + primary_key_columns=["id"], + ) + ], + ) + + def test_add_read_back_and_delete_cdc_sink_task(self): + config = self._valid_config() + try: + add_result = self.store.maintenance.send(AddCdcSinkOperation(config)) + except RavenException as e: + # Server rejected the task (e.g. community server without a CDC Sink + # license): the rejection must surface as a RavenException whose message + # embeds the server's error text, never the Message field alone. + self.assertIsInstance(e, RavenException) + self.assertIn("Your license doesn't support using the CDC sink feature.", str(e)) + return + + self.assertIsNotNone(add_result.raft_command_index) + self.assertIsNotNone(add_result.task_id) + self._created_task_ids.append(add_result.task_id) + + ongoing_task = self.store.maintenance.send( + GetOngoingTaskInfoOperation(add_result.task_id, OngoingTaskType.CDC_SINK) + ) + self.assertIsInstance(ongoing_task, OngoingTaskCdcSink) + self.assertEqual(OngoingTaskType.CDC_SINK, ongoing_task.task_type) + self.assertEqual(self.CONNECTION_STRING_NAME, ongoing_task.connection_string_name) + self.assertEqual("cdc-sink-integration-task", ongoing_task.configuration.name) + + def test_update_cdc_sink_task(self): + config = self._valid_config() + try: + add_result = self.store.maintenance.send(AddCdcSinkOperation(config)) + except RavenException as e: + self.assertIsInstance(e, RavenException) + self.assertIn("Your license doesn't support using the CDC sink feature.", str(e)) + return + + self._created_task_ids.append(add_result.task_id) + + config.disabled = True + config.task_id = add_result.task_id + update_result = self.store.maintenance.send(UpdateCdcSinkOperation(add_result.task_id, config)) + self.assertIsNotNone(update_result.raft_command_index) + # The update deletes and re-adds the task; the response TaskId is the + # raft index of the update command (the new task id), not the old one. + self.assertNotEqual(add_result.task_id, update_result.task_id) + self._created_task_ids.append(update_result.task_id) + + ongoing_task = self.store.maintenance.send( + GetOngoingTaskInfoOperation(update_result.task_id, OngoingTaskType.CDC_SINK) + ) + self.assertIsInstance(ongoing_task, OngoingTaskCdcSink) + self.assertEqual(update_result.task_id, ongoing_task.task_id) + self.assertTrue(ongoing_task.configuration.disabled) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/cdc_sink_tests/test_cdc_sink_operations.py b/ravendb/tests/cdc_sink_tests/test_cdc_sink_operations.py new file mode 100644 index 00000000..1e98ec83 --- /dev/null +++ b/ravendb/tests/cdc_sink_tests/test_cdc_sink_operations.py @@ -0,0 +1,381 @@ +"""Wire and dispatch tests for the CDC Sink operations: add/update request +shapes and results, server-error surfacing, ongoing-task integration, and +the DatabaseRecord.CdcSinks list. +""" + +import json +import unittest + +from ravendb.documents.operations.cdc_sink.add_cdc_sink_operation import ( + AddCdcSinkCommand, + AddCdcSinkOperation, + AddCdcSinkOperationResult, +) +from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration +from ravendb.documents.operations.cdc_sink.update_cdc_sink_operation import ( + UpdateCdcSinkCommand, + UpdateCdcSinkOperation, + UpdateCdcSinkOperationResult, +) +from ravendb.documents.operations.ongoing_tasks import ( + DeleteOngoingTaskOperation, + GetOngoingTaskInfoOperation, + OngoingTask, + OngoingTaskCdcSink, + OngoingTaskType, +) +from ravendb.exceptions.exception_dispatcher import ExceptionDispatcher +from ravendb.exceptions.raven_exceptions import RavenException +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.database_record import DatabaseRecord + + +class TestAddCdcSinkOperationWire(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + self.config = CdcSinkConfiguration(name="cdc-1", connection_string_name="sql-cs") + self.command = AddCdcSinkCommand(None, self.config) + + def test_put_to_admin_cdc_sink(self): + request = self.command.create_request(self.node) + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/databases/db1/admin/cdc-sink", request.url) + + def test_configuration_json_is_the_body(self): + request = self.command.create_request(self.node) + self.assertEqual("cdc-1", json.loads(request.data)["Name"]) + + def test_not_a_read_request(self): + self.assertFalse(self.command.is_read_request()) + + def test_implements_raft_command_marker(self): + self.assertIsInstance(self.command, RaftCommand) + self.assertTrue(self.command.get_raft_unique_request_id()) + + def test_null_response_raises(self): + with self.assertRaises(ValueError): + self.command.set_response(None, False) + + def test_result_parses_raft_command_index_and_task_id(self): + self.command.set_response(json.dumps({"RaftCommandIndex": 5, "TaskId": 9}), False) + self.assertEqual(5, self.command.result.raft_command_index) + self.assertEqual(9, self.command.result.task_id) + + def test_result_round_trips_beyond_2_31(self): + result = AddCdcSinkOperationResult.from_json({"RaftCommandIndex": 99999999999999, "TaskId": 5000000000}) + self.assertEqual(99999999999999, result.raft_command_index) + self.assertEqual(5000000000, result.task_id) + + def test_result_missing_keys_default_to_zero(self): + result = AddCdcSinkOperationResult.from_json({}) + self.assertEqual(0, result.raft_command_index) + self.assertEqual(0, result.task_id) + + def test_operation_get_command(self): + operation = AddCdcSinkOperation(self.config) + self.assertIsInstance(operation.get_command(None), AddCdcSinkCommand) + + +class TestUpdateCdcSinkOperationWire(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + self.config = CdcSinkConfiguration(name="cdc-1", connection_string_name="sql-cs") + self.command = UpdateCdcSinkCommand(None, 42, self.config) + + def test_put_to_admin_cdc_sink_with_id_query(self): + request = self.command.create_request(self.node) + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/databases/db1/admin/cdc-sink?id=42", request.url) + + def test_not_a_read_request(self): + self.assertFalse(self.command.is_read_request()) + + def test_implements_raft_command_marker(self): + self.assertIsInstance(self.command, RaftCommand) + self.assertTrue(self.command.get_raft_unique_request_id()) + + def test_null_response_raises(self): + with self.assertRaises(ValueError): + self.command.set_response(None, False) + + def test_result_parses(self): + self.command.set_response(json.dumps({"RaftCommandIndex": 5, "TaskId": 42}), False) + self.assertIsInstance(self.command.result, UpdateCdcSinkOperationResult) + self.assertEqual(5, self.command.result.raft_command_index) + self.assertEqual(42, self.command.result.task_id) + + def test_result_missing_keys_default_to_zero(self): + self.command.set_response(json.dumps({}), False) + self.assertEqual(0, self.command.result.raft_command_index) + self.assertEqual(0, self.command.result.task_id) + + def test_operation_get_command(self): + operation = UpdateCdcSinkOperation(42, self.config) + self.assertIsInstance(operation.get_command(None), UpdateCdcSinkCommand) + + +class TestExceptionSurfacing(unittest.TestCase): + def _dispatch(self, error_type, message, error, code=402): + schema = ExceptionDispatcher.ExceptionSchema( + url="http://localhost:8080", object_type=error_type, message=message, error=error + ) + return ExceptionDispatcher.get(schema, code) + + def test_unmapped_type_surfaces_base_raven_exception_with_full_error_text(self): + exception = self._dispatch( + "Raven.Client.Exceptions.Commercial.LicenseLimitException", + "Your license doesn't support using the CDC sink feature.", + "Raven.Client.Exceptions.Commercial.LicenseLimitException: Your license doesn't support using the CDC sink feature.", + ) + self.assertIsInstance(exception, RavenException) + self.assertIn("Your license doesn't support using the CDC sink feature.", str(exception)) + self.assertIn("The server at http://localhost:8080 responded with status code: 402", str(exception)) + + def test_500_lookup_failure_message_embeds_inner_error(self): + error = ( + "System.InvalidOperationException: Invalid CDC Sink configuration.\n" + "Errors:\n" + "- Could not find connection string named 'sql-cs'. Please supply an existing connection string.\n" + "Configuration:\n{json}\n" + ) + exception = self._dispatch( + "System.InvalidOperationException", + "Invalid CDC Sink configuration.", + error, + code=500, + ) + self.assertIsInstance(exception, RavenException) + self.assertIn( + "Could not find connection string named 'sql-cs'. Please supply an existing connection string.", + str(exception), + ) + + def test_500_nonexistent_task_message_embeds_inner_error(self): + error = ( + "Raven.Server.Rachis.RachisApplyException: Failed to update database record.\n" + " ---> System.InvalidOperationException: CDC Sink task with ID 999999 does not exist.\n" + ) + exception = self._dispatch( + "Raven.Server.Rachis.RachisApplyException", + "Failed to update database record.", + error, + code=500, + ) + self.assertIsInstance(exception, RavenException) + self.assertIn("Failed to update database record.", str(exception)) + self.assertIn("CDC Sink task with ID 999999 does not exist.", str(exception)) + + +class TestOngoingTaskType(unittest.TestCase): + def test_cdc_sink_wire_value(self): + self.assertEqual("CdcSink", OngoingTaskType.CDC_SINK.value) + + def test_delete_operation_flows_type_to_wire(self): + operation = DeleteOngoingTaskOperation(42, OngoingTaskType.CDC_SINK) + command = operation.get_command(None) + request = command.create_request(ServerNode("http://localhost:8080", "db1")) + self.assertEqual("http://localhost:8080/databases/db1/admin/tasks?id=42&type=CdcSink", request.url) + + +class TestOngoingTaskCdcSink(unittest.TestCase): + def _task_dict(self, **overrides): + base = { + "TaskId": 7, + "TaskType": "CdcSink", + "ResponsibleNode": {"NodeTag": "A", "NodeUrl": "http://x", "ResponsibleNode": "A"}, + "TaskState": "Enabled", + "TaskConnectionStatus": "Active", + "TaskName": "cdc-1", + "Error": None, + "MentorNode": "A", + "PinToMentorNode": True, + "ConnectionStringName": "sql-cs", + "FactoryName": "System.Data.SqlClient", + "Configuration": { + "Name": "cdc-1", + "TaskId": 7, + "Disabled": False, + "ConnectionStringName": "sql-cs", + "MentorNode": None, + "PinToMentorNode": True, + "Tables": [], + "Postgres": None, + "SkipInitialLoad": False, + }, + "LastBatchTime": "2026-08-18T10:00:00.0000000Z", + "LastCheckpoint": "0/1", + "SecondsSinceLastBatch": 12.5, + "LastActivityTime": "2026-08-18T10:05:00.0000000Z", + "SecondsSinceLastActivity": 3.25, + "HealthIssue": None, + } + base.update(overrides) + return base + + def test_from_json_parses_all_keys(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + self.assertIsInstance(task, OngoingTaskCdcSink) + self.assertEqual(OngoingTaskType.CDC_SINK, task.task_type) + self.assertEqual(7, task.task_id) + self.assertEqual("cdc-1", task.task_name) + self.assertEqual("sql-cs", task.connection_string_name) + self.assertEqual("System.Data.SqlClient", task.factory_name) + self.assertEqual("0/1", task.last_checkpoint) + self.assertEqual(12.5, task.seconds_since_last_batch) + self.assertEqual(3.25, task.seconds_since_last_activity) + self.assertIsNone(task.health_issue) + + def test_datetimes_parse_as_naive_utc(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + self.assertEqual("2026-08-18 10:00:00", str(task.last_batch_time)) + self.assertEqual("2026-08-18 10:05:00", str(task.last_activity_time)) + self.assertIsNone(task.last_batch_time.tzinfo) + + def test_configuration_parses_as_object(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration + + self.assertIsInstance(task.configuration, CdcSinkConfiguration) + self.assertEqual("cdc-1", task.configuration.name) + + def test_nullable_fields_stay_none_when_absent(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + task = OngoingTaskCdcSink.from_json( + { + "TaskId": 7, + "TaskType": "CdcSink", + "TaskName": "cdc-1", + } + ) + self.assertIsNone(task.last_batch_time) + self.assertIsNone(task.last_checkpoint) + self.assertIsNone(task.seconds_since_last_batch) + self.assertIsNone(task.configuration) + + def test_to_json_key_order(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + out = task.to_json() + self.assertEqual( + [ + "TaskId", + "TaskType", + "ResponsibleNode", + "TaskState", + "TaskConnectionStatus", + "TaskName", + "Error", + "MentorNode", + "PinToMentorNode", + "ConnectionStringName", + "FactoryName", + "Configuration", + "LastBatchTime", + "LastCheckpoint", + "SecondsSinceLastBatch", + "LastActivityTime", + "SecondsSinceLastActivity", + "HealthIssue", + ], + list(out.keys()), + ) + self.assertEqual("CdcSink", out["TaskType"]) + self.assertEqual("System.Data.SqlClient", out["FactoryName"]) + + def test_configuration_serializes_with_type_omission_rule(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + configuration_out = task.to_json()["Configuration"] + self.assertNotIn("TestMode", configuration_out) + self.assertIsNone(configuration_out["Postgres"]) + + def test_to_json_writes_none_for_unset_configuration(self): + task = OngoingTaskCdcSink.from_json({"TaskId": 1, "TaskType": "CdcSink"}) + self.assertIsNone(task.to_json()["Configuration"]) + + def test_fractional_seconds_round_trip(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + out = task.to_json() + self.assertEqual(12.5, out["SecondsSinceLastBatch"]) + self.assertEqual(3.25, out["SecondsSinceLastActivity"]) + + +class TestGetOngoingTaskInfoDispatch(unittest.TestCase): + def test_cdc_sink_dispatches_to_ongoing_task_cdc_sink(self): + operation = GetOngoingTaskInfoOperation(7, OngoingTaskType.CDC_SINK) + command = operation.get_command(None) + json_dict = { + "TaskId": 7, + "TaskType": "CdcSink", + "ConnectionStringName": "sql-cs", + "Configuration": None, + } + task = command._deserialize_task(json_dict) + self.assertIsInstance(task, OngoingTaskCdcSink) + + def test_get_request_url(self): + operation = GetOngoingTaskInfoOperation(7, OngoingTaskType.CDC_SINK) + command = operation.get_command(None) + request = command.create_request(ServerNode("http://localhost:8080", "db1")) + self.assertEqual("http://localhost:8080/databases/db1/task?key=7&type=CdcSink", request.url) + + +class TestDatabaseRecordCdcSinks(unittest.TestCase): + def _base_record(self, **overrides): + record = {"DatabaseName": "db", "LockMode": "Unlock", "AutoIndexes": {}} + record.update(overrides) + return record + + def test_from_json_parses_cdc_sinks(self): + raw = self._base_record( + CdcSinks=[ + { + "Name": "cdc-1", + "TaskId": 9, + "Disabled": False, + "ConnectionStringName": "sql-cs", + "MentorNode": None, + "PinToMentorNode": False, + "Tables": [], + "Postgres": None, + "SkipInitialLoad": False, + } + ] + ) + record = DatabaseRecord.from_json(raw) + self.assertEqual(1, len(record.cdc_sinks)) + self.assertEqual("cdc-1", record.cdc_sinks[0].name) + self.assertEqual("sql-cs", record.cdc_sinks[0].connection_string_name) + + def test_missing_cdc_sinks_key_yields_empty_list(self): + record = DatabaseRecord.from_json(self._base_record()) + self.assertEqual([], record.cdc_sinks) + + def test_to_json_emits_cdc_sinks_with_configuration_shape(self): + record = DatabaseRecord.from_json( + self._base_record( + CdcSinks=[ + { + "Name": "cdc-1", + "TaskId": 9, + "Disabled": False, + "ConnectionStringName": "sql-cs", + "MentorNode": None, + "PinToMentorNode": False, + "Tables": [], + "Postgres": None, + "SkipInitialLoad": False, + } + ] + ) + ) + out = record.to_json() + self.assertIn("CdcSinks", out) + self.assertEqual("cdc-1", out["CdcSinks"][0]["Name"]) + + def test_empty_cdc_sinks_serialize_as_empty_array(self): + record = DatabaseRecord.from_json(self._base_record()) + self.assertEqual([], record.to_json()["CdcSinks"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/cdc_sink_tests/test_client_surface_additions.py b/ravendb/tests/cdc_sink_tests/test_client_surface_additions.py new file mode 100644 index 00000000..4170aab2 --- /dev/null +++ b/ravendb/tests/cdc_sink_tests/test_client_surface_additions.py @@ -0,0 +1,255 @@ +"""Tests for the client version header, the S3 checksum-validation flag, the +connection-string UsedBy metadata, and the pull-replication sink cursors. +""" + +import json +import unittest + +import requests + +from ravendb.documents.operations.attachments import RemoteAttachmentsS3Settings +from ravendb.documents.operations.backups.settings import ( + GetBackupConfigurationScript, + S3Settings, +) +from ravendb.documents.operations.connection_string.get_connection_string_operation import ( + GetConnectionStringsOperation, + GetConnectionStringsResult, +) +from ravendb.documents.operations.connection_strings import ConnectionStringUsage +from ravendb.documents.operations.ongoing_tasks import ( + OngoingTaskPullReplicationAsSink, + OngoingTaskType, +) +from ravendb.http.request_executor import RequestExecutor +from ravendb.http.server_node import ServerNode +from ravendb.primitives import constants + + +class TestClientVersionHeader(unittest.TestCase): + def test_client_version_is_7_2_5(self): + self.assertEqual("7.2.5", RequestExecutor.CLIENT_VERSION) + + def test_wire_header_carries_client_version(self): + executor = object.__new__(RequestExecutor) + executor._disable_client_configuration_updates = False + executor._client_configuration_etag = 0 + executor._disable_topology_updates = False + executor._topology_etag = 0 + + request = requests.Request("GET", "http://localhost:8080/databases/db1/") + executor._set_request_headers(None, None, request) + self.assertEqual("7.2.5", request.headers[constants.Headers.CLIENT_VERSION]) + + +class TestS3ChecksumValidation(unittest.TestCase): + def _script(self): + return GetBackupConfigurationScript.default() + + def test_to_json_writes_disable_checksum_validation(self): + settings = S3Settings( + bucket_name="b", disable_checksum_validation=True, get_backup_configuration_script=self._script() + ) + out = settings.to_json() + self.assertTrue(out["DisableChecksumValidation"]) + + def test_from_json_parses_flag(self): + script = self._script().to_json() + settings = S3Settings.from_json( + { + "Disabled": False, + "GetBackupConfigurationScript": script, + "AwsAccessKey": None, + "AwsSecretKey": None, + "AwsSessionToken": None, + "AwsRegionName": None, + "RemoteFolderName": None, + "BucketName": "b", + "CustomServerUrl": None, + "ForcePathStyle": False, + "DisableChecksumValidation": True, + } + ) + self.assertTrue(settings.disable_checksum_validation) + + def test_from_json_defaults_flag_to_false_when_absent(self): + script = self._script().to_json() + settings = S3Settings.from_json( + { + "Disabled": False, + "GetBackupConfigurationScript": script, + "AwsAccessKey": None, + "AwsSecretKey": None, + "AwsSessionToken": None, + "AwsRegionName": None, + "RemoteFolderName": None, + "BucketName": "b", + "CustomServerUrl": None, + "ForcePathStyle": False, + } + ) + self.assertFalse(settings.disable_checksum_validation) + + def test_equality_includes_flag(self): + a = S3Settings(bucket_name="b", disable_checksum_validation=False) + b = S3Settings(bucket_name="b", disable_checksum_validation=True) + self.assertNotEqual(a, b) + same = S3Settings(bucket_name="b", disable_checksum_validation=False) + self.assertEqual(a, same) + + def test_stays_hashable_and_hash_consistent_with_equality(self): + a = S3Settings(bucket_name="b", disable_checksum_validation=False) + same = S3Settings(bucket_name="b", disable_checksum_validation=False) + b = S3Settings(bucket_name="b", disable_checksum_validation=True) + self.assertEqual(hash(a), hash(same)) + self.assertNotEqual(hash(a), hash(b)) + self.assertEqual(len({a, same, b}), 2) + + +class TestRemoteAttachmentsS3ChecksumValidation(unittest.TestCase): + def test_to_json_writes_disable_checksum_validation(self): + settings = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=True) + self.assertTrue(settings.to_json()["DisableChecksumValidation"]) + + def test_from_json_parses_flag_and_defaults_false(self): + self.assertTrue( + RemoteAttachmentsS3Settings.from_json( + {"BucketName": "b", "DisableChecksumValidation": True} + ).disable_checksum_validation + ) + self.assertFalse(RemoteAttachmentsS3Settings.from_json({"BucketName": "b"}).disable_checksum_validation) + + def test_equality_includes_flag(self): + a = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=False) + b = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=True) + self.assertNotEqual(a, b) + self.assertEqual(a, RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=False)) + + def test_stays_hashable_and_hash_consistent_with_equality(self): + a = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=False) + same = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=False) + b = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=True) + self.assertEqual(hash(a), hash(same)) + self.assertNotEqual(hash(a), hash(b)) + self.assertEqual(len({a, same, b}), 2) + + +class TestConnectionStringUsedBy(unittest.TestCase): + def _result_dict(self, used_by=None): + return { + "RavenConnectionStrings": { + "raven": { + "Name": "raven", + "Database": "db", + "TopologyDiscoveryUrls": [], + "UsedBy": used_by if used_by is not None else [], + } + }, + "SqlConnectionStrings": {}, + "OlapConnectionStrings": {}, + "AiConnectionStrings": {}, + "ElasticSearchConnectionStrings": {}, + "QueueConnectionStrings": {}, + "SnowflakeConnectionStrings": {}, + } + + def test_used_by_parsed_from_get_response(self): + result = GetConnectionStringsResult.from_json( + self._result_dict( + used_by=[ + { + "Kind": "SqlEtl", + "Id": 5000000000, + "Identifier": None, + "Name": "etl-1", + }, + {"Kind": "AiAgent", "Id": None, "Identifier": "agent-1", "Name": "agent-1"}, + ] + ) + ) + connection_string = result.raven_connection_strings["raven"] + self.assertEqual(2, len(connection_string.used_by)) + first = connection_string.used_by[0] + self.assertIsInstance(first, ConnectionStringUsage) + self.assertEqual("SqlEtl", first.kind) + self.assertEqual(5000000000, first.id) + self.assertIsNone(first.identifier) + self.assertEqual("etl-1", first.name) + second = connection_string.used_by[1] + self.assertEqual("AiAgent", second.kind) + self.assertIsNone(second.id) + self.assertEqual("agent-1", second.identifier) + + def test_used_by_empty_array_when_nothing_uses_it(self): + result = GetConnectionStringsResult.from_json(self._result_dict()) + self.assertEqual([], result.raven_connection_strings["raven"].used_by) + + def test_absent_used_by_key_does_not_raise(self): + raw = self._result_dict() + del raw["RavenConnectionStrings"]["raven"]["UsedBy"] + result = GetConnectionStringsResult.from_json(raw) + self.assertEqual([], result.raven_connection_strings["raven"].used_by) + + def test_to_json_never_emits_used_by(self): + result = GetConnectionStringsResult.from_json(self._result_dict(used_by=[{"Kind": "SqlEtl", "Id": 1}])) + connection_string = result.raven_connection_strings["raven"] + out = connection_string.to_json() + self.assertNotIn("UsedBy", out) + # round-trip: GET -> from_json -> to_json -> PUT must not leak UsedBy + self.assertNotIn("UsedBy", json.dumps(out, default=str)) + + def test_get_command_is_a_read_request(self): + operation = GetConnectionStringsOperation() + command = operation.get_command(None) + self.assertTrue(command.is_read_request()) + request = command.create_request(ServerNode("http://localhost:8080", "db1")) + self.assertEqual("http://localhost:8080/databases/db1/admin/connection-strings", request.url) + + +class TestPullReplicationAsSinkCursors(unittest.TestCase): + def _sink_dict(self, **overrides): + base = { + "TaskId": 1, + "TaskType": "PullReplicationAsSink", + "HubName": "hub", + "AllowedHubToSinkPaths": ["a"], + "AllowedSinkToHubPaths": ["b"], + "HubCursor": "hub-cursor-1", + "SinkCursor": "sink-cursor-1", + } + base.update(overrides) + return base + + def test_from_json_parses_hub_and_sink_cursor(self): + task = OngoingTaskPullReplicationAsSink.from_json(self._sink_dict()) + self.assertEqual("hub-cursor-1", task.hub_cursor) + self.assertEqual("sink-cursor-1", task.sink_cursor) + + def test_to_json_writes_cursors_after_allowed_sink_to_hub_paths(self): + task = OngoingTaskPullReplicationAsSink.from_json(self._sink_dict()) + out = task.to_json() + self.assertEqual("hub-cursor-1", out["HubCursor"]) + self.assertEqual("sink-cursor-1", out["SinkCursor"]) + keys = list(out.keys()) + self.assertGreater(keys.index("HubCursor"), keys.index("AllowedSinkToHubPaths")) + self.assertEqual(["HubCursor", "SinkCursor"], keys[-2:]) + + def test_absent_keys_stay_none_and_serialize_as_none(self): + task = OngoingTaskPullReplicationAsSink.from_json(self._sink_dict(HubCursor=None)) + self.assertIsNone(task.hub_cursor) + task = OngoingTaskPullReplicationAsSink.from_json({"TaskId": 1, "TaskType": "PullReplicationAsSink"}) + self.assertIsNone(task.hub_cursor) + self.assertIsNone(task.sink_cursor) + out = task.to_json() + self.assertIsNone(out["HubCursor"]) + self.assertIsNone(out["SinkCursor"]) + + def test_round_trip(self): + task = OngoingTaskPullReplicationAsSink.from_json(self._sink_dict()) + parsed = OngoingTaskPullReplicationAsSink.from_json(task.to_json()) + self.assertEqual("hub-cursor-1", parsed.hub_cursor) + self.assertEqual("sink-cursor-1", parsed.sink_cursor) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/queue_tests/__init__.py b/ravendb/tests/queue_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ravendb/tests/queue_tests/test_azure_service_bus.py b/ravendb/tests/queue_tests/test_azure_service_bus.py new file mode 100644 index 00000000..cfd4aabe --- /dev/null +++ b/ravendb/tests/queue_tests/test_azure_service_bus.py @@ -0,0 +1,259 @@ +"""Tests for the Azure Service Bus surface: the broker type, +the connection settings classes with exactly-one-auth validation, the queue +connection-string field, and the sink-source helpers with entry validation. +""" + +import unittest + +from ravendb.documents.operations.etl.queue.azure_service_bus_connection_settings import ( + AzureServiceBusConnectionSettings, + AzureServiceBusEntraId, + AzureServiceBusPasswordless, +) +from ravendb.documents.operations.etl.queue.azure_service_bus_sink_source import AzureServiceBusSinkSource +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType, QueueConnectionString +from ravendb.exceptions.exceptions import InvalidOperationException + + +class TestQueueBrokerType(unittest.TestCase): + def test_azure_service_bus_value(self): + self.assertEqual("AzureServiceBus", QueueBrokerType.AZURE_SERVICE_BUS.value) + + +class TestAzureServiceBusConnectionSettings(unittest.TestCase): + def test_connection_string_valid_when_contains_sb_protocol(self): + settings = AzureServiceBusConnectionSettings( + connection_string="Endpoint=sb://ns.servicebus.windows.net/;SharedAccessKeyName=k;SharedAccessKey=v" + ) + self.assertTrue(settings.is_valid_connection()) + + def test_sb_search_is_case_insensitive(self): + settings = AzureServiceBusConnectionSettings(connection_string="SB://ns.servicebus.windows.net") + self.assertTrue(settings.is_valid_connection()) + + def test_connection_string_invalid_without_sb_protocol(self): + for value in ( + "SharedAccessKeyName=key;SharedAccessKey=abc", + "Endpoint=https://ns.servicebus.windows.net/;SharedAccessKey=abc", + "nothing-useful-here", + ): + self.assertFalse(AzureServiceBusConnectionSettings(connection_string=value).is_valid_connection()) + + def test_whitespace_connection_string_counts_as_unset(self): + settings = AzureServiceBusConnectionSettings(connection_string=" ") + self.assertFalse(settings.is_valid_connection()) + # ToJson asymmetry: IsNullOrEmpty writes a whitespace-only value. + self.assertEqual({"ConnectionString": " "}, settings.to_json()) + + def test_exactly_one_auth_method_required(self): + none_set = AzureServiceBusConnectionSettings() + self.assertFalse(none_set.is_valid_connection()) + + two_set = AzureServiceBusConnectionSettings( + connection_string="Endpoint=sb://x/", + entra_id=AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s"), + ) + self.assertFalse(two_set.is_valid_connection()) + + def test_entra_id_requires_all_four_fields(self): + complete = AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s") + self.assertTrue(AzureServiceBusConnectionSettings(entra_id=complete).is_valid_connection()) + for kwargs in ( + {"namespace": "ns", "tenant_id": "t", "client_id": "c"}, + {"namespace": "ns", "tenant_id": "t", "client_id": "c", "client_secret": " "}, + ): + self.assertFalse( + AzureServiceBusConnectionSettings(entra_id=AzureServiceBusEntraId(**kwargs)).is_valid_connection() + ) + + def test_passwordless_requires_namespace(self): + self.assertTrue( + AzureServiceBusConnectionSettings( + passwordless=AzureServiceBusPasswordless(namespace="ns") + ).is_valid_connection() + ) + self.assertFalse( + AzureServiceBusConnectionSettings( + passwordless=AzureServiceBusPasswordless(namespace=" ") + ).is_valid_connection() + ) + + def test_get_service_bus_url_extracts_endpoint(self): + settings = AzureServiceBusConnectionSettings( + connection_string="Endpoint=sb://my.servicebus.windows.net;SharedAccessKeyName=key" + ) + self.assertEqual("sb://my.servicebus.windows.net", settings.get_service_bus_url()) + + def test_get_service_bus_url_extraction_starts_at_found_index(self): + settings = AzureServiceBusConnectionSettings( + connection_string="SharedAccessKeyName=key;Endpoint=sb://ns.servicebus.windows.net/;SharedAccessKey=abc" + ) + self.assertEqual("sb://ns.servicebus.windows.net/", settings.get_service_bus_url()) + + def test_get_service_bus_url_case_insensitive_extraction(self): + settings = AzureServiceBusConnectionSettings(connection_string="Endpoint=SB://ns.servicebus.windows.net/") + self.assertEqual("SB://ns.servicebus.windows.net/", settings.get_service_bus_url()) + + def test_get_service_bus_url_to_end_without_semicolon(self): + settings = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://ns.servicebus.windows.net") + self.assertEqual("sb://ns.servicebus.windows.net", settings.get_service_bus_url()) + + def test_get_service_bus_url_from_namespace(self): + settings = AzureServiceBusConnectionSettings( + entra_id=AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s") + ) + self.assertEqual("sb://ns/", settings.get_service_bus_url()) + settings = AzureServiceBusConnectionSettings(passwordless=AzureServiceBusPasswordless(namespace="ns")) + self.assertEqual("sb://ns/", settings.get_service_bus_url()) + + def test_get_service_bus_url_throws_when_no_endpoint(self): + with self.assertRaises(InvalidOperationException) as ctx: + AzureServiceBusConnectionSettings(connection_string="no sb here").get_service_bus_url() + self.assertEqual("No endpoint provided", str(ctx.exception)) + + def test_get_service_bus_url_throws_when_no_namespace(self): + with self.assertRaises(InvalidOperationException) as ctx: + AzureServiceBusConnectionSettings().get_service_bus_url() + self.assertEqual("No namespace provided", str(ctx.exception)) + + def test_to_json_writes_only_set_fields(self): + self.assertEqual({}, AzureServiceBusConnectionSettings().to_json()) + settings = AzureServiceBusConnectionSettings( + connection_string="Endpoint=sb://x/", + entra_id=AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s"), + ) + result = settings.to_json() + self.assertEqual({"Endpoint=sb://x/"}, {result["ConnectionString"]}) + self.assertIn("EntraId", result) + self.assertNotIn("Passwordless", result) + + def test_audit_json_masks_secrets(self): + settings = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://x/;Key=secret") + self.assertEqual({"ConnectionString": ""}, settings.to_audit_json()) + + settings = AzureServiceBusConnectionSettings( + entra_id=AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="secret") + ) + audit = settings.to_audit_json() + self.assertNotIn("ClientSecret", audit["EntraId"]) + + settings = AzureServiceBusConnectionSettings(passwordless=AzureServiceBusPasswordless(namespace="ns")) + self.assertEqual({"Passwordless": {"Namespace": "ns"}}, settings.to_audit_json()) + + def test_equality_and_hash(self): + a = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://x/") + b = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://x/") + c = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://y/") + self.assertEqual(a, b) + self.assertNotEqual(a, c) + self.assertEqual(hash(a), hash(b)) + + e1 = AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s") + e2 = AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s") + self.assertEqual(e1, e2) + self.assertEqual(hash(e1), hash(e2)) + + p1 = AzureServiceBusPasswordless(namespace="ns") + p2 = AzureServiceBusPasswordless(namespace="ns") + self.assertEqual(p1, p2) + self.assertEqual(hash(p1), hash(p2)) + + +class TestQueueConnectionStringAsbField(unittest.TestCase): + def test_to_json_writes_key_between_amazon_sqs_and_type(self): + connection_string = QueueConnectionString(name="q", broker_type=QueueBrokerType.AZURE_SERVICE_BUS) + keys = list(connection_string.to_json().keys()) + self.assertEqual("AmazonSqsConnectionSettings", keys[-3]) + self.assertEqual("AzureServiceBusConnectionSettings", keys[-2]) + self.assertEqual("Type", keys[-1]) + + def test_unset_field_is_null_on_wire(self): + connection_string = QueueConnectionString(name="q", broker_type=QueueBrokerType.AZURE_SERVICE_BUS) + self.assertIsNone(connection_string.to_json()["AzureServiceBusConnectionSettings"]) + + def test_from_json_parses_full_settings_object(self): + payload = { + "Name": "q", + "BrokerType": "AzureServiceBus", + "KafkaConnectionSettings": None, + "RabbitMqConnectionSettings": None, + "AzureQueueStorageConnectionSettings": None, + "AmazonSqsConnectionSettings": None, + "AzureServiceBusConnectionSettings": {"ConnectionString": "Endpoint=sb://ns/"}, # type: ignore + "Type": "Queue", + } + parsed = QueueConnectionString.from_json(payload) + self.assertIsInstance(parsed.azure_service_bus_settings, AzureServiceBusConnectionSettings) + self.assertEqual("Endpoint=sb://ns/", parsed.azure_service_bus_settings.connection_string) + + def test_round_trip(self): + settings = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://ns/") + original = QueueConnectionString( + name="q", broker_type=QueueBrokerType.AZURE_SERVICE_BUS, azure_service_bus_settings=settings + ) + back = QueueConnectionString.from_json(original.to_json()) + self.assertEqual(original.to_json(), back.to_json()) + + +class TestAzureServiceBusSinkSource(unittest.TestCase): + def test_queue_returns_name_when_valid(self): + self.assertEqual("my-queue", AzureServiceBusSinkSource.queue("my-queue")) + + def test_queue_throws_when_empty(self): + for value in (None, "", " "): + with self.assertRaises(ValueError) as ctx: + AzureServiceBusSinkSource.queue(value) + self.assertEqual("Queue name must be non-empty.", str(ctx.exception)) + + def test_queue_throws_when_contains_separator(self): + with self.assertRaises(ValueError) as ctx: + AzureServiceBusSinkSource.queue("foo;bar") + self.assertEqual("Queue name must not contain the ';' character.", str(ctx.exception)) + + def test_subscription_encodes_topic_and_subscription(self): + self.assertEqual("topic;sub", AzureServiceBusSinkSource.subscription("topic", "sub")) + + def test_subscription_throws_when_empty(self): + for topic, subscription in ( + (None, "sub"), + ("", "sub"), + (" ", "sub"), + ("topic", None), + ("topic", ""), + ("topic", " "), + ): + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.subscription(topic, subscription) + + def test_subscription_throws_when_contains_separator(self): + with self.assertRaises(ValueError) as ctx: + AzureServiceBusSinkSource.subscription("to;pic", "sub") + self.assertEqual("Topic name must not contain the ';' character.", str(ctx.exception)) + with self.assertRaises(ValueError) as ctx: + AzureServiceBusSinkSource.subscription("topic", "su;b") + self.assertEqual("Subscription name must not contain the ';' character.", str(ctx.exception)) + + def test_validate_entry_accepts_valid_entries(self): + self.assertIsNone(AzureServiceBusSinkSource.validate_entry("my-queue")) + self.assertIsNone(AzureServiceBusSinkSource.validate_entry("topic;sub")) + + def test_validate_entry_rejects_invalid_entries(self): + for entry in ("", " ", ";sub", "topic;", ";", "topic;sub;extra"): + self.assertIsNotNone(AzureServiceBusSinkSource.validate_entry(entry), entry) + + def test_validate_script_names_script_and_entry(self): + errors = AzureServiceBusSinkSource.validate_script("script", ["ok", "a;b;c"]) + self.assertEqual( + [ + "Script 'script': Azure Service Bus subscription source 'a;b;c' is invalid. " + "Use ';' with a single ';' separator and both parts non-empty." + ], + errors, + ) + errors = AzureServiceBusSinkSource.validate_script("script", [" "]) + self.assertEqual(["Script 'script': Azure Service Bus source entry cannot be empty."], errors) + self.assertEqual([], AzureServiceBusSinkSource.validate_script("script", ["my-queue"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/serverwide_tests/test_certificates_sso.py b/ravendb/tests/serverwide_tests/test_certificates_sso.py new file mode 100644 index 00000000..e9396d53 --- /dev/null +++ b/ravendb/tests/serverwide_tests/test_certificates_sso.py @@ -0,0 +1,185 @@ +"""Tests for the SSO certificate surface: CertificateMetadata +SSO fields, the CertificateUsage/SsoProvider enums, SsoIdentifier, and the +edit-operation body rules. +""" + +import unittest + +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.operations.certificates import ( + CertificateDefinition, + CertificateMetadata, + CertificateUsage, + DatabaseAccess, + EditClientCertificateOperation, + SecurityClearance, + SsoIdentifier, + SsoProvider, +) + + +class TestSsoEnums(unittest.TestCase): + def test_certificate_usage_values_are_csharp_names(self): + self.assertEqual( + ["RavenServer", "RavenServerForCommunication", "Client", "SsoServer", "SsoClient", "WellKnownIssuer"], + [m.value for m in CertificateUsage], + ) + + def test_sso_provider_values_are_csharp_names(self): + self.assertEqual(["Github", "Google", "Microsoft", "Windows"], [m.value for m in SsoProvider]) + + def test_sso_identifier_round_trip(self): + sso = SsoIdentifier(provider=SsoProvider.GITHUB, domain="github.com", identifier="alice") + self.assertEqual({"Provider": "Github", "Domain": "github.com", "Identifier": "alice"}, sso.to_json()) + back = SsoIdentifier.from_json({"Provider": "Google", "Domain": None, "Identifier": "bob"}) + self.assertEqual(SsoProvider.GOOGLE, back.provider) + self.assertIsNone(back.domain) + self.assertEqual("bob", back.identifier) + + +class TestCertificateMetadataSsoFields(unittest.TestCase): + def test_to_json_writes_sso_keys(self): + definition = CertificateDefinition( + name="n", + usage=CertificateUsage.SSO_CLIENT, + sso_server_public_key_pinning_hashes=["hash1"], + allow_any_sso_server=True, + sso_identifiers=[SsoIdentifier(provider=SsoProvider.WINDOWS, domain="corp", identifier="win\\u")], + ) + result = definition.to_json() + self.assertEqual("SsoClient", result["Usage"]) + self.assertEqual(["hash1"], result["SsoServerPublicKeyPinningHashes"]) + self.assertIs(True, result["AllowAnySsoServer"]) + self.assertEqual([{"Provider": "Windows", "Domain": "corp", "Identifier": "win\\u"}], result["SsoIdentifiers"]) + + def test_to_json_usage_null_and_defaults(self): + definition = CertificateDefinition(name="n") + result = definition.to_json() + self.assertIsNone(result["Usage"]) + self.assertEqual([], result["SsoServerPublicKeyPinningHashes"]) + self.assertIs(False, result["AllowAnySsoServer"]) + self.assertEqual([], result["SsoIdentifiers"]) + + def test_from_json_parses_sso_fields(self): + metadata = CertificateMetadata.from_json( + { + "Name": "n", + "SecurityClearance": "ClusterAdmin", + "Usage": "SsoServer", + "SsoServerPublicKeyPinningHashes": ["h"], + "AllowAnySsoServer": True, + "SsoIdentifiers": [{"Provider": "Github", "Domain": None, "Identifier": "id"}], + } + ) + self.assertEqual(CertificateUsage.SSO_SERVER, metadata.usage) + self.assertEqual(["h"], metadata.sso_server_public_key_pinning_hashes) + self.assertIs(True, metadata.allow_any_sso_server) + self.assertEqual(1, len(metadata.sso_identifiers)) + self.assertEqual(SsoProvider.GITHUB, metadata.sso_identifiers[0].provider) + + def test_from_json_missing_sso_keys_use_defaults(self): + metadata = CertificateMetadata.from_json({"Name": "n", "SecurityClearance": "ClusterAdmin"}) + self.assertIsNone(metadata.usage) + self.assertEqual([], metadata.sso_server_public_key_pinning_hashes) + self.assertIs(False, metadata.allow_any_sso_server) + self.assertEqual([], metadata.sso_identifiers) + + def test_definition_from_json_parses_sso_fields(self): + definition = CertificateDefinition.from_json( + { + "Certificate": None, + "Name": "n", + "SecurityClearance": "ClusterAdmin", + "Thumbprint": "tp", + "NotAfter": None, + "Permissions": {"db": "ReadWrite"}, + "CollectionSecondaryKeys": [], + "CollectionPrimaryKey": "", + "PublicKeyPinningHash": None, + "Usage": "Client", + "SsoServerPublicKeyPinningHashes": ["h"], + "AllowAnySsoServer": False, + "SsoIdentifiers": [], + } + ) + self.assertEqual(CertificateUsage.CLIENT, definition.usage) + self.assertEqual(["h"], definition.sso_server_public_key_pinning_hashes) + + +class TestEditClientCertificateOperationSso(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + + def _body(self, **parameters_kwargs): + parameters = EditClientCertificateOperation.Parameters( + thumbprint="tp", + permissions={"db1": DatabaseAccess.ADMIN}, + name="n", + clearance=SecurityClearance.CLUSTER_ADMIN, + **parameters_kwargs, + ) + operation = EditClientCertificateOperation(parameters) + command = operation.get_command(None) + request = command.create_request(self.node) + return request, command + + def test_base_keys_always_present(self): + request, command = self._body() + self.assertEqual("POST", request.method) + self.assertEqual("http://localhost:8080/admin/certificates/edit", request.url) + body = request.data + self.assertEqual("tp", body["Thumbprint"]) + self.assertEqual("n", body["Name"]) + self.assertEqual("ClusterAdmin", body["SecurityClearance"]) + self.assertIs(False, body["Disabled"]) + self.assertEqual({"db1": "Admin"}, body["Permissions"]) + + def test_sso_keys_omitted_when_none(self): + request, _ = self._body() + body = request.data + self.assertNotIn("SsoServerPublicKeyPinningHashes", body) + self.assertNotIn("AllowAnySsoServer", body) + self.assertNotIn("SsoIdentifiers", body) + + def test_sso_keys_written_when_provided(self): + request, _ = self._body( + sso_server_public_key_pinning_hashes=["h1", "h2"], + allow_any_sso_server=True, + sso_identifiers=[SsoIdentifier(provider=SsoProvider.GOOGLE, domain="g.com", identifier="id")], + ) + body = request.data + self.assertEqual(["h1", "h2"], body["SsoServerPublicKeyPinningHashes"]) + self.assertIs(True, body["AllowAnySsoServer"]) + self.assertEqual([{"Provider": "Google", "Identifier": "id", "Domain": "g.com"}], body["SsoIdentifiers"]) + + def test_empty_list_sent_to_clear(self): + request, _ = self._body(sso_server_public_key_pinning_hashes=[]) + self.assertEqual([], request.data["SsoServerPublicKeyPinningHashes"]) + + def test_domain_omitted_when_empty_in_edit_body(self): + request, _ = self._body( + sso_identifiers=[SsoIdentifier(provider=SsoProvider.WINDOWS, domain="", identifier="id")] + ) + self.assertEqual([{"Provider": "Windows", "Identifier": "id"}], request.data["SsoIdentifiers"]) + + def test_body_not_built_from_certificate_definition_to_json(self): + # The edit body must never carry the unconditional SSO keys that + # CertificateDefinition.to_json writes for its own serialization. + request, _ = self._body() + body = request.data + for key in ("Usage", "SsoServerPublicKeyPinningHashes", "AllowAnySsoServer", "SsoIdentifiers"): + self.assertNotIn(key, body) + + def test_command_is_raft_command(self): + request, command = self._body() + self.assertIsInstance(command, RaftCommand) + self.assertTrue(command.get_raft_unique_request_id()) + + def test_constructor_validation(self): + with self.assertRaises(ValueError): + EditClientCertificateOperation(None) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/serverwide_tests/test_server_wide_connection_strings.py b/ravendb/tests/serverwide_tests/test_server_wide_connection_strings.py new file mode 100644 index 00000000..f960739f --- /dev/null +++ b/ravendb/tests/serverwide_tests/test_server_wide_connection_strings.py @@ -0,0 +1,244 @@ +"""Tests for the server-wide connection strings surface: +the wrapper class, and the put/get/remove operations. +""" + +import json +import unittest + +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType, QueueConnectionString +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.operations.connection_strings import ( + GetServerWideConnectionStringsOperation, + PutServerWideConnectionStringOperation, + PutServerWideConnectionStringResult, + RemoveServerWideConnectionStringOperation, + RemoveServerWideConnectionStringResult, + ServerWideConnectionString, + ServerWideConnectionStringUsage, +) +from ravendb.serverwide.server_operation_executor import ConnectionStringType + + +class TestServerWideConnectionString(unittest.TestCase): + def _queue(self): + return QueueConnectionString(name="q1", broker_type=QueueBrokerType.KAFKA) + + def test_to_json_inner_keys_plus_type_and_excluded(self): + wrapper = ServerWideConnectionString(connection_string=self._queue(), excluded_databases=["db2"]) + result = wrapper.to_json() + self.assertEqual("Queue", result["Type"]) + self.assertEqual(["db2"], result["ExcludedDatabases"]) + self.assertEqual("Kafka", result["BrokerType"]) + self.assertNotIn("UsedBy", result) + self.assertEqual(1, len([k for k in result if k == "Type"])) + + def test_excluded_databases_null_when_unset(self): + wrapper = ServerWideConnectionString(connection_string=self._queue()) + self.assertIsNone(wrapper.to_json()["ExcludedDatabases"]) + + def test_name_and_type_delegate_to_inner(self): + wrapper = ServerWideConnectionString(connection_string=self._queue()) + self.assertEqual("q1", wrapper.name) + self.assertEqual("Queue", wrapper.get_type) + + def test_none_inner_delegates_to_none(self): + wrapper = ServerWideConnectionString() + self.assertIsNone(wrapper.name) + self.assertEqual("None", wrapper.get_type) + self.assertEqual({"Type": "None", "ExcludedDatabases": None}, wrapper.to_json()) + + def test_from_json_dispatches_by_type(self): + item = { + "Name": "q1", + "BrokerType": "Kafka", + "KafkaConnectionSettings": None, + "RabbitMqConnectionSettings": None, + "AzureQueueStorageConnectionSettings": None, + "AmazonSqsConnectionSettings": None, + "AzureServiceBusConnectionSettings": None, + "Type": "Queue", + "ExcludedDatabases": ["db2"], + "UsedBy": [ + { + "Kind": "QueueSink", + "Id": 99999999999, + "Identifier": "ident", + "Name": "task", + "DatabaseName": "db1", + } + ], + } + parsed = ServerWideConnectionString.from_json(item) + self.assertIsInstance(parsed, ServerWideConnectionString) + self.assertEqual("q1", parsed.name) + self.assertEqual("Queue", parsed.get_type) + self.assertEqual(["db2"], parsed.excluded_databases) + self.assertIsInstance(parsed.connection_string, QueueConnectionString) + usage = parsed.used_by[0] + self.assertIsInstance(usage, ServerWideConnectionStringUsage) + self.assertEqual("QueueSink", usage.kind) + self.assertEqual(99999999999, usage.id) + self.assertEqual("db1", usage.database_name) + + def test_from_json_returns_none_when_type_missing(self): + self.assertIsNone(ServerWideConnectionString.from_json({"Name": "x"})) + + def test_from_json_tolerates_missing_used_by(self): + parsed = ServerWideConnectionString.from_json( + { + "Name": "r1", + "Database": "d", + "TopologyDiscoveryUrls": ["http://localhost:8080"], + "Type": "Raven", + } + ) + self.assertEqual([], parsed.used_by) + + +class TestPutServerWideConnectionStringOperation(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + + def _wrapper(self): + inner = QueueConnectionString(name="q1", broker_type=QueueBrokerType.KAFKA) + return ServerWideConnectionString(connection_string=inner, excluded_databases=["db2"]) + + def test_put_url_and_body(self): + operation = PutServerWideConnectionStringOperation(self._wrapper()) + command = operation.get_command(None) + request = command.create_request(self.node) + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/admin/configuration/server-wide/connection-strings", request.url) + body = json.loads(request.data) + self.assertEqual("Queue", body["Type"]) + self.assertEqual(["db2"], body["ExcludedDatabases"]) + self.assertNotIn("UsedBy", body) + + def test_not_a_read_request(self): + command = PutServerWideConnectionStringOperation(self._wrapper()).get_command(None) + self.assertFalse(command.is_read_request()) + + def test_implements_raft_command(self): + command = PutServerWideConnectionStringOperation(self._wrapper()).get_command(None) + self.assertIsInstance(command, RaftCommand) + self.assertTrue(command.get_raft_unique_request_id()) + + def test_null_response_raises(self): + command = PutServerWideConnectionStringOperation(self._wrapper()).get_command(None) + with self.assertRaises(ValueError): + command.set_response(None, False) + + def test_result_parses_raft_command_index(self): + result = PutServerWideConnectionStringResult.from_json({"RaftCommandIndex": 1234567890123}) + self.assertEqual(1234567890123, result.raft_command_index) + + def test_constructor_validation(self): + with self.assertRaises(ValueError): + PutServerWideConnectionStringOperation(None) + with self.assertRaises(ValueError): + PutServerWideConnectionStringOperation(ServerWideConnectionString()) + + +class TestGetServerWideConnectionStringsOperation(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + + def test_no_query_when_no_params(self): + operation = GetServerWideConnectionStringsOperation() + request = operation.get_command(None).create_request(self.node) + self.assertEqual("GET", request.method) + self.assertEqual("http://localhost:8080/admin/configuration/server-wide/connection-strings", request.url) + + def test_name_then_type_query_order(self): + operation = GetServerWideConnectionStringsOperation("my cs", ConnectionStringType.QUEUE) + request = operation.get_command(None).create_request(self.node) + self.assertEqual( + "http://localhost:8080/admin/configuration/server-wide/connection-strings?name=my%20cs&type=Queue", + request.url, + ) + + def test_type_only_query(self): + operation = GetServerWideConnectionStringsOperation(None, ConnectionStringType.QUEUE) + request = operation.get_command(None).create_request(self.node) + self.assertEqual( + "http://localhost:8080/admin/configuration/server-wide/connection-strings?type=Queue", request.url + ) + + def test_type_value_is_enum_name_not_repr(self): + operation = GetServerWideConnectionStringsOperation("n", ConnectionStringType.QUEUE) + url = operation.get_command(None).create_request(self.node).url + self.assertNotIn("ConnectionStringType.QUEUE", url) + self.assertIn("type=Queue", url) + + def test_blank_name_raises(self): + with self.assertRaises(ValueError) as ctx: + GetServerWideConnectionStringsOperation(" ") + self.assertIn("Connection string name must not be null or empty", str(ctx.exception)) + + def test_is_read_request(self): + command = GetServerWideConnectionStringsOperation().get_command(None) + self.assertTrue(command.is_read_request()) + + def test_null_response_raises(self): + command = GetServerWideConnectionStringsOperation().get_command(None) + with self.assertRaises(ValueError): + command.set_response(None, False) + + def test_result_parses_results_with_database_name(self): + payload = { + "Results": [ + { + "Name": "q1", + "BrokerType": "Kafka", + "KafkaConnectionSettings": None, + "RabbitMqConnectionSettings": None, + "AzureQueueStorageConnectionSettings": None, + "AmazonSqsConnectionSettings": None, + "AzureServiceBusConnectionSettings": None, + "Type": "Queue", + "ExcludedDatabases": None, + "UsedBy": [{"Kind": "QueueSink", "Id": 1, "Identifier": None, "Name": "t", "DatabaseName": "db"}], + } + ] + } + command = GetServerWideConnectionStringsOperation().get_command(None) + command.set_response(json.dumps(payload), False) + self.assertEqual(1, len(command.result.results)) + self.assertEqual("q1", command.result.results[0].name) + self.assertEqual("db", command.result.results[0].used_by[0].database_name) + + +class TestRemoveServerWideConnectionStringOperation(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + + def test_delete_url_with_name_and_type(self): + inner = QueueConnectionString(name="q1", broker_type=QueueBrokerType.KAFKA) + operation = RemoveServerWideConnectionStringOperation(inner) + command = operation.get_command(None) + request = command.create_request(self.node) + self.assertEqual("DELETE", request.method) + self.assertEqual( + "http://localhost:8080/admin/configuration/server-wide/connection-strings?name=q1&type=Queue", + request.url, + ) + + def test_implements_raft_command(self): + inner = QueueConnectionString(name="q1", broker_type=QueueBrokerType.KAFKA) + command = RemoveServerWideConnectionStringOperation(inner).get_command(None) + self.assertIsInstance(command, RaftCommand) + + def test_constructor_validation(self): + with self.assertRaises(ValueError): + RemoveServerWideConnectionStringOperation(None) + with self.assertRaises(ValueError): + RemoveServerWideConnectionStringOperation(QueueConnectionString(name=" ")) + + def test_result_parses_raft_command_index(self): + result = RemoveServerWideConnectionStringResult.from_json({"RaftCommandIndex": 7}) + self.assertEqual(7, result.raft_command_index) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/serverwide_tests/test_server_wide_connection_strings_live.py b/ravendb/tests/serverwide_tests/test_server_wide_connection_strings_live.py new file mode 100644 index 00000000..2ef98312 --- /dev/null +++ b/ravendb/tests/serverwide_tests/test_server_wide_connection_strings_live.py @@ -0,0 +1,83 @@ +"""Live-server integration tests for server-wide connection strings. + +These assert the wire facts a 7.2.5 community server enforces: the 400 for a +Type-less PUT body, the 402 license rejection for PUT and DELETE, and the GET +shapes. The module skips when no server is reachable at localhost:8081. +""" + +import os +import unittest + +import requests + +from ravendb.documents.operations.etl.configuration import RavenConnectionString +from ravendb.documents.store.definition import DocumentStore +from ravendb.exceptions.raven_exceptions import RavenException +from ravendb.serverwide.operations.connection_strings import ( + GetServerWideConnectionStringsOperation, + PutServerWideConnectionStringOperation, + RemoveServerWideConnectionStringOperation, + ServerWideConnectionString, +) + +_LIVE_SERVER_URL = os.environ.get("RAVENDB_LIVE_SERVER_URL", "http://localhost:8081") + + +def _server_reachable() -> bool: + try: + response = requests.get(f"{_LIVE_SERVER_URL}/build/version", timeout=3) + return response.status_code == 200 and '"7.2.5"' in response.text + except Exception: + return False + + +@unittest.skipUnless(_server_reachable(), f"no live 7.2.5 server at {_LIVE_SERVER_URL}") +class TestServerWideConnectionStringsLive(unittest.TestCase): + def setUp(self): + self.store = DocumentStore(urls=[_LIVE_SERVER_URL]) + self.store.initialize() + + def tearDown(self): + self.store.close() + + def test_put_without_type_answers_bad_request(self): + # The server's deserializer returns null when Type is missing, which it + # rejects as 400. The wrapper always writes Type, so the probe uses the + # raw endpoint. + response = requests.put( + f"{_LIVE_SERVER_URL}/admin/configuration/server-wide/connection-strings", + json={"Name": "x"}, + ) + self.assertEqual(400, response.status_code) + self.assertIn("Connection string is missing or invalid", response.text) + + def test_valid_put_surfaces_license_rejection_as_base_raven_exception(self): + wrapper = ServerWideConnectionString( + connection_string=RavenConnectionString( + name="live-cs", database="db1", topology_discovery_urls=[_LIVE_SERVER_URL] + ) + ) + with self.assertRaises(RavenException) as ctx: + self.store.maintenance.server.send(PutServerWideConnectionStringOperation(wrapper)) + self.assertIn("Your license doesn't support adding server wide connection strings.", str(ctx.exception)) + + def test_delete_surfaces_license_rejection_as_base_raven_exception(self): + connection_string = RavenConnectionString( + name="live-cs", database="db1", topology_discovery_urls=[_LIVE_SERVER_URL] + ) + with self.assertRaises(RavenException) as ctx: + self.store.maintenance.server.send(RemoveServerWideConnectionStringOperation(connection_string)) + self.assertIn("Your license doesn't support adding server wide connection strings.", str(ctx.exception)) + + def test_get_returns_results(self): + result = self.store.maintenance.server.send(GetServerWideConnectionStringsOperation()) + self.assertIsNotNone(result) + self.assertIsInstance(result.results, list) + + def test_get_type_query_value_is_enum_name(self): + result = self.store.maintenance.server.send(GetServerWideConnectionStringsOperation(None, None)) + self.assertIsNotNone(result) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/session_tests/test_cluster_transaction_change_vector.py b/ravendb/tests/session_tests/test_cluster_transaction_change_vector.py new file mode 100644 index 00000000..a2cbc543 --- /dev/null +++ b/ravendb/tests/session_tests/test_cluster_transaction_change_vector.py @@ -0,0 +1,137 @@ +"""Tests for the session cluster-transaction change-vector fix: +the Database-Cluster-Tx-Id header capture, the last-part etag rule, the +>2-parts throw, and the null change-vector no-op. +""" + +import unittest + +from ravendb.documents.session.document_info import DocumentInfo +from ravendb.documents.session.document_session_operations.in_memory_document_session_operations import ( + InMemoryDocumentSessionOperations, +) +from ravendb.documents.session.misc import SessionInfo +from ravendb.exceptions.exceptions import InvalidOperationException +from ravendb.primitives import constants +from ravendb.util.client_change_vector_utils import ClientChangeVectorUtils + + +class TestClientChangeVectorUtils(unittest.TestCase): + def test_etag_between_last_colon_and_dash(self): + # Format of a cluster-transaction segment: 'Trxn:{index}-{clusterId}'. + self.assertEqual(15, ClientChangeVectorUtils.get_etag_by_id("Trxn:15-A:42", "A:42")) + + def test_no_match_returns_zero(self): + self.assertEqual(0, ClientChangeVectorUtils.get_etag_by_id("A:1-node:2", "A:42")) + self.assertEqual(0, ClientChangeVectorUtils.get_etag_by_id("Trxn:0-A:42", "A:42")) + + def test_null_change_vector_returns_zero(self): + self.assertEqual(0, ClientChangeVectorUtils.get_etag_by_id(None, "A:42")) + + def test_separator_guard(self): + with self.assertRaises(ValueError): + ClientChangeVectorUtils.get_etag_by_id("a|b", "x") + + +def _new_actions(session_info): + actions = InMemoryDocumentSessionOperations.SaveChangesData.ActionsToRunOnSuccess.__new__( + InMemoryDocumentSessionOperations.SaveChangesData.ActionsToRunOnSuccess + ) + session = object.__new__(InMemoryDocumentSessionOperations) + session.session_info = session_info + actions._ActionsToRunOnSuccess__session = session + actions._ActionsToRunOnSuccess__document_infos_to_update = [] + return actions + + +class TestUpdateEntityDocumentInfo(unittest.TestCase): + def test_last_part_etag_advances_last_cluster_transaction_index(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = "A:42" + session_info.last_cluster_transaction_index = None + + actions = _new_actions(session_info) + info = DocumentInfo(key="users/1", change_vector="A:1-node:2|Trxn:15-A:42") + actions.update_entity_document_info(info, {"Name": "u"}) + + self.assertEqual(15, session_info.last_cluster_transaction_index) + self.assertEqual(1, len(actions._ActionsToRunOnSuccess__document_infos_to_update)) + + def test_etag_read_from_last_part_only(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = "A:42" + session_info.last_cluster_transaction_index = None + + actions = _new_actions(session_info) + # The first part also contains 'A:42'; only the last part may contribute. + info = DocumentInfo(key="users/1", change_vector="A:5-A:42|Trxn:7-A:42") + actions.update_entity_document_info(info, {}) + + self.assertEqual(7, session_info.last_cluster_transaction_index) + + def test_max_with_existing_index(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = "A:42" + session_info.last_cluster_transaction_index = 20 + + actions = _new_actions(session_info) + actions.update_entity_document_info(DocumentInfo(key="users/1", change_vector="A:1-node:2|Trxn:15-A:42"), {}) + self.assertEqual(20, session_info.last_cluster_transaction_index) + + actions.update_entity_document_info(DocumentInfo(key="users/2", change_vector="A:1-node:2|Trxn:25-A:42"), {}) + self.assertEqual(25, session_info.last_cluster_transaction_index) + + def test_more_than_two_parts_throws(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = "A:42" + session_info.last_cluster_transaction_index = None + + actions = _new_actions(session_info) + with self.assertRaises(InvalidOperationException) as ctx: + actions.update_entity_document_info(DocumentInfo(key="users/1", change_vector="A:1|B:2|Trxn:3-A:42"), {}) + self.assertEqual("The document 'users/1' has invalid change vector 'A:1|B:2|Trxn:3-A:42'", str(ctx.exception)) + + def test_null_change_vector_is_no_op(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = "A:42" + session_info.last_cluster_transaction_index = None + + actions = _new_actions(session_info) + actions.update_entity_document_info(DocumentInfo(key="users/1", change_vector=None), {}) + self.assertIsNone(session_info.last_cluster_transaction_index) + + def test_no_cluster_id_is_no_op(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = None + session_info.last_cluster_transaction_index = None + + actions = _new_actions(session_info) + actions.update_entity_document_info(DocumentInfo(key="users/1", change_vector="A:1-node:2"), {}) + self.assertIsNone(session_info.last_cluster_transaction_index) + + def test_split_separator_is_pipe(self): + self.assertEqual("|", ClientChangeVectorUtils.SEPARATOR) + + def test_session_info_has_cluster_transaction_id_field(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = None + self.assertIsNone(session_info.cluster_transaction_id) + + +class TestClusterTransactionHeaderCapture(unittest.TestCase): + def test_constant_is_database_cluster_tx_id(self): + self.assertEqual("Database-Cluster-Tx-Id", constants.Headers.DATABASE_CLUSTER_TRANSACTION_ID) + + def test_request_executor_captures_header_in_success_path(self): + from ravendb.http.request_executor import RequestExecutor + + source = open("ravendb/http/request_executor.py").read() + # The capture must sit in the success path, before process_response. + success_index = source.index("command.process_response(self._cache, response, url)") + header_index = source.index("DATABASE_CLUSTER_TRANSACTION_ID in response.headers") + self.assertLess(header_index, success_index) + # It must guard the absent header. + self.assertIn("DATABASE_CLUSTER_TRANSACTION_ID in response.headers", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/session_tests/test_cluster_transaction_change_vector_integration.py b/ravendb/tests/session_tests/test_cluster_transaction_change_vector_integration.py new file mode 100644 index 00000000..a050e11f --- /dev/null +++ b/ravendb/tests/session_tests/test_cluster_transaction_change_vector_integration.py @@ -0,0 +1,38 @@ +"""Integration test for the cluster-transaction change-vector fix. + +After saving a document in a CLUSTER_WIDE transaction, the session's +last_cluster_transaction_index must be advanced to the cluster-transaction etag +of the stored document (the etag read from the last '|'-separated part of the +change vector). Skipped when RAVENDB_LICENSE is not set, following the +AI-agent/CDC integration pattern. +""" + +import os +import unittest + +from ravendb.documents.session.misc import SessionOptions, TransactionMode +from ravendb.infrastructure.entities import User +from ravendb.tests.test_base import TestBase + + +@unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") +class TestClusterTransactionChangeVector(TestBase): + def test_last_cluster_transaction_index_advanced_after_cluster_transaction(self): + user = User(name="Karmel") + + session_options = SessionOptions(transaction_mode=TransactionMode.CLUSTER_WIDE) + session_options.disable_atomic_document_writes_in_cluster_wide_transaction = True + + with self.store.open_session(session_options=session_options) as session: + session.store(user, "users/1") + session.save_changes() + + # The Database-Cluster-Tx-Id header captured on the save response gives the + # cluster id; the stored change vector carries the cluster-transaction segment + # as its last '|'-separated part, so the session index must advance. + self.assertGreater(session.session_info.last_cluster_transaction_index or 0, 0) + self.assertIsNotNone(session.session_info.cluster_transaction_id) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/util/client_change_vector_utils.py b/ravendb/util/client_change_vector_utils.py new file mode 100644 index 00000000..4677b410 --- /dev/null +++ b/ravendb/util/client_change_vector_utils.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from typing import Optional + + +class ClientChangeVectorUtils: + SEPARATOR = "|" + + @staticmethod + def get_etag_by_id(change_vector: Optional[str], id: str) -> int: + """Returns the etag for the given cluster-transaction id in a change-vector + part, or 0 when the id is not present (C# GetEtagById).""" + if change_vector is None: + return 0 + + if id is None: + raise ValueError("id cannot be None") + + if ClientChangeVectorUtils.SEPARATOR in change_vector: + raise ValueError( + f"Change vector contains '{ClientChangeVectorUtils.SEPARATOR}', " + "which is not supported for this operation." + ) + + index = change_vector.find("-" + id) + if index == -1: + return 0 + + end = index - 1 + start = change_vector.rfind(":", 0, end + 1) + 1 + + return int(change_vector[start : end + 1])