From 1f908de1b9f9778c19b3cf6b1dc29d01ee6ec1e7 Mon Sep 17 00:00:00 2001 From: reforge Date: Wed, 19 Aug 2026 01:59:05 -0300 Subject: [PATCH 1/7] Add the CDC Sink task surface to the Python client Add the CdcSinkConfiguration family and the add/update operations, and wire the task into ongoing-task info, the DatabaseRecord, and the public exports. Serialization matches the reference client: CdcColumnMapping omits Type when it is Default, CdcSinkTaskState keeps a case-insensitive Tables dict whose keys keep their stored casing on the wire, and CdcSinkTableLoadState writes null lists instead of empty arrays. The add/update commands are MaintenanceOperation/RavenCommand pairs with the RaftCommand marker and a null-response throw, so server rejections surface as RavenException through ExceptionDispatcher; no client-side validation runs before the request is sent. Reforge-Run: 20260819T014403Z-2668162-reforge --- ravendb/__init__.py | 18 + .../documents/operations/cdc_sink/__init__.py | 43 ++ .../cdc_sink/add_cdc_sink_operation.py | 66 +++ .../cdc_sink/cdc_sink_configuration.py | 294 ++++++++++++ .../cdc_sink/cdc_sink_task_state.py | 137 ++++++ .../cdc_sink/update_cdc_sink_operation.py | 68 +++ ravendb/documents/operations/ongoing_tasks.py | 113 ++++- ravendb/serverwide/database_record.py | 9 + ravendb/tests/cdc_sink_tests/__init__.py | 0 .../test_cdc_sink_configuration.py | 440 ++++++++++++++++++ .../test_cdc_sink_integration.py | 139 ++++++ .../test_cdc_sink_operations.py | 381 +++++++++++++++ 12 files changed, 1704 insertions(+), 4 deletions(-) create mode 100644 ravendb/documents/operations/cdc_sink/__init__.py create mode 100644 ravendb/documents/operations/cdc_sink/add_cdc_sink_operation.py create mode 100644 ravendb/documents/operations/cdc_sink/cdc_sink_configuration.py create mode 100644 ravendb/documents/operations/cdc_sink/cdc_sink_task_state.py create mode 100644 ravendb/documents/operations/cdc_sink/update_cdc_sink_operation.py create mode 100644 ravendb/tests/cdc_sink_tests/__init__.py create mode 100644 ravendb/tests/cdc_sink_tests/test_cdc_sink_configuration.py create mode 100644 ravendb/tests/cdc_sink_tests/test_cdc_sink_integration.py create mode 100644 ravendb/tests/cdc_sink_tests/test_cdc_sink_operations.py diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 172d57b8..129d923e 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -188,6 +188,24 @@ from ravendb.documents.operations.ongoing_tasks import ( OngoingTaskPullReplicationAsSink, OngoingTaskPullReplicationAsHub, + OngoingTaskCdcSink, +) +from ravendb.documents.operations.cdc_sink import ( + CdcSinkConfiguration, + CdcSinkTableConfig, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcColumnMapping, + CdcColumnType, + CdcSinkRelationType, + CdcSinkTaskState, + CdcSinkTableLoadState, + AddCdcSinkOperation, + AddCdcSinkOperationResult, + UpdateCdcSinkOperation, + UpdateCdcSinkOperationResult, ) from ravendb.documents.operations.revisions import ( RevisionsCollectionConfiguration, diff --git a/ravendb/documents/operations/cdc_sink/__init__.py b/ravendb/documents/operations/cdc_sink/__init__.py new file mode 100644 index 00000000..b65269f5 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/__init__.py @@ -0,0 +1,43 @@ +from ravendb.documents.operations.cdc_sink.add_cdc_sink_operation import ( + AddCdcSinkOperation, + AddCdcSinkOperationResult, +) +from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import ( + CdcColumnMapping, + CdcColumnType, + CdcSinkConfiguration, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcSinkRelationType, + CdcSinkTableConfig, +) +from ravendb.documents.operations.cdc_sink.cdc_sink_task_state import ( + CdcSinkTableLoadState, + CdcSinkTablesDict, + CdcSinkTaskState, +) +from ravendb.documents.operations.cdc_sink.update_cdc_sink_operation import ( + UpdateCdcSinkOperation, + UpdateCdcSinkOperationResult, +) + +__all__ = [ + "AddCdcSinkOperation", + "AddCdcSinkOperationResult", + "UpdateCdcSinkOperation", + "UpdateCdcSinkOperationResult", + "CdcSinkConfiguration", + "CdcSinkTableConfig", + "CdcSinkEmbeddedTableConfig", + "CdcSinkLinkedTableConfig", + "CdcSinkOnDeleteConfig", + "CdcSinkPostgresSettings", + "CdcColumnMapping", + "CdcColumnType", + "CdcSinkRelationType", + "CdcSinkTaskState", + "CdcSinkTableLoadState", + "CdcSinkTablesDict", +] diff --git a/ravendb/documents/operations/cdc_sink/add_cdc_sink_operation.py b/ravendb/documents/operations/cdc_sink/add_cdc_sink_operation.py new file mode 100644 index 00000000..cabe5189 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/add_cdc_sink_operation.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, TYPE_CHECKING + +import requests + +from ravendb.documents.conventions import DocumentConventions +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration + + +class AddCdcSinkOperationResult: + def __init__(self, raft_command_index: int = 0, task_id: int = 0): + self.raft_command_index = raft_command_index + self.task_id = task_id + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "AddCdcSinkOperationResult": + return cls( + raft_command_index=json_dict.get("RaftCommandIndex", 0), + task_id=json_dict.get("TaskId", 0), + ) + + +class AddCdcSinkOperation(MaintenanceOperation[AddCdcSinkOperationResult]): + def __init__(self, configuration: "CdcSinkConfiguration"): + if configuration is None: + raise ValueError("configuration cannot be None") + self._configuration = configuration + + def get_command(self, conventions: DocumentConventions) -> RavenCommand[AddCdcSinkOperationResult]: + return AddCdcSinkCommand(conventions, self._configuration) + + +class AddCdcSinkCommand(RavenCommand[AddCdcSinkOperationResult], RaftCommand): + def __init__(self, conventions: DocumentConventions, configuration: "CdcSinkConfiguration"): + super().__init__(AddCdcSinkOperationResult) + self._conventions = conventions + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink" + + request = requests.Request("PUT", url) + request.headers = {"Content-Type": "application/json"} + request.data = json.dumps(self._configuration.to_json()) + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = AddCdcSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() diff --git a/ravendb/documents/operations/cdc_sink/cdc_sink_configuration.py b/ravendb/documents/operations/cdc_sink/cdc_sink_configuration.py new file mode 100644 index 00000000..eee6e9c9 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/cdc_sink_configuration.py @@ -0,0 +1,294 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, List, Optional + + +class CdcColumnType(Enum): + DEFAULT = "Default" + JSON = "Json" + ATTACHMENT = "Attachment" + + +class CdcSinkRelationType(Enum): + ARRAY = "Array" + MAP = "Map" + VALUE = "Value" + + +class CdcColumnMapping: + def __init__( + self, + column: Optional[str] = None, + name: Optional[str] = None, + type: CdcColumnType = CdcColumnType.DEFAULT, + ): + self.column = column + self.name = name + self.type = type + + def to_json(self) -> Dict[str, Any]: + json_dict = {"Column": self.column, "Name": self.name} + if self.type != CdcColumnType.DEFAULT: + json_dict["Type"] = self.type.value + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcColumnMapping": + type_raw = json_dict.get("Type") + return cls( + column=json_dict.get("Column"), + name=json_dict.get("Name"), + type=CdcColumnType(type_raw) if type_raw is not None else CdcColumnType.DEFAULT, + ) + + +class CdcSinkOnDeleteConfig: + def __init__(self, patch: Optional[str] = None, ignore_deletes: bool = False): + self.patch = patch + self.ignore_deletes = ignore_deletes + + def to_json(self) -> Dict[str, Any]: + return {"Patch": self.patch, "IgnoreDeletes": self.ignore_deletes} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkOnDeleteConfig": + return cls( + patch=json_dict.get("Patch"), + ignore_deletes=json_dict.get("IgnoreDeletes", False), + ) + + +class CdcSinkPostgresSettings: + def __init__(self, publication_name: Optional[str] = None, slot_name: Optional[str] = None): + self.publication_name = publication_name + self.slot_name = slot_name + + def to_json(self) -> Dict[str, Any]: + return {"PublicationName": self.publication_name, "SlotName": self.slot_name} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkPostgresSettings": + return cls( + publication_name=json_dict.get("PublicationName"), + slot_name=json_dict.get("SlotName"), + ) + + +class CdcSinkLinkedTableConfig: + def __init__( + self, + source_table_schema: Optional[str] = None, + source_table_name: Optional[str] = None, + property_name: Optional[str] = None, + join_columns: Optional[List[str]] = None, + linked_collection_name: Optional[str] = None, + ): + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.property_name = property_name + self.join_columns = join_columns if join_columns is not None else [] + self.linked_collection_name = linked_collection_name + + def to_json(self) -> Dict[str, Any]: + return { + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "PropertyName": self.property_name, + "JoinColumns": self.join_columns, + "LinkedCollectionName": self.linked_collection_name, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkLinkedTableConfig": + return cls( + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + property_name=json_dict.get("PropertyName"), + join_columns=json_dict.get("JoinColumns") or [], + linked_collection_name=json_dict.get("LinkedCollectionName"), + ) + + +class CdcSinkEmbeddedTableConfig: + def __init__( + self, + source_table_schema: Optional[str] = None, + source_table_name: Optional[str] = None, + property_name: Optional[str] = None, + columns: Optional[List[CdcColumnMapping]] = None, + primary_key_columns: Optional[List[str]] = None, + join_columns: Optional[List[str]] = None, + type: CdcSinkRelationType = CdcSinkRelationType.ARRAY, + patch: Optional[str] = None, + on_delete: Optional[CdcSinkOnDeleteConfig] = None, + case_sensitive_keys: bool = False, + embedded_tables: Optional[List["CdcSinkEmbeddedTableConfig"]] = None, + linked_tables: Optional[List[CdcSinkLinkedTableConfig]] = None, + ): + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.property_name = property_name + self.columns = columns if columns is not None else [] + self.primary_key_columns = primary_key_columns if primary_key_columns is not None else [] + self.join_columns = join_columns if join_columns is not None else [] + self.type = type + self.patch = patch + self.on_delete = on_delete + self.case_sensitive_keys = case_sensitive_keys + self.embedded_tables = embedded_tables if embedded_tables is not None else [] + self.linked_tables = linked_tables if linked_tables is not None else [] + + def to_json(self) -> Dict[str, Any]: + return { + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "PropertyName": self.property_name, + "Columns": [column.to_json() for column in self.columns], + "PrimaryKeyColumns": self.primary_key_columns, + "JoinColumns": self.join_columns, + "Type": self.type.value, + "Patch": self.patch, + "OnDelete": self.on_delete.to_json() if self.on_delete is not None else None, + "CaseSensitiveKeys": self.case_sensitive_keys, + "EmbeddedTables": [embedded.to_json() for embedded in self.embedded_tables], + "LinkedTables": [linked.to_json() for linked in self.linked_tables], + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkEmbeddedTableConfig": + type_raw = json_dict.get("Type") + on_delete_raw = json_dict.get("OnDelete") + return cls( + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + property_name=json_dict.get("PropertyName"), + columns=[CdcColumnMapping.from_json(column) for column in (json_dict.get("Columns") or [])], + primary_key_columns=json_dict.get("PrimaryKeyColumns") or [], + join_columns=json_dict.get("JoinColumns") or [], + type=CdcSinkRelationType(type_raw) if type_raw is not None else CdcSinkRelationType.ARRAY, + patch=json_dict.get("Patch"), + on_delete=CdcSinkOnDeleteConfig.from_json(on_delete_raw) if on_delete_raw is not None else None, + case_sensitive_keys=json_dict.get("CaseSensitiveKeys", False), + embedded_tables=[ + CdcSinkEmbeddedTableConfig.from_json(embedded) for embedded in (json_dict.get("EmbeddedTables") or []) + ], + linked_tables=[ + CdcSinkLinkedTableConfig.from_json(linked) for linked in (json_dict.get("LinkedTables") or []) + ], + ) + + +class CdcSinkTableConfig: + def __init__( + self, + collection_name: Optional[str] = None, + source_table_schema: Optional[str] = None, + source_table_name: Optional[str] = None, + columns: Optional[List[CdcColumnMapping]] = None, + primary_key_columns: Optional[List[str]] = None, + patch: Optional[str] = None, + on_delete: Optional[CdcSinkOnDeleteConfig] = None, + disabled: bool = False, + embedded_tables: Optional[List[CdcSinkEmbeddedTableConfig]] = None, + linked_tables: Optional[List[CdcSinkLinkedTableConfig]] = None, + ): + self.collection_name = collection_name + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.columns = columns if columns is not None else [] + self.primary_key_columns = primary_key_columns if primary_key_columns is not None else [] + self.patch = patch + self.on_delete = on_delete + self.disabled = disabled + self.embedded_tables = embedded_tables if embedded_tables is not None else [] + self.linked_tables = linked_tables if linked_tables is not None else [] + + def to_json(self) -> Dict[str, Any]: + return { + "CollectionName": self.collection_name, + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "Columns": [column.to_json() for column in self.columns], + "PrimaryKeyColumns": self.primary_key_columns, + "Patch": self.patch, + "OnDelete": self.on_delete.to_json() if self.on_delete is not None else None, + "Disabled": self.disabled, + "EmbeddedTables": [embedded.to_json() for embedded in self.embedded_tables], + "LinkedTables": [linked.to_json() for linked in self.linked_tables], + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkTableConfig": + on_delete_raw = json_dict.get("OnDelete") + return cls( + collection_name=json_dict.get("CollectionName"), + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + columns=[CdcColumnMapping.from_json(column) for column in (json_dict.get("Columns") or [])], + primary_key_columns=json_dict.get("PrimaryKeyColumns") or [], + patch=json_dict.get("Patch"), + on_delete=CdcSinkOnDeleteConfig.from_json(on_delete_raw) if on_delete_raw is not None else None, + disabled=json_dict.get("Disabled", False), + embedded_tables=[ + CdcSinkEmbeddedTableConfig.from_json(embedded) for embedded in (json_dict.get("EmbeddedTables") or []) + ], + linked_tables=[ + CdcSinkLinkedTableConfig.from_json(linked) for linked in (json_dict.get("LinkedTables") or []) + ], + ) + + +class CdcSinkConfiguration: + def __init__( + self, + task_id: int = 0, + disabled: bool = False, + name: Optional[str] = None, + mentor_node: Optional[str] = None, + pin_to_mentor_node: bool = False, + connection_string_name: Optional[str] = None, + tables: Optional[List[CdcSinkTableConfig]] = None, + postgres: Optional[CdcSinkPostgresSettings] = None, + skip_initial_load: bool = False, + ): + self.task_id = task_id + self.disabled = disabled + self.name = name + self.mentor_node = mentor_node + self.pin_to_mentor_node = pin_to_mentor_node + self.connection_string_name = connection_string_name + self.tables = tables if tables is not None else [] + self.postgres = postgres + self.skip_initial_load = skip_initial_load + + def get_default_task_name(self) -> str: + return f"CDC Sink to {self.connection_string_name}" + + def to_json(self) -> Dict[str, Any]: + return { + "Name": self.name, + "TaskId": self.task_id, + "Disabled": self.disabled, + "ConnectionStringName": self.connection_string_name, + "MentorNode": self.mentor_node, + "PinToMentorNode": self.pin_to_mentor_node, + "Tables": [table.to_json() for table in self.tables], + "Postgres": self.postgres.to_json() if self.postgres is not None else None, + "SkipInitialLoad": self.skip_initial_load, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkConfiguration": + postgres_raw = json_dict.get("Postgres") + return cls( + task_id=json_dict.get("TaskId", 0), + disabled=json_dict.get("Disabled", False), + name=json_dict.get("Name"), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode", False), + connection_string_name=json_dict.get("ConnectionStringName"), + tables=[CdcSinkTableConfig.from_json(table) for table in (json_dict.get("Tables") or [])], + postgres=CdcSinkPostgresSettings.from_json(postgres_raw) if postgres_raw is not None else None, + skip_initial_load=json_dict.get("SkipInitialLoad", False), + ) diff --git a/ravendb/documents/operations/cdc_sink/cdc_sink_task_state.py b/ravendb/documents/operations/cdc_sink/cdc_sink_task_state.py new file mode 100644 index 00000000..481512b7 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/cdc_sink_task_state.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +from typing import Any, Dict, Iterator, List, Optional, Tuple + + +class CdcSinkTablesDict(dict): + """A dict whose string-key lookups ignore case while iteration keeps the stored key casing. + + Mirrors the C# Dictionary(StringComparer.OrdinalIgnoreCase): + lookups compare case-insensitively, but enumeration yields the original keys. + """ + + def __init__(self, *args, **kwargs): + super().__init__() + self._original_keys: Dict[str, str] = {} + self.update(*args, **kwargs) + + @staticmethod + def _lower(key): + return key.lower() if isinstance(key, str) else key + + def __setitem__(self, key, value) -> None: + lowered = self._lower(key) + self._original_keys[lowered] = key + super().__setitem__(lowered, value) + + def __getitem__(self, key): + return super().__getitem__(self._lower(key)) + + def __delitem__(self, key) -> None: + lowered = self._lower(key) + self._original_keys.pop(lowered, None) + super().__delitem__(lowered) + + def __contains__(self, key) -> bool: + return super().__contains__(self._lower(key)) + + def get(self, key, default=None): + return super().get(self._lower(key), default) + + def pop(self, key, *args): + lowered = self._lower(key) + self._original_keys.pop(lowered, None) + return super().pop(lowered, *args) + + def setdefault(self, key, default=None): + if key not in self: + self[key] = default + return self[key] + + def update(self, *args, **kwargs) -> None: + for key, value in dict(*args, **kwargs).items(): + self[key] = value + + def keys(self): + return list(self._original_keys.values()) + + def items(self): + return [(self._original_keys[lowered], value) for lowered, value in super().items()] + + def values(self): + return list(super().values()) + + def __iter__(self) -> Iterator: + for key in self._original_keys.values(): + yield key + + def __len__(self) -> int: + return super().__len__() + + +class CdcSinkTableLoadState: + def __init__( + self, + initial_load_completed: bool = False, + last_key_values: Optional[List[str]] = None, + key_columns: Optional[List[str]] = None, + ): + self.initial_load_completed = initial_load_completed + self.last_key_values = last_key_values + self.key_columns = key_columns + + def to_json(self) -> Dict[str, Any]: + return { + "InitialLoadCompleted": self.initial_load_completed, + "LastKeyValues": self.last_key_values if self.last_key_values is not None else None, + "KeyColumns": self.key_columns if self.key_columns is not None else None, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkTableLoadState": + return cls( + initial_load_completed=json_dict.get("InitialLoadCompleted", False), + last_key_values=json_dict.get("LastKeyValues"), + key_columns=json_dict.get("KeyColumns"), + ) + + +class CdcSinkTaskState: + collection_name = "@cdc-states" + + def __init__( + self, + last_lsn: Optional[str] = None, + tables: Optional[CdcSinkTablesDict] = None, + configuration_name: Optional[str] = None, + ): + self.last_lsn = last_lsn + self.tables = tables if tables is not None else CdcSinkTablesDict() + self.configuration_name = configuration_name + + @classmethod + def get_document_id(cls, configuration_name: str) -> str: + return f"{cls.collection_name}/{configuration_name}" + + def to_json(self) -> Dict[str, Any]: + tables_json: Dict[str, Any] = {} + for key in self.tables.keys(): + tables_json[key] = self.tables[key].to_json() + return { + "ConfigurationName": self.configuration_name, + "LastLsn": self.last_lsn, + "Tables": tables_json, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "CdcSinkTaskState": + tables_raw = json_dict.get("Tables") + tables = CdcSinkTablesDict() + if tables_raw: + for key, value in tables_raw.items(): + tables[key] = CdcSinkTableLoadState.from_json(value) + return cls( + last_lsn=json_dict.get("LastLsn"), + tables=tables, + configuration_name=json_dict.get("ConfigurationName"), + ) diff --git a/ravendb/documents/operations/cdc_sink/update_cdc_sink_operation.py b/ravendb/documents/operations/cdc_sink/update_cdc_sink_operation.py new file mode 100644 index 00000000..7c066327 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/update_cdc_sink_operation.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, TYPE_CHECKING + +import requests + +from ravendb.documents.conventions import DocumentConventions +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration + + +class UpdateCdcSinkOperationResult: + def __init__(self, raft_command_index: int = 0, task_id: int = 0): + self.raft_command_index = raft_command_index + self.task_id = task_id + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "UpdateCdcSinkOperationResult": + return cls( + raft_command_index=json_dict.get("RaftCommandIndex", 0), + task_id=json_dict.get("TaskId", 0), + ) + + +class UpdateCdcSinkOperation(MaintenanceOperation[UpdateCdcSinkOperationResult]): + def __init__(self, task_id: int, configuration: "CdcSinkConfiguration"): + if configuration is None: + raise ValueError("configuration cannot be None") + self._task_id = task_id + self._configuration = configuration + + def get_command(self, conventions: DocumentConventions) -> RavenCommand[UpdateCdcSinkOperationResult]: + return UpdateCdcSinkCommand(conventions, self._task_id, self._configuration) + + +class UpdateCdcSinkCommand(RavenCommand[UpdateCdcSinkOperationResult], RaftCommand): + def __init__(self, conventions: DocumentConventions, task_id: int, configuration: "CdcSinkConfiguration"): + super().__init__(UpdateCdcSinkOperationResult) + self._conventions = conventions + self._task_id = task_id + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink?id={self._task_id}" + + request = requests.Request("PUT", url) + request.headers = {"Content-Type": "application/json"} + request.data = json.dumps(self._configuration.to_json()) + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = UpdateCdcSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() diff --git a/ravendb/documents/operations/ongoing_tasks.py b/ravendb/documents/operations/ongoing_tasks.py index 62b869c6..8b8c7bc4 100644 --- a/ravendb/documents/operations/ongoing_tasks.py +++ b/ravendb/documents/operations/ongoing_tasks.py @@ -18,6 +18,7 @@ from ravendb.documents.conventions import DocumentConventions from ravendb.documents.operations.ai.gen_ai_configuration import GenAiConfiguration from ravendb.documents.operations.ai.embeddings_generation_configuration import EmbeddingsGenerationConfiguration + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration class OngoingTaskType(Enum): @@ -33,6 +34,7 @@ class OngoingTaskType(Enum): PULL_REPLICATION_AS_HUB = "PullReplicationAsHub" PULL_REPLICATION_AS_SINK = "PullReplicationAsSink" QUEUE_SINK = "QueueSink" + CDC_SINK = "CdcSink" EMBEDDINGS_GENERATION = "EmbeddingsGeneration" GEN_AI = "GenAi" @@ -280,6 +282,97 @@ def from_json(cls, json_dict: dict) -> "OngoingTaskEmbeddingsGeneration": ) +class OngoingTaskCdcSink(OngoingTask): + """Ongoing task information for CDC Sink tasks.""" + + def __init__( + self, + task_id: Optional[int] = None, + responsible_node: Optional[NodeId] = None, + task_state: Optional[OngoingTaskState] = None, + task_connection_status: Optional[OngoingTaskConnectionStatus] = None, + task_name: Optional[str] = None, + error: Optional[str] = None, + mentor_node: Optional[str] = None, + pin_to_mentor_node: Optional[bool] = None, + configuration: Optional["CdcSinkConfiguration"] = None, + connection_string_name: Optional[str] = None, + factory_name: Optional[str] = None, + last_batch_time=None, + last_checkpoint: Optional[str] = None, + seconds_since_last_batch: Optional[float] = None, + last_activity_time=None, + seconds_since_last_activity: Optional[float] = None, + health_issue: Optional[str] = None, + ): + super().__init__( + task_id=task_id, + task_type=OngoingTaskType.CDC_SINK, + responsible_node=responsible_node, + task_state=task_state, + task_connection_status=task_connection_status, + task_name=task_name, + error=error, + mentor_node=mentor_node, + pin_to_mentor_node=pin_to_mentor_node, + ) + self.configuration = configuration + self.connection_string_name = connection_string_name + self.factory_name = factory_name + self.last_batch_time = last_batch_time + self.last_checkpoint = last_checkpoint + self.seconds_since_last_batch = seconds_since_last_batch + self.last_activity_time = last_activity_time + self.seconds_since_last_activity = seconds_since_last_activity + self.health_issue = health_issue + + def to_json(self) -> dict: + result = super().to_json() + result["ConnectionStringName"] = self.connection_string_name + result["FactoryName"] = self.factory_name + result["Configuration"] = self.configuration.to_json() if self.configuration else None + result["LastBatchTime"] = Utils.datetime_to_string(self.last_batch_time) + result["LastCheckpoint"] = self.last_checkpoint + result["SecondsSinceLastBatch"] = self.seconds_since_last_batch + result["LastActivityTime"] = Utils.datetime_to_string(self.last_activity_time) + result["SecondsSinceLastActivity"] = self.seconds_since_last_activity + result["HealthIssue"] = self.health_issue + return result + + @classmethod + def from_json(cls, json_dict: dict) -> "OngoingTaskCdcSink": + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration + + if json_dict is None: + return None + + task_state_str = json_dict.get("TaskState") + task_connection_status_str = json_dict.get("TaskConnectionStatus") + config_dict = json_dict.get("Configuration") + + return cls( + task_id=json_dict.get("TaskId"), + responsible_node=NodeId.from_json(json_dict.get("ResponsibleNode")), + task_state=OngoingTaskState(task_state_str) if task_state_str else None, + task_connection_status=( + OngoingTaskConnectionStatus(task_connection_status_str) if task_connection_status_str else None + ), + task_name=json_dict.get("TaskName"), + error=json_dict.get("Error"), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode"), + configuration=CdcSinkConfiguration.from_json(config_dict) if config_dict else None, + connection_string_name=json_dict.get("ConnectionStringName"), + factory_name=json_dict.get("FactoryName"), + last_batch_time=Utils.string_to_datetime(json_dict.get("LastBatchTime")), + last_checkpoint=json_dict.get("LastCheckpoint"), + seconds_since_last_batch=json_dict.get("SecondsSinceLastBatch"), + last_activity_time=Utils.string_to_datetime(json_dict.get("LastActivityTime")), + seconds_since_last_activity=json_dict.get("SecondsSinceLastActivity"), + health_issue=json_dict.get("HealthIssue"), + ) + + class OngoingTaskPullReplicationAsHub(OngoingTask): """Ongoing task information for a single pull-replication hub connection.""" @@ -392,6 +485,8 @@ def __init__( access_name: Optional[str] = None, allowed_hub_to_sink_paths: Optional[list] = None, allowed_sink_to_hub_paths: Optional[list] = None, + hub_cursor: Optional[str] = None, + sink_cursor: Optional[str] = None, ): super().__init__( task_id=task_id, @@ -414,6 +509,8 @@ def __init__( self.access_name = access_name self.allowed_hub_to_sink_paths = allowed_hub_to_sink_paths self.allowed_sink_to_hub_paths = allowed_sink_to_hub_paths + self.hub_cursor = hub_cursor + self.sink_cursor = sink_cursor def to_json(self) -> dict: result = super().to_json() @@ -427,6 +524,8 @@ def to_json(self) -> dict: result["AccessName"] = self.access_name result["AllowedHubToSinkPaths"] = self.allowed_hub_to_sink_paths result["AllowedSinkToHubPaths"] = self.allowed_sink_to_hub_paths + result["HubCursor"] = self.hub_cursor + result["SinkCursor"] = self.sink_cursor return result @classmethod @@ -457,6 +556,8 @@ def from_json(cls, json_dict: dict) -> Optional["OngoingTaskPullReplicationAsSin access_name=json_dict.get("AccessName"), allowed_hub_to_sink_paths=json_dict.get("AllowedHubToSinkPaths"), allowed_sink_to_hub_paths=json_dict.get("AllowedSinkToHubPaths"), + hub_cursor=json_dict.get("HubCursor"), + sink_cursor=json_dict.get("SinkCursor"), ) @@ -557,7 +658,7 @@ def get_raft_unique_request_id(self) -> str: class GetOngoingTaskInfoOperation( - MaintenanceOperation[Union[OngoingTask, OngoingTaskGenAi, OngoingTaskEmbeddingsGeneration]] + MaintenanceOperation[Union[OngoingTask, OngoingTaskGenAi, OngoingTaskEmbeddingsGeneration, OngoingTaskCdcSink]] ): """ Operation to retrieve detailed information about a specific ongoing task. @@ -593,14 +694,16 @@ def __init__(self, task_id_or_name: Union[int, str], task_type: OngoingTaskType) def get_command( self, conventions: "DocumentConventions" - ) -> RavenCommand[OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration]: + ) -> RavenCommand[OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration | OngoingTaskCdcSink]: if self._task_name is not None: return GetOngoingTaskInfoOperation._GetOngoingTaskInfoCommand( task_name=self._task_name, task_type=self._task_type ) return GetOngoingTaskInfoOperation._GetOngoingTaskInfoCommand(task_id=self._task_id, task_type=self._task_type) - class _GetOngoingTaskInfoCommand(RavenCommand[OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration]): + class _GetOngoingTaskInfoCommand( + RavenCommand[OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration | OngoingTaskCdcSink] + ): def __init__( self, task_type: OngoingTaskType, @@ -629,12 +732,14 @@ def set_response(self, response: Optional[str], from_cache: bool) -> None: def _deserialize_task( self, json_dict: dict - ) -> OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration: + ) -> OngoingTask | OngoingTaskGenAi | OngoingTaskEmbeddingsGeneration | OngoingTaskCdcSink: """Deserialize the task based on its type.""" if self._task_type == OngoingTaskType.GEN_AI: return OngoingTaskGenAi.from_json(json_dict) elif self._task_type == OngoingTaskType.EMBEDDINGS_GENERATION: return OngoingTaskEmbeddingsGeneration.from_json(json_dict) + elif self._task_type == OngoingTaskType.CDC_SINK: + return OngoingTaskCdcSink.from_json(json_dict) elif self._task_type == OngoingTaskType.PULL_REPLICATION_AS_SINK: return OngoingTaskPullReplicationAsSink.from_json(json_dict) else: diff --git a/ravendb/serverwide/database_record.py b/ravendb/serverwide/database_record.py index 955b1e84..2e5ed29d 100644 --- a/ravendb/serverwide/database_record.py +++ b/ravendb/serverwide/database_record.py @@ -70,6 +70,7 @@ def __init__(self, database_name: Optional[str] = None): self.sql_etls: List[SqlEtlConfiguration] = [] self.olap_etls: List[OlapEtlConfiguration] = [] self.embeddings_generations: List = [] + self.cdc_sinks: List = [] self.client: Optional[ClientConfiguration] = None self.studio: Optional[StudioConfiguration] = None self.truncated_cluster_transaction_commands_count: int = 0 @@ -119,6 +120,7 @@ def to_json(self): "RavenEtls": self.raven_etls, "SqlEtls": self.sql_etls, "OlapEtls": self.olap_etls, + "CdcSinks": [cdc_sink.to_json() for cdc_sink in self.cdc_sinks], "Client": self.client, "Studio": self.studio, "TruncatedClusterTransactionCommand": self.truncated_cluster_transaction_commands_count, @@ -178,6 +180,13 @@ def from_json(cls, json_dict: dict) -> DatabaseRecord: ] else: record.embeddings_generations = [] + cdc_sinks_data = json_dict.get("CdcSinks", []) + if cdc_sinks_data: + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration + + record.cdc_sinks = [CdcSinkConfiguration.from_json(c) for c in cdc_sinks_data] + else: + record.cdc_sinks = [] record.client = json_dict.get("Client", None) record.studio = json_dict.get("Studio", None) record.truncated_cluster_transaction_commands_count = json_dict.get("TruncatedClusterTransactionCommand", None) diff --git a/ravendb/tests/cdc_sink_tests/__init__.py b/ravendb/tests/cdc_sink_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ravendb/tests/cdc_sink_tests/test_cdc_sink_configuration.py b/ravendb/tests/cdc_sink_tests/test_cdc_sink_configuration.py new file mode 100644 index 00000000..06189a47 --- /dev/null +++ b/ravendb/tests/cdc_sink_tests/test_cdc_sink_configuration.py @@ -0,0 +1,440 @@ +"""Unit tests for the CDC Sink configuration classes: CdcSinkConfiguration, +its nested config classes, and the CdcSinkTaskState / CdcSinkTableLoadState +serialization shape. +""" + +import unittest + +from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import ( + CdcColumnMapping, + CdcColumnType, + CdcSinkConfiguration, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcSinkRelationType, + CdcSinkTableConfig, +) +from ravendb.documents.operations.cdc_sink.cdc_sink_task_state import ( + CdcSinkTableLoadState, + CdcSinkTablesDict, + CdcSinkTaskState, +) + + +class TestCdcSinkConfiguration(unittest.TestCase): + def test_to_json_key_order_and_values(self): + config = CdcSinkConfiguration( + task_id=42, + disabled=True, + name="cdc-1", + mentor_node="A", + pin_to_mentor_node=True, + connection_string_name="sql-cs", + tables=[CdcSinkTableConfig(collection_name="Orders", source_table_name="orders")], + postgres=CdcSinkPostgresSettings(publication_name="pub", slot_name="slot"), + skip_initial_load=True, + ) + out = config.to_json() + self.assertEqual( + [ + "Name", + "TaskId", + "Disabled", + "ConnectionStringName", + "MentorNode", + "PinToMentorNode", + "Tables", + "Postgres", + "SkipInitialLoad", + ], + list(out.keys()), + ) + self.assertEqual("cdc-1", out["Name"]) + self.assertEqual(42, out["TaskId"]) + self.assertTrue(out["Disabled"]) + self.assertEqual("sql-cs", out["ConnectionStringName"]) + self.assertEqual("A", out["MentorNode"]) + self.assertTrue(out["PinToMentorNode"]) + self.assertTrue(out["SkipInitialLoad"]) + self.assertEqual({"PublicationName": "pub", "SlotName": "slot"}, out["Postgres"]) + + def test_postgres_is_null_when_unset(self): + out = CdcSinkConfiguration(name="c", connection_string_name="sql").to_json() + self.assertIsNone(out["Postgres"]) + + def test_test_mode_is_never_serialized(self): + out = CdcSinkConfiguration(name="c", connection_string_name="sql").to_json() + self.assertNotIn("TestMode", out) + + def test_from_json_round_trip_with_null_postgres(self): + raw = { + "Name": "c", + "TaskId": 0, + "Disabled": False, + "ConnectionStringName": "sql", + "MentorNode": None, + "PinToMentorNode": False, + "Tables": [], + "Postgres": None, + "SkipInitialLoad": False, + } + config = CdcSinkConfiguration.from_json(raw) + self.assertEqual("c", config.name) + self.assertEqual("sql", config.connection_string_name) + self.assertIsNone(config.postgres) + self.assertEqual([], config.tables) + self.assertEqual(raw, config.to_json()) + + def test_from_json_missing_keys_use_defaults(self): + config = CdcSinkConfiguration.from_json({"ConnectionStringName": "sql"}) + self.assertFalse(config.disabled) + self.assertEqual(0, config.task_id) + self.assertIsNone(config.name) + self.assertFalse(config.pin_to_mentor_node) + self.assertEqual([], config.tables) + self.assertIsNone(config.postgres) + self.assertFalse(config.skip_initial_load) + + def test_get_default_task_name(self): + config = CdcSinkConfiguration(connection_string_name="my-sql") + self.assertEqual("CDC Sink to my-sql", config.get_default_task_name()) + + def test_empty_tables_serialize_as_array_not_omitted(self): + out = CdcSinkConfiguration(name="c", connection_string_name="sql").to_json() + self.assertEqual([], out["Tables"]) + + def test_task_id_round_trips_beyond_2_31(self): + config = CdcSinkConfiguration(task_id=5000000000, name="c", connection_string_name="sql") + self.assertEqual(5000000000, CdcSinkConfiguration.from_json(config.to_json()).task_id) + + +class TestCdcSinkTableConfig(unittest.TestCase): + def test_to_json_keys(self): + table = CdcSinkTableConfig( + collection_name="Orders", + source_table_schema="dbo", + source_table_name="orders", + columns=[CdcColumnMapping(column="id", name="Id")], + primary_key_columns=["id"], + patch="this.Total = $row.total;", + on_delete=CdcSinkOnDeleteConfig(patch="this.Archived = true;", ignore_deletes=True), + disabled=False, + ) + out = table.to_json() + self.assertEqual( + [ + "CollectionName", + "SourceTableSchema", + "SourceTableName", + "Columns", + "PrimaryKeyColumns", + "Patch", + "OnDelete", + "Disabled", + "EmbeddedTables", + "LinkedTables", + ], + list(out.keys()), + ) + self.assertEqual("Orders", out["CollectionName"]) + self.assertEqual(["id"], out["PrimaryKeyColumns"]) + self.assertEqual({"Patch": "this.Archived = true;", "IgnoreDeletes": True}, out["OnDelete"]) + self.assertEqual([], out["EmbeddedTables"]) + self.assertEqual([], out["LinkedTables"]) + + def test_empty_lists_are_arrays_not_omitted(self): + out = CdcSinkTableConfig(collection_name="Orders").to_json() + self.assertEqual([], out["Columns"]) + self.assertEqual([], out["PrimaryKeyColumns"]) + self.assertIsNone(out["OnDelete"]) + + def test_from_json_round_trip(self): + table = CdcSinkTableConfig( + collection_name="Orders", + source_table_schema="dbo", + source_table_name="orders", + columns=[CdcColumnMapping(column="id", name="Id")], + primary_key_columns=["id"], + patch="this.Total = $row.total;", + on_delete=CdcSinkOnDeleteConfig(ignore_deletes=True), + disabled=True, + ) + parsed = CdcSinkTableConfig.from_json(table.to_json()) + self.assertEqual("Orders", parsed.collection_name) + self.assertEqual("dbo", parsed.source_table_schema) + self.assertTrue(parsed.disabled) + self.assertTrue(parsed.on_delete.ignore_deletes) + self.assertEqual("this.Total = $row.total;", parsed.patch) + + +class TestCdcColumnMapping(unittest.TestCase): + def test_type_omitted_when_default(self): + out = CdcColumnMapping(column="id", name="Id").to_json() + self.assertEqual(["Column", "Name"], list(out.keys())) + + def test_type_written_as_enum_name_when_not_default(self): + self.assertEqual( + "Json", CdcColumnMapping(column="data", name="Data", type=CdcColumnType.JSON).to_json()["Type"] + ) + self.assertEqual( + "Attachment", CdcColumnMapping(column="file", name="File", type=CdcColumnType.ATTACHMENT).to_json()["Type"] + ) + + def test_enum_wire_values(self): + self.assertEqual("Default", CdcColumnType.DEFAULT.value) + self.assertEqual("Json", CdcColumnType.JSON.value) + self.assertEqual("Attachment", CdcColumnType.ATTACHMENT.value) + + def test_from_json_default_type_when_absent(self): + mapping = CdcColumnMapping.from_json({"Column": "id", "Name": "Id"}) + self.assertEqual(CdcColumnType.DEFAULT, mapping.type) + mapping = CdcColumnMapping.from_json({"Column": "data", "Name": "Data", "Type": "Json"}) + self.assertEqual(CdcColumnType.JSON, mapping.type) + + +class TestCdcSinkEmbeddedTableConfig(unittest.TestCase): + def test_type_always_written_as_enum_name(self): + for relation_type in CdcSinkRelationType: + config = CdcSinkEmbeddedTableConfig(source_table_name="lines", property_name="Lines", type=relation_type) + self.assertEqual(relation_type.value, config.to_json()["Type"]) + + def test_to_json_keys(self): + config = CdcSinkEmbeddedTableConfig( + source_table_schema="dbo", + source_table_name="order_lines", + property_name="Lines", + columns=[CdcColumnMapping(column="id", name="Id")], + primary_key_columns=["id"], + join_columns=["order_id"], + type=CdcSinkRelationType.ARRAY, + patch="this.Total = $row.total;", + on_delete=CdcSinkOnDeleteConfig(ignore_deletes=True), + case_sensitive_keys=True, + embedded_tables=[CdcSinkEmbeddedTableConfig(source_table_name="child", property_name="Child")], + linked_tables=[ + CdcSinkLinkedTableConfig( + source_table_name="customers", + property_name="Customer", + join_columns=["customer_id"], + linked_collection_name="Customers", + ) + ], + ) + out = config.to_json() + self.assertEqual( + [ + "SourceTableSchema", + "SourceTableName", + "PropertyName", + "Columns", + "PrimaryKeyColumns", + "JoinColumns", + "Type", + "Patch", + "OnDelete", + "CaseSensitiveKeys", + "EmbeddedTables", + "LinkedTables", + ], + list(out.keys()), + ) + self.assertEqual("Array", out["Type"]) + self.assertTrue(out["CaseSensitiveKeys"]) + self.assertEqual(1, len(out["EmbeddedTables"])) + self.assertEqual(1, len(out["LinkedTables"])) + + def test_round_trip(self): + config = CdcSinkEmbeddedTableConfig( + source_table_name="order_lines", + property_name="Lines", + type=CdcSinkRelationType.MAP, + case_sensitive_keys=True, + embedded_tables=[CdcSinkEmbeddedTableConfig(source_table_name="child", property_name="Child")], + linked_tables=[ + CdcSinkLinkedTableConfig( + source_table_name="customers", + property_name="Customer", + join_columns=["customer_id"], + linked_collection_name="Customers", + ) + ], + ) + parsed = CdcSinkEmbeddedTableConfig.from_json(config.to_json()) + self.assertEqual(CdcSinkRelationType.MAP, parsed.type) + self.assertTrue(parsed.case_sensitive_keys) + self.assertEqual("child", parsed.embedded_tables[0].source_table_name) + self.assertEqual("Customers", parsed.linked_tables[0].linked_collection_name) + + +class TestCdcSinkOnDeleteConfig(unittest.TestCase): + def test_to_json_keys(self): + config = CdcSinkOnDeleteConfig(patch="this.Archived = true;", ignore_deletes=True) + self.assertEqual({"Patch": "this.Archived = true;", "IgnoreDeletes": True}, config.to_json()) + + def test_round_trip(self): + parsed = CdcSinkOnDeleteConfig.from_json({"Patch": "p", "IgnoreDeletes": False}) + self.assertEqual("p", parsed.patch) + self.assertFalse(parsed.ignore_deletes) + + +class TestCdcSinkPostgresSettings(unittest.TestCase): + def test_to_json_keys(self): + self.assertEqual( + {"PublicationName": "pub", "SlotName": "slot"}, + CdcSinkPostgresSettings(publication_name="pub", slot_name="slot").to_json(), + ) + + def test_round_trip(self): + parsed = CdcSinkPostgresSettings.from_json({"PublicationName": "pub", "SlotName": "slot"}) + self.assertEqual("pub", parsed.publication_name) + self.assertEqual("slot", parsed.slot_name) + + +class TestCdcSinkTaskState(unittest.TestCase): + def test_collection_name_constant(self): + self.assertEqual("@cdc-states", CdcSinkTaskState.collection_name) + + def test_get_document_id_preserves_casing(self): + self.assertEqual("@cdc-states/MyCdc", CdcSinkTaskState.get_document_id("MyCdc")) + + def test_to_json_shape(self): + state = CdcSinkTaskState(last_lsn="0/1", configuration_name="cdc-1") + state.tables["orders"] = CdcSinkTableLoadState( + initial_load_completed=True, last_key_values=["1"], key_columns=["id"] + ) + out = state.to_json() + self.assertEqual(["ConfigurationName", "LastLsn", "Tables"], list(out.keys())) + self.assertIsInstance(out["Tables"], dict) + self.assertIsInstance(out["Tables"]["orders"], dict) + + def test_to_json_preserves_table_key_casing(self): + state = CdcSinkTaskState(configuration_name="cdc-1") + state.tables["Orders"] = CdcSinkTableLoadState() + out = state.to_json() + self.assertIn("Orders", out["Tables"]) + + def test_tables_dict_lookup_is_case_insensitive(self): + state = CdcSinkTaskState(configuration_name="cdc-1") + state.tables["orders"] = CdcSinkTableLoadState(initial_load_completed=True) + self.assertIn("ORDERS", state.tables) + self.assertTrue(state.tables["ORDERS"].initial_load_completed) + + def test_tables_dict_reinsert_with_different_casing_updates_single_entry(self): + state = CdcSinkTaskState(configuration_name="cdc-1") + state.tables["orders"] = CdcSinkTableLoadState(initial_load_completed=False) + state.tables["ORDERS"] = CdcSinkTableLoadState(initial_load_completed=True) + self.assertEqual(1, len(state.tables)) + self.assertTrue(state.tables["orders"].initial_load_completed) + out = state.to_json() + self.assertEqual(1, len(out["Tables"])) + self.assertTrue(out["Tables"]["ORDERS"]["InitialLoadCompleted"]) + + def test_tables_dict_removal_through_any_casing_cleans_up(self): + state = CdcSinkTaskState(configuration_name="cdc-1") + state.tables["orders"] = CdcSinkTableLoadState(initial_load_completed=True) + state.tables["products"] = CdcSinkTableLoadState() + del state.tables["ORDERS"] + self.assertEqual(1, len(state.tables)) + self.assertNotIn("orders", state.tables) + out = state.to_json() + self.assertEqual(["products"], list(out["Tables"].keys())) + self.assertTrue(out["Tables"]["products"]["InitialLoadCompleted"] is False) + + def test_from_json_missing_tables_yields_empty_dict(self): + state = CdcSinkTaskState.from_json({"ConfigurationName": "cdc-1", "LastLsn": "0/1"}) + self.assertEqual({}, state.tables) + self.assertEqual("0/1", state.last_lsn) + + def test_from_json_builds_case_insensitive_tables_at_parse_time(self): + raw = { + "ConfigurationName": "cdc-1", + "LastLsn": "0/1", + "Tables": {"orders": {"InitialLoadCompleted": True, "LastKeyValues": ["1"], "KeyColumns": ["id"]}}, + } + state = CdcSinkTaskState.from_json(raw) + self.assertTrue(state.tables["ORDERS"].initial_load_completed) + self.assertEqual(["1"], state.tables["Orders"].last_key_values) + self.assertEqual(["id"], state.tables["orders"].key_columns) + + def test_table_load_state_null_lists_on_wire(self): + out = CdcSinkTableLoadState().to_json() + self.assertEqual(["InitialLoadCompleted", "LastKeyValues", "KeyColumns"], list(out.keys())) + self.assertIsNone(out["LastKeyValues"]) + self.assertIsNone(out["KeyColumns"]) + + def test_table_load_state_from_json_nulls_stay_none(self): + load = CdcSinkTableLoadState.from_json({"InitialLoadCompleted": False}) + self.assertIsNone(load.last_key_values) + self.assertIsNone(load.key_columns) + load = CdcSinkTableLoadState.from_json( + {"InitialLoadCompleted": True, "LastKeyValues": None, "KeyColumns": None} + ) + self.assertIsNone(load.last_key_values) + self.assertTrue(load.initial_load_completed) + + def test_tables_dict_type(self): + self.assertIsInstance(CdcSinkTablesDict(), dict) + + +class TestNestedConfigurationRoundTrip(unittest.TestCase): + def test_fully_populated_configuration_survives_round_trip(self): + config = CdcSinkConfiguration( + task_id=7, + name="cdc-nested", + connection_string_name="sql-cs", + skip_initial_load=True, + tables=[ + CdcSinkTableConfig( + collection_name="Orders", + source_table_schema="dbo", + source_table_name="orders", + columns=[ + CdcColumnMapping(column="id", name="Id"), + CdcColumnMapping(column="data", name="Data", type=CdcColumnType.JSON), + ], + primary_key_columns=["id"], + patch="this.Total = $row.total;", + on_delete=CdcSinkOnDeleteConfig(patch="this.Archived = true;", ignore_deletes=True), + disabled=False, + embedded_tables=[ + CdcSinkEmbeddedTableConfig( + source_table_name="order_lines", + property_name="Lines", + columns=[CdcColumnMapping(column="id", name="Id")], + primary_key_columns=["id"], + join_columns=["order_id"], + type=CdcSinkRelationType.ARRAY, + case_sensitive_keys=True, + embedded_tables=[ + CdcSinkEmbeddedTableConfig(source_table_name="line_items", property_name="Items") + ], + ) + ], + linked_tables=[ + CdcSinkLinkedTableConfig( + source_table_name="customers", + property_name="Customer", + join_columns=["customer_id"], + linked_collection_name="Customers", + ) + ], + ), + CdcSinkTableConfig(collection_name="Products", source_table_name="products"), + ], + postgres=CdcSinkPostgresSettings(publication_name="rvn_cdc_p_1", slot_name="rvn_cdc_s_1"), + ) + parsed = CdcSinkConfiguration.from_json(config.to_json()) + self.assertEqual(config.to_json(), parsed.to_json()) + self.assertEqual("Orders", parsed.tables[0].collection_name) + self.assertEqual(CdcColumnType.JSON, parsed.tables[0].columns[1].type) + self.assertEqual("Array", parsed.tables[0].embedded_tables[0].type.value) + self.assertEqual("Customers", parsed.tables[0].linked_tables[0].linked_collection_name) + self.assertEqual("rvn_cdc_p_1", parsed.postgres.publication_name) + # table order is preserved + self.assertEqual(["Orders", "Products"], [t.collection_name for t in parsed.tables]) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/cdc_sink_tests/test_cdc_sink_integration.py b/ravendb/tests/cdc_sink_tests/test_cdc_sink_integration.py new file mode 100644 index 00000000..e08c5041 --- /dev/null +++ b/ravendb/tests/cdc_sink_tests/test_cdc_sink_integration.py @@ -0,0 +1,139 @@ +"""Integration tests for the CDC Sink task surface against a live RavenDB server. + +The server gates CDC Sink behind a license, so the tests follow the AI-agent +integration pattern: skipped when RAVENDB_LICENSE is not set. The add request +is attempted through the standard maintenance executor; a server rejection +surfaces as a base RavenException whose message embeds the server error text, +and an accepted task is read back through GetOngoingTaskInfoOperation to +verify the OngoingTaskType.CDC_SINK dispatch. +""" + +import os +import unittest + +from ravendb.exceptions.raven_exceptions import RavenException +from ravendb.documents.operations.cdc_sink import ( + AddCdcSinkOperation, + CdcColumnMapping, + CdcSinkConfiguration, + CdcSinkTableConfig, + UpdateCdcSinkOperation, +) +from ravendb.documents.operations.connection_string.put_connection_string_operation import ( + PutConnectionStringOperation, +) +from ravendb.documents.operations.connection_string.remove_connection_string_operation import ( + RemoveConnectionStringOperation, +) +from ravendb.documents.operations.etl.sql import SqlConnectionString +from ravendb.documents.operations.ongoing_tasks import ( + DeleteOngoingTaskOperation, + GetOngoingTaskInfoOperation, + OngoingTaskCdcSink, + OngoingTaskType, +) +from ravendb.tests.test_base import TestBase + + +@unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") +class TestCdcSinkIntegration(TestBase): + CONNECTION_STRING_NAME = "cdc-sink-test-sql-cs" + + def setUp(self): + super().setUp() + sql_connection_string = SqlConnectionString( + name=self.CONNECTION_STRING_NAME, + connection_string="Data Source=localhost;Initial Catalog=test;Integrated Security=true", + factory_name="System.Data.SqlClient", + ) + self.store.maintenance.send(PutConnectionStringOperation(sql_connection_string)) + self._created_task_ids = [] + + def tearDown(self): + for task_id in self._created_task_ids: + try: + self.store.maintenance.send(DeleteOngoingTaskOperation(task_id, OngoingTaskType.CDC_SINK)) + except Exception: + pass + try: + self.store.maintenance.send( + RemoveConnectionStringOperation( + SqlConnectionString(name=self.CONNECTION_STRING_NAME, connection_string="", factory_name="") + ) + ) + except Exception: + pass + super().tearDown() + + def _valid_config(self, name="cdc-sink-integration-task"): + return CdcSinkConfiguration( + name=name, + connection_string_name=self.CONNECTION_STRING_NAME, + tables=[ + CdcSinkTableConfig( + collection_name="Orders", + source_table_schema="dbo", + source_table_name="orders", + columns=[ + CdcColumnMapping(column="id", name="Id"), + CdcColumnMapping(column="customer", name="Customer"), + ], + primary_key_columns=["id"], + ) + ], + ) + + def test_add_read_back_and_delete_cdc_sink_task(self): + config = self._valid_config() + try: + add_result = self.store.maintenance.send(AddCdcSinkOperation(config)) + except RavenException as e: + # Server rejected the task (e.g. community server without a CDC Sink + # license): the rejection must surface as a RavenException whose message + # embeds the server's error text, never the Message field alone. + self.assertIsInstance(e, RavenException) + self.assertIn("Your license doesn't support using the CDC sink feature.", str(e)) + return + + self.assertIsNotNone(add_result.raft_command_index) + self.assertIsNotNone(add_result.task_id) + self._created_task_ids.append(add_result.task_id) + + ongoing_task = self.store.maintenance.send( + GetOngoingTaskInfoOperation(add_result.task_id, OngoingTaskType.CDC_SINK) + ) + self.assertIsInstance(ongoing_task, OngoingTaskCdcSink) + self.assertEqual(OngoingTaskType.CDC_SINK, ongoing_task.task_type) + self.assertEqual(self.CONNECTION_STRING_NAME, ongoing_task.connection_string_name) + self.assertEqual("cdc-sink-integration-task", ongoing_task.configuration.name) + + def test_update_cdc_sink_task(self): + config = self._valid_config() + try: + add_result = self.store.maintenance.send(AddCdcSinkOperation(config)) + except RavenException as e: + self.assertIsInstance(e, RavenException) + self.assertIn("Your license doesn't support using the CDC sink feature.", str(e)) + return + + self._created_task_ids.append(add_result.task_id) + + config.disabled = True + config.task_id = add_result.task_id + update_result = self.store.maintenance.send(UpdateCdcSinkOperation(add_result.task_id, config)) + self.assertIsNotNone(update_result.raft_command_index) + # The update deletes and re-adds the task; the response TaskId is the + # raft index of the update command (the new task id), not the old one. + self.assertNotEqual(add_result.task_id, update_result.task_id) + self._created_task_ids.append(update_result.task_id) + + ongoing_task = self.store.maintenance.send( + GetOngoingTaskInfoOperation(update_result.task_id, OngoingTaskType.CDC_SINK) + ) + self.assertIsInstance(ongoing_task, OngoingTaskCdcSink) + self.assertEqual(update_result.task_id, ongoing_task.task_id) + self.assertTrue(ongoing_task.configuration.disabled) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/cdc_sink_tests/test_cdc_sink_operations.py b/ravendb/tests/cdc_sink_tests/test_cdc_sink_operations.py new file mode 100644 index 00000000..1e98ec83 --- /dev/null +++ b/ravendb/tests/cdc_sink_tests/test_cdc_sink_operations.py @@ -0,0 +1,381 @@ +"""Wire and dispatch tests for the CDC Sink operations: add/update request +shapes and results, server-error surfacing, ongoing-task integration, and +the DatabaseRecord.CdcSinks list. +""" + +import json +import unittest + +from ravendb.documents.operations.cdc_sink.add_cdc_sink_operation import ( + AddCdcSinkCommand, + AddCdcSinkOperation, + AddCdcSinkOperationResult, +) +from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration +from ravendb.documents.operations.cdc_sink.update_cdc_sink_operation import ( + UpdateCdcSinkCommand, + UpdateCdcSinkOperation, + UpdateCdcSinkOperationResult, +) +from ravendb.documents.operations.ongoing_tasks import ( + DeleteOngoingTaskOperation, + GetOngoingTaskInfoOperation, + OngoingTask, + OngoingTaskCdcSink, + OngoingTaskType, +) +from ravendb.exceptions.exception_dispatcher import ExceptionDispatcher +from ravendb.exceptions.raven_exceptions import RavenException +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.database_record import DatabaseRecord + + +class TestAddCdcSinkOperationWire(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + self.config = CdcSinkConfiguration(name="cdc-1", connection_string_name="sql-cs") + self.command = AddCdcSinkCommand(None, self.config) + + def test_put_to_admin_cdc_sink(self): + request = self.command.create_request(self.node) + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/databases/db1/admin/cdc-sink", request.url) + + def test_configuration_json_is_the_body(self): + request = self.command.create_request(self.node) + self.assertEqual("cdc-1", json.loads(request.data)["Name"]) + + def test_not_a_read_request(self): + self.assertFalse(self.command.is_read_request()) + + def test_implements_raft_command_marker(self): + self.assertIsInstance(self.command, RaftCommand) + self.assertTrue(self.command.get_raft_unique_request_id()) + + def test_null_response_raises(self): + with self.assertRaises(ValueError): + self.command.set_response(None, False) + + def test_result_parses_raft_command_index_and_task_id(self): + self.command.set_response(json.dumps({"RaftCommandIndex": 5, "TaskId": 9}), False) + self.assertEqual(5, self.command.result.raft_command_index) + self.assertEqual(9, self.command.result.task_id) + + def test_result_round_trips_beyond_2_31(self): + result = AddCdcSinkOperationResult.from_json({"RaftCommandIndex": 99999999999999, "TaskId": 5000000000}) + self.assertEqual(99999999999999, result.raft_command_index) + self.assertEqual(5000000000, result.task_id) + + def test_result_missing_keys_default_to_zero(self): + result = AddCdcSinkOperationResult.from_json({}) + self.assertEqual(0, result.raft_command_index) + self.assertEqual(0, result.task_id) + + def test_operation_get_command(self): + operation = AddCdcSinkOperation(self.config) + self.assertIsInstance(operation.get_command(None), AddCdcSinkCommand) + + +class TestUpdateCdcSinkOperationWire(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + self.config = CdcSinkConfiguration(name="cdc-1", connection_string_name="sql-cs") + self.command = UpdateCdcSinkCommand(None, 42, self.config) + + def test_put_to_admin_cdc_sink_with_id_query(self): + request = self.command.create_request(self.node) + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/databases/db1/admin/cdc-sink?id=42", request.url) + + def test_not_a_read_request(self): + self.assertFalse(self.command.is_read_request()) + + def test_implements_raft_command_marker(self): + self.assertIsInstance(self.command, RaftCommand) + self.assertTrue(self.command.get_raft_unique_request_id()) + + def test_null_response_raises(self): + with self.assertRaises(ValueError): + self.command.set_response(None, False) + + def test_result_parses(self): + self.command.set_response(json.dumps({"RaftCommandIndex": 5, "TaskId": 42}), False) + self.assertIsInstance(self.command.result, UpdateCdcSinkOperationResult) + self.assertEqual(5, self.command.result.raft_command_index) + self.assertEqual(42, self.command.result.task_id) + + def test_result_missing_keys_default_to_zero(self): + self.command.set_response(json.dumps({}), False) + self.assertEqual(0, self.command.result.raft_command_index) + self.assertEqual(0, self.command.result.task_id) + + def test_operation_get_command(self): + operation = UpdateCdcSinkOperation(42, self.config) + self.assertIsInstance(operation.get_command(None), UpdateCdcSinkCommand) + + +class TestExceptionSurfacing(unittest.TestCase): + def _dispatch(self, error_type, message, error, code=402): + schema = ExceptionDispatcher.ExceptionSchema( + url="http://localhost:8080", object_type=error_type, message=message, error=error + ) + return ExceptionDispatcher.get(schema, code) + + def test_unmapped_type_surfaces_base_raven_exception_with_full_error_text(self): + exception = self._dispatch( + "Raven.Client.Exceptions.Commercial.LicenseLimitException", + "Your license doesn't support using the CDC sink feature.", + "Raven.Client.Exceptions.Commercial.LicenseLimitException: Your license doesn't support using the CDC sink feature.", + ) + self.assertIsInstance(exception, RavenException) + self.assertIn("Your license doesn't support using the CDC sink feature.", str(exception)) + self.assertIn("The server at http://localhost:8080 responded with status code: 402", str(exception)) + + def test_500_lookup_failure_message_embeds_inner_error(self): + error = ( + "System.InvalidOperationException: Invalid CDC Sink configuration.\n" + "Errors:\n" + "- Could not find connection string named 'sql-cs'. Please supply an existing connection string.\n" + "Configuration:\n{json}\n" + ) + exception = self._dispatch( + "System.InvalidOperationException", + "Invalid CDC Sink configuration.", + error, + code=500, + ) + self.assertIsInstance(exception, RavenException) + self.assertIn( + "Could not find connection string named 'sql-cs'. Please supply an existing connection string.", + str(exception), + ) + + def test_500_nonexistent_task_message_embeds_inner_error(self): + error = ( + "Raven.Server.Rachis.RachisApplyException: Failed to update database record.\n" + " ---> System.InvalidOperationException: CDC Sink task with ID 999999 does not exist.\n" + ) + exception = self._dispatch( + "Raven.Server.Rachis.RachisApplyException", + "Failed to update database record.", + error, + code=500, + ) + self.assertIsInstance(exception, RavenException) + self.assertIn("Failed to update database record.", str(exception)) + self.assertIn("CDC Sink task with ID 999999 does not exist.", str(exception)) + + +class TestOngoingTaskType(unittest.TestCase): + def test_cdc_sink_wire_value(self): + self.assertEqual("CdcSink", OngoingTaskType.CDC_SINK.value) + + def test_delete_operation_flows_type_to_wire(self): + operation = DeleteOngoingTaskOperation(42, OngoingTaskType.CDC_SINK) + command = operation.get_command(None) + request = command.create_request(ServerNode("http://localhost:8080", "db1")) + self.assertEqual("http://localhost:8080/databases/db1/admin/tasks?id=42&type=CdcSink", request.url) + + +class TestOngoingTaskCdcSink(unittest.TestCase): + def _task_dict(self, **overrides): + base = { + "TaskId": 7, + "TaskType": "CdcSink", + "ResponsibleNode": {"NodeTag": "A", "NodeUrl": "http://x", "ResponsibleNode": "A"}, + "TaskState": "Enabled", + "TaskConnectionStatus": "Active", + "TaskName": "cdc-1", + "Error": None, + "MentorNode": "A", + "PinToMentorNode": True, + "ConnectionStringName": "sql-cs", + "FactoryName": "System.Data.SqlClient", + "Configuration": { + "Name": "cdc-1", + "TaskId": 7, + "Disabled": False, + "ConnectionStringName": "sql-cs", + "MentorNode": None, + "PinToMentorNode": True, + "Tables": [], + "Postgres": None, + "SkipInitialLoad": False, + }, + "LastBatchTime": "2026-08-18T10:00:00.0000000Z", + "LastCheckpoint": "0/1", + "SecondsSinceLastBatch": 12.5, + "LastActivityTime": "2026-08-18T10:05:00.0000000Z", + "SecondsSinceLastActivity": 3.25, + "HealthIssue": None, + } + base.update(overrides) + return base + + def test_from_json_parses_all_keys(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + self.assertIsInstance(task, OngoingTaskCdcSink) + self.assertEqual(OngoingTaskType.CDC_SINK, task.task_type) + self.assertEqual(7, task.task_id) + self.assertEqual("cdc-1", task.task_name) + self.assertEqual("sql-cs", task.connection_string_name) + self.assertEqual("System.Data.SqlClient", task.factory_name) + self.assertEqual("0/1", task.last_checkpoint) + self.assertEqual(12.5, task.seconds_since_last_batch) + self.assertEqual(3.25, task.seconds_since_last_activity) + self.assertIsNone(task.health_issue) + + def test_datetimes_parse_as_naive_utc(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + self.assertEqual("2026-08-18 10:00:00", str(task.last_batch_time)) + self.assertEqual("2026-08-18 10:05:00", str(task.last_activity_time)) + self.assertIsNone(task.last_batch_time.tzinfo) + + def test_configuration_parses_as_object(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + from ravendb.documents.operations.cdc_sink.cdc_sink_configuration import CdcSinkConfiguration + + self.assertIsInstance(task.configuration, CdcSinkConfiguration) + self.assertEqual("cdc-1", task.configuration.name) + + def test_nullable_fields_stay_none_when_absent(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + task = OngoingTaskCdcSink.from_json( + { + "TaskId": 7, + "TaskType": "CdcSink", + "TaskName": "cdc-1", + } + ) + self.assertIsNone(task.last_batch_time) + self.assertIsNone(task.last_checkpoint) + self.assertIsNone(task.seconds_since_last_batch) + self.assertIsNone(task.configuration) + + def test_to_json_key_order(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + out = task.to_json() + self.assertEqual( + [ + "TaskId", + "TaskType", + "ResponsibleNode", + "TaskState", + "TaskConnectionStatus", + "TaskName", + "Error", + "MentorNode", + "PinToMentorNode", + "ConnectionStringName", + "FactoryName", + "Configuration", + "LastBatchTime", + "LastCheckpoint", + "SecondsSinceLastBatch", + "LastActivityTime", + "SecondsSinceLastActivity", + "HealthIssue", + ], + list(out.keys()), + ) + self.assertEqual("CdcSink", out["TaskType"]) + self.assertEqual("System.Data.SqlClient", out["FactoryName"]) + + def test_configuration_serializes_with_type_omission_rule(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + configuration_out = task.to_json()["Configuration"] + self.assertNotIn("TestMode", configuration_out) + self.assertIsNone(configuration_out["Postgres"]) + + def test_to_json_writes_none_for_unset_configuration(self): + task = OngoingTaskCdcSink.from_json({"TaskId": 1, "TaskType": "CdcSink"}) + self.assertIsNone(task.to_json()["Configuration"]) + + def test_fractional_seconds_round_trip(self): + task = OngoingTaskCdcSink.from_json(self._task_dict()) + out = task.to_json() + self.assertEqual(12.5, out["SecondsSinceLastBatch"]) + self.assertEqual(3.25, out["SecondsSinceLastActivity"]) + + +class TestGetOngoingTaskInfoDispatch(unittest.TestCase): + def test_cdc_sink_dispatches_to_ongoing_task_cdc_sink(self): + operation = GetOngoingTaskInfoOperation(7, OngoingTaskType.CDC_SINK) + command = operation.get_command(None) + json_dict = { + "TaskId": 7, + "TaskType": "CdcSink", + "ConnectionStringName": "sql-cs", + "Configuration": None, + } + task = command._deserialize_task(json_dict) + self.assertIsInstance(task, OngoingTaskCdcSink) + + def test_get_request_url(self): + operation = GetOngoingTaskInfoOperation(7, OngoingTaskType.CDC_SINK) + command = operation.get_command(None) + request = command.create_request(ServerNode("http://localhost:8080", "db1")) + self.assertEqual("http://localhost:8080/databases/db1/task?key=7&type=CdcSink", request.url) + + +class TestDatabaseRecordCdcSinks(unittest.TestCase): + def _base_record(self, **overrides): + record = {"DatabaseName": "db", "LockMode": "Unlock", "AutoIndexes": {}} + record.update(overrides) + return record + + def test_from_json_parses_cdc_sinks(self): + raw = self._base_record( + CdcSinks=[ + { + "Name": "cdc-1", + "TaskId": 9, + "Disabled": False, + "ConnectionStringName": "sql-cs", + "MentorNode": None, + "PinToMentorNode": False, + "Tables": [], + "Postgres": None, + "SkipInitialLoad": False, + } + ] + ) + record = DatabaseRecord.from_json(raw) + self.assertEqual(1, len(record.cdc_sinks)) + self.assertEqual("cdc-1", record.cdc_sinks[0].name) + self.assertEqual("sql-cs", record.cdc_sinks[0].connection_string_name) + + def test_missing_cdc_sinks_key_yields_empty_list(self): + record = DatabaseRecord.from_json(self._base_record()) + self.assertEqual([], record.cdc_sinks) + + def test_to_json_emits_cdc_sinks_with_configuration_shape(self): + record = DatabaseRecord.from_json( + self._base_record( + CdcSinks=[ + { + "Name": "cdc-1", + "TaskId": 9, + "Disabled": False, + "ConnectionStringName": "sql-cs", + "MentorNode": None, + "PinToMentorNode": False, + "Tables": [], + "Postgres": None, + "SkipInitialLoad": False, + } + ] + ) + ) + out = record.to_json() + self.assertIn("CdcSinks", out) + self.assertEqual("cdc-1", out["CdcSinks"][0]["Name"]) + + def test_empty_cdc_sinks_serialize_as_empty_array(self): + record = DatabaseRecord.from_json(self._base_record()) + self.assertEqual([], record.to_json()["CdcSinks"]) + + +if __name__ == "__main__": + unittest.main() From 6921de15867c1f7a4fc9e9c18441b19da96b9d62 Mon Sep 17 00:00:00 2001 From: reforge Date: Wed, 19 Aug 2026 01:59:05 -0300 Subject: [PATCH 2/7] Add the 7.2.5 version header, S3 checksum flag, UsedBy, sink cursors Bump RequestExecutor.CLIENT_VERSION to 7.2.5 so the Raven-Client-Version header carries the new version. Add DisableChecksumValidation to the S3 backup and remote-attachment settings with value equality and hashability that include the flag, matching the reference Equals/GetHashCode overrides. Expose UsedBy on every connection-string class through a typed ConnectionStringUsage object parsed from the GET response; to_json never writes it, so a GET -> from_json -> to_json round-trip does not leak the server-computed metadata. Append HubCursor and SinkCursor to the pull-replication-as-sink task info after AllowedSinkToHubPaths. Reforge-Run: 20260819T014403Z-2668162-reforge --- ravendb/__init__.py | 2 +- .../operations/ai/ai_connection_string.py | 8 +- .../operations/attachments/__init__.py | 37 +++ .../documents/operations/backups/settings.py | 29 ++ .../operations/connection_strings.py | 30 ++- .../documents/operations/etl/configuration.py | 13 +- .../etl/elastic_search/connection.py | 15 +- .../operations/etl/olap/connection.py | 8 +- .../operations/etl/queue/connection.py | 7 +- .../operations/etl/snowflake/connection.py | 14 +- .../documents/operations/etl/sql/__init__.py | 15 +- ravendb/http/request_executor.py | 2 +- .../test_client_surface_additions.py | 255 ++++++++++++++++++ 13 files changed, 408 insertions(+), 27 deletions(-) create mode 100644 ravendb/tests/cdc_sink_tests/test_client_surface_additions.py diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 129d923e..65c9c32d 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -75,7 +75,7 @@ ) from ravendb.documents.operations.configuration.definitions import StudioConfiguration, StudioEnvironment -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage # AI Operations from ravendb.documents.ai import ( diff --git a/ravendb/documents/operations/ai/ai_connection_string.py b/ravendb/documents/operations/ai/ai_connection_string.py index 9e39e520..1ed2a11b 100644 --- a/ravendb/documents/operations/ai/ai_connection_string.py +++ b/ravendb/documents/operations/ai/ai_connection_string.py @@ -1,5 +1,5 @@ import enum -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, List from ravendb.serverwide.server_operation_executor import ConnectionStringType from ravendb.documents.operations.ai.azure_open_ai_settings import AzureOpenAiSettings @@ -11,7 +11,7 @@ from ravendb.documents.operations.ai.open_ai_settings import OpenAiSettings from ravendb.documents.operations.ai.vertex_settings import VertexSettings -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage class AiModelType(enum.Enum): @@ -45,8 +45,9 @@ def __init__( mistral_ai_settings: Optional[MistralAiSettings] = None, vertex_settings: Optional[VertexSettings] = None, model_type: AiModelType = None, + used_by: Optional[List[ConnectionStringUsage]] = None, ): - super().__init__(name) + super().__init__(name, used_by) self.identifier = identifier self.openai_settings = openai_settings self.azure_openai_settings = azure_openai_settings @@ -195,4 +196,5 @@ def from_json(cls, json_dict: Dict[str, Any]) -> "AiConnectionString": VertexSettings.from_json(json_dict["VertexSettings"]) if json_dict.get("VertexSettings") else None ), model_type=AiModelType(json_dict["ModelType"]) if json_dict.get("ModelType") else None, + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/attachments/__init__.py b/ravendb/documents/operations/attachments/__init__.py index 1dfdcd40..19241389 100644 --- a/ravendb/documents/operations/attachments/__init__.py +++ b/ravendb/documents/operations/attachments/__init__.py @@ -404,6 +404,7 @@ def __init__( custom_server_url: str = None, force_path_style: bool = None, storage_class: Optional[S3StorageClass] = None, + disable_checksum_validation: bool = False, ): self.aws_access_key = aws_access_key self.aws_secret_key = aws_secret_key @@ -414,6 +415,7 @@ def __init__( self.custom_server_url = custom_server_url self.force_path_style = force_path_style self.storage_class = storage_class + self.disable_checksum_validation = disable_checksum_validation @classmethod def from_json(cls, json_dict: dict) -> RemoteAttachmentsS3Settings: @@ -428,6 +430,7 @@ def from_json(cls, json_dict: dict) -> RemoteAttachmentsS3Settings: json_dict.get("CustomServerUrl"), json_dict.get("ForcePathStyle"), S3StorageClass(storage_class_raw) if storage_class_raw is not None else None, + json_dict.get("DisableChecksumValidation", False), ) def to_json(self) -> dict: @@ -440,11 +443,45 @@ def to_json(self) -> dict: "BucketName": self.bucket_name, "CustomServerUrl": self.custom_server_url, "ForcePathStyle": self.force_path_style, + "DisableChecksumValidation": self.disable_checksum_validation, } if self.storage_class is not None: result["StorageClass"] = self.storage_class.value return result + def __eq__(self, other) -> bool: + if not isinstance(other, RemoteAttachmentsS3Settings): + return False + + return ( + self.aws_region_name == other.aws_region_name + and self.bucket_name == other.bucket_name + and self.remote_folder_name == other.remote_folder_name + and self.custom_server_url == other.custom_server_url + and self.force_path_style == other.force_path_style + and self.disable_checksum_validation == other.disable_checksum_validation + and self.storage_class == other.storage_class + and self.aws_access_key == other.aws_access_key + and self.aws_secret_key == other.aws_secret_key + and self.aws_session_token == other.aws_session_token + ) + + def __hash__(self) -> int: + return hash( + ( + self.aws_region_name, + self.bucket_name, + self.remote_folder_name, + self.custom_server_url, + self.force_path_style, + self.disable_checksum_validation, + self.storage_class, + self.aws_access_key, + self.aws_secret_key, + self.aws_session_token, + ) + ) + class RemoteAttachmentsAzureSettings: def __init__( diff --git a/ravendb/documents/operations/backups/settings.py b/ravendb/documents/operations/backups/settings.py index 565e3df8..10a99f2d 100644 --- a/ravendb/documents/operations/backups/settings.py +++ b/ravendb/documents/operations/backups/settings.py @@ -130,6 +130,7 @@ def __init__( bucket_name: str = None, custom_server_url: str = None, force_path_style: bool = None, + disable_checksum_validation: bool = False, ): super().__init__( disabled, @@ -143,6 +144,7 @@ def __init__( self.bucket_name = bucket_name self.custom_server_url = custom_server_url self.force_path_style = force_path_style + self.disable_checksum_validation = disable_checksum_validation @classmethod def from_json(cls, json_dict: Dict[str, Any]) -> S3Settings: @@ -157,6 +159,7 @@ def from_json(cls, json_dict: Dict[str, Any]) -> S3Settings: json_dict["BucketName"], json_dict["CustomServerUrl"], json_dict["ForcePathStyle"], + json_dict.get("DisableChecksumValidation", False), ) def to_json(self) -> Dict[str, Any]: @@ -171,8 +174,34 @@ def to_json(self) -> Dict[str, Any]: "BucketName": self.bucket_name, "CustomServerUrl": self.custom_server_url, "ForcePathStyle": self.force_path_style, + "DisableChecksumValidation": self.disable_checksum_validation, } + def __eq__(self, other) -> bool: + if not isinstance(other, S3Settings): + return False + + return ( + self.aws_region_name == other.aws_region_name + and self.bucket_name == other.bucket_name + and self.remote_folder_name == other.remote_folder_name + and self.custom_server_url == other.custom_server_url + and self.force_path_style == other.force_path_style + and self.disable_checksum_validation == other.disable_checksum_validation + ) + + def __hash__(self) -> int: + return hash( + ( + self.aws_region_name, + self.bucket_name, + self.remote_folder_name, + self.custom_server_url, + self.force_path_style, + self.disable_checksum_validation, + ) + ) + class GlacierSettings(AmazonSettings): def __init__( diff --git a/ravendb/documents/operations/connection_strings.py b/ravendb/documents/operations/connection_strings.py index be5c9a44..bf69d730 100644 --- a/ravendb/documents/operations/connection_strings.py +++ b/ravendb/documents/operations/connection_strings.py @@ -1,10 +1,36 @@ from abc import abstractmethod -from typing import Dict, Any +from typing import Any, Dict, List, Optional + + +class ConnectionStringUsage: + """A single usage of a connection string, as reported by the server on GET.""" + + def __init__( + self, + kind: Optional[str] = None, + id: Optional[int] = None, + identifier: Optional[str] = None, + name: Optional[str] = None, + ): + self.kind = kind + self.id = id + self.identifier = identifier + self.name = name + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "ConnectionStringUsage": + return cls( + kind=json_dict.get("Kind"), + id=json_dict.get("Id"), + identifier=json_dict.get("Identifier"), + name=json_dict.get("Name"), + ) class ConnectionString: - def __init__(self, name: str): + def __init__(self, name: str, used_by: Optional[List[ConnectionStringUsage]] = None): self.name = name + self.used_by = used_by if used_by is not None else [] @abstractmethod def get_type(self): diff --git a/ravendb/documents/operations/etl/configuration.py b/ravendb/documents/operations/etl/configuration.py index 205a4757..bee0f13a 100644 --- a/ravendb/documents/operations/etl/configuration.py +++ b/ravendb/documents/operations/etl/configuration.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from typing import Optional, Generic, TypeVar, List, Dict, Any -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage from ravendb.documents.operations.etl.etl_type import EtlType from ravendb.documents.operations.etl.transformation import Transformation import ravendb.serverwide.server_operation_executor @@ -11,8 +11,14 @@ class RavenConnectionString(ConnectionString): - def __init__(self, name: str, database: Optional[str] = None, topology_discovery_urls: Optional[List[str]] = None): - super().__init__(name) + def __init__( + self, + name: str, + database: Optional[str] = None, + topology_discovery_urls: Optional[List[str]] = None, + used_by: Optional[List[ConnectionStringUsage]] = None, + ): + super().__init__(name, used_by) self.database = database self.topology_discovery_urls = topology_discovery_urls @@ -34,6 +40,7 @@ def from_json(cls, json_dict: Dict) -> "RavenConnectionString": name=json_dict["Name"], database=json_dict["Database"], topology_discovery_urls=json_dict["TopologyDiscoveryUrls"], + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/etl/elastic_search/connection.py b/ravendb/documents/operations/etl/elastic_search/connection.py index aa464574..6f4ffafa 100644 --- a/ravendb/documents/operations/etl/elastic_search/connection.py +++ b/ravendb/documents/operations/etl/elastic_search/connection.py @@ -1,6 +1,6 @@ -from typing import List, Dict, Any +from typing import List, Dict, Any, Optional -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage from ravendb.serverwide.server_operation_executor import ConnectionStringType @@ -92,8 +92,14 @@ def from_json(cls, data: Dict[str, Any]) -> "Authentication": class ElasticSearchConnectionString(ConnectionString): - def __init__(self, name: str, nodes: List[str] = None, authentication: Authentication = None): - super().__init__(name) + def __init__( + self, + name: str, + nodes: List[str] = None, + authentication: Authentication = None, + used_by: Optional[List[ConnectionStringUsage]] = None, + ): + super().__init__(name, used_by) self.nodes = nodes self.authentication = authentication @@ -117,4 +123,5 @@ def from_json(cls, json_dict: Dict[str, Any]) -> Any: authentication=( Authentication.from_json(json_dict["Authentication"]) if json_dict["Authentication"] else None ), + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/etl/olap/connection.py b/ravendb/documents/operations/etl/olap/connection.py index f6e7ea76..a5654380 100644 --- a/ravendb/documents/operations/etl/olap/connection.py +++ b/ravendb/documents/operations/etl/olap/connection.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, List from ravendb.documents.operations.backups.settings import ( LocalSettings, @@ -8,7 +8,7 @@ GoogleCloudSettings, FtpSettings, ) -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage import ravendb.serverwide.server_operation_executor from ravendb.documents.operations.etl.configuration import EtlConfiguration @@ -23,8 +23,9 @@ def __init__( glacier_settings: Optional[GlacierSettings] = None, google_cloud_settings: Optional[GoogleCloudSettings] = None, ftp_settings: Optional[FtpSettings] = None, + used_by: Optional[List[ConnectionStringUsage]] = None, ): - super().__init__(name) + super().__init__(name, used_by) self.local_settings = local_settings self.s3_settings = s3_settings self.azure_settings = azure_settings @@ -64,6 +65,7 @@ def from_json(cls, json_dict: dict) -> "OlapConnectionString": else None ), ftp_settings=FtpSettings.from_json(json_dict["FtpSettings"]) if json_dict["FtpSettings"] else None, + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/etl/queue/connection.py b/ravendb/documents/operations/etl/queue/connection.py index 985d65d0..ebaefad3 100644 --- a/ravendb/documents/operations/etl/queue/connection.py +++ b/ravendb/documents/operations/etl/queue/connection.py @@ -1,6 +1,7 @@ from enum import Enum +from typing import List, Optional -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage import ravendb.serverwide.server_operation_executor from ravendb.documents.operations.etl.queue.amazon_sqs_connection_settings import AmazonSqsConnectionSettings from ravendb.documents.operations.etl.queue.azure_queue_storage_connection_settings import ( @@ -27,8 +28,9 @@ def __init__( rabbit_mq_settings: RabbitMqConnectionSettings = None, azure_queue_storage_settings: AzureQueueStorageConnectionSettings = None, amazon_sqs_settings: AmazonSqsConnectionSettings = None, + used_by: Optional[List[ConnectionStringUsage]] = None, ): - super().__init__(name) + super().__init__(name, used_by) self.broker_type = broker_type self.kafka_settings = kafka_settings self.rabbit_mq_settings = rabbit_mq_settings @@ -77,4 +79,5 @@ def from_json(cls, json_dict: dict) -> "QueueConnectionString": if json_dict["AmazonSqsConnectionSettings"] else None ), + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/etl/snowflake/connection.py b/ravendb/documents/operations/etl/snowflake/connection.py index b2d0e87b..e622d73f 100644 --- a/ravendb/documents/operations/etl/snowflake/connection.py +++ b/ravendb/documents/operations/etl/snowflake/connection.py @@ -1,12 +1,17 @@ -from typing import Optional +from typing import Optional, List -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage import ravendb.serverwide.server_operation_executor class SnowflakeConnectionString(ConnectionString): - def __init__(self, name: str, connection_string: Optional[str] = None): - super().__init__(name) + def __init__( + self, + name: str, + connection_string: Optional[str] = None, + used_by: Optional[List[ConnectionStringUsage]] = None, + ): + super().__init__(name, used_by) self.connection_string = connection_string @property @@ -25,4 +30,5 @@ def from_json(cls, json_dict: dict) -> "SnowflakeConnectionString": return cls( name=json_dict["Name"], connection_string=json_dict["ConnectionString"], + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/documents/operations/etl/sql/__init__.py b/ravendb/documents/operations/etl/sql/__init__.py index bcd5ec39..10f0d200 100644 --- a/ravendb/documents/operations/etl/sql/__init__.py +++ b/ravendb/documents/operations/etl/sql/__init__.py @@ -1,13 +1,19 @@ -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, List -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage import ravendb.serverwide.server_operation_executor from ravendb.documents.operations.etl.configuration import EtlConfiguration class SqlConnectionString(ConnectionString): - def __init__(self, name: str, connection_string: Optional[str] = None, factory_name: Optional[str] = None): - super().__init__(name) + def __init__( + self, + name: str, + connection_string: Optional[str] = None, + factory_name: Optional[str] = None, + used_by: Optional[List[ConnectionStringUsage]] = None, + ): + super().__init__(name, used_by) self.connection_string = connection_string self.factory_name = factory_name @@ -29,6 +35,7 @@ def from_json(cls, json_dict: Dict[str, Any]) -> "SqlConnectionString": name=json_dict["Name"], connection_string=json_dict["ConnectionString"], factory_name=json_dict["FactoryName"], + used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/http/request_executor.py b/ravendb/http/request_executor.py index a9402629..360da4dc 100644 --- a/ravendb/http/request_executor.py +++ b/ravendb/http/request_executor.py @@ -56,7 +56,7 @@ class RequestExecutor: __INITIAL_TOPOLOGY_ETAG = -2 __GLOBAL_APPLICATION_IDENTIFIER = uuid.uuid4() - CLIENT_VERSION = "7.2.3" + CLIENT_VERSION = "7.2.5" logger = logging.getLogger("request_executor") # todo: initializer should take also cryptography certificates diff --git a/ravendb/tests/cdc_sink_tests/test_client_surface_additions.py b/ravendb/tests/cdc_sink_tests/test_client_surface_additions.py new file mode 100644 index 00000000..4170aab2 --- /dev/null +++ b/ravendb/tests/cdc_sink_tests/test_client_surface_additions.py @@ -0,0 +1,255 @@ +"""Tests for the client version header, the S3 checksum-validation flag, the +connection-string UsedBy metadata, and the pull-replication sink cursors. +""" + +import json +import unittest + +import requests + +from ravendb.documents.operations.attachments import RemoteAttachmentsS3Settings +from ravendb.documents.operations.backups.settings import ( + GetBackupConfigurationScript, + S3Settings, +) +from ravendb.documents.operations.connection_string.get_connection_string_operation import ( + GetConnectionStringsOperation, + GetConnectionStringsResult, +) +from ravendb.documents.operations.connection_strings import ConnectionStringUsage +from ravendb.documents.operations.ongoing_tasks import ( + OngoingTaskPullReplicationAsSink, + OngoingTaskType, +) +from ravendb.http.request_executor import RequestExecutor +from ravendb.http.server_node import ServerNode +from ravendb.primitives import constants + + +class TestClientVersionHeader(unittest.TestCase): + def test_client_version_is_7_2_5(self): + self.assertEqual("7.2.5", RequestExecutor.CLIENT_VERSION) + + def test_wire_header_carries_client_version(self): + executor = object.__new__(RequestExecutor) + executor._disable_client_configuration_updates = False + executor._client_configuration_etag = 0 + executor._disable_topology_updates = False + executor._topology_etag = 0 + + request = requests.Request("GET", "http://localhost:8080/databases/db1/") + executor._set_request_headers(None, None, request) + self.assertEqual("7.2.5", request.headers[constants.Headers.CLIENT_VERSION]) + + +class TestS3ChecksumValidation(unittest.TestCase): + def _script(self): + return GetBackupConfigurationScript.default() + + def test_to_json_writes_disable_checksum_validation(self): + settings = S3Settings( + bucket_name="b", disable_checksum_validation=True, get_backup_configuration_script=self._script() + ) + out = settings.to_json() + self.assertTrue(out["DisableChecksumValidation"]) + + def test_from_json_parses_flag(self): + script = self._script().to_json() + settings = S3Settings.from_json( + { + "Disabled": False, + "GetBackupConfigurationScript": script, + "AwsAccessKey": None, + "AwsSecretKey": None, + "AwsSessionToken": None, + "AwsRegionName": None, + "RemoteFolderName": None, + "BucketName": "b", + "CustomServerUrl": None, + "ForcePathStyle": False, + "DisableChecksumValidation": True, + } + ) + self.assertTrue(settings.disable_checksum_validation) + + def test_from_json_defaults_flag_to_false_when_absent(self): + script = self._script().to_json() + settings = S3Settings.from_json( + { + "Disabled": False, + "GetBackupConfigurationScript": script, + "AwsAccessKey": None, + "AwsSecretKey": None, + "AwsSessionToken": None, + "AwsRegionName": None, + "RemoteFolderName": None, + "BucketName": "b", + "CustomServerUrl": None, + "ForcePathStyle": False, + } + ) + self.assertFalse(settings.disable_checksum_validation) + + def test_equality_includes_flag(self): + a = S3Settings(bucket_name="b", disable_checksum_validation=False) + b = S3Settings(bucket_name="b", disable_checksum_validation=True) + self.assertNotEqual(a, b) + same = S3Settings(bucket_name="b", disable_checksum_validation=False) + self.assertEqual(a, same) + + def test_stays_hashable_and_hash_consistent_with_equality(self): + a = S3Settings(bucket_name="b", disable_checksum_validation=False) + same = S3Settings(bucket_name="b", disable_checksum_validation=False) + b = S3Settings(bucket_name="b", disable_checksum_validation=True) + self.assertEqual(hash(a), hash(same)) + self.assertNotEqual(hash(a), hash(b)) + self.assertEqual(len({a, same, b}), 2) + + +class TestRemoteAttachmentsS3ChecksumValidation(unittest.TestCase): + def test_to_json_writes_disable_checksum_validation(self): + settings = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=True) + self.assertTrue(settings.to_json()["DisableChecksumValidation"]) + + def test_from_json_parses_flag_and_defaults_false(self): + self.assertTrue( + RemoteAttachmentsS3Settings.from_json( + {"BucketName": "b", "DisableChecksumValidation": True} + ).disable_checksum_validation + ) + self.assertFalse(RemoteAttachmentsS3Settings.from_json({"BucketName": "b"}).disable_checksum_validation) + + def test_equality_includes_flag(self): + a = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=False) + b = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=True) + self.assertNotEqual(a, b) + self.assertEqual(a, RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=False)) + + def test_stays_hashable_and_hash_consistent_with_equality(self): + a = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=False) + same = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=False) + b = RemoteAttachmentsS3Settings(bucket_name="b", disable_checksum_validation=True) + self.assertEqual(hash(a), hash(same)) + self.assertNotEqual(hash(a), hash(b)) + self.assertEqual(len({a, same, b}), 2) + + +class TestConnectionStringUsedBy(unittest.TestCase): + def _result_dict(self, used_by=None): + return { + "RavenConnectionStrings": { + "raven": { + "Name": "raven", + "Database": "db", + "TopologyDiscoveryUrls": [], + "UsedBy": used_by if used_by is not None else [], + } + }, + "SqlConnectionStrings": {}, + "OlapConnectionStrings": {}, + "AiConnectionStrings": {}, + "ElasticSearchConnectionStrings": {}, + "QueueConnectionStrings": {}, + "SnowflakeConnectionStrings": {}, + } + + def test_used_by_parsed_from_get_response(self): + result = GetConnectionStringsResult.from_json( + self._result_dict( + used_by=[ + { + "Kind": "SqlEtl", + "Id": 5000000000, + "Identifier": None, + "Name": "etl-1", + }, + {"Kind": "AiAgent", "Id": None, "Identifier": "agent-1", "Name": "agent-1"}, + ] + ) + ) + connection_string = result.raven_connection_strings["raven"] + self.assertEqual(2, len(connection_string.used_by)) + first = connection_string.used_by[0] + self.assertIsInstance(first, ConnectionStringUsage) + self.assertEqual("SqlEtl", first.kind) + self.assertEqual(5000000000, first.id) + self.assertIsNone(first.identifier) + self.assertEqual("etl-1", first.name) + second = connection_string.used_by[1] + self.assertEqual("AiAgent", second.kind) + self.assertIsNone(second.id) + self.assertEqual("agent-1", second.identifier) + + def test_used_by_empty_array_when_nothing_uses_it(self): + result = GetConnectionStringsResult.from_json(self._result_dict()) + self.assertEqual([], result.raven_connection_strings["raven"].used_by) + + def test_absent_used_by_key_does_not_raise(self): + raw = self._result_dict() + del raw["RavenConnectionStrings"]["raven"]["UsedBy"] + result = GetConnectionStringsResult.from_json(raw) + self.assertEqual([], result.raven_connection_strings["raven"].used_by) + + def test_to_json_never_emits_used_by(self): + result = GetConnectionStringsResult.from_json(self._result_dict(used_by=[{"Kind": "SqlEtl", "Id": 1}])) + connection_string = result.raven_connection_strings["raven"] + out = connection_string.to_json() + self.assertNotIn("UsedBy", out) + # round-trip: GET -> from_json -> to_json -> PUT must not leak UsedBy + self.assertNotIn("UsedBy", json.dumps(out, default=str)) + + def test_get_command_is_a_read_request(self): + operation = GetConnectionStringsOperation() + command = operation.get_command(None) + self.assertTrue(command.is_read_request()) + request = command.create_request(ServerNode("http://localhost:8080", "db1")) + self.assertEqual("http://localhost:8080/databases/db1/admin/connection-strings", request.url) + + +class TestPullReplicationAsSinkCursors(unittest.TestCase): + def _sink_dict(self, **overrides): + base = { + "TaskId": 1, + "TaskType": "PullReplicationAsSink", + "HubName": "hub", + "AllowedHubToSinkPaths": ["a"], + "AllowedSinkToHubPaths": ["b"], + "HubCursor": "hub-cursor-1", + "SinkCursor": "sink-cursor-1", + } + base.update(overrides) + return base + + def test_from_json_parses_hub_and_sink_cursor(self): + task = OngoingTaskPullReplicationAsSink.from_json(self._sink_dict()) + self.assertEqual("hub-cursor-1", task.hub_cursor) + self.assertEqual("sink-cursor-1", task.sink_cursor) + + def test_to_json_writes_cursors_after_allowed_sink_to_hub_paths(self): + task = OngoingTaskPullReplicationAsSink.from_json(self._sink_dict()) + out = task.to_json() + self.assertEqual("hub-cursor-1", out["HubCursor"]) + self.assertEqual("sink-cursor-1", out["SinkCursor"]) + keys = list(out.keys()) + self.assertGreater(keys.index("HubCursor"), keys.index("AllowedSinkToHubPaths")) + self.assertEqual(["HubCursor", "SinkCursor"], keys[-2:]) + + def test_absent_keys_stay_none_and_serialize_as_none(self): + task = OngoingTaskPullReplicationAsSink.from_json(self._sink_dict(HubCursor=None)) + self.assertIsNone(task.hub_cursor) + task = OngoingTaskPullReplicationAsSink.from_json({"TaskId": 1, "TaskType": "PullReplicationAsSink"}) + self.assertIsNone(task.hub_cursor) + self.assertIsNone(task.sink_cursor) + out = task.to_json() + self.assertIsNone(out["HubCursor"]) + self.assertIsNone(out["SinkCursor"]) + + def test_round_trip(self): + task = OngoingTaskPullReplicationAsSink.from_json(self._sink_dict()) + parsed = OngoingTaskPullReplicationAsSink.from_json(task.to_json()) + self.assertEqual("hub-cursor-1", parsed.hub_cursor) + self.assertEqual("sink-cursor-1", parsed.sink_cursor) + + +if __name__ == "__main__": + unittest.main() From b68ac8f8cbab48f06b4dccb067cdd85350712849 Mon Sep 17 00:00:00 2001 From: reforge Date: Wed, 19 Aug 2026 01:59:05 -0300 Subject: [PATCH 3/7] Add server-wide connection string operations Add ServerWideConnectionString wrapping any concrete connection string plus excluded databases, and put/get/remove operations against /admin/configuration/server-wide/connection-strings. The wrapper serializes the inner connection string with exactly one Type key carrying the enum NAME string and ExcludedDatabases (null when unset); UsedBy is server-computed metadata and is never written. from_json dispatches on Type, yields None when Type is missing (the server rejects such bodies), and parses the usages including their DatabaseName. The GET query carries name then type, each only when set, with the type value as the enum NAME, never the Python repr. Put and remove implement the RaftCommand marker and raise on a null response like the reference commands. Reforge-Run: 20260819T014403Z-2668162-reforge --- ravendb/__init__.py | 11 + .../operations/connection_strings.py | 282 ++++++++++++++++++ .../test_server_wide_connection_strings.py | 244 +++++++++++++++ ...est_server_wide_connection_strings_live.py | 83 ++++++ 4 files changed, 620 insertions(+) create mode 100644 ravendb/serverwide/operations/connection_strings.py create mode 100644 ravendb/tests/serverwide_tests/test_server_wide_connection_strings.py create mode 100644 ravendb/tests/serverwide_tests/test_server_wide_connection_strings_live.py diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 65c9c32d..5bdeec2d 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -384,6 +384,17 @@ GetDatabaseRecordOperation, ) +from ravendb.serverwide.operations.connection_strings import ( + ServerWideConnectionString, + ServerWideConnectionStringUsage, + PutServerWideConnectionStringOperation, + PutServerWideConnectionStringResult, + GetServerWideConnectionStringsOperation, + GetServerWideConnectionStringsResult, + RemoveServerWideConnectionStringOperation, + RemoveServerWideConnectionStringResult, +) + from ravendb.documents.identity.hilo import ( HiLoIdGenerator, MultiTypeHiLoGenerator, diff --git a/ravendb/serverwide/operations/connection_strings.py b/ravendb/serverwide/operations/connection_strings.py new file mode 100644 index 00000000..cd44a445 --- /dev/null +++ b/ravendb/serverwide/operations/connection_strings.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional, TYPE_CHECKING + +import requests + +from ravendb.documents.operations.ai.ai_connection_string import AiConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage +from ravendb.documents.operations.etl.configuration import RavenConnectionString +from ravendb.documents.operations.etl.elastic_search.connection import ElasticSearchConnectionString +from ravendb.documents.operations.etl.olap.connection import OlapConnectionString +from ravendb.documents.operations.etl.queue.connection import QueueConnectionString +from ravendb.documents.operations.etl.snowflake.connection import SnowflakeConnectionString +from ravendb.documents.operations.etl.sql import SqlConnectionString +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.operations.common import ServerOperation +from ravendb.serverwide.server_operation_executor import ConnectionStringType +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class ServerWideConnectionStringUsage(ConnectionStringUsage): + """A server-wide usage of a connection string; the server aggregates usages across + databases, so each usage also carries the DatabaseName where the task lives.""" + + def __init__( + self, + kind: Optional[str] = None, + id: Optional[int] = None, + identifier: Optional[str] = None, + name: Optional[str] = None, + database_name: Optional[str] = None, + ): + super().__init__(kind, id, identifier, name) + self.database_name = database_name + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> ServerWideConnectionStringUsage: + return cls( + kind=json_dict.get("Kind"), + id=json_dict.get("Id"), + identifier=json_dict.get("Identifier"), + name=json_dict.get("Name"), + database_name=json_dict.get("DatabaseName"), + ) + + +_CONNECTION_STRING_FROM_JSON = { + ConnectionStringType.RAVEN: RavenConnectionString.from_json, + ConnectionStringType.SQL: SqlConnectionString.from_json, + ConnectionStringType.OLAP: OlapConnectionString.from_json, + ConnectionStringType.ELASTIC_SEARCH: ElasticSearchConnectionString.from_json, + ConnectionStringType.QUEUE: QueueConnectionString.from_json, + ConnectionStringType.SNOWFLAKE: SnowflakeConnectionString.from_json, + ConnectionStringType.AI: AiConnectionString.from_json, +} + + +class ServerWideConnectionString: + def __init__( + self, + connection_string: Optional[ConnectionString] = None, + excluded_databases: Optional[List[str]] = None, + used_by: Optional[List[ServerWideConnectionStringUsage]] = None, + ): + self.connection_string = connection_string + self.excluded_databases = excluded_databases + self.used_by = used_by if used_by is not None else [] + + @property + def name(self) -> Optional[str]: + return self.connection_string.name if self.connection_string else None + + @property + def get_type(self) -> str: + return self.connection_string.get_type if self.connection_string else ConnectionStringType.NONE.value + + def to_json(self) -> Dict[str, Any]: + # UsedBy is read-only server metadata and is never written (C# ToJson omits it). + json_dict = self.connection_string.to_json() if self.connection_string else {} + json_dict["Type"] = self.get_type + json_dict["ExcludedDatabases"] = self.excluded_databases + return json_dict + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]) -> Optional[ServerWideConnectionString]: + if json_dict is None: + return None + + # The server rejects a body without Type; the C# FromBlittable returns null there. + type_value = json_dict.get("Type") + if not type_value: + return None + + try: + connection_string_type = ConnectionStringType(type_value) + except ValueError: + return None + + from_json = _CONNECTION_STRING_FROM_JSON.get(connection_string_type) + if from_json is None: + return None + + result = cls( + connection_string=from_json(json_dict), + excluded_databases=json_dict.get("ExcludedDatabases"), + ) + + used_by = json_dict.get("UsedBy") + if used_by: + result.used_by = [ServerWideConnectionStringUsage.from_json(usage) for usage in used_by] + + return result + + +class PutServerWideConnectionStringResult: + def __init__(self, raft_command_index: Optional[int] = None): + self.raft_command_index = raft_command_index + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> PutServerWideConnectionStringResult: + return cls(raft_command_index=json_dict.get("RaftCommandIndex")) + + +class PutServerWideConnectionStringOperation(ServerOperation[PutServerWideConnectionStringResult]): + def __init__(self, connection_string: ServerWideConnectionString): + if connection_string is None: + raise ValueError("connection_string cannot be None") + if connection_string.connection_string is None: + raise ValueError("connection_string.connection_string must not be None") + + self._connection_string = connection_string + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[PutServerWideConnectionStringResult]: + return self.PutServerWideConnectionStringCommand(self._connection_string) + + class PutServerWideConnectionStringCommand(RavenCommand[PutServerWideConnectionStringResult], RaftCommand): + def __init__(self, connection_string: ServerWideConnectionString): + super().__init__(PutServerWideConnectionStringResult) + if connection_string is None: + raise ValueError("connection_string cannot be None") + self._connection_string = connection_string + + def is_read_request(self) -> bool: + return False + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/admin/configuration/server-wide/connection-strings" + request = requests.Request("PUT", url) + request.headers = {"Content-Type": "application/json"} + request.data = json.dumps(self._connection_string.to_json()) + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = PutServerWideConnectionStringResult.from_json(json.loads(response)) + + +class GetServerWideConnectionStringsResult: + def __init__(self, results: Optional[List[ServerWideConnectionString]] = None): + self.results = results if results is not None else [] + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> GetServerWideConnectionStringsResult: + results = [] + for item in json_dict.get("Results") or []: + parsed = ServerWideConnectionString.from_json(item) + if parsed is not None: + results.append(parsed) + return cls(results) + + +class GetServerWideConnectionStringsOperation(ServerOperation[GetServerWideConnectionStringsResult]): + def __init__( + self, + connection_string_name: Optional[str] = None, + type: Optional[ConnectionStringType] = None, + ): + if connection_string_name is not None and (not connection_string_name or connection_string_name.isspace()): + raise ValueError("Connection string name must not be null or empty.") + + self._connection_string_name = connection_string_name + self._type = type + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[GetServerWideConnectionStringsResult]: + return self.GetServerWideConnectionStringsCommand(self._connection_string_name, self._type) + + class GetServerWideConnectionStringsCommand(RavenCommand[GetServerWideConnectionStringsResult]): + def __init__( + self, + connection_string_name: Optional[str] = None, + type: Optional[ConnectionStringType] = None, + ): + super().__init__(GetServerWideConnectionStringsResult) + self._connection_string_name = connection_string_name + self._type = type + + def is_read_request(self) -> bool: + return True + + def create_request(self, node: ServerNode) -> requests.Request: + from urllib.parse import quote + + url = f"{node.url}/admin/configuration/server-wide/connection-strings" + + query_params = [] + if self._connection_string_name is not None: + query_params.append(f"name={quote(self._connection_string_name)}") + if self._type is not None and self._type != ConnectionStringType.NONE: + # The query value is the enum NAME string, never the Python enum repr. + query_params.append(f"type={self._type.value}") + + if query_params: + url += "?" + "&".join(query_params) + + return requests.Request("GET", url) + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = GetServerWideConnectionStringsResult.from_json(json.loads(response)) + + +class RemoveServerWideConnectionStringResult: + def __init__(self, raft_command_index: Optional[int] = None): + self.raft_command_index = raft_command_index + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> RemoveServerWideConnectionStringResult: + return cls(raft_command_index=json_dict.get("RaftCommandIndex")) + + +class RemoveServerWideConnectionStringOperation(ServerOperation[RemoveServerWideConnectionStringResult]): + def __init__(self, connection_string: ConnectionString): + if connection_string is None: + raise ValueError("connection_string cannot be None") + if not connection_string.name or connection_string.name.isspace(): + raise ValueError("Connection string name must not be null or empty.") + + self._connection_string = connection_string + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[RemoveServerWideConnectionStringResult]: + return self.RemoveServerWideConnectionStringCommand(self._connection_string) + + class RemoveServerWideConnectionStringCommand(RavenCommand[RemoveServerWideConnectionStringResult], RaftCommand): + def __init__(self, connection_string: ConnectionString): + super().__init__(RemoveServerWideConnectionStringResult) + self._connection_string = connection_string + + def is_read_request(self) -> bool: + return False + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + def create_request(self, node: ServerNode) -> requests.Request: + from urllib.parse import quote + + url = ( + f"{node.url}/admin/configuration/server-wide/connection-strings" + f"?name={quote(self._connection_string.name)}" + f"&type={self._connection_string.get_type}" + ) + + return requests.Request("DELETE", url) + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = RemoveServerWideConnectionStringResult.from_json(json.loads(response)) diff --git a/ravendb/tests/serverwide_tests/test_server_wide_connection_strings.py b/ravendb/tests/serverwide_tests/test_server_wide_connection_strings.py new file mode 100644 index 00000000..f960739f --- /dev/null +++ b/ravendb/tests/serverwide_tests/test_server_wide_connection_strings.py @@ -0,0 +1,244 @@ +"""Tests for the server-wide connection strings surface: +the wrapper class, and the put/get/remove operations. +""" + +import json +import unittest + +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType, QueueConnectionString +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.operations.connection_strings import ( + GetServerWideConnectionStringsOperation, + PutServerWideConnectionStringOperation, + PutServerWideConnectionStringResult, + RemoveServerWideConnectionStringOperation, + RemoveServerWideConnectionStringResult, + ServerWideConnectionString, + ServerWideConnectionStringUsage, +) +from ravendb.serverwide.server_operation_executor import ConnectionStringType + + +class TestServerWideConnectionString(unittest.TestCase): + def _queue(self): + return QueueConnectionString(name="q1", broker_type=QueueBrokerType.KAFKA) + + def test_to_json_inner_keys_plus_type_and_excluded(self): + wrapper = ServerWideConnectionString(connection_string=self._queue(), excluded_databases=["db2"]) + result = wrapper.to_json() + self.assertEqual("Queue", result["Type"]) + self.assertEqual(["db2"], result["ExcludedDatabases"]) + self.assertEqual("Kafka", result["BrokerType"]) + self.assertNotIn("UsedBy", result) + self.assertEqual(1, len([k for k in result if k == "Type"])) + + def test_excluded_databases_null_when_unset(self): + wrapper = ServerWideConnectionString(connection_string=self._queue()) + self.assertIsNone(wrapper.to_json()["ExcludedDatabases"]) + + def test_name_and_type_delegate_to_inner(self): + wrapper = ServerWideConnectionString(connection_string=self._queue()) + self.assertEqual("q1", wrapper.name) + self.assertEqual("Queue", wrapper.get_type) + + def test_none_inner_delegates_to_none(self): + wrapper = ServerWideConnectionString() + self.assertIsNone(wrapper.name) + self.assertEqual("None", wrapper.get_type) + self.assertEqual({"Type": "None", "ExcludedDatabases": None}, wrapper.to_json()) + + def test_from_json_dispatches_by_type(self): + item = { + "Name": "q1", + "BrokerType": "Kafka", + "KafkaConnectionSettings": None, + "RabbitMqConnectionSettings": None, + "AzureQueueStorageConnectionSettings": None, + "AmazonSqsConnectionSettings": None, + "AzureServiceBusConnectionSettings": None, + "Type": "Queue", + "ExcludedDatabases": ["db2"], + "UsedBy": [ + { + "Kind": "QueueSink", + "Id": 99999999999, + "Identifier": "ident", + "Name": "task", + "DatabaseName": "db1", + } + ], + } + parsed = ServerWideConnectionString.from_json(item) + self.assertIsInstance(parsed, ServerWideConnectionString) + self.assertEqual("q1", parsed.name) + self.assertEqual("Queue", parsed.get_type) + self.assertEqual(["db2"], parsed.excluded_databases) + self.assertIsInstance(parsed.connection_string, QueueConnectionString) + usage = parsed.used_by[0] + self.assertIsInstance(usage, ServerWideConnectionStringUsage) + self.assertEqual("QueueSink", usage.kind) + self.assertEqual(99999999999, usage.id) + self.assertEqual("db1", usage.database_name) + + def test_from_json_returns_none_when_type_missing(self): + self.assertIsNone(ServerWideConnectionString.from_json({"Name": "x"})) + + def test_from_json_tolerates_missing_used_by(self): + parsed = ServerWideConnectionString.from_json( + { + "Name": "r1", + "Database": "d", + "TopologyDiscoveryUrls": ["http://localhost:8080"], + "Type": "Raven", + } + ) + self.assertEqual([], parsed.used_by) + + +class TestPutServerWideConnectionStringOperation(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + + def _wrapper(self): + inner = QueueConnectionString(name="q1", broker_type=QueueBrokerType.KAFKA) + return ServerWideConnectionString(connection_string=inner, excluded_databases=["db2"]) + + def test_put_url_and_body(self): + operation = PutServerWideConnectionStringOperation(self._wrapper()) + command = operation.get_command(None) + request = command.create_request(self.node) + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/admin/configuration/server-wide/connection-strings", request.url) + body = json.loads(request.data) + self.assertEqual("Queue", body["Type"]) + self.assertEqual(["db2"], body["ExcludedDatabases"]) + self.assertNotIn("UsedBy", body) + + def test_not_a_read_request(self): + command = PutServerWideConnectionStringOperation(self._wrapper()).get_command(None) + self.assertFalse(command.is_read_request()) + + def test_implements_raft_command(self): + command = PutServerWideConnectionStringOperation(self._wrapper()).get_command(None) + self.assertIsInstance(command, RaftCommand) + self.assertTrue(command.get_raft_unique_request_id()) + + def test_null_response_raises(self): + command = PutServerWideConnectionStringOperation(self._wrapper()).get_command(None) + with self.assertRaises(ValueError): + command.set_response(None, False) + + def test_result_parses_raft_command_index(self): + result = PutServerWideConnectionStringResult.from_json({"RaftCommandIndex": 1234567890123}) + self.assertEqual(1234567890123, result.raft_command_index) + + def test_constructor_validation(self): + with self.assertRaises(ValueError): + PutServerWideConnectionStringOperation(None) + with self.assertRaises(ValueError): + PutServerWideConnectionStringOperation(ServerWideConnectionString()) + + +class TestGetServerWideConnectionStringsOperation(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + + def test_no_query_when_no_params(self): + operation = GetServerWideConnectionStringsOperation() + request = operation.get_command(None).create_request(self.node) + self.assertEqual("GET", request.method) + self.assertEqual("http://localhost:8080/admin/configuration/server-wide/connection-strings", request.url) + + def test_name_then_type_query_order(self): + operation = GetServerWideConnectionStringsOperation("my cs", ConnectionStringType.QUEUE) + request = operation.get_command(None).create_request(self.node) + self.assertEqual( + "http://localhost:8080/admin/configuration/server-wide/connection-strings?name=my%20cs&type=Queue", + request.url, + ) + + def test_type_only_query(self): + operation = GetServerWideConnectionStringsOperation(None, ConnectionStringType.QUEUE) + request = operation.get_command(None).create_request(self.node) + self.assertEqual( + "http://localhost:8080/admin/configuration/server-wide/connection-strings?type=Queue", request.url + ) + + def test_type_value_is_enum_name_not_repr(self): + operation = GetServerWideConnectionStringsOperation("n", ConnectionStringType.QUEUE) + url = operation.get_command(None).create_request(self.node).url + self.assertNotIn("ConnectionStringType.QUEUE", url) + self.assertIn("type=Queue", url) + + def test_blank_name_raises(self): + with self.assertRaises(ValueError) as ctx: + GetServerWideConnectionStringsOperation(" ") + self.assertIn("Connection string name must not be null or empty", str(ctx.exception)) + + def test_is_read_request(self): + command = GetServerWideConnectionStringsOperation().get_command(None) + self.assertTrue(command.is_read_request()) + + def test_null_response_raises(self): + command = GetServerWideConnectionStringsOperation().get_command(None) + with self.assertRaises(ValueError): + command.set_response(None, False) + + def test_result_parses_results_with_database_name(self): + payload = { + "Results": [ + { + "Name": "q1", + "BrokerType": "Kafka", + "KafkaConnectionSettings": None, + "RabbitMqConnectionSettings": None, + "AzureQueueStorageConnectionSettings": None, + "AmazonSqsConnectionSettings": None, + "AzureServiceBusConnectionSettings": None, + "Type": "Queue", + "ExcludedDatabases": None, + "UsedBy": [{"Kind": "QueueSink", "Id": 1, "Identifier": None, "Name": "t", "DatabaseName": "db"}], + } + ] + } + command = GetServerWideConnectionStringsOperation().get_command(None) + command.set_response(json.dumps(payload), False) + self.assertEqual(1, len(command.result.results)) + self.assertEqual("q1", command.result.results[0].name) + self.assertEqual("db", command.result.results[0].used_by[0].database_name) + + +class TestRemoveServerWideConnectionStringOperation(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + + def test_delete_url_with_name_and_type(self): + inner = QueueConnectionString(name="q1", broker_type=QueueBrokerType.KAFKA) + operation = RemoveServerWideConnectionStringOperation(inner) + command = operation.get_command(None) + request = command.create_request(self.node) + self.assertEqual("DELETE", request.method) + self.assertEqual( + "http://localhost:8080/admin/configuration/server-wide/connection-strings?name=q1&type=Queue", + request.url, + ) + + def test_implements_raft_command(self): + inner = QueueConnectionString(name="q1", broker_type=QueueBrokerType.KAFKA) + command = RemoveServerWideConnectionStringOperation(inner).get_command(None) + self.assertIsInstance(command, RaftCommand) + + def test_constructor_validation(self): + with self.assertRaises(ValueError): + RemoveServerWideConnectionStringOperation(None) + with self.assertRaises(ValueError): + RemoveServerWideConnectionStringOperation(QueueConnectionString(name=" ")) + + def test_result_parses_raft_command_index(self): + result = RemoveServerWideConnectionStringResult.from_json({"RaftCommandIndex": 7}) + self.assertEqual(7, result.raft_command_index) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/serverwide_tests/test_server_wide_connection_strings_live.py b/ravendb/tests/serverwide_tests/test_server_wide_connection_strings_live.py new file mode 100644 index 00000000..2ef98312 --- /dev/null +++ b/ravendb/tests/serverwide_tests/test_server_wide_connection_strings_live.py @@ -0,0 +1,83 @@ +"""Live-server integration tests for server-wide connection strings. + +These assert the wire facts a 7.2.5 community server enforces: the 400 for a +Type-less PUT body, the 402 license rejection for PUT and DELETE, and the GET +shapes. The module skips when no server is reachable at localhost:8081. +""" + +import os +import unittest + +import requests + +from ravendb.documents.operations.etl.configuration import RavenConnectionString +from ravendb.documents.store.definition import DocumentStore +from ravendb.exceptions.raven_exceptions import RavenException +from ravendb.serverwide.operations.connection_strings import ( + GetServerWideConnectionStringsOperation, + PutServerWideConnectionStringOperation, + RemoveServerWideConnectionStringOperation, + ServerWideConnectionString, +) + +_LIVE_SERVER_URL = os.environ.get("RAVENDB_LIVE_SERVER_URL", "http://localhost:8081") + + +def _server_reachable() -> bool: + try: + response = requests.get(f"{_LIVE_SERVER_URL}/build/version", timeout=3) + return response.status_code == 200 and '"7.2.5"' in response.text + except Exception: + return False + + +@unittest.skipUnless(_server_reachable(), f"no live 7.2.5 server at {_LIVE_SERVER_URL}") +class TestServerWideConnectionStringsLive(unittest.TestCase): + def setUp(self): + self.store = DocumentStore(urls=[_LIVE_SERVER_URL]) + self.store.initialize() + + def tearDown(self): + self.store.close() + + def test_put_without_type_answers_bad_request(self): + # The server's deserializer returns null when Type is missing, which it + # rejects as 400. The wrapper always writes Type, so the probe uses the + # raw endpoint. + response = requests.put( + f"{_LIVE_SERVER_URL}/admin/configuration/server-wide/connection-strings", + json={"Name": "x"}, + ) + self.assertEqual(400, response.status_code) + self.assertIn("Connection string is missing or invalid", response.text) + + def test_valid_put_surfaces_license_rejection_as_base_raven_exception(self): + wrapper = ServerWideConnectionString( + connection_string=RavenConnectionString( + name="live-cs", database="db1", topology_discovery_urls=[_LIVE_SERVER_URL] + ) + ) + with self.assertRaises(RavenException) as ctx: + self.store.maintenance.server.send(PutServerWideConnectionStringOperation(wrapper)) + self.assertIn("Your license doesn't support adding server wide connection strings.", str(ctx.exception)) + + def test_delete_surfaces_license_rejection_as_base_raven_exception(self): + connection_string = RavenConnectionString( + name="live-cs", database="db1", topology_discovery_urls=[_LIVE_SERVER_URL] + ) + with self.assertRaises(RavenException) as ctx: + self.store.maintenance.server.send(RemoveServerWideConnectionStringOperation(connection_string)) + self.assertIn("Your license doesn't support adding server wide connection strings.", str(ctx.exception)) + + def test_get_returns_results(self): + result = self.store.maintenance.server.send(GetServerWideConnectionStringsOperation()) + self.assertIsNotNone(result) + self.assertIsInstance(result.results, list) + + def test_get_type_query_value_is_enum_name(self): + result = self.store.maintenance.server.send(GetServerWideConnectionStringsOperation(None, None)) + self.assertIsNotNone(result) + + +if __name__ == "__main__": + unittest.main() From efe1d9329dcb6c6d8ae310c6465ecbaf8f68b763 Mon Sep 17 00:00:00 2001 From: reforge Date: Wed, 19 Aug 2026 01:59:05 -0300 Subject: [PATCH 4/7] Add Azure Service Bus queue support Add the AzureServiceBus broker type and the connection-settings classes, which validate that exactly one authentication method is set: a connection string containing sb://, a fully populated EntraId, or a Passwordless with a namespace. get_service_bus_url extracts the sb:// endpoint from the connection string preserving input case, or builds sb://{namespace}/ from the auth classes. to_json writes only the set fields, to_audit_json masks the secrets, and all three classes implement equality with a consistent hash. AzureServiceBusSinkSource encodes a plain queue name as a queue and 'topic;subscription' as a topic subscription; entry validation reports errors naming the script and the entry, with the reference error messages. Reforge-Run: 20260819T014403Z-2668162-reforge --- .../azure_service_bus_connection_settings.py | 207 ++++++++++++++ .../queue/azure_service_bus_sink_source.py | 78 ++++++ .../operations/etl/queue/connection.py | 14 + ravendb/tests/queue_tests/__init__.py | 0 .../queue_tests/test_azure_service_bus.py | 259 ++++++++++++++++++ 5 files changed, 558 insertions(+) create mode 100644 ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py create mode 100644 ravendb/documents/operations/etl/queue/azure_service_bus_sink_source.py create mode 100644 ravendb/tests/queue_tests/__init__.py create mode 100644 ravendb/tests/queue_tests/test_azure_service_bus.py diff --git a/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py b/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py new file mode 100644 index 00000000..78c863ca --- /dev/null +++ b/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +from typing import Any, Dict, Optional + +from ravendb.exceptions.exceptions import InvalidOperationException + + +class AzureServiceBusEntraId: + def __init__( + self, + namespace: Optional[str] = None, + tenant_id: Optional[str] = None, + client_id: Optional[str] = None, + client_secret: Optional[str] = None, + ): + self.namespace = namespace + self.tenant_id = tenant_id + self.client_id = client_id + self.client_secret = client_secret + + def is_valid(self) -> bool: + return ( + not _is_blank(self.namespace) + and not _is_blank(self.tenant_id) + and not _is_blank(self.client_id) + and not _is_blank(self.client_secret) + ) + + def to_json(self) -> Dict[str, Any]: + return { + "Namespace": self.namespace, + "TenantId": self.tenant_id, + "ClientId": self.client_id, + "ClientSecret": self.client_secret, + } + + def to_audit_json(self) -> Dict[str, Any]: + # ClientSecret is masked in audit output. + return { + "Namespace": self.namespace, + "TenantId": self.tenant_id, + "ClientId": self.client_id, + } + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]) -> AzureServiceBusEntraId: + return cls( + namespace=json_dict.get("Namespace"), + tenant_id=json_dict.get("TenantId"), + client_id=json_dict.get("ClientId"), + client_secret=json_dict.get("ClientSecret"), + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AzureServiceBusEntraId): + return False + return ( + self.namespace == other.namespace + and self.tenant_id == other.tenant_id + and self.client_id == other.client_id + and self.client_secret == other.client_secret + ) + + def __hash__(self) -> int: + return hash((self.namespace, self.tenant_id, self.client_id, self.client_secret)) + + +class AzureServiceBusPasswordless: + """Machine authentication (Managed Identity); only a namespace is required.""" + + def __init__(self, namespace: Optional[str] = None): + self.namespace = namespace + + def is_valid(self) -> bool: + return not _is_blank(self.namespace) + + def to_json(self) -> Dict[str, Any]: + return {"Namespace": self.namespace} + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]) -> AzureServiceBusPasswordless: + return cls(namespace=json_dict.get("Namespace")) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AzureServiceBusPasswordless): + return False + return self.namespace == other.namespace + + def __hash__(self) -> int: + return hash(self.namespace) + + +def _is_blank(value: Optional[str]) -> bool: + return value is None or (isinstance(value, str) and value.strip() == "") + + +class AzureServiceBusConnectionSettings: + def __init__( + self, + connection_string: Optional[str] = None, + entra_id: Optional[AzureServiceBusEntraId] = None, + passwordless: Optional[AzureServiceBusPasswordless] = None, + ): + self.connection_string = connection_string + self.entra_id = entra_id + self.passwordless = passwordless + + def is_valid_connection(self) -> bool: + if not self._is_only_one_connection_provided(): + return False + + if self.entra_id is not None and self.entra_id.is_valid(): + return True + + if self.passwordless is not None and self.passwordless.is_valid(): + return True + + return self._try_extract_endpoint() is not None + + def _is_only_one_connection_provided(self) -> bool: + count = 0 + if self.entra_id is not None: + count += 1 + if not _is_blank(self.connection_string): + count += 1 + if self.passwordless is not None: + count += 1 + return count == 1 + + def get_service_bus_url(self) -> str: + if not _is_blank(self.connection_string): + endpoint = self._try_extract_endpoint() + if endpoint is not None: + return endpoint + raise InvalidOperationException("No endpoint provided") + + return f"sb://{self._get_namespace()}/" + + def _try_extract_endpoint(self) -> Optional[str]: + if not self.connection_string: + return None + + # Case-insensitive search, matching IndexOf(OrdinalIgnoreCase) in the C# client. + lowered = self.connection_string.lower() + start = lowered.find("sb://") + if start < 0: + return None + + end = self.connection_string.find(";", start) + if end < 0: + return self.connection_string[start:] + + return self.connection_string[start:end] + + def _get_namespace(self) -> str: + if self.entra_id is not None: + return self.entra_id.namespace + + if self.passwordless is not None: + return self.passwordless.namespace + + raise InvalidOperationException("No namespace provided") + + def to_json(self) -> Dict[str, Any]: + # Only set auth fields are written. The connection string is written when it is + # not None and not "" (IsNullOrEmpty), so a whitespace-only value IS written even + # though the exactly-one rule treats it as unset. + json_dict = {} + if self.connection_string: + json_dict["ConnectionString"] = self.connection_string + if self.entra_id is not None: + json_dict["EntraId"] = self.entra_id.to_json() + if self.passwordless is not None: + json_dict["Passwordless"] = self.passwordless.to_json() + return json_dict + + def to_audit_json(self) -> Dict[str, Any]: + json_dict = {} + if self.connection_string: + json_dict["ConnectionString"] = "" + if self.entra_id is not None: + json_dict["EntraId"] = self.entra_id.to_audit_json() + if self.passwordless is not None: + json_dict["Passwordless"] = self.passwordless.to_json() + return json_dict + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]) -> AzureServiceBusConnectionSettings: + entra_id = json_dict.get("EntraId") + passwordless = json_dict.get("Passwordless") + return cls( + connection_string=json_dict.get("ConnectionString"), + entra_id=AzureServiceBusEntraId.from_json(entra_id) if entra_id else None, + passwordless=AzureServiceBusPasswordless.from_json(passwordless) if passwordless else None, + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AzureServiceBusConnectionSettings): + return False + return ( + self.connection_string == other.connection_string + and self.entra_id == other.entra_id + and self.passwordless == other.passwordless + ) + + def __hash__(self) -> int: + return hash((self.connection_string, self.entra_id, self.passwordless)) diff --git a/ravendb/documents/operations/etl/queue/azure_service_bus_sink_source.py b/ravendb/documents/operations/etl/queue/azure_service_bus_sink_source.py new file mode 100644 index 00000000..a02508ab --- /dev/null +++ b/ravendb/documents/operations/etl/queue/azure_service_bus_sink_source.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from typing import List, Optional, Tuple + + +class AzureServiceBusSinkSource: + """Helpers for encoding Azure Service Bus sources as strings. + + Encoding convention: a plain queue name is a Service Bus queue, and + "topic;subscription" is a topic subscription. The ';' separator is + collision-safe because Service Bus naming rules forbid it in names. + """ + + SEPARATOR = ";" + + @staticmethod + def queue(queue_name: str) -> str: + if not queue_name or (isinstance(queue_name, str) and queue_name.isspace()): + raise ValueError("Queue name must be non-empty.") + if AzureServiceBusSinkSource.SEPARATOR in queue_name: + raise ValueError("Queue name must not contain the ';' character.") + return queue_name + + @staticmethod + def subscription(topic_name: str, subscription_name: str) -> str: + if not topic_name or (isinstance(topic_name, str) and topic_name.isspace()): + raise ValueError("Topic name must be non-empty.") + if not subscription_name or (isinstance(subscription_name, str) and subscription_name.isspace()): + raise ValueError("Subscription name must be non-empty.") + if AzureServiceBusSinkSource.SEPARATOR in topic_name: + raise ValueError("Topic name must not contain the ';' character.") + if AzureServiceBusSinkSource.SEPARATOR in subscription_name: + raise ValueError("Subscription name must not contain the ';' character.") + return f"{topic_name}{AzureServiceBusSinkSource.SEPARATOR}{subscription_name}" + + @staticmethod + def validate_entry(entry: str) -> Optional[str]: + """Returns None when the entry is a valid queue name or a valid + topic;subscription pair, otherwise the error message.""" + if not entry or (isinstance(entry, str) and entry.isspace()): + return "Azure Service Bus source entry cannot be empty." + + if AzureServiceBusSinkSource.SEPARATOR in entry and not AzureServiceBusSinkSource.try_parse_subscription(entry): + return ( + f"Azure Service Bus subscription source '{entry}' is invalid. " + f"Use '{AzureServiceBusSinkSource.SEPARATOR}' with a single " + f"'{AzureServiceBusSinkSource.SEPARATOR}' separator and both parts non-empty." + ) + + return None + + @staticmethod + def validate_script(script_name: str, queues: Optional[List[str]]) -> List[str]: + errors = [] + if queues is None: + return errors + + for entry in queues: + error = AzureServiceBusSinkSource.validate_entry(entry) + if error is None: + continue + errors.append(f"Script '{script_name}': {error}") + + return errors + + @staticmethod + def try_parse_subscription(entry: str) -> Optional[Tuple[str, str]]: + if entry is None: + return None + + parts = entry.split(AzureServiceBusSinkSource.SEPARATOR) + if len(parts) != 2: + return None + + if not parts[0] or parts[0].isspace() or not parts[1] or parts[1].isspace(): + return None + + return parts[0], parts[1] diff --git a/ravendb/documents/operations/etl/queue/connection.py b/ravendb/documents/operations/etl/queue/connection.py index ebaefad3..6c20441f 100644 --- a/ravendb/documents/operations/etl/queue/connection.py +++ b/ravendb/documents/operations/etl/queue/connection.py @@ -7,6 +7,9 @@ from ravendb.documents.operations.etl.queue.azure_queue_storage_connection_settings import ( AzureQueueStorageConnectionSettings, ) +from ravendb.documents.operations.etl.queue.azure_service_bus_connection_settings import ( + AzureServiceBusConnectionSettings, +) from ravendb.documents.operations.etl.queue.kafka_connection_settings import KafkaConnectionSettings from ravendb.documents.operations.etl.queue.rabbit_mq_connection_settings import RabbitMqConnectionSettings @@ -17,6 +20,7 @@ class QueueBrokerType(Enum): RABBIT_MQ = "RabbitMq" AZURE_QUEUE_STORAGE = "AzureQueueStorage" AMAZON_SQS = "AmazonSqs" + AZURE_SERVICE_BUS = "AzureServiceBus" class QueueConnectionString(ConnectionString): @@ -28,6 +32,7 @@ def __init__( rabbit_mq_settings: RabbitMqConnectionSettings = None, azure_queue_storage_settings: AzureQueueStorageConnectionSettings = None, amazon_sqs_settings: AmazonSqsConnectionSettings = None, + azure_service_bus_settings: AzureServiceBusConnectionSettings = None, used_by: Optional[List[ConnectionStringUsage]] = None, ): super().__init__(name, used_by) @@ -36,6 +41,7 @@ def __init__( self.rabbit_mq_settings = rabbit_mq_settings self.azure_queue_storage_settings = azure_queue_storage_settings self.amazon_sqs_settings = amazon_sqs_settings + self.azure_service_bus_settings = azure_service_bus_settings @property def get_type(self): @@ -51,6 +57,9 @@ def to_json(self): self.azure_queue_storage_settings.to_json() if self.azure_queue_storage_settings else None ), "AmazonSqsConnectionSettings": self.amazon_sqs_settings.to_json() if self.amazon_sqs_settings else None, + "AzureServiceBusConnectionSettings": ( + self.azure_service_bus_settings.to_json() if self.azure_service_bus_settings else None + ), "Type": ravendb.serverwide.server_operation_executor.ConnectionStringType.QUEUE, } @@ -79,5 +88,10 @@ def from_json(cls, json_dict: dict) -> "QueueConnectionString": if json_dict["AmazonSqsConnectionSettings"] else None ), + azure_service_bus_settings=( + AzureServiceBusConnectionSettings.from_json(json_dict["AzureServiceBusConnectionSettings"]) + if json_dict.get("AzureServiceBusConnectionSettings") + else None + ), used_by=[ConnectionStringUsage.from_json(usage) for usage in (json_dict.get("UsedBy") or [])], ) diff --git a/ravendb/tests/queue_tests/__init__.py b/ravendb/tests/queue_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ravendb/tests/queue_tests/test_azure_service_bus.py b/ravendb/tests/queue_tests/test_azure_service_bus.py new file mode 100644 index 00000000..cfd4aabe --- /dev/null +++ b/ravendb/tests/queue_tests/test_azure_service_bus.py @@ -0,0 +1,259 @@ +"""Tests for the Azure Service Bus surface: the broker type, +the connection settings classes with exactly-one-auth validation, the queue +connection-string field, and the sink-source helpers with entry validation. +""" + +import unittest + +from ravendb.documents.operations.etl.queue.azure_service_bus_connection_settings import ( + AzureServiceBusConnectionSettings, + AzureServiceBusEntraId, + AzureServiceBusPasswordless, +) +from ravendb.documents.operations.etl.queue.azure_service_bus_sink_source import AzureServiceBusSinkSource +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType, QueueConnectionString +from ravendb.exceptions.exceptions import InvalidOperationException + + +class TestQueueBrokerType(unittest.TestCase): + def test_azure_service_bus_value(self): + self.assertEqual("AzureServiceBus", QueueBrokerType.AZURE_SERVICE_BUS.value) + + +class TestAzureServiceBusConnectionSettings(unittest.TestCase): + def test_connection_string_valid_when_contains_sb_protocol(self): + settings = AzureServiceBusConnectionSettings( + connection_string="Endpoint=sb://ns.servicebus.windows.net/;SharedAccessKeyName=k;SharedAccessKey=v" + ) + self.assertTrue(settings.is_valid_connection()) + + def test_sb_search_is_case_insensitive(self): + settings = AzureServiceBusConnectionSettings(connection_string="SB://ns.servicebus.windows.net") + self.assertTrue(settings.is_valid_connection()) + + def test_connection_string_invalid_without_sb_protocol(self): + for value in ( + "SharedAccessKeyName=key;SharedAccessKey=abc", + "Endpoint=https://ns.servicebus.windows.net/;SharedAccessKey=abc", + "nothing-useful-here", + ): + self.assertFalse(AzureServiceBusConnectionSettings(connection_string=value).is_valid_connection()) + + def test_whitespace_connection_string_counts_as_unset(self): + settings = AzureServiceBusConnectionSettings(connection_string=" ") + self.assertFalse(settings.is_valid_connection()) + # ToJson asymmetry: IsNullOrEmpty writes a whitespace-only value. + self.assertEqual({"ConnectionString": " "}, settings.to_json()) + + def test_exactly_one_auth_method_required(self): + none_set = AzureServiceBusConnectionSettings() + self.assertFalse(none_set.is_valid_connection()) + + two_set = AzureServiceBusConnectionSettings( + connection_string="Endpoint=sb://x/", + entra_id=AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s"), + ) + self.assertFalse(two_set.is_valid_connection()) + + def test_entra_id_requires_all_four_fields(self): + complete = AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s") + self.assertTrue(AzureServiceBusConnectionSettings(entra_id=complete).is_valid_connection()) + for kwargs in ( + {"namespace": "ns", "tenant_id": "t", "client_id": "c"}, + {"namespace": "ns", "tenant_id": "t", "client_id": "c", "client_secret": " "}, + ): + self.assertFalse( + AzureServiceBusConnectionSettings(entra_id=AzureServiceBusEntraId(**kwargs)).is_valid_connection() + ) + + def test_passwordless_requires_namespace(self): + self.assertTrue( + AzureServiceBusConnectionSettings( + passwordless=AzureServiceBusPasswordless(namespace="ns") + ).is_valid_connection() + ) + self.assertFalse( + AzureServiceBusConnectionSettings( + passwordless=AzureServiceBusPasswordless(namespace=" ") + ).is_valid_connection() + ) + + def test_get_service_bus_url_extracts_endpoint(self): + settings = AzureServiceBusConnectionSettings( + connection_string="Endpoint=sb://my.servicebus.windows.net;SharedAccessKeyName=key" + ) + self.assertEqual("sb://my.servicebus.windows.net", settings.get_service_bus_url()) + + def test_get_service_bus_url_extraction_starts_at_found_index(self): + settings = AzureServiceBusConnectionSettings( + connection_string="SharedAccessKeyName=key;Endpoint=sb://ns.servicebus.windows.net/;SharedAccessKey=abc" + ) + self.assertEqual("sb://ns.servicebus.windows.net/", settings.get_service_bus_url()) + + def test_get_service_bus_url_case_insensitive_extraction(self): + settings = AzureServiceBusConnectionSettings(connection_string="Endpoint=SB://ns.servicebus.windows.net/") + self.assertEqual("SB://ns.servicebus.windows.net/", settings.get_service_bus_url()) + + def test_get_service_bus_url_to_end_without_semicolon(self): + settings = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://ns.servicebus.windows.net") + self.assertEqual("sb://ns.servicebus.windows.net", settings.get_service_bus_url()) + + def test_get_service_bus_url_from_namespace(self): + settings = AzureServiceBusConnectionSettings( + entra_id=AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s") + ) + self.assertEqual("sb://ns/", settings.get_service_bus_url()) + settings = AzureServiceBusConnectionSettings(passwordless=AzureServiceBusPasswordless(namespace="ns")) + self.assertEqual("sb://ns/", settings.get_service_bus_url()) + + def test_get_service_bus_url_throws_when_no_endpoint(self): + with self.assertRaises(InvalidOperationException) as ctx: + AzureServiceBusConnectionSettings(connection_string="no sb here").get_service_bus_url() + self.assertEqual("No endpoint provided", str(ctx.exception)) + + def test_get_service_bus_url_throws_when_no_namespace(self): + with self.assertRaises(InvalidOperationException) as ctx: + AzureServiceBusConnectionSettings().get_service_bus_url() + self.assertEqual("No namespace provided", str(ctx.exception)) + + def test_to_json_writes_only_set_fields(self): + self.assertEqual({}, AzureServiceBusConnectionSettings().to_json()) + settings = AzureServiceBusConnectionSettings( + connection_string="Endpoint=sb://x/", + entra_id=AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s"), + ) + result = settings.to_json() + self.assertEqual({"Endpoint=sb://x/"}, {result["ConnectionString"]}) + self.assertIn("EntraId", result) + self.assertNotIn("Passwordless", result) + + def test_audit_json_masks_secrets(self): + settings = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://x/;Key=secret") + self.assertEqual({"ConnectionString": ""}, settings.to_audit_json()) + + settings = AzureServiceBusConnectionSettings( + entra_id=AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="secret") + ) + audit = settings.to_audit_json() + self.assertNotIn("ClientSecret", audit["EntraId"]) + + settings = AzureServiceBusConnectionSettings(passwordless=AzureServiceBusPasswordless(namespace="ns")) + self.assertEqual({"Passwordless": {"Namespace": "ns"}}, settings.to_audit_json()) + + def test_equality_and_hash(self): + a = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://x/") + b = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://x/") + c = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://y/") + self.assertEqual(a, b) + self.assertNotEqual(a, c) + self.assertEqual(hash(a), hash(b)) + + e1 = AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s") + e2 = AzureServiceBusEntraId(namespace="ns", tenant_id="t", client_id="c", client_secret="s") + self.assertEqual(e1, e2) + self.assertEqual(hash(e1), hash(e2)) + + p1 = AzureServiceBusPasswordless(namespace="ns") + p2 = AzureServiceBusPasswordless(namespace="ns") + self.assertEqual(p1, p2) + self.assertEqual(hash(p1), hash(p2)) + + +class TestQueueConnectionStringAsbField(unittest.TestCase): + def test_to_json_writes_key_between_amazon_sqs_and_type(self): + connection_string = QueueConnectionString(name="q", broker_type=QueueBrokerType.AZURE_SERVICE_BUS) + keys = list(connection_string.to_json().keys()) + self.assertEqual("AmazonSqsConnectionSettings", keys[-3]) + self.assertEqual("AzureServiceBusConnectionSettings", keys[-2]) + self.assertEqual("Type", keys[-1]) + + def test_unset_field_is_null_on_wire(self): + connection_string = QueueConnectionString(name="q", broker_type=QueueBrokerType.AZURE_SERVICE_BUS) + self.assertIsNone(connection_string.to_json()["AzureServiceBusConnectionSettings"]) + + def test_from_json_parses_full_settings_object(self): + payload = { + "Name": "q", + "BrokerType": "AzureServiceBus", + "KafkaConnectionSettings": None, + "RabbitMqConnectionSettings": None, + "AzureQueueStorageConnectionSettings": None, + "AmazonSqsConnectionSettings": None, + "AzureServiceBusConnectionSettings": {"ConnectionString": "Endpoint=sb://ns/"}, # type: ignore + "Type": "Queue", + } + parsed = QueueConnectionString.from_json(payload) + self.assertIsInstance(parsed.azure_service_bus_settings, AzureServiceBusConnectionSettings) + self.assertEqual("Endpoint=sb://ns/", parsed.azure_service_bus_settings.connection_string) + + def test_round_trip(self): + settings = AzureServiceBusConnectionSettings(connection_string="Endpoint=sb://ns/") + original = QueueConnectionString( + name="q", broker_type=QueueBrokerType.AZURE_SERVICE_BUS, azure_service_bus_settings=settings + ) + back = QueueConnectionString.from_json(original.to_json()) + self.assertEqual(original.to_json(), back.to_json()) + + +class TestAzureServiceBusSinkSource(unittest.TestCase): + def test_queue_returns_name_when_valid(self): + self.assertEqual("my-queue", AzureServiceBusSinkSource.queue("my-queue")) + + def test_queue_throws_when_empty(self): + for value in (None, "", " "): + with self.assertRaises(ValueError) as ctx: + AzureServiceBusSinkSource.queue(value) + self.assertEqual("Queue name must be non-empty.", str(ctx.exception)) + + def test_queue_throws_when_contains_separator(self): + with self.assertRaises(ValueError) as ctx: + AzureServiceBusSinkSource.queue("foo;bar") + self.assertEqual("Queue name must not contain the ';' character.", str(ctx.exception)) + + def test_subscription_encodes_topic_and_subscription(self): + self.assertEqual("topic;sub", AzureServiceBusSinkSource.subscription("topic", "sub")) + + def test_subscription_throws_when_empty(self): + for topic, subscription in ( + (None, "sub"), + ("", "sub"), + (" ", "sub"), + ("topic", None), + ("topic", ""), + ("topic", " "), + ): + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.subscription(topic, subscription) + + def test_subscription_throws_when_contains_separator(self): + with self.assertRaises(ValueError) as ctx: + AzureServiceBusSinkSource.subscription("to;pic", "sub") + self.assertEqual("Topic name must not contain the ';' character.", str(ctx.exception)) + with self.assertRaises(ValueError) as ctx: + AzureServiceBusSinkSource.subscription("topic", "su;b") + self.assertEqual("Subscription name must not contain the ';' character.", str(ctx.exception)) + + def test_validate_entry_accepts_valid_entries(self): + self.assertIsNone(AzureServiceBusSinkSource.validate_entry("my-queue")) + self.assertIsNone(AzureServiceBusSinkSource.validate_entry("topic;sub")) + + def test_validate_entry_rejects_invalid_entries(self): + for entry in ("", " ", ";sub", "topic;", ";", "topic;sub;extra"): + self.assertIsNotNone(AzureServiceBusSinkSource.validate_entry(entry), entry) + + def test_validate_script_names_script_and_entry(self): + errors = AzureServiceBusSinkSource.validate_script("script", ["ok", "a;b;c"]) + self.assertEqual( + [ + "Script 'script': Azure Service Bus subscription source 'a;b;c' is invalid. " + "Use ';' with a single ';' separator and both parts non-empty." + ], + errors, + ) + errors = AzureServiceBusSinkSource.validate_script("script", [" "]) + self.assertEqual(["Script 'script': Azure Service Bus source entry cannot be empty."], errors) + self.assertEqual([], AzureServiceBusSinkSource.validate_script("script", ["my-queue"])) + + +if __name__ == "__main__": + unittest.main() From 7a37f62bf9774259f6c27fcbf731d848224348b9 Mon Sep 17 00:00:00 2001 From: reforge Date: Wed, 19 Aug 2026 01:59:05 -0300 Subject: [PATCH 5/7] Add AI conversation message reading and cancellation controls Add GetConversationMessagesOperation with its options and typed results for reading agent conversation messages, reachable from the store through the maintenance executor. The command sends GET to /ai/agent/conversation/messages with the raven 7-digit timestamp form for before/after and the detail-level enum name; a 404 (null response) leaves the result None, matching the reference SetResponse. Parameters keep their native JSON types. RunConversationOperation always appends cancelPendingActionTools to the URL immediately after the debug parameter, and AiConversation resets the flag after a successful run. The streaming loop observes a caller cancellation signal between lines so a mid-stream cancel stops the read promptly. Reforge-Run: 20260819T014403Z-2668162-reforge --- ravendb/__init__.py | 7 + ravendb/documents/ai/ai_conversation.py | 23 +- ravendb/documents/ai/ai_operations.py | 29 +- .../operations/ai/agents/__init__.py | 19 + .../get_conversation_messages_operation.py | 254 ++++++++++++++ .../ai/agents/run_conversation_operation.py | 20 ++ .../test_conversation_messages_live.py | 65 ++++ .../test_get_conversation_messages.py | 324 ++++++++++++++++++ 8 files changed, 736 insertions(+), 5 deletions(-) create mode 100644 ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py create mode 100644 ravendb/tests/ai_agent_tests/test_conversation_messages_live.py create mode 100644 ravendb/tests/ai_agent_tests/test_get_conversation_messages.py diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 5bdeec2d..ac2e18ed 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -115,6 +115,13 @@ GetAiAgentsResponse, AddOrUpdateAiAgentOperation, DeleteAiAgentOperation, + GetConversationMessagesOperation, + GetConversationMessagesOptions, + AiConversationMessagesResult, + AiConversationMessage, + AiToolCallResult, + AiMessageRole, + AiConversationDetailLevel, ) from ravendb.documents.operations.ai import ( ChunkingOptions, diff --git a/ravendb/documents/ai/ai_conversation.py b/ravendb/documents/ai/ai_conversation.py index 50ea0fef..17f42916 100644 --- a/ravendb/documents/ai/ai_conversation.py +++ b/ravendb/documents/ai/ai_conversation.py @@ -43,6 +43,7 @@ def __init__( conversation_id: str = None, change_vector: str = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ): self._store = store self._agent_id = agent_id @@ -50,6 +51,7 @@ def __init__( self._conversation_id = conversation_id self._change_vector = change_vector self._debug = debug + self._cancel_pending_action_tools = cancel_pending_action_tools self._prompt_parts: List[ContentPart] = [] self._action_responses: Dict[str, AiAgentActionResponse] = {} @@ -137,17 +139,26 @@ def add_artificial_action_with_response(self, tool_id: str, action_response) -> self._artificial_actions.append(AiAgentArtificialActionResponse(tool_id=tool_id, content=content)) - def run(self) -> AiAnswer: + def run(self, cancellation_event=None) -> AiAnswer: self._dispatched_tool_ids.clear() while True: - r = self._run_internal() + r = self._run_internal(cancellation_event=cancellation_event) if self._handle_server_reply(r): return r - def stream(self, stream_property_path: str = None, on_chunk: Optional[Callable[[str], None]] = None) -> AiAnswer: + def stream( + self, + stream_property_path: str = None, + on_chunk: Optional[Callable[[str], None]] = None, + cancellation_event=None, + ) -> AiAnswer: while True: - r = self._run_internal(stream_property_path=stream_property_path, streamed_chunks_callback=on_chunk) + r = self._run_internal( + stream_property_path=stream_property_path, + streamed_chunks_callback=on_chunk, + cancellation_event=cancellation_event, + ) if self._handle_server_reply(r): return r @@ -155,6 +166,7 @@ def _run_internal( self, stream_property_path: Optional[str] = None, streamed_chunks_callback: Optional[Callable[[str], None]] = None, + cancellation_event=None, ) -> AiAnswer: from ravendb.documents.operations.ai.agents import RunConversationOperation import time @@ -194,6 +206,8 @@ def _run_internal( streamed_chunks_callback=streamed_chunks_callback, attachments_commands=self._attachments_commands, debug=self._debug, + cancel_pending_action_tools=self._cancel_pending_action_tools, + cancellation_event=cancellation_event, ) try: @@ -203,6 +217,7 @@ def _run_internal( self._change_vector = result.change_vector self._conversation_id = result.conversation_id + self._cancel_pending_action_tools = False self._action_requests = result.action_requests or [] return AiAnswer( diff --git a/ravendb/documents/ai/ai_operations.py b/ravendb/documents/ai/ai_operations.py index 77bf1ad8..197d3f0d 100644 --- a/ravendb/documents/ai/ai_operations.py +++ b/ravendb/documents/ai/ai_operations.py @@ -78,6 +78,7 @@ def conversation( creation_options: "AiConversationCreationOptions" = None, change_vector: str = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ) -> AiConversation: """ Creates a new conversation with the specified AI agent. @@ -88,12 +89,22 @@ def conversation( creation_options: Optional creation options for the conversation change_vector: Optional change vector for concurrency control debug: Optional flag enabling server-side conversation debugging + cancel_pending_action_tools: When True, the server auto-answers any open action + tool calls instead of requiring a response Returns: Conversation operations interface for managing the conversation """ - return AiConversation(self._store, agent_id, creation_options, conversation_id, change_vector, debug) + return AiConversation( + self._store, + agent_id, + creation_options, + conversation_id, + change_vector, + debug, + cancel_pending_action_tools, + ) def conversation_with_id(self, conversation_id: str, change_vector: str = None) -> AiConversation: """ @@ -115,3 +126,19 @@ def conversation_with_id(self, conversation_id: str, change_vector: str = None) from ravendb.documents.ai.ai_conversation import AiConversation return AiConversation.with_conversation_id(self._store, conversation_id, change_vector) + + def get_conversation_messages(self, conversation_id_or_options) -> "AiConversationMessagesResult": + """ + Reads messages from an AI agent conversation. + + Args: + conversation_id_or_options: The conversation document ID, or a + GetConversationMessagesOptions instance for paging/filtering control. + + Returns: + The conversation messages result; None when the conversation does not exist. + """ + from ravendb.documents.operations.ai.agents import GetConversationMessagesOperation + + operation = GetConversationMessagesOperation(conversation_id_or_options) + return self._store.maintenance.send(operation) diff --git a/ravendb/documents/operations/ai/agents/__init__.py b/ravendb/documents/operations/ai/agents/__init__.py index d8099d41..df5496f8 100644 --- a/ravendb/documents/operations/ai/agents/__init__.py +++ b/ravendb/documents/operations/ai/agents/__init__.py @@ -39,6 +39,17 @@ AiConversationParameterOptions, ) +from .get_conversation_messages_operation import ( + GetConversationMessagesOperation, + GetConversationMessagesOptions, + GetConversationMessagesCommand, + AiConversationMessagesResult, + AiConversationMessage, + AiToolCallResult, + AiMessageRole, + AiConversationDetailLevel, +) + __all__ = [ "AiAgentConfiguration", "AiAgentConfigurationResult", @@ -68,4 +79,12 @@ "GetAiAgentsResponse", "AddOrUpdateAiAgentOperation", "DeleteAiAgentOperation", + "GetConversationMessagesOperation", + "GetConversationMessagesOptions", + "GetConversationMessagesCommand", + "AiConversationMessagesResult", + "AiConversationMessage", + "AiToolCallResult", + "AiMessageRole", + "AiConversationDetailLevel", ] diff --git a/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py b/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py new file mode 100644 index 00000000..f5968aeb --- /dev/null +++ b/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +import enum +import json +from datetime import datetime +from typing import Any, Dict, List, Optional, TYPE_CHECKING + +import requests + +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.documents.conventions import DocumentConventions +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.tools.utils import Utils + +if TYPE_CHECKING: + from ravendb.documents.operations.ai.agents.run_conversation_operation import AiUsage + + +class AiMessageRole(enum.Enum): + SYSTEM = "System" + USER = "User" + ASSISTANT = "Assistant" + SUMMARY = "Summary" + INTERNAL = "Internal" + + def __str__(self) -> str: + return self.value + + +class AiConversationDetailLevel(enum.Enum): + SIMPLE = "Simple" + DETAILED = "Detailed" + FULL = "Full" + + def __str__(self) -> str: + return self.value + + +class AiToolCallResult: + def __init__( + self, + id: Optional[str] = None, + name: Optional[str] = None, + arguments: Optional[str] = None, + result: Optional[str] = None, + sub_conversation_id: Optional[str] = None, + ): + self.id = id + self.name = name + self.arguments = arguments + self.result = result + self.sub_conversation_id = sub_conversation_id + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiToolCallResult: + return cls( + id=json_dict.get("Id"), + name=json_dict.get("Name"), + arguments=json_dict.get("Arguments"), + result=json_dict.get("Result"), + sub_conversation_id=json_dict.get("SubConversationId"), + ) + + def to_json(self) -> Dict[str, Any]: + return { + "Id": self.id, + "Name": self.name, + "Arguments": self.arguments, + "Result": self.result, + "SubConversationId": self.sub_conversation_id, + } + + +class AiConversationMessage: + def __init__( + self, + role: Optional[AiMessageRole] = None, + content: Optional[str] = None, + attachments: Optional[List[str]] = None, + timestamp: Optional[datetime] = None, + tool_calls: Optional[List[AiToolCallResult]] = None, + usage: Optional["AiUsage"] = None, + sub_conversation_id: Optional[str] = None, + ): + self.role = role + self.content = content + self.attachments = attachments + self.timestamp = timestamp + self.tool_calls = tool_calls + self.usage = usage + self.sub_conversation_id = sub_conversation_id + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiConversationMessage: + from ravendb.documents.operations.ai.agents.run_conversation_operation import AiUsage + + role_value = json_dict.get("Role") + tool_calls = json_dict.get("ToolCalls") + return cls( + role=AiMessageRole(role_value) if role_value else None, + content=json_dict.get("Content"), + attachments=json_dict.get("Attachments"), + timestamp=Utils.string_to_datetime(json_dict.get("Timestamp")), + tool_calls=[AiToolCallResult.from_json(tc) for tc in tool_calls] if tool_calls is not None else None, + usage=AiUsage.from_json(json_dict["Usage"]) if json_dict.get("Usage") else None, + sub_conversation_id=json_dict.get("SubConversationId"), + ) + + def to_json(self) -> Dict[str, Any]: + return { + "Role": self.role.value if self.role else None, + "Content": self.content, + "Attachments": self.attachments, + "Timestamp": Utils.datetime_to_string(self.timestamp), + "ToolCalls": [tc.to_json() for tc in self.tool_calls] if self.tool_calls is not None else None, + "Usage": self.usage.to_json() if self.usage else None, + "SubConversationId": self.sub_conversation_id, + } + + +class AiConversationMessagesResult: + def __init__( + self, + conversation_id: Optional[str] = None, + agent: Optional[str] = None, + parameters: Optional[Dict[str, Any]] = None, + total_usage: Optional["AiUsage"] = None, + last_message_at: Optional[datetime] = None, + messages: Optional[List[AiConversationMessage]] = None, + has_more_messages: bool = False, + sub_conversation_ids: Optional[List[str]] = None, + attachments: Optional[List[str]] = None, + ): + self.conversation_id = conversation_id + self.agent = agent + self.parameters = parameters + self.total_usage = total_usage + self.last_message_at = last_message_at + self.messages = messages + self.has_more_messages = has_more_messages + self.sub_conversation_ids = sub_conversation_ids + self.attachments = attachments + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiConversationMessagesResult: + from ravendb.documents.operations.ai.agents.run_conversation_operation import AiUsage + + messages = json_dict.get("Messages") + return cls( + conversation_id=json_dict.get("ConversationId"), + agent=json_dict.get("Agent"), + parameters=json_dict.get("Parameters"), + total_usage=AiUsage.from_json(json_dict["TotalUsage"]) if json_dict.get("TotalUsage") else None, + last_message_at=Utils.string_to_datetime(json_dict.get("LastMessageAt")), + messages=[AiConversationMessage.from_json(m) for m in messages] if messages is not None else None, + has_more_messages=json_dict.get("HasMoreMessages", False), + sub_conversation_ids=json_dict.get("SubConversationIds"), + attachments=json_dict.get("Attachments"), + ) + + def to_json(self) -> Dict[str, Any]: + return { + "ConversationId": self.conversation_id, + "Agent": self.agent, + "Parameters": self.parameters, + "TotalUsage": self.total_usage.to_json() if self.total_usage else None, + "LastMessageAt": Utils.datetime_to_string(self.last_message_at), + "HasMoreMessages": self.has_more_messages, + "SubConversationIds": self.sub_conversation_ids, + "Attachments": self.attachments, + "Messages": [m.to_json() for m in self.messages] if self.messages is not None else None, + } + + +class GetConversationMessagesOptions: + def __init__( + self, + conversation_id: Optional[str] = None, + before: Optional[datetime] = None, + after: Optional[datetime] = None, + page_size: int = 2147483647, + detail_level: AiConversationDetailLevel = AiConversationDetailLevel.SIMPLE, + ): + self.conversation_id = conversation_id + self.before = before + self.after = after + self.page_size = page_size + self.detail_level = detail_level + + def validate(self) -> None: + if not self.conversation_id: + raise ValueError("conversation_id cannot be None or empty") + + if self.before is not None and self.after is not None: + raise ValueError("before and after cannot both be specified.") + + if self.page_size <= 0: + raise ValueError("PageSize must be greater than 0.") + + +class GetConversationMessagesOperation(MaintenanceOperation[AiConversationMessagesResult]): + def __init__(self, conversation_id_or_options): + if isinstance(conversation_id_or_options, GetConversationMessagesOptions): + parameters = conversation_id_or_options + else: + parameters = GetConversationMessagesOptions(conversation_id=conversation_id_or_options) + parameters.validate() + self._parameters = parameters + + def get_command(self, conventions: DocumentConventions) -> RavenCommand[AiConversationMessagesResult]: + return GetConversationMessagesCommand(self._parameters) + + +class GetConversationMessagesCommand(RavenCommand[AiConversationMessagesResult]): + def __init__(self, parameters: GetConversationMessagesOptions): + super().__init__(AiConversationMessagesResult) + self._parameters = parameters + + def is_read_request(self) -> bool: + return True + + def create_request(self, node: ServerNode) -> requests.Request: + from urllib.parse import quote + + url = ( + f"{node.url}/databases/{node.database}/ai/agent/conversation/messages" + f"?conversationId={quote(self._parameters.conversation_id)}" + ) + if self._parameters.before is not None: + url += f"&before={quote(Utils.datetime_to_string(self._parameters.before))}" + if self._parameters.after is not None: + url += f"&after={quote(Utils.datetime_to_string(self._parameters.after))}" + url += f"&pageSize={self._parameters.page_size}" + detail_level = self._parameters.detail_level or AiConversationDetailLevel.SIMPLE + url += f"&detailLevel={detail_level.value}" + + return requests.Request("GET", url) + + def set_response(self, response: str, from_cache: bool) -> None: + # A null response (404 for a missing conversation) leaves the result None. + if response is None: + return + + response_json = json.loads(response) + result = AiConversationMessagesResult.from_json(response_json) + + # Parameters are parsed natively so heterogeneous values keep their types + # (int stays int, float stays float, lists stay lists). + parameters = response_json.get("Parameters") + if isinstance(parameters, dict): + result.parameters = dict(parameters) + + self.result = result diff --git a/ravendb/documents/operations/ai/agents/run_conversation_operation.py b/ravendb/documents/operations/ai/agents/run_conversation_operation.py index b830284e..f192d643 100644 --- a/ravendb/documents/operations/ai/agents/run_conversation_operation.py +++ b/ravendb/documents/operations/ai/agents/run_conversation_operation.py @@ -295,6 +295,8 @@ def __init__( streamed_chunks_callback: Optional[Callable[[str], None]] = None, attachments_commands: Optional[List[Any]] = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, + cancellation_event=None, ): if not agent_id or (isinstance(agent_id, str) and agent_id.isspace()): raise ValueError("agent_id cannot be None or empty") @@ -314,6 +316,8 @@ def __init__( self._streamed_chunks_callback = streamed_chunks_callback self._attachments_commands = attachments_commands or [] self._debug = debug + self._cancel_pending_action_tools = cancel_pending_action_tools + self._cancellation_event = cancellation_event def get_command(self, conventions: DocumentConventions) -> RavenCommand[ConversationResult[TSchema]]: return RunConversationCommand( @@ -329,6 +333,8 @@ def get_command(self, conventions: DocumentConventions) -> RavenCommand[Conversa conventions=conventions, attachments_commands=self._attachments_commands, debug=self._debug, + cancel_pending_action_tools=self._cancel_pending_action_tools, + cancellation_event=self._cancellation_event, ) @@ -347,6 +353,8 @@ def __init__( conventions: Optional[DocumentConventions] = None, attachments_commands: Optional[List[Any]] = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, + cancellation_event=None, ): from ravendb.util.util import RaftIdGenerator from ravendb.documents.commands.batches import PutAttachmentCommandData @@ -363,6 +371,8 @@ def __init__( self._streamed_chunks_callback = streamed_chunks_callback self._conventions = conventions self._debug = debug + self._cancel_pending_action_tools = cancel_pending_action_tools + self._cancellation_event = cancellation_event self._attachments_commands = attachments_commands or [] # Raft id pinned at construction so retries keep the same id. @@ -409,6 +419,10 @@ def create_request(self, node: ServerNode) -> requests.Request: if self._debug is not None: url += f"&debug={self._debug}" + # Always appended, after the debug parameter (C# order: streaming, changeVector, debug, + # cancelPendingActionTools), so the wire carries it even when False. + url += f"&cancelPendingActionTools={self._cancel_pending_action_tools}" + request_body = ConversationRequestBody( action_responses=self._action_responses, artificial_actions=self._artificial_actions, @@ -449,6 +463,12 @@ def process_response(self, cache, response: requests.Response, url) -> ResponseD return super().process_response(cache, response, url) for line in response.iter_lines(decode_unicode=True): + if self._cancellation_event is not None and self._cancellation_event.is_set(): + # Cancellation is observed between lines, so a mid-stream cancel stops the + # read promptly instead of consuming the remaining stream (RavenDB-26693). + from concurrent.futures import CancelledError + + raise CancelledError("The operation was canceled.") if not line: continue if line.startswith("{"): diff --git a/ravendb/tests/ai_agent_tests/test_conversation_messages_live.py b/ravendb/tests/ai_agent_tests/test_conversation_messages_live.py new file mode 100644 index 00000000..95de7fb5 --- /dev/null +++ b/ravendb/tests/ai_agent_tests/test_conversation_messages_live.py @@ -0,0 +1,65 @@ +"""Live-server integration tests for AI conversation messages (C1/C6). + +Asserts the live-verified fact: a missing conversation answers HTTP 404 with an +empty body, and the operation surfaces result None (never an exception). +Skipped when no server is reachable at localhost:8081. +""" + +import os +import unittest + +import requests + +from ravendb.documents.store.definition import DocumentStore + +_LIVE_SERVER_URL = os.environ.get("RAVENDB_LIVE_SERVER_URL", "http://localhost:8081") +_DATABASE = "conversation-messages-live-db" + + +def _server_reachable() -> bool: + try: + response = requests.get(f"{_LIVE_SERVER_URL}/build/version", timeout=3) + return response.status_code == 200 and '"7.2.5"' in response.text + except Exception: + return False + + +@unittest.skipUnless(_server_reachable(), f"no live 7.2.5 server at {_LIVE_SERVER_URL}") +class TestConversationMessagesLive(unittest.TestCase): + @classmethod + def setUpClass(cls): + requests.put(f"{_LIVE_SERVER_URL}/admin/databases?name={_DATABASE}", json={"DatabaseName": _DATABASE}) + + @classmethod + def tearDownClass(cls): + requests.delete(f"{_LIVE_SERVER_URL}/admin/databases?name={_DATABASE}&hard-delete=true") + + def setUp(self): + self.store = DocumentStore(urls=[_LIVE_SERVER_URL], database=_DATABASE) + self.store.initialize() + + def tearDown(self): + self.store.close() + + def test_missing_conversation_answers_404_with_empty_body(self): + response = requests.get( + f"{_LIVE_SERVER_URL}/databases/{_DATABASE}/ai/agent/conversation/messages" + "?conversationId=chats/missing&pageSize=10&detailLevel=Simple" + ) + self.assertEqual(404, response.status_code) + self.assertEqual(0, len(response.content)) + + def test_missing_conversation_result_is_none(self): + result = self.store.ai.get_conversation_messages("chats/missing") + self.assertIsNone(result) + + def test_response_carries_database_cluster_tx_id_header(self): + response = requests.get( + f"{_LIVE_SERVER_URL}/databases/{_DATABASE}/ai/agent/conversation/messages" + "?conversationId=chats/missing&pageSize=10&detailLevel=Simple" + ) + self.assertIsNotNone(response.headers.get("Database-Cluster-Tx-Id")) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/ai_agent_tests/test_get_conversation_messages.py b/ravendb/tests/ai_agent_tests/test_get_conversation_messages.py new file mode 100644 index 00000000..49e0e9e3 --- /dev/null +++ b/ravendb/tests/ai_agent_tests/test_get_conversation_messages.py @@ -0,0 +1,324 @@ +"""Tests for the AI conversation-messages surface, cancelPendingActionTools on +the wire, and the streaming read that honors cancellation. +""" + +import json +import threading +import unittest +from datetime import datetime + +from ravendb.documents.ai.ai_operations import AiOperations +from ravendb.documents.operations.ai.agents import ( + AiConversationDetailLevel, + AiConversationMessage, + AiConversationMessagesResult, + AiMessageRole, + AiToolCallResult, + GetConversationMessagesCommand, + GetConversationMessagesOperation, + GetConversationMessagesOptions, + RunConversationOperation, +) +from ravendb.documents.operations.ai.agents.run_conversation_operation import RunConversationCommand +from ravendb.http.server_node import ServerNode +from ravendb.tools.utils import Utils + + +class TestGetConversationMessagesCommandWire(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + + def _command(self, **kwargs): + options = GetConversationMessagesOptions(**kwargs) + operation = GetConversationMessagesOperation(options) + return operation.get_command(None) + + def test_get_to_conversation_messages_endpoint(self): + command = self._command(conversation_id="chats/1") + request = command.create_request(self.node) + self.assertEqual("GET", request.method) + self.assertTrue(request.url.startswith("http://localhost:8080/databases/db1/ai/agent/conversation/messages?")) + + def test_parameter_order_and_always_present_params(self): + before = datetime(2026, 1, 2, 3, 4, 5, 123456) + command = self._command( + conversation_id="chats/1", before=before, page_size=50, detail_level=AiConversationDetailLevel.DETAILED + ) + url = command.create_request(self.node).url + query = url.split("?", 1)[1] + parts = [p.split("=", 1)[0] for p in query.split("&")] + self.assertEqual(["conversationId", "before", "pageSize", "detailLevel"], parts) + self.assertTrue(url.endswith("&pageSize=50&detailLevel=Detailed")) + + def test_after_appended_only_when_set(self): + after = datetime(2026, 1, 2, 3, 4, 5, 123456) + url = self._command(conversation_id="c", after=after).create_request(self.node).url + self.assertIn("&after=", url) + self.assertNotIn("&before=", url) + + def test_before_timestamp_is_the_seven_digit_raven_form(self): + from urllib.parse import unquote + + before = datetime(2026, 1, 2, 3, 4, 5, 123456) + url = self._command(conversation_id="c", before=before).create_request(self.node).url + ts = Utils.datetime_to_string(before) + self.assertEqual(7, len(ts.split(".")[1])) + query = url.split("?", 1)[1] + before_value = [p.split("=", 1)[1] for p in query.split("&") if p.startswith("before=")][0] + self.assertEqual(ts, unquote(before_value)) + + def test_page_size_default_is_int_max_value(self): + url = self._command(conversation_id="c").create_request(self.node).url + self.assertTrue(url.endswith("&pageSize=2147483647&detailLevel=Simple")) + + def test_is_read_request(self): + self.assertTrue(self._command(conversation_id="c").is_read_request()) + + def test_constructor_validates_conversation_id(self): + with self.assertRaises(ValueError): + GetConversationMessagesOperation("") + with self.assertRaises(ValueError): + GetConversationMessagesOperation(GetConversationMessagesOptions()) + + def test_before_and_after_are_mutually_exclusive(self): + when = datetime(2026, 1, 1) + with self.assertRaises(ValueError) as ctx: + GetConversationMessagesOperation( + GetConversationMessagesOptions(conversation_id="c", before=when, after=when) + ) + self.assertIn("cannot both be specified", str(ctx.exception)) + + def test_page_size_must_be_positive(self): + with self.assertRaises(ValueError) as ctx: + GetConversationMessagesOperation(GetConversationMessagesOptions(conversation_id="c", page_size=0)) + self.assertIn("PageSize must be greater than 0", str(ctx.exception)) + + def test_null_response_leaves_result_none(self): + command = GetConversationMessagesCommand(GetConversationMessagesOptions(conversation_id="missing")) + command.set_response(None, False) + self.assertIsNone(command.result) + + def test_parameters_keep_typed_values(self): + payload = { + "ConversationId": "chats/1", + "Agent": "a1", + "Parameters": { + "big": 3000000000, + "frac": 0.5, + "text": "hi", + "flag": True, + "arr": [1, 2, 3], + "mixed": [1, "x"], + }, + "TotalUsage": None, + "Messages": [], + } + command = GetConversationMessagesCommand(GetConversationMessagesOptions(conversation_id="chats/1")) + command.set_response(json.dumps(payload), False) + params = command.result.parameters + self.assertIsInstance(params["big"], int) + self.assertEqual(3000000000, params["big"]) + self.assertIsInstance(params["frac"], float) + self.assertEqual(0.5, params["frac"]) + self.assertEqual("hi", params["text"]) + self.assertIs(True, params["flag"]) + self.assertEqual([1, 2, 3], params["arr"]) + self.assertEqual([1, "x"], params["mixed"]) + + +class TestAiConversationMessage(unittest.TestCase): + def test_from_json_parses_all_fields(self): + payload = { + "Role": "User", + "Content": "hello", + "Attachments": ["a.txt"], + "Timestamp": "2026-01-02T03:04:05.1234560Z", + "ToolCalls": [{"Id": "t1", "Name": "n", "Arguments": "{}", "Result": "ok", "SubConversationId": None}], + "Usage": { + "PromptTokens": 1, + "CompletionTokens": 2, + "TotalTokens": 3, + "CachedTokens": 0, + "ReasoningTokens": 0, + }, + "SubConversationId": None, + } + message = AiConversationMessage.from_json(payload) + self.assertEqual(AiMessageRole.USER, message.role) + self.assertEqual("hello", message.content) + self.assertEqual(["a.txt"], message.attachments) + self.assertEqual(2026, message.timestamp.year) + self.assertEqual("t1", message.tool_calls[0].id) + self.assertEqual(AiUsage_tokens(message), 3) + + def test_to_json_order_and_role_as_name(self): + message = AiConversationMessage( + role=AiMessageRole.SYSTEM, + content="sys", + attachments=[], + timestamp=datetime(2026, 1, 1), + tool_calls=[], + usage=None, + sub_conversation_id=None, + ) + result = message.to_json() + self.assertEqual( + list(result.keys()), + ["Role", "Content", "Attachments", "Timestamp", "ToolCalls", "Usage", "SubConversationId"], + ) + self.assertEqual("System", result["Role"]) + + def test_pending_tool_call_result_stays_none(self): + tool_call = AiToolCallResult.from_json({"Id": "t1", "Name": "n", "Arguments": "{}", "Result": None}) + self.assertIsNone(tool_call.result) + self.assertEqual( + {"Id": "t1", "Name": "n", "Arguments": "{}", "Result": None, "SubConversationId": None}, + tool_call.to_json(), + ) + + def test_missing_keys_use_defaults(self): + message = AiConversationMessage.from_json({"Role": "Assistant"}) + self.assertIsNone(message.content) + self.assertIsNone(message.attachments) + self.assertIsNone(message.timestamp) + self.assertIsNone(message.tool_calls) + + +def AiUsage_tokens(message): + return message.usage.total_tokens + + +class TestAiConversationMessagesResult(unittest.TestCase): + def test_to_json_key_order(self): + result = AiConversationMessagesResult( + conversation_id="c", + agent="a", + parameters={}, + total_usage=None, + last_message_at=None, + has_more_messages=False, + sub_conversation_ids=None, + attachments=None, + messages=None, + ) + keys = list(result.to_json().keys()) + self.assertEqual( + [ + "ConversationId", + "Agent", + "Parameters", + "TotalUsage", + "LastMessageAt", + "HasMoreMessages", + "SubConversationIds", + "Attachments", + "Messages", + ], + keys, + ) + + def test_missing_key_defaults(self): + result = AiConversationMessagesResult.from_json({"ConversationId": "c"}) + self.assertEqual("c", result.conversation_id) + self.assertIsNone(result.agent) + self.assertIsNone(result.parameters) + self.assertIsNone(result.total_usage) + self.assertIsNone(result.last_message_at) + self.assertIsNone(result.messages) + self.assertFalse(result.has_more_messages) + self.assertIsNone(result.sub_conversation_ids) + self.assertIsNone(result.attachments) + + +class TestAiOperationsIntegration(unittest.TestCase): + def test_get_conversation_messages_reachable_from_store(self): + self.assertTrue(hasattr(AiOperations, "get_conversation_messages")) + + def test_ai_operations_exported_from_package(self): + import ravendb + + self.assertTrue(hasattr(ravendb, "AiConversationMessagesResult")) + self.assertTrue(hasattr(ravendb, "GetConversationMessagesOperation")) + self.assertTrue(hasattr(ravendb, "GetConversationMessagesOptions")) + self.assertTrue(hasattr(ravendb, "AiMessageRole")) + self.assertTrue(hasattr(ravendb, "AiConversationDetailLevel")) + + +class TestCancelPendingActionTools(unittest.TestCase): + def _command(self, cancel=False, debug=None): + operation = RunConversationOperation( + agent_id="a", + conversation_id="c", + debug=debug, + cancel_pending_action_tools=cancel, + ) + return operation.get_command(None) + + def test_flag_always_on_wire_after_debug(self): + url = self._command().create_request(ServerNode("http://localhost:8080", "db1")).url + self.assertTrue(url.endswith("&cancelPendingActionTools=False"), url) + + def test_flag_after_debug_parameter(self): + url = self._command(debug=True).create_request(ServerNode("http://localhost:8080", "db1")).url + self.assertIn("&debug=True&cancelPendingActionTools=False", url) + + def test_flag_true_when_set(self): + url = self._command(cancel=True).create_request(ServerNode("http://localhost:8080", "db1")).url + self.assertTrue(url.endswith("&cancelPendingActionTools=True"), url) + + +class TestStreamingCancellation(unittest.TestCase): + def test_cancelled_stream_stops_without_reading_rest(self): + from ravendb.http.misc import ResponseDisposeHandling + from ravendb.http.http_cache import HttpCache + + lines_read = [] + + class FakeResponse: + def iter_lines(self, decode_unicode=True): + for i in range(100): + lines_read.append(i) + # Real streamed chunks are JSON-encoded strings, not bare objects. + yield f'"chunk {i}"' + + def close(self): + pass + + event = threading.Event() + command = RunConversationCommand( + agent_id="a", + conversation_id="c", + stream_property_path="answer", + streamed_chunks_callback=lambda chunk: event.set(), + cancellation_event=event, + ) + response = FakeResponse() + with self.assertRaises(Exception) as ctx: + command.process_response(HttpCache(), response, "http://x") + self.assertIsInstance(ctx.exception, Exception) + self.assertLess(len(lines_read), 100, "the stream must not be fully consumed") + + def test_no_cancellation_event_reads_to_end(self): + from ravendb.http.misc import ResponseDisposeHandling + from ravendb.http.http_cache import HttpCache + + class FakeResponse: + def iter_lines(self, decode_unicode=True): + yield '{"a": 1}' + yield '{"a": 2}' + + def close(self): + pass + + command = RunConversationCommand( + agent_id="a", + conversation_id="c", + stream_property_path="answer", + streamed_chunks_callback=lambda chunk: None, + ) + handling = command.process_response(HttpCache(), FakeResponse(), "http://x") + self.assertEqual(ResponseDisposeHandling.AUTOMATIC, handling) + + +if __name__ == "__main__": + unittest.main() From 30cef13dc35d8654e15506d157f6eaceb7ed728d Mon Sep 17 00:00:00 2001 From: reforge Date: Wed, 19 Aug 2026 01:59:05 -0300 Subject: [PATCH 6/7] Add SSO certificate metadata and edit parameters Add usage, sso_server_public_key_pinning_hashes, allow_any_sso_server, and sso_identifiers to CertificateMetadata, with the CertificateUsage and SsoProvider enums and the SsoIdentifier class. CertificateDefinition.to_json writes the SSO keys after Disabled; from_json defaults a missing Usage to None, missing lists to [], and missing AllowAnySsoServer to False. EditClientCertificateOperation.Parameters gains the three nullable SSO fields. The edit body is written manually with conditional SSO keys: a field is written only when provided (an empty list clears the stored value), and each SsoIdentifier writes Domain only when non-empty. The body is never built from CertificateDefinition.to_json, whose unconditional SSO defaults would wipe a stored SSO configuration on a plain permission edit. Reforge-Run: 20260819T014403Z-2668162-reforge --- ravendb/__init__.py | 3 + ravendb/serverwide/operations/certificates.py | 143 +++++++++++++- .../serverwide_tests/test_certificates_sso.py | 185 ++++++++++++++++++ 3 files changed, 322 insertions(+), 9 deletions(-) create mode 100644 ravendb/tests/serverwide_tests/test_certificates_sso.py diff --git a/ravendb/__init__.py b/ravendb/__init__.py index ac2e18ed..156cfd10 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -381,6 +381,9 @@ GetCertificatesResponse, PutClientCertificateOperation, SecurityClearance, + CertificateUsage, + SsoProvider, + SsoIdentifier, ) from ravendb.serverwide.operations.common import ( BuildNumber, diff --git a/ravendb/serverwide/operations/certificates.py b/ravendb/serverwide/operations/certificates.py index 29a22cba..9cb4c0c7 100644 --- a/ravendb/serverwide/operations/certificates.py +++ b/ravendb/serverwide/operations/certificates.py @@ -4,7 +4,7 @@ import enum import json from datetime import datetime -from typing import TYPE_CHECKING, Dict, List +from typing import TYPE_CHECKING, Dict, List, Optional import requests @@ -39,6 +39,51 @@ def __str__(self): return self.value +class CertificateUsage(enum.Enum): + RAVEN_SERVER = "RavenServer" + RAVEN_SERVER_FOR_COMMUNICATION = "RavenServerForCommunication" + CLIENT = "Client" + SSO_SERVER = "SsoServer" + SSO_CLIENT = "SsoClient" + WELL_KNOWN_ISSUER = "WellKnownIssuer" + + def __str__(self): + return self.value + + +class SsoProvider(enum.Enum): + GITHUB = "Github" + GOOGLE = "Google" + MICROSOFT = "Microsoft" + WINDOWS = "Windows" + + def __str__(self): + return self.value + + +class SsoIdentifier: + def __init__(self, provider: SsoProvider = None, domain: str = None, identifier: str = None): + self.provider = provider + self.domain = domain + self.identifier = identifier + + def to_json(self) -> dict: + return { + "Provider": self.provider.value if self.provider else None, + "Domain": self.domain, + "Identifier": self.identifier, + } + + @classmethod + def from_json(cls, json_dict: dict) -> SsoIdentifier: + provider_value = json_dict.get("Provider") + return cls( + provider=SsoProvider(provider_value) if provider_value else None, + domain=json_dict.get("Domain"), + identifier=json_dict.get("Identifier"), + ) + + class CertificateRawData: def __init__(self, raw_data: bytes = None): self.raw_data = raw_data @@ -57,6 +102,10 @@ def __init__( public_key_pinning_hash: str = None, not_before: datetime = None, disabled: bool = False, + usage: CertificateUsage = None, + sso_server_public_key_pinning_hashes: List[str] = None, + allow_any_sso_server: bool = False, + sso_identifiers: List[SsoIdentifier] = None, ): self.name = name self.security_clearance = security_clearance @@ -68,6 +117,12 @@ def __init__( self.public_key_pinning_hash = public_key_pinning_hash self.not_before = not_before self.disabled = disabled + self.usage = usage + self.sso_server_public_key_pinning_hashes = ( + sso_server_public_key_pinning_hashes if sso_server_public_key_pinning_hashes is not None else [] + ) + self.allow_any_sso_server = allow_any_sso_server + self.sso_identifiers = sso_identifiers if sso_identifiers is not None else [] @classmethod def from_json(cls, json_dict: dict) -> CertificateMetadata: @@ -82,6 +137,14 @@ def from_json(cls, json_dict: dict) -> CertificateMetadata: json_dict.get("PublicKeyPinningHash", None), Utils.string_to_datetime(json_dict["NotBefore"]) if "NotBefore" in json_dict else None, json_dict.get("Disabled", False), + CertificateUsage(json_dict["Usage"]) if json_dict.get("Usage") else None, + json_dict.get("SsoServerPublicKeyPinningHashes") or [], + json_dict.get("AllowAnySsoServer", False), + ( + [SsoIdentifier.from_json(sso) for sso in json_dict["SsoIdentifiers"]] + if json_dict.get("SsoIdentifiers") + else [] + ), ) @@ -99,6 +162,10 @@ def __init__( collection_primary_key: str = None, public_key_pinning_hash: str = None, disabled: bool = False, + usage: CertificateUsage = None, + sso_server_public_key_pinning_hashes: List[str] = None, + allow_any_sso_server: bool = False, + sso_identifiers: List[SsoIdentifier] = None, ): super().__init__( name, @@ -110,6 +177,10 @@ def __init__( collection_primary_key, public_key_pinning_hash, disabled=disabled, + usage=usage, + sso_server_public_key_pinning_hashes=sso_server_public_key_pinning_hashes, + allow_any_sso_server=allow_any_sso_server, + sso_identifiers=sso_identifiers, ) self.certificate = certificate self.password = password @@ -126,6 +197,10 @@ def to_json(self) -> dict: "Certificate": self.certificate, "Password": self.password, "Disabled": self.disabled, + "Usage": self.usage.value if self.usage else None, + "SsoServerPublicKeyPinningHashes": self.sso_server_public_key_pinning_hashes, + "AllowAnySsoServer": self.allow_any_sso_server, + "SsoIdentifiers": [sso.to_json() for sso in self.sso_identifiers] if self.sso_identifiers else [], } if self.not_after: json_dict.update({"NotAfter": Utils.datetime_to_string(self.not_after)}) @@ -145,6 +220,14 @@ def from_json(cls, json_dict: dict) -> CertificateDefinition: json_dict["CollectionPrimaryKey"], json_dict["PublicKeyPinningHash"], disabled=json_dict.get("Disabled", False), + usage=CertificateUsage(json_dict["Usage"]) if json_dict.get("Usage") else None, + sso_server_public_key_pinning_hashes=json_dict.get("SsoServerPublicKeyPinningHashes") or [], + allow_any_sso_server=json_dict.get("AllowAnySsoServer", False), + sso_identifiers=( + [SsoIdentifier.from_json(sso) for sso in json_dict["SsoIdentifiers"]] + if json_dict.get("SsoIdentifiers") + else [] + ), ) @@ -456,12 +539,18 @@ def __init__( name: str, clearance: SecurityClearance, disabled: bool = False, + sso_server_public_key_pinning_hashes: Optional[List[str]] = None, + allow_any_sso_server: Optional[bool] = None, + sso_identifiers: Optional[List[SsoIdentifier]] = None, ): self.thumbprint = thumbprint self.permissions = permissions self.name = name self.clearance = clearance self.disabled = disabled + self.sso_server_public_key_pinning_hashes = sso_server_public_key_pinning_hashes + self.allow_any_sso_server = allow_any_sso_server + self.sso_identifiers = sso_identifiers def __init__(self, parameters: Parameters): if parameters is None: @@ -481,10 +570,20 @@ def __init__(self, parameters: Parameters): self.__permissions = parameters.permissions self.__clearance = parameters.clearance self.__disabled = parameters.disabled + self.__sso_server_public_key_pinning_hashes = parameters.sso_server_public_key_pinning_hashes + self.__allow_any_sso_server = parameters.allow_any_sso_server + self.__sso_identifiers = parameters.sso_identifiers def get_command(self, conventions: "DocumentConventions") -> "VoidRavenCommand": return self.__EditCertificateClientCommand( - self.__thumbprint, self.__name, self.__permissions, self.__clearance, self.__disabled + self.__thumbprint, + self.__name, + self.__permissions, + self.__clearance, + self.__disabled, + self.__sso_server_public_key_pinning_hashes, + self.__allow_any_sso_server, + self.__sso_identifiers, ) class __EditCertificateClientCommand(VoidRavenCommand, RaftCommand): @@ -495,6 +594,9 @@ def __init__( permissions: Dict[str, DatabaseAccess], clearance: SecurityClearance, disabled: bool, + sso_server_public_key_pinning_hashes: Optional[List[str]] = None, + allow_any_sso_server: Optional[bool] = None, + sso_identifiers: Optional[List[SsoIdentifier]] = None, ): super().__init__() self.__thumbprint = thumbprint @@ -502,6 +604,9 @@ def __init__( self.__permissions = permissions self.__clearance = clearance self.__disabled = disabled + self.__sso_server_public_key_pinning_hashes = sso_server_public_key_pinning_hashes + self.__allow_any_sso_server = allow_any_sso_server + self.__sso_identifiers = sso_identifiers def is_read_request(self) -> bool: return False @@ -509,15 +614,35 @@ def is_read_request(self) -> bool: def create_request(self, node: ServerNode) -> requests.Request: url = f"{node.url}/admin/certificates/edit" - definition = CertificateDefinition() - definition.thumbprint = self.__thumbprint - definition.permissions = self.__permissions - definition.security_clearance = self.__clearance - definition.name = self.__name - definition.disabled = self.__disabled + # The body is written manually, never from CertificateDefinition.to_json: the SSO + # keys must be present only when the caller provided them, because the server's + # edit handler treats a present key as an explicit replacement (even an empty list + # clears the stored value) and an absent key as "leave untouched". + body = { + "Thumbprint": self.__thumbprint, + "Name": self.__name, + "SecurityClearance": self.__clearance.value if self.__clearance else None, + "Disabled": self.__disabled, + "Permissions": {key: str(value) for key, value in self.__permissions.items()}, + } + if self.__sso_server_public_key_pinning_hashes is not None: + body["SsoServerPublicKeyPinningHashes"] = self.__sso_server_public_key_pinning_hashes + if self.__allow_any_sso_server is not None: + body["AllowAnySsoServer"] = self.__allow_any_sso_server + if self.__sso_identifiers is not None: + sso_ids = [] + for sso_id in self.__sso_identifiers: + entry = { + "Provider": sso_id.provider.value if sso_id.provider else None, + "Identifier": sso_id.identifier, + } + if sso_id.domain: + entry["Domain"] = sso_id.domain + sso_ids.append(entry) + body["SsoIdentifiers"] = sso_ids request = requests.Request("POST", url) - request.data = definition.to_json() + request.data = body return request diff --git a/ravendb/tests/serverwide_tests/test_certificates_sso.py b/ravendb/tests/serverwide_tests/test_certificates_sso.py new file mode 100644 index 00000000..e9396d53 --- /dev/null +++ b/ravendb/tests/serverwide_tests/test_certificates_sso.py @@ -0,0 +1,185 @@ +"""Tests for the SSO certificate surface: CertificateMetadata +SSO fields, the CertificateUsage/SsoProvider enums, SsoIdentifier, and the +edit-operation body rules. +""" + +import unittest + +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.operations.certificates import ( + CertificateDefinition, + CertificateMetadata, + CertificateUsage, + DatabaseAccess, + EditClientCertificateOperation, + SecurityClearance, + SsoIdentifier, + SsoProvider, +) + + +class TestSsoEnums(unittest.TestCase): + def test_certificate_usage_values_are_csharp_names(self): + self.assertEqual( + ["RavenServer", "RavenServerForCommunication", "Client", "SsoServer", "SsoClient", "WellKnownIssuer"], + [m.value for m in CertificateUsage], + ) + + def test_sso_provider_values_are_csharp_names(self): + self.assertEqual(["Github", "Google", "Microsoft", "Windows"], [m.value for m in SsoProvider]) + + def test_sso_identifier_round_trip(self): + sso = SsoIdentifier(provider=SsoProvider.GITHUB, domain="github.com", identifier="alice") + self.assertEqual({"Provider": "Github", "Domain": "github.com", "Identifier": "alice"}, sso.to_json()) + back = SsoIdentifier.from_json({"Provider": "Google", "Domain": None, "Identifier": "bob"}) + self.assertEqual(SsoProvider.GOOGLE, back.provider) + self.assertIsNone(back.domain) + self.assertEqual("bob", back.identifier) + + +class TestCertificateMetadataSsoFields(unittest.TestCase): + def test_to_json_writes_sso_keys(self): + definition = CertificateDefinition( + name="n", + usage=CertificateUsage.SSO_CLIENT, + sso_server_public_key_pinning_hashes=["hash1"], + allow_any_sso_server=True, + sso_identifiers=[SsoIdentifier(provider=SsoProvider.WINDOWS, domain="corp", identifier="win\\u")], + ) + result = definition.to_json() + self.assertEqual("SsoClient", result["Usage"]) + self.assertEqual(["hash1"], result["SsoServerPublicKeyPinningHashes"]) + self.assertIs(True, result["AllowAnySsoServer"]) + self.assertEqual([{"Provider": "Windows", "Domain": "corp", "Identifier": "win\\u"}], result["SsoIdentifiers"]) + + def test_to_json_usage_null_and_defaults(self): + definition = CertificateDefinition(name="n") + result = definition.to_json() + self.assertIsNone(result["Usage"]) + self.assertEqual([], result["SsoServerPublicKeyPinningHashes"]) + self.assertIs(False, result["AllowAnySsoServer"]) + self.assertEqual([], result["SsoIdentifiers"]) + + def test_from_json_parses_sso_fields(self): + metadata = CertificateMetadata.from_json( + { + "Name": "n", + "SecurityClearance": "ClusterAdmin", + "Usage": "SsoServer", + "SsoServerPublicKeyPinningHashes": ["h"], + "AllowAnySsoServer": True, + "SsoIdentifiers": [{"Provider": "Github", "Domain": None, "Identifier": "id"}], + } + ) + self.assertEqual(CertificateUsage.SSO_SERVER, metadata.usage) + self.assertEqual(["h"], metadata.sso_server_public_key_pinning_hashes) + self.assertIs(True, metadata.allow_any_sso_server) + self.assertEqual(1, len(metadata.sso_identifiers)) + self.assertEqual(SsoProvider.GITHUB, metadata.sso_identifiers[0].provider) + + def test_from_json_missing_sso_keys_use_defaults(self): + metadata = CertificateMetadata.from_json({"Name": "n", "SecurityClearance": "ClusterAdmin"}) + self.assertIsNone(metadata.usage) + self.assertEqual([], metadata.sso_server_public_key_pinning_hashes) + self.assertIs(False, metadata.allow_any_sso_server) + self.assertEqual([], metadata.sso_identifiers) + + def test_definition_from_json_parses_sso_fields(self): + definition = CertificateDefinition.from_json( + { + "Certificate": None, + "Name": "n", + "SecurityClearance": "ClusterAdmin", + "Thumbprint": "tp", + "NotAfter": None, + "Permissions": {"db": "ReadWrite"}, + "CollectionSecondaryKeys": [], + "CollectionPrimaryKey": "", + "PublicKeyPinningHash": None, + "Usage": "Client", + "SsoServerPublicKeyPinningHashes": ["h"], + "AllowAnySsoServer": False, + "SsoIdentifiers": [], + } + ) + self.assertEqual(CertificateUsage.CLIENT, definition.usage) + self.assertEqual(["h"], definition.sso_server_public_key_pinning_hashes) + + +class TestEditClientCertificateOperationSso(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db1") + + def _body(self, **parameters_kwargs): + parameters = EditClientCertificateOperation.Parameters( + thumbprint="tp", + permissions={"db1": DatabaseAccess.ADMIN}, + name="n", + clearance=SecurityClearance.CLUSTER_ADMIN, + **parameters_kwargs, + ) + operation = EditClientCertificateOperation(parameters) + command = operation.get_command(None) + request = command.create_request(self.node) + return request, command + + def test_base_keys_always_present(self): + request, command = self._body() + self.assertEqual("POST", request.method) + self.assertEqual("http://localhost:8080/admin/certificates/edit", request.url) + body = request.data + self.assertEqual("tp", body["Thumbprint"]) + self.assertEqual("n", body["Name"]) + self.assertEqual("ClusterAdmin", body["SecurityClearance"]) + self.assertIs(False, body["Disabled"]) + self.assertEqual({"db1": "Admin"}, body["Permissions"]) + + def test_sso_keys_omitted_when_none(self): + request, _ = self._body() + body = request.data + self.assertNotIn("SsoServerPublicKeyPinningHashes", body) + self.assertNotIn("AllowAnySsoServer", body) + self.assertNotIn("SsoIdentifiers", body) + + def test_sso_keys_written_when_provided(self): + request, _ = self._body( + sso_server_public_key_pinning_hashes=["h1", "h2"], + allow_any_sso_server=True, + sso_identifiers=[SsoIdentifier(provider=SsoProvider.GOOGLE, domain="g.com", identifier="id")], + ) + body = request.data + self.assertEqual(["h1", "h2"], body["SsoServerPublicKeyPinningHashes"]) + self.assertIs(True, body["AllowAnySsoServer"]) + self.assertEqual([{"Provider": "Google", "Identifier": "id", "Domain": "g.com"}], body["SsoIdentifiers"]) + + def test_empty_list_sent_to_clear(self): + request, _ = self._body(sso_server_public_key_pinning_hashes=[]) + self.assertEqual([], request.data["SsoServerPublicKeyPinningHashes"]) + + def test_domain_omitted_when_empty_in_edit_body(self): + request, _ = self._body( + sso_identifiers=[SsoIdentifier(provider=SsoProvider.WINDOWS, domain="", identifier="id")] + ) + self.assertEqual([{"Provider": "Windows", "Identifier": "id"}], request.data["SsoIdentifiers"]) + + def test_body_not_built_from_certificate_definition_to_json(self): + # The edit body must never carry the unconditional SSO keys that + # CertificateDefinition.to_json writes for its own serialization. + request, _ = self._body() + body = request.data + for key in ("Usage", "SsoServerPublicKeyPinningHashes", "AllowAnySsoServer", "SsoIdentifiers"): + self.assertNotIn(key, body) + + def test_command_is_raft_command(self): + request, command = self._body() + self.assertIsInstance(command, RaftCommand) + self.assertTrue(command.get_raft_unique_request_id()) + + def test_constructor_validation(self): + with self.assertRaises(ValueError): + EditClientCertificateOperation(None) + + +if __name__ == "__main__": + unittest.main() From 940f2968cba7f8fdd52c5200d5f2f5b48e1e67e1 Mon Sep 17 00:00:00 2001 From: reforge Date: Wed, 19 Aug 2026 01:59:05 -0300 Subject: [PATCH 7/7] Fix session cluster-transaction change-vector handling Capture the Database-Cluster-Tx-Id response header into SessionInfo.cluster_transaction_id in the request-executor success path, guarded on the header's presence, before process_response. The session needs the cluster id for the change-vector fix, mirroring the reference client's SessionInfo chain. UpdateEntityDocumentInfo splits a document's change vector on '|' when the session has a cluster-transaction id: more than two parts throw with the document id and vector in the message, otherwise the etag for the cluster id is read from the LAST part only and last_cluster_transaction_index advances to the max of the current value and the etag. A null change vector is a no-op. ClientChangeVectorUtils carries the separator and the GetEtagById logic (etag between the last ':' and the '-{id}' marker, 0 when the id is absent). Reforge-Run: 20260819T014403Z-2668162-reforge --- .../in_memory_document_session_operations.py | 17 +++ ravendb/documents/session/misc.py | 1 + ravendb/http/request_executor.py | 4 + .../test_cluster_transaction_change_vector.py | 137 ++++++++++++++++++ ...r_transaction_change_vector_integration.py | 38 +++++ ravendb/util/client_change_vector_utils.py | 32 ++++ 6 files changed, 229 insertions(+) create mode 100644 ravendb/tests/session_tests/test_cluster_transaction_change_vector.py create mode 100644 ravendb/tests/session_tests/test_cluster_transaction_change_vector_integration.py create mode 100644 ravendb/util/client_change_vector_utils.py diff --git a/ravendb/documents/session/document_session_operations/in_memory_document_session_operations.py b/ravendb/documents/session/document_session_operations/in_memory_document_session_operations.py index 35f42d95..686fa59a 100644 --- a/ravendb/documents/session/document_session_operations/in_memory_document_session_operations.py +++ b/ravendb/documents/session/document_session_operations/in_memory_document_session_operations.py @@ -2062,6 +2062,23 @@ def remove_document_by_entity(self, entity): self.__documents_by_entity_to_remove.append(entity) def update_entity_document_info(self, document_info: DocumentInfo, document: dict): + session_info = self.__session.session_info + cluster_id = session_info.cluster_transaction_id if session_info else None + if cluster_id is not None and document_info.change_vector is not None: + from ravendb.util.client_change_vector_utils import ClientChangeVectorUtils + + cv = document_info.change_vector.split(ClientChangeVectorUtils.SEPARATOR) + if len(cv) > 2: + raise InvalidOperationException( + f"The document '{document_info.key}' has invalid change vector " + f"'{document_info.change_vector}'" + ) + + cluster_tx_index = ClientChangeVectorUtils.get_etag_by_id(cv[-1], cluster_id) + if cluster_tx_index > 0: + session_info.last_cluster_transaction_index = max( + session_info.last_cluster_transaction_index or 0, cluster_tx_index + ) self.__document_infos_to_update.append((document_info, document)) def clear_session_state_after_successful_save_changes(self): diff --git a/ravendb/documents/session/misc.py b/ravendb/documents/session/misc.py index 84d201e1..07697321 100644 --- a/ravendb/documents/session/misc.py +++ b/ravendb/documents/session/misc.py @@ -74,6 +74,7 @@ def __init__( self.no_caching = options.no_caching self.last_cluster_transaction_index: Union[None, int] = None + self.cluster_transaction_id: Union[None, str] = None @property def can_use_load_balance_behavior(self) -> bool: diff --git a/ravendb/http/request_executor.py b/ravendb/http/request_executor.py index 360da4dc..b0911175 100644 --- a/ravendb/http/request_executor.py +++ b/ravendb/http/request_executor.py @@ -660,6 +660,10 @@ def execute( self._throw_failed_to_contact_all_nodes(command, request) return # we either handled this already in the unsuccessful response or we are throwing self._on_succeed_request_invoke(self._database_name, url, response, request, attempt_num) + if session_info is not None and constants.Headers.DATABASE_CLUSTER_TRANSACTION_ID in response.headers: + session_info.cluster_transaction_id = response.headers[ + constants.Headers.DATABASE_CLUSTER_TRANSACTION_ID + ] response_dispose = command.process_response(self._cache, response, url) self._last_returned_response = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) finally: diff --git a/ravendb/tests/session_tests/test_cluster_transaction_change_vector.py b/ravendb/tests/session_tests/test_cluster_transaction_change_vector.py new file mode 100644 index 00000000..a2cbc543 --- /dev/null +++ b/ravendb/tests/session_tests/test_cluster_transaction_change_vector.py @@ -0,0 +1,137 @@ +"""Tests for the session cluster-transaction change-vector fix: +the Database-Cluster-Tx-Id header capture, the last-part etag rule, the +>2-parts throw, and the null change-vector no-op. +""" + +import unittest + +from ravendb.documents.session.document_info import DocumentInfo +from ravendb.documents.session.document_session_operations.in_memory_document_session_operations import ( + InMemoryDocumentSessionOperations, +) +from ravendb.documents.session.misc import SessionInfo +from ravendb.exceptions.exceptions import InvalidOperationException +from ravendb.primitives import constants +from ravendb.util.client_change_vector_utils import ClientChangeVectorUtils + + +class TestClientChangeVectorUtils(unittest.TestCase): + def test_etag_between_last_colon_and_dash(self): + # Format of a cluster-transaction segment: 'Trxn:{index}-{clusterId}'. + self.assertEqual(15, ClientChangeVectorUtils.get_etag_by_id("Trxn:15-A:42", "A:42")) + + def test_no_match_returns_zero(self): + self.assertEqual(0, ClientChangeVectorUtils.get_etag_by_id("A:1-node:2", "A:42")) + self.assertEqual(0, ClientChangeVectorUtils.get_etag_by_id("Trxn:0-A:42", "A:42")) + + def test_null_change_vector_returns_zero(self): + self.assertEqual(0, ClientChangeVectorUtils.get_etag_by_id(None, "A:42")) + + def test_separator_guard(self): + with self.assertRaises(ValueError): + ClientChangeVectorUtils.get_etag_by_id("a|b", "x") + + +def _new_actions(session_info): + actions = InMemoryDocumentSessionOperations.SaveChangesData.ActionsToRunOnSuccess.__new__( + InMemoryDocumentSessionOperations.SaveChangesData.ActionsToRunOnSuccess + ) + session = object.__new__(InMemoryDocumentSessionOperations) + session.session_info = session_info + actions._ActionsToRunOnSuccess__session = session + actions._ActionsToRunOnSuccess__document_infos_to_update = [] + return actions + + +class TestUpdateEntityDocumentInfo(unittest.TestCase): + def test_last_part_etag_advances_last_cluster_transaction_index(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = "A:42" + session_info.last_cluster_transaction_index = None + + actions = _new_actions(session_info) + info = DocumentInfo(key="users/1", change_vector="A:1-node:2|Trxn:15-A:42") + actions.update_entity_document_info(info, {"Name": "u"}) + + self.assertEqual(15, session_info.last_cluster_transaction_index) + self.assertEqual(1, len(actions._ActionsToRunOnSuccess__document_infos_to_update)) + + def test_etag_read_from_last_part_only(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = "A:42" + session_info.last_cluster_transaction_index = None + + actions = _new_actions(session_info) + # The first part also contains 'A:42'; only the last part may contribute. + info = DocumentInfo(key="users/1", change_vector="A:5-A:42|Trxn:7-A:42") + actions.update_entity_document_info(info, {}) + + self.assertEqual(7, session_info.last_cluster_transaction_index) + + def test_max_with_existing_index(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = "A:42" + session_info.last_cluster_transaction_index = 20 + + actions = _new_actions(session_info) + actions.update_entity_document_info(DocumentInfo(key="users/1", change_vector="A:1-node:2|Trxn:15-A:42"), {}) + self.assertEqual(20, session_info.last_cluster_transaction_index) + + actions.update_entity_document_info(DocumentInfo(key="users/2", change_vector="A:1-node:2|Trxn:25-A:42"), {}) + self.assertEqual(25, session_info.last_cluster_transaction_index) + + def test_more_than_two_parts_throws(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = "A:42" + session_info.last_cluster_transaction_index = None + + actions = _new_actions(session_info) + with self.assertRaises(InvalidOperationException) as ctx: + actions.update_entity_document_info(DocumentInfo(key="users/1", change_vector="A:1|B:2|Trxn:3-A:42"), {}) + self.assertEqual("The document 'users/1' has invalid change vector 'A:1|B:2|Trxn:3-A:42'", str(ctx.exception)) + + def test_null_change_vector_is_no_op(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = "A:42" + session_info.last_cluster_transaction_index = None + + actions = _new_actions(session_info) + actions.update_entity_document_info(DocumentInfo(key="users/1", change_vector=None), {}) + self.assertIsNone(session_info.last_cluster_transaction_index) + + def test_no_cluster_id_is_no_op(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = None + session_info.last_cluster_transaction_index = None + + actions = _new_actions(session_info) + actions.update_entity_document_info(DocumentInfo(key="users/1", change_vector="A:1-node:2"), {}) + self.assertIsNone(session_info.last_cluster_transaction_index) + + def test_split_separator_is_pipe(self): + self.assertEqual("|", ClientChangeVectorUtils.SEPARATOR) + + def test_session_info_has_cluster_transaction_id_field(self): + session_info = SessionInfo.__new__(SessionInfo) + session_info.cluster_transaction_id = None + self.assertIsNone(session_info.cluster_transaction_id) + + +class TestClusterTransactionHeaderCapture(unittest.TestCase): + def test_constant_is_database_cluster_tx_id(self): + self.assertEqual("Database-Cluster-Tx-Id", constants.Headers.DATABASE_CLUSTER_TRANSACTION_ID) + + def test_request_executor_captures_header_in_success_path(self): + from ravendb.http.request_executor import RequestExecutor + + source = open("ravendb/http/request_executor.py").read() + # The capture must sit in the success path, before process_response. + success_index = source.index("command.process_response(self._cache, response, url)") + header_index = source.index("DATABASE_CLUSTER_TRANSACTION_ID in response.headers") + self.assertLess(header_index, success_index) + # It must guard the absent header. + self.assertIn("DATABASE_CLUSTER_TRANSACTION_ID in response.headers", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/session_tests/test_cluster_transaction_change_vector_integration.py b/ravendb/tests/session_tests/test_cluster_transaction_change_vector_integration.py new file mode 100644 index 00000000..a050e11f --- /dev/null +++ b/ravendb/tests/session_tests/test_cluster_transaction_change_vector_integration.py @@ -0,0 +1,38 @@ +"""Integration test for the cluster-transaction change-vector fix. + +After saving a document in a CLUSTER_WIDE transaction, the session's +last_cluster_transaction_index must be advanced to the cluster-transaction etag +of the stored document (the etag read from the last '|'-separated part of the +change vector). Skipped when RAVENDB_LICENSE is not set, following the +AI-agent/CDC integration pattern. +""" + +import os +import unittest + +from ravendb.documents.session.misc import SessionOptions, TransactionMode +from ravendb.infrastructure.entities import User +from ravendb.tests.test_base import TestBase + + +@unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") +class TestClusterTransactionChangeVector(TestBase): + def test_last_cluster_transaction_index_advanced_after_cluster_transaction(self): + user = User(name="Karmel") + + session_options = SessionOptions(transaction_mode=TransactionMode.CLUSTER_WIDE) + session_options.disable_atomic_document_writes_in_cluster_wide_transaction = True + + with self.store.open_session(session_options=session_options) as session: + session.store(user, "users/1") + session.save_changes() + + # The Database-Cluster-Tx-Id header captured on the save response gives the + # cluster id; the stored change vector carries the cluster-transaction segment + # as its last '|'-separated part, so the session index must advance. + self.assertGreater(session.session_info.last_cluster_transaction_index or 0, 0) + self.assertIsNotNone(session.session_info.cluster_transaction_id) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/util/client_change_vector_utils.py b/ravendb/util/client_change_vector_utils.py new file mode 100644 index 00000000..4677b410 --- /dev/null +++ b/ravendb/util/client_change_vector_utils.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from typing import Optional + + +class ClientChangeVectorUtils: + SEPARATOR = "|" + + @staticmethod + def get_etag_by_id(change_vector: Optional[str], id: str) -> int: + """Returns the etag for the given cluster-transaction id in a change-vector + part, or 0 when the id is not present (C# GetEtagById).""" + if change_vector is None: + return 0 + + if id is None: + raise ValueError("id cannot be None") + + if ClientChangeVectorUtils.SEPARATOR in change_vector: + raise ValueError( + f"Change vector contains '{ClientChangeVectorUtils.SEPARATOR}', " + "which is not supported for this operation." + ) + + index = change_vector.find("-" + id) + if index == -1: + return 0 + + end = index - 1 + start = change_vector.rfind(":", 0, end + 1) + 1 + + return int(change_vector[start : end + 1])