RDBC-889 7.0 Python client with Vector API + half of the tests

Author: poissoncorp

.github/workflows/RavenClient.yml

@@ -2,9 +2,9 @@ name: tests/python
 
 on:
   push:
-    branches: [v5.2]
+    branches: [v7.0]
   pull_request:
-    branches: [v5.2]
+    branches: [v7.0]
   schedule:
     - cron: '0 10 * * *'
   workflow_dispatch:
@@ -30,7 +30,7 @@ jobs:
       strategy:
         matrix:
           python-version: [ '3.8', '3.9', '3.10' ,'3.11', '3.12']
-          serverVersion: [ '5.4', '6.2', '7.0' ]
+          serverVersion: [ '7.0' ]
         fail-fast: false
 
       steps:

README.md

@@ -7,7 +7,7 @@ Install from [PyPi](https://pypi.python.org/pypi), as [ravendb](https://pypi.org
 pip install ravendb
 ````
 ## Introduction and changelog
-Python client API (v5.2) for [RavenDB](https://ravendb.net/) , a NoSQL document database.
+Python client API (v7.0) for [RavenDB](https://ravendb.net/) , a NoSQL document database.
 
 Although new API isn't compatible with the previous one, it comes with **many improvements and new features**.
 
@@ -17,8 +17,6 @@ Although new API isn't compatible with the previous one, it comes with **many im
 
 ## Releases
 
-* All client versions 5.2.x are fully compatible with and support RavenDB server releases 5.4 and 6.0.
-
 * [Click here](https://github.com/ravendb/ravendb-python-client/releases) to view all Releases and Changelog.
 
 ---

README_pypi.md

@@ -8,7 +8,7 @@ pip install ravendb
 ```
 
 ## Introduction
-Python client API (v5.2) for [RavenDB](https://ravendb.net/) , a NoSQL document database.
+Python client API (v7.0) for [RavenDB](https://ravendb.net/) , a NoSQL document database.
 
 Although new API isn't compatible with the previous one, it comes with **many improvements and new features**.
 
@@ -19,8 +19,6 @@ Although new API isn't compatible with the previous one, it comes with **many im
 
 ## Releases
 
-* All client versions 5.2.x are fully compatible with and support RavenDB server releases 5.4 and 6.0.
-
 * [Click here](https://github.com/ravendb/ravendb-python-client/releases) to view all Releases and Changelog.
 
 ## What's new?

ravendb/documents/indexes/abstract_index_creation_tasks.py

@@ -1,5 +1,5 @@
 from abc import abstractmethod, ABC
-from typing import Generic, TypeVar, Union, Dict, Set, Callable, Optional, List, TYPE_CHECKING
+from typing import Generic, TypeVar, Union, Dict, Set, Callable, Optional, List, TYPE_CHECKING, Any
 
 from ravendb.documents.conventions import DocumentConventions
 from ravendb.documents.indexes.definitions import (
@@ -18,6 +18,7 @@
     IndexType,
 )
 from ravendb.documents.indexes.spatial.configuration import SpatialOptionsFactory
+from ravendb.documents.indexes.vector.options import VectorOptions
 from ravendb.documents.operations.indexes import PutIndexesOperation
 from ravendb.documents.store.definition import DocumentStoreBase
 from ravendb.primitives import constants
@@ -88,6 +89,7 @@ def __init__(self):
         self._index_suggestions: Set[str] = set()
         self._term_vectors_strings: Dict[str, FieldTermVector] = {}
         self._spatial_options_strings: Dict[str, SpatialOptions] = {}
+        self._vector_indexes_strings: Dict[str, VectorOptions] = {}
 
         self._output_reduce_to_collection: Union[None, str] = None
         self._pattern_for_output_reduce_to_collection_references: Union[None, str] = None
@@ -150,6 +152,9 @@ def _add_assembly(self, assembly: AdditionalAssembly) -> None:
 
         self.additional_assemblies.add(assembly)
 
+    def _vector(self, field: str, vector_options: VectorOptions) -> None:
+        self._vector_indexes_strings[field] = vector_options
+
 
 class AbstractIndexDefinitionBuilder(Generic[_T_IndexDefinition]):
     def __init__(self, index_name: str):
@@ -165,6 +170,7 @@ def __init__(self, index_name: str):
         self.suggestions_options: Set[str] = set()
         self.term_vectors_strings: Dict[str, FieldTermVector] = {}
         self.spatial_indexes_strings: Dict[str, SpatialOptions] = {}
+        self.vector_indexes_strings: Dict[str, VectorOptions] = {}
 
         self.lock_mode: Optional[IndexLockMode] = None
         self.priority: Optional[IndexLockMode] = None
@@ -191,7 +197,7 @@ def __apply_values(
         self,
         index_definition: IndexDefinition,
         values: Dict[str, object],
-        action: Callable[[IndexFieldOptions, object], None],
+        action: Callable[[IndexFieldOptions, Any], None],
     ) -> None:
         for key, value in values.items():
             field = index_definition.fields.get(key, IndexFieldOptions())
@@ -216,29 +222,33 @@ def to_index_definition(self, conventions: DocumentConventions, validate_map: bo
             for suggestions_option in self.suggestions_options:
                 suggestions[suggestions_option] = True
 
-            def __set_indexing(options, value):
+            def __set_indexing(options: IndexFieldOptions, value: FieldIndexing):
                 options.indexing = value
 
-            def __set_storage(options, value):
+            def __set_storage(options: IndexFieldOptions, value: FieldStorage):
                 options.storage = value
 
-            def __set_analyzer(options, value):
+            def __set_analyzer(options: IndexFieldOptions, value: str):
                 options.analyzer = value
 
-            def __set_term_vector(options, value):
+            def __set_term_vector(options: IndexFieldOptions, value: FieldTermVector):
                 options.term_vector = value
 
-            def __set_spatial(options, value):
+            def __set_spatial(options: IndexFieldOptions, value: SpatialOptions):
                 options.spatial = value
 
-            def __set_suggestions(options, value):
+            def __set_vector(options: IndexFieldOptions, value: VectorOptions):
+                options.vector = value
+
+            def __set_suggestions(options: IndexFieldOptions, value: bool):
                 options.suggestions = value
 
             self.__apply_values(index_definition, self.indexes_strings, __set_indexing)
             self.__apply_values(index_definition, self.stores_strings, __set_storage)
             self.__apply_values(index_definition, self.analyzers_strings, __set_analyzer)
             self.__apply_values(index_definition, self.term_vectors_strings, __set_term_vector)
             self.__apply_values(index_definition, self.spatial_indexes_strings, __set_spatial)
+            self.__apply_values(index_definition, self.vector_indexes_strings, __set_vector)
             self.__apply_values(index_definition, suggestions, __set_suggestions)
 
             index_definition.additional_sources = self.additional_sources
@@ -302,6 +312,7 @@ def create_index_definition(self) -> IndexDefinition:
         index_definition_builder.suggestions_options = self._index_suggestions
         index_definition_builder.term_vectors_strings = self._term_vectors_strings
         index_definition_builder.spatial_indexes_strings = self._spatial_options_strings
+        index_definition_builder.vector_indexes_strings = self._vector_indexes_strings
         index_definition_builder.output_reduce_to_collection = self._output_reduce_to_collection
         index_definition_builder.pattern_for_output_reduce_to_collection_references = (
             self._pattern_for_output_reduce_to_collection_references

ravendb/documents/indexes/definitions.py

@@ -6,6 +6,7 @@
 from abc import ABC
 from typing import Union, Optional, List, Dict, Set, Iterable
 from ravendb.documents.indexes.spatial.configuration import SpatialOptions, AutoSpatialOptions
+from ravendb.documents.indexes.vector.options import VectorOptions, AutoVectorOptions
 from ravendb.tools.utils import Utils
 
 
@@ -144,13 +145,15 @@ def __init__(
         indexing: Optional[FieldIndexing] = None,
         term_vector: Optional[FieldTermVector] = None,
         spatial: Optional[SpatialOptions] = None,
+        vector: Optional[VectorOptions] = None,
         analyzer: Optional[str] = None,
         suggestions: Optional[bool] = None,
     ):
         self.storage = storage
         self.indexing = indexing
         self.term_vector = term_vector
         self.spatial = spatial
+        self.vector = vector
         self.analyzer = analyzer
         self.suggestions = suggestions
 
@@ -160,6 +163,7 @@ def to_json(self):
             "Indexing": self.indexing,
             "TermVector": self.term_vector,
             "Spatial": self.spatial.to_json() if self.spatial else None,
+            "Vector": self.vector.to_json() if self.vector else None,
             "Analyzer": self.analyzer,
             "Suggestions": self.suggestions,
         }
@@ -370,6 +374,7 @@ def __init__(
         indexing: Optional[AutoFieldIndexing] = None,
         aggregation: Optional[AggregationOperation] = None,
         spatial: Optional[AutoSpatialOptions] = None,
+        vector: Optional[AutoVectorOptions] = None,
         group_by_array_behavior: Optional[GroupByArrayBehavior] = None,
         suggestions: Optional[bool] = None,
         is_name_quoted: Optional[bool] = None,
@@ -378,6 +383,7 @@ def __init__(
         self.indexing = indexing
         self.aggregation = aggregation
         self.spatial = spatial
+        self.vector = vector
         self.group_by_array_behavior = group_by_array_behavior
         self.suggestions = suggestions
         self.is_name_quoted = is_name_quoted
@@ -389,6 +395,7 @@ def from_json(cls, json_dict: Dict) -> AutoIndexFieldOptions:
             AutoFieldIndexing(json_dict.get("Indexing")),
             AggregationOperation(json_dict.get("Aggregation")) if json_dict.get("Aggregation", None) else None,
             AutoSpatialOptions.from_json(json_dict.get("Spatial")) if json_dict.get("Spatial", None) else None,
+            AutoVectorOptions.from_json(json_dict.get("Vector")) if json_dict.get("Vector", None) else None,
             GroupByArrayBehavior(json_dict.get("GroupByArrayBehavior")),
             json_dict.get("Suggestions"),
             json_dict.get("IsNameQuoted"),
@@ -400,6 +407,9 @@ def to_json(self) -> Dict:
             "Indexing": self.indexing.value,
             "Aggregation": self.aggregation.value if self.aggregation is not None else None,
             "Spatial": self.spatial.type if self.spatial is not None else None,
+            "Vector": (
+                self.vector.to_json() if self.vector is not None else None
+            ),  # todo; check if vector.to_json() is valid here
             "GroupByArrayBehavior": self.group_by_array_behavior.value,
             "Suggestions": self.suggestions,
             "IsNameQuoted": self.is_name_quoted,

ravendb/documents/indexes/vector/__init__.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class VectorEmbeddingType(Enum):
+    SINGLE = "Single"  # float
+    INT8 = "Int8"  # quantized int
+    BINARY = "Binary"  # 1/0 quantized int
+    TEXT = "Text"  # str

ravendb/documents/indexes/vector/embedding.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from typing import Dict, Any
+
+from ravendb.documents.indexes.vector.embedding import VectorEmbeddingType
+from ravendb.primitives import constants
+
+
+class VectorOptions:
+    def __init__(
+        self,
+        source_embedding_type: VectorEmbeddingType = constants.VectorSearch.DEFAULT_EMBEDDING_TYPE,
+        destination_embedding_type: VectorEmbeddingType = constants.VectorSearch.DEFAULT_EMBEDDING_TYPE,
+        dimensions: int = None,
+        number_of_edges: int = None,
+        number_of_candidates_for_indexing: int = None,
+    ):
+        self.dimensions = dimensions
+        self.source_embedding_type = source_embedding_type
+        self.destination_embedding_type = destination_embedding_type
+        self.numbers_of_candidates_for_indexing = number_of_candidates_for_indexing
+        self.number_of_edges = number_of_edges
+
+    @classmethod
+    def from_json(cls, json_dict: Dict[str, Any]) -> VectorOptions:
+        return cls(
+            json_dict["SourceEmbeddingType"],
+            json_dict["DestinationEmbeddingType"],
+            json_dict["Dimensions"],
+            json_dict["NumberOfEdges"],
+            json_dict["NumberOfCandidatesForIndexing"],
+        )
+
+    def to_json(self) -> Dict[str, Any]:
+        return {
+            "SourceEmbeddingType": self.source_embedding_type.value,
+            "DestinationEmbeddingType": self.destination_embedding_type.value,
+            "Dimensions": self.dimensions,
+            "NumberOfCandidatesForIndexing": self.numbers_of_candidates_for_indexing,
+            "NumberOfEdges": self.number_of_edges,
+        }
+
+
+class AutoVectorOptions(VectorOptions):
+    def __init__(
+        self,
+        source_embedding_type: VectorEmbeddingType = constants.VectorSearch.DEFAULT_EMBEDDING_TYPE,
+        destination_embedding_type: VectorEmbeddingType = constants.VectorSearch.DEFAULT_EMBEDDING_TYPE,
+        dimensions: int = None,
+        number_of_edges: int = None,
+        number_of_candidates_for_indexing: int = None,
+        source_field_name: str = None,
+    ):
+        super().__init__(
+            source_embedding_type,
+            destination_embedding_type,
+            dimensions,
+            number_of_edges,
+            number_of_candidates_for_indexing,
+        )
+        self.source_field_name = source_field_name
+
+    @classmethod
+    def from_vector_options(cls, vector_options: VectorOptions):
+        return cls(
+            source_embedding_type=vector_options.source_embedding_type,
+            destination_embedding_type=vector_options.destination_embedding_type,
+            dimensions=vector_options.dimensions,
+            number_of_edges=vector_options.number_of_edges,
+            number_of_candidates_for_indexing=vector_options.numbers_of_candidates_for_indexing,
+        )
+
+    @classmethod
+    def from_json(cls, json_dict: Dict[str, Any]):
+        vec_options = super().from_json(json_dict)
+        auto_vect_options = cls.from_vector_options(vec_options)
+        auto_vect_options.source_field_name = json_dict["SourceFieldName"]
+        return auto_vect_options
+
+    def to_json(self) -> Dict[str, Any]:
+        json_dict = super().to_json()
+        json_dict["SourceFieldName"] = self.source_field_name
+        return json_dict

ravendb/documents/indexes/vector/options.py

@@ -0,0 +1,66 @@
+import struct
+from typing import List, Tuple
+
+
+class VectorQuantizer:
+    @staticmethod
+    def to_int8(raw_embedding: List[float]) -> bytes:
+        """
+        Converts a list of floats to a packed byte array of signed 8-bit integers (int8).
+        The maximum absolute value is appended as a 4-byte float at the end.
+
+        Args:
+            raw_embedding (List[float]): List of floating-point numbers to be quantized.
+
+        Returns:
+            bytes: Packed byte array containing the quantized int8 values and the max component.
+        """
+        if not raw_embedding:
+            return b""
+
+        # Find the maximum absolute value in the input array
+        max_component: float = max(abs(x) for x in raw_embedding)
+
+        # If all elements are zero, set quantized to all zeros
+        if max_component == 0:
+            quantized: List[int] = [0] * len(raw_embedding)
+        else:
+            # Scale all elements to the range [-127, 127]
+            scale_factor: float = 127.0 / max_component
+            quantized: List[int] = [int(x * scale_factor) for x in raw_embedding]
+
+        # Pack the quantized values into signed bytes (int8)
+        packed: bytes = struct.pack("b" * len(quantized), *quantized)
+
+        # Append the max_component as a little-endian float
+        packed += struct.pack("<f", max_component)
+
+        return packed
+
+    @staticmethod
+    def to_int1(raw_embedding: List[float]) -> bytes:
+        """
+        Converts a list of floats to a packed byte array of binary values (int1).
+        Each byte represents 8 consecutive float values, where each bit corresponds to
+        whether the float is non-negative (1) or negative (0).
+
+        Args:
+            raw_embedding (List[float]): List of floating-point numbers to be quantized.
+
+        Returns:
+            bytes: Packed byte array containing the binary-packed values.
+        """
+        # Calculate the number of bytes needed to store the binary-packed values
+        output_length: int = (len(raw_embedding) + 7) // 8
+
+        # Initialize a bytearray to store the packed bits
+        bytes_list: bytearray = bytearray(output_length)
+
+        # Iterate over each float value and pack it into the appropriate bit
+        for i, val in enumerate(raw_embedding):
+            if val >= 0:
+                byte_index: int = i // 8  # Determine which byte to modify
+                bit_pos: int = 7 - (i % 8)  # Determine the bit position within the byte
+                bytes_list[byte_index] |= 1 << bit_pos  # Set the bit to 1 if the value is non-negative
+
+        return bytes(bytes_list)