RDBC-899 Refactor VectorSearch utilities by removing QueryFieldUtil and introducing VectorQuantizer with updated tests

Author: M4xymm

src/Documents/Queries/VectorSearch/Fields/VectorField.ts

@@ -1,8 +1,9 @@
 import { IVectorField } from "../../../Session/IVectorFieldFactory.js";
 import { VectorFieldBase } from "../Common/VectorFieldBase.js";
+import { Field } from "../../../../Types/index.js";
 
 export class VectorField<T> extends VectorFieldBase<T> implements IVectorField {
-    constructor(fieldName: string) {
+    constructor(fieldName: Field<T>) {
         super(fieldName);
         this._byFieldMethodUsed = true;
         this.fieldName = fieldName;

src/Documents/Queries/VectorSearch/Utils/PropertyPathUtil.ts

@@ -9,27 +9,28 @@ export class VectorQuantizer {
      */
     public static toInt8(rawEmbedding: number[] | Float32Array): number[] {
         const length = rawEmbedding.length;
-        const result = new Int8Array(length + 4); // Extra 4 bytes for the float
+        const result = new Array(length + 4);
 
-        // Find the maximum absolute value
         let maxAbsValue = 0;
         for (let i = 0; i < length; i++) {
             maxAbsValue = Math.max(maxAbsValue, Math.abs(rawEmbedding[i]));
         }
 
-        // Scale factor to fit in int8 range
         const scaleFactor = maxAbsValue === 0 ? 1 : 127 / maxAbsValue;
 
-        // Convert and scale values
         for (let i = 0; i < length; i++) {
             result[i] = Math.round(rawEmbedding[i] * scaleFactor);
         }
 
-        // Append the scale factor as a float32 at the end
-        const scaleFactorView = new DataView(result.buffer, length, 4);
-        scaleFactorView.setFloat32(0, maxAbsValue, true); // true for little-endian
+        const buffer = new ArrayBuffer(4);
+        const dataView = new DataView(buffer);
+        dataView.setFloat32(0, maxAbsValue, true);
 
-        return Array.from(result);
+        for (let i = 0; i < 4; i++) {
+            result[length + i] = dataView.getInt8(i);
+        }
+
+        return result;
     }
 
     /**

src/Documents/Queries/VectorSearch/Utils/PropertyPathUtilTest.ts

@@ -96,7 +96,6 @@ import {
     IVectorFieldFactory, IVectorFieldValueFactory, VectorEmbeddingFieldValueFactory
 } from "./IVectorFieldFactory.js";
 import { VectorEmbeddingFieldFactory } from "../Queries/VectorSearch/VectorEmbeddingFieldFactory.js";
-import { VectorFieldFactory } from "./VectorFieldFactory.js";
 import { Field } from "../../Types/index.js";
 
 /**

…nts/Queries/VectorSearch/VectorSearch.ts …/Queries/VectorSearch/VectorQuantizer.tssrc/Documents/Queries/VectorSearch/VectorSearch.ts renamed to src/Documents/Queries/VectorSearch/VectorQuantizer.ts src/Documents/Queries/VectorSearch/VectorSearch.ts renamed to src/Documents/Queries/VectorSearch/VectorQuantizer.ts

@@ -419,7 +419,7 @@ export * from "./Documents/Indexes/TimeSeries/AbstractRawJavaScriptTimeSeriesInd
 export * from "./Documents/Indexes/TimeSeries/TimeSeriesIndexDefinition.js";
 export * from "./Documents/Indexes/TimeSeries/TimeSeriesIndexDefinitionBuilder.js";
 // VECTOR SEARCH
-export * from "./Documents/Queries/VectorSearch/VectorSearch.js"
+export * from "./Documents/Queries/VectorSearch/VectorQuantizer.js"
 export * from "./Utility/VectorSearchUtil.js"
 
 // REPLICATION

src/Documents/Session/AbstractDocumentQuery.ts

@@ -0,0 +1,96 @@
+import assert from "node:assert";
+import {VectorQuantizer} from "../../../../src/Documents/Queries/VectorSearch/VectorQuantizer.js";
+
+describe("VectorQuantizer", function () {
+    describe("toInt8", function () {
+        it("should correctly quantize vector elements and store scale factor", function () {
+            const input = [0.1, 0.2];
+            const result = VectorQuantizer.toInt8(input);
+
+            const expectedResult = [64, 127, -51, -52, 76, 62];
+
+            for (let i = 0; i < expectedResult.length; i++) {
+                assert.equal(result[i], expectedResult[i]);
+            }
+        });
+
+        it("should maintain expected byte values in quantized output", function () {
+            const input = [0.1, 0.2];
+            const result = VectorQuantizer.toInt8(input);
+
+            const expectedResult = [64, 127, -51, -52, 76, 62];
+
+            for (let i = 0; i < expectedResult.length; i++) {
+                assert.equal(result[i], expectedResult[i]);
+            }
+        });
+
+        it("should handle zero vector input correctly", function () {
+            const input = [0, 0, 0, 0];
+            const result = VectorQuantizer.toInt8(input);
+
+            const expectedResult = Array(8).fill(0);
+
+            for (let i = 0; i < expectedResult.length; i++) {
+                assert.equal(result[i], expectedResult[i]);
+            }
+        });
+
+        it("should correctly quantize array of floats with positive and negative values", function () {
+            const input = [0.5, -1.5, 2.5, -3.5];
+            const result = VectorQuantizer.toInt8(input);
+
+            const expectedResult = [18, -54, 91, -127, 0, 0, 96, 64];
+
+            for (let i = 0; i < expectedResult.length; i++) {
+                assert.equal(result[i], expectedResult[i]);
+            }
+        });
+    });
+
+    describe("toInt1", function () {
+        it("should correctly convert vector to binary bit representation", function () {
+            const input = [1.0, -2.0, 3.0, -4.0, 5.0, -6.0, 7.0, -8.0, 9.0];
+            const result = VectorQuantizer.toInt1(input);
+
+            const expectedResult = [0xAA, 0x80];
+
+            for (let i = 0; i < expectedResult.length; i++) {
+                assert.equal(result[i], expectedResult[i]);
+            }
+        });
+
+        it("should represent zero values as positive bits", function () {
+            const input = [0, 0, 0, 0, 0, 0, 0, 0];
+            const result = VectorQuantizer.toInt1(input);
+
+            const expectedResult = [0xFF];
+
+            for (let i = 0; i < expectedResult.length; i++) {
+                assert.equal(result[i], expectedResult[i]);
+            }
+        });
+
+        it("should properly pad vectors with length not divisible by 8", function () {
+            const input = [1, 2, 3, 4, 5];
+            const result = VectorQuantizer.toInt1(input);
+
+            const expectedResult = [0xF8];
+
+            for (let i = 0; i < expectedResult.length; i++) {
+                assert.equal(result[i], expectedResult[i]);
+            }
+        });
+
+        it("should correctly convert Float32Array with alternating signs", function () {
+            const input = [-1, 2, -3, 4, -5, 6, -7, 8];
+            const result = VectorQuantizer.toInt1(input);
+
+            const expectedResult = [0x55];
+
+            for (let i = 0; i < expectedResult.length; i++) {
+                assert.equal(result[i], expectedResult[i]);
+            }
+        });
+    });
+});