diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 111c8b32bb68..6a9a3f4df782 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -4854,6 +4854,35 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + # valkey-store + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-cpu-valkey-store' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "valkey-store" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-valkey-store' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "valkey-store" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' # rfdetr - build-type: '' cuda-major-version: "" @@ -5556,6 +5585,10 @@ includeDarwin: tag-suffix: "-metal-darwin-arm64-local-store" build-type: "metal" lang: "go" + - backend: "valkey-store" + tag-suffix: "-metal-darwin-arm64-valkey-store" + build-type: "metal" + lang: "go" - backend: "llama-cpp-quantization" tag-suffix: "-metal-darwin-arm64-llama-cpp-quantization" build-type: "mps" diff --git a/.gitignore b/.gitignore index 666b81df934e..1b3efa14be44 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ LocalAI # Go backend packages whose main lives under backend/go/. /cloud-proxy /local-store +/valkey-store # prevent above rules from omitting the helm chart !charts/* # prevent above rules from omitting the api/localai folder diff --git a/Makefile b/Makefile index da2e638b07c4..dc8ad4f2908a 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin GOCMD=go GOTEST=$(GOCMD) test @@ -69,7 +69,7 @@ else GORELEASER=$(shell which goreleaser) endif -TEST_PATHS?=./api/... ./pkg/... ./core/... ./backend/go/cloud-proxy/... ./backend/go/local-store/... +TEST_PATHS?=./api/... ./pkg/... ./core/... ./backend/go/cloud-proxy/... ./backend/go/local-store/... ./backend/go/valkey-store/... ## Coverage output and the committed baseline that CI compares against. ## The gate is strict: total coverage must never decrease (no tolerance). @@ -1223,6 +1223,7 @@ BACKEND_PRIVACY_FILTER = privacy-filter|privacy-filter|.|false|false # Golang backends BACKEND_PIPER = piper|golang|.|false|true BACKEND_LOCAL_STORE = local-store|golang|.|false|true +BACKEND_VALKEY_STORE = valkey-store|golang|.|false|true BACKEND_CLOUD_PROXY = cloud-proxy|golang|.|false|true BACKEND_HUGGINGFACE = huggingface|golang|.|false|true BACKEND_SILERO_VAD = silero-vad|golang|.|false|true @@ -1318,6 +1319,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_DS4))) $(eval $(call generate-docker-build-target,$(BACKEND_PRIVACY_FILTER))) $(eval $(call generate-docker-build-target,$(BACKEND_PIPER))) $(eval $(call generate-docker-build-target,$(BACKEND_LOCAL_STORE))) +$(eval $(call generate-docker-build-target,$(BACKEND_VALKEY_STORE))) $(eval $(call generate-docker-build-target,$(BACKEND_CLOUD_PROXY))) $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE))) $(eval $(call generate-docker-build-target,$(BACKEND_SILERO_VAD))) diff --git a/backend/README.md b/backend/README.md index 0e92a0f03c58..e2f9b275dc71 100644 --- a/backend/README.md +++ b/backend/README.md @@ -56,6 +56,7 @@ The backend system provides language-specific Dockerfiles that handle the build - **stablediffusion-ggml**: Stable Diffusion in Go with GGML Cpp backend - **piper**: Text-to-speech synthesis Golang with C bindings using rhaspy/piper - **local-store**: Vector storage backend +- **valkey-store**: Vector storage backend persisting to a Valkey server (Valkey Search module) #### C++ Backends (`cpp/`) - **llama-cpp**: Llama.cpp integration diff --git a/backend/go/valkey-store/Makefile b/backend/go/valkey-store/Makefile new file mode 100644 index 000000000000..ea9b0ae7c41d --- /dev/null +++ b/backend/go/valkey-store/Makefile @@ -0,0 +1,12 @@ +GOCMD=go + +valkey-store: + CGO_ENABLED=0 $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o valkey-store ./ + +package: + bash package.sh + +build: valkey-store package + +clean: + rm -f valkey-store diff --git a/backend/go/valkey-store/main.go b/backend/go/valkey-store/main.go new file mode 100644 index 000000000000..f06dfa6f5113 --- /dev/null +++ b/backend/go/valkey-store/main.go @@ -0,0 +1,25 @@ +package main + +// Note: this is started internally by LocalAI and a server is allocated for each store + +import ( + "flag" + "os" + + grpc "github.com/mudler/LocalAI/pkg/grpc" + "github.com/mudler/xlog" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") +) + +func main() { + xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(os.Getenv("LOCALAI_LOG_LEVEL")), os.Getenv("LOCALAI_LOG_FORMAT"))) + + flag.Parse() + + if err := grpc.StartServer(*addr, NewStore()); err != nil { + panic(err) + } +} diff --git a/backend/go/valkey-store/package.sh b/backend/go/valkey-store/package.sh new file mode 100644 index 000000000000..c6b487836eaa --- /dev/null +++ b/backend/go/valkey-store/package.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Script to copy the appropriate libraries based on architecture +# This script is used in the final stage of the Dockerfile + +set -e + +CURDIR=$(dirname "$(realpath $0)") + +mkdir -p $CURDIR/package +cp -avf $CURDIR/valkey-store $CURDIR/package/ +cp -rfv $CURDIR/run.sh $CURDIR/package/ diff --git a/backend/go/valkey-store/run.sh b/backend/go/valkey-store/run.sh new file mode 100644 index 000000000000..caf370e2e78c --- /dev/null +++ b/backend/go/valkey-store/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -ex + +CURDIR=$(dirname "$(realpath "$0")") + +exec "$CURDIR"/valkey-store "$@" diff --git a/backend/go/valkey-store/store.go b/backend/go/valkey-store/store.go new file mode 100644 index 000000000000..ee1dbd440571 --- /dev/null +++ b/backend/go/valkey-store/store.go @@ -0,0 +1,408 @@ +package main + +// Valkey-backed vector store, exposed as a gRPC backend. Realizes +// option (2) of the vector-search design discussion (#1792) through +// the same pluggable seam as local-store: core selects it via the +// `backend` field on /stores/* requests, no proto or HTTP change. +// +// Compared to local-store this trades in-process speed for +// durability (Valkey RDB/AOF persistence) and a scaling path +// (opt-in HNSW). Vectors live in the Valkey Search module +// (`FT.*` on the valkey/valkey-bundle image). +// +// Data model: the vector IS the key in the store contract, so each +// entry is a HASH at `` with fields +// `vec` (raw float32 bytes, the indexed VECTOR attribute) and `val` +// (opaque payload). Set is an upsert for free — same vector, same +// hash key. A `meta` HASH records the index dimension so a +// restarted backend process can validate dimensions without parsing +// FT.INFO ("meta" is outside the hex alphabet, so it can never +// collide with an encoded vector). +// +// Search parity: the index defaults to FLAT with COSINE distance, so +// Find is an exact scan whose ranking matches local-store; Valkey +// returns cosine *distance* and similarity is derived as +// `1 - distance`. Set VALKEY_INDEX_ALGO=HNSW (or the valkey_index_algo +// model option) for approximate ANN on large corpora. +// +// Connection settings resolve per-model first, then fall back to the +// process-wide env vars: `valkey_addr` / VALKEY_ADDR, `valkey_index_algo` +// / VALKEY_INDEX_ALGO, and — mirroring cloud-proxy's api_key_env — +// `valkey_username_env` / `valkey_password_env` name the env vars that +// actually hold the credential, so distinct model configs can each +// point at a different Valkey server with its own credentials. +// +// Concurrency: base.SingleThread serialises gRPC calls, matching +// local-store; Valkey itself is the durable shared state. + +import ( + "context" + "encoding/binary" + "encoding/hex" + "fmt" + "math" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/mudler/LocalAI/pkg/grpc/base" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/store" + "github.com/valkey-io/valkey-go" +) + +const ( + keyspacePrefix = "localai:store:" + vecField = "vec" + valField = "val" + scoreAlias = "__score" +) + +type Store struct { + base.SingleThread + + client valkey.Client + + // prefix namespaces every hash key and the index name, derived + // from the store namespace core sends via opts.Model. Two + // namespaces on the same Valkey server never see each other's + // vectors because each index is created with PREFIX . + prefix string + index string + + // keyLen mirrors local-store: the dimension of every stored key, + // -1 while unknown. It becomes known at index creation (first + // Set) or at Load time from the meta hash of a pre-existing + // index, so dimension mismatches fail loudly instead of + // producing bogus cosine matches. + keyLen int + + // indexAlgo is resolved once at Load (model option, else env var, + // else FLAT) and reused by createIndex, which runs later from + // StoresSet and has no access to the ModelOptions that carried it. + indexAlgo string +} + +func NewStore() *Store { + return &Store{keyLen: -1} +} + +// optString reads a model option (key:value form) from ModelOptions, +// returning def when absent. Mirrors optInt in sibling backends +// (e.g. moss-transcribe-cpp) for the model YAML's `options:` list. +func optString(opts *pb.ModelOptions, key, def string) string { + for _, o := range opts.GetOptions() { + k, v, ok := strings.Cut(o, ":") + if ok && strings.TrimSpace(k) == key { + return strings.TrimSpace(v) + } + } + return def +} + +// optEnvIndirect resolves a credential the way cloud-proxy's +// api_key_env does: the model option names an env var to read from, +// so distinct model configs can each reference a different credential +// pair without putting secrets in the YAML. Falls back to a single +// process-wide env var when the model doesn't set the option, keeping +// the single-Valkey-server default working unchanged. +func optEnvIndirect(opts *pb.ModelOptions, optKey, fallbackEnvVar string) string { + if envVar := optString(opts, optKey, ""); envVar != "" { + return os.Getenv(envVar) + } + return os.Getenv(fallbackEnvVar) +} + +// Load validates the namespace, connects to Valkey and, when the +// namespace was already populated by a previous process, restores the +// index dimension from the meta hash. The namespace-prefix gate keeps +// the model loader's greedy autoload probing (real LLM names) from +// binding an LLM to the vector store — same contract as local-store. +func (s *Store) Load(opts *pb.ModelOptions) error { + if !strings.HasPrefix(opts.GetModel(), store.NamespacePrefix) { + return fmt.Errorf("valkey-store: refusing to load %q: not a store namespace (expected %q prefix)", opts.GetModel(), store.NamespacePrefix) + } + ns := strings.TrimPrefix(opts.GetModel(), store.NamespacePrefix) + if ns == "" { + ns = "default" + } + s.prefix = keyspacePrefix + ns + ":" + s.index = s.prefix + "idx" + + addr := optString(opts, "valkey_addr", os.Getenv("VALKEY_ADDR")) + if addr == "" { + addr = "127.0.0.1:6379" + } + s.indexAlgo = optString(opts, "valkey_index_algo", os.Getenv("VALKEY_INDEX_ALGO")) + client, err := valkey.NewClient(valkey.ClientOption{ + InitAddress: []string{addr}, + Username: optEnvIndirect(opts, "valkey_username_env", "VALKEY_USERNAME"), + Password: optEnvIndirect(opts, "valkey_password_env", "VALKEY_PASSWORD"), + // The store contract is request/response; the client-side + // cache would only add invalidation traffic. + DisableCache: true, + }) + if err != nil { + return fmt.Errorf("valkey-store: connect %s: %w", addr, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := client.Do(ctx, client.B().Ping().Build()).Error(); err != nil { + client.Close() + return fmt.Errorf("valkey-store: ping %s: %w", addr, err) + } + s.client = client + + dim, err := client.Do(ctx, client.B().Hget().Key(s.prefix+"meta").Field("dim").Build()).AsInt64() + if err == nil && dim > 0 { + s.keyLen = int(dim) + } + return nil +} + +func (s *Store) StoresSet(opts *pb.StoresSetOptions) error { + keys := store.UnwrapKeys(opts.Keys) + values := store.UnwrapValues(opts.Values) + if len(keys) == 0 { + return fmt.Errorf("valkey-store: Set: no keys to add") + } + if len(keys) != len(values) { + return fmt.Errorf("valkey-store: Set: len(keys) = %d, len(values) = %d", len(keys), len(values)) + } + if s.keyLen == -1 { + if err := s.createIndex(len(keys[0])); err != nil { + return err + } + } + for i, k := range keys { + if len(k) != s.keyLen { + return fmt.Errorf("valkey-store: Set: key %d length %d does not match existing %d", i, len(k), s.keyLen) + } + } + + ctx := context.Background() + cmds := make(valkey.Commands, 0, len(keys)) + for i, k := range keys { + vec := encodeVector(k) + cmds = append(cmds, s.client.B().Hset().Key(s.prefix+hex.EncodeToString(vec)). + FieldValue(). + FieldValue(vecField, valkey.BinaryString(vec)). + FieldValue(valField, valkey.BinaryString(values[i])). + Build()) + } + for _, resp := range s.client.DoMulti(ctx, cmds...) { + if err := resp.Error(); err != nil { + return fmt.Errorf("valkey-store: Set: %w", err) + } + } + return nil +} + +// createIndex creates the vector index for this namespace with the +// given dimension and records the dimension in the meta hash. If the +// index already exists (created by another process after our Load +// checked), adopt its recorded dimension instead of failing the Set. +func (s *Store) createIndex(dim int) error { + if dim == 0 { + return fmt.Errorf("valkey-store: Set: cannot index zero-length vectors") + } + algo := strings.ToUpper(s.indexAlgo) + if algo == "" { + algo = "FLAT" + } + if algo != "FLAT" && algo != "HNSW" { + return fmt.Errorf("valkey-store: valkey_index_algo/VALKEY_INDEX_ALGO %q not supported (FLAT or HNSW)", algo) + } + + ctx := context.Background() + err := s.client.Do(ctx, s.client.B().Arbitrary("FT.CREATE").Args( + s.index, "ON", "HASH", "PREFIX", "1", s.prefix, "SCHEMA", + vecField, "VECTOR", algo, "6", + "TYPE", "FLOAT32", + "DIM", strconv.Itoa(dim), + "DISTANCE_METRIC", "COSINE", + ).Build()).Error() + if err != nil { + if !strings.Contains(strings.ToLower(err.Error()), "already exists") { + return fmt.Errorf("valkey-store: FT.CREATE %s: %w", s.index, err) + } + existing, herr := s.client.Do(ctx, s.client.B().Hget().Key(s.prefix+"meta").Field("dim").Build()).AsInt64() + if herr != nil || existing <= 0 { + return fmt.Errorf("valkey-store: index %s exists but its dimension is unknown: %v", s.index, herr) + } + s.keyLen = int(existing) + return nil + } + if err := s.client.Do(ctx, s.client.B().Hset().Key(s.prefix+"meta"). + FieldValue().FieldValue("dim", strconv.Itoa(dim)).Build()).Error(); err != nil { + return fmt.Errorf("valkey-store: record index dimension: %w", err) + } + s.keyLen = dim + return nil +} + +func (s *Store) StoresDelete(opts *pb.StoresDeleteOptions) error { + keys := store.UnwrapKeys(opts.Keys) + if len(keys) == 0 { + return fmt.Errorf("valkey-store: Delete: no keys to delete") + } + if s.keyLen != -1 { + for i, k := range keys { + if len(k) != s.keyLen { + return fmt.Errorf("valkey-store: Delete: key %d length %d does not match existing %d", i, len(k), s.keyLen) + } + } + } + hashKeys := make([]string, len(keys)) + for i, k := range keys { + hashKeys[i] = s.prefix + hex.EncodeToString(encodeVector(k)) + } + if err := s.client.Do(context.Background(), s.client.B().Del().Key(hashKeys...).Build()).Error(); err != nil { + return fmt.Errorf("valkey-store: Delete: %w", err) + } + return nil +} + +// StoresGet fetches values for the given keys. Missing keys are +// omitted from the result rather than reported as an error — callers +// compare returned-key length against requested-key length to detect +// them. Returned slices are aligned. +func (s *Store) StoresGet(opts *pb.StoresGetOptions) (pb.StoresGetResult, error) { + keys := store.UnwrapKeys(opts.Keys) + if s.keyLen == -1 { + return pb.StoresGetResult{}, nil + } + for i, k := range keys { + if len(k) != s.keyLen { + return pb.StoresGetResult{}, fmt.Errorf("valkey-store: Get: key %d length %d does not match existing %d", i, len(k), s.keyLen) + } + } + + cmds := make(valkey.Commands, 0, len(keys)) + for _, k := range keys { + cmds = append(cmds, s.client.B().Hget(). + Key(s.prefix+hex.EncodeToString(encodeVector(k))).Field(valField).Build()) + } + var foundKeys [][]float32 + var foundValues [][]byte + for i, resp := range s.client.DoMulti(context.Background(), cmds...) { + val, err := resp.AsBytes() + if err != nil { + if valkey.IsValkeyNil(err) { + continue + } + return pb.StoresGetResult{}, fmt.Errorf("valkey-store: Get: %w", err) + } + foundKeys = append(foundKeys, keys[i]) + foundValues = append(foundValues, val) + } + return pb.StoresGetResult{ + Keys: store.WrapKeys(foundKeys), + Values: store.WrapValues(foundValues), + }, nil +} + +// StoresFind returns the topK nearest stored entries by cosine +// similarity, ordered most-similar first. An empty store returns +// empty slices and no error. +func (s *Store) StoresFind(opts *pb.StoresFindOptions) (pb.StoresFindResult, error) { + query := opts.Key.Floats + topK := int(opts.TopK) + if topK < 1 { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: topK = %d, must be >= 1", topK) + } + if s.keyLen == -1 { + return pb.StoresFindResult{}, nil + } + if len(query) != s.keyLen { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: query length %d does not match existing %d", len(query), s.keyLen) + } + + // No SORTBY: Valkey Search rejects sorting on the KNN score alias + // (it is not an indexed field); results are re-ordered client-side. + resp := s.client.Do(context.Background(), s.client.B().Arbitrary("FT.SEARCH").Args( + s.index, + fmt.Sprintf("*=>[KNN %d @%s $query_vector AS %s]", topK, vecField, scoreAlias), + "PARAMS", "2", "query_vector", valkey.BinaryString(encodeVector(query)), + "RETURN", "2", scoreAlias, valField, + "LIMIT", "0", strconv.Itoa(topK), + "DIALECT", "2", + ).Build()) + rows, err := resp.ToArray() + if err != nil { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: FT.SEARCH %s: %w", s.index, err) + } + // Reply shape: [count, key1, [field, value, ...], key2, ...]. + var keys [][]float32 + var values [][]byte + var sims []float32 + for i := 1; i+1 < len(rows); i += 2 { + hashKey, err := rows[i].ToString() + if err != nil { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: parse key: %w", err) + } + key, err := decodeVectorKey(strings.TrimPrefix(hashKey, s.prefix)) + if err != nil { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: %q: %w", hashKey, err) + } + fields, err := rows[i+1].AsStrMap() + if err != nil { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: parse fields of %q: %w", hashKey, err) + } + dist, err := strconv.ParseFloat(fields[scoreAlias], 32) + if err != nil { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: parse score of %q: %w", hashKey, err) + } + keys = append(keys, key) + values = append(values, []byte(fields[valField])) + sims = append(sims, 1-float32(dist)) + } + order := make([]int, len(sims)) + for i := range order { + order[i] = i + } + sort.SliceStable(order, func(a, b int) bool { return sims[order[a]] > sims[order[b]] }) + sortedKeys := make([][]float32, len(order)) + sortedValues := make([][]byte, len(order)) + sortedSims := make([]float32, len(order)) + for i, j := range order { + sortedKeys[i] = keys[j] + sortedValues[i] = values[j] + sortedSims[i] = sims[j] + } + keys, values, sims = sortedKeys, sortedValues, sortedSims + return pb.StoresFindResult{ + Keys: store.WrapKeys(keys), + Values: store.WrapValues(values), + Similarities: sims, + }, nil +} + +// encodeVector serialises a vector as little-endian float32 bytes — +// the wire format Valkey Search expects for VECTOR fields and the +// basis of the hex hash-key encoding. +func encodeVector(v []float32) []byte { + out := make([]byte, 4*len(v)) + for i, f := range v { + binary.LittleEndian.PutUint32(out[i*4:], math.Float32bits(f)) + } + return out +} + +func decodeVectorKey(hexKey string) ([]float32, error) { + raw, err := hex.DecodeString(hexKey) + if err != nil { + return nil, fmt.Errorf("not a vector hash key: %w", err) + } + if len(raw)%4 != 0 { + return nil, fmt.Errorf("not a vector hash key: %d bytes", len(raw)) + } + v := make([]float32, len(raw)/4) + for i := range v { + v[i] = math.Float32frombits(binary.LittleEndian.Uint32(raw[i*4:])) + } + return v, nil +} diff --git a/backend/go/valkey-store/store_suite_test.go b/backend/go/valkey-store/store_suite_test.go new file mode 100644 index 000000000000..d15f3fe2a079 --- /dev/null +++ b/backend/go/valkey-store/store_suite_test.go @@ -0,0 +1,13 @@ +package main + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestValkeyStore(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "valkey-store test suite") +} diff --git a/backend/go/valkey-store/store_test.go b/backend/go/valkey-store/store_test.go new file mode 100644 index 000000000000..2347a57f1636 --- /dev/null +++ b/backend/go/valkey-store/store_test.go @@ -0,0 +1,448 @@ +package main + +// Regression suite for the valkey-store gRPC backend, mirroring the +// local-store suite: the Stores{Set,Get,Find,Delete} surface is the +// only public contract, so the two backends are drop-in alternatives. +// +// The suite needs a Valkey server with the Valkey Search module +// (valkey/valkey-bundle). Resolution order: +// 1. VALKEY_TEST_ADDR — use an already-running server (local dev), +// 2. otherwise start a valkey/valkey-bundle testcontainer +// (skipped on darwin, following the repo-wide pattern). +// Load's namespace-gate tests run everywhere: they fail before any +// connection is attempted. + +import ( + "context" + "fmt" + "math" + "os" + "runtime" + "time" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/store" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("StoresLoad namespace gate", func() { + It("refuses model names without the namespace prefix", func() { + err := NewStore().Load(&pb.ModelOptions{Model: "some-llm.gguf"}) + Expect(err).To(MatchError(ContainSubstring("not a store namespace"))) + Expect(NewStore().Load(&pb.ModelOptions{})).NotTo(Succeed()) + }) +}) + +var _ = Describe("model-config option resolution", func() { + It("optString prefers the model option over the default", func() { + opts := &pb.ModelOptions{Options: []string{"valkey_addr:10.0.0.1:6379"}} + Expect(optString(opts, "valkey_addr", "127.0.0.1:6379")).To(Equal("10.0.0.1:6379")) + }) + + It("optString falls back to the default when the option is absent", func() { + Expect(optString(&pb.ModelOptions{}, "valkey_addr", "127.0.0.1:6379")).To(Equal("127.0.0.1:6379")) + }) + + It("optEnvIndirect reads the env var named by the model option", func() { + Expect(os.Setenv("VALKEY_STORE_TEST_CRED_A", "creds-a")).To(Succeed()) + defer func() { _ = os.Unsetenv("VALKEY_STORE_TEST_CRED_A") }() + opts := &pb.ModelOptions{Options: []string{"valkey_password_env:VALKEY_STORE_TEST_CRED_A"}} + Expect(optEnvIndirect(opts, "valkey_password_env", "VALKEY_PASSWORD")).To(Equal("creds-a")) + }) + + It("optEnvIndirect falls back to the process-wide env var when unset", func() { + Expect(os.Setenv("VALKEY_PASSWORD", "fallback-cred")).To(Succeed()) + defer func() { _ = os.Unsetenv("VALKEY_PASSWORD") }() + Expect(optEnvIndirect(&pb.ModelOptions{}, "valkey_password_env", "VALKEY_PASSWORD")).To(Equal("fallback-cred")) + }) + + It("lets two models pick different credential env vars against the same server", func() { + Expect(os.Setenv("VALKEY_STORE_TEST_CRED_A", "creds-a")).To(Succeed()) + Expect(os.Setenv("VALKEY_STORE_TEST_CRED_B", "creds-b")).To(Succeed()) + defer func() { _ = os.Unsetenv("VALKEY_STORE_TEST_CRED_A") }() + defer func() { _ = os.Unsetenv("VALKEY_STORE_TEST_CRED_B") }() + a := &pb.ModelOptions{Options: []string{"valkey_username_env:VALKEY_STORE_TEST_CRED_A"}} + b := &pb.ModelOptions{Options: []string{"valkey_username_env:VALKEY_STORE_TEST_CRED_B"}} + Expect(optEnvIndirect(a, "valkey_username_env", "VALKEY_USERNAME")).To(Equal("creds-a")) + Expect(optEnvIndirect(b, "valkey_username_env", "VALKEY_USERNAME")).To(Equal("creds-b")) + }) +}) + +var _ = Describe("valkey-store against a live server", Ordered, func() { + var nsCounter int + + BeforeAll(func() { + if os.Getenv("VALKEY_TEST_ADDR") != "" { + Expect(os.Setenv("VALKEY_ADDR", os.Getenv("VALKEY_TEST_ADDR"))).To(Succeed()) + return + } + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI (set VALKEY_TEST_ADDR to run against a local server)") + } + ctx := context.Background() + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "valkey/valkey-bundle:latest", + ExposedPorts: []string{"6379/tcp"}, + WaitingFor: wait.ForListeningPort("6379/tcp").WithStartupTimeout(60 * time.Second), + }, + Started: true, + }) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = c.Terminate(context.Background()) }) + host, err := c.Host(ctx) + Expect(err).NotTo(HaveOccurred()) + port, err := c.MappedPort(ctx, "6379/tcp") + Expect(err).NotTo(HaveOccurred()) + Expect(os.Setenv("VALKEY_ADDR", fmt.Sprintf("%s:%s", host, port.Port()))).To(Succeed()) + }) + + // newTestStore loads a Store bound to a fresh namespace so specs + // can't see each other's vectors, and drops the namespace's index + // and keys when the spec ends so reruns against a long-lived + // server (VALKEY_TEST_ADDR) start clean too. + newTestStore := func() *Store { + nsCounter++ + ns := fmt.Sprintf("test-%d-%d", GinkgoRandomSeed(), nsCounter) + s := NewStore() + ExpectWithOffset(1, s.Load(&pb.ModelOptions{Model: store.NamespacePrefix + ns})).To(Succeed()) + DeferCleanup(func() { dropNamespace(s) }) + return s + } + + // reopen simulates a backend process restart: a brand-new Store + // loading the same namespace against the same server. + reopen := func(s *Store) *Store { + ns := s.prefix[len(keyspacePrefix) : len(s.prefix)-1] + fresh := NewStore() + ExpectWithOffset(1, fresh.Load(&pb.ModelOptions{Model: store.NamespacePrefix + ns})).To(Succeed()) + return fresh + } + + Describe("Load", func() { + It("accepts prefixed store namespaces", func() { + s := newTestStore() + Expect(s.client).NotTo(BeNil()) + }) + + It("accepts the prefix alone (default store)", func() { + s := NewStore() + Expect(s.Load(&pb.ModelOptions{Model: store.NamespacePrefix})).To(Succeed()) + DeferCleanup(func() { dropNamespace(s) }) + }) + + It("takes the index algorithm from the model option over VALKEY_INDEX_ALGO", func() { + Expect(os.Setenv("VALKEY_INDEX_ALGO", "HNSW")).To(Succeed()) + defer func() { _ = os.Unsetenv("VALKEY_INDEX_ALGO") }() + + nsCounter++ + ns := fmt.Sprintf("test-%d-%d", GinkgoRandomSeed(), nsCounter) + s := NewStore() + Expect(s.Load(&pb.ModelOptions{ + Model: store.NamespacePrefix + ns, + Options: []string{"valkey_index_algo:FLAT"}, + })).To(Succeed()) + DeferCleanup(func() { dropNamespace(s) }) + Expect(s.indexAlgo).To(Equal("FLAT"), "model option should win over the env var") + + mustSet(s, [][]float32{{1, 0, 0}}, [][]byte{[]byte("x")}) + info, err := s.client.Do(context.Background(), s.client.B().Arbitrary("FT.INFO").Args(s.index).Build()).ToAny() + Expect(err).NotTo(HaveOccurred()) + Expect(fmt.Sprintf("%v", info)).To(ContainSubstring("FLAT"), "index should have been created with the option's algorithm, not the env var's") + }) + }) + + Describe("StoresSet", func() { + It("rejects empty input", func() { + Expect(newTestStore().StoresSet(&pb.StoresSetOptions{})).NotTo(Succeed(), "Set with no keys should fail") + }) + + It("rejects key/value length mismatch", func() { + err := newTestStore().StoresSet(&pb.StoresSetOptions{ + Keys: wrapKeys([][]float32{{1, 0, 0}}), + Values: wrapValues([][]byte{[]byte("a"), []byte("b")}), + }) + Expect(err).To(HaveOccurred(), "len(keys) != len(values) should fail") + }) + + It("rejects dimension mismatch on later add", func() { + s := newTestStore() + mustSet(s, [][]float32{{1, 0, 0}}, [][]byte{[]byte("3d")}) + err := s.StoresSet(&pb.StoresSetOptions{ + Keys: wrapKeys([][]float32{{1, 0}}), + Values: wrapValues([][]byte{[]byte("2d")}), + }) + Expect(err).To(HaveOccurred(), "dimension mismatch on later Set should fail") + }) + + It("rejects dimension mismatch within batch", func() { + err := newTestStore().StoresSet(&pb.StoresSetOptions{ + Keys: wrapKeys([][]float32{{1, 0, 0}, {1, 0}}), + Values: wrapValues([][]byte{[]byte("3d"), []byte("2d")}), + }) + Expect(err).To(HaveOccurred(), "mixed-dimension within one batch should fail") + }) + + It("upserts existing keys", func() { + s := newTestStore() + mustSet(s, [][]float32{{0.3, 0, 0}, {0.1, 0, 0}}, [][]byte{[]byte("c"), []byte("a")}) + mustSet(s, [][]float32{{0.2, 0, 0}, {0.1, 0, 0}}, [][]byte{[]byte("b"), []byte("a-updated")}) + got := singleGet(s, []float32{0.1, 0, 0}) + Expect(string(got)).To(Equal("a-updated")) + res, err := s.StoresGet(&pb.StoresGetOptions{ + Keys: wrapKeys([][]float32{{0.1, 0, 0}, {0.2, 0, 0}, {0.3, 0, 0}}), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Keys).To(HaveLen(3)) + }) + }) + + Describe("StoresGet", func() { + It("round-trips multi-key", func() { + s := newTestStore() + mustSet(s, + [][]float32{{0.1, 0.2, 0.3}, {0.4, 0.5, 0.6}, {0.7, 0.8, 0.9}}, + [][]byte{[]byte("a"), []byte("b"), []byte("c")}, + ) + res, err := s.StoresGet(&pb.StoresGetOptions{ + Keys: wrapKeys([][]float32{{0.7, 0.8, 0.9}, {0.1, 0.2, 0.3}}), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Keys).To(HaveLen(2)) + Expect(string(res.Values[0].Bytes)).To(Equal("c")) + Expect(string(res.Values[1].Bytes)).To(Equal("a")) + }) + + It("omits missing keys rather than erroring", func() { + s := newTestStore() + mustSet(s, [][]float32{{0.1, 0, 0}}, [][]byte{[]byte("a")}) + res, err := s.StoresGet(&pb.StoresGetOptions{ + Keys: wrapKeys([][]float32{{0.1, 0, 0}, {0.9, 0, 0}}), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Keys).To(HaveLen(1)) + }) + + It("returns empty on a never-written namespace", func() { + res, err := newTestStore().StoresGet(&pb.StoresGetOptions{ + Keys: wrapKeys([][]float32{{0.1, 0, 0}}), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Keys).To(BeEmpty()) + }) + }) + + Describe("StoresDelete", func() { + It("removes keys", func() { + s := newTestStore() + mustSet(s, + [][]float32{{0.1, 0, 0}, {0.2, 0, 0}, {0.3, 0, 0}, {0.4, 0, 0}}, + [][]byte{[]byte("a"), []byte("b"), []byte("c"), []byte("d")}, + ) + Expect(s.StoresDelete(&pb.StoresDeleteOptions{ + Keys: wrapKeys([][]float32{{0.2, 0, 0}, {0.4, 0, 0}}), + })).To(Succeed()) + res, err := s.StoresGet(&pb.StoresGetOptions{ + Keys: wrapKeys([][]float32{{0.1, 0, 0}, {0.2, 0, 0}, {0.3, 0, 0}, {0.4, 0, 0}}), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Keys).To(HaveLen(2)) + }) + + It("tolerates missing keys", func() { + s := newTestStore() + mustSet(s, [][]float32{{0.1, 0, 0}}, [][]byte{[]byte("a")}) + Expect(s.StoresDelete(&pb.StoresDeleteOptions{ + Keys: wrapKeys([][]float32{{0.9, 0, 0}}), + })).To(Succeed(), "delete of missing key should succeed") + Expect(singleGet(s, []float32{0.1, 0, 0})).NotTo(BeNil()) + }) + }) + + Describe("StoresFind", func() { + It("returns normalized top-K most-similar first", func() { + s := newTestStore() + mustSet(s, + [][]float32{ + normalizeVec([]float32{1, 0, 0}), + normalizeVec([]float32{0, 1, 0}), + normalizeVec([]float32{0, 0, 1}), + }, + [][]byte{[]byte("x"), []byte("y"), []byte("z")}, + ) + res, err := s.StoresFind(&pb.StoresFindOptions{ + Key: &pb.StoresKey{Floats: normalizeVec([]float32{0.9, 0.1, 0})}, + TopK: 2, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Keys).To(HaveLen(2)) + Expect(res.Similarities[0]).To(BeNumerically(">=", res.Similarities[1]), "results not sorted desc by similarity") + Expect(string(res.Values[0].Bytes)).To(Equal("x")) + }) + + It("ranks non-normalized keys by cosine similarity", func() { + s := newTestStore() + mustSet(s, [][]float32{{2, 0, 0}, {0, 3, 0}}, [][]byte{[]byte("x"), []byte("y")}) + res, err := s.StoresFind(&pb.StoresFindOptions{ + Key: &pb.StoresKey{Floats: []float32{4, 0, 0}}, + TopK: 1, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(string(res.Values[0].Bytes)).To(Equal("x")) + Expect(res.Similarities[0]).To(BeNumerically(">=", float32(0.99))) + Expect(res.Similarities[0]).To(BeNumerically("<=", float32(1.01))) + }) + + It("rejects zero topK", func() { + s := newTestStore() + mustSet(s, [][]float32{{1, 0, 0}}, [][]byte{[]byte("x")}) + _, err := s.StoresFind(&pb.StoresFindOptions{ + Key: &pb.StoresKey{Floats: []float32{1, 0, 0}}, + TopK: 0, + }) + Expect(err).To(HaveOccurred(), "Find with topK=0 should fail") + }) + + It("rejects dimension mismatch", func() { + s := newTestStore() + mustSet(s, [][]float32{{1, 0, 0}}, [][]byte{[]byte("x")}) + _, err := s.StoresFind(&pb.StoresFindOptions{ + Key: &pb.StoresKey{Floats: []float32{1, 0}}, + TopK: 1, + }) + Expect(err).To(HaveOccurred(), "Find with mismatched dimension should fail") + }) + + It("returns empty result on empty store", func() { + res, err := newTestStore().StoresFind(&pb.StoresFindOptions{ + Key: &pb.StoresKey{Floats: []float32{1, 0, 0}}, + TopK: 5, + }) + Expect(err).NotTo(HaveOccurred(), "Find on empty store should succeed") + Expect(res.Keys).To(BeEmpty()) + }) + + It("handles topK larger than store", func() { + s := newTestStore() + mustSet(s, + [][]float32{normalizeVec([]float32{1, 0, 0}), normalizeVec([]float32{0, 1, 0})}, + [][]byte{[]byte("x"), []byte("y")}, + ) + res, err := s.StoresFind(&pb.StoresFindOptions{ + Key: &pb.StoresKey{Floats: normalizeVec([]float32{1, 0, 0})}, + TopK: 10, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Keys).To(HaveLen(2)) + }) + }) + + Describe("durability across process restarts", func() { + It("serves Get and Find from a fresh Store on the same namespace", func() { + s := newTestStore() + mustSet(s, [][]float32{normalizeVec([]float32{1, 2, 3})}, [][]byte{[]byte("persisted")}) + + fresh := reopen(s) + Expect(fresh.keyLen).To(Equal(3), "restarted process should restore the index dimension") + Expect(string(singleGet(fresh, normalizeVec([]float32{1, 2, 3})))).To(Equal("persisted")) + res, err := fresh.StoresFind(&pb.StoresFindOptions{ + Key: &pb.StoresKey{Floats: normalizeVec([]float32{1, 2, 3})}, + TopK: 1, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Values).To(HaveLen(1)) + Expect(string(res.Values[0].Bytes)).To(Equal("persisted")) + }) + }) + + Describe("namespace isolation", func() { + It("keeps two namespaces invisible to each other", func() { + a := newTestStore() + b := newTestStore() + mustSet(a, [][]float32{{1, 0, 0}}, [][]byte{[]byte("a-only")}) + mustSet(b, [][]float32{{0, 1, 0}}, [][]byte{[]byte("b-only")}) + + res, err := a.StoresFind(&pb.StoresFindOptions{ + Key: &pb.StoresKey{Floats: []float32{0, 1, 0}}, + TopK: 10, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Values).To(HaveLen(1)) + Expect(string(res.Values[0].Bytes)).To(Equal("a-only")) + }) + }) +}) + +// --- test helpers --- + +func dropNamespace(s *Store) { + if s.client == nil { + return + } + ctx := context.Background() + s.client.Do(ctx, s.client.B().Arbitrary("FT.DROPINDEX").Args(s.index).Build()) + var cursor uint64 + for { + scan, err := s.client.Do(ctx, s.client.B().Scan().Cursor(cursor).Match(s.prefix+"*").Count(100).Build()).AsScanEntry() + if err != nil { + break + } + if len(scan.Elements) > 0 { + s.client.Do(ctx, s.client.B().Del().Key(scan.Elements...).Build()) + } + cursor = scan.Cursor + if cursor == 0 { + break + } + } + s.client.Close() +} + +func mustSet(s *Store, keys [][]float32, values [][]byte) { + ExpectWithOffset(1, s.StoresSet(&pb.StoresSetOptions{Keys: wrapKeys(keys), Values: wrapValues(values)})).To(Succeed()) +} + +func singleGet(s *Store, key []float32) []byte { + res, err := s.StoresGet(&pb.StoresGetOptions{Keys: wrapKeys([][]float32{key})}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + if len(res.Values) == 0 { + return nil + } + return res.Values[0].Bytes +} + +func wrapKeys(in [][]float32) []*pb.StoresKey { + out := make([]*pb.StoresKey, len(in)) + for i, k := range in { + out[i] = &pb.StoresKey{Floats: k} + } + return out +} + +func wrapValues(in [][]byte) []*pb.StoresValue { + out := make([]*pb.StoresValue, len(in)) + for i, v := range in { + out[i] = &pb.StoresValue{Bytes: v} + } + return out +} + +func normalizeVec(v []float32) []float32 { + var sum float64 + for _, x := range v { + sum += float64(x) * float64(x) + } + mag := math.Sqrt(sum) + if mag == 0 { + return v + } + out := make([]float32, len(v)) + for i, x := range v { + out[i] = float32(float64(x) / mag) + } + return out +} diff --git a/backend/index.yaml b/backend/index.yaml index 3374a9d713fd..4c8c7deaf1ef 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -1664,6 +1664,24 @@ capabilities: default: "cpu-local-store" metal: "metal-local-store" +- &valkey-store + name: "valkey-store" + urls: + - https://github.com/mudler/LocalAI + - https://valkey.io/topics/search/ + description: | + Valkey Store is a vector store backed by a Valkey server running the + Valkey Search module (valkey/valkey-bundle). Vectors persist across + restarts and can optionally be indexed with HNSW for approximate + nearest-neighbor search on large corpora. + tags: + - vector-database + - open-source + - CPU + license: MIT + capabilities: + default: "cpu-valkey-store" + metal: "metal-valkey-store" - &kitten-tts name: "kitten-tts" urls: @@ -2158,6 +2176,35 @@ uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-local-store" mirrors: - localai/localai-backends:master-metal-darwin-arm64-local-store +- !!merge <<: *valkey-store + name: "cpu-valkey-store" + alias: "valkey-store" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-valkey-store" + mirrors: + - localai/localai-backends:latest-cpu-valkey-store +- !!merge <<: *valkey-store + name: "cpu-valkey-store-development" + alias: "valkey-store" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-valkey-store" + mirrors: + - localai/localai-backends:master-cpu-valkey-store +- !!merge <<: *valkey-store + name: "valkey-store-development" + alias: "valkey-store" + capabilities: + default: "cpu-valkey-store-development" + metal: "metal-valkey-store-development" +- !!merge <<: *valkey-store + name: "metal-valkey-store" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-valkey-store" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-valkey-store +- !!merge <<: *valkey-store + name: "metal-valkey-store-development" + alias: "valkey-store" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-valkey-store" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-valkey-store - !!merge <<: *opus name: "cpu-opus" uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-opus" diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index 8b26dd8c963f..fbc9bed79150 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -132,6 +132,6 @@ LocalAI supports various types of backends: - **Image & Video Generation Backends**: For diffusion and audio-conditioned avatar models (e.g., stable-diffusion.cpp, diffusers, vLLM-Omni, [LongCat-Video]({{%relref "features/longcat-video" %}})) - **Vision & Detection Backends**: For object detection, segmentation, depth, and face/voice recognition (e.g., rf-detr.cpp, locate-anything.cpp, sam3.cpp, insightface) - **Audio Processing Backends**: For voice activity detection and audio enhancement (e.g., Silero VAD, LocalVQE) -- **Utility Backends**: For reranking, PII/NER token classification, fine-tuning, quantization, and vector storage (e.g., rerankers, privacy-filter.cpp, TRL, local-store) +- **Utility Backends**: For reranking, PII/NER token classification, fine-tuning, quantization, and vector storage (e.g., rerankers, privacy-filter.cpp, TRL, local-store, valkey-store) See the [Backend & Model Compatibility Table]({{%relref "reference/compatibility-table" %}}) for the full catalog. diff --git a/docs/content/features/stores.md b/docs/content/features/stores.md index 02947db8149f..b86037215977 100644 --- a/docs/content/features/stores.md +++ b/docs/content/features/stores.md @@ -44,8 +44,37 @@ Keys are arrays of floating point numbers with a maximum width of 32bits. Values The key vectors must all be the same length and it's best for search performance if they are normalized. When addings keys it will be detected if they are not normalized and what length they are. -All endpoints accept a `store` field which specifies which store to operate on. Presently they are created -on the fly and there is only one store backend so no configuration is required. +All endpoints accept a `store` field which specifies which store to operate on. Stores are created +on the fly. + +All endpoints also accept an optional `backend` field selecting which store backend serves the +request. The default is `local-store`, an in-memory exact-search store. Alternatively, +`valkey-store` (alias `valkey`) stores the vectors in a Valkey server running the +[Valkey Search](https://valkey.io/topics/search/) module (the `valkey/valkey-bundle` image), which +makes them durable across restarts. `valkey-store` is configured through environment variables, +or per-model via the model config's `options:` list, which takes priority over the env var: + +| Variable | Model option | Default | Description | +|---|---|---|---| +| `VALKEY_ADDR` | `valkey_addr` | `127.0.0.1:6379` | Valkey server address | +| `VALKEY_USERNAME` | `valkey_username_env` | | Username, if the server requires AUTH | +| `VALKEY_PASSWORD` | `valkey_password_env` | | Password, if the server requires AUTH | +| `VALKEY_INDEX_ALGO` | `valkey_index_algo` | `FLAT` | Vector index algorithm: `FLAT` (exact, matches `local-store` results) or `HNSW` (approximate, for large corpora) | + +The username/password options name an environment variable to read the credential from, rather +than taking the credential directly, so each model config can point at its own Valkey server with +its own credentials without putting secrets in the YAML (the same pattern `cloud-proxy`'s +`api_key_env` uses): + +```yaml +name: my-valkey-store +backend: valkey-store +options: + - "valkey_addr:valkey.internal:6379" + - "valkey_username_env:MY_VALKEY_USERNAME" + - "valkey_password_env:MY_VALKEY_PASSWORD" + - "valkey_index_algo:HNSW" +``` ## Set diff --git a/docs/content/reference/compatibility-table.md b/docs/content/reference/compatibility-table.md index 0e9551b3bc3c..4ac55b8ddd9c 100644 --- a/docs/content/reference/compatibility-table.md +++ b/docs/content/reference/compatibility-table.md @@ -117,6 +117,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ | [rerankers](https://github.com/AnswerDotAI/rerankers) | Document reranking for RAG | CUDA 12, ROCm, Intel SYCL, Metal | | [privacy-filter.cpp](https://github.com/localai-org/privacy-filter.cpp) | Standalone GGML engine for the openai-privacy-filter PII/NER token-classification model family (powers LocalAI's PII redaction tier) | CPU, CUDA 13, Vulkan | | [local-store](https://github.com/mudler/LocalAI) | Local-first vector database for embeddings | CPU, Metal | +| [valkey-store](https://github.com/mudler/LocalAI) | Durable vector store for embeddings backed by Valkey Search (FLAT or HNSW) | CPU, Metal | | [TRL](https://github.com/huggingface/trl) | Fine-tuning (SFT, DPO, GRPO, RLOO, KTO, ORPO) | CPU, CUDA 12/13 | | [llama.cpp quantization](https://github.com/ggml-org/llama.cpp) | HuggingFace → GGUF model conversion and quantization | CPU, Metal | diff --git a/go.mod b/go.mod index fa3608c620e0..4485301f95f5 100644 --- a/go.mod +++ b/go.mod @@ -155,6 +155,7 @@ require ( github.com/tmc/langchaingo v0.1.14 // indirect github.com/transparency-dev/formats v0.0.0-20251017110053-404c0d5b696c // indirect github.com/transparency-dev/merkle v0.0.2 // indirect + github.com/valkey-io/valkey-go v1.0.76 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect go.mongodb.org/mongo-driver v1.17.6 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect diff --git a/go.sum b/go.sum index 129976bbcd0f..b0c5826082d3 100644 --- a/go.sum +++ b/go.sum @@ -1351,6 +1351,8 @@ github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW github.com/ulikunitz/xz v0.5.14 h1:uv/0Bq533iFdnMHZdRBTOlaNMdb1+ZxXIlHDZHIHcvg= github.com/ulikunitz/xz v0.5.14/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/valkey-io/valkey-go v1.0.76 h1:Rcown7FFseVhG9b0+4MWfMs4xWu8otPzHjrsK044ET4= +github.com/valkey-io/valkey-go v1.0.76/go.mod h1:6X581PhgfeMkJmyfjIsa2eFdq6dy3Qkkg9zwjM1p42M= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.68.0 h1:v12Nx16iepr8r9ySOwqI+5RBJ/DqTxhOy1HrHoDFnok= diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index 43273a662946..1d26ecc3fd2b 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -26,6 +26,7 @@ var Aliases = map[string]string{ "ik_llama": IKLLamaCPP, "ik-llama": IKLLamaCPP, "embedded-store": LocalStoreBackend, + "valkey": ValkeyStoreBackend, "huggingface-embeddings": TransformersBackend, "transformers-musicgen": TransformersBackend, "sentencetransformers": TransformersBackend, @@ -46,6 +47,7 @@ const ( TransformersBackend = "transformers" LocalStoreBackend = "local-store" + ValkeyStoreBackend = "valkey-store" ) // starts the grpcModelProcess for the backend, and returns a grpc client