diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index a4067d4623b2..cbac9284a90a 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -3,6 +3,8 @@ package config import ( "slices" "strings" + + "github.com/mudler/LocalAI/pkg/model" ) // Usecase name constants — the canonical string values used in gallery entries, @@ -678,6 +680,42 @@ func GetBackendCapability(backend string) *BackendCapability { return nil } +// llmAutoLoadUsecases are the usecases that mark a backend able to serve a +// text/LLM GGUF model. A GGUF model that declares no explicit backend must only +// be auto-tried against backends carrying one of these usecases - never against +// audio/codec/image backends (e.g. opus) that happen to be installed alongside +// it (see issue #9287). +var llmAutoLoadUsecases = []string{ + UsecaseChat, + UsecaseCompletion, + UsecaseEdit, + UsecaseEmbeddings, +} + +// isLLMCapableForAutoLoad reports whether the named backend is known to serve +// text/LLM models, for pkg/model's GGUF backend auto-detection (#9287). Backends +// absent from the capability table are treated as not LLM-capable. +func isLLMCapableForAutoLoad(name string) bool { + capability := GetBackendCapability(name) + if capability == nil { + return false + } + for _, u := range capability.PossibleUsecases { + if slices.Contains(llmAutoLoadUsecases, u) { + return true + } + } + return false +} + +func init() { + // Wire the LLM-capability filter into pkg/model's GGUF backend + // auto-detection. pkg/model is a lower-level package and must not import + // core/config (that would form a core/config -> pkg/model -> core/config + // import cycle), so core/config registers the predicate here instead (#9287). + model.RegisterLLMCapableBackendFunc(isLLMCapableForAutoLoad) +} + // VoiceCloningForModel returns the reference-audio contract only when the // installed model variant can honor it. Several backends serve both Base // (voice cloning) and CustomVoice/VoiceDesign models, so backend name alone is diff --git a/pkg/model/autoload.go b/pkg/model/autoload.go new file mode 100644 index 000000000000..d53186283647 --- /dev/null +++ b/pkg/model/autoload.go @@ -0,0 +1,97 @@ +package model + +import ( + "sort" + "strings" +) + +// preferredGGUFBackend is tried first when auto-detecting the backend for a +// GGUF model, since GGUF is overwhelmingly llama.cpp's native format. +const preferredGGUFBackend = "llama-cpp" + +// llmCapableBackend reports whether the named backend can serve a text/LLM GGUF +// model. The backend capability table lives in core/config, which is a +// higher-level package that already imports pkg/model; importing it back here +// would form a core/config -> pkg/model -> core/config cycle. So core/config +// registers the predicate via RegisterLLMCapableBackendFunc instead (see #9287). +// When unset (e.g. a build that never imports core/config) GGUF capability +// filtering is skipped and auto-detect falls back to the deterministic set. +var llmCapableBackend func(name string) bool + +// RegisterLLMCapableBackendFunc wires the LLM-capability predicate used by +// SelectAutoLoadBackends. It is called from core/config's init so pkg/model +// need not import core/config (see #9287). +func RegisterLLMCapableBackendFunc(fn func(name string) bool) { + llmCapableBackend = fn +} + +// SelectAutoLoadBackends returns the ordered, deterministic list of backend +// names to try when loading a model that declares no explicit backend. +// +// available is the set of installed backend names (unordered, as it comes from a +// Go map). modelFile is the model file name/path (may be empty). +// +// The trial loop in (*ModelLoader).Load picks the first backend whose gRPC +// LoadModel succeeds, so the order and membership of this list directly decide +// which backend wins. The previous implementation ranged a Go map (random +// order) with no filtering, so an unrelated installed backend such as the +// "opus" audio codec could win a GGUF/LLM model load (#9287). +// +// Behaviour: +// - The result is always deterministically ordered, so auto-detect no longer +// depends on map iteration order. +// - For a GGUF model file the list is filtered to LLM-capable backends and +// llama-cpp is placed first, so an incompatible audio/codec/image backend +// can never win the trial loop. +// - If filtering would leave no candidate, the full sorted set is returned +// instead, so a model that previously loaded never becomes unloadable. +func SelectAutoLoadBackends(available []string, modelFile string) []string { + sorted := append([]string(nil), available...) + sort.Strings(sorted) + + if !isGGUFModelFile(modelFile) { + return sorted + } + + // No capability predicate wired (core/config not linked in): skip filtering + // rather than risk dropping a valid candidate. + if llmCapableBackend == nil { + return sorted + } + + filtered := make([]string, 0, len(sorted)) + hasLlama := false + for _, b := range sorted { + if b == preferredGGUFBackend { + hasLlama = true + continue // added explicitly first below + } + if isLLMCapableBackend(b) { + filtered = append(filtered, b) + } + } + if hasLlama { + filtered = append([]string{preferredGGUFBackend}, filtered...) + } + + if len(filtered) == 0 { + // Conservative fallback: no known LLM-capable backend is installed, so + // rather than refuse to load, fall back to the previous behaviour of + // trying every installed backend (now at least in a deterministic order). + return sorted + } + return filtered +} + +func isGGUFModelFile(modelFile string) bool { + return strings.HasSuffix(strings.ToLower(modelFile), ".gguf") +} + +// isLLMCapableBackend reports whether a backend is known to serve text/LLM +// models. Backends absent from the capability map (unknown) are treated as +// not LLM-capable here: for GGUF auto-detection we only want backends we can +// positively confirm handle LLMs, and the zero-candidate fallback keeps unknown +// setups working. Callers must ensure llmCapableBackend is non-nil. +func isLLMCapableBackend(name string) bool { + return llmCapableBackend(name) +} diff --git a/pkg/model/autoload_test.go b/pkg/model/autoload_test.go new file mode 100644 index 000000000000..9142639afd44 --- /dev/null +++ b/pkg/model/autoload_test.go @@ -0,0 +1,55 @@ +package model_test + +import ( + "github.com/mudler/LocalAI/pkg/model" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func init() { + // The real LLM-capability table lives in core/config, which pkg/model must + // not import (cycle). Register a fake predicate so the selection algorithm + // is exercised here independent of the capability table (#9287). + model.RegisterLLMCapableBackendFunc(func(name string) bool { + return name == "llama-cpp" || name == "vllm" + }) +} + +var _ = Describe("SelectAutoLoadBackends (#9287)", func() { + Describe("GGUF model auto-detection", func() { + It("excludes incompatible audio/codec backends (e.g. opus) for a .gguf model", func() { + // Regression for #9287: installing an unrelated audio backend like + // "opus" must never win the GGUF auto-detect trial loop. + got := model.SelectAutoLoadBackends([]string{"opus", "llama-cpp"}, "Qwen3.5-9b.gguf") + Expect(got).NotTo(ContainElement("opus")) + Expect(got).To(ContainElement("llama-cpp")) + }) + + It("places llama-cpp first for a .gguf model", func() { + got := model.SelectAutoLoadBackends([]string{"vllm", "opus", "llama-cpp"}, "model.gguf") + Expect(got).NotTo(BeEmpty()) + Expect(got[0]).To(Equal("llama-cpp")) + }) + + It("is deterministic regardless of input ordering", func() { + a := model.SelectAutoLoadBackends([]string{"opus", "vllm", "llama-cpp", "whisper"}, "m.gguf") + b := model.SelectAutoLoadBackends([]string{"whisper", "llama-cpp", "vllm", "opus"}, "m.gguf") + Expect(a).To(Equal(b)) + }) + + It("falls back to the full sorted set when filtering leaves no candidate", func() { + // No LLM-capable backend installed: never make a previously-loadable + // model unloadable, return the original set (sorted). + got := model.SelectAutoLoadBackends([]string{"opus"}, "model.gguf") + Expect(got).To(Equal([]string{"opus"})) + }) + }) + + Describe("non-GGUF model auto-detection", func() { + It("returns a deterministic (sorted) set without filtering", func() { + got := model.SelectAutoLoadBackends([]string{"opus", "llama-cpp", "diffusers"}, "model-dir") + Expect(got).To(Equal([]string{"diffusers", "llama-cpp", "opus"})) + }) + }) +}) diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index 7d9afca050fc..bdec48e2f91e 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -424,14 +424,17 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) { // Otherwise scan for backends in the asset directory var err error - // get backends embedded in the binary - autoLoadBackends := []string{} - - // append externalBackends supplied by the user via the CLI + // Collect the installed/external backends (the map is unordered). + available := []string{} for b := range ml.GetAllExternalBackends(o) { - autoLoadBackends = append(autoLoadBackends, b) + available = append(available, b) } + // Build a deterministic, file-type-filtered candidate list so an + // incompatible backend (e.g. an audio codec like opus) can never win the + // trial loop for a GGUF/LLM model. See SelectAutoLoadBackends / #9287. + autoLoadBackends := SelectAutoLoadBackends(available, o.model) + if len(autoLoadBackends) == 0 { xlog.Error("No backends found") return nil, fmt.Errorf("no backends found")