Add async inference loading with CATNIP_INFERENCE env flag

- Implement non-blocking background initialization for inference service
- Add state management (initializing/ready/failed/disabled) with progress tracking
- Return 503 with status info while model downloads in background
- Add retry logic with exponential backoff (3 attempts)
- Use golang.org/x/sys/unix for cross-platform stderr suppression
- Clean up .gitignore (remove models/) and .goreleaser.yml (remove bundled libs)

The inference service now starts immediately and downloads libraries/model
in the background. Enable with CATNIP_INFERENCE=1 environment variable.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

Author: Claude

.gitignore

@@ -64,9 +64,3 @@ container/internal/assets/dist/*
 # Xcode user-specific files
 **/xcuserdata/
 xcode/build/
-
-# Inference: llama.cpp libraries and GGUF models
-# Libraries are downloaded at build time, not committed
-container/models/lib/
-models/*.gguf
-*.gguf

container/.goreleaser.yml

@@ -111,17 +111,6 @@ archives:
       # Copy the entire signed app bundle - keep the Catnip.app directory name
       - src: "dist/catnip-macos_darwin_amd64_v1/Catnip.app"
         dst: "Catnip.app"
-      # Include llama.cpp libraries for local inference
-      - src: "models/lib/darwin/amd64/build/bin/libllama.dylib"
-        dst: "lib/libllama.dylib"
-      - src: "models/lib/darwin/amd64/build/bin/libggml.dylib"
-        dst: "lib/libggml.dylib"
-      - src: "models/lib/darwin/amd64/build/bin/libggml-metal.dylib"
-        dst: "lib/libggml-metal.dylib"
-      - src: "models/lib/darwin/amd64/build/bin/libggml-base.dylib"
-        dst: "lib/libggml-base.dylib"
-      - src: "models/lib/darwin/amd64/build/bin/libggml-cpu.dylib"
-        dst: "lib/libggml-cpu.dylib"
       # Documentation files
       - README.md
       - LICENSE
@@ -139,17 +128,6 @@ archives:
       # Copy the entire signed app bundle - keep the Catnip.app directory name
       - src: "dist/catnip-macos_darwin_arm64_v8.0/Catnip.app"
         dst: "Catnip.app"
-      # Include llama.cpp libraries for local inference
-      - src: "models/lib/darwin/arm64/build/bin/libllama.dylib"
-        dst: "lib/libllama.dylib"
-      - src: "models/lib/darwin/arm64/build/bin/libggml.dylib"
-        dst: "lib/libggml.dylib"
-      - src: "models/lib/darwin/arm64/build/bin/libggml-metal.dylib"
-        dst: "lib/libggml-metal.dylib"
-      - src: "models/lib/darwin/arm64/build/bin/libggml-base.dylib"
-        dst: "lib/libggml-base.dylib"
-      - src: "models/lib/darwin/arm64/build/bin/libggml-cpu.dylib"
-        dst: "lib/libggml-cpu.dylib"
       # Documentation files
       - README.md
       - LICENSE

container/internal/cmd/serve.go

@@ -154,20 +154,21 @@ func startServer(cmd *cobra.Command) {
 	claudeService := services.NewClaudeService()
 	sessionService := services.NewSessionService()
 
-	// Initialize inference service (cross-platform support via yzma FFI)
+	// Initialize inference service if enabled via CATNIP_INFERENCE=1
 	var inferenceService *services.InferenceService
-	inferenceConfig := services.InferenceConfig{
-		ModelURL: "https://huggingface.co/vanpelt/catnip-summarizer/resolve/main/gemma3-270m-summarizer-Q4_K_M.gguf",
-		Checksum: "", // Optional checksum for verification
-	}
-	var err error
-	inferenceService, err = services.NewInferenceService(inferenceConfig)
-	if err != nil {
-		logger.Warnf("⚠️  Failed to initialize inference service: %v", err)
-		logger.Warnf("   Run 'catnip download' to pre-download dependencies")
-		inferenceService = nil
+	if os.Getenv("CATNIP_INFERENCE") == "1" {
+		inferenceConfig := services.InferenceConfig{
+			ModelURL: "https://huggingface.co/vanpelt/catnip-summarizer/resolve/main/gemma3-270m-summarizer-Q4_K_M.gguf",
+			Checksum: "", // Optional checksum for verification
+		}
+		inferenceService = services.NewInferenceService(inferenceConfig)
+
+		// Start background initialization (non-blocking)
+		go inferenceService.InitializeAsync()
+
+		logger.Infof("🧠 Inference service enabled, downloading in background... (%s/%s)", goruntime.GOOS, goruntime.GOARCH)
 	} else {
-		logger.Infof("✅ Inference service initialized (%s/%s)", goruntime.GOOS, goruntime.GOARCH)
+		logger.Debugf("🧠 Inference service disabled (set CATNIP_INFERENCE=1 to enable)")
 	}
 
 	// Wire up SessionService to ClaudeService for best session file selection
@@ -224,7 +225,11 @@ func startServer(cmd *cobra.Command) {
 	defer eventsHandler.Stop()
 	portsHandler := handlers.NewPortsHandler(portMonitor).WithEvents(eventsHandler)
 	proxyHandler := handlers.NewProxyHandler(portMonitor)
-	inferenceHandler := handlers.NewInferenceHandler(inferenceService)
+	// Only create inference handler if service is enabled
+	var inferenceHandler *handlers.InferenceHandler
+	if inferenceService != nil {
+		inferenceHandler = handlers.NewInferenceHandler(inferenceService)
+	}
 
 	// Connect events handler to GitService for worktree status events
 	gitService.SetEventsHandler(eventsHandler)
@@ -309,9 +314,11 @@ func startServer(cmd *cobra.Command) {
 	v1.Post("/ports/mappings", portsHandler.SetPortMapping)
 	v1.Delete("/ports/mappings/:port", portsHandler.DeletePortMapping)
 
-	// Inference routes (cross-platform local inference)
-	v1.Post("/inference/summarize", inferenceHandler.HandleSummarize)
-	v1.Get("/inference/status", inferenceHandler.HandleInferenceStatus)
+	// Inference routes (only if enabled via CATNIP_INFERENCE=1)
+	if inferenceHandler != nil {
+		v1.Post("/inference/summarize", inferenceHandler.HandleSummarize)
+		v1.Get("/inference/status", inferenceHandler.HandleInferenceStatus)
+	}
 
 	// Server info route
 	v1.Get("/info", func(c *fiber.Ctx) error {

container/internal/cmd/summarize.go

@@ -61,9 +61,15 @@ func runSummarize(cmd *cobra.Command, args []string) error {
 		Checksum: "",
 	}
 
-	inferenceService, err := services.NewInferenceService(inferenceConfig)
-	if err != nil {
-		return fmt.Errorf("failed to initialize inference service: %w\n\nTry running: catnip download", err)
+	inferenceService := services.NewInferenceService(inferenceConfig)
+
+	// Run initialization synchronously for CLI usage
+	inferenceService.InitializeAsync()
+
+	// Check if initialization succeeded
+	if !inferenceService.IsReady() {
+		state, message, _ := inferenceService.GetStatus()
+		return fmt.Errorf("failed to initialize inference service: %s (%s)\n\nTry running: catnip download", message, state)
 	}
 
 	// Run inference

container/internal/handlers/inference.go

@@ -40,16 +40,18 @@ type SummarizeResponse struct {
 // InferenceStatusResponse represents the inference service status
 // @Description Status of the local inference service
 type InferenceStatusResponse struct {
-	// Whether inference is available on this platform
+	// Whether inference is ready for requests
 	Available bool `json:"available" example:"true"`
+	// Current status: initializing, ready, failed
+	Status string `json:"status" example:"ready"`
+	// Human-readable status message
+	Message string `json:"message,omitempty" example:"Inference service ready"`
+	// Download progress (when initializing)
+	Progress *services.DownloadProgress `json:"progress,omitempty"`
 	// Platform name (darwin, linux, windows)
 	Platform string `json:"platform" example:"darwin"`
 	// Architecture (amd64, arm64)
 	Architecture string `json:"architecture" example:"arm64"`
-	// Model path if loaded
-	ModelPath string `json:"modelPath,omitempty" example:"/Users/user/.catnip/models/gemma3-270m-summarizer-Q4_K_M.gguf"`
-	// Error message if initialization failed
-	Error string `json:"error,omitempty" example:"model not found"`
 }
 
 // HandleSummarize godoc
@@ -65,10 +67,20 @@ type InferenceStatusResponse struct {
 // @Failure 503 {object} fiber.Map "Inference not available on this platform"
 // @Router /v1/inference/summarize [post]
 func (h *InferenceHandler) HandleSummarize(c *fiber.Ctx) error {
-	// Check if service is available
+	// Check if service is available and ready
 	if h.service == nil {
 		return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
-			"error": "Inference service not available on this platform",
+			"error": "Inference service not configured",
+		})
+	}
+
+	// Check if service is ready
+	if !h.service.IsReady() {
+		state, message, progress := h.service.GetStatus()
+		return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
+			"error":    fmt.Sprintf("Inference service not ready: %s", message),
+			"status":   string(state),
+			"progress": progress,
 		})
 	}
 
@@ -114,18 +126,27 @@ func (h *InferenceHandler) HandleSummarize(c *fiber.Ctx) error {
 // @Success 200 {object} InferenceStatusResponse "Inference service status"
 // @Router /v1/inference/status [get]
 func (h *InferenceHandler) HandleInferenceStatus(c *fiber.Ctx) error {
-	status := InferenceStatusResponse{
-		Available:    h.service != nil,
+	resp := InferenceStatusResponse{
 		Platform:     runtime.GOOS,
 		Architecture: runtime.GOARCH,
 	}
 
-	if h.service != nil {
-		// Try to get model path (implementation would need to expose this)
-		status.ModelPath = "~/.catnip/models/gemma3-270m-summarizer-Q4_K_M.gguf"
-	} else {
-		status.Error = "Inference only available on macOS currently"
+	if h.service == nil {
+		resp.Available = false
+		resp.Status = "disabled"
+		resp.Message = "Inference service not configured"
+		return c.JSON(resp)
+	}
+
+	state, message, progress := h.service.GetStatus()
+	resp.Available = h.service.IsReady()
+	resp.Status = string(state)
+	resp.Message = message
+
+	// Include progress if still initializing
+	if state == services.InferenceStateInitializing {
+		resp.Progress = &progress
 	}
 
-	return c.JSON(status)
+	return c.JSON(resp)
 }