Local rag

build.gradle.kts

@@ -1,3 +1,6 @@
+import java.io.File
+import java.net.URL
+
 plugins {
     java
     // Must be compatible with Kotlin compiler bundled with IntelliJ IDEA 2025.3 (metadata 2.2.0)
@@ -8,6 +11,8 @@ plugins {
 
 val ideaVersion = "2025.3"
 val javaVersion = 21
+val onnxRuntimeVersion = "1.20.0"
+val djlTokenizersVersion = "0.33.0"
 
 group = "com.lsfusion"
 version = file("META-INF/plugin.xml").let {
@@ -27,6 +32,15 @@ repositories {
     }
 }
 
+configurations.all {
+    resolutionStrategy.eachDependency {
+        if (requested.group == "com.microsoft.onnxruntime") {
+            useVersion(onnxRuntimeVersion)
+            because("Keep all ONNX Runtime jars on one version")
+        }
+    }
+}
+
 java {
     // Compile with a stable JDK toolchain.
     // `runIde` in your environment was using javac 22.0.1 and crashed with StackOverflowError inside javac parser.
@@ -111,6 +125,10 @@ dependencies {
     implementation("org.json:json:20180813")
     implementation("org.jsoup:jsoup:1.15.3")
     implementation("net.gcardone.junidecode:junidecode:0.5.2")
+    implementation("com.microsoft.onnxruntime:onnxruntime:$onnxRuntimeVersion")
+    implementation("ai.djl.huggingface:tokenizers:$djlTokenizersVersion")
+    implementation("org.apache.lucene:lucene-core:10.3.2")
+    implementation("org.apache.lucene:lucene-analysis-common:10.3.2")
 
     // Needed by MCP toolset DTOs; compiler plugin generates serializers.
     compileOnly("org.jetbrains.kotlinx:kotlinx-serialization-json:1.7.3")
@@ -132,6 +150,78 @@ dependencies {
     }
 }
 
+val modelDir = layout.projectDirectory.dir(".mcp-model")
+val e5ModelUrl = "https://huggingface.co/intfloat/e5-small/resolve/main/model.onnx?download=true"
+val e5TokenizerUrl = "https://huggingface.co/intfloat/e5-small/resolve/main/tokenizer.json?download=true"
+val onnxNativeDir = layout.buildDirectory.dir("onnxruntime-native")
+val onnxTempDir = layout.buildDirectory.dir("onnxruntime-tmp")
+val onnxRuntimeDll = onnxNativeDir.map { it.file("onnxruntime.dll") }
+val onnxRuntimeJniDll = onnxNativeDir.map { it.file("onnxruntime4j_jni.dll") }
+
+tasks.register("downloadE5Model") {
+    group = "mcp"
+    description = "Download e5-small ONNX model and tokenizer if missing"
+    notCompatibleWithConfigurationCache("Downloads model files to project directory")
+    doLast {
+        val dirFile = modelDir.asFile
+        if (!dirFile.exists()) {
+            dirFile.mkdirs()
+        }
+        val modelFile = modelDir.file("model.onnx").asFile
+        val tokenizerFile = modelDir.file("tokenizer.json").asFile
+
+        fun downloadIfMissing(url: String, target: File) {
+            if (target.exists() && target.length() > 0) return
+            URL(url).openStream().use { input ->
+                target.outputStream().use { output -> input.copyTo(output) }
+            }
+        }
+
+        downloadIfMissing(e5ModelUrl, modelFile)
+        downloadIfMissing(e5TokenizerUrl, tokenizerFile)
+    }
+}
+
+val extractOnnxRuntimeNative by tasks.registering(Sync::class) {
+    group = "mcp"
+    description = "Extract ONNX Runtime native libs into build dir"
+    val runtimeClasspath = configurations.runtimeClasspath
+    from({
+        runtimeClasspath.get()
+            .filter { it.name.startsWith("onnxruntime-") && it.extension == "jar" }
+            .map { zipTree(it) }
+    }) {
+        include("**/onnxruntime*.dll")
+        include("**/onnxruntime_providers_*.dll")
+        include("**/onnxruntime*.so")
+        include("**/onnxruntime*.dylib")
+        include("**/libonnxruntime*.dylib")
+        include("**/onnxruntime4j_jni*")
+        include("**/libonnxruntime4j_jni*")
+    }
+    into(onnxNativeDir)
+    includeEmptyDirs = false
+    duplicatesStrategy = org.gradle.api.file.DuplicatesStrategy.EXCLUDE
+    eachFile { path = name }
+}
+
+val createOnnxTempDir by tasks.registering {
+    group = "mcp"
+    description = "Create ONNX Runtime temp dir"
+    doLast {
+        onnxTempDir.get().asFile.mkdirs()
+    }
+}
+
+tasks.withType<org.jetbrains.intellij.platform.gradle.tasks.RunIdeTask>().configureEach {
+    dependsOn("downloadE5Model", extractOnnxRuntimeNative, createOnnxTempDir)
+    jvmArgs("-Dlsfusion.mcp.embedding.modelDir=${modelDir.asFile.absolutePath}")
+    jvmArgs("-Donnxruntime.native.path=${onnxNativeDir.get().asFile.absolutePath}")
+    jvmArgs("-Donnxruntime.native.onnxruntime.path=${onnxRuntimeDll.get().asFile.absolutePath}")
+    jvmArgs("-Donnxruntime.native.onnxruntime4j_jni.path=${onnxRuntimeJniDll.get().asFile.absolutePath}")
+    jvmArgs("-Djava.io.tmpdir=${onnxTempDir.get().asFile.absolutePath}")
+}
+
 intellijPlatform {
     pluginVerification {
         ides {

src/com/lsfusion/LSFBaseStartupActivity.java

@@ -7,6 +7,8 @@
 import com.intellij.openapi.vfs.VirtualFileManager;
 import com.intellij.openapi.vfs.impl.BulkVirtualFileListenerAdapter;
 import com.lsfusion.actions.locale.LSFPropertiesFileListener;
+import com.lsfusion.mcp.LSFMcpRagFileListener;
+import com.lsfusion.mcp.LocalMcpRagService;
 import kotlin.Unit;
 import kotlin.coroutines.Continuation;
 import org.jetbrains.annotations.NotNull;
@@ -17,6 +19,12 @@ public class LSFBaseStartupActivity implements ProjectActivity, DumbAware {
     public @Nullable Object execute(@NotNull Project project, @NotNull Continuation<? super Unit> continuation) {
         project.getMessageBus().connect().subscribe(FileEditorManagerListener.FILE_EDITOR_MANAGER, new LSFFileEditorManagerListener());
         project.getMessageBus().connect().subscribe(VirtualFileManager.VFS_CHANGES, new BulkVirtualFileListenerAdapter(new LSFPropertiesFileListener(project)));
+        project.getMessageBus().connect().subscribe(VirtualFileManager.VFS_CHANGES, new BulkVirtualFileListenerAdapter(new LSFMcpRagFileListener(project)));
+        try {
+            LocalMcpRagService.getInstance(project).indexProjectAsync();
+        } catch (Exception ignored) {
+            // indexing service not available
+        }
         return Unit.INSTANCE;
     }
 }

src/com/lsfusion/mcp/EmbeddingProvider.java

@@ -0,0 +1,11 @@
+package com.lsfusion.mcp;
+
+public interface EmbeddingProvider extends AutoCloseable {
+    float[] embed(String text) throws Exception;
+    int dimension();
+
+    @Override
+    default void close() throws Exception {
+        // no-op by default
+    }
+}

src/com/lsfusion/mcp/LSFMcpRagFileListener.java

@@ -0,0 +1,64 @@
+package com.lsfusion.mcp;
+
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.VirtualFileCopyEvent;
+import com.intellij.openapi.vfs.VirtualFileEvent;
+import com.intellij.openapi.vfs.VirtualFileListener;
+import com.intellij.openapi.vfs.VirtualFileMoveEvent;
+import com.intellij.openapi.vfs.VirtualFilePropertyEvent;
+import org.jetbrains.annotations.NotNull;
+
+public class LSFMcpRagFileListener implements VirtualFileListener {
+    private final Project project;
+
+    public LSFMcpRagFileListener(Project project) {
+        this.project = project;
+    }
+
+    @Override
+    public void contentsChanged(@NotNull VirtualFileEvent event) {
+        update(event.getFile());
+    }
+
+    @Override
+    public void fileCreated(@NotNull VirtualFileEvent event) {
+        update(event.getFile());
+    }
+
+    @Override
+    public void fileDeleted(@NotNull VirtualFileEvent event) {
+        delete(event.getFile());
+    }
+
+    @Override
+    public void fileMoved(@NotNull VirtualFileMoveEvent event) {
+        update(event.getFile());
+    }
+
+    @Override
+    public void fileCopied(@NotNull VirtualFileCopyEvent event) {
+        update(event.getFile());
+    }
+
+    @Override
+    public void propertyChanged(@NotNull VirtualFilePropertyEvent event) {
+        update(event.getFile());
+    }
+
+    private void update(VirtualFile file) {
+        try {
+            LocalMcpRagService.getInstance(project).updateFile(file);
+        } catch (Exception ignored) {
+            // service not available
+        }
+    }
+
+    private void delete(VirtualFile file) {
+        try {
+            LocalMcpRagService.getInstance(project).deleteFile(file);
+        } catch (Exception ignored) {
+            // service not available
+        }
+    }
+}