Solr Cell: index Tika 4.x tk:chunks into a dense_vector field

changelog/unreleased/PR#4853-update-org-apache-tika-tika-core.yml

@@ -0,0 +1,7 @@
+title: Update org.apache.tika:tika-core to v4
+type: dependency_update
+authors:
+- name: solrbot
+links:
+- name: PR#4853
+  url: https://github.com/apache/solr/pull/4853

changelog/unreleased/SolrCellTikaChunks.yml

@@ -0,0 +1,4 @@
+title: Solr Cell can index Tika 4.x chunk embeddings into dense_vector fields via new tikaserver.chunks parameter
+type: added
+authors:
+  - name: Eric Pugh

gradle/libs.versions.toml

@@ -42,7 +42,7 @@ apache-log4j = "2.26.1"
 apache-lucene = "10.4.0"
 apache-opennlp = "2.5.10"
 apache-rat = "0.18"
-apache-tika = "3.3.1"
+apache-tika = "4.0.0"
 apache-tomcat = "6.0.53"
 apache-zookeeper = "3.9.5"
 # @keep for version alignment

solr/licenses/commonmark-0.30.0.jar.sha1

@@ -0,0 +1 @@
+fc611136b3191a08d2b84c5d5a26116e3531afee

solr/licenses/commonmark-LICENSE-BSD.txt

@@ -0,0 +1,23 @@
+Copyright (c) 2015, Robin Stocker
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

solr/licenses/commonmark-ext-gfm-strikethrough-0.30.0.jar.sha1

@@ -0,0 +1 @@
+e9addacba704ffce26d6b8bcd1c50e235ec2c991

solr/licenses/commonmark-ext-gfm-tables-0.30.0.jar.sha1

@@ -0,0 +1 @@
+84560a4af5afe4808543bd9c5587a660ce148b14

solr/licenses/tika-core-4.0.0.jar.sha1

@@ -0,0 +1 @@
+f370d8619de9e19329b2cc117943c83468c2ff9f

solr/modules/extraction/build.gradle

@@ -46,4 +46,6 @@ dependencies {
   testImplementation libs.junit.junit
   testImplementation libs.testcontainers
   testImplementation libs.carrotsearch.randomizedtesting.runner
+  // Embedded mock HTTP server for TikaServerExtractionBackendTest's chunk-embeddings tests
+  testImplementation libs.eclipse.jetty.server
 }

solr/modules/extraction/gradle.lockfile

@@ -118,11 +118,14 @@ org.apache.lucene:lucene-spatial-extras:10.4.0=jarValidation,runtimeClasspath,ru
 org.apache.lucene:lucene-spatial3d:10.4.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
 org.apache.lucene:lucene-suggest:10.4.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
 org.apache.lucene:lucene-test-framework:10.4.0=jarValidation,testCompileClasspath,testRuntimeClasspath
-org.apache.tika:tika-core:3.3.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
+org.apache.tika:tika-core:4.0.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
 org.apache.zookeeper:zookeeper-jute:3.9.5=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
 org.apache.zookeeper:zookeeper:3.9.5=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
 org.apiguardian:apiguardian-api:1.1.2=jarValidation,testRuntimeClasspath
 org.codehaus.woodstox:stax2-api:4.3.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
+org.commonmark:commonmark-ext-gfm-strikethrough:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
+org.commonmark:commonmark-ext-gfm-tables:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
+org.commonmark:commonmark:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
 org.eclipse.jetty.compression:jetty-compression-common:12.1.10=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
 org.eclipse.jetty.compression:jetty-compression-gzip:12.1.10=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
 org.eclipse.jetty.ee10:jetty-ee10-servlet:12.1.10=jarValidation,testRuntimeClasspath
@@ -140,7 +143,7 @@ org.eclipse.jetty:jetty-http:12.1.10=compileClasspath,jarValidation,runtimeClass
 org.eclipse.jetty:jetty-io:12.1.10=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
 org.eclipse.jetty:jetty-rewrite:12.1.10=jarValidation,testRuntimeClasspath
 org.eclipse.jetty:jetty-security:12.1.10=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
-org.eclipse.jetty:jetty-server:12.1.10=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
+org.eclipse.jetty:jetty-server:12.1.10=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
 org.eclipse.jetty:jetty-session:12.1.10=jarValidation,testRuntimeClasspath
 org.eclipse.jetty:jetty-util:12.1.10=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
 org.glassfish.hk2.external:aopalliance-repackaged:4.0.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath

solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java

@@ -19,10 +19,14 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.lang.invoke.MethodHandles;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import java.util.regex.Pattern;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.params.UpdateParams;
 import org.apache.solr.common.util.ContentStream;
@@ -143,8 +147,24 @@ public void load(
               .tikaServerRecursive(tikaserverRecursive)
               .tikaServerTimeoutSeconds(tikaTimeoutSecs)
               .tikaServerRequestHeaders(Map.of())
+              .tikaServerConfigJson(params.get(ExtractingParams.TIKASERVER_CONFIG_JSON))
+              .tikaServerChunks(params.getBool(ExtractingParams.TIKASERVER_CHUNKS, false))
               .build();
 
+      if (extractionRequest.tikaServerChunks) {
+        try {
+          loadChunks(req, params, stream, inputStream, extractionRequest);
+        } catch (Exception e) {
+          if (ignoreTikaException) {
+            if (log.isWarnEnabled())
+              log.warn("skip extracting chunks due to {}.", e.getLocalizedMessage(), e);
+            return;
+          }
+          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+        }
+        return;
+      }
+
       boolean captureAttr = params.getBool(ExtractingParams.CAPTURE_ATTRIBUTES, false);
       String[] captureElems = params.getParams(ExtractingParams.CAPTURE_ELEMENTS);
       boolean needsSaxParsing =
@@ -231,6 +251,69 @@ public void load(
     }
   }
 
+  /**
+   * Indexes one Solr document per Tika 4.x {@code tk:chunks} entry instead of one document for the
+   * whole source file. See {@link ExtractingParams#TIKASERVER_CHUNKS}.
+   */
+  private void loadChunks(
+      SolrQueryRequest req,
+      SolrParams params,
+      ContentStream stream,
+      InputStream inputStream,
+      ExtractionRequest extractionRequest)
+      throws Exception {
+    if (!(backend instanceof TikaServerExtractionBackend tikaServerBackend)) {
+      throw new SolrException(
+          SolrException.ErrorCode.BAD_REQUEST,
+          ExtractingParams.TIKASERVER_CHUNKS
+              + "=true requires the "
+              + TikaServerExtractionBackend.NAME
+              + " extraction backend");
+    }
+    List<TikaServerExtractionBackend.Chunk> chunks =
+        tikaServerBackend.extractChunks(inputStream, extractionRequest);
+
+    String contentField = params.get(ExtractingParams.TIKASERVER_CHUNKS_CONTENT_FIELD, "content");
+    String vectorField = params.get(ExtractingParams.TIKASERVER_CHUNKS_VECTOR_FIELD, "vector");
+    String parentField =
+        params.get(ExtractingParams.TIKASERVER_CHUNKS_PARENT_FIELD, "chunk_parent_id");
+
+    String uniqueKeyField =
+        req.getSchema().getUniqueKeyField() != null
+            ? req.getSchema().getUniqueKeyField().getName()
+            : "id";
+
+    String parentId = params.get(ExtractingParams.LITERALS_PREFIX + uniqueKeyField);
+    if (parentId == null) parentId = params.get(ExtractingParams.RESOURCE_NAME);
+    if (parentId == null) parentId = stream.getName();
+    if (parentId == null) parentId = UUID.randomUUID().toString();
+
+    int n = 0;
+    for (TikaServerExtractionBackend.Chunk chunk : chunks) {
+      SolrInputDocument doc = new SolrInputDocument();
+      doc.setField(uniqueKeyField, parentId + "-chunk-" + n);
+      doc.setField(contentField, chunk.text);
+      doc.setField(vectorField, chunk.vector);
+      doc.setField(parentField, parentId);
+
+      Iterator<String> paramNames = params.getParameterNamesIterator();
+      while (paramNames.hasNext()) {
+        String pname = paramNames.next();
+        if (!pname.startsWith(ExtractingParams.LITERALS_PREFIX)) continue;
+        String name = pname.substring(ExtractingParams.LITERALS_PREFIX.length());
+        if (name.equals(uniqueKeyField)) continue;
+        for (String v : params.getParams(pname)) {
+          doc.addField(name, v);
+        }
+      }
+
+      templateAdd.clear();
+      templateAdd.solrDoc = doc;
+      processor.processAdd(templateAdd);
+      n++;
+    }
+  }
+
   /*
    * Extracts content from the given input stream using an optional XPath expression
    * and a SAX content handler. The extraction process may filter content based on

solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java

@@ -157,4 +157,39 @@ public interface ExtractingParams {
 
   /** Default or per-request timeout in seconds for TikaServer HTTP calls. */
   String TIKASERVER_TIMEOUT_SECS = "tikaserver.timeoutSeconds";
+
+  /**
+   * Optional raw JSON object sent as the "config" part of a per-request TikaServer configuration
+   * call (e.g. {@code {"pdf-parser":{"ocr":{"strategy":"no_ocr"}}}}). Tika Server 4.x removed its
+   * X-Tika-* configuration headers in favor of this JSON mechanism; the server must additionally
+   * have {@code allowPerRequestConfig=true} set, or the request is rejected with 403. Ignored for
+   * recursive (tikaserver.recursive) requests, since TikaServer has no XML-output variant of
+   * /rmeta/config.
+   */
+  String TIKASERVER_CONFIG_JSON = "tikaserver.config";
+
+  /**
+   * If {@code true}, splits the extracted document into Tika 4.x's {@code tk:chunks} (produced by a
+   * server-side embedding metadata filter, e.g. {@code openai-embedding-filter}) and indexes one
+   * Solr document per chunk, each carrying its own embedding vector, instead of one document for
+   * the whole source file. Requires the Tika Server to be configured with an embedding filter;
+   * throws if the response carries no {@code tk:chunks}. See {@link
+   * #TIKASERVER_CHUNKS_CONTENT_FIELD}, {@link #TIKASERVER_CHUNKS_VECTOR_FIELD}, and {@link
+   * #TIKASERVER_CHUNKS_PARENT_FIELD} for the Solr field names used; all three must already exist in
+   * the schema.
+   */
+  String TIKASERVER_CHUNKS = "tikaserver.chunks";
+
+  /** Solr field to hold a chunk's text when {@link #TIKASERVER_CHUNKS} is enabled. */
+  String TIKASERVER_CHUNKS_CONTENT_FIELD = "tikaserver.chunks.contentField";
+
+  /** Solr field to hold a chunk's embedding vector when {@link #TIKASERVER_CHUNKS} is enabled. */
+  String TIKASERVER_CHUNKS_VECTOR_FIELD = "tikaserver.chunks.vectorField";
+
+  /**
+   * Solr field to hold the source document's id on every chunk document when {@link
+   * #TIKASERVER_CHUNKS} is enabled. Chunks are indexed as plain sibling documents referencing their
+   * parent by this field, not as Solr nested/block-join child documents.
+   */
+  String TIKASERVER_CHUNKS_PARENT_FIELD = "tikaserver.chunks.parentField";
 }

solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java

@@ -38,6 +38,8 @@ public class ExtractionRequest {
   public final boolean tikaServerRecursive;
   public final Integer tikaServerTimeoutSeconds; // optional per-request override
   public final Map<String, String> tikaServerRequestHeaders = new HashMap<>();
+  public final String tikaServerConfigJson; // optional raw JSON "config" part; TikaServer 4.x only
+  public final boolean tikaServerChunks; // split into tk:chunks instead of one whole document
 
   /**
    * Constructs an ExtractionRequest object containing metadata and configurations for extraction
@@ -59,6 +61,10 @@ public class ExtractionRequest {
    *     only). If null or ≤ 0, the default timeout will be used
    * @param tikaServerRequestHeaders optional headers to be included in requests to the extraction
    *     service. TikaServer only
+   * @param tikaServerConfigJson optional raw JSON object sent as the per-request "config" part.
+   *     TikaServer 4.x only; requires allowPerRequestConfig=true on the server
+   * @param tikaServerChunks if true, extract Tika 4.x's tk:chunks (one per embedding-filter chunk)
+   *     instead of a single whole-document result. TikaServer 4.x only
    */
   private ExtractionRequest(
       String streamType,
@@ -73,7 +79,9 @@ private ExtractionRequest(
       String extractFormat,
       boolean tikaServerRecursive,
       Integer tikaServerTimeoutSeconds,
-      Map<String, String> tikaServerRequestHeaders) {
+      Map<String, String> tikaServerRequestHeaders,
+      String tikaServerConfigJson,
+      boolean tikaServerChunks) {
     this.streamType = streamType;
     this.resourceName = resourceName;
     this.contentType = contentType;
@@ -89,6 +97,8 @@ private ExtractionRequest(
     if (tikaServerRequestHeaders != null) {
       this.tikaServerRequestHeaders.putAll(tikaServerRequestHeaders);
     }
+    this.tikaServerConfigJson = tikaServerConfigJson;
+    this.tikaServerChunks = tikaServerChunks;
   }
 
   /** Creates a new Builder for constructing ExtractionRequest instances. */
@@ -111,6 +121,8 @@ public static class Builder {
     private boolean tikaServerRecursive = false;
     private Integer tikaServerTimeoutSeconds;
     private Map<String, String> tikaServerRequestHeaders;
+    private String tikaServerConfigJson;
+    private boolean tikaServerChunks = false;
 
     private Builder() {}
 
@@ -179,6 +191,16 @@ public Builder tikaServerRequestHeaders(Map<String, String> tikaServerRequestHea
       return this;
     }
 
+    public Builder tikaServerConfigJson(String tikaServerConfigJson) {
+      this.tikaServerConfigJson = tikaServerConfigJson;
+      return this;
+    }
+
+    public Builder tikaServerChunks(boolean tikaServerChunks) {
+      this.tikaServerChunks = tikaServerChunks;
+      return this;
+    }
+
     public ExtractionRequest build() {
       return new ExtractionRequest(
           streamType,
@@ -193,7 +215,9 @@ public ExtractionRequest build() {
           extractFormat,
           tikaServerRecursive,
           tikaServerTimeoutSeconds,
-          tikaServerRequestHeaders);
+          tikaServerRequestHeaders,
+          tikaServerConfigJson,
+          tikaServerChunks);
     }
   }
 }