diff --git a/pom.xml b/pom.xml index d0a9958c8d4..ffde688d560 100644 --- a/pom.xml +++ b/pom.xml @@ -50,6 +50,8 @@ tika-langdetect tika-pipes + tika-grpc-api + tika-grpc-mapper tika-grpc tika-app tika-server diff --git a/tika-bom/pom.xml b/tika-bom/pom.xml index e208ff61b72..b9afdba9c64 100644 --- a/tika-bom/pom.xml +++ b/tika-bom/pom.xml @@ -452,6 +452,21 @@ tika-async-cli ${revision} + + org.apache.tika + tika-grpc-api + ${revision} + + + org.apache.tika + tika-grpc-mapper + ${revision} + + + org.apache.tika + tika-grpc + ${revision} + org.apache.tika tika-httpclient-commons diff --git a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java index edcfd9a445b..6ba3d51d1ad 100644 --- a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java +++ b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ExternalTestBase.java @@ -243,7 +243,8 @@ private static void loadGovdocs1() throws IOException, InterruptedException { public static void copyTestFixtures() throws IOException { Path targetDir = TEST_FOLDER.toPath(); Files.createDirectories(targetDir); - String[] fixtures = {"sample.txt", "sample.html", "sample.csv", "sample.xml"}; + String[] fixtures = {"sample.txt", "sample.html", "sample.csv", "sample.xml", + "testPDF.pdf", "test_recursive_embedded.docx"}; for (String fixture : fixtures) { URL resource = ExternalTestBase.class.getClassLoader() .getResource("test-fixtures/" + fixture); diff --git a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/FileSystemFetcherTest.java b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/FileSystemFetcherTest.java index 065aa7890a1..f06ff4f51df 100644 --- a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/FileSystemFetcherTest.java +++ b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/FileSystemFetcherTest.java @@ -81,8 +81,8 @@ void testFileSystemFetcher() throws Exception { @Override public void onNext(FetchAndParseReply fetchAndParseReply) { LOG.debug("Reply from fetch-and-parse - key={}, status={}", - fetchAndParseReply.getFetchKey(), fetchAndParseReply.getStatus()); - if ("FETCH_AND_PARSE_EXCEPTION".equals(fetchAndParseReply.getStatus())) { + fetchAndParseReply.getFetchKey(), fetchAndParseReply.getDocument().getStatus().getPipesStatus()); + if ("FETCH_AND_PARSE_EXCEPTION".equals(fetchAndParseReply.getDocument().getStatus().getPipesStatus())) { errors.add(fetchAndParseReply); } else { successes.add(fetchAndParseReply); diff --git a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java index 9cf66973898..429e88280fe 100644 --- a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java +++ b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/filesystem/HandlerTypeTest.java @@ -45,18 +45,27 @@ import org.apache.tika.SaveFetcherReply; import org.apache.tika.SaveFetcherRequest; import org.apache.tika.TikaGrpc; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.FormatCategory; +import org.apache.tika.grpc.v1.MetadataField; +import org.apache.tika.grpc.v1.MetadataValue; +import org.apache.tika.grpc.v1.ParseStatus; import org.apache.tika.pipes.ExternalTestBase; import org.apache.tika.pipes.fetcher.fs.FileSystemFetcherConfig; /** - * Tests per-request ParseContext configuration via FetchAndParseRequest.parse_context_json. + * End-to-end tests against a live tika-grpc server (real gRPC wire serialization, real + * forked PipesServer, real fetcher plumbing). + * + * Covers per-request ParseContext configuration via + * FetchAndParseRequest.parse_context_json (e.g. + * {"basic-content-handler-factory": {"type": "HTML"}}), and the typed + * FetchAndParseReply.document contract: DocumentMetadata typed fields, the tagged + * `extra` tail (typed where Tika declares a type), the structured `blocks` tree, + * `format_category`, and `embedded` recursion for container formats. * * Uses the Ignite ConfigStore so that fetchers registered via saveFetcher are visible * to both the gRPC server JVM and the forked PipesServer JVM. - * - * Verifies that clients can override any parse context component on a per-request basis - * by providing a JSON object with component names as keys. - * Example: {"basic-content-handler-factory": {"type": "HTML"}} */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @Tag("E2ETest") @@ -286,35 +295,50 @@ private static boolean isParentProcess(long pid) { return false; } + private void registerFileSystemFetcher(TikaGrpc.TikaBlockingStub blockingStub, String fetcherId) + throws Exception { + FileSystemFetcherConfig config = new FileSystemFetcherConfig(); + config.setBasePath(TEST_FOLDER.getAbsolutePath()); + + SaveFetcherReply saveReply = blockingStub.saveFetcher(SaveFetcherRequest.newBuilder() + .setFetcherId(fetcherId) + .setFetcherType("file-system-fetcher") + .setFetcherConfigJson(ExternalTestBase.OBJECT_MAPPER.writeValueAsString(config)) + .build()); + LOG.info("Fetcher created: {}", saveReply.getFetcherId()); + } + + private static MetadataField findExtra(Document document, String key) { + return document.getExtraList().stream() + .filter(field -> field.getKey().equals(key)) + .findFirst() + .orElseThrow(() -> new AssertionError( + "expected document.extra to contain key '" + key + "', got: " + + document.getExtraList().stream().map(MetadataField::getKey).toList())); + } + @Test void testParseContextJson() throws Exception { String fetcherId = "handlerTypeFetcher"; ManagedChannel channel = getManagedChannel(); try { TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel); + registerFileSystemFetcher(blockingStub, fetcherId); - FileSystemFetcherConfig config = new FileSystemFetcherConfig(); - config.setBasePath(TEST_FOLDER.getAbsolutePath()); - - SaveFetcherReply saveReply = blockingStub.saveFetcher(SaveFetcherRequest.newBuilder() - .setFetcherId(fetcherId) - .setFetcherType("file-system-fetcher") - .setFetcherConfigJson(ExternalTestBase.OBJECT_MAPPER.writeValueAsString(config)) - .build()); - LOG.info("Fetcher created: {}", saveReply.getFetcherId()); - - // Parse sample.html requesting HTML output + // Parse sample.html requesting HTML output. render_markdown carries the + // handler's flat output in Document.markdown so it can be asserted directly. FetchAndParseReply htmlReply = blockingStub.fetchAndParse(FetchAndParseRequest.newBuilder() .setFetcherId(fetcherId) .setFetchKey("sample.html") .setParseContextJson("{\"basic-content-handler-factory\": {\"type\": \"HTML\"}}") + .setRenderMarkdown(true) .build()); - LOG.info("HTML parse status: {}", htmlReply.getStatus()); - Assertions.assertEquals("PARSE_SUCCESS", htmlReply.getStatus(), + LOG.info("HTML parse status: {}", htmlReply.getDocument().getStatus().getPipesStatus()); + Assertions.assertEquals("PARSE_SUCCESS", htmlReply.getDocument().getStatus().getPipesStatus(), "Parse should succeed with HTML handler type"); - String htmlContent = htmlReply.getFieldsMap().get("X-TIKA:content"); + String htmlContent = htmlReply.getDocument().getMarkdown(); Assertions.assertNotNull(htmlContent, "Content should be present in HTML response"); LOG.info("HTML content (first 200 chars): {}", htmlContent.substring(0, Math.min(200, htmlContent.length()))); Assertions.assertTrue( @@ -326,13 +350,14 @@ void testParseContextJson() throws Exception { .setFetcherId(fetcherId) .setFetchKey("sample.html") .setParseContextJson("{\"basic-content-handler-factory\": {\"type\": \"TEXT\"}}") + .setRenderMarkdown(true) .build()); - LOG.info("Text parse status: {}", textReply.getStatus()); - Assertions.assertEquals("PARSE_SUCCESS", textReply.getStatus(), + LOG.info("Text parse status: {}", textReply.getDocument().getStatus().getPipesStatus()); + Assertions.assertEquals("PARSE_SUCCESS", textReply.getDocument().getStatus().getPipesStatus(), "Parse should succeed with TEXT handler type"); - String textContent = textReply.getFieldsMap().get("X-TIKA:content"); + String textContent = textReply.getDocument().getMarkdown(); Assertions.assertNotNull(textContent, "Content should be present in text response"); LOG.info("Text content (first 200 chars): {}", textContent.substring(0, Math.min(200, textContent.length()))); Assertions.assertFalse( @@ -343,15 +368,160 @@ void testParseContextJson() throws Exception { "HTML and TEXT outputs should differ for the same document"); } finally { - channel.shutdown(); - try { - if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { - channel.shutdownNow(); - } - } catch (InterruptedException e) { + shutdownChannel(channel); + } + } + + /** + * Exercises the typed Document contract end-to-end for a PDF: typed + * DocumentMetadata fields, the tagged `extra` tail (a boolean-typed key and a + * string-typed key), the structured block tree, format_category, and ParseStatus -- + * all through the live server and real gRPC wire serialization, not the in-process + * mapper tests. + */ + @Test + void typedDocumentContractOverLiveServerPdf() throws Exception { + String fetcherId = "typedContractPdfFetcher"; + ManagedChannel channel = getManagedChannel(); + try { + TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel); + registerFileSystemFetcher(blockingStub, fetcherId); + + FetchAndParseReply reply = blockingStub.fetchAndParse(FetchAndParseRequest.newBuilder() + .setFetcherId(fetcherId) + .setFetchKey("testPDF.pdf") + .setRenderMarkdown(true) + .build()); + + Assertions.assertEquals("PARSE_SUCCESS", reply.getDocument().getStatus().getPipesStatus()); + Assertions.assertTrue(reply.hasDocument(), "reply should carry a typed Document"); + Document document = reply.getDocument(); + + // envelope + Assertions.assertEquals("application/pdf", document.getContentType()); + Assertions.assertEquals(FormatCategory.FORMAT_CATEGORY_PDF, document.getFormatCategory()); + Assertions.assertEquals(ParseStatus.Status.SUCCESS, document.getStatus().getStatus()); + + // typed DocumentMetadata fields + Assertions.assertEquals("Apache Tika - Apache Tika", document.getMetadata().getTitle()); + Assertions.assertEquals(java.util.List.of("Bertrand Delacr\u00e9taz"), + document.getMetadata().getAuthorsList()); + Assertions.assertEquals(1, document.getMetadata().getPageCount()); + Assertions.assertTrue(document.getMetadata().hasCreated(), + "created date should map to a typed Timestamp"); + + // content: the canonical block tree, plus the flat rendering this request + // opted into with render_markdown + Assertions.assertFalse(document.getBlocksList().isEmpty(), + "structured block tree should be populated"); + Assertions.assertTrue(document.getMarkdown().contains("Tika"), + "requested markdown rendering should contain extracted text"); + + // tagged tail: pdf:encrypted is declared boolean by Tika, so it must arrive + // typed as a boolean over the wire, not as a string + MetadataValue encrypted = findExtra(document, "pdf:encrypted").getValue(); + Assertions.assertEquals(MetadataValue.ValueCase.BOOLEAN, encrypted.getValueCase(), + "pdf:encrypted has a declared boolean type and must be tagged as one"); + Assertions.assertFalse(encrypted.getBoolean(), "testPDF.pdf is not encrypted"); + + // tagged tail: a text-typed key stays a string, value intact + MetadataValue creatorTool = findExtra(document, "xmp:CreatorTool").getValue(); + Assertions.assertEquals(MetadataValue.ValueCase.STRINGS, creatorTool.getValueCase()); + Assertions.assertEquals(java.util.List.of("Firefox"), + creatorTool.getStrings().getValuesList()); + } finally { + shutdownChannel(channel); + } + } + + /** + * Same contract for HTML through the live server: the typed title and a tagged tail + * key, proving the format-specific transformer ran server-side. + */ + @Test + void typedDocumentContractOverLiveServerHtml() throws Exception { + String fetcherId = "typedContractHtmlFetcher"; + ManagedChannel channel = getManagedChannel(); + try { + TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel); + registerFileSystemFetcher(blockingStub, fetcherId); + + FetchAndParseReply reply = blockingStub.fetchAndParse(FetchAndParseRequest.newBuilder() + .setFetcherId(fetcherId) + .setFetchKey("sample.html") + .build()); + + Assertions.assertEquals("PARSE_SUCCESS", reply.getDocument().getStatus().getPipesStatus()); + Document document = reply.getDocument(); + + Assertions.assertEquals(FormatCategory.FORMAT_CATEGORY_HTML, document.getFormatCategory()); + Assertions.assertEquals("Sample E2E Test Document", document.getMetadata().getTitle(), + "html should map to the typed title field"); + // this request did not set render_markdown, so the reply carries the content + // once: the canonical block tree, with the flat rendering left empty + Assertions.assertFalse(document.getBlocksList().isEmpty()); + Assertions.assertTrue(document.getMarkdown().isEmpty(), + "markdown rendering is opt-in and was not requested"); + + // tagged tail still carries the HTML-specific keys (encoding, etc.) + MetadataValue encoding = findExtra(document, "Content-Encoding").getValue(); + Assertions.assertEquals(MetadataValue.ValueCase.STRINGS, encoding.getValueCase()); + Assertions.assertFalse(encoding.getStrings().getValuesList().isEmpty()); + } finally { + shutdownChannel(channel); + } + } + + /** + * Container formats must recurse: an Office document with embedded resources should + * come back with each embedded child as its own fully typed Document, through the + * live server. + */ + @Test + void embeddedDocumentsRecurseOverLiveServer() throws Exception { + String fetcherId = "embeddedDocsFetcher"; + ManagedChannel channel = getManagedChannel(); + try { + TikaGrpc.TikaBlockingStub blockingStub = TikaGrpc.newBlockingStub(channel); + registerFileSystemFetcher(blockingStub, fetcherId); + + FetchAndParseReply reply = blockingStub.fetchAndParse(FetchAndParseRequest.newBuilder() + .setFetcherId(fetcherId) + .setFetchKey("test_recursive_embedded.docx") + .build()); + + Assertions.assertEquals("PARSE_SUCCESS", reply.getDocument().getStatus().getPipesStatus()); + Document document = reply.getDocument(); + + Assertions.assertEquals(FormatCategory.FORMAT_CATEGORY_OFFICE, document.getFormatCategory()); + // the container's own typed metadata (docProps/core.xml carries dcterms dates) + Assertions.assertTrue(document.getMetadata().hasCreated()); + Assertions.assertTrue(document.getMetadata().hasModified()); + + Assertions.assertTrue(document.getEmbeddedCount() > 0, + "container format should recurse into embedded children"); + for (Document child : document.getEmbeddedList()) { + Assertions.assertFalse(child.getContentType().isEmpty(), + "every embedded child should carry a detected content type"); + } + Assertions.assertTrue( + document.getEmbeddedList().stream() + .anyMatch(child -> !child.getOrigin().getFilename().isEmpty()), + "embedded children should carry their resource names in origin.filename"); + } finally { + shutdownChannel(channel); + } + } + + private static void shutdownChannel(ManagedChannel channel) { + channel.shutdown(); + try { + if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { channel.shutdownNow(); - Thread.currentThread().interrupt(); } + } catch (InterruptedException e) { + channel.shutdownNow(); + Thread.currentThread().interrupt(); } } } diff --git a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ignite/IgniteConfigStoreTest.java b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ignite/IgniteConfigStoreTest.java index 347d13f1d0f..588d4fb8eef 100644 --- a/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ignite/IgniteConfigStoreTest.java +++ b/tika-e2e-tests/tika-grpc/src/test/java/org/apache/tika/pipes/ignite/IgniteConfigStoreTest.java @@ -425,8 +425,10 @@ void testIgniteConfigStore() throws Exception { @Override public void onNext(FetchAndParseReply fetchAndParseReply) { LOG.debug("Reply from fetch-and-parse - key={}, status={}", - fetchAndParseReply.getFetchKey(), fetchAndParseReply.getStatus()); - if ("FETCH_AND_PARSE_EXCEPTION".equals(fetchAndParseReply.getStatus())) { + fetchAndParseReply.getFetchKey(), + fetchAndParseReply.getDocument().getStatus().getPipesStatus()); + if ("FETCH_AND_PARSE_EXCEPTION".equals( + fetchAndParseReply.getDocument().getStatus().getPipesStatus())) { errors.add(fetchAndParseReply); } else { successes.add(fetchAndParseReply); diff --git a/tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/testPDF.pdf b/tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/testPDF.pdf new file mode 100644 index 00000000000..1f1bcff6fe9 Binary files /dev/null and b/tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/testPDF.pdf differ diff --git a/tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/test_recursive_embedded.docx b/tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/test_recursive_embedded.docx new file mode 100644 index 00000000000..cd562cbb82d Binary files /dev/null and b/tika-e2e-tests/tika-grpc/src/test/resources/test-fixtures/test_recursive_embedded.docx differ diff --git a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-handlertype.json b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-handlertype.json index 48798bd0ebe..3e79e594359 100644 --- a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-handlertype.json +++ b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-handlertype.json @@ -1,5 +1,9 @@ { "plugin-roots": ["/var/cache/tika/plugins"], + "grpc": { + "allowComponentManagement": true, + "allowPerRequestConfig": true + }, "pipes": { "numClients": 1, "configStoreType": "ignite", diff --git a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-local.json b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-local.json index 93499c56695..b79fe642bce 100644 --- a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-local.json +++ b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config-ignite-local.json @@ -1,5 +1,8 @@ { "plugin-roots": ["/var/cache/tika/plugins"], + "grpc": { + "allowComponentManagement": true + }, "pipes": { "numClients": 1, "configStoreType": "ignite", diff --git a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json index 8863a4d90ac..5639e397a46 100644 --- a/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json +++ b/tika-e2e-tests/tika-grpc/src/test/resources/tika-config.json @@ -1,5 +1,8 @@ { "plugin-roots": ["/var/cache/tika/plugins"], + "grpc": { + "allowComponentManagement": true + }, "pipes": { "numClients": 1, "forkedJvmArgs": [ diff --git a/tika-grpc-api/README.md b/tika-grpc-api/README.md new file mode 100644 index 00000000000..3f74dcc9a0e --- /dev/null +++ b/tika-grpc-api/README.md @@ -0,0 +1,55 @@ +# Apache Tika gRPC API + +Typed protobuf messages for Tika parse output under `org.apache.tika.grpc.v1`. + +## Contents + +- **Document** (`document.proto`) — the single, small, stable parse-result contract: a + structured markdown content tree (`blocks`/`markdown`), typed common metadata + (`DocumentMetadata`), and a tagged metadata tail (`extra`) for everything + format-specific. +- **Bundled descriptors** — `META-INF/org.apache.tika.grpc.v1.descriptors` in the + published jar. + +## Usage + +```xml +<dependency> + <groupId>org.apache.tika</groupId> + <artifactId>tika-grpc-api</artifactId> + <version>${tika.version}</version> +</dependency> +``` + +## The Document shape + +Rather than one proto message per source format, `Document` models content and +metadata by *concern*, not by *format*: + +- **Content** lives in `markdown` (the authoritative render) and `blocks` (the same + content parsed into a structured tree of headings/paragraphs/lists/tables/code + blocks/inline runs). This is format-agnostic: every Tika parser's output reaches + this shape the same way, via markdown. +- **Metadata** has a small, bounded set of typed common fields on `DocumentMetadata` + (title, authors, description, keywords, languages, dates, counts, dimensions, + rights) plus a tagged tail (`extra`, a `repeated MetadataField`) for everything + format-specific. Tail values are typed where Tika's own `Property` declares a type + (integer/number/boolean/timestamp), and a string otherwise — never guessed. +- **`format_category`** is a cheap routing hint (`FormatCategory` enum: PDF, OFFICE, + IMAGE, HTML, RTF, EPUB, WARC, GENERIC). It is not mutually exclusive with anything + in `extra` — a document can be `FORMAT_CATEGORY_PDF` and still carry Creative + Commons rights metadata, for example. +- **`embedded`** recurses: a PDF with an embedded image is one `Document` whose + `embedded` list contains a fully-typed child `Document` for the image. + +Format-specific mapping (which Tika `Property` becomes which typed field, and what +falls through to `extra`) lives in `tika-grpc-mapper`'s +`org.apache.tika.grpc.mapper.transform.DocumentTransformer` implementations, one per +format — code, not schema. Adding a parser means adding a transformer; the wire +contract does not change. + +## Lint + +```bash +cd tika-grpc-api && buf lint +``` diff --git a/tika-grpc-api/buf.yaml b/tika-grpc-api/buf.yaml new file mode 100644 index 00000000000..69f8cd15936 --- /dev/null +++ b/tika-grpc-api/buf.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: v2 +modules: + - path: src/main/proto + name: buf.build/apache/tika-grpc-api +deps: + - buf.build/googleapis/googleapis +lint: + use: + - STANDARD +breaking: + use: + - FILE diff --git a/tika-grpc-api/pom.xml b/tika-grpc-api/pom.xml new file mode 100644 index 00000000000..175c2cb3b4f --- /dev/null +++ b/tika-grpc-api/pom.xml @@ -0,0 +1,121 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <artifactId>tika-grpc-api</artifactId> + <packaging>jar</packaging> + <name>Apache Tika gRPC API (typed Document protos)</name> + <url>https://tika.apache.org/</url> + + <parent> + <groupId>org.apache.tika</groupId> + <artifactId>tika-parent</artifactId> + <version>${revision}</version> + <relativePath>../tika-parent/pom.xml</relativePath> + </parent> + + <dependencies> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </dependency> + </dependencies> + + <build> + <extensions> + <extension> + <groupId>kr.motd.maven</groupId> + <artifactId>os-maven-plugin</artifactId> + <version>1.7.1</version> + </extension> + </extensions> + <plugins> + <plugin> + <groupId>org.xolstice.maven.plugins</groupId> + <artifactId>protobuf-maven-plugin</artifactId> + <version>0.6.1</version> + <configuration> + <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact> + <protoSourceRoot>${project.basedir}/src/main/proto</protoSourceRoot> + <includes> + <include>org/apache/tika/grpc/v1/*.proto</include> + </includes> + <writeDescriptorSet>true</writeDescriptorSet> + <includeDependenciesInDescriptorSet>true</includeDependenciesInDescriptorSet> + <descriptorSetOutputDirectory>${project.build.outputDirectory}/META-INF</descriptorSetOutputDirectory> + <descriptorSetFileName>org.apache.tika.grpc.v1.descriptors</descriptorSetFileName> + </configuration> + <executions> + <execution> + <goals> + <goal>compile</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>3.6.1</version> + <executions> + <execution> + <id>add-protobuf-sources</id> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>${project.build.directory}/generated-sources/protobuf/java</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <configuration> + <archive> + <manifestEntries> + <Automatic-Module-Name>org.apache.tika.grpc.api</Automatic-Module-Name> + </manifestEntries> + </archive> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <configuration> + <excludeGeneratedSources>true</excludeGeneratedSources> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <inputExcludes> + <inputExclude>README.md</inputExclude> + </inputExcludes> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v1/document.proto b/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v1/document.proto new file mode 100644 index 00000000000..355c8d7e677 --- /dev/null +++ b/tika-grpc-api/src/main/proto/org/apache/tika/grpc/v1/document.proto @@ -0,0 +1,235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The Document shape: a single, small, stable contract for a parsed Tika document, +// instead of a proto message per source format. +// +// Content model: a structured document tree (blocks + inline runs, with tables / +// task lists / strikethrough), anchored to the standard markdown block shape so it +// does not churn. The tree is the canonical content: structured consumers read it +// directly instead of re-parsing a string. A markdown rendering of the same tree is +// available on request for tools that want a flat string, and it round-trips because +// the shape is markdown's. +// +// Metadata: typed common fields (by concern, not by source format) plus a TAGGED tail. +// Tags come from Tika's declared Property value type; unknown keys are strings, never +// guessed. +// +// Format specifics arrive via per-parser DocumentTransformer code, not a proto-per-type, +// so adding a parser never touches the wire. + +syntax = "proto3"; + +package org.apache.tika.grpc.v1; + +import "google/protobuf/timestamp.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.tika.grpc.v1"; + +// ========================================================================= +// Top level +// ========================================================================= +message Document { + // envelope + string id = 1; + string content_type = 2; // Tika-detected MIME type + FormatCategory format_category = 3; // coarse routing hint; not mutually exclusive with `extra` + google.protobuf.Timestamp parsed_at = 4; + ParseStatus status = 5; + SourceOrigin origin = 6; + + // content. `blocks` is the canonical structured content and is always populated for + // a parse that produced content. `markdown` is a flat rendering of the same tree, + // populated only when the request asks for it (FetchAndParseRequest.render_markdown), + // so a reply does not carry the content twice. + string markdown = 10; + repeated Block blocks = 11; + + // metadata + DocumentMetadata metadata = 20; // typed common fields + repeated MetadataField extra = 21; // tagged tail: everything a transformer didn't type + + // embedded children, recursive (parent PDF + child image, each fully typed) + repeated Document embedded = 30; +} + +// Coarse format category for routing. Cross-cutting concerns (e.g. Creative Commons +// licensing) are NOT categories here -- they surface via `extra`/DocumentMetadata +// regardless of the primary category, since they can coexist with any format. +enum FormatCategory { + FORMAT_CATEGORY_UNSPECIFIED = 0; + FORMAT_CATEGORY_PDF = 1; + FORMAT_CATEGORY_OFFICE = 2; + FORMAT_CATEGORY_IMAGE = 3; + FORMAT_CATEGORY_HTML = 4; + FORMAT_CATEGORY_RTF = 5; + FORMAT_CATEGORY_EPUB = 6; + FORMAT_CATEGORY_WARC = 7; + FORMAT_CATEGORY_GENERIC = 8; +} + +message SourceOrigin { + string filename = 1; + string uri = 2; + int64 byte_size = 3; + string parser = 4; + // Hex-encoded SHA-256 of the source bytes, taken from the digest the parse pipeline + // recorded (the X-TIKA:digest:SHA256 metadata value). Empty when the pipeline has no + // digester configured. Downstream consumers use it to tie derived artifacts + // (annotations, index entries) back to the exact bytes that were parsed. + string sha256 = 5; +} + +// ========================================================================= +// Blocks (markdown block structure) +// ========================================================================= +message Block { + oneof block { + Heading heading = 1; // ATX/setext, level 1..6 + Paragraph paragraph = 2; + BlockQuote block_quote = 3; + BulletList bullet_list = 4; + OrderedList ordered_list = 5; + CodeBlock code_block = 6; // fenced or indented + HtmlBlock html_block = 7; + ThematicBreak thematic_break = 8; + Table table = 9; // GFM + } +} + +message Heading { int32 level = 1; repeated Inline content = 2; } +message Paragraph { repeated Inline content = 1; } +message BlockQuote { repeated Block blocks = 1; } +message ThematicBreak {} +message HtmlBlock { string literal = 1; } + +message BulletList { string marker = 1; bool tight = 2; repeated ListItem items = 3; } // marker: - + * +message OrderedList { int32 start = 1; string delimiter = 2; bool tight = 3; repeated ListItem items = 4; } // delimiter: . ) +message ListItem { + TaskStatus task = 1; // GFM task list checkbox + repeated Block blocks = 2; +} +enum TaskStatus { TASK_NONE = 0; TASK_UNCHECKED = 1; TASK_CHECKED = 2; } + +message CodeBlock { bool fenced = 1; string info = 2; string literal = 3; } // info = fence language/info + +message Table { repeated TableCell header = 1; repeated TableRow rows = 2; } +message TableRow { repeated TableCell cells = 1; } +message TableCell { Alignment alignment = 1; repeated Inline content = 2; } +enum Alignment { ALIGN_NONE = 0; ALIGN_LEFT = 1; ALIGN_CENTER = 2; ALIGN_RIGHT = 3; } + +// ========================================================================= +// Inline runs (structured markdown formatting) +// ========================================================================= +message Inline { + oneof inline { + string text = 1; // plain run + Emphasis emphasis = 2; // *italic* + Strong strong = 3; // **bold** + Strikethrough strikethrough = 4; // ~~del~~ (GFM) + Code code = 5; // `code` + Link link = 6; + Image image = 7; + string html = 8; // raw inline html + LineBreak line_break = 9; + } +} +message Emphasis { repeated Inline content = 1; } +message Strong { repeated Inline content = 1; } +message Strikethrough { repeated Inline content = 1; } +message Code { string literal = 1; } +message Link { string destination = 1; string title = 2; repeated Inline content = 3; } +message LineBreak { bool hard = 1; } + +message Image { + string destination = 1; + string title = 2; + string alt = 3; +} + +// ========================================================================= +// Metadata: typed common fields (grouped by concern, not by source format). +// The descriptive core follows the Dublin Core element set (title, creator, +// description, subject, language, publisher, identifier, date, rights), which is +// also the vocabulary Tika's own property model and XMP anchor to. +// ========================================================================= +message DocumentMetadata { + string title = 1; + repeated string authors = 2; + string description = 3; + repeated string keywords = 4; + repeated string languages = 5; + repeated string publishers = 6; + // Unambiguous references to the resource as declared by the document itself + // (DOI, ISBN, ...), from dc:identifier. + repeated string identifiers = 7; + + google.protobuf.Timestamp created = 10; + google.protobuf.Timestamp modified = 11; + + int32 page_count = 20; + int64 word_count = 21; + int64 character_count = 22; + + string rights = 30; // free-text rights/license statement + string license_url = 31; // e.g. a Creative Commons license URI + + int32 width = 40; + int32 height = 41; + double duration_seconds = 42; + + // Extend by concern (fonts, geo, security) as new common needs appear. This is + // where the "few hundred fields" would live if we chased them -- bounded, + // common, and cross-format. Anything narrower belongs in `extra`. +} + +// ========================================================================= +// Tagged tail: typed where Tika declares the type, string otherwise (never guessed) +// ========================================================================= +message MetadataField { string key = 1; MetadataValue value = 2; } +message MetadataValue { + oneof value { + StringList strings = 1; // default for untyped / multivalue text + int64 integer = 2; + double number = 3; + bool boolean = 4; + google.protobuf.Timestamp timestamp = 5; + } +} +message StringList { repeated string values = 1; } + +message ParseStatus { + enum Status { UNSPECIFIED = 0; SUCCESS = 1; PARTIAL = 2; FAILED = 3; } + Status status = 1; + // Wall-clock milliseconds for the whole fetch+parse round trip as observed by the + // server. Fetch and parse happen together inside the forked pipes worker, so a + // parse-only time is not separable here -- the name says what is actually measured. + int64 fetch_parse_time_ms = 2; + string parser_used = 3; + repeated string parsers_used = 4; + repeated string warnings = 5; + repeated string errors = 6; + // Version string of the Tika that produced this document (e.g. "Apache Tika 4.0.0"), + // so long-lived stores can tell which producer wrote an entry. + string tika_version = 7; + // The raw pipes result name this document's status was derived from (e.g. + // "PARSE_SUCCESS", "TIMEOUT"; see org.apache.tika.pipes.api.PipesResult.RESULT_STATUS). + // Diagnostic detail behind the typed `status`, not a stable enumeration: branch on + // `status`, log this. + string pipes_status = 8; +} diff --git a/tika-grpc-mapper/docs/EXTENSIONS.md b/tika-grpc-mapper/docs/EXTENSIONS.md new file mode 100644 index 00000000000..98090d3f8de --- /dev/null +++ b/tika-grpc-mapper/docs/EXTENSIONS.md @@ -0,0 +1,43 @@ +# Document extensions + +Core mapping lives in `org.apache.tika.grpc.mapper.DocumentBuilder`, which delegates +metadata mapping to `org.apache.tika.grpc.mapper.transform.DocumentTransformer` +implementations (one per format concern) and content-tree building to +`org.apache.tika.grpc.mapper.content.MarkdownBlockTreeBuilder` (format-agnostic, since +every Tika parser's output reaches this module as markdown). + +## Adding a new format + +Adding support for a new format means adding a `DocumentTransformer`, not touching the +wire contract: + +1. Implement `DocumentTransformer`: `appliesTo(Metadata)` decides whether this + transformer applies (usually a `Metadata.CONTENT_TYPE` check; cross-cutting + transformers like `CreativeCommonsDocumentTransformer` may inspect the full + metadata instead); `transform(Metadata, Document.Builder)` maps the common, + cross-format facts into `DocumentMetadata` via `TransformSupport`, then calls + `MetadataTagger.appendTail(...)` so every remaining Tika key lands in the tagged + tail, typed where Tika declares the type and string otherwise. +2. Register it in `DocumentTransformers.defaults()`. +3. Add a test that parses a real fixture (via the module's existing + `tika-parser-*-module` test-jar dependencies) and asserts on the fields that + actually populate. + +`PdfDocumentTransformer` is the reference implementation for this pattern. + +## Document outlines / structure + +PDF bookmarks, HTML/Markdown heading trees, and section boundaries are largely already +captured by `Document.blocks` (a `Heading` block carries its `level`, so a heading tree +is just a filter over `blocks`). A dedicated outline extraction pass (e.g. PDF +bookmarks, which are metadata rather than content) is not yet implemented; it would be +a `DocumentTransformer` that populates `DocumentMetadata` or the tagged tail from +PDFBox's outline API, not a proto change. + +## Pluggable, format-unknown output + +For output that does not fit the typed shape at all (e.g. a document-layout model's +own tree), the plan is a pluggable external-parser mechanism carrying opaque +`google.protobuf.Any` payloads on the `Document` — a follow-up change, so that this +contract never has to model a plugin's output shape. Do not add per-format proto +messages here. diff --git a/tika-grpc-mapper/docs/SOURCE_DESTINATION_MAPPING.md b/tika-grpc-mapper/docs/SOURCE_DESTINATION_MAPPING.md new file mode 100644 index 00000000000..058b9e4cfc5 --- /dev/null +++ b/tika-grpc-mapper/docs/SOURCE_DESTINATION_MAPPING.md @@ -0,0 +1,369 @@ +# Source to Destination Field Mapping + +## 🎯 Purpose +This document provides **exact** mappings from Tika interface properties to our protobuf fields. Use this as the definitive reference when implementing metadata builders. + +--- + +## 📋 PDF Documents + +### Source: `org.apache.tika.metadata.PDF.java` +### Destination: `io.pipeline.parsed.data.pdf.v1.PdfMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `PDF.DOC_INFO_TITLE` | `doc_info_title` | string | PDF document title | +| `PDF.DOC_INFO_AUTHOR` | `doc_info_author` | string | PDF document author | +| `PDF.DOC_INFO_SUBJECT` | `doc_info_subject` | string | PDF document subject | +| `PDF.DOC_INFO_KEYWORDS` | `doc_info_keywords` | string | PDF document keywords | +| `PDF.DOC_INFO_CREATOR` | `doc_info_creator` | string | PDF document creator | +| `PDF.DOC_INFO_PRODUCER` | `doc_info_producer` | string | PDF document producer | +| `PDF.DOC_INFO_CREATED` | `doc_info_created` | Timestamp | PDF creation date | +| `PDF.DOC_INFO_MODIFICATION_DATE` | `doc_info_modification_date` | Timestamp | PDF modification date | +| `PDF.DOC_INFO_TRAPPED` | `doc_info_trapped` | string | PDF trapped status | +| `PDF.PDF_VERSION` | `pdf_version` | string | PDF version | +| `PDF.PDFA_VERSION` | `pdfa_version` | string | PDF/A version | +| `PDF.PDF_EXTENSION_VERSION` | `pdf_extension_version` | string | PDF extension version | +| `PDF.PDFAID_CONFORMANCE` | `pdfaid_conformance` | string | PDF/A ID conformance | +| `PDF.PDFAID_PART` | `pdfaid_part` | int32 | PDF/A ID part | +| `PDF.PDFUAID_PART` | `pdfuaid_part` | int32 | PDF/UA ID part | +| `PDF.PDFVT_VERSION` | `pdfvt_version` | string | PDF/VT version | +| `PDF.PDFVT_MODIFIED` | `pdfvt_modified` | Timestamp | PDF/VT modified date | +| `PDF.PDFXID_VERSION` | `pdfxid_version` | string | PDF/X ID version | +| `PDF.PDFX_VERSION` | `pdfx_version` | string | PDF/X version | +| `PDF.PDFX_CONFORMANCE` | `pdfx_conformance` | string | PDF/X conformance | +| `PDF.IS_ENCRYPTED` | `is_encrypted` | bool | PDF encryption status | +| `PDF.PRODUCER` | `producer` | string | PDF producer | +| `PDF.HAS_XFA` | `has_xfa` | bool | Has XFA forms | +| `PDF.HAS_XMP` | `has_xmp` | bool | Has XMP metadata | +| `PDF.XMP_LOCATION` | `xmp_location` | string | XMP metadata location | +| `PDF.HAS_ACROFORM_FIELDS` | `has_acroform_fields` | bool | Has AcroForm fields | +| `PDF.HAS_MARKED_CONTENT` | `has_marked_content` | bool | Has marked content | +| `PDF.HAS_COLLECTION` | `has_collection` | bool | Has collection (portfolio) | +| `PDF.HAS_3D` | `has_3d` | bool | Has 3D annotations | +| `PDF.NUM_3D_ANNOTATIONS` | `num_3d_annotations` | int32 | Number of 3D annotations | +| `PDF.ACTION_TRIGGER` | `action_trigger` | string | Action trigger location | +| `PDF.ACTION_TRIGGERS` | `action_triggers` | repeated string | All action triggers | +| `PDF.ACTION_TYPES` | `action_types` | repeated string | Action types | +| `PDF.CHARACTERS_PER_PAGE` | `characters_per_page` | repeated int32 | Characters per page | +| `PDF.UNMAPPED_UNICODE_CHARS_PER_PAGE` | `unmapped_unicode_chars_per_page` | repeated int32 | Unmapped unicode chars per page | +| `PDF.TOTAL_UNMAPPED_UNICODE_CHARS` | `total_unmapped_unicode_chars` | int32 | Total unmapped unicode chars | +| `PDF.OVERALL_PERCENTAGE_UNMAPPED_UNICODE_CHARS` | `overall_percentage_unmapped_unicode_chars` | double | Percentage unmapped unicode | +| `PDF.CONTAINS_DAMAGED_FONT` | `contains_damaged_font` | bool | Contains damaged fonts | +| `PDF.CONTAINS_NON_EMBEDDED_FONT` | `contains_non_embedded_font` | bool | Contains non-embedded fonts | +| `PDF.EMBEDDED_FILE_DESCRIPTION` | `embedded_file_description` | string | Embedded file description | +| `PDF.EMBEDDED_FILE_ANNOTATION_TYPE` | `embedded_file_annotation_type` | string | Embedded file annotation type | +| `PDF.EMBEDDED_FILE_SUBTYPE` | `embedded_file_subtype` | string | Embedded file subtype | +| `PDF.ANNOTATION_TYPES` | `annotation_types` | repeated string | Annotation types | +| `PDF.ANNOTATION_SUBTYPES` | `annotation_subtypes` | repeated string | Annotation subtypes | +| `PDF.ASSOCIATED_FILE_RELATIONSHIP` | `associated_file_relationship` | string | Associated file relationship | +| `PDF.INCREMENTAL_UPDATE_NUMBER` | `incremental_update_number` | int32 | Incremental update number | +| `PDF.PDF_INCREMENTAL_UPDATE_COUNT` | `pdf_incremental_update_count` | int32 | Incremental update count | +| `PDF.OCR_PAGE_COUNT` | `ocr_page_count` | int32 | OCR page count | +| `PDF.EOF_OFFSETS` | `eof_offsets` | repeated double | EOF offsets | + +### Source: `org.apache.tika.metadata.XMPPDF.java` +### Destination: `io.pipeline.parsed.data.pdf.v1.PdfMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `XMPPDF.KEY_WORDS` | `xmp_keywords` | string | XMP PDF keywords (constant is KEY_WORDS in Tika 2.x snapshot) | +| `XMPPDF.PDF_VERSION` | `pdf_version` | string | XMP PDF version (normalized to `pdf_version`) | +| `XMPPDF.PRODUCER` | `producer` | string | XMP PDF producer (normalized to `producer`) | + +### Source: `org.apache.tika.metadata.AccessPermissions.java` +### Destination: `io.pipeline.parsed.data.pdf.v1.PdfMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `AccessPermissions.ASSEMBLE_DOCUMENT` | `can_assemble_document` | bool | Can assemble document | +| `AccessPermissions.EXTRACT_CONTENT` | `can_extract_content` | bool | Can extract content | +| `AccessPermissions.EXTRACT_FOR_ACCESSIBILITY` | `can_extract_for_accessibility` | bool | Can extract for accessibility | +| `AccessPermissions.FILL_IN_FORM` | `can_fill_in_form` | bool | Can fill in forms | +| `AccessPermissions.CAN_MODIFY_ANNOTATIONS` | `can_modify_annotations` | bool | Can modify annotations | +| `AccessPermissions.CAN_MODIFY` | `can_modify_document` | bool | Can modify document | +| `AccessPermissions.CAN_PRINT` | `can_print` | bool | Can print | +| `AccessPermissions.CAN_PRINT_FAITHFUL` | `can_print_faithful` | bool | Can print faithful | + +Notes: +- We normalize to expressive `can_*` boolean fields and align names/types with Tika snapshot. +### Additional PDF-related fields +### Sources: `org.apache.tika.metadata.PagedText.java`, PDF parser literals +### Destination: `io.pipeline.parsed.data.pdf.v1.PdfMetadata` + +| Source | Protobuf Field | Type | Notes | +|--------|----------------|------|-------| +| `PagedText.N_PAGES` | `n_pages` | int32 | Total number of pages | +| `"X-TIKA:pdf:metadata-xmp-parse-failed"` | `xmp_parse_failed` | repeated string | XMP parse failure messages | +| `PDF.ILLUSTRATOR_TYPE` | `illustrator_type` | string | Illustrator type if present | +| `"pdf:foundNonAdobeExtensionName"` | — | — | Kept in `ParseResponse.metadata` mirror (literal key) | + +--- + +## 📋 Office Documents + +### Source: `org.apache.tika.metadata.Office.java` +### Destination: `io.pipeline.parsed.data.office.v1.OfficeMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `Office.CHARACTER_COUNT` | `character_count` | int32 | Character count | +| `Office.CHARACTER_COUNT_WITH_SPACES` | `character_count_with_spaces` | int32 | Character count with spaces | +| `Office.LINE_COUNT` | `line_count` | int32 | Line count | +| `Office.PAGE_COUNT` | `page_count` | int32 | Page count | +| `Office.PARAGRAPH_COUNT` | `paragraph_count` | int32 | Paragraph count | +| `Office.WORD_COUNT` | `word_count` | int32 | Word count | +| `Office.SLIDE_COUNT` | `slide_count` | int32 | Slide count | +| `Office.NOTES_COUNT` | `notes_count` | int32 | Notes count | +| `Office.HIDDEN_COUNT` | `hidden_count` | int32 | Hidden slide count | +| `Office.MULTIMEDIA_OBJECT_COUNT` | `multimedia_object_count` | int32 | Multimedia object count | + +### Source: `org.apache.tika.metadata.OfficeOpenXMLCore.java` +### Destination: `io.pipeline.parsed.data.office.v1.OfficeMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `OfficeOpenXMLCore.CATEGORY` | `category` | string | Document category | +| `OfficeOpenXMLCore.CONTENT_STATUS` | `content_status` | string | Content status | +| `OfficeOpenXMLCore.CREATED` | `created` | Timestamp | Creation date | +| `OfficeOpenXMLCore.CREATOR` | `creator` | string | Creator | +| `OfficeOpenXMLCore.DESCRIPTION` | `description` | string | Description | +| `OfficeOpenXMLCore.IDENTIFIER` | `identifier` | string | Identifier | +| `OfficeOpenXMLCore.KEYWORDS` | `keywords` | string | Keywords | +| `OfficeOpenXMLCore.LANGUAGE` | `language` | string | Language | +| `OfficeOpenXMLCore.LAST_MODIFIED_BY` | `last_modified_by` | string | Last modified by | +| `OfficeOpenXMLCore.LAST_PRINTED` | `last_printed` | Timestamp | Last printed | +| `OfficeOpenXMLCore.MODIFIED` | `modified` | Timestamp | Modified date | +| `OfficeOpenXMLCore.REVISION` | `revision` | string | Revision | +| `OfficeOpenXMLCore.SUBJECT` | `subject` | string | Subject | +| `OfficeOpenXMLCore.TITLE` | `title` | string | Title | +| `OfficeOpenXMLCore.VERSION` | `version` | string | Version | + +### Source: `org.apache.tika.metadata.OfficeOpenXMLExtended.java` +### Destination: `io.pipeline.parsed.data.office.v1.OfficeMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `OfficeOpenXMLExtended.APPLICATION` | `application` | string | Application name | +| `OfficeOpenXMLExtended.APPLICATION_VERSION` | `application_version` | string | Application version | +| `OfficeOpenXMLExtended.CHARACTERS` | `characters` | int64 | Character count | +| `OfficeOpenXMLExtended.CHARACTERS_WITH_SPACES` | `characters_with_spaces` | int64 | Characters with spaces | +| `OfficeOpenXMLExtended.COMPANY` | `company` | string | Company | +| `OfficeOpenXMLExtended.DOC_SECURITY` | `doc_security` | int32 | Document security | +| `OfficeOpenXMLExtended.HIDDEN_SLIDES` | `hidden_slides` | int32 | Hidden slides | +| `OfficeOpenXMLExtended.LINES` | `lines` | int32 | Lines | +| `OfficeOpenXMLExtended.MANAGER` | `manager` | string | Manager | +| `OfficeOpenXMLExtended.MULTIMEDIA_CLIPS` | `multimedia_clips` | int32 | Multimedia clips | +| `OfficeOpenXMLExtended.NOTES` | `notes` | int32 | Notes | +| `OfficeOpenXMLExtended.PAGES` | `pages` | int32 | Pages | +| `OfficeOpenXMLExtended.PARAGRAPHS` | `paragraphs` | int32 | Paragraphs | +| `OfficeOpenXMLExtended.PRESENTATION_FORMAT` | `presentation_format` | string | Presentation format | +| `OfficeOpenXMLExtended.SLIDES` | `slides` | int32 | Slides | +| `OfficeOpenXMLExtended.TEMPLATE` | `template` | string | Template | +| `OfficeOpenXMLExtended.TOTAL_TIME` | `total_time_raw` / `total_time_seconds` | string / int32 | Raw duration and parsed seconds | +| `OfficeOpenXMLExtended.WORDS` | `words` | int32 | Words | + +--- + +## 📋 Image Documents + +### Source: `org.apache.tika.metadata.TIFF.java` +### Destination: `io.pipeline.parsed.data.image.v1.ImageMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `TIFF.IMAGE_WIDTH` | `image_width` | int32 | Image width | +| `TIFF.IMAGE_LENGTH` | `image_length` | int32 | Image height | +| `TIFF.BITS_PER_SAMPLE` | `bits_per_sample` | repeated int32 | Bits per sample | +| `TIFF.COMPRESSION` | `compression` | int32 | Compression type | +| `TIFF.PHOTOMETRIC_INTERPRETATION` | `photometric_interpretation` | int32 | Photometric interpretation | +| `TIFF.SAMPLES_PER_PIXEL` | `samples_per_pixel` | int32 | Samples per pixel | +| `TIFF.PLANAR_CONFIGURATION` | `planar_configuration` | int32 | Planar configuration | +| `TIFF.RESOLUTION_UNIT` | `resolution_unit` | int32 | Resolution unit | +| `TIFF.X_RESOLUTION` | `x_resolution` | double | X resolution | +| `TIFF.Y_RESOLUTION` | `y_resolution` | double | Y resolution | +| `TIFF.ORIENTATION` | `orientation` | int32 | Image orientation | + +### Source: `org.apache.tika.metadata.IPTC.java` +### Destination: `io.pipeline.parsed.data.image.v1.ImageMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `IPTC.KEYWORDS` | `iptc_keywords` | repeated string | IPTC keywords | +| `IPTC.CAPTION_ABSTRACT` | `iptc_caption` | string | IPTC caption | +| `IPTC.HEADLINE` | `iptc_headline` | string | IPTC headline | +| `IPTC.CREDIT` | `iptc_credit` | string | IPTC credit | +| `IPTC.SOURCE` | `iptc_source` | string | IPTC source | +| `IPTC.COPYRIGHT_NOTICE` | `iptc_copyright` | string | IPTC copyright | +| `IPTC.CATEGORY` | `iptc_category` | string | IPTC category | +| `IPTC.SUPPLEMENTAL_CATEGORIES` | `iptc_supplemental_categories` | repeated string | IPTC supplemental categories | + +--- + +## 📋 Email Documents + +### Source: `org.apache.tika.metadata.Message.java` +### Destination: `io.pipeline.parsed.data.email.v1.EmailMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `Message.MESSAGE_FROM` | `from_address` | string | From address | +| `Message.MESSAGE_TO` | `to_addresses` | repeated string | To addresses | +| `Message.MESSAGE_CC` | `cc_addresses` | repeated string | CC addresses | +| `Message.MESSAGE_BCC` | `bcc_addresses` | repeated string | BCC addresses | +| `Message.MESSAGE_SUBJECT` | `subject` | string | Email subject | +| `Message.MESSAGE_DATE` | `message_date` | Timestamp | Message date | +| `Message.MESSAGE_ID` | `message_id` | string | Message ID | +| `Message.MESSAGE_IN_REPLY_TO` | `in_reply_to` | string | In reply to | +| `Message.MESSAGE_REFERENCES` | `references` | repeated string | References | + +### Source: `org.apache.tika.metadata.MAPI.java` +### Destination: `io.pipeline.parsed.data.email.v1.EmailMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `MAPI.MESSAGE_CLASS` | `mapi_message_class` | string | MAPI message class | +| `MAPI.CONVERSATION_TOPIC` | `mapi_conversation_topic` | string | MAPI conversation topic | + +--- + +## 📋 Media Documents + +### Source: `org.apache.tika.metadata.XMPDM.java` +### Destination: `io.pipeline.parsed.data.media.v1.MediaMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `XMPDM.DURATION` | `duration_seconds` | double | Duration in seconds | +| `XMPDM.AUDIO_SAMPLE_RATE` | `audio_sample_rate` | int32 | Audio sample rate | +| `XMPDM.AUDIO_CHANNELS` | `audio_channels` | int32 | Audio channels | +| `XMPDM.VIDEO_FRAME_RATE` | `video_frame_rate` | double | Video frame rate | +| `XMPDM.VIDEO_WIDTH` | `video_width` | int32 | Video width | +| `XMPDM.VIDEO_HEIGHT` | `video_height` | int32 | Video height | +| `XMPDM.ARTIST` | `artist` | string | Artist | +| `XMPDM.ALBUM` | `album` | string | Album | +| `XMPDM.TRACK_NUMBER` | `track_number` | int32 | Track number | +| `XMPDM.GENRE` | `genre` | string | Genre | +| `XMPDM.COMPOSER` | `composer` | string | Composer | + +--- + +## 📋 HTML Documents + +### Source: `org.apache.tika.metadata.HTML.java` +### Destination: `io.pipeline.parsed.data.html.v1.HtmlMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `HTML.DESCRIPTION` | `description` | string | HTML description | +| `HTML.KEYWORDS` | `keywords` | repeated string | HTML keywords | +| `HTML.REFRESH` | `refresh` | string | HTML refresh | + +--- + +## 📋 Creative Commons Documents + +### Source: `org.apache.tika.metadata.CreativeCommons.java` +### Destination: `io.pipeline.parsed.data.creative_commons.v1.CreativeCommonsMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `CreativeCommons.LICENSE_URL` | `license_url` | string | License URL | +| `CreativeCommons.LICENSE_LOCATION` | `license_location` | string | License location | +| `CreativeCommons.WORK_TYPE` | `work_type` | string | Work type | + +### Source: `org.apache.tika.metadata.XMPRights.java` +### Destination: `io.pipeline.parsed.data.creative_commons.v1.CreativeCommonsMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `XMPRights.CERTIFICATE` | `rights_certificate` | string | Rights certificate | +| `XMPRights.MARKED` | `rights_marked` | bool | Rights marked | +| `XMPRights.OWNER` | `rights_owners` | repeated string | Rights owners | +| `XMPRights.USAGE_TERMS` | `usage_terms` | string | Usage terms | +| `XMPRights.WEB_STATEMENT` | `web_statement` | string | Web statement | + +--- + +## 📋 Font Documents + +### Source: `org.apache.tika.metadata.Font.java` +### Destination: `io.pipeline.parsed.data.tika.font.v1.FontMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `Font.FONT_NAME` | `font_name` | string | Font name | + +--- + +## 📋 EPUB Documents + +### Source: `org.apache.tika.metadata.Epub.java` +### Destination: `io.pipeline.parsed.data.epub.v1.EpubMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `Epub.RENDITION_LAYOUT` | `rendition_layout` | string | Rendition layout | +| `Epub.VERSION` | `version` | string | EPUB version | + +--- + +## 📋 WARC Documents + +### Source: `org.apache.tika.metadata.WARC.java` +### Destination: `io.pipeline.parsed.data.warc.v1.WarcMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `WARC.WARC_RECORD_ID` | `warc_record_id` | string | WARC record ID | +| `WARC.WARC_RECORD_CONTENT_TYPE` | `warc_record_content_type` | string | WARC record content type | +| `WARC.WARC_RECORD_DATE` | `warc_record_date` | Timestamp | WARC record date | +| `WARC.WARC_RECORD_TYPE` | `warc_record_type` | string | WARC record type | + +--- + +## 📋 Climate Forecast Documents + +### Source: `org.apache.tika.metadata.ClimateForcast.java` +### Destination: `io.pipeline.parsed.data.climate.v1.ClimateForcastMetadata` + +| Tika Property | Protobuf Field | Type | Notes | +|---------------|----------------|------|-------| +| `ClimateForcast.PROGRAM_ID` | `program_id` | string | Program ID | +| `ClimateForcast.INSTITUTION` | `institution` | string | Institution | +| `ClimateForcast.EXPERIMENT_ID` | `experiment_id` | string | Experiment ID | +| `ClimateForcast.MODEL_ID` | `model_id` | string | Model ID | +| `ClimateForcast.REALIZATION` | `realization` | string | Realization | +| `ClimateForcast.FREQUENCY` | `frequency` | string | Frequency | +| `ClimateForcast.REALM` | `realm` | string | Realm | +| `ClimateForcast.VARIABLE_ID` | `variable_id` | string | Variable ID | +| `ClimateForcast.ENSEMBLE` | `ensemble` | string | Ensemble | +| `ClimateForcast.GRID_LABEL` | `grid_label` | string | Grid label | +| `ClimateForcast.NOMINAL_RESOLUTION` | `nominal_resolution` | string | Nominal resolution | +| `ClimateForcast.SOURCE_ID` | `source_id` | string | Source ID | +| `ClimateForcast.SUB_EXPERIMENT_ID` | `sub_experiment_id` | string | Sub experiment ID | +| `ClimateForcast.TABLE_ID` | `table_id` | string | Table ID | +| `ClimateForcast.VARIANT_LABEL` | `variant_label` | string | Variant label | +| `ClimateForcast.ACTIVITY_ID` | `activity_id` | string | Activity ID | + +--- + +## 🔧 Usage Instructions + +### For Implementing Metadata Builders: + +1. **Find your document type** in this mapping +2. **Use the exact Tika Property names** from the "Tika Property" column +3. **Map to the exact protobuf field names** from the "Protobuf Field" column +4. **Use the correct data types** as specified in the "Type" column +5. **Any unmapped fields** are still present in the `ParseResponse.metadata` mirror (every key, typed) + +### Example Implementation: +```java +// For PDF metadata builder +MetadataUtils.mapStringField(tikaMetadata, PDF.DOC_INFO_TITLE, builder::setDocInfoTitle, mappedFields); +MetadataUtils.mapTimestampField(tikaMetadata, PDF.DOC_INFO_CREATED, builder::setDocInfoCreated, mappedFields); +MetadataUtils.mapBooleanField(tikaMetadata, PDF.IS_ENCRYPTED, builder::setIsEncrypted, mappedFields); +``` + +This mapping ensures **exact** correspondence between what Tika extracts and what our protobuf captures! diff --git a/tika-grpc-mapper/docs/TIKA_INTERFACE_MAPPING.md b/tika-grpc-mapper/docs/TIKA_INTERFACE_MAPPING.md new file mode 100644 index 00000000000..2369ef0fc14 --- /dev/null +++ b/tika-grpc-mapper/docs/TIKA_INTERFACE_MAPPING.md @@ -0,0 +1,253 @@ +# Tika Interface to Protobuf Mapping + +## 🎯 Purpose +This document maps **actual** Tika metadata interfaces to our protobuf structures. Every interface listed here has been verified to exist in the Tika source code. + +## 📋 Verified Tika Metadata Interfaces + +### Core Document Interfaces +- ✅ `PDF.java` - PDF document properties +- ✅ `Office.java` - Basic Office document properties +- ✅ `OfficeOpenXMLCore.java` - OOXML core properties +- ✅ `OfficeOpenXMLExtended.java` - OOXML extended properties +- ✅ `Message.java` - Email message properties +- ✅ `HTML.java` - HTML document properties +- ✅ `RTFMetadata.java` - RTF document properties +- ✅ `Database.java` - Database file properties +- ✅ `Font.java` - Font file properties +- ✅ `Epub.java` - EPUB book properties +- ✅ `WARC.java` - Web archive properties +- ✅ `ClimateForcast.java` - NetCDF/Climate data properties +- ✅ `CreativeCommons.java` - Creative Commons licensing + +### Image and Media Interfaces +- ✅ `TIFF.java` - TIFF image properties +- ✅ `IPTC.java` - IPTC image metadata +- ✅ `Photoshop.java` - Photoshop-specific metadata +- ✅ `XMPDM.java` - XMP Digital Media properties + +### XMP Interfaces +- ✅ `XMP.java` - XMP Basic Schema +- ✅ `XMPDC.java` - XMP Dublin Core +- ✅ `XMPMM.java` - XMP Media Management +- ✅ `XMPPDF.java` - XMP PDF properties +- ✅ `XMPRights.java` - XMP Rights Management +- ✅ `XMPIdq.java` - XMP Identifier Qualifier + +### Specialized Interfaces +- ✅ `MAPI.java` - MAPI (Outlook) properties +- ✅ `PST.java` - PST file properties +- ✅ `QuattroPro.java` - QuattroPro spreadsheet +- ✅ `WordPerfect.java` - WordPerfect document +- ✅ `MachineMetadata.java` - Machine/system metadata +- ✅ `ExternalProcess.java` - External process metadata +- ✅ `Geographic.java` - Geographic/location data +- ✅ `AccessPermissions.java` - Document access permissions +- ✅ `FileSystem.java` - File system metadata +- ✅ `HttpHeaders.java` - HTTP header metadata +- ✅ `Rendering.java` - Rendering properties +- ✅ `PagedText.java` - Paged text properties +- ✅ `TikaPagedText.java` - Tika-specific paged text + +### Core Framework +- ✅ `DublinCore.java` - Dublin Core standard +- ✅ `TikaCoreProperties.java` - Core Tika properties +- ✅ `TikaMimeKeys.java` - MIME type keys + +--- + +## 🗺️ Interface to Protobuf Mapping + +### 1. PDF Documents +**Tika Interfaces:** +- `PDF.java` - 50+ properties (DOC_INFO_*, PDF_VERSION, IS_ENCRYPTED, etc.) +- `XMPPDF.java` - XMP PDF-specific properties +- `AccessPermissions.java` - PDF security permissions + +**Protobuf Target:** +- `io.pipeline.parsed.data.pdf.v1.PdfMetadata` + +**Key Properties to Map:** +```java +// From PDF.java +PDF.DOC_INFO_TITLE → title +PDF.DOC_INFO_AUTHOR → author +PDF.DOC_INFO_SUBJECT → subject +PDF.DOC_INFO_KEYWORDS → keywords +PDF.DOC_INFO_CREATOR → creator +PDF.DOC_INFO_PRODUCER → producer +PDF.DOC_INFO_CREATED → creation_date +PDF.DOC_INFO_MODIFICATION_DATE → modification_date +PDF.PDF_VERSION → pdf_version +PDF.PDFA_VERSION → pdfa_version +PDF.IS_ENCRYPTED → is_encrypted +PDF.HAS_XFA → has_xfa +PDF.HAS_ACROFORM_FIELDS → has_acroform_fields +PDF.HAS_MARKED_CONTENT → has_marked_content +PDF.HAS_COLLECTION → has_collection +PDF.HAS_3D → has_3d +// ... and 35+ more actual properties + +// From XMPPDF.java (note: constant is KEY_WORDS in snapshot) +XMPPDF.KEY_WORDS → xmp_keywords (string) + +// From AccessPermissions.java (booleans) +AccessPermissions.CAN_MODIFY → can_modify_document +AccessPermissions.EXTRACT_CONTENT → can_extract_content +AccessPermissions.EXTRACT_FOR_ACCESSIBILITY → can_extract_for_accessibility +AccessPermissions.ASSEMBLE_DOCUMENT → can_assemble_document +AccessPermissions.FILL_IN_FORM → can_fill_in_form +AccessPermissions.CAN_MODIFY_ANNOTATIONS → can_modify_annotations +AccessPermissions.CAN_PRINT → can_print +AccessPermissions.CAN_PRINT_FAITHFUL → can_print_faithful + +// From PagedText.java +PagedText.N_PAGES → n_pages (int32) + +// Literal keys from PDF parsers +"X-TIKA:pdf:metadata-xmp-parse-failed" → xmp_parse_failed (repeated string) +"pdf:foundNonAdobeExtensionName" → kept in the `ParseResponse.metadata` mirror +``` + +### 2. Office Documents +**Tika Interfaces:** +- `Office.java` - Basic Office properties +- `OfficeOpenXMLCore.java` - OOXML core metadata +- `OfficeOpenXMLExtended.java` - OOXML extended metadata + +**Protobuf Target:** +- `io.pipeline.parsed.data.office.v1.OfficeMetadata` + +### 3. Image Documents +**Tika Interfaces:** +- `TIFF.java` - TIFF-specific properties +- `IPTC.java` - IPTC metadata standard +- `Photoshop.java` - Photoshop metadata +- `XMP.java` - XMP basic properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.image.v1.ImageMetadata` + +### 4. Email Documents +**Tika Interfaces:** +- `Message.java` - Email message properties +- `MAPI.java` - Outlook/MAPI properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.email.v1.EmailMetadata` + +### 5. Media Documents +**Tika Interfaces:** +- `XMPDM.java` - XMP Digital Media properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.media.v1.MediaMetadata` + +### 6. HTML Documents +**Tika Interfaces:** +- `HTML.java` - HTML metadata properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.html.v1.HtmlMetadata` + +### 7. RTF Documents +**Tika Interfaces:** +- `RTFMetadata.java` - RTF-specific properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.rtf.v1.RtfMetadata` + +### 8. Database Documents +**Tika Interfaces:** +- `Database.java` - Database file properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.database.v1.DatabaseMetadata` + +### 9. Font Documents +**Tika Interfaces:** +- `Font.java` - Font file properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.tika.font.v1.FontMetadata` + +### 10. EPUB Documents +**Tika Interfaces:** +- `Epub.java` - EPUB book properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.epub.v1.EpubMetadata` + +### 11. WARC Documents +**Tika Interfaces:** +- `WARC.java` - Web archive properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.warc.v1.WarcMetadata` + +### 12. Climate Forecast Documents +**Tika Interfaces:** +- `ClimateForcast.java` - NetCDF/Climate properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.climate.v1.ClimateForcastMetadata` + +### 13. Creative Commons Documents +**Tika Interfaces:** +- `CreativeCommons.java` - CC licensing properties +- `XMPRights.java` - XMP rights management + +**Protobuf Target:** +- `io.pipeline.parsed.data.creative_commons.v1.CreativeCommonsMetadata` + +### 14. Generic Documents +**Tika Interfaces:** +- `PST.java` - PST file properties +- `QuattroPro.java` - QuattroPro spreadsheet +- `WordPerfect.java` - WordPerfect document +- `MachineMetadata.java` - Machine metadata +- `ExternalProcess.java` - External process metadata +- `Geographic.java` - Geographic data +- `FileSystem.java` - File system metadata +- `HttpHeaders.java` - HTTP headers +- `Rendering.java` - Rendering properties +- `PagedText.java` - Paged text properties + +**Protobuf Target:** +- `io.pipeline.parsed.data.generic.v1.GenericMetadata` + +--- + +## 🔧 Implementation Strategy + +### Phase 1: Verify Protobuf Fields Match Tika Interfaces +For each document type, ensure our protobuf has fields that correspond to the **actual** Tika interface properties. + +### Phase 2: Create Accurate Builders +Build metadata extractors that map **only** the properties that actually exist in the Tika interfaces. + +### Phase 3: Handle Unmapped Fields +Every Tika key — mapped or not — is mirrored into `ParseResponse.metadata` as a typed, +multivalue-preserving `MetadataEntry`. This single lossless channel replaces the former +per-format `google.protobuf.Struct` catch-alls. + +--- + +## ⚠️ Important Notes + +1. **Only use properties that actually exist** in the Tika interfaces (or clearly documented literal keys from parser code) +2. **Check property types** - some are `Property.internalText()`, others are `Property.internalInteger()`, etc. +3. **Handle multiple interfaces** - some document types (like Office) have multiple related interfaces +4. **Use proper Property constants** - don't use string literals, use the actual Property constants +5. **Lossless mirror** - every key (mapped or not) also appears in `ParseResponse.metadata` + +--- + +## 🎯 Next Steps + +1. **Verify our protobuf fields** match the actual Tika interface properties. If we intentionally rename for clarity, document it in SOURCE_DESTINATION_MAPPING.md. +2. **Create simple, accurate builders** that map only what exists +3. **Test with real documents** to ensure we capture everything Tika extracts +4. **Follow the principle**: "Whatever Tika extracts, we save - strongly-typed if we recognize it, and always in the `metadata` mirror regardless" + +This mapping ensures we build **accurate** metadata extractors based on what Tika actually provides, not what we assume it should provide. diff --git a/tika-grpc-mapper/pom.xml b/tika-grpc-mapper/pom.xml new file mode 100644 index 00000000000..8abed989f84 --- /dev/null +++ b/tika-grpc-mapper/pom.xml @@ -0,0 +1,213 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <artifactId>tika-grpc-mapper</artifactId> + <packaging>jar</packaging> + <name>Apache Tika gRPC mapper (Metadata to Document)</name> + <url>https://tika.apache.org/</url> + + <parent> + <groupId>org.apache.tika</groupId> + <artifactId>tika-parent</artifactId> + <version>${revision}</version> + <relativePath>../tika-parent/pom.xml</relativePath> + </parent> + + <dependencies> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-grpc-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.commonmark</groupId> + <artifactId>commonmark</artifactId> + </dependency> + <dependency> + <groupId>org.commonmark</groupId> + <artifactId>commonmark-ext-gfm-tables</artifactId> + </dependency> + <dependency> + <groupId>org.commonmark</groupId> + <artifactId>commonmark-ext-gfm-strikethrough</artifactId> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parsers-standard-package</artifactId> + <version>${project.version}</version> + <type>pom</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-core</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-pdf-module</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-html-module</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-mail-module</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-microsoft-module</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-image-module</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-text-module</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-font-module</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-audiovideo-module</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-webarchive-module</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-miscoffice-module</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-scientific-module</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-scientific-package</artifactId> + <version>${project.version}</version> + <type>pom</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-scientific-module</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j2-impl</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <configuration> + <archive> + <manifestEntries> + <Automatic-Module-Name>org.apache.tika.grpc.mapper</Automatic-Module-Name> + </manifestEntries> + </archive> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <configuration> + <excludeGeneratedSources>true</excludeGeneratedSources> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <inputExcludes> + <inputExclude>docs/**</inputExclude> + <inputExclude>**/test-documents/**</inputExclude> + </inputExcludes> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/DocumentBuilder.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/DocumentBuilder.java new file mode 100644 index 00000000000..75d28aeb980 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/DocumentBuilder.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper; + +import java.time.Instant; +import java.util.List; + +import org.apache.tika.Tika; +import org.apache.tika.grpc.mapper.content.MarkdownBlockTreeBuilder; +import org.apache.tika.grpc.mapper.transform.DocumentTransformers; +import org.apache.tika.grpc.mapper.transform.FormatCategoryDetector; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.ParseStatus; +import org.apache.tika.grpc.v1.SourceOrigin; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * Builds a {@link Document} from Tika's parse output: the pipes default content handler + * renders the extracted content as markdown, so this class parses that markdown into + * the structured {@code Block} tree once (format-agnostic) and delegates metadata + * mapping to {@link DocumentTransformers} (format-specific, one transformer per + * concern). Embedded documents recurse into {@link Document#getEmbeddedList()}. + * + * <p>The block tree is the canonical content and is always populated. The flat + * {@code markdown} field is a rendering of the same content and is only carried when + * the caller asks for it, so a reply does not ship the content twice.</p> + */ +public final class DocumentBuilder { + + private static final DocumentTransformers TRANSFORMERS = DocumentTransformers.defaults(); + + // The metadata key DigestDef.metadataKey() produces for SHA-256 with the default hex + // encoding; the parse pipeline records the source digest there when a digester is + // configured. + private static final String SHA256_DIGEST_KEY = TikaCoreProperties.TIKA_META_PREFIX + + "digest" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER + "SHA256"; + + private DocumentBuilder() { + } + + /** + * Maps Tika parse output (primary metadata plus optional embedded metadata list) to + * {@link Document}. Pass {@code primary == null} when the pipes result carried no + * metadata at all (e.g. a fetch failure, or a crash before the parse produced + * anything): the returned Document then has only an id and a status. The status still + * derives from {@code pipesStatus} -- {@code EMPTY_OUTPUT} with no metadata is a + * legitimate success -- and an explicit error is recorded for the non-success cases. + * + * @param renderMarkdown whether to also carry the flat markdown rendering in + * {@code Document.markdown} (the block tree is populated either + * way) + */ + public static Document build(Metadata primary, List<Metadata> allMetadata, String markdownBody, + String docId, String pipesStatus, long fetchParseTimeMs, + boolean renderMarkdown) { + if (primary == null) { + ParseStatus.Builder statusBuilder = ParseStatus.newBuilder() + .setStatus(mapPipesStatus(pipesStatus)) + .setFetchParseTimeMs(fetchParseTimeMs) + .setTikaVersion(Tika.getString()); + if (pipesStatus != null && !pipesStatus.isEmpty()) { + statusBuilder.setPipesStatus(pipesStatus); + } + if (statusBuilder.getStatus() != ParseStatus.Status.SUCCESS) { + statusBuilder.addErrors("No metadata returned from parse"); + } + Document.Builder document = Document.newBuilder().setStatus(statusBuilder.build()); + if (docId != null && !docId.isEmpty()) { + document.setId(docId); + } + return document.build(); + } + + Document.Builder document = buildOne(primary, markdownBody, docId, renderMarkdown); + + String parserClass = primary.get(TikaCoreProperties.TIKA_PARSED_BY); + ParseStatus.Builder statusBuilder = ParseStatus.newBuilder() + .setStatus(mapPipesStatus(pipesStatus)) + .setFetchParseTimeMs(fetchParseTimeMs) + .setTikaVersion(Tika.getString()); + if (pipesStatus != null && !pipesStatus.isEmpty()) { + statusBuilder.setPipesStatus(pipesStatus); + } + if (parserClass != null && !parserClass.isEmpty()) { + statusBuilder.setParserUsed(parserClass); + } + String parsedByFull = primary.get(TikaCoreProperties.TIKA_PARSED_BY_FULL_SET); + if (parsedByFull != null && !parsedByFull.isEmpty()) { + for (String p : parsedByFull.split(",")) { + if (!p.trim().isEmpty()) { + statusBuilder.addParsersUsed(p.trim()); + } + } + } + document.setStatus(statusBuilder.build()); + + if (allMetadata != null && allMetadata.size() > 1) { + for (int i = 1; i < allMetadata.size(); i++) { + Metadata embedded = allMetadata.get(i); + String embeddedBody = embedded.get(TikaCoreProperties.TIKA_CONTENT); + document.addEmbedded( + buildOne(embedded, embeddedBody, docId + "#" + i, renderMarkdown).build()); + } + } + + return document.build(); + } + + private static Document.Builder buildOne(Metadata tika, String markdownBody, String docId, + boolean renderMarkdown) { + Document.Builder document = Document.newBuilder(); + if (docId != null && !docId.isEmpty()) { + document.setId(docId); + } + + String contentType = tika.get(Metadata.CONTENT_TYPE); + if (contentType != null && !contentType.isBlank()) { + document.setContentType(contentType.trim()); + } + document.setFormatCategory(FormatCategoryDetector.detect(tika)); + + Instant now = Instant.now(); + document.setParsedAt(com.google.protobuf.Timestamp.newBuilder() + .setSeconds(now.getEpochSecond()) + .setNanos(now.getNano()) + .build()); + + SourceOrigin.Builder origin = SourceOrigin.newBuilder(); + String resourceName = tika.get(TikaCoreProperties.RESOURCE_NAME_KEY); + if (resourceName != null && !resourceName.isBlank()) { + origin.setFilename(resourceName.trim()); + } + String parserClass = tika.get(TikaCoreProperties.TIKA_PARSED_BY); + if (parserClass != null && !parserClass.isBlank()) { + origin.setParser(parserClass.trim()); + } + String sha256 = tika.get(SHA256_DIGEST_KEY); + if (sha256 != null && !sha256.isBlank()) { + origin.setSha256(sha256.trim()); + } + document.setOrigin(origin.build()); + + if (markdownBody != null && !markdownBody.isEmpty()) { + document.addAllBlocks(MarkdownBlockTreeBuilder.toBlocks(markdownBody)); + if (renderMarkdown) { + document.setMarkdown(markdownBody); + } + } + + TRANSFORMERS.transform(tika, document); + + return document; + } + + /** + * Maps {@code org.apache.tika.pipes.api.PipesResult.RESULT_STATUS.name()} to + * {@link ParseStatus.Status}. Values are named after the real enum constants (not + * guessed) since this module intentionally has no compile dependency on tika-pipes-api: + * a clean success (e.g. {@code PARSE_SUCCESS}, {@code EMIT_SUCCESS}) is {@code SUCCESS}; + * a success that happened alongside a caught exception (e.g. + * {@code PARSE_SUCCESS_WITH_EXCEPTION}) is {@code PARTIAL}; everything else -- including + * process crashes like {@code TIMEOUT} and {@code OOM}, which are not partial successes -- + * is {@code FAILED}. + */ + private static ParseStatus.Status mapPipesStatus(String pipesStatus) { + if (pipesStatus == null || pipesStatus.isEmpty()) { + return ParseStatus.Status.UNSPECIFIED; + } + return switch (pipesStatus) { + case "EMPTY_OUTPUT", "PARSE_SUCCESS", "EMIT_SUCCESS", "EMIT_SUCCESS_PASSBACK" -> + ParseStatus.Status.SUCCESS; + case "PARSE_SUCCESS_WITH_EXCEPTION", "PARSE_EXCEPTION_NO_EMIT", "EMIT_SUCCESS_PARSE_EXCEPTION" -> + ParseStatus.Status.PARTIAL; + default -> ParseStatus.Status.FAILED; + }; + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilder.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilder.java new file mode 100644 index 00000000000..fadbb3be7d9 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilder.java @@ -0,0 +1,333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.content; + +import java.util.ArrayList; +import java.util.List; + +import org.commonmark.Extension; +import org.commonmark.ext.gfm.strikethrough.Strikethrough; +import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension; +import org.commonmark.ext.gfm.tables.TableBlock; +import org.commonmark.ext.gfm.tables.TableBody; +import org.commonmark.ext.gfm.tables.TableCell; +import org.commonmark.ext.gfm.tables.TableHead; +import org.commonmark.ext.gfm.tables.TableRow; +import org.commonmark.ext.gfm.tables.TablesExtension; +import org.commonmark.node.BlockQuote; +import org.commonmark.node.BulletList; +import org.commonmark.node.Code; +import org.commonmark.node.Emphasis; +import org.commonmark.node.FencedCodeBlock; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.Heading; +import org.commonmark.node.HtmlBlock; +import org.commonmark.node.HtmlInline; +import org.commonmark.node.Image; +import org.commonmark.node.IndentedCodeBlock; +import org.commonmark.node.Link; +import org.commonmark.node.ListItem; +import org.commonmark.node.Node; +import org.commonmark.node.OrderedList; +import org.commonmark.node.Paragraph; +import org.commonmark.node.SoftLineBreak; +import org.commonmark.node.StrongEmphasis; +import org.commonmark.node.Text; +import org.commonmark.node.ThematicBreak; +import org.commonmark.parser.Parser; + +/** + * Converts a markdown string into the {@code Document} proto's structured + * {@code Block}/{@code Inline} tree. + * <p> + * Every Tika parser's SAX output already reaches this module as markdown text (the + * pipes default content handler renders {@code TikaCoreProperties.TIKA_CONTENT} as + * markdown via {@code ToMarkdownContentHandler}). Reusing that single rendering path + * plus commonmark-java's parser means the content tree is built once, generically, for + * every source format -- no per-format content handling is needed, only per-format + * metadata handling ({@link org.apache.tika.grpc.mapper.transform.DocumentTransformer}). + * <p> + * Naming note: every {@code org.apache.tika.grpc.v1.*} proto type is referenced fully + * qualified in this file. Many proto message names (Heading, Paragraph, BlockQuote, + * Emphasis, Code, Link, Image, TableRow, TableCell, ...) collide with commonmark-java's + * own node class names, which are imported normally; fully qualifying one side avoids + * any ambiguity. + */ +public final class MarkdownBlockTreeBuilder { + + private static final List<Extension> EXTENSIONS = + List.of(TablesExtension.create(), StrikethroughExtension.create()); + private static final Parser COMMONMARK = Parser.builder().extensions(EXTENSIONS).build(); + + private MarkdownBlockTreeBuilder() { + } + + /** Parses {@code markdown} into a flat list of top-level {@code Block}s. */ + public static List<org.apache.tika.grpc.v1.Block> toBlocks(String markdown) { + if (markdown == null || markdown.isEmpty()) { + return List.of(); + } + return blockChildren(COMMONMARK.parse(markdown)); + } + + private static List<org.apache.tika.grpc.v1.Block> blockChildren(Node parent) { + List<org.apache.tika.grpc.v1.Block> blocks = new ArrayList<>(); + for (Node child = parent.getFirstChild(); child != null; child = child.getNext()) { + org.apache.tika.grpc.v1.Block block = toBlock(child); + if (block != null) { + blocks.add(block); + } + } + return blocks; + } + + private static org.apache.tika.grpc.v1.Block toBlock(Node node) { + if (node instanceof Heading) { + Heading heading = (Heading) node; + return org.apache.tika.grpc.v1.Block.newBuilder() + .setHeading(org.apache.tika.grpc.v1.Heading.newBuilder() + .setLevel(heading.getLevel()) + .addAllContent(inlineChildren(heading))) + .build(); + } + if (node instanceof Paragraph) { + return org.apache.tika.grpc.v1.Block.newBuilder() + .setParagraph(org.apache.tika.grpc.v1.Paragraph.newBuilder() + .addAllContent(inlineChildren(node))) + .build(); + } + if (node instanceof BlockQuote) { + return org.apache.tika.grpc.v1.Block.newBuilder() + .setBlockQuote(org.apache.tika.grpc.v1.BlockQuote.newBuilder() + .addAllBlocks(blockChildren(node))) + .build(); + } + if (node instanceof BulletList) { + BulletList bulletList = (BulletList) node; + return org.apache.tika.grpc.v1.Block.newBuilder() + .setBulletList(org.apache.tika.grpc.v1.BulletList.newBuilder() + .setMarker(String.valueOf(bulletList.getBulletMarker())) + .setTight(bulletList.isTight()) + .addAllItems(listItems(bulletList))) + .build(); + } + if (node instanceof OrderedList) { + OrderedList orderedList = (OrderedList) node; + return org.apache.tika.grpc.v1.Block.newBuilder() + .setOrderedList(org.apache.tika.grpc.v1.OrderedList.newBuilder() + .setStart(orderedList.getStartNumber()) + .setDelimiter(String.valueOf(orderedList.getDelimiter())) + .setTight(orderedList.isTight()) + .addAllItems(listItems(orderedList))) + .build(); + } + if (node instanceof FencedCodeBlock) { + FencedCodeBlock codeBlock = (FencedCodeBlock) node; + return org.apache.tika.grpc.v1.Block.newBuilder() + .setCodeBlock(org.apache.tika.grpc.v1.CodeBlock.newBuilder() + .setFenced(true) + .setInfo(nullToEmpty(codeBlock.getInfo())) + .setLiteral(nullToEmpty(codeBlock.getLiteral()))) + .build(); + } + if (node instanceof IndentedCodeBlock) { + return org.apache.tika.grpc.v1.Block.newBuilder() + .setCodeBlock(org.apache.tika.grpc.v1.CodeBlock.newBuilder() + .setFenced(false) + .setLiteral(nullToEmpty(((IndentedCodeBlock) node).getLiteral()))) + .build(); + } + if (node instanceof HtmlBlock) { + return org.apache.tika.grpc.v1.Block.newBuilder() + .setHtmlBlock(org.apache.tika.grpc.v1.HtmlBlock.newBuilder() + .setLiteral(nullToEmpty(((HtmlBlock) node).getLiteral()))) + .build(); + } + if (node instanceof ThematicBreak) { + return org.apache.tika.grpc.v1.Block.newBuilder() + .setThematicBreak(org.apache.tika.grpc.v1.ThematicBreak.getDefaultInstance()) + .build(); + } + if (node instanceof TableBlock) { + return org.apache.tika.grpc.v1.Block.newBuilder().setTable(toTable(node)).build(); + } + // Link reference definitions and any other unmodeled block produce no rendered output. + return null; + } + + private static List<org.apache.tika.grpc.v1.ListItem> listItems(Node list) { + List<org.apache.tika.grpc.v1.ListItem> items = new ArrayList<>(); + for (Node child = list.getFirstChild(); child != null; child = child.getNext()) { + if (child instanceof ListItem) { + items.add(org.apache.tika.grpc.v1.ListItem.newBuilder() + .addAllBlocks(blockChildren(child)) + .build()); + } + } + return items; + } + + private static org.apache.tika.grpc.v1.Table toTable(Node tableBlock) { + org.apache.tika.grpc.v1.Table.Builder table = org.apache.tika.grpc.v1.Table.newBuilder(); + for (Node section = tableBlock.getFirstChild(); section != null; section = section.getNext()) { + if (section instanceof TableHead) { + for (Node row = section.getFirstChild(); row != null; row = row.getNext()) { + if (!(row instanceof TableRow)) { + continue; + } + for (Node cell = row.getFirstChild(); cell != null; cell = cell.getNext()) { + if (cell instanceof TableCell) { + table.addHeader(toTableCell((TableCell) cell)); + } + } + } + } else if (section instanceof TableBody) { + for (Node row = section.getFirstChild(); row != null; row = row.getNext()) { + if (!(row instanceof TableRow)) { + continue; + } + org.apache.tika.grpc.v1.TableRow.Builder rowBuilder = org.apache.tika.grpc.v1.TableRow.newBuilder(); + for (Node cell = row.getFirstChild(); cell != null; cell = cell.getNext()) { + if (cell instanceof TableCell) { + rowBuilder.addCells(toTableCell((TableCell) cell)); + } + } + table.addRows(rowBuilder.build()); + } + } + } + return table.build(); + } + + private static org.apache.tika.grpc.v1.TableCell toTableCell(TableCell cell) { + return org.apache.tika.grpc.v1.TableCell.newBuilder() + .setAlignment(toAlignment(cell.getAlignment())) + .addAllContent(inlineChildren(cell)) + .build(); + } + + private static org.apache.tika.grpc.v1.Alignment toAlignment(TableCell.Alignment alignment) { + if (alignment == null) { + return org.apache.tika.grpc.v1.Alignment.ALIGN_NONE; + } + switch (alignment) { + case LEFT: + return org.apache.tika.grpc.v1.Alignment.ALIGN_LEFT; + case CENTER: + return org.apache.tika.grpc.v1.Alignment.ALIGN_CENTER; + case RIGHT: + return org.apache.tika.grpc.v1.Alignment.ALIGN_RIGHT; + default: + return org.apache.tika.grpc.v1.Alignment.ALIGN_NONE; + } + } + + private static List<org.apache.tika.grpc.v1.Inline> inlineChildren(Node parent) { + List<org.apache.tika.grpc.v1.Inline> inlines = new ArrayList<>(); + for (Node child = parent.getFirstChild(); child != null; child = child.getNext()) { + org.apache.tika.grpc.v1.Inline inline = toInline(child); + if (inline != null) { + inlines.add(inline); + } + } + return inlines; + } + + private static org.apache.tika.grpc.v1.Inline toInline(Node node) { + if (node instanceof Text) { + return org.apache.tika.grpc.v1.Inline.newBuilder().setText(((Text) node).getLiteral()).build(); + } + if (node instanceof Emphasis) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setEmphasis(org.apache.tika.grpc.v1.Emphasis.newBuilder().addAllContent(inlineChildren(node))) + .build(); + } + if (node instanceof StrongEmphasis) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setStrong(org.apache.tika.grpc.v1.Strong.newBuilder().addAllContent(inlineChildren(node))) + .build(); + } + if (node instanceof Strikethrough) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setStrikethrough(org.apache.tika.grpc.v1.Strikethrough.newBuilder().addAllContent(inlineChildren(node))) + .build(); + } + if (node instanceof Code) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setCode(org.apache.tika.grpc.v1.Code.newBuilder().setLiteral(((Code) node).getLiteral())) + .build(); + } + if (node instanceof Link) { + Link link = (Link) node; + org.apache.tika.grpc.v1.Link.Builder linkBuilder = org.apache.tika.grpc.v1.Link.newBuilder() + .setDestination(nullToEmpty(link.getDestination())) + .addAllContent(inlineChildren(link)); + if (link.getTitle() != null) { + linkBuilder.setTitle(link.getTitle()); + } + return org.apache.tika.grpc.v1.Inline.newBuilder().setLink(linkBuilder).build(); + } + if (node instanceof Image) { + Image image = (Image) node; + org.apache.tika.grpc.v1.Image.Builder imageBuilder = org.apache.tika.grpc.v1.Image.newBuilder() + .setDestination(nullToEmpty(image.getDestination())) + .setAlt(collectText(image)); + if (image.getTitle() != null) { + imageBuilder.setTitle(image.getTitle()); + } + return org.apache.tika.grpc.v1.Inline.newBuilder().setImage(imageBuilder).build(); + } + if (node instanceof HardLineBreak) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setLineBreak(org.apache.tika.grpc.v1.LineBreak.newBuilder().setHard(true)) + .build(); + } + if (node instanceof SoftLineBreak) { + return org.apache.tika.grpc.v1.Inline.newBuilder() + .setLineBreak(org.apache.tika.grpc.v1.LineBreak.newBuilder().setHard(false)) + .build(); + } + if (node instanceof HtmlInline) { + return org.apache.tika.grpc.v1.Inline.newBuilder().setHtml(((HtmlInline) node).getLiteral()).build(); + } + // Reference definitions and any other unmodeled inline node produce no rendered output. + return null; + } + + /** Flattens inline content to plain text (for image alt), losing no literals. */ + private static String collectText(Node node) { + StringBuilder sb = new StringBuilder(); + for (Node child = node.getFirstChild(); child != null; child = child.getNext()) { + if (child instanceof Text) { + sb.append(((Text) child).getLiteral()); + } else if (child instanceof Code) { + sb.append(((Code) child).getLiteral()); + } else if (child instanceof HtmlInline) { + sb.append(((HtmlInline) child).getLiteral()); + } else if (child instanceof SoftLineBreak || child instanceof HardLineBreak) { + sb.append(' '); + } else { + sb.append(collectText(child)); + } + } + return sb.toString(); + } + + private static String nullToEmpty(String value) { + return value == null ? "" : value; + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/text/DocumentTextFlattener.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/text/DocumentTextFlattener.java new file mode 100644 index 00000000000..bb3d64ec778 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/text/DocumentTextFlattener.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.text; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.tika.grpc.v1.Block; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.Inline; +import org.apache.tika.grpc.v1.Table; +import org.apache.tika.grpc.v1.TableCell; +import org.apache.tika.grpc.v1.TableRow; + +/** + * Flattens a Tika Document's canonical block tree into analyzable text, constructing the + * exact block-path offset map as it walks (see DESIGN.md for the per-type policy). The + * mapping is exact by construction: every text-bearing block records the range it + * contributed while contributing it, so no offsets are required from, or stamped into, + * the Tika contract. + * + * <p>Blocks are separated by a blank line so sentence detection treats block boundaries + * as hard boundaries. Code blocks and raw html are skipped for analysis and reported in + * {@link FlattenedText#skippedPaths()}. Embedded documents are flattened separately + * by the caller ({@link #flatten(Document)} handles one document's own blocks only).</p> + */ +public final class DocumentTextFlattener { + + /** + * Offsets are only meaningful against the exact flattening that produced them; store + * this next to any persisted annotation, and bump it on any policy change. + */ + public static final String FLATTEN_VERSION = "tika-block-flatten/1"; + + private static final String BLOCK_SEPARATOR = "\n\n"; + private static final String CELL_SEPARATOR = " | "; + + private final StringBuilder text = new StringBuilder(); + private final List<FlattenedText.BlockRange> ranges = new ArrayList<>(); + private final List<String> skipped = new ArrayList<>(); + + private DocumentTextFlattener() { + } + + /** + * Flattens one document's block tree. + * + * @param document the parsed document; must not be null + * @return the flattened text with anchors and provenance carried over + */ + public static FlattenedText flatten(Document document) { + DocumentTextFlattener flattener = new DocumentTextFlattener(); + List<Block> blocks = document.getBlocksList(); + for (int i = 0; i < blocks.size(); i++) { + flattener.block(blocks.get(i), String.valueOf(i)); + } + return new FlattenedText( + document.getId(), + document.getOrigin().getSha256(), + FLATTEN_VERSION, + flattener.text.toString(), + List.copyOf(flattener.ranges), + List.copyOf(flattener.skipped)); + } + + private void block(Block block, String path) { + switch (block.getBlockCase()) { + case HEADING -> anchored(path, () -> inlines(block.getHeading().getContentList())); + case PARAGRAPH -> anchored(path, () -> inlines(block.getParagraph().getContentList())); + case BLOCK_QUOTE -> children(block.getBlockQuote().getBlocksList(), path); + case BULLET_LIST -> items(block.getBulletList().getItemsList().stream() + .map(item -> item.getBlocksList()).toList(), path); + case ORDERED_LIST -> items(block.getOrderedList().getItemsList().stream() + .map(item -> item.getBlocksList()).toList(), path); + case TABLE -> table(block.getTable(), path); + case CODE_BLOCK, HTML_BLOCK -> skipped.add(path); + default -> { + // ThematicBreak and future block types carry no analyzable text. + } + } + } + + private void children(List<Block> blocks, String parentPath) { + for (int i = 0; i < blocks.size(); i++) { + block(blocks.get(i), parentPath + "." + i); + } + } + + private void items(List<List<Block>> itemBlocks, String parentPath) { + for (int i = 0; i < itemBlocks.size(); i++) { + children(itemBlocks.get(i), parentPath + "." + i); + } + } + + private void table(Table table, String path) { + if (!table.getHeaderList().isEmpty()) { + row(table.getHeaderList(), path + ".header"); + } + List<TableRow> rows = table.getRowsList(); + for (int r = 0; r < rows.size(); r++) { + row(rows.get(r).getCellsList(), path + ".rows." + r); + } + } + + // One line per row; every cell gets its own anchor so table-fact consumers can point + // at the exact cell, and the row line as a whole reads as one statement. + private void row(List<TableCell> cells, String rowPath) { + boolean wroteAny = false; + for (int c = 0; c < cells.size(); c++) { + String cellText = renderInlines(cells.get(c).getContentList()); + if (cellText.isEmpty()) { + continue; + } + if (wroteAny) { + text.append(CELL_SEPARATOR); + } else { + separateFromPrevious(); + } + int start = text.length(); + text.append(cellText); + ranges.add(new FlattenedText.BlockRange(rowPath + ".cells." + c, start, + text.length())); + wroteAny = true; + } + } + + // Runs the emitter between separator handling and anchor recording; empty output + // leaves no anchor and no separator. + private void anchored(String path, Runnable emitter) { + int lengthBefore = text.length(); + separateFromPrevious(); + int start = text.length(); + emitter.run(); + if (text.length() == start) { + text.setLength(lengthBefore); + return; + } + ranges.add(new FlattenedText.BlockRange(path, start, text.length())); + } + + private void separateFromPrevious() { + if (!text.isEmpty()) { + text.append(BLOCK_SEPARATOR); + } + } + + private void inlines(List<Inline> content) { + text.append(renderInlines(content)); + } + + private static String renderInlines(List<Inline> content) { + StringBuilder out = new StringBuilder(); + renderInto(content, out); + return out.toString(); + } + + private static void renderInto(List<Inline> content, StringBuilder out) { + for (Inline inline : content) { + switch (inline.getInlineCase()) { + case TEXT -> out.append(inline.getText()); + case EMPHASIS -> renderInto(inline.getEmphasis().getContentList(), out); + case STRONG -> renderInto(inline.getStrong().getContentList(), out); + case STRIKETHROUGH -> + renderInto(inline.getStrikethrough().getContentList(), out); + case CODE -> out.append(inline.getCode().getLiteral()); + case LINK -> renderInto(inline.getLink().getContentList(), out); + case IMAGE -> out.append(inline.getImage().getAlt()); + case LINE_BREAK -> out.append(inline.getLineBreak().getHard() ? "\n" : " "); + default -> { + // raw inline html contributes nothing analyzable + } + } + } + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/text/FlattenedText.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/text/FlattenedText.java new file mode 100644 index 00000000000..d9872a89b3e --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/text/FlattenedText.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.text; + +import java.util.List; + +/** + * The result of flattening one Tika Document (or one embedded child): the analyzable + * text, the exact block-path anchors constructed during the walk, and the provenance + * needed to tie downstream annotations back to the parsed bytes. + * + * @param documentId the Tika Document id ("" when the source carried none) + * @param sourceSha256 SourceOrigin.sha256 ("" when the parse pipeline had no digester) + * @param flattenVersion the flattening algorithm version the offsets are valid against + * @param text the flattened text OpenNLP analyzes + * @param ranges text-bearing anchors in ascending start order, non-overlapping + * @param skippedPaths block paths deliberately not analyzed (code blocks, raw html) + */ +public record FlattenedText(String documentId, String sourceSha256, String flattenVersion, + String text, List<BlockRange> ranges, + List<String> skippedPaths) { + + /** + * One text-bearing anchor: the block (or table cell) at {@code path} produced + * {@code text[start, end)}. + */ + public record BlockRange(String path, int start, int end) { + } + + /** + * Projects a span in flattened coordinates onto the block anchors it overlaps, in + * order. The common case is a single anchor; a span crossing a block boundary (rare, + * since blocks are separated by blank lines) reports every anchor it touches with + * the local range inside each. + * + * @param start inclusive start in flattened coordinates + * @param end exclusive end; must be greater than {@code start} and within the text + * @return the overlapped anchors with span-local ranges, block order + * @throws IllegalArgumentException if the span is empty, inverted, or out of bounds + */ + public List<Projection> project(int start, int end) { + if (start < 0 || end <= start || end > text.length()) { + throw new IllegalArgumentException( + "Span [" + start + ", " + end + ") is not a valid range of a text of length " + + text.length()); + } + return ranges.stream() + .filter(r -> r.start() < end && start < r.end()) + .map(r -> new Projection(r.path(), + Math.max(start, r.start()) - r.start(), + Math.min(end, r.end()) - r.start())) + .toList(); + } + + /** + * A span projected into one block's local coordinates: {@code [localStart, localEnd)} + * within the text that block contributed. + */ + public record Projection(String path, int localStart, int localEnd) { + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformer.java new file mode 100644 index 00000000000..ef5f9133390 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformer.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Property; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.metadata.XMPRights; + +/** + * Creative Commons / XMP rights detection. Unlike the format transformers, this one is + * cross-cutting: Creative Commons or other rights-management metadata can ride along with + * any document format (a PDF, a JPEG, an Office doc, ...), so it does not key off + * content-type at all. Instead it inspects the full metadata for CC/license-shaped field + * names and values, or for any populated XMP rights-management property. + * + * <p>Rights-management specifics (marked, owner, usage terms, certificate, ...) are NOT + * given their own proto fields. They flow into the tagged tail, typed where Tika declares + * the type and string otherwise - same philosophy as the format transformers. + */ +public final class CreativeCommonsDocumentTransformer implements DocumentTransformer { + + private static final String[] NAME_HINTS = {"license", "creative", "cc:", "rights"}; + + private static final Property[] XMP_RIGHTS_PROPERTIES = { + XMPRights.MARKED, + XMPRights.OWNER, + XMPRights.USAGE_TERMS, + XMPRights.WEB_STATEMENT, + XMPRights.CERTIFICATE + }; + + @Override + public boolean isCrossCutting() { + return true; + } + + @Override + public boolean appliesTo(Metadata tika) { + for (String name : tika.names()) { + String lowerName = name.toLowerCase(Locale.ROOT); + for (String hint : NAME_HINTS) { + if (lowerName.contains(hint)) { + String value = tika.get(name); + if (value != null && value.toLowerCase(Locale.ROOT).contains("creative")) { + return true; + } + break; + } + } + } + + for (Property property : XMP_RIGHTS_PROPERTIES) { + String value = tika.get(property); + if (value != null && !value.trim().isEmpty()) { + return true; + } + } + + return false; + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + TransformSupport.setString(tika, TikaCoreProperties.RIGHTS, meta::setRights, consumed); + // A CC license URL, e.g. https://creativecommons.org/licenses/by/4.0/, typically + // lives in the XMP rights "web statement" property. + TransformSupport.setString(tika, XMPRights.WEB_STATEMENT, meta::setLicenseUrl, consumed); + + // Everything else (xmpRights:Marked, xmpRights:Owner, xmpRights:UsageTerms, + // xmpRights:Certificate, ...) lands in the tagged tail, appended once by + // DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformer.java new file mode 100644 index 00000000000..80bacf815ca --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformer.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; + +/** + * Maps one Tika {@link Metadata} (a document, or an embedded part) onto the typed + * {@link Document}. Transformers are code, not schema: adding a parser adds a + * transformer, and the wire contract never changes. More than one transformer may + * apply to a single document (a format transformer plus cross-cutting ones, such as + * Creative Commons rights detection alongside a PDF or Office transformer). + */ +public interface DocumentTransformer { + + /** + * True if this transformer applies to the given document. Most transformers only + * look at {@code tika.get(Metadata.CONTENT_TYPE)}; cross-cutting ones (e.g. Creative + * Commons rights detection) may inspect the full metadata instead. + */ + boolean appliesTo(Metadata tika); + + /** + * Populate typed fields on the builder, and mark every Tika key consumed in + * {@code consumed}. {@code consumed} is shared across every transformer that applies to + * this document (not private to this transformer), so the tagged tail -- appended once, + * after every applicable transformer has run -- never duplicates a key that a different + * transformer already mapped to a typed field. + */ + void transform(Metadata tika, Document.Builder document, Set<String> consumed); + + /** + * True if this transformer is cross-cutting (may apply alongside, not instead of, a + * format transformer -- e.g. Creative Commons rights detection). Cross-cutting + * transformers do not count toward whether a document "matched" a format, so they never + * suppress the generic fallback: a plain-text document with CC rights metadata still + * gets the universal title/author/description mapping. + */ + default boolean isCrossCutting() { + return false; + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformers.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformers.java new file mode 100644 index 00000000000..d97e740c0bc --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/DocumentTransformers.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; + +/** + * Picks and runs the applicable transformer(s) for a parsed document. Multiple may + * apply (e.g. a format transformer plus the Creative Commons rights transformer); the + * generic fallback runs whenever no format-specific transformer matched, so an unknown + * format still yields a useful, lossless Document. + * + * Adding support for a new format means adding a transformer to {@link #defaults()}. + * The proto does not change, so clients never rebuild for it. + */ +public final class DocumentTransformers { + + private final List<DocumentTransformer> transformers; + private final DocumentTransformer fallback = new GenericDocumentTransformer(); + + public DocumentTransformers(List<DocumentTransformer> transformers) { + this.transformers = transformers; + } + + public static DocumentTransformers defaults() { + return new DocumentTransformers(List.of( + new PdfDocumentTransformer(), + new OfficeDocumentTransformer(), + new ImageDocumentTransformer(), + new HtmlDocumentTransformer(), + new RtfDocumentTransformer(), + new EpubDocumentTransformer(), + new WarcDocumentTransformer(), + new CreativeCommonsDocumentTransformer() + )); + } + + /** Populate the typed metadata/tagged-tail portion of {@code document} for {@code tika}. */ + public void transform(Metadata tika, Document.Builder document) { + // Shared across every applicable transformer so the tagged tail, appended exactly + // once below, never duplicates a key that a different transformer already typed. + Set<String> consumed = new HashSet<>(); + boolean matchedFormat = false; + for (DocumentTransformer transformer : transformers) { + if (transformer.appliesTo(tika)) { + transformer.transform(tika, document, consumed); + matchedFormat |= !transformer.isCrossCutting(); + } + } + if (!matchedFormat) { + fallback.transform(tika, document, consumed); + } + MetadataTagger.appendTail(tika, consumed, document); + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformer.java new file mode 100644 index 00000000000..0a302d9d241 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformer.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; + +/** + * EPUB. Maps the common, cross-format facts into typed DocumentMetadata. + * + * EPUB-specific properties (rendition layout, spec version, OPF/NAV structural fields, ...) + * are NOT given their own proto fields. They flow into the tagged tail, typed where Tika + * declares the type and string otherwise. That is the whole point: format richness lives + * in code plus the tail, not in the wire contract - so the schema stays small and stable + * and clients never rebuild when we add or change an EPUB property. + */ +public final class EpubDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + return contentType != null && contentType.toLowerCase(Locale.ROOT).contains("epub"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { + // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + TransformSupport.mapCommonFields(tika, document.getMetadataBuilder(), consumed); + + // Everything EPUB-specific (epub:*, dc:identifier, OPF/NAV structural fields, ...) + // lands in the tagged tail, appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/FormatCategoryDetector.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/FormatCategoryDetector.java new file mode 100644 index 00000000000..0e570edde82 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/FormatCategoryDetector.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; + +import org.apache.tika.grpc.v1.FormatCategory; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * Detects the coarse {@link FormatCategory} routing hint from Tika metadata. This is + * independent of which {@link DocumentTransformer}s actually run -- transformers + * self-select via {@link DocumentTransformer#appliesTo(String)} and are not mutually + * exclusive (e.g. Creative Commons rights can coexist with any category here), so this + * enum exists purely as a cheap client-side routing hint. + */ +public final class FormatCategoryDetector { + + private FormatCategoryDetector() { + } + + public static FormatCategory detect(Metadata metadata) { + String mimeType = metadata.get(Metadata.CONTENT_TYPE); + String resourceName = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY); + + if (mimeType != null) { + String mime = mimeType.toLowerCase(Locale.ROOT); + if (mime.contains("pdf")) { + return FormatCategory.FORMAT_CATEGORY_PDF; + } + if (mime.contains("officedocument") || mime.contains("msword") + || mime.contains("ms-excel") || mime.contains("ms-powerpoint") + || mime.contains("opendocument") || mime.contains("vnd.ms-") + || mime.contains("vnd.openxmlformats")) { + return FormatCategory.FORMAT_CATEGORY_OFFICE; + } + if (mime.startsWith("image/")) { + return FormatCategory.FORMAT_CATEGORY_IMAGE; + } + if (mime.contains("text/html") || mime.contains("application/xhtml")) { + return FormatCategory.FORMAT_CATEGORY_HTML; + } + if (mime.contains("rtf")) { + return FormatCategory.FORMAT_CATEGORY_RTF; + } + if (mime.contains("epub")) { + return FormatCategory.FORMAT_CATEGORY_EPUB; + } + if (mime.contains("warc")) { + return FormatCategory.FORMAT_CATEGORY_WARC; + } + } + + if (resourceName != null) { + String name = resourceName.toLowerCase(Locale.ROOT); + if (name.endsWith(".pdf")) { + return FormatCategory.FORMAT_CATEGORY_PDF; + } + if (name.endsWith(".doc") || name.endsWith(".docx") || name.endsWith(".xls") + || name.endsWith(".xlsx") || name.endsWith(".ppt") || name.endsWith(".pptx") + || name.endsWith(".odt") || name.endsWith(".ods") || name.endsWith(".odp")) { + return FormatCategory.FORMAT_CATEGORY_OFFICE; + } + if (name.endsWith(".jpg") || name.endsWith(".jpeg") || name.endsWith(".png") + || name.endsWith(".gif") || name.endsWith(".tiff") || name.endsWith(".tif") + || name.endsWith(".bmp")) { + return FormatCategory.FORMAT_CATEGORY_IMAGE; + } + if (name.endsWith(".html") || name.endsWith(".htm") || name.endsWith(".xhtml")) { + return FormatCategory.FORMAT_CATEGORY_HTML; + } + if (name.endsWith(".rtf")) { + return FormatCategory.FORMAT_CATEGORY_RTF; + } + if (name.endsWith(".epub")) { + return FormatCategory.FORMAT_CATEGORY_EPUB; + } + if (name.endsWith(".warc") || name.endsWith(".arc")) { + return FormatCategory.FORMAT_CATEGORY_WARC; + } + } + + return FormatCategory.FORMAT_CATEGORY_GENERIC; + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/GenericDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/GenericDocumentTransformer.java new file mode 100644 index 00000000000..0a435028439 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/GenericDocumentTransformer.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; + +/** + * Always-applicable fallback. Pulls the universal Dublin Core fields into typed metadata + * and routes everything else to the tagged tail, so an unknown or unsupported format + * still produces a useful, lossless Document. + */ +public final class GenericDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + return true; + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { + TransformSupport.mapCommonFields(tika, document.getMetadataBuilder(), consumed); + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformer.java new file mode 100644 index 00000000000..7d7bd41bcea --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformer.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; + +/** + * HTML / XHTML. Maps the common, cross-format facts into typed DocumentMetadata. + * + * HTML-specific properties (html:meta:*, Open Graph og:*, Twitter Card twitter:*, + * html:link:*, ICBM geo, encoding-detector fields, ...) are NOT given their own proto + * fields. They flow into the tagged tail, typed where Tika declares the type and string + * otherwise. That is the whole point: format richness lives in code plus the tail, not + * in the wire contract - so the schema stays small and stable and clients never rebuild + * when we add or change an HTML property. + */ +public final class HtmlDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + if (contentType == null) { + return false; + } + String lower = contentType.toLowerCase(Locale.ROOT); + return lower.contains("text/html") || lower.contains("application/xhtml"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { + // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + TransformSupport.mapCommonFields(tika, document.getMetadataBuilder(), consumed); + + // Everything HTML-specific (html:meta:*, og:*, twitter:*, html:link:*, ICBM, ...) + // lands in the tagged tail, appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformer.java new file mode 100644 index 00000000000..e9ce47d8974 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformer.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.IPTC; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TIFF; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * Images. Maps the common, cross-format facts into typed DocumentMetadata. + * + * Image-specific properties (EXIF exposure/f-number/focal-length/flash/ISO, GPS + * coordinates, IPTC headline/category/credit-line/copyright-notice, Photoshop + * city/country/state, equipment make/model, resolution, orientation, ...) are NOT + * given their own proto fields. They flow into the tagged tail, typed where Tika + * declares the type and string otherwise. That is the whole point: format richness + * lives in code plus the tail, not in the wire contract - so the schema stays small + * and stable and clients never rebuild when we add or change an image property. + */ +public final class ImageDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + return contentType != null && contentType.toLowerCase(Locale.ROOT).startsWith("image/"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + // Images deliberately do NOT use TransformSupport.mapCommonFields: their common + // facts come from image-native properties (IPTC keywords, EXIF/TIFF original date), + // and dc:creator / dc:language / dcterms:created rarely mean for a photo what they + // mean for a document, so those stay in the tagged tail untouched. + TransformSupport.setString(tika, TikaCoreProperties.TITLE, meta::setTitle, consumed); + TransformSupport.setString(tika, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); + TransformSupport.addStrings(tika, IPTC.KEYWORDS, meta::addAllKeywords, consumed); + TransformSupport.setTimestamp(tika, TIFF.ORIGINAL_DATE, meta::setCreated, consumed); + TransformSupport.setTimestamp(tika, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + TransformSupport.setInt(tika, TIFF.IMAGE_WIDTH, meta::setWidth, consumed); + TransformSupport.setInt(tika, TIFF.IMAGE_LENGTH, meta::setHeight, consumed); + + // Everything image-specific (exif:*, tiff:*, IPTC-*, photoshop:*, geo:*, ...) lands + // in the tagged tail, appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/MetadataTagger.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/MetadataTagger.java new file mode 100644 index 00000000000..330469ac0f4 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/MetadataTagger.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.time.Instant; +import java.util.Date; +import java.util.Locale; +import java.util.Set; + +import com.google.protobuf.Timestamp; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.MetadataField; +import org.apache.tika.grpc.v1.MetadataValue; +import org.apache.tika.grpc.v1.StringList; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Property; + +/** + * Builds the tagged metadata tail. The tag comes from Tika's <em>declared</em> + * {@link Property} value type. Keys with no declared type are emitted as strings, + * never guessed - so we never turn "8 8 8 8" into a broken integer. + */ +public final class MetadataTagger { + + /** + * {@link Property#get(String)} can only see Property constants whose declaring class + * has already been initialized by the JVM. In the gRPC server the parse runs in a + * forked pipes process, so nothing in the server JVM necessarily touches e.g. + * {@code org.apache.tika.metadata.PDF} before mapping -- and declared types + * ({@code pdf:encrypted} is a boolean, ...) would silently degrade to plain strings + * depending on class-loading order. Force-initialize the tika-core property + * vocabularies once so tagging is deterministic. + */ + private static final Class<?>[] PROPERTY_VOCABULARIES = { + org.apache.tika.metadata.AccessPermissions.class, + org.apache.tika.metadata.ClimateForcast.class, + org.apache.tika.metadata.CreativeCommons.class, + org.apache.tika.metadata.Database.class, + org.apache.tika.metadata.DublinCore.class, + org.apache.tika.metadata.DWG.class, + org.apache.tika.metadata.Epub.class, + org.apache.tika.metadata.ExternalProcess.class, + org.apache.tika.metadata.FileSystem.class, + org.apache.tika.metadata.Font.class, + org.apache.tika.metadata.Geographic.class, + org.apache.tika.metadata.HTML.class, + org.apache.tika.metadata.HttpHeaders.class, + org.apache.tika.metadata.IPTC.class, + org.apache.tika.metadata.MachineMetadata.class, + org.apache.tika.metadata.MAPI.class, + org.apache.tika.metadata.Message.class, + org.apache.tika.metadata.Office.class, + org.apache.tika.metadata.OfficeOpenXMLCore.class, + org.apache.tika.metadata.OfficeOpenXMLExtended.class, + org.apache.tika.metadata.PageAnchoring.class, + org.apache.tika.metadata.PagedText.class, + org.apache.tika.metadata.PDF.class, + org.apache.tika.metadata.Photoshop.class, + org.apache.tika.metadata.PST.class, + org.apache.tika.metadata.QuattroPro.class, + org.apache.tika.metadata.Rendering.class, + org.apache.tika.metadata.RTFMetadata.class, + org.apache.tika.metadata.TIFF.class, + org.apache.tika.metadata.TikaCoreProperties.class, + org.apache.tika.metadata.TikaPagedText.class, + org.apache.tika.metadata.WARC.class, + org.apache.tika.metadata.WordPerfect.class, + org.apache.tika.metadata.XMP.class, + org.apache.tika.metadata.XMPDC.class, + org.apache.tika.metadata.XMPDM.class, + org.apache.tika.metadata.XMPIdq.class, + org.apache.tika.metadata.XMPMM.class, + org.apache.tika.metadata.XMPPDF.class, + org.apache.tika.metadata.XMPRights.class, + org.apache.tika.metadata.Zip.class, + }; + + static { + for (Class<?> vocabulary : PROPERTY_VOCABULARIES) { + try { + // A class literal alone does NOT initialize the class; forName(…, true, …) does. + Class.forName(vocabulary.getName(), true, vocabulary.getClassLoader()); + } catch (ClassNotFoundException e) { + throw new ExceptionInInitializerError(e); + } + } + } + + private MetadataTagger() { + } + + /** Append every key not already consumed by a typed field into {@code Document.extra}. */ + public static void appendTail(Metadata tika, Set<String> consumed, Document.Builder document) { + for (String key : tika.names()) { + if (consumed.contains(key)) { + continue; + } + MetadataValue value = tag(tika, key); + if (value != null) { + document.addExtra(MetadataField.newBuilder().setKey(key).setValue(value).build()); + } + } + } + + /** Tag one key by its declared Property type; fall back to multivalue strings. */ + public static MetadataValue tag(Metadata tika, String key) { + String[] values = tika.getValues(key); + if (values == null || values.length == 0) { + return null; + } + Property property = Property.get(key); + if (values.length == 1 && property != null) { + MetadataValue typed = tagByDeclaredType(tika, property, values[0].trim()); + if (typed != null) { + return typed; + } + } + StringList.Builder strings = StringList.newBuilder(); + for (String v : values) { + if (v != null && !v.trim().isEmpty()) { + strings.addValues(v.trim()); + } + } + return strings.getValuesCount() == 0 + ? null + : MetadataValue.newBuilder().setStrings(strings).build(); + } + + private static MetadataValue tagByDeclaredType(Metadata tika, Property property, String raw) { + switch (property.getPrimaryProperty().getValueType()) { + case INTEGER: + try { + return MetadataValue.newBuilder().setInteger(Long.parseLong(raw)).build(); + } catch (NumberFormatException ignored) { + return null; + } + case REAL: + case RATIONAL: + try { + return MetadataValue.newBuilder().setNumber(Double.parseDouble(raw)).build(); + } catch (NumberFormatException ignored) { + return null; + } + case BOOLEAN: + return MetadataValue.newBuilder().setBoolean(parseBoolean(raw)).build(); + case DATE: + Date date = tika.getDate(property); + return date == null + ? null + : MetadataValue.newBuilder().setTimestamp(toTimestamp(date)).build(); + default: + return null; // not a declared scalar -> caller falls back to strings + } + } + + static boolean parseBoolean(String v) { + String s = v.trim().toLowerCase(Locale.ROOT); + return "true".equals(s) || "yes".equals(s) || "1".equals(s); + } + + static Timestamp toTimestamp(Date date) { + Instant i = date.toInstant(); + return Timestamp.newBuilder().setSeconds(i.getEpochSecond()).setNanos(i.getNano()).build(); + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformer.java new file mode 100644 index 00000000000..1dd66fb33ec --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformer.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Office; + +/** + * Office (MS Word/Excel/PowerPoint, OOXML, OpenDocument). Maps the common, cross-format + * facts into typed DocumentMetadata. + * + * Office-specific properties (slide/paragraph/line/table/image/object counts, OOXML + * core/extended properties, hidden sheets/slides, comments, track changes, ...) are NOT + * given their own proto fields. They flow into the tagged tail, typed where Tika declares + * the type and string otherwise. That is the whole point: format richness lives in code + * plus the tail, not in the wire contract - so the schema stays small and stable and + * clients never rebuild when we add or change an Office property. + */ +public final class OfficeDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + if (contentType == null) { + return false; + } + String lower = contentType.toLowerCase(Locale.ROOT); + return lower.contains("officedocument") + || lower.contains("msword") + || lower.contains("ms-excel") + || lower.contains("ms-powerpoint") + || lower.contains("opendocument") + || lower.contains("vnd.ms-") + || lower.contains("vnd.openxmlformats"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + // Office keywords live under meta:keyword, not dc:subject. + TransformSupport.mapCommonFields(tika, meta, consumed, Office.KEYWORDS); + TransformSupport.setInt(tika, Office.PAGE_COUNT, meta::setPageCount, consumed); + TransformSupport.setLong(tika, Office.WORD_COUNT, meta::setWordCount, consumed); + TransformSupport.setLong(tika, Office.CHARACTER_COUNT, meta::setCharacterCount, consumed); + + // Everything Office-specific (slide/paragraph/line/table/image/object counts, + // OOXML core/extended properties, hidden sheets/slides, comments, track changes, ...) + // lands in the tagged tail, appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformer.java new file mode 100644 index 00000000000..0439a0cc595 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformer.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Office; +import org.apache.tika.metadata.PagedText; + +/** + * PDF. Maps the common, cross-format facts into typed DocumentMetadata. + * + * PDF-specific properties (encryption flags, XMP, permissions, incremental updates, ...) + * are NOT given their own proto fields. They flow into the tagged tail, typed where Tika + * declares the type and string otherwise. That is the whole point: format richness lives + * in code plus the tail, not in the wire contract - so the schema stays small and stable + * and clients never rebuild when we add or change a PDF property. + */ +public final class PdfDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + return contentType != null && contentType.toLowerCase(Locale.ROOT).contains("pdf"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + TransformSupport.mapCommonFields(tika, meta, consumed); + TransformSupport.setInt(tika, PagedText.N_PAGES, meta::setPageCount, consumed); + TransformSupport.setLong(tika, Office.WORD_COUNT, meta::setWordCount, consumed); + + // Everything PDF-specific (pdf:*, xmpPDF:*, access_permission:*, ...) lands in the + // tagged tail, appended once by DocumentTransformers after every applicable + // transformer has contributed to `consumed`. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformer.java new file mode 100644 index 00000000000..5d298356149 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformer.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Office; + +/** + * RTF. Maps the common, cross-format facts into typed DocumentMetadata. + * + * RTF-specific properties (encapsulated HTML flags, embedded-object thumbnails and + * application/class/topic/item strings, OOXML core/extended fields such as category, + * manager, company, template, ...) are NOT given their own proto fields. They flow into + * the tagged tail, typed where Tika declares the type and string otherwise. That is the + * whole point: format richness lives in code plus the tail, not in the wire contract - so + * the schema stays small and stable and clients never rebuild when we add or change an + * RTF property. + */ +public final class RtfDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + return contentType != null && contentType.toLowerCase(Locale.ROOT).contains("rtf"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { + DocumentMetadata.Builder meta = document.getMetadataBuilder(); + + // Common, cross-format fields -> typed (a date is a Timestamp, a count is an int). + // RTF keywords ride on the Office property set (meta:keyword), not dc:subject. + TransformSupport.mapCommonFields(tika, meta, consumed, Office.KEYWORDS); + TransformSupport.setInt(tika, Office.PAGE_COUNT, meta::setPageCount, consumed); + TransformSupport.setLong(tika, Office.WORD_COUNT, meta::setWordCount, consumed); + TransformSupport.setLong(tika, Office.CHARACTER_COUNT, meta::setCharacterCount, consumed); + + // Everything RTF-specific (rtf_meta:*, extended-properties:*, ...) lands in the + // tagged tail, appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/TransformSupport.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/TransformSupport.java new file mode 100644 index 00000000000..0a116a3189c --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/TransformSupport.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import com.google.protobuf.Timestamp; + +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Property; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * Helpers for transformers: set a typed Document field from a Tika Property and mark the + * key consumed so it is not duplicated into the tagged tail. + */ +public final class TransformSupport { + + private TransformSupport() { + } + + /** + * Maps the common, cross-format fields -- title, description, authors, keywords, + * languages, publishers, identifiers, created, modified -- into typed + * {@link DocumentMetadata}. These are the Dublin Core descriptive elements; keywords + * come from {@link TikaCoreProperties#SUBJECT}. Kept in one place so the common + * mapping cannot drift between format transformers. + */ + public static void mapCommonFields(Metadata md, DocumentMetadata.Builder meta, Set<String> consumed) { + mapCommonFields(md, meta, consumed, TikaCoreProperties.SUBJECT); + } + + /** + * Same as {@link #mapCommonFields(Metadata, DocumentMetadata.Builder, Set)} but with + * the keyword source overridden, for formats whose keyword bag lives under a + * different property (e.g. Office and RTF use {@code meta:keyword}); {@code dc:subject} + * then stays in the tagged tail because those formats treat it as a distinct field. + */ + public static void mapCommonFields(Metadata md, DocumentMetadata.Builder meta, Set<String> consumed, + Property keywordsProperty) { + setString(md, TikaCoreProperties.TITLE, meta::setTitle, consumed); + setString(md, TikaCoreProperties.DESCRIPTION, meta::setDescription, consumed); + addStrings(md, TikaCoreProperties.CREATOR, meta::addAllAuthors, consumed); + addStrings(md, keywordsProperty, meta::addAllKeywords, consumed); + addStrings(md, TikaCoreProperties.LANGUAGE, meta::addAllLanguages, consumed); + addStrings(md, TikaCoreProperties.PUBLISHER, meta::addAllPublishers, consumed); + addStrings(md, TikaCoreProperties.IDENTIFIER, meta::addAllIdentifiers, consumed); + setTimestamp(md, TikaCoreProperties.CREATED, meta::setCreated, consumed); + setTimestamp(md, TikaCoreProperties.MODIFIED, meta::setModified, consumed); + } + + public static void setString(Metadata md, Property key, Consumer<String> setter, Set<String> consumed) { + String v = md.get(key); + if (v != null && !v.trim().isEmpty()) { + setter.accept(v.trim()); + consumed.add(key.getName()); + } + } + + public static void addStrings(Metadata md, Property key, Consumer<Iterable<String>> setter, Set<String> consumed) { + String[] values = md.getValues(key); + if (values != null && values.length > 0) { + List<String> list = Arrays.stream(values) + .filter(s -> s != null && !s.trim().isEmpty()) + .map(String::trim) + .collect(Collectors.toList()); + if (!list.isEmpty()) { + setter.accept(list); + consumed.add(key.getName()); + } + } + } + + public static void setInt(Metadata md, Property key, Consumer<Integer> setter, Set<String> consumed) { + Integer v = md.getInt(key); + if (v != null) { + setter.accept(v); + consumed.add(key.getName()); + } + } + + public static void setLong(Metadata md, Property key, Consumer<Long> setter, Set<String> consumed) { + Integer v = md.getInt(key); + if (v != null) { + setter.accept(v.longValue()); + consumed.add(key.getName()); + } + } + + public static void setDouble(Metadata md, Property key, Consumer<Double> setter, Set<String> consumed) { + String v = md.get(key); + if (v != null && !v.trim().isEmpty()) { + try { + setter.accept(Double.parseDouble(v.trim())); + consumed.add(key.getName()); + } catch (NumberFormatException ignored) { + // leave unconsumed; falls through to the tagged tail + } + } + } + + public static void setTimestamp(Metadata md, Property key, Consumer<Timestamp> setter, Set<String> consumed) { + Date d = md.getDate(key); + if (d != null) { + setter.accept(MetadataTagger.toTimestamp(d)); + consumed.add(key.getName()); + } + } +} diff --git a/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformer.java b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformer.java new file mode 100644 index 00000000000..195e39f7793 --- /dev/null +++ b/tika-grpc-mapper/src/main/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformer.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import java.util.Locale; +import java.util.Set; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; + +/** + * WARC (Web ARChive) containers. Unlike most formats, WARC's useful facts - WARC-Date, + * WARC-Target-URI, WARC-Record-ID, HTTP status, HTTP response headers, ... - are not + * normalized by {@code org.apache.tika.parser.warc.WARCParser} onto declared + * {@link org.apache.tika.metadata.Property} constants. The parser adds them under literal + * string keys instead ({@code warc:WARC-Date}, {@code warc:WARC-Target-URI}, + * {@code warc:http:status}, {@code warc:http:*}, ...). {@code org.apache.tika.metadata.WARC} + * only declares a handful of properties (WARC_RECORD_CONTENT_TYPE, WARC_PAYLOAD_CONTENT_TYPE, + * WARC_RECORD_ID, WARC_WARNING), and none of the common, cross-format facts that this + * transformer could map (title, created, ...) are actually populated by the parser for a + * WARC record. + * + * There is therefore nothing here that fairly maps onto the small set of typed + * DocumentMetadata fields - forcing one would be dishonest, not useful. That is fine: the + * tagged tail already handles arbitrary string keys, typed by Tika's declared Property type + * where one exists and as plain strings otherwise, so none of the WARC/HTTP richness is + * lost - it just does not need its own proto fields. + */ +public final class WarcDocumentTransformer implements DocumentTransformer { + + @Override + public boolean appliesTo(Metadata tika) { + String contentType = tika.get(Metadata.CONTENT_TYPE); + return contentType != null && contentType.toLowerCase(Locale.ROOT).contains("warc"); + } + + @Override + public void transform(Metadata tika, Document.Builder document, Set<String> consumed) { + // No common field fairly maps to the typed DocumentMetadata for a WARC record (see + // class javadoc). Everything (warc:WARC-Date, warc:WARC-Target-URI, + // warc:WARC-Record-ID, warc:http:status, warc:http:*, ...) lands in the tagged tail, + // appended once by DocumentTransformers. + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ClasspathTestDocuments.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ClasspathTestDocuments.java new file mode 100644 index 00000000000..fba712e0a1e --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ClasspathTestDocuments.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper; + +import java.io.IOException; +import java.net.JarURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +/** + * Discovers {@code test-documents/*} fixtures contributed by parser module test-jars on the classpath. + */ +final class ClasspathTestDocuments { + + private static final String TEST_DOCUMENTS_DIR = "test-documents"; + + private ClasspathTestDocuments() { + } + + static List<String> listByExtension(String extension) throws IOException { + String suffix = extension.startsWith(".") ? extension : "." + extension; + Set<String> found = new TreeSet<>(); + ClassLoader classLoader = ClasspathTestDocuments.class.getClassLoader(); + Enumeration<URL> roots = classLoader.getResources(TEST_DOCUMENTS_DIR); + while (roots.hasMoreElements()) { + collectFromUrl(roots.nextElement(), suffix, found); + } + return new ArrayList<>(found); + } + + static List<String> listByExtensions(String... extensions) throws IOException { + Set<String> found = new TreeSet<>(); + for (String extension : extensions) { + found.addAll(listByExtension(extension)); + } + return new ArrayList<>(found); + } + + private static void collectFromUrl(URL url, String suffix, Set<String> found) throws IOException { + if ("file".equals(url.getProtocol())) { + try { + Path dir = Paths.get(url.toURI()); + if (Files.isDirectory(dir)) { + collectFromDirectory(dir, suffix, found); + } + } catch (java.net.URISyntaxException e) { + throw new IOException("Bad test-documents URL: " + url, e); + } + return; + } + if ("jar".equals(url.getProtocol())) { + collectFromJarUrl(url, suffix, found); + } + } + + private static void collectFromDirectory(Path dir, String suffix, Set<String> found) throws IOException { + try (var paths = Files.walk(dir)) { + paths.filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().endsWith(suffix)) + .forEach(path -> found.add(path.getFileName().toString())); + } + } + + private static void collectFromJarUrl(URL url, String suffix, Set<String> found) throws IOException { + JarURLConnection connection = (JarURLConnection) url.openConnection(); + String prefix = connection.getEntryName(); + if (prefix == null) { + prefix = TEST_DOCUMENTS_DIR; + } + if (!prefix.endsWith("/")) { + prefix = prefix + "/"; + } + JarFile jarFile = connection.getJarFile(); + final String entryPrefix = prefix; + for (JarEntry entry : jarFile.stream().toList()) { + if (entry.isDirectory()) { + continue; + } + String name = entry.getName(); + if (!name.startsWith(entryPrefix) || !name.endsWith(suffix)) { + continue; + } + int slash = name.lastIndexOf('/'); + found.add(slash >= 0 ? name.substring(slash + 1) : name); + } + } + +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/DocumentBuilderTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/DocumentBuilderTest.java new file mode 100644 index 00000000000..96c3fcd98ad --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/DocumentBuilderTest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.FormatCategory; +import org.apache.tika.grpc.v1.ParseStatus; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Property; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * End-to-end coverage of {@link DocumentBuilder}: the content tree (parsed from the + * pipes-default markdown rendering), typed/tagged metadata (via {@link + * org.apache.tika.grpc.mapper.transform.DocumentTransformers}), format-category routing, + * status, and embedded-document recursion. + */ +class DocumentBuilderTest extends ParseFixtureSupport { + + @Test + void buildsContentTreeAndMetadataFromRealPdf() throws Exception { + Document document = map(parseBody("testPDF.pdf"), "doc-1", 42L); + + assertFalse(document.getBlocksList().isEmpty()); + assertTrue(blockText(document).contains("Tika")); + assertEquals(FormatCategory.FORMAT_CATEGORY_PDF, document.getFormatCategory()); + assertEquals("application/pdf", document.getContentType()); + assertEquals("Apache Tika - Apache Tika", document.getMetadata().getTitle()); + assertTrue(document.getExtraCount() > 0); + + assertEquals(ParseStatus.Status.SUCCESS, document.getStatus().getStatus()); + assertEquals("PARSE_SUCCESS", document.getStatus().getPipesStatus()); + assertEquals(42L, document.getStatus().getFetchParseTimeMs()); + } + + /** + * The block tree is the canonical content and is always populated; the flat markdown + * string is a rendering of the same tree and is only carried on request, so a reply + * does not ship the content twice. + */ + @Test + void markdownRenderingIsOnlyCarriedOnRequest() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "text/plain"); + String body = "# Title\n\nSome text.\n"; + + Document withoutRender = DocumentBuilder.build( + metadata, List.of(metadata), body, "d", "PARSE_SUCCESS", 1L, false); + assertFalse(withoutRender.getBlocksList().isEmpty()); + assertTrue(withoutRender.getMarkdown().isEmpty()); + + Document withRender = DocumentBuilder.build( + metadata, List.of(metadata), body, "d", "PARSE_SUCCESS", 1L, true); + assertFalse(withRender.getBlocksList().isEmpty()); + assertEquals(body, withRender.getMarkdown()); + assertEquals(withoutRender.getBlocksList(), withRender.getBlocksList()); + } + + @Test + void mapsSourceDigestWhenThePipelineRecordedOne() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "text/plain"); + // Reserved X-TIKA keys only accept Property writes (TIKA-4769); this mirrors how + // InputStreamDigester records the digest. + metadata.set(Property.internalText(TikaCoreProperties.TIKA_META_PREFIX + "digest" + + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER + "SHA256"), "0a1b2c3d"); + + Document document = DocumentBuilder.build( + metadata, List.of(metadata), "text", "d", "PARSE_SUCCESS", 1L, false); + assertEquals("0a1b2c3d", document.getOrigin().getSha256()); + + Metadata undigested = new Metadata(); + undigested.set(Metadata.CONTENT_TYPE, "text/plain"); + Document withoutDigest = DocumentBuilder.build( + undigested, List.of(undigested), "text", "d", "PARSE_SUCCESS", 1L, false); + assertTrue(withoutDigest.getOrigin().getSha256().isEmpty()); + } + + @Test + void mapsDublinCorePublishersAndIdentifiers() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "application/pdf"); + metadata.add(TikaCoreProperties.PUBLISHER, "Apache Software Foundation"); + metadata.add(TikaCoreProperties.IDENTIFIER, "urn:isbn:9780000000000"); + + Document document = DocumentBuilder.build( + metadata, List.of(metadata), "text", "d", "PARSE_SUCCESS", 1L, false); + + assertEquals(List.of("Apache Software Foundation"), + document.getMetadata().getPublishersList()); + assertEquals(List.of("urn:isbn:9780000000000"), + document.getMetadata().getIdentifiersList()); + // consumed into the typed fields, so the keys must not duplicate into the tail + assertTrue(document.getExtraList().stream() + .noneMatch(f -> f.getKey().equals(TikaCoreProperties.PUBLISHER.getName()) + || f.getKey().equals(TikaCoreProperties.IDENTIFIER.getName()))); + } + + @Test + void recordsProducerVersion() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "text/plain"); + + Document document = DocumentBuilder.build( + metadata, List.of(metadata), "text", "d", "PARSE_SUCCESS", 1L, false); + assertTrue(document.getStatus().getTikaVersion().startsWith("Apache Tika")); + + // Provenance is recorded even when the parse produced no metadata at all. + Document failed = DocumentBuilder.build(null, null, null, "d", "FETCH_EXCEPTION", 1L, false); + assertTrue(failed.getStatus().getTikaVersion().startsWith("Apache Tika")); + } + + @Test + void recursesIntoEmbeddedDocuments() { + Metadata primary = new Metadata(); + primary.set(Metadata.CONTENT_TYPE, "application/pdf"); + primary.set(TikaCoreProperties.TITLE, "Container"); + + Metadata embedded = new Metadata(); + embedded.set(Metadata.CONTENT_TYPE, "image/jpeg"); + embedded.set(TikaCoreProperties.RESOURCE_NAME_KEY, "photo.jpg"); + embedded.set(TikaCoreProperties.TIKA_CONTENT, "an embedded photo"); + + Document document = DocumentBuilder.build( + primary, List.of(primary, embedded), "# Container\n", "doc-1", "PARSE_SUCCESS", 5L, + false); + + assertEquals(1, document.getEmbeddedCount()); + Document child = document.getEmbedded(0); + assertEquals("image/jpeg", child.getContentType()); + assertEquals(FormatCategory.FORMAT_CATEGORY_IMAGE, child.getFormatCategory()); + assertEquals("photo.jpg", child.getOrigin().getFilename()); + assertTrue(blockText(child).contains("an embedded photo")); + } + + /** + * {@code primary == null} is the server's signal that the pipes result carried no + * metadata at all (see TikaGrpcServerImpl). On a failure status that surfaces as an + * explicit error; on {@code EMPTY_OUTPUT} -- a success with nothing to map -- it must + * NOT be reported as a failure. + */ + @Test + void reportsFailureWhenNoMetadata() { + Document document = DocumentBuilder.build(null, null, null, "doc-1", "FETCH_EXCEPTION", 1L, false); + + assertEquals("doc-1", document.getId()); + assertEquals(ParseStatus.Status.FAILED, document.getStatus().getStatus()); + assertEquals("FETCH_EXCEPTION", document.getStatus().getPipesStatus()); + assertFalse(document.getStatus().getErrorsList().isEmpty()); + } + + @Test + void emptyOutputWithNoMetadataIsSuccessNotFailure() { + Document document = DocumentBuilder.build(null, null, null, "doc-1", "EMPTY_OUTPUT", 1L, false); + + assertEquals(ParseStatus.Status.SUCCESS, document.getStatus().getStatus()); + assertTrue(document.getStatus().getErrorsList().isEmpty()); + } + + /** + * {@code pipesStatus} is populated from {@code PipesResult.RESULT_STATUS.name()} (see + * org.apache.tika.pipes.api.PipesResult), never from a plain "OK" -- there is no such + * enum constant. A real successful parse reports as e.g. "PARSE_SUCCESS" or + * "EMIT_SUCCESS", and a real timeout reports as "TIMEOUT" (categorized as a process + * crash, not a partial success). The status mapping must recognize the real names. + */ + @Test + void mapsRealPipesResultStatusNames() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "text/plain"); + + assertEquals(ParseStatus.Status.SUCCESS, + DocumentBuilder.build(metadata, null, null, "d", "PARSE_SUCCESS", 1L, false).getStatus().getStatus(), + "PARSE_SUCCESS is a real, clean success status"); + assertEquals(ParseStatus.Status.SUCCESS, + DocumentBuilder.build(metadata, null, null, "d", "EMIT_SUCCESS", 1L, false).getStatus().getStatus()); + assertEquals(ParseStatus.Status.SUCCESS, + DocumentBuilder.build(metadata, null, null, "d", "EMPTY_OUTPUT", 1L, false).getStatus().getStatus()); + + assertEquals(ParseStatus.Status.PARTIAL, + DocumentBuilder.build(metadata, null, null, "d", "PARSE_SUCCESS_WITH_EXCEPTION", 1L, false) + .getStatus().getStatus(), + "succeeded, but with a caveat along the way, is a partial success not a clean one"); + + assertEquals(ParseStatus.Status.FAILED, + DocumentBuilder.build(metadata, null, null, "d", "TIMEOUT", 1L, false).getStatus().getStatus(), + "TIMEOUT is categorized as a process crash, not a partial success"); + assertEquals(ParseStatus.Status.FAILED, + DocumentBuilder.build(metadata, null, null, "d", "OOM", 1L, false).getStatus().getStatus()); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ParseFixtureSupport.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ParseFixtureSupport.java new file mode 100644 index 00000000000..91120d7f41a --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/ParseFixtureSupport.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper; + +import java.util.List; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Block; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.Inline; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.ToMarkdownContentHandler; + +/** + * Shared helpers: parse Tika test fixtures, map them through {@link DocumentBuilder}, + * and read plain text back out of the block tree for content assertions. + */ +public abstract class ParseFixtureSupport extends TikaTest { + + /** + * Result of parsing a fixture with markdown body extraction enabled. + */ + public record ParseFixture(Metadata primary, List<Metadata> allMetadata, String body) { + } + + protected ParseFixture parseBody(String fileName) throws Exception { + java.io.StringWriter writer = new java.io.StringWriter(); + ToMarkdownContentHandler handler = new ToMarkdownContentHandler(writer); + Metadata metadata = new Metadata(); + try (TikaInputStream input = getResourceAsStream("/test-documents/" + fileName)) { + AUTO_DETECT_PARSER.parse(input, handler, metadata, new ParseContext()); + } + return new ParseFixture(metadata, List.of(metadata), writer.toString()); + } + + protected Document map(ParseFixture fixture, String docId, long fetchParseTimeMs) { + // "PARSE_SUCCESS" mirrors a real org.apache.tika.pipes.api.PipesResult.RESULT_STATUS + // name -- there is no "OK" constant on that enum. + return DocumentBuilder.build( + fixture.primary(), fixture.allMetadata(), fixture.body(), docId, "PARSE_SUCCESS", + fetchParseTimeMs, false); + } + + protected Document map(ParseFixture fixture, String docId) { + return map(fixture, docId, 1L); + } + + /** + * Concatenates the plain text carried by a document's block tree, so content + * assertions can read the canonical representation instead of the optional flat + * rendering. Formatting wrappers (emphasis, links, ...) contribute their inner text; + * structural nodes recurse. + */ + protected static String blockText(Document document) { + StringBuilder text = new StringBuilder(); + for (Block block : document.getBlocksList()) { + appendBlockText(block, text); + } + return text.toString(); + } + + private static void appendBlockText(Block block, StringBuilder text) { + switch (block.getBlockCase()) { + case HEADING -> appendInlineText(block.getHeading().getContentList(), text); + case PARAGRAPH -> appendInlineText(block.getParagraph().getContentList(), text); + case BLOCK_QUOTE -> block.getBlockQuote().getBlocksList() + .forEach(b -> appendBlockText(b, text)); + case BULLET_LIST -> block.getBulletList().getItemsList().forEach( + item -> item.getBlocksList().forEach(b -> appendBlockText(b, text))); + case ORDERED_LIST -> block.getOrderedList().getItemsList().forEach( + item -> item.getBlocksList().forEach(b -> appendBlockText(b, text))); + case CODE_BLOCK -> text.append(block.getCodeBlock().getLiteral()); + case TABLE -> { + block.getTable().getHeaderList() + .forEach(cell -> appendInlineText(cell.getContentList(), text)); + block.getTable().getRowsList().forEach(row -> row.getCellsList() + .forEach(cell -> appendInlineText(cell.getContentList(), text))); + } + default -> { + } + } + text.append('\n'); + } + + private static void appendInlineText(List<Inline> inlines, StringBuilder text) { + for (Inline inline : inlines) { + switch (inline.getInlineCase()) { + case TEXT -> text.append(inline.getText()); + case EMPHASIS -> appendInlineText(inline.getEmphasis().getContentList(), text); + case STRONG -> appendInlineText(inline.getStrong().getContentList(), text); + case STRIKETHROUGH -> + appendInlineText(inline.getStrikethrough().getContentList(), text); + case CODE -> text.append(inline.getCode().getLiteral()); + case LINK -> appendInlineText(inline.getLink().getContentList(), text); + default -> { + } + } + } + } + +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilderTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilderTest.java new file mode 100644 index 00000000000..39c9c2ee4e9 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/content/MarkdownBlockTreeBuilderTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.content; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.grpc.v1.Alignment; +import org.apache.tika.grpc.v1.Block; +import org.apache.tika.grpc.v1.Inline; + +class MarkdownBlockTreeBuilderTest { + + @Test + void emptyOrNullMarkdownYieldsNoBlocks() { + assertTrue(MarkdownBlockTreeBuilder.toBlocks(null).isEmpty()); + assertTrue(MarkdownBlockTreeBuilder.toBlocks("").isEmpty()); + } + + @Test + void nestedBulletListInsideAnOrderedListItemIsPreserved() { + String markdown = "1. first\n" + + " - nested a\n" + + " - nested b\n" + + "2. second\n"; + + List<Block> blocks = MarkdownBlockTreeBuilder.toBlocks(markdown); + assertEquals(1, blocks.size()); + Block ordered = blocks.get(0); + assertTrue(ordered.hasOrderedList()); + assertEquals(2, ordered.getOrderedList().getItemsCount()); + + org.apache.tika.grpc.v1.ListItem first = ordered.getOrderedList().getItems(0); + boolean sawNestedBulletList = first.getBlocksList().stream().anyMatch(Block::hasBulletList); + assertTrue(sawNestedBulletList, "the nested bullet list under item 1 must survive the recursive conversion"); + + org.apache.tika.grpc.v1.BulletList nested = first.getBlocksList().stream() + .filter(Block::hasBulletList) + .map(Block::getBulletList) + .findFirst() + .orElseThrow(); + assertEquals(2, nested.getItemsCount()); + } + + @Test + void gfmTableWithAlignmentAndMultipleRows() { + String markdown = "| Left | Center | Right |\n" + + "|:---|:---:|---:|\n" + + "| a | b | c |\n" + + "| d | e | f |\n"; + + List<Block> blocks = MarkdownBlockTreeBuilder.toBlocks(markdown); + assertEquals(1, blocks.size()); + assertTrue(blocks.get(0).hasTable()); + + org.apache.tika.grpc.v1.Table table = blocks.get(0).getTable(); + assertEquals(3, table.getHeaderCount()); + assertEquals(Alignment.ALIGN_LEFT, table.getHeader(0).getAlignment()); + assertEquals(Alignment.ALIGN_CENTER, table.getHeader(1).getAlignment()); + assertEquals(Alignment.ALIGN_RIGHT, table.getHeader(2).getAlignment()); + + assertEquals(2, table.getRowsCount()); + assertEquals(3, table.getRows(0).getCellsCount()); + assertEquals(3, table.getRows(1).getCellsCount()); + } + + @Test + void linkAndImageCarryDestinationAndOptionalTitle() { + String markdown = "[link text](https://example.com/page \"a title\")\n\n" + + "![alt text](https://example.com/img.png)\n"; + + List<Block> blocks = MarkdownBlockTreeBuilder.toBlocks(markdown); + assertEquals(2, blocks.size()); + + Inline linkInline = blocks.get(0).getParagraph().getContent(0); + assertTrue(linkInline.hasLink()); + assertEquals("https://example.com/page", linkInline.getLink().getDestination()); + assertEquals("a title", linkInline.getLink().getTitle()); + + Inline imageInline = blocks.get(1).getParagraph().getContent(0); + assertTrue(imageInline.hasImage()); + assertEquals("https://example.com/img.png", imageInline.getImage().getDestination()); + assertEquals("alt text", imageInline.getImage().getAlt()); + assertEquals("", imageInline.getImage().getTitle(), "no title given in the markdown -> unset, not null-ish garbage"); + } + + @Test + void imageAltTextKeepsCodeSpansAndLineBreaks() { + String markdown = "![see `config.json` for\ndetails](img.png)\n"; + + List<Block> blocks = MarkdownBlockTreeBuilder.toBlocks(markdown); + Inline imageInline = blocks.get(0).getParagraph().getContent(0); + assertTrue(imageInline.hasImage()); + assertEquals("see config.json for details", imageInline.getImage().getAlt(), + "code-span literals and soft breaks inside alt must not be dropped"); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/text/DocumentTextFlattenerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/text/DocumentTextFlattenerTest.java new file mode 100644 index 00000000000..aaa781e484b --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/text/DocumentTextFlattenerTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.text; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.grpc.v1.Block; +import org.apache.tika.grpc.v1.BlockQuote; +import org.apache.tika.grpc.v1.CodeBlock; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.Emphasis; +import org.apache.tika.grpc.v1.Heading; +import org.apache.tika.grpc.v1.Inline; +import org.apache.tika.grpc.v1.Paragraph; +import org.apache.tika.grpc.v1.SourceOrigin; +import org.apache.tika.grpc.v1.Table; +import org.apache.tika.grpc.v1.TableCell; +import org.apache.tika.grpc.v1.TableRow; + +class DocumentTextFlattenerTest { + + private static Inline text(String value) { + return Inline.newBuilder().setText(value).build(); + } + + private static Block paragraph(String value) { + return Block.newBuilder() + .setParagraph(Paragraph.newBuilder().addContent(text(value))).build(); + } + + private static TableCell cell(String value) { + return TableCell.newBuilder().addContent(text(value)).build(); + } + + @Test + void anchorsAreExactByConstruction() { + Document document = Document.newBuilder() + .setId("doc-1") + .setOrigin(SourceOrigin.newBuilder().setSha256("abc123")) + .addBlocks(Block.newBuilder().setHeading(Heading.newBuilder() + .setLevel(1).addContent(text("Quarterly Report")))) + .addBlocks(paragraph("Revenue grew in EMEA.")) + .build(); + + FlattenedText flat = DocumentTextFlattener.flatten(document); + + assertEquals("Quarterly Report\n\nRevenue grew in EMEA.", flat.text()); + assertEquals("doc-1", flat.documentId()); + assertEquals("abc123", flat.sourceSha256()); + assertEquals(DocumentTextFlattener.FLATTEN_VERSION, flat.flattenVersion()); + // every anchor's range must reproduce its block's text exactly + assertEquals(2, flat.ranges().size()); + for (FlattenedText.BlockRange range : flat.ranges()) { + String slice = flat.text().substring(range.start(), range.end()); + assertTrue(!slice.isBlank() && !slice.contains("\n\n"), + "anchor " + range.path() + " must cover exactly one block: " + slice); + } + assertEquals("0", flat.ranges().get(0).path()); + assertEquals("1", flat.ranges().get(1).path()); + } + + @Test + void formattingWrappersContributeInnerTextOnly() { + Document document = Document.newBuilder() + .addBlocks(Block.newBuilder().setParagraph(Paragraph.newBuilder() + .addContent(text("The ")) + .addContent(Inline.newBuilder().setEmphasis( + Emphasis.newBuilder().addContent(text("quick")))) + .addContent(text(" fox")))) + .build(); + + assertEquals("The quick fox", DocumentTextFlattener.flatten(document).text()); + } + + @Test + void nestedBlocksGetDottedPaths() { + Document document = Document.newBuilder() + .addBlocks(paragraph("Intro.")) + .addBlocks(Block.newBuilder().setBlockQuote(BlockQuote.newBuilder() + .addBlocks(paragraph("Quoted line one.")) + .addBlocks(paragraph("Quoted line two.")))) + .build(); + + FlattenedText flat = DocumentTextFlattener.flatten(document); + + assertEquals(List.of("0", "1.0", "1.1"), + flat.ranges().stream().map(FlattenedText.BlockRange::path).toList()); + } + + @Test + void tablesLinearizeRowWiseWithCellAnchors() { + Table table = Table.newBuilder() + .addHeader(cell("Region")).addHeader(cell("Revenue")) + .addRows(TableRow.newBuilder().addCells(cell("EMEA")).addCells(cell("4.2M"))) + .build(); + Document document = Document.newBuilder() + .addBlocks(Block.newBuilder().setTable(table)) + .build(); + + FlattenedText flat = DocumentTextFlattener.flatten(document); + + assertEquals("Region | Revenue\n\nEMEA | 4.2M", flat.text()); + assertEquals(List.of("0.header.cells.0", "0.header.cells.1", + "0.rows.0.cells.0", "0.rows.0.cells.1"), + flat.ranges().stream().map(FlattenedText.BlockRange::path).toList()); + // the revenue figure's anchor points at exactly its cell text + FlattenedText.BlockRange revenue = flat.ranges().get(3); + assertEquals("4.2M", flat.text().substring(revenue.start(), revenue.end())); + } + + @Test + void codeAndHtmlBlocksAreSkippedAndReported() { + Document document = Document.newBuilder() + .addBlocks(paragraph("Before.")) + .addBlocks(Block.newBuilder().setCodeBlock( + CodeBlock.newBuilder().setLiteral("int x = 1;"))) + .addBlocks(paragraph("After.")) + .build(); + + FlattenedText flat = DocumentTextFlattener.flatten(document); + + assertEquals("Before.\n\nAfter.", flat.text()); + assertEquals(List.of("1"), flat.skippedPaths()); + assertEquals(List.of("0", "2"), + flat.ranges().stream().map(FlattenedText.BlockRange::path).toList()); + } + + @Test + void projectionMapsSpansBackToBlockLocalCoordinates() { + Document document = Document.newBuilder() + .addBlocks(paragraph("Alice went to Paris.")) + .addBlocks(paragraph("Bob stayed home.")) + .build(); + FlattenedText flat = DocumentTextFlattener.flatten(document); + + int paris = flat.text().indexOf("Paris"); + List<FlattenedText.Projection> projections = flat.project(paris, paris + 5); + + assertEquals(1, projections.size()); + assertEquals("0", projections.get(0).path()); + assertEquals("Alice went to Paris.".indexOf("Paris"), projections.get(0).localStart()); + assertEquals(projections.get(0).localStart() + 5, projections.get(0).localEnd()); + + // a second-block span projects into the second block's local coordinates + int bob = flat.text().indexOf("Bob"); + List<FlattenedText.Projection> second = flat.project(bob, bob + 3); + assertEquals("1", second.get(0).path()); + assertEquals(0, second.get(0).localStart()); + + assertThrows(IllegalArgumentException.class, () -> flat.project(5, 5)); + assertThrows(IllegalArgumentException.class, + () -> flat.project(0, flat.text().length() + 1)); + } + + @Test + void emptyBlocksLeaveNoAnchorAndNoSeparator() { + Document document = Document.newBuilder() + .addBlocks(paragraph("Only real content.")) + .addBlocks(Block.newBuilder().setParagraph(Paragraph.newBuilder())) + .build(); + + FlattenedText flat = DocumentTextFlattener.flatten(document); + + assertEquals("Only real content.", flat.text()); + assertEquals(1, flat.ranges().size()); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformerTest.java new file mode 100644 index 00000000000..60aad02f587 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/CreativeCommonsDocumentTransformerTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.metadata.XMPRights; + +/** + * Verifies {@link CreativeCommonsDocumentTransformer} against a synthetic XMP rights + * metadata set (this module's test-document corpus has no dedicated CC-licensed fixture), + * and confirms it does not fire on documents with no rights-related metadata at all. + */ +class CreativeCommonsDocumentTransformerTest { + + @Test + void mapsRightsAndLicenseUrlAndTagsTheRest() { + Metadata metadata = new Metadata(); + metadata.set(XMPRights.MARKED, "True"); + metadata.set(XMPRights.WEB_STATEMENT, "https://creativecommons.org/licenses/by/4.0/"); + metadata.set(TikaCoreProperties.RIGHTS, "Creative Commons Attribution 4.0"); + + CreativeCommonsDocumentTransformer transformer = new CreativeCommonsDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set<String> consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertEquals("https://creativecommons.org/licenses/by/4.0/", builder.getMetadata().getLicenseUrl()); + assertEquals("Creative Commons Attribution 4.0", builder.getMetadata().getRights()); + assertTrue(builder.getExtraCount() > 0); + } + + @Test + void doesNotApplyToDocumentsWithNoRightsMetadata() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "text/plain"); + + CreativeCommonsDocumentTransformer transformer = new CreativeCommonsDocumentTransformer(); + assertFalse(transformer.appliesTo(metadata)); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/DocumentTransformersTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/DocumentTransformersTest.java new file mode 100644 index 00000000000..ab0165b68e2 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/DocumentTransformersTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.metadata.XMPRights; + +class DocumentTransformersTest { + + /** + * A plain-text document (no format transformer applies) that also happens to carry + * Creative Commons rights metadata must still get the universal Dublin-Core-ish fields + * (title, authors, ...) mapped by the generic fallback. Before the fix, the + * cross-cutting {@link CreativeCommonsDocumentTransformer} matching on its own set + * {@code matched = true} and suppressed the fallback entirely, silently dropping title/ + * author/description/keywords/languages/dates for any generic document that happens to + * carry rights metadata. + */ + @Test + void crossCuttingTransformerDoesNotSuppressGenericFallback() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "text/plain"); + metadata.set(TikaCoreProperties.TITLE, "A Generic Document"); + metadata.add(TikaCoreProperties.CREATOR, "Jane Doe"); + metadata.set(XMPRights.MARKED, "True"); + + Document.Builder builder = Document.newBuilder(); + DocumentTransformers.defaults().transform(metadata, builder); + + assertEquals("A Generic Document", builder.getMetadata().getTitle(), + "generic fallback must still run alongside a matching cross-cutting transformer"); + assertTrue(builder.getMetadata().getAuthorsList().contains("Jane Doe")); + // the cross-cutting transformer's own mapping must also still apply + assertTrue(builder.getExtraList().stream().anyMatch(f -> f.getKey().equals(XMPRights.MARKED.getName()))); + } + + /** + * When two transformers both apply to the same document (e.g. a PDF that also carries + * Creative Commons rights metadata -- an explicitly supported, documented scenario), each + * transformer must not re-tag a key the other one already mapped to a typed field. Every + * key should appear in the tagged tail at most once. + */ + @Test + void doesNotDuplicateTaggedTailWhenMultipleTransformersApply() { + Metadata metadata = new Metadata(); + metadata.set(Metadata.CONTENT_TYPE, "application/pdf"); + metadata.set(TikaCoreProperties.TITLE, "Report"); + metadata.set(XMPRights.MARKED, "True"); + + Document.Builder builder = Document.newBuilder(); + DocumentTransformers.defaults().transform(metadata, builder); + + long titleTagCount = builder.getExtraList().stream() + .filter(f -> f.getKey().equals(TikaCoreProperties.TITLE.getName())) + .count(); + assertEquals(0, titleTagCount, + "title is a typed field (already on document.metadata.title); it must not also " + + "be duplicated into the tagged tail by a second transformer"); + + long markedTagCount = builder.getExtraList().stream() + .filter(f -> f.getKey().equals(XMPRights.MARKED.getName())) + .count(); + assertEquals(1, markedTagCount, "every tail key should appear at most once"); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformerTest.java new file mode 100644 index 00000000000..06d3932c3cb --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/EpubDocumentTransformerTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Verifies {@link EpubDocumentTransformer} against a real EPUB fixture: typed common + * fields land on {@link org.apache.tika.grpc.v1.DocumentMetadata}, and the many + * EPUB-specific properties (epub:*, dc:identifier, OPF/NAV structural fields, ...) land + * in the tagged tail. + */ +class EpubDocumentTransformerTest extends TikaTest { + + @Test + void transformsRealEpubFixture() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testEPUB.epub")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + EpubDocumentTransformer transformer = new EpubDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set<String> consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertFalse(builder.getMetadata().getTitle().isBlank()); + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformerTest.java new file mode 100644 index 00000000000..26a3486053c --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/HtmlDocumentTransformerTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Verifies {@link HtmlDocumentTransformer} against a real HTML fixture: typed common + * fields land on {@link org.apache.tika.grpc.v1.DocumentMetadata}, and the many + * HTML-specific properties (html:meta:*, Open Graph, Twitter Card, ...) land in the + * tagged tail. + */ +class HtmlDocumentTransformerTest extends TikaTest { + + @Test + void transformsRealHtmlFixture() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testHTML_metadata.html")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + HtmlDocumentTransformer transformer = new HtmlDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set<String> consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertFalse(builder.getMetadata().getTitle().isBlank()); + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformerTest.java new file mode 100644 index 00000000000..02dc464bb98 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/ImageDocumentTransformerTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.image.JpegParser; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Verifies {@link ImageDocumentTransformer} against a real JPEG fixture carrying EXIF, GPS, + * and IPTC metadata. Uses {@link JpegParser} directly rather than the auto-detecting parser: + * on a machine with GDAL command-line tools installed, MIME-based auto-detection can route + * {@code image/jpeg} to the (unrelated) GDAL raster parser instead, which does not extract + * EXIF/TIFF metadata at all. Targeting the parser this transformer actually cares about keeps + * the test deterministic regardless of what else is on the host. + */ +class ImageDocumentTransformerTest extends TikaTest { + + @Test + void mapsTiffDimensionsAndTagsTheRest() throws Exception { + Metadata metadata = new Metadata(); + // Calling JpegParser directly skips detection, so set Content-Type the way + // AutoDetectParser normally would before delegating. + metadata.set(Metadata.CONTENT_TYPE, "image/jpeg"); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testJPEG_EXIF.jpg")) { + new JpegParser().parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + ImageDocumentTransformer transformer = new ImageDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set<String> consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertTrue(builder.getMetadata().getWidth() > 0); + assertTrue(builder.getMetadata().getHeight() > 0); + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformerTest.java new file mode 100644 index 00000000000..62de41b102c --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/OfficeDocumentTransformerTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.grpc.v1.DocumentMetadata; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +class OfficeDocumentTransformerTest extends TikaTest { + + @Test + void mapsTypedFieldsAndTagsTheRest() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testWORD.doc")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + OfficeDocumentTransformer transformer = new OfficeDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata), "should apply to application/msword"); + + Document.Builder builder = Document.newBuilder(); + java.util.Set<String> consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + DocumentMetadata meta = builder.getMetadata(); + assertFalse(meta.getTitle().isBlank(), "title should be populated"); + assertEquals("Sample Word Document", meta.getTitle()); + assertEquals(1, meta.getAuthorsCount()); + assertEquals("Keith Bennett", meta.getAuthors(0)); + assertTrue(meta.hasCreated(), "created should be populated"); + assertTrue(meta.hasModified(), "modified should be populated"); + assertEquals(2, meta.getPageCount()); + assertEquals(122L, meta.getWordCount()); + assertEquals(699L, meta.getCharacterCount()); + + // Everything not mapped to a typed field (revision, extended properties, + // last-author, comment info, hidden-text flag, etc.) lands in the tagged tail. + assertTrue(builder.getExtraCount() > 0, "tagged tail should be populated"); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformerTest.java new file mode 100644 index 00000000000..e88db845ff7 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/PdfDocumentTransformerTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Verifies {@link PdfDocumentTransformer} against the real {@code testPDF.pdf} fixture. + */ +class PdfDocumentTransformerTest extends TikaTest { + + @Test + void mapsTypedFieldsAndTagsTheRest() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testPDF.pdf")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + PdfDocumentTransformer transformer = new PdfDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set<String> consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertEquals("Apache Tika - Apache Tika", builder.getMetadata().getTitle()); + assertEquals(1, builder.getMetadata().getPageCount()); + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformerTest.java new file mode 100644 index 00000000000..8a0506e6503 --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/RtfDocumentTransformerTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +class RtfDocumentTransformerTest extends TikaTest { + + @Test + void mapsCommonFieldsAndTagsTheRest() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testRTF.rtf")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + RtfDocumentTransformer transformer = new RtfDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set<String> consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + // testRTF.rtf carries a title, an author, a creation date and word/page/character + // counts. Assert on what this real fixture actually has, not on every field the + // transformer is capable of mapping. + assertEquals("Test d’indexation Word", builder.getMetadata().getTitle()); + assertEquals(1, builder.getMetadata().getAuthorsCount()); + assertEquals("Bibliotheque", builder.getMetadata().getAuthors(0)); + assertTrue(builder.getMetadata().hasCreated()); + assertEquals(1, builder.getMetadata().getPageCount()); + assertEquals(3, builder.getMetadata().getWordCount()); + assertEquals(21, builder.getMetadata().getCharacterCount()); + + // RTF/OOXML-specific properties (e.g. extended-properties:Company) are not typed + // fields; they must still show up in the tagged tail. + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformerTest.java b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformerTest.java new file mode 100644 index 00000000000..3f3e16e4beb --- /dev/null +++ b/tika-grpc-mapper/src/test/java/org/apache/tika/grpc/mapper/transform/WarcDocumentTransformerTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.grpc.mapper.transform; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.grpc.v1.Document; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Verifies {@link WarcDocumentTransformer} against a real WARC fixture: it applies to + * WARC content types and routes the WARC/HTTP header fields into the tagged tail. + */ +class WarcDocumentTransformerTest extends TikaTest { + + @Test + void appliesToWarcAndTagsTheTail() throws Exception { + Metadata metadata = new Metadata(); + try (TikaInputStream tis = getResourceAsStream("/test-documents/testWARC_multiple.warc")) { + AUTO_DETECT_PARSER.parse(tis, new BodyContentHandler(-1), metadata, new ParseContext()); + } + + WarcDocumentTransformer transformer = new WarcDocumentTransformer(); + assertTrue(transformer.appliesTo(metadata)); + + Document.Builder builder = Document.newBuilder(); + java.util.Set<String> consumed = new java.util.HashSet<>(); + transformer.transform(metadata, builder, consumed); + MetadataTagger.appendTail(metadata, consumed, builder); + + assertTrue(builder.getExtraCount() > 0); + } +} diff --git a/tika-grpc-mapper/src/test/resources/test-documents/sample.txt b/tika-grpc-mapper/src/test/resources/test-documents/sample.txt new file mode 100644 index 00000000000..1461a4ea78a --- /dev/null +++ b/tika-grpc-mapper/src/test/resources/test-documents/sample.txt @@ -0,0 +1 @@ +Sample plain text document for generic metadata mapping tests. diff --git a/tika-grpc/README.md b/tika-grpc/README.md index 2ebea44c002..2268b995898 100644 --- a/tika-grpc/README.md +++ b/tika-grpc/README.md @@ -1,8 +1,8 @@ -# Tika Pipes GRPC Server +# Tika Pipes gRPC Server -The following is the Tika Pipes GRPC Server. - -This server will manage a pool of Tika Pipes clients. +The Tika Pipes gRPC server exposes fetcher and iterator management and document +fetch-and-parse over gRPC. It runs a pool of Tika Pipes worker processes and routes +requests through the configured fetchers. * Tika Pipes Fetcher CRUD operations * Create @@ -19,6 +19,59 @@ This server will manage a pool of Tika Pipes clients. > the config. See the > [Tika gRPC security configuration docs](../docs/modules/ROOT/pages/using-tika/grpc/index.adoc). +## Typed parse output + +Parse results are returned as `org.apache.tika.grpc.v1.Document` on +`FetchAndParseReply.document`. The previous `FetchAndParseReply.fields` +(`map<string,string>`) has been removed. + +Rather than one proto message per source format, `Document` models content and +metadata by concern: + +1. **Content tree**: `markdown` (the authoritative render) and `blocks` — the same + content parsed into a structured tree of headings/paragraphs/lists/tables/code + blocks/inline runs. This is format-agnostic: every Tika parser's output reaches + this shape via markdown, so content handling does not vary per format. +2. **Typed metadata**: a small, bounded set of common fields on `DocumentMetadata` + (title, authors, description, keywords, languages, dates, counts, dimensions, + rights) — the cross-format facts every consumer wants, typed. +3. **Tagged tail**: `extra` (`repeated MetadataField`) carries everything + format-specific (PDF permissions, EXIF/GPS, OOXML core properties, …), typed where + Tika's own `Property` declares a type and a string otherwise — never guessed. This + is the lossless catch-all; nothing is dropped. +4. **`format_category`**: a cheap routing hint, not mutually exclusive with `extra` — + a PDF can still carry Creative Commons rights metadata, for example. +5. **`embedded`**: recursive — a PDF with an embedded image is one `Document` whose + `embedded` list holds a fully-typed child `Document` for the image. + +Supporting artifacts live in sibling Maven modules (also listed in `tika-bom`): + +| Module | Role | +|--------|------| +| `tika-grpc-api` | Protobuf definitions (`org.apache.tika.grpc.v1`: `document.proto`), generated Java stubs, bundled `FileDescriptorSet` under `META-INF/` | +| `tika-grpc-mapper` | Maps Tika `Metadata` to `Document` via per-format `DocumentTransformer`s (code, not schema) plus a format-agnostic markdown-to-block-tree builder | +| `tika-grpc` | gRPC service implementation (this module) | + +Client migration (summary): + +| Before | After | +|--------|-------| +| `FetchAndParseReply.fields["content"]` | `document.markdown` | +| `FetchAndParseReply.fields["Some-Key"]` | `document.extra` (find by key; typed by Tika's declared `Property` type, string otherwise) | +| `FetchAndParseReply.fields["Content-Type"]` | `document.content_type` | +| Flat string keys for PDF/Office/etc. | `document.metadata` (common fields) plus `document.extra` (format-specific) | +| Ad hoc title/author strings | `document.metadata.title` / `document.metadata.authors` | + +See [tika-grpc-api/README.md](../tika-grpc-api/README.md) for the full shape and +[tika-grpc-mapper/docs/EXTENSIONS.md](../tika-grpc-mapper/docs/EXTENSIONS.md) for how +to add a new format. Transformer tests live under `tika-grpc-mapper/src/test/java`. + +Build the API and mapper with the rest of the reactor: + +```bash +./mvnw -pl tika-grpc-api,tika-grpc-mapper,tika-grpc test +``` + ## Distribution and Maven Artifact **tika-grpc is designed to be run via Docker — it is not a standalone runnable artifact published to Maven Central.** diff --git a/tika-grpc/pom.xml b/tika-grpc/pom.xml index 7d021a11e65..f791489bfdc 100644 --- a/tika-grpc/pom.xml +++ b/tika-grpc/pom.xml @@ -135,6 +135,16 @@ <artifactId>j2objc-annotations</artifactId> <version>${j2objc-annotations.version}</version> <!-- prevent downgrade of version in guava --> </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-grpc-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-grpc-mapper</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>org.apache.tika</groupId> <artifactId>tika-async-cli</artifactId> @@ -380,6 +390,9 @@ <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact> <pluginId>grpc-java</pluginId> <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact> + <additionalProtoPathElements> + <additionalProtoPathElement>${project.basedir}/../tika-grpc-api/src/main/proto</additionalProtoPathElement> + </additionalProtoPathElements> </configuration> <executions> <execution> diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java index 7213217edb9..a700b4ca9d5 100644 --- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java +++ b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java @@ -19,6 +19,8 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -59,7 +61,10 @@ import org.apache.tika.config.loader.TikaJsonConfig; import org.apache.tika.exception.TikaConfigException; import org.apache.tika.exception.TikaException; +import org.apache.tika.grpc.mapper.DocumentBuilder; +import org.apache.tika.grpc.v1.Document; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.ParseContext; import org.apache.tika.pipes.api.FetchEmitTuple; import org.apache.tika.pipes.api.PipesResult; @@ -106,6 +111,12 @@ class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase { } Path configPath = tikaConfigFile.toPath(); + // ParseBytes needs a fetcher the FORKED worker can resolve. Runtime store writes do + // not reach the worker with the default in-memory config store, so the server hands + // every component (its own managers and the forked worker alike) an augmented copy + // of the config with an internal file-system fetcher rooted at a server temp dir. + this.parseBytesDir = java.nio.file.Files.createTempDirectory("tika-grpc-parse-bytes"); + configPath = augmentWithParseBytesFetcher(configPath, parseBytesDir); this.tikaConfigPath = configPath; TikaJsonConfig tikaJsonConfig = TikaJsonConfig.load(configPath); @@ -281,6 +292,93 @@ public void fetchAndParse(FetchAndParseRequest request, responseObserver.onCompleted(); } + private static final String PARSE_BYTES_FETCHER_ID = "__tika_grpc_parse_bytes"; + private final java.nio.file.Path parseBytesDir; + + // Writes a sibling temp copy of the config with the internal parse-bytes fetcher added + // to the `fetchers` section, so it is loaded exactly like a user-defined fetcher. + private static Path augmentWithParseBytesFetcher(Path configPath, java.nio.file.Path dir) + throws IOException { + com.fasterxml.jackson.databind.node.ObjectNode root = + (com.fasterxml.jackson.databind.node.ObjectNode) + OBJECT_MAPPER.readTree(configPath.toFile()); + com.fasterxml.jackson.databind.node.ObjectNode fetchers = + root.has("fetchers") && root.get("fetchers").isObject() + ? (com.fasterxml.jackson.databind.node.ObjectNode) root.get("fetchers") + : root.putObject("fetchers"); + fetchers.putObject(PARSE_BYTES_FETCHER_ID) + .putObject("file-system-fetcher") + .put("basePath", dir.toAbsolutePath().toString()); + Path augmented = java.nio.file.Files.createTempFile("tika-grpc-config", ".json"); + java.nio.file.Files.writeString(augmented, + OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(root)); + return augmented; + } + + @Override + public void parseBytes(org.apache.tika.ParseBytesRequest request, + StreamObserver<FetchAndParseReply> responseObserver) { + if (request.getRequestId().isBlank()) { + responseObserver.onError(io.grpc.Status.INVALID_ARGUMENT + .withDescription("request_id is required").asRuntimeException()); + return; + } + if (request.getContent().isEmpty()) { + responseObserver.onError(io.grpc.Status.INVALID_ARGUMENT + .withDescription("content is required").asRuntimeException()); + return; + } + FetchAndParseRequest internal = FetchAndParseRequest.newBuilder() + .setFetcherId(PARSE_BYTES_FETCHER_ID) + .setFetchKey(request.getRequestId()) + .setParseContextJson(request.getParseContextJson()) + .setRenderMarkdown(request.getRenderMarkdown()) + .build(); + if (denyPerRequestConfig(internal, responseObserver)) { + return; + } + java.nio.file.Path file = null; + try { + // The forked worker fetches by key relative to the internal fetcher's base path; + // the caller-visible id stays request_id while the on-disk name stays unique. + String tempName = java.util.UUID.randomUUID().toString(); + file = parseBytesDir.resolve(tempName); + java.nio.file.Files.write(file, request.getContent().toByteArray()); + FetchAndParseRequest keyed = internal.toBuilder().setFetchKey(tempName).build(); + fetchAndParseImpl(keyed, new StreamObserver<>() { + @Override + public void onNext(FetchAndParseReply reply) { + Document document = reply.getDocument().toBuilder() + .setId(request.getRequestId()).build(); + responseObserver.onNext(reply.toBuilder() + .setFetchKey(request.getRequestId()) + .setDocument(document).build()); + } + + @Override + public void onError(Throwable t) { + responseObserver.onError(t); + } + + @Override + public void onCompleted() { + } + }); + responseObserver.onCompleted(); + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + if (file != null) { + try { + java.nio.file.Files.deleteIfExists(file); + } catch (IOException ignore) { + // best effort; the directory is a server temp dir + } + } + } + } + + private void fetchAndParseImpl(FetchAndParseRequest request, StreamObserver<FetchAndParseReply> responseObserver) { Fetcher fetcher; @@ -291,6 +389,9 @@ private void fetchAndParseImpl(FetchAndParseRequest request, } Metadata tikaMetadata = new Metadata(); + // Times the whole pipesClient.process() round trip: fetch and parse both happen + // inside the forked pipes worker, so this is fetch+parse latency, not parse-only. + long fetchParseStart = System.nanoTime(); try { ParseContext parseContext = new ParseContext(); String additionalFetchConfigJson = request.getAdditionalFetchConfigJson(); @@ -307,21 +408,28 @@ private void fetchAndParseImpl(FetchAndParseRequest request, new EmitKey(), tikaMetadata, parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP)); FetchAndParseReply.Builder fetchReplyBuilder = FetchAndParseReply.newBuilder() - .setFetchKey(request.getFetchKey()) - .setStatus(pipesResult.status().name()); + .setFetchKey(request.getFetchKey()); if (pipesResult.status().equals(PipesResult.RESULT_STATUS.FETCH_EXCEPTION)) { fetchReplyBuilder.setErrorMessage(pipesResult.message()); } + long fetchParseTimeMs = (System.nanoTime() - fetchParseStart) / 1_000_000L; + List<Metadata> metadataList = new ArrayList<>(); if (pipesResult.emitData() != null && pipesResult.emitData().getMetadataList() != null) { - for (Metadata metadata : pipesResult.emitData().getMetadataList()) { - for (String name : metadata.names()) { - String value = metadata.get(name); - if (value != null) { - fetchReplyBuilder.putFields(name, value); - } - } - } + metadataList.addAll(pipesResult.emitData().getMetadataList()); } + // null (not an empty Metadata) when the pipes result carried no metadata, so + // DocumentBuilder can distinguish "nothing came back" from an empty parse. + Metadata primary = metadataList.isEmpty() ? null : metadataList.get(0); + String body = primary == null ? null : primary.get(TikaCoreProperties.TIKA_CONTENT); + Document document = DocumentBuilder.build( + primary, + metadataList, + body, + request.getFetchKey(), + pipesResult.status().name(), + fetchParseTimeMs, + request.getRenderMarkdown()); + fetchReplyBuilder.setDocument(document); responseObserver.onNext(fetchReplyBuilder.build()); } catch (IOException e) { throw new RuntimeException(e); diff --git a/tika-grpc/src/main/proto/tika.proto b/tika-grpc/src/main/proto/tika.proto index 7d365f95d14..c81a2dd6c8b 100644 --- a/tika-grpc/src/main/proto/tika.proto +++ b/tika-grpc/src/main/proto/tika.proto @@ -14,6 +14,8 @@ syntax = "proto3"; package tika; +import "org/apache/tika/grpc/v1/document.proto"; + option java_multiple_files = true; option java_package = "org.apache.tika"; option java_outer_classname = "TikaProto"; @@ -42,6 +44,10 @@ service Tika { the FetchParseTuple data output from Tika Pipes. This is a synchronous call that immediately returns 1 result. */ rpc FetchAndParse(FetchAndParseRequest) returns (FetchAndParseReply) {} + + // Parses bytes the caller already has: no fetcher registration, no server-side + // state. The parse still runs in the isolated forked worker. + rpc ParseBytes(ParseBytesRequest) returns (FetchAndParseReply) {} /* Using a Fetcher in the fetcher store, send a FetchAndParse request. This will fetch, parse, and return the FetchParseTuple data output from Tika Pipes. This will stream the data from the server in response. @@ -68,8 +74,8 @@ service Tika { Get a pipes iterator's data from the iterator store. */ rpc GetPipesIterator(GetPipesIteratorRequest) returns (GetPipesIteratorReply) {} - /* - Delete a pipes iterator from the iterator store. + /* + Delete a pipes iterator from the iterator store. */ rpc DeletePipesIterator(DeletePipesIteratorRequest) returns (DeletePipesIteratorReply) {} } @@ -106,17 +112,35 @@ message FetchAndParseRequest { // Example: {"basic-content-handler-factory": {"type": "HTML"}, "timeout-limits": {"progressTimeoutMillis": 30000}} // See the parse-context.idx component registry for available component names. string parse_context_json = 5; + // Also populate Document.markdown with a flat rendering of the content. The structured + // Document.blocks tree is always returned; the flat string is off by default so a reply + // does not carry the content twice. + bool render_markdown = 6; +} + +message ParseBytesRequest { + // Caller-chosen identifier echoed back as the reply's fetch_key and used as the + // Document id. Required; must be unique per in-flight request. + string request_id = 1; + // The raw document bytes to parse. + bytes content = 2; + // Optional JSON object to configure the ParseContext for this request; same + // semantics as FetchAndParseRequest.parse_context_json. + string parse_context_json = 3; + // Also populate Document.markdown; same semantics as + // FetchAndParseRequest.render_markdown. + bool render_markdown = 4; } message FetchAndParseReply { // Echoes the fetch_key that was sent in the request. string fetch_key = 1; - // Metadata fields from the parse output. - map<string, string> fields = 2; - // The status from the message. See javadoc for org.apache.tika.pipes.PipesResult.STATUS for the list of status. - string status = 3; + reserved 2, 3; + reserved "fields", "status"; // If there was an error, this will contain the error message. string error_message = 4; + // Strongly-typed parse output from Tika. + org.apache.tika.grpc.v1.Document document = 5; } message DeleteFetcherRequest { diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java index 508c672e733..d87523b508e 100644 --- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java +++ b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/PipesBiDirectionalStreamingIntegrationTest.java @@ -179,12 +179,12 @@ static void cleanConfig() throws Exception { void testHttpFetchScenario() throws Exception { AtomicInteger numParsed = new AtomicInteger(); AtomicInteger numErrors = new AtomicInteger(); - Map<String, Map<String, String>> result = Collections.synchronizedMap(new HashMap<>()); + Map<String, String> result = Collections.synchronizedMap(new HashMap<>()); List<String> errorMessages = Collections.synchronizedList(new ArrayList<>()); StreamObserver<FetchAndParseReply> responseObserver = new StreamObserver<>() { @Override public void onNext(FetchAndParseReply fetchAndParseReply) { - String status = fetchAndParseReply.getStatus(); + String status = fetchAndParseReply.getDocument().getStatus().getPipesStatus(); LOGGER.info("Parsed: {} with status: {}", fetchAndParseReply.getFetchKey(), status); // Check if this is an error status @@ -198,7 +198,10 @@ public void onNext(FetchAndParseReply fetchAndParseReply) { LOGGER.error(errorMsg); } else { numParsed.incrementAndGet(); - result.put(fetchAndParseReply.getFetchKey(), fetchAndParseReply.getFieldsMap()); + if (fetchAndParseReply.hasDocument()) { + result.put(fetchAndParseReply.getFetchKey(), + fetchAndParseReply.getDocument().getMarkdown()); + } } } @@ -220,6 +223,7 @@ public void onCompleted() { .newBuilder() .setFetcherId(httpFetcherId) .setFetchKey(httpServerUrl + "/" + file) + .setRenderMarkdown(true) .build()); } request.onCompleted(); diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java index 2380735a65d..2a068fbcebf 100644 --- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java +++ b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java @@ -228,6 +228,27 @@ private static String createFetcherId(int i) { return "nick" + i + ":is:cool:super/" + FileSystemFetcher.class; } + @Test + public void testParseBytesNeedsNoFetcherRegistration(Resources resources) throws Exception { + // Component management is denied by default, and parseBytes must still work: the + // caller sends bytes, no fetcher CRUD, no server-side state visible to the client. + TikaGrpc.TikaBlockingStub blockingStub = startServer(resources, tikaConfig); + + FetchAndParseReply reply = blockingStub.parseBytes(org.apache.tika.ParseBytesRequest + .newBuilder() + .setRequestId("bytes-1") + .setContent(com.google.protobuf.ByteString.copyFromUtf8("Hello from raw bytes.")) + .setRenderMarkdown(true) + .build()); + + assertEquals("bytes-1", reply.getFetchKey()); + assertEquals("bytes-1", reply.getDocument().getId()); + assertEquals("PARSE_SUCCESS", reply.getDocument().getStatus().getPipesStatus(), + "error_message=" + reply.getErrorMessage() + " errors=" + reply.getDocument().getStatus().getErrorsList()); + Assertions.assertTrue(reply.getDocument().getMarkdown().contains("Hello from raw bytes")); + Assertions.assertFalse(reply.getDocument().getBlocksList().isEmpty()); + } + @Test public void testComponentManagementDeniedByDefault(Resources resources) throws Exception { TikaGrpc.TikaBlockingStub blockingStub = startServer(resources, tikaConfig); @@ -420,8 +441,10 @@ public void testBiStream(Resources resources) throws Exception { StreamObserver<FetchAndParseReply> replyStreamObserver = new StreamObserver<>() { @Override public void onNext(FetchAndParseReply fetchAndParseReply) { - LOG.debug("Fetched {} with metadata {}", fetchAndParseReply.getFetchKey(), fetchAndParseReply.getFieldsMap()); - if (PipesResult.RESULT_STATUS.FETCH_EXCEPTION.name().equals(fetchAndParseReply.getStatus())) { + LOG.debug("Fetched {} with document present={}", + fetchAndParseReply.getFetchKey(), fetchAndParseReply.hasDocument()); + if (PipesResult.RESULT_STATUS.FETCH_EXCEPTION.name().equals( + fetchAndParseReply.getDocument().getStatus().getPipesStatus())) { errors.add(fetchAndParseReply); } else { successes.add(fetchAndParseReply); @@ -474,10 +497,12 @@ public void onCompleted() { // Log what we got for debugging LOG.info("Successes: {}, Errors: {}", successes.size(), errors.size()); for (FetchAndParseReply success : successes) { - LOG.info("Success: {} - status: {}", success.getFetchKey(), success.getStatus()); + LOG.info("Success: {} - status: {}", success.getFetchKey(), + success.getDocument().getStatus().getPipesStatus()); } for (FetchAndParseReply error : errors) { - LOG.info("Error: {} - status: {}", error.getFetchKey(), error.getStatus()); + LOG.info("Error: {} - status: {}", error.getFetchKey(), + error.getDocument().getStatus().getPipesStatus()); } assertEquals(NUM_TEST_DOCS, successes.size());