✨ Add tika fallback for text extraction

Author: simonwoerpel

.gitignore

@@ -3,6 +3,8 @@ data/archive
 data/model_type_prediction.ftz
 debug.sqlite3
 data/servicelayer-archive
+# tika
+contrib/tika-server.jar
 # documentation
 site
 # Byte-compiled / optimized / DLL files

ingestors/ingestor.py

@@ -1,7 +1,11 @@
 import logging
+import typing
 
 from rigour.mime import normalize_extension, normalize_mimetype
 
+if typing.TYPE_CHECKING:
+    from ingestors.manager import Manager
+
 log = logging.getLogger(__name__)
 
 
@@ -12,7 +16,7 @@ class Ingestor(object):
     EXTENSIONS = []
     SCORE = 3
 
-    def __init__(self, manager):
+    def __init__(self, manager: "Manager"):
         self.manager = manager
 
     def ingest(self, file_path, entity):

ingestors/manager.py

@@ -26,6 +26,8 @@
 from ingestors import __version__
 from ingestors.directory import DirectoryIngestor
 from ingestors.exc import ENCRYPTED_MSG, ProcessingException
+from ingestors.ingestor import Ingestor
+from ingestors.misc.tika import TikaIngestor
 from ingestors.settings import Settings
 from ingestors.util import filter_text, remove_directory
 
@@ -146,7 +148,7 @@ def emit_text_fragment(self, entity, texts, fragment):
             doc.add("indexText", texts)
             self.emit_entity(doc, fragment=safe_fragment(fragment))
 
-    def auction(self, file_path, entity):
+    def auction(self, file_path, entity) -> type[Ingestor]:
         if not entity.has("mimeType"):
             if file_path.is_dir():
                 entity.add("mimeType", DirectoryIngestor.MIME_TYPE)
@@ -163,6 +165,10 @@ def auction(self, file_path, entity):
                 best_score = score
                 best_cls = cls
 
+        settings = Settings()
+        if settings.tika_fallback:
+            best_cls = TikaIngestor
+
         if best_cls is None:
             raise ProcessingException("Format not supported")
         return best_cls

ingestors/misc/tika.py

@@ -0,0 +1,32 @@
+from typing import Any
+
+from followthemoney import EntityProxy, model
+from rigour.mime import normalize_mimetype
+
+from ingestors.exc import ProcessingException
+from ingestors.ingestor import Ingestor
+from ingestors.support.tika import TikaSupport
+
+
+def extract_mimetype(result: dict[str, Any]) -> str | None:
+    mime = result.get("Content-Type")
+    if mime and ";" in mime:
+        return normalize_mimetype(mime.split(";")[0])
+    return normalize_mimetype(mime)
+
+
+class TikaIngestor(Ingestor, TikaSupport):
+    SCORE = 1
+
+    def ingest(self, file_path: str, entity: EntityProxy):
+        with open(file_path, "rb") as fh:
+            try:
+                result = self.extract_tika(fh, cache_key=entity.first("contentHash"))
+                if result:
+                    patch = model.make_entity(entity.schema)
+                    patch.id = entity.id
+                    patch.add("mimeType", extract_mimetype(result))
+                    patch.add("bodyText", result["content"])
+                    self.manager.emit_entity(patch, fragment="tika")
+            except Exception as exc:
+                raise ProcessingException("Cannot extract tika text: %s" % exc) from exc

ingestors/settings.py

@@ -23,6 +23,9 @@ class Settings(OpenAlephSettings):
     convert_timeout: int = 300
     """Headless libreoffice document convert timeout in seconds"""
 
+    tika_fallback: bool = False
+    """Use Apache Tika as a text extraction fallback"""
+
 
 _settings = OpenAlephSettings()
 

ingestors/support/tika.py

@@ -0,0 +1,32 @@
+import logging
+from io import BytesIO
+from typing import Any
+
+from normality import collapse_spaces
+from tika import parser
+
+from ingestors.support.cache import CacheSupport
+
+log = logging.getLogger(__name__)
+
+
+class TikaSupport(CacheSupport):
+    def extract_tika(
+        self, fh: BytesIO, cache_key: str | None = None
+    ) -> dict[str, Any] | None:
+        _cache_key = None
+        if cache_key:
+            _cache_key = self.cache_key("tika", cache_key)
+            result = self.tags.get(_cache_key)
+            if result is not None:
+                log.info("Tika: cached result for checksum %s" % cache_key)
+                return result
+
+        result = parser.from_file(fh)
+        if isinstance(result, dict):
+            text = result.get("content")
+            if text:
+                result["content"] = collapse_spaces(text)
+            if _cache_key:
+                self.tags.set(_cache_key, result)
+            return result

poetry.lock

@@ -41,6 +41,7 @@ dependencies = [
     "boto3 (>=1.11.9,<2.0.0)",
     "openaleph-procrastinate (>=5.0.0,<6.0.0)",
     "openaleph-servicelayer (>=1.23.3, <1.24.0)",
+    "tika (>=3.1.0,<4.0.0)",
 ]
 
 [project.scripts]
@@ -87,6 +88,7 @@ svg = "ingestors.media.svg:SVGIngestor"
 audio = "ingestors.media.audio:AudioIngestor"
 video = "ingestors.media.video:VideoIngestor"
 json = "ingestors.misc.jsonfile:JSONIngestor"
+tika = "ingestors.misc.tika:TikaIngestor"
 
 [project.optional-dependencies]
 ocr = ["tesserocr (==2.6.2)"]
@@ -116,3 +118,5 @@ OPENALEPH_ANALYZE_DEFER = 0
 OPENALEPH_INDEX_DEFER = 0
 PROCRASTINATE_DB_URI = "memory://"
 FTM_FRAGMENTS_URI = "sqlite:///ingest_test.db"
+TIKA_PATH = "./contrib"
+INGESTORS_TIKA_FALLBACK = 1

pyproject.toml

@@ -4,7 +4,7 @@ bump2version==1.0.1 ; python_version >= "3.11" and python_version < "3.14"
 cfgv==3.4.0 ; python_version >= "3.11" and python_version < "3.14"
 click==8.2.1 ; python_version >= "3.11" and python_version < "3.14"
 colorama==0.4.6 ; python_version >= "3.11" and python_version < "3.14" and (platform_system == "Windows" or sys_platform == "win32")
-coverage==7.10.5 ; python_version >= "3.11" and python_version < "3.14"
+coverage==7.10.6 ; python_version >= "3.11" and python_version < "3.14"
 distlib==0.4.0 ; python_version >= "3.11" and python_version < "3.14"
 fakeredis==2.30.3 ; python_version >= "3.11" and python_version < "3.14"
 filelock==3.19.1 ; python_version >= "3.11" and python_version < "3.14"

requirements-dev.txt

@@ -75,7 +75,7 @@ openaleph-servicelayer==1.23.3 ; python_version >= "3.11" and python_version < "
 openpyxl==3.1.5 ; python_version >= "3.11" and python_version < "3.14"
 orjson==3.11.3 ; python_version >= "3.11" and python_version < "3.14"
 pathspec==0.12.1 ; python_version >= "3.11" and python_version < "3.14"
-phonenumbers==9.0.12 ; python_version >= "3.11" and python_version < "3.14"
+phonenumbers==9.0.13 ; python_version >= "3.11" and python_version < "3.14"
 pillow==11.3.0 ; python_version >= "3.11" and python_version < "3.14"
 platformdirs==4.4.0 ; python_version >= "3.11" and python_version < "3.14"
 prefixdate==0.5.0 ; python_version >= "3.11" and python_version < "3.14"
@@ -111,7 +111,7 @@ rapidfuzz==3.14.0 ; python_version >= "3.11" and python_version < "3.14"
 rarfile==4.2 ; python_version >= "3.11" and python_version < "3.14"
 rdflib==7.1.4 ; python_version >= "3.11" and python_version < "3.14"
 redis==6.4.0 ; python_version >= "3.11" and python_version < "3.14"
-regex==2025.7.34 ; python_version >= "3.11" and python_version < "3.14"
+regex==2025.8.29 ; python_version >= "3.11" and python_version < "3.14"
 requests==2.32.5 ; python_version >= "3.11" and python_version < "3.14"
 rich==14.1.0 ; python_version >= "3.11" and python_version < "3.14"
 rigour==1.3.0 ; python_version >= "3.11" and python_version < "3.14"
@@ -120,6 +120,7 @@ s3transfer==0.13.1 ; python_version >= "3.11" and python_version < "3.14"
 scikit-learn==1.7.1 ; python_version >= "3.11" and python_version < "3.14"
 scipy==1.16.1 ; python_version >= "3.11" and python_version < "3.14"
 sentry-sdk==2.0.1 ; python_version >= "3.11" and python_version < "3.14"
+setuptools==80.9.0 ; python_version >= "3.11" and python_version < "3.14"
 shellingham==1.5.4 ; python_version >= "3.11" and python_version < "3.14"
 shortuuid==1.0.13 ; python_version >= "3.11" and python_version < "3.14"
 six==1.17.0 ; python_version >= "3.11" and python_version < "3.14"
@@ -130,6 +131,7 @@ structlog==24.4.0 ; python_version >= "3.11" and python_version < "3.14"
 texttable==1.7.0 ; python_version >= "3.11" and python_version < "3.14"
 textual==4.0.0 ; python_version >= "3.11" and python_version < "3.14"
 threadpoolctl==3.6.0 ; python_version >= "3.11" and python_version < "3.14"
+tika==3.1.0 ; python_version >= "3.11" and python_version < "3.14"
 tqdm==4.67.1 ; python_version >= "3.11" and python_version < "3.14"
 typer==0.17.3 ; python_version >= "3.11" and python_version < "3.14"
 typing-extensions==4.15.0 ; python_version >= "3.11" and python_version < "3.14"