From 59601fde3111d983b1d3058c22a3d257783cba90 Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Fri, 6 Nov 2026 17:24:33 +0100 Subject: [PATCH] Name loose S3 objects relative to their common prefix Remote objects were named by basename only, so pdfs/a/x.pdf and pdfs/b/x.pdf listed under pdfs/ both mapped to x.grobid.tei.xml: within a chunk the second overwrote the first, across chunks it was skipped as already done. Objects are now named by their key relative to the common prefix of the listing, mirroring what local directories and archives do, and the citation-list download route keeps that relative path on disk. --- grobid_client/grobid_client.py | 32 +++++++++- tests/test_s3_output_names.py | 104 +++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 tests/test_s3_output_names.py diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index 34de3bb..33ce6de 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -824,6 +824,30 @@ def _s3_basename(self, uri: str) -> str: """Return the last path component of an s3:// key.""" return self._split_s3(uri)[1].rsplit("/", 1)[-1] + def _s3_common_prefix(self, uris: list) -> str: + """Return the key prefix (up to a '/') shared by all the given objects. + + Plays the role _common_base plays for local files: loose remote + objects are named relative to it, so pdfs/a/x.pdf and pdfs/b/x.pdf + listed under pdfs/ become a/x.pdf and b/x.pdf and do not collide on + the same output file. + """ + keys = [self._split_s3(uri)[1] for uri in uris] + if not keys: + return "" + if len(keys) == 1: + prefix = keys[0] + else: + prefix = os.path.commonprefix(keys) + return prefix[:prefix.rfind("/") + 1] if "/" in prefix else "" + + def _s3_relative_name(self, uri: str, prefix: str) -> str: + """Name a remote object by its key relative to prefix (see above).""" + key = self._split_s3(uri)[1] + name = key[len(prefix):] if prefix and key.startswith(prefix) else key + safe = self._safe_member_path("", name) + return safe if safe else self._s3_basename(uri) + def _import_smart_open(self) -> Any: try: import smart_open # noqa: F401 @@ -1373,6 +1397,9 @@ def _process_remote_files( queue_size = self._effective_queue_size(n) print(f"Found {total} remote file(s) to process") + # Objects are named relative to their common prefix so that the + # output mirrors the listed "directory", as it does for local ones + prefix = self._s3_common_prefix(uris) processed_count = 0 error_count = 0 skipped_count = 0 @@ -1397,7 +1424,7 @@ def _process_remote_files( self.logger.error(f"Failed to fetch {uri}: {str(e)}") error_count += 1 continue - document.name = self._s3_basename(uri) + document.name = self._s3_relative_name(uri, prefix) documents.append(document) if not documents: @@ -1434,8 +1461,9 @@ def _process_remote_files( for uri in chunk: if verbose: self.logger.info(f"Fetching {uri}") - dest = os.path.join(temp_dir, self._s3_basename(uri)) + dest = os.path.join(temp_dir, self._s3_relative_name(uri, prefix)) try: + os.makedirs(os.path.dirname(dest), exist_ok=True) with self._s3_open(uri) as src, open(dest, "wb") as out_file: shutil.copyfileobj(src, out_file) local_files.append(dest) diff --git a/tests/test_s3_output_names.py b/tests/test_s3_output_names.py new file mode 100644 index 0000000..970d135 --- /dev/null +++ b/tests/test_s3_output_names.py @@ -0,0 +1,104 @@ +""" +Loose S3 objects are named relative to their common prefix, so two keys +that share a basename (a/x.pdf, b/x.pdf) get two output files. +""" +import os + +import pytest +from unittest.mock import Mock, patch + +boto3 = pytest.importorskip("boto3") +pytest.importorskip("smart_open") +pytest.importorskip("moto") +try: + from moto import mock_aws +except ImportError: # moto < 5 + from moto import mock_s3 as mock_aws + +from grobid_client.grobid_client import GrobidClient + +BUCKET = "test-bucket" +REGION = "us-east-1" + + +@pytest.fixture(autouse=True) +def _aws_env(monkeypatch): + for var in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SECURITY_TOKEN", "AWS_SESSION_TOKEN"): + monkeypatch.setenv(var, "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", REGION) + + +def _client(): + with patch.object(GrobidClient, "_test_server_connection"): + with patch.object(GrobidClient, "_configure_logging"): + c = GrobidClient(check_server=False) + c.logger = Mock() + c.config["queue_size"] = 1 # one object per chunk: collisions across chunks too + return c + + +def _tei(out): + found = [] + for root, _, files in os.walk(out): + for f in files: + if f.endswith(".grobid.tei.xml"): + found.append(os.path.relpath(os.path.join(root, f), out).replace(os.sep, "/")) + return sorted(found) + + +def test_common_prefix(): + c = _client() + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/a/x.pdf", f"s3://{BUCKET}/pdfs/b/x.pdf"]) == "pdfs/" + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/x.pdf", f"s3://{BUCKET}/pdfs/y.pdf"]) == "pdfs/" + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/x.pdf"]) == "pdfs/" + assert c._s3_common_prefix([f"s3://{BUCKET}/x.pdf", f"s3://{BUCKET}/y.pdf"]) == "" + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/xa.pdf", f"s3://{BUCKET}/pdfs/xb.pdf"]) == "pdfs/" + assert c._s3_relative_name(f"s3://{BUCKET}/pdfs/a/x.pdf", "pdfs/") == os.path.join("a", "x.pdf") + assert c._s3_relative_name(f"s3://{BUCKET}/pdfs/x.pdf", "pdfs/") == "x.pdf" + + +@mock_aws +def test_same_basename_under_different_prefixes(tmp_path): + s3 = boto3.client("s3", region_name=REGION) + s3.create_bucket(Bucket=BUCKET) + for k in ["pdfs/a/x.pdf", "pdfs/b/x.pdf", "pdfs/top.pdf"]: + s3.put_object(Bucket=BUCKET, Key=k, Body=b"%PDF-" + k.encode()) + c = _client() + out = str(tmp_path / "out") + posted = [] + + def fake_post(url, files=None, data=None, headers=None, timeout=None): + posted.append(files["input"][0]) + resp = Mock() + resp.text = "ok" + return (resp, 200) + + with patch.object(GrobidClient, "post", side_effect=fake_post): + c.process("processFulltextDocument", f"s3://{BUCKET}/pdfs/", output=out, force=True) + assert sorted(posted) == [os.path.join("a", "x.pdf"), os.path.join("b", "x.pdf"), "top.pdf"] + assert _tei(out) == ["a/x.grobid.tei.xml", "b/x.grobid.tei.xml", "top.grobid.tei.xml"] + + # a rerun without --force skips all three, none is mistaken for another + posted.clear() + with patch.object(GrobidClient, "post", side_effect=fake_post): + c.process("processFulltextDocument", f"s3://{BUCKET}/pdfs/", output=out, force=False) + assert posted == [] + + +@mock_aws +def test_citation_lists_keep_their_prefix_on_the_disk_route(tmp_path): + s3 = boto3.client("s3", region_name=REGION) + s3.create_bucket(Bucket=BUCKET) + for k in ["lists/a/refs.txt", "lists/b/refs.txt"]: + s3.put_object(Bucket=BUCKET, Key=k, Body=b"one ref\n") + c = _client() + out = str(tmp_path / "out") + + def fake_post(url, files=None, data=None, headers=None, timeout=None): + resp = Mock() + resp.text = "refs" + return (resp, 200) + + with patch.object(GrobidClient, "post", side_effect=fake_post): + c.process("processCitationList", f"s3://{BUCKET}/lists/", output=out, force=True) + assert _tei(out) == ["a/refs.grobid.tei.xml", "b/refs.grobid.tei.xml"]