From 077d5f26d5b35f07d2b3b9307a9de7d3e1b71497 Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Fri, 6 Nov 2026 17:22:41 +0100 Subject: [PATCH 1/2] Do not treat error markers as citation-list inputs Any .txt was an eligible input for processCitationList, including the _.txt markers written next to the inputs when no output directory is given. A rerun posted the previous run's markers as citation lists, producing marker-of-marker files and inflated counts. PDFs are no longer eligible for this service either, since process_txt reads them as text and fails. --- grobid_client/grobid_client.py | 7 +++- tests/test_citation_list_inputs.py | 62 ++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 tests/test_citation_list_inputs.py diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index 34de3bb..e3edc8a 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -984,7 +984,12 @@ def _is_eligible_input(self, filename: str, service: str) -> bool: return False if service == 'processCitationList': # The input is a text file with one raw citation per line; a PDF - # would be read as text by process_txt and fail. + # would be read as text by process_txt and fail. An error marker + # (_.txt, written next to the inputs when no output + # directory is given) is not an input either, or a rerun would + # post the markers of the previous run as citation lists. + if self.ERROR_FILE_RE.search(filename): + return False return filename.endswith(".txt") or filename.endswith(".TXT") if filename.endswith(".pdf") or filename.endswith(".PDF"): return True diff --git a/tests/test_citation_list_inputs.py b/tests/test_citation_list_inputs.py new file mode 100644 index 0000000..356db47 --- /dev/null +++ b/tests/test_citation_list_inputs.py @@ -0,0 +1,62 @@ +""" +processCitationList must not pick up its own error markers as inputs. + +Without --output the marker of a failed list (refs_500.txt) is written next +to refs.txt; on the next run it is a .txt in the input directory. +""" +import os +import tempfile +from unittest.mock import Mock, patch + +from grobid_client.grobid_client import GrobidClient + + +def _client(): + with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'): + with patch('grobid_client.grobid_client.GrobidClient._configure_logging'): + client = GrobidClient(check_server=False) + client.logger = Mock() + return client + + +def _run(client, input_dir, failing=()): + sent = [] + + def post(url, files=None, data=None, headers=None, timeout=None): + name = data['citations'][0] if files is None else files['input'][0] + sent.append(name) + resp = Mock() + if name in failing: + resp.text = 'boom' + return (resp, 500) + resp.text = 'ok' + return (resp, 200) + + with patch.object(GrobidClient, 'post', side_effect=post): + client.process('processCitationList', input_dir, output=None, force=False) + return sent + + +def test_eligibility(): + c = _client() + assert c._is_eligible_input('refs.txt', 'processCitationList') is True + assert c._is_eligible_input('refs_500.txt', 'processCitationList') is False + assert c._is_eligible_input('refs_408.TXT', 'processCitationList') is True # markers are lower-case + assert c._is_eligible_input('refs_2020.txt', 'processCitationList') is True # not a status + assert c._is_eligible_input('paper.pdf', 'processCitationList') is False + + +def test_markers_of_a_previous_run_are_not_reprocessed(): + c = _client() + with tempfile.TemporaryDirectory() as d: + inp = os.path.join(d, 'in') + os.makedirs(inp) + for name in ['good.txt', 'bad.txt']: + with open(os.path.join(inp, name), 'w') as fh: + fh.write(f'{name} citation\n') + # first run: bad.txt fails and leaves bad_500.txt next to it + assert sorted(_run(c, inp, failing={'bad.txt citation'})) == ['bad.txt citation', 'good.txt citation'] + assert os.path.isfile(os.path.join(inp, 'bad_500.txt')) + # second run: only bad.txt is retried; the marker is not an input + assert _run(c, inp, failing={'bad.txt citation'}) == ['bad.txt citation'] + assert sorted(os.listdir(inp)) == ['bad.txt', 'bad_500.txt', 'good.grobid.tei.xml', 'good.txt'] From 9a34be4a3d3bea2b7291c0e0f80371be6e76d20d Mon Sep 17 00:00:00 2001 From: Luca <15426+lfoppiano@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:20:12 +0000 Subject: [PATCH 2/2] Name loose S3 objects relative to their common prefix (#133) Remote objects were named by basename only, so pdfs/a/x.pdf and pdfs/b/x.pdf listed under pdfs/ both mapped to x.grobid.tei.xml: within a chunk the second overwrote the first, across chunks it was skipped as already done. Objects are now named by their key relative to the common prefix of the listing, mirroring what local directories and archives do, and the citation-list download route keeps that relative path on disk. --- grobid_client/grobid_client.py | 32 +++++++++- tests/test_s3_output_names.py | 104 +++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 tests/test_s3_output_names.py diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index e3edc8a..ed48701 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -824,6 +824,30 @@ def _s3_basename(self, uri: str) -> str: """Return the last path component of an s3:// key.""" return self._split_s3(uri)[1].rsplit("/", 1)[-1] + def _s3_common_prefix(self, uris: list) -> str: + """Return the key prefix (up to a '/') shared by all the given objects. + + Plays the role _common_base plays for local files: loose remote + objects are named relative to it, so pdfs/a/x.pdf and pdfs/b/x.pdf + listed under pdfs/ become a/x.pdf and b/x.pdf and do not collide on + the same output file. + """ + keys = [self._split_s3(uri)[1] for uri in uris] + if not keys: + return "" + if len(keys) == 1: + prefix = keys[0] + else: + prefix = os.path.commonprefix(keys) + return prefix[:prefix.rfind("/") + 1] if "/" in prefix else "" + + def _s3_relative_name(self, uri: str, prefix: str) -> str: + """Name a remote object by its key relative to prefix (see above).""" + key = self._split_s3(uri)[1] + name = key[len(prefix):] if prefix and key.startswith(prefix) else key + safe = self._safe_member_path("", name) + return safe if safe else self._s3_basename(uri) + def _import_smart_open(self) -> Any: try: import smart_open # noqa: F401 @@ -1378,6 +1402,9 @@ def _process_remote_files( queue_size = self._effective_queue_size(n) print(f"Found {total} remote file(s) to process") + # Objects are named relative to their common prefix so that the + # output mirrors the listed "directory", as it does for local ones + prefix = self._s3_common_prefix(uris) processed_count = 0 error_count = 0 skipped_count = 0 @@ -1402,7 +1429,7 @@ def _process_remote_files( self.logger.error(f"Failed to fetch {uri}: {str(e)}") error_count += 1 continue - document.name = self._s3_basename(uri) + document.name = self._s3_relative_name(uri, prefix) documents.append(document) if not documents: @@ -1439,8 +1466,9 @@ def _process_remote_files( for uri in chunk: if verbose: self.logger.info(f"Fetching {uri}") - dest = os.path.join(temp_dir, self._s3_basename(uri)) + dest = os.path.join(temp_dir, self._s3_relative_name(uri, prefix)) try: + os.makedirs(os.path.dirname(dest), exist_ok=True) with self._s3_open(uri) as src, open(dest, "wb") as out_file: shutil.copyfileobj(src, out_file) local_files.append(dest) diff --git a/tests/test_s3_output_names.py b/tests/test_s3_output_names.py new file mode 100644 index 0000000..970d135 --- /dev/null +++ b/tests/test_s3_output_names.py @@ -0,0 +1,104 @@ +""" +Loose S3 objects are named relative to their common prefix, so two keys +that share a basename (a/x.pdf, b/x.pdf) get two output files. +""" +import os + +import pytest +from unittest.mock import Mock, patch + +boto3 = pytest.importorskip("boto3") +pytest.importorskip("smart_open") +pytest.importorskip("moto") +try: + from moto import mock_aws +except ImportError: # moto < 5 + from moto import mock_s3 as mock_aws + +from grobid_client.grobid_client import GrobidClient + +BUCKET = "test-bucket" +REGION = "us-east-1" + + +@pytest.fixture(autouse=True) +def _aws_env(monkeypatch): + for var in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SECURITY_TOKEN", "AWS_SESSION_TOKEN"): + monkeypatch.setenv(var, "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", REGION) + + +def _client(): + with patch.object(GrobidClient, "_test_server_connection"): + with patch.object(GrobidClient, "_configure_logging"): + c = GrobidClient(check_server=False) + c.logger = Mock() + c.config["queue_size"] = 1 # one object per chunk: collisions across chunks too + return c + + +def _tei(out): + found = [] + for root, _, files in os.walk(out): + for f in files: + if f.endswith(".grobid.tei.xml"): + found.append(os.path.relpath(os.path.join(root, f), out).replace(os.sep, "/")) + return sorted(found) + + +def test_common_prefix(): + c = _client() + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/a/x.pdf", f"s3://{BUCKET}/pdfs/b/x.pdf"]) == "pdfs/" + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/x.pdf", f"s3://{BUCKET}/pdfs/y.pdf"]) == "pdfs/" + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/x.pdf"]) == "pdfs/" + assert c._s3_common_prefix([f"s3://{BUCKET}/x.pdf", f"s3://{BUCKET}/y.pdf"]) == "" + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/xa.pdf", f"s3://{BUCKET}/pdfs/xb.pdf"]) == "pdfs/" + assert c._s3_relative_name(f"s3://{BUCKET}/pdfs/a/x.pdf", "pdfs/") == os.path.join("a", "x.pdf") + assert c._s3_relative_name(f"s3://{BUCKET}/pdfs/x.pdf", "pdfs/") == "x.pdf" + + +@mock_aws +def test_same_basename_under_different_prefixes(tmp_path): + s3 = boto3.client("s3", region_name=REGION) + s3.create_bucket(Bucket=BUCKET) + for k in ["pdfs/a/x.pdf", "pdfs/b/x.pdf", "pdfs/top.pdf"]: + s3.put_object(Bucket=BUCKET, Key=k, Body=b"%PDF-" + k.encode()) + c = _client() + out = str(tmp_path / "out") + posted = [] + + def fake_post(url, files=None, data=None, headers=None, timeout=None): + posted.append(files["input"][0]) + resp = Mock() + resp.text = "ok" + return (resp, 200) + + with patch.object(GrobidClient, "post", side_effect=fake_post): + c.process("processFulltextDocument", f"s3://{BUCKET}/pdfs/", output=out, force=True) + assert sorted(posted) == [os.path.join("a", "x.pdf"), os.path.join("b", "x.pdf"), "top.pdf"] + assert _tei(out) == ["a/x.grobid.tei.xml", "b/x.grobid.tei.xml", "top.grobid.tei.xml"] + + # a rerun without --force skips all three, none is mistaken for another + posted.clear() + with patch.object(GrobidClient, "post", side_effect=fake_post): + c.process("processFulltextDocument", f"s3://{BUCKET}/pdfs/", output=out, force=False) + assert posted == [] + + +@mock_aws +def test_citation_lists_keep_their_prefix_on_the_disk_route(tmp_path): + s3 = boto3.client("s3", region_name=REGION) + s3.create_bucket(Bucket=BUCKET) + for k in ["lists/a/refs.txt", "lists/b/refs.txt"]: + s3.put_object(Bucket=BUCKET, Key=k, Body=b"one ref\n") + c = _client() + out = str(tmp_path / "out") + + def fake_post(url, files=None, data=None, headers=None, timeout=None): + resp = Mock() + resp.text = "refs" + return (resp, 200) + + with patch.object(GrobidClient, "post", side_effect=fake_post): + c.process("processCitationList", f"s3://{BUCKET}/lists/", output=out, force=True) + assert _tei(out) == ["a/refs.grobid.tei.xml", "b/refs.grobid.tei.xml"]