diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index 34de3bb..ed48701 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -824,6 +824,30 @@ def _s3_basename(self, uri: str) -> str: """Return the last path component of an s3:// key.""" return self._split_s3(uri)[1].rsplit("/", 1)[-1] + def _s3_common_prefix(self, uris: list) -> str: + """Return the key prefix (up to a '/') shared by all the given objects. + + Plays the role _common_base plays for local files: loose remote + objects are named relative to it, so pdfs/a/x.pdf and pdfs/b/x.pdf + listed under pdfs/ become a/x.pdf and b/x.pdf and do not collide on + the same output file. + """ + keys = [self._split_s3(uri)[1] for uri in uris] + if not keys: + return "" + if len(keys) == 1: + prefix = keys[0] + else: + prefix = os.path.commonprefix(keys) + return prefix[:prefix.rfind("/") + 1] if "/" in prefix else "" + + def _s3_relative_name(self, uri: str, prefix: str) -> str: + """Name a remote object by its key relative to prefix (see above).""" + key = self._split_s3(uri)[1] + name = key[len(prefix):] if prefix and key.startswith(prefix) else key + safe = self._safe_member_path("", name) + return safe if safe else self._s3_basename(uri) + def _import_smart_open(self) -> Any: try: import smart_open # noqa: F401 @@ -984,7 +1008,12 @@ def _is_eligible_input(self, filename: str, service: str) -> bool: return False if service == 'processCitationList': # The input is a text file with one raw citation per line; a PDF - # would be read as text by process_txt and fail. + # would be read as text by process_txt and fail. An error marker + # (_.txt, written next to the inputs when no output + # directory is given) is not an input either, or a rerun would + # post the markers of the previous run as citation lists. + if self.ERROR_FILE_RE.search(filename): + return False return filename.endswith(".txt") or filename.endswith(".TXT") if filename.endswith(".pdf") or filename.endswith(".PDF"): return True @@ -1373,6 +1402,9 @@ def _process_remote_files( queue_size = self._effective_queue_size(n) print(f"Found {total} remote file(s) to process") + # Objects are named relative to their common prefix so that the + # output mirrors the listed "directory", as it does for local ones + prefix = self._s3_common_prefix(uris) processed_count = 0 error_count = 0 skipped_count = 0 @@ -1397,7 +1429,7 @@ def _process_remote_files( self.logger.error(f"Failed to fetch {uri}: {str(e)}") error_count += 1 continue - document.name = self._s3_basename(uri) + document.name = self._s3_relative_name(uri, prefix) documents.append(document) if not documents: @@ -1434,8 +1466,9 @@ def _process_remote_files( for uri in chunk: if verbose: self.logger.info(f"Fetching {uri}") - dest = os.path.join(temp_dir, self._s3_basename(uri)) + dest = os.path.join(temp_dir, self._s3_relative_name(uri, prefix)) try: + os.makedirs(os.path.dirname(dest), exist_ok=True) with self._s3_open(uri) as src, open(dest, "wb") as out_file: shutil.copyfileobj(src, out_file) local_files.append(dest) diff --git a/tests/test_citation_list_inputs.py b/tests/test_citation_list_inputs.py new file mode 100644 index 0000000..356db47 --- /dev/null +++ b/tests/test_citation_list_inputs.py @@ -0,0 +1,62 @@ +""" +processCitationList must not pick up its own error markers as inputs. + +Without --output the marker of a failed list (refs_500.txt) is written next +to refs.txt; on the next run it is a .txt in the input directory. +""" +import os +import tempfile +from unittest.mock import Mock, patch + +from grobid_client.grobid_client import GrobidClient + + +def _client(): + with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'): + with patch('grobid_client.grobid_client.GrobidClient._configure_logging'): + client = GrobidClient(check_server=False) + client.logger = Mock() + return client + + +def _run(client, input_dir, failing=()): + sent = [] + + def post(url, files=None, data=None, headers=None, timeout=None): + name = data['citations'][0] if files is None else files['input'][0] + sent.append(name) + resp = Mock() + if name in failing: + resp.text = 'boom' + return (resp, 500) + resp.text = 'ok' + return (resp, 200) + + with patch.object(GrobidClient, 'post', side_effect=post): + client.process('processCitationList', input_dir, output=None, force=False) + return sent + + +def test_eligibility(): + c = _client() + assert c._is_eligible_input('refs.txt', 'processCitationList') is True + assert c._is_eligible_input('refs_500.txt', 'processCitationList') is False + assert c._is_eligible_input('refs_408.TXT', 'processCitationList') is True # markers are lower-case + assert c._is_eligible_input('refs_2020.txt', 'processCitationList') is True # not a status + assert c._is_eligible_input('paper.pdf', 'processCitationList') is False + + +def test_markers_of_a_previous_run_are_not_reprocessed(): + c = _client() + with tempfile.TemporaryDirectory() as d: + inp = os.path.join(d, 'in') + os.makedirs(inp) + for name in ['good.txt', 'bad.txt']: + with open(os.path.join(inp, name), 'w') as fh: + fh.write(f'{name} citation\n') + # first run: bad.txt fails and leaves bad_500.txt next to it + assert sorted(_run(c, inp, failing={'bad.txt citation'})) == ['bad.txt citation', 'good.txt citation'] + assert os.path.isfile(os.path.join(inp, 'bad_500.txt')) + # second run: only bad.txt is retried; the marker is not an input + assert _run(c, inp, failing={'bad.txt citation'}) == ['bad.txt citation'] + assert sorted(os.listdir(inp)) == ['bad.txt', 'bad_500.txt', 'good.grobid.tei.xml', 'good.txt'] diff --git a/tests/test_s3_output_names.py b/tests/test_s3_output_names.py new file mode 100644 index 0000000..970d135 --- /dev/null +++ b/tests/test_s3_output_names.py @@ -0,0 +1,104 @@ +""" +Loose S3 objects are named relative to their common prefix, so two keys +that share a basename (a/x.pdf, b/x.pdf) get two output files. +""" +import os + +import pytest +from unittest.mock import Mock, patch + +boto3 = pytest.importorskip("boto3") +pytest.importorskip("smart_open") +pytest.importorskip("moto") +try: + from moto import mock_aws +except ImportError: # moto < 5 + from moto import mock_s3 as mock_aws + +from grobid_client.grobid_client import GrobidClient + +BUCKET = "test-bucket" +REGION = "us-east-1" + + +@pytest.fixture(autouse=True) +def _aws_env(monkeypatch): + for var in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SECURITY_TOKEN", "AWS_SESSION_TOKEN"): + monkeypatch.setenv(var, "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", REGION) + + +def _client(): + with patch.object(GrobidClient, "_test_server_connection"): + with patch.object(GrobidClient, "_configure_logging"): + c = GrobidClient(check_server=False) + c.logger = Mock() + c.config["queue_size"] = 1 # one object per chunk: collisions across chunks too + return c + + +def _tei(out): + found = [] + for root, _, files in os.walk(out): + for f in files: + if f.endswith(".grobid.tei.xml"): + found.append(os.path.relpath(os.path.join(root, f), out).replace(os.sep, "/")) + return sorted(found) + + +def test_common_prefix(): + c = _client() + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/a/x.pdf", f"s3://{BUCKET}/pdfs/b/x.pdf"]) == "pdfs/" + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/x.pdf", f"s3://{BUCKET}/pdfs/y.pdf"]) == "pdfs/" + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/x.pdf"]) == "pdfs/" + assert c._s3_common_prefix([f"s3://{BUCKET}/x.pdf", f"s3://{BUCKET}/y.pdf"]) == "" + assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/xa.pdf", f"s3://{BUCKET}/pdfs/xb.pdf"]) == "pdfs/" + assert c._s3_relative_name(f"s3://{BUCKET}/pdfs/a/x.pdf", "pdfs/") == os.path.join("a", "x.pdf") + assert c._s3_relative_name(f"s3://{BUCKET}/pdfs/x.pdf", "pdfs/") == "x.pdf" + + +@mock_aws +def test_same_basename_under_different_prefixes(tmp_path): + s3 = boto3.client("s3", region_name=REGION) + s3.create_bucket(Bucket=BUCKET) + for k in ["pdfs/a/x.pdf", "pdfs/b/x.pdf", "pdfs/top.pdf"]: + s3.put_object(Bucket=BUCKET, Key=k, Body=b"%PDF-" + k.encode()) + c = _client() + out = str(tmp_path / "out") + posted = [] + + def fake_post(url, files=None, data=None, headers=None, timeout=None): + posted.append(files["input"][0]) + resp = Mock() + resp.text = "ok" + return (resp, 200) + + with patch.object(GrobidClient, "post", side_effect=fake_post): + c.process("processFulltextDocument", f"s3://{BUCKET}/pdfs/", output=out, force=True) + assert sorted(posted) == [os.path.join("a", "x.pdf"), os.path.join("b", "x.pdf"), "top.pdf"] + assert _tei(out) == ["a/x.grobid.tei.xml", "b/x.grobid.tei.xml", "top.grobid.tei.xml"] + + # a rerun without --force skips all three, none is mistaken for another + posted.clear() + with patch.object(GrobidClient, "post", side_effect=fake_post): + c.process("processFulltextDocument", f"s3://{BUCKET}/pdfs/", output=out, force=False) + assert posted == [] + + +@mock_aws +def test_citation_lists_keep_their_prefix_on_the_disk_route(tmp_path): + s3 = boto3.client("s3", region_name=REGION) + s3.create_bucket(Bucket=BUCKET) + for k in ["lists/a/refs.txt", "lists/b/refs.txt"]: + s3.put_object(Bucket=BUCKET, Key=k, Body=b"one ref\n") + c = _client() + out = str(tmp_path / "out") + + def fake_post(url, files=None, data=None, headers=None, timeout=None): + resp = Mock() + resp.text = "refs" + return (resp, 200) + + with patch.object(GrobidClient, "post", side_effect=fake_post): + c.process("processCitationList", f"s3://{BUCKET}/lists/", output=out, force=True) + assert _tei(out) == ["a/refs.grobid.tei.xml", "b/refs.grobid.tei.xml"]