Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 36 additions & 3 deletions grobid_client/grobid_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,30 @@ def _s3_basename(self, uri: str) -> str:
"""Return the last path component of an s3:// key."""
return self._split_s3(uri)[1].rsplit("/", 1)[-1]

def _s3_common_prefix(self, uris: list) -> str:
"""Return the key prefix (up to a '/') shared by all the given objects.

Plays the role _common_base plays for local files: loose remote
objects are named relative to it, so pdfs/a/x.pdf and pdfs/b/x.pdf
listed under pdfs/ become a/x.pdf and b/x.pdf and do not collide on
the same output file.
"""
keys = [self._split_s3(uri)[1] for uri in uris]
if not keys:
return ""
if len(keys) == 1:
prefix = keys[0]
else:
prefix = os.path.commonprefix(keys)
return prefix[:prefix.rfind("/") + 1] if "/" in prefix else ""

def _s3_relative_name(self, uri: str, prefix: str) -> str:
"""Name a remote object by its key relative to prefix (see above)."""
key = self._split_s3(uri)[1]
name = key[len(prefix):] if prefix and key.startswith(prefix) else key
safe = self._safe_member_path("", name)
return safe if safe else self._s3_basename(uri)

def _import_smart_open(self) -> Any:
try:
import smart_open # noqa: F401
Expand Down Expand Up @@ -984,7 +1008,12 @@ def _is_eligible_input(self, filename: str, service: str) -> bool:
return False
if service == 'processCitationList':
# The input is a text file with one raw citation per line; a PDF
# would be read as text by process_txt and fail.
# would be read as text by process_txt and fail. An error marker
# (<stem>_<status>.txt, written next to the inputs when no output
# directory is given) is not an input either, or a rerun would
# post the markers of the previous run as citation lists.
if self.ERROR_FILE_RE.search(filename):
return False
return filename.endswith(".txt") or filename.endswith(".TXT")
if filename.endswith(".pdf") or filename.endswith(".PDF"):
return True
Expand Down Expand Up @@ -1373,6 +1402,9 @@ def _process_remote_files(

queue_size = self._effective_queue_size(n)
print(f"Found {total} remote file(s) to process")
# Objects are named relative to their common prefix so that the
# output mirrors the listed "directory", as it does for local ones
prefix = self._s3_common_prefix(uris)
processed_count = 0
error_count = 0
skipped_count = 0
Expand All @@ -1397,7 +1429,7 @@ def _process_remote_files(
self.logger.error(f"Failed to fetch {uri}: {str(e)}")
error_count += 1
continue
document.name = self._s3_basename(uri)
document.name = self._s3_relative_name(uri, prefix)
documents.append(document)

if not documents:
Expand Down Expand Up @@ -1434,8 +1466,9 @@ def _process_remote_files(
for uri in chunk:
if verbose:
self.logger.info(f"Fetching {uri}")
dest = os.path.join(temp_dir, self._s3_basename(uri))
dest = os.path.join(temp_dir, self._s3_relative_name(uri, prefix))
try:
os.makedirs(os.path.dirname(dest), exist_ok=True)
with self._s3_open(uri) as src, open(dest, "wb") as out_file:
shutil.copyfileobj(src, out_file)
local_files.append(dest)
Expand Down
62 changes: 62 additions & 0 deletions tests/test_citation_list_inputs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
"""
processCitationList must not pick up its own error markers as inputs.

Without --output the marker of a failed list (refs_500.txt) is written next
to refs.txt; on the next run it is a .txt in the input directory.
"""
import os
import tempfile
from unittest.mock import Mock, patch

from grobid_client.grobid_client import GrobidClient


def _client():
with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'):
with patch('grobid_client.grobid_client.GrobidClient._configure_logging'):
client = GrobidClient(check_server=False)
client.logger = Mock()
return client


def _run(client, input_dir, failing=()):
sent = []

def post(url, files=None, data=None, headers=None, timeout=None):
name = data['citations'][0] if files is None else files['input'][0]
sent.append(name)
resp = Mock()
if name in failing:
resp.text = 'boom'
return (resp, 500)
resp.text = '<TEI>ok</TEI>'
return (resp, 200)

with patch.object(GrobidClient, 'post', side_effect=post):
client.process('processCitationList', input_dir, output=None, force=False)
return sent


def test_eligibility():
c = _client()
assert c._is_eligible_input('refs.txt', 'processCitationList') is True
assert c._is_eligible_input('refs_500.txt', 'processCitationList') is False
assert c._is_eligible_input('refs_408.TXT', 'processCitationList') is True # markers are lower-case
assert c._is_eligible_input('refs_2020.txt', 'processCitationList') is True # not a status
assert c._is_eligible_input('paper.pdf', 'processCitationList') is False


def test_markers_of_a_previous_run_are_not_reprocessed():
c = _client()
with tempfile.TemporaryDirectory() as d:
inp = os.path.join(d, 'in')
os.makedirs(inp)
for name in ['good.txt', 'bad.txt']:
with open(os.path.join(inp, name), 'w') as fh:
fh.write(f'{name} citation\n')
# first run: bad.txt fails and leaves bad_500.txt next to it
assert sorted(_run(c, inp, failing={'bad.txt citation'})) == ['bad.txt citation', 'good.txt citation']
assert os.path.isfile(os.path.join(inp, 'bad_500.txt'))
# second run: only bad.txt is retried; the marker is not an input
assert _run(c, inp, failing={'bad.txt citation'}) == ['bad.txt citation']
assert sorted(os.listdir(inp)) == ['bad.txt', 'bad_500.txt', 'good.grobid.tei.xml', 'good.txt']
104 changes: 104 additions & 0 deletions tests/test_s3_output_names.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
"""
Loose S3 objects are named relative to their common prefix, so two keys
that share a basename (a/x.pdf, b/x.pdf) get two output files.
"""
import os

import pytest
from unittest.mock import Mock, patch

boto3 = pytest.importorskip("boto3")
pytest.importorskip("smart_open")
pytest.importorskip("moto")
try:
from moto import mock_aws
except ImportError: # moto < 5
from moto import mock_s3 as mock_aws

from grobid_client.grobid_client import GrobidClient

BUCKET = "test-bucket"
REGION = "us-east-1"


@pytest.fixture(autouse=True)
def _aws_env(monkeypatch):
for var in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SECURITY_TOKEN", "AWS_SESSION_TOKEN"):
monkeypatch.setenv(var, "testing")
monkeypatch.setenv("AWS_DEFAULT_REGION", REGION)


def _client():
with patch.object(GrobidClient, "_test_server_connection"):
with patch.object(GrobidClient, "_configure_logging"):
c = GrobidClient(check_server=False)
c.logger = Mock()
c.config["queue_size"] = 1 # one object per chunk: collisions across chunks too
return c


def _tei(out):
found = []
for root, _, files in os.walk(out):
for f in files:
if f.endswith(".grobid.tei.xml"):
found.append(os.path.relpath(os.path.join(root, f), out).replace(os.sep, "/"))
return sorted(found)


def test_common_prefix():
c = _client()
assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/a/x.pdf", f"s3://{BUCKET}/pdfs/b/x.pdf"]) == "pdfs/"
assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/x.pdf", f"s3://{BUCKET}/pdfs/y.pdf"]) == "pdfs/"
assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/x.pdf"]) == "pdfs/"
assert c._s3_common_prefix([f"s3://{BUCKET}/x.pdf", f"s3://{BUCKET}/y.pdf"]) == ""
assert c._s3_common_prefix([f"s3://{BUCKET}/pdfs/xa.pdf", f"s3://{BUCKET}/pdfs/xb.pdf"]) == "pdfs/"
assert c._s3_relative_name(f"s3://{BUCKET}/pdfs/a/x.pdf", "pdfs/") == os.path.join("a", "x.pdf")
assert c._s3_relative_name(f"s3://{BUCKET}/pdfs/x.pdf", "pdfs/") == "x.pdf"


@mock_aws
def test_same_basename_under_different_prefixes(tmp_path):
s3 = boto3.client("s3", region_name=REGION)
s3.create_bucket(Bucket=BUCKET)
for k in ["pdfs/a/x.pdf", "pdfs/b/x.pdf", "pdfs/top.pdf"]:
s3.put_object(Bucket=BUCKET, Key=k, Body=b"%PDF-" + k.encode())
c = _client()
out = str(tmp_path / "out")
posted = []

def fake_post(url, files=None, data=None, headers=None, timeout=None):
posted.append(files["input"][0])
resp = Mock()
resp.text = "<TEI>ok</TEI>"
return (resp, 200)

with patch.object(GrobidClient, "post", side_effect=fake_post):
c.process("processFulltextDocument", f"s3://{BUCKET}/pdfs/", output=out, force=True)
assert sorted(posted) == [os.path.join("a", "x.pdf"), os.path.join("b", "x.pdf"), "top.pdf"]
assert _tei(out) == ["a/x.grobid.tei.xml", "b/x.grobid.tei.xml", "top.grobid.tei.xml"]

# a rerun without --force skips all three, none is mistaken for another
posted.clear()
with patch.object(GrobidClient, "post", side_effect=fake_post):
c.process("processFulltextDocument", f"s3://{BUCKET}/pdfs/", output=out, force=False)
assert posted == []


@mock_aws
def test_citation_lists_keep_their_prefix_on_the_disk_route(tmp_path):
s3 = boto3.client("s3", region_name=REGION)
s3.create_bucket(Bucket=BUCKET)
for k in ["lists/a/refs.txt", "lists/b/refs.txt"]:
s3.put_object(Bucket=BUCKET, Key=k, Body=b"one ref\n")
c = _client()
out = str(tmp_path / "out")

def fake_post(url, files=None, data=None, headers=None, timeout=None):
resp = Mock()
resp.text = "<TEI>refs</TEI>"
return (resp, 200)

with patch.object(GrobidClient, "post", side_effect=fake_post):
c.process("processCitationList", f"s3://{BUCKET}/lists/", output=out, force=True)
assert _tei(out) == ["a/refs.grobid.tei.xml", "b/refs.grobid.tei.xml"]
Loading