Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,9 @@ grobid_client --input "~/data/**/*.pdf" --output ~/results processFulltextDocu
> - **Directories** are processed recursively and the output mirrors the input tree: `~/pdfs/2023/a.pdf` →
> `~/results/2023/a.grobid.tei.xml` (subdirectories of `--output` are created as needed, and the JSON/Markdown
> outputs and error files sit next to each TEI file). Archives keep their internal paths the same way.
> When two inputs of one run would get the same output (`paper.pdf` and `paper.PDF`, or two archives that both
> hold `doc.pdf`), the first one met keeps the name and the other gets its extension or its archive's name added
> (`paper.pdf.grobid.tei.xml`, `doc.<archive name>.grobid.tei.xml`); a warning names both.
> - **Archives** (`.zip`, `.tar`, `.tar.gz`/`.tgz`, `.tar.bz2`/`.tbz2`) are streamed: eligible entries are read into
> memory in chunks of `queue_size` and sent to GROBID straight from there — the archive is never fully decompressed and
> nothing but the results in `--output` ever touches the disk. If `--output` is omitted, results go to a directory named
Expand Down
542 changes: 343 additions & 199 deletions grobid_client/grobid_client.py

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions tests/test_busy_retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def test_each_attempt_uses_a_fresh_closed_stream(self):
client = _client()
post, calls = _busy_then_ok(3)
with patch.object(GrobidClient, 'post', side_effect=post):
with patch('grobid_client.grobid_client.time.sleep') as sleep:
with patch.object(client._stopping, 'wait', return_value=False) as sleep:
name, status, text = client.process_pdf('processFulltextDocument', b'%PDF-bytes')
assert (status, text) == (200, '<TEI>ok</TEI>')
assert sleep.call_count == 3
Expand All @@ -60,7 +60,7 @@ def test_a_file_on_disk_is_reopened_each_time(self):
with open(pdf, 'wb') as fh:
fh.write(b'%PDF-disk')
with patch.object(GrobidClient, 'post', side_effect=post):
with patch('grobid_client.grobid_client.time.sleep'):
with patch.object(client._stopping, 'wait', return_value=False):
_, status, _ = client.process_pdf('processFulltextDocument', pdf)
assert status == 200
assert all(h.closed for h in calls)
Expand All @@ -69,7 +69,7 @@ def test_a_long_busy_spell_does_not_recurse(self):
client = _client()
post, calls = _busy_then_ok(5000)
with patch.object(GrobidClient, 'post', side_effect=post):
with patch('grobid_client.grobid_client.time.sleep'):
with patch.object(client._stopping, 'wait', return_value=False):
_, status, _ = client.process_pdf('processFulltextDocument', b'%PDF')
assert status == 200
assert len(calls) == 5001
Expand All @@ -85,7 +85,7 @@ def post(url, files=None, data=None, headers=None, timeout=None):
return answer

with patch.object(GrobidClient, 'post', side_effect=post):
with patch('grobid_client.grobid_client.time.sleep'):
with patch.object(client._stopping, 'wait', return_value=False):
_, status, text = client.process_pdf('processFulltextDocument', b'%PDF')
assert status == 408 and 'timeout' in text.lower()

Expand All @@ -103,7 +103,7 @@ def test_503_then_ok(self):
post, calls = _busy_then_ok(2)
with tempfile.TemporaryDirectory() as d:
with patch.object(GrobidClient, 'post', side_effect=post):
with patch('grobid_client.grobid_client.time.sleep') as sleep:
with patch.object(client._stopping, 'wait', return_value=False) as sleep:
name, status, text = client.process_txt(
'processCitationList', self._refs(d), False, False, False, False, False, False, False)
assert (status, text) == (200, '<TEI>ok</TEI>')
Expand Down
52 changes: 52 additions & 0 deletions tests/test_empty_answer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
"""A 200 without content is an error, not an empty TEI that blocks every rerun."""
import os
from unittest.mock import Mock, patch

import pytest

from grobid_client.grobid_client import GrobidClient


def _client():
with patch.object(GrobidClient, '_configure_logging'):
client = GrobidClient(check_server=False)
client.logger = Mock()
return client


@pytest.mark.parametrize('body', ['', ' \n'])
def test_process_pdf_reports_an_empty_answer(body):
client = _client()
with patch.object(client, 'post', return_value=(Mock(text=body), 200)):
name, status, text = client.process_pdf('processFulltextDocument', b'%PDF')
assert (status, text) == (500, GrobidClient.EMPTY_ANSWER)


def test_process_txt_reports_an_empty_answer(tmp_path):
refs = tmp_path / 'refs.txt'
refs.write_text('a ref\n')
client = _client()
with patch.object(client, 'post', return_value=(Mock(text=''), 200)):
_, status, _ = client.process_txt('processCitationList', str(refs), False, False, False, False,
False, False, False)
assert status == 500


def test_an_empty_answer_leaves_a_marker_and_is_retried(tmp_path):
src = tmp_path / 'in'
src.mkdir()
(src / 'p.pdf').write_bytes(b'%PDF')
out = tmp_path / 'out'
answers = iter([(Mock(text=''), 200), (Mock(text='<TEI>ok</TEI>'), 200)])
client = _client()
with patch.object(client, 'post', side_effect=lambda *a, **k: next(answers)):
client.process_paths('processFulltextDocument', [str(src)], output=str(out), force=False)
assert sorted(os.listdir(out)) == ['p_500.txt']
client.process_paths('processFulltextDocument', [str(src)], output=str(out), force=False)
assert sorted(os.listdir(out)) == ['p.grobid.tei.xml']


def test_a_real_answer_is_untouched():
client = _client()
with patch.object(client, 'post', return_value=(Mock(text='<TEI/>'), 200)):
assert client.process_pdf('processFulltextDocument', b'%PDF')[1:] == (200, '<TEI/>')
17 changes: 17 additions & 0 deletions tests/test_end_to_end.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,3 +160,20 @@ def test_without_output_results_sit_next_to_the_inputs(server, pdf_tree, tmp_pat
assert result.returncode == 0, result.stderr
assert (pdf_tree / 'a' / 'x.grobid.tei.xml').exists()
assert (pdf_tree / 'a' / 'deep' / 'y.grobid.tei.xml').exists()


def test_progress_lines_come_before_the_errors_in_a_log_file(server, pdf_tree, tmp_path):
# stdout and stderr into one file, as with `grobid_client ... > run.log 2>&1`:
# "Found N file(s)" used to sit in stdout's buffer and land at the very end
log = tmp_path / 'run.log'
with open(log, 'w') as fh:
subprocess.run(
[sys.executable, '-c', 'from grobid_client.grobid_client import main; main()',
'--server', server, '--input', str(pdf_tree), '--output', str(tmp_path / 'out'),
'processFulltextDocument'],
stdout=fh, stderr=subprocess.STDOUT, cwd=tmp_path, timeout=120)
lines = log.read_text().splitlines()
found = next(i for i, line in enumerate(lines) if line.startswith('Found 5 local file(s)'))
error = next(i for i, line in enumerate(lines) if 'broken.pdf failed' in line)
summary = next(i for i, line in enumerate(lines) if line.startswith('Processing completed: 4 out of 5'))
assert found < error < summary
10 changes: 5 additions & 5 deletions tests/test_grobid_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -407,8 +407,8 @@ def test_process_pdf_server_busy_retry(self, mock_post):
with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'):
with patch('grobid_client.grobid_client.GrobidClient._configure_logging'):
with patch('builtins.open', mock_open()):
with patch('time.sleep') as mock_sleep:
client = GrobidClient(check_server=False)
client = GrobidClient(check_server=False)
with patch.object(client._stopping, 'wait', return_value=False) as mock_sleep:
client.logger = Mock()

result = client.process_pdf(
Expand All @@ -423,7 +423,7 @@ def test_process_pdf_server_busy_retry(self, mock_post):
segment_sentences=False
)

# Should have called sleep due to 503 response
# Should have waited once due to the 503 response
mock_sleep.assert_called_once()
assert result[1] == 200

Expand Down Expand Up @@ -1635,7 +1635,7 @@ def test_retry_after_503_resends_the_whole_document(self):
client = self._client()
post, sent = self._post_spy(statuses=(503, 200))

with patch('time.sleep'):
with patch.object(client._stopping, 'wait', return_value=False):
with patch.object(GrobidClient, 'post', side_effect=post):
result = client.process_pdf(
'processFulltextDocument', self._named(self.PDF, 'busy.pdf'))
Expand All @@ -1653,7 +1653,7 @@ def test_retry_after_503_reopens_a_file_on_disk(self):
with open(path, 'wb') as f:
f.write(self.PDF)

with patch('time.sleep'):
with patch.object(client._stopping, 'wait', return_value=False):
with patch.object(GrobidClient, 'post', side_effect=post):
result = client.process_pdf('processFulltextDocument', path)

Expand Down
150 changes: 150 additions & 0 deletions tests/test_interrupt_busy_server.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
"""
Ctrl-C while the server is busy.

A document the server answers 503 to is re-posted after ``sleep_time`` for as
long as the server stays busy. That retry ran in the pool's worker threads, which
the interrupted main thread then waited for: the client printed "Interrupted"
and kept posting, so a permanently busy server needed a second Ctrl-C (or a kill).
"""
import http.server
import json
import signal
import subprocess
import sys
import threading
import time
from unittest.mock import Mock, patch

import pytest

from grobid_client.grobid_client import GrobidClient


def _client(sleep_time=60):
with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'):
with patch('grobid_client.grobid_client.GrobidClient._configure_logging'):
client = GrobidClient(check_server=False)
client.logger = Mock()
client.config['sleep_time'] = sleep_time
return client


def _always_busy():
calls = []

def post(url, files=None, data=None, headers=None, timeout=None):
calls.append(url)
return Mock(text='busy'), 503

return post, calls


def test_a_stopped_client_does_not_wait_for_a_busy_server():
client = _client()
client._stopping.set()
post, calls = _always_busy()
with patch.object(GrobidClient, 'post', side_effect=post):
start = time.time()
name, status, text = client.process_pdf('processFulltextDocument', b'%PDF')
assert time.time() - start < 5
assert (status, text) == (503, GrobidClient.BUSY_SERVER_INTERRUPTED)
assert len(calls) == 1


def test_interrupting_a_batch_ends_the_busy_retries(tmp_path):
"""KeyboardInterrupt in the main thread must not leave workers retrying."""
client = _client(sleep_time=60)
post, calls = _always_busy()
pdf = tmp_path / 'busy.pdf'
pdf.write_bytes(b'%PDF')

def interrupted(futures):
deadline = time.time() + 10
while not calls and time.time() < deadline:
time.sleep(0.01)
raise KeyboardInterrupt

with patch.object(GrobidClient, 'post', side_effect=post):
with patch('grobid_client.grobid_client.concurrent.futures.as_completed', side_effect=interrupted):
start = time.time()
with pytest.raises(KeyboardInterrupt):
client.process_batch('processFulltextDocument', [str(pdf)], str(tmp_path), str(tmp_path / 'out'),
2, False, False, False, False, False, False, False, False)
assert time.time() - start < 10 # not the 60 s sleep_time
assert len(calls) == 1
assert not (tmp_path / 'out').exists() # an interrupted document leaves no error marker


def test_a_new_batch_after_an_interrupt_retries_again(tmp_path):
client = _client(sleep_time=0)
client._stopping.set() # left over from an interrupted batch
answers = iter([503, 200])

def post(url, files=None, data=None, headers=None, timeout=None):
return Mock(text='<TEI/>'), next(answers)

pdf = tmp_path / 'p.pdf'
pdf.write_bytes(b'%PDF')
with patch.object(GrobidClient, 'post', side_effect=post):
processed, errors, skipped = client.process_batch(
'processFulltextDocument', [str(pdf)], str(tmp_path), str(tmp_path / 'out'),
1, False, False, False, False, False, False, False, False)
assert (processed, errors) == (1, 0)


class _Busy(http.server.BaseHTTPRequestHandler):
posts = 0

def log_message(self, *args):
pass

def do_GET(self):
self.send_response(200)
self.send_header('Content-Length', '4')
self.end_headers()
self.wfile.write(b'true')

def do_POST(self):
self.rfile.read(int(self.headers.get('Content-Length', 0)))
type(self).posts += 1
self.send_response(503)
self.send_header('Content-Length', '4')
self.end_headers()
self.wfile.write(b'busy')


@pytest.mark.skipif(sys.platform == 'win32', reason='needs POSIX signals')
def test_cli_stops_on_the_first_ctrl_c_against_a_busy_server(tmp_path):
httpd = http.server.ThreadingHTTPServer(('127.0.0.1', 0), _Busy)
threading.Thread(target=httpd.serve_forever, daemon=True).start()
try:
url = f'http://127.0.0.1:{httpd.server_address[1]}'
(tmp_path / 'in').mkdir()
for name in ('a.pdf', 'b.pdf', 'c.pdf'):
(tmp_path / 'in' / name).write_bytes(b'%PDF')
config = tmp_path / 'config.json'
config.write_text(json.dumps({'grobid_server': url, 'sleep_time': 30}))
proc = subprocess.Popen(
[sys.executable, '-c', 'from grobid_client.grobid_client import main; main()',
'--config', str(config), '--input', str(tmp_path / 'in'), '--output', str(tmp_path / 'out'),
'--n', '2', 'processFulltextDocument'],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, cwd=str(tmp_path))
deadline = time.time() + 30
while _Busy.posts < 2 and time.time() < deadline:
time.sleep(0.05)
assert _Busy.posts >= 2, 'the client never reached the server'
proc.send_signal(signal.SIGINT)
start = time.time()
try:
log, _ = proc.communicate(timeout=15)
except subprocess.TimeoutExpired:
proc.kill()
pytest.fail('still retrying the busy server 15 s after Ctrl-C')
assert proc.returncode == 130, log
assert time.time() - start < 15
assert 'run the same command again' in log
# the documents interrupted while waiting are not recorded as failures
assert not list((tmp_path / 'out').glob('*_503.txt'))
finally:
httpd.shutdown()
httpd.server_close()
Loading
Loading