From 661a73ea983e4586fcceb578f5c682380c0e4019 Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Tue, 10 Nov 2026 20:01:23 +0100 Subject: [PATCH 1/7] Stop retrying a busy server on Ctrl-C A document the server answers 503 to is re-posted every sleep_time seconds for as long as the server stays busy. The retry runs in the pool's worker threads, and the interrupted main thread waited for them: the client kept posting after Ctrl-C and only a second Ctrl-C or a kill stopped it. The busy wait is now an Event wait. When a batch is interrupted the event is set and queued work is cancelled, so waiting workers return at once and the process exits with 130. An interrupted document leaves no error marker. The event is cleared at the start of the next batch. --- grobid_client/grobid_client.py | 387 +++++++++++++++------------- tests/test_busy_retry.py | 10 +- tests/test_grobid_client.py | 10 +- tests/test_interrupt_busy_server.py | 150 +++++++++++ 4 files changed, 371 insertions(+), 186 deletions(-) create mode 100644 tests/test_interrupt_busy_server.py diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index 2bd99fd..0c6397e 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -34,6 +34,7 @@ import shutil import tarfile import tempfile +import threading import zipfile from typing import Any, BinaryIO, Callable, List, Optional, Tuple, Union import copy @@ -141,6 +142,9 @@ def __init__( # --no-sign-request) instead of with the default credential chain self.s3_no_sign_request = s3_no_sign_request self._s3_client_instance: Any = None + # Set when a run is being interrupted: the worker threads waiting to + # re-post a document to a busy server give up instead of retrying + self._stopping = threading.Event() # Initialize config with defaults self.config = copy.deepcopy(self.DEFAULT_CONFIG) @@ -220,10 +224,25 @@ def _warn_on_consolidation_timeout(self, consolidate_citations: bool) -> None: f"(2-3 minutes is recommended)." ) - def _wait_for_busy_server(self, file_path: str) -> None: - """Pause before re-posting a document the server answered 503 to.""" + BUSY_SERVER_INTERRUPTED = "Interrupted while waiting for the busy server" + + def _wait_for_busy_server(self, file_path: str) -> bool: + """Pause before re-posting a document the server answered 503 to. + + Returns False, at once, when the run is being interrupted: a server + that stays busy would otherwise keep the worker threads retrying, and + the process alive, long after Ctrl-C. + """ + if self._stopping.is_set(): + return False self.logger.warning(f"Server busy (503), retrying {file_path} after {self.config['sleep_time']} seconds") - time.sleep(self.config["sleep_time"]) + return not self._stopping.wait(self.config["sleep_time"]) + + def _stop_workers(self, futures: list) -> None: + """Called when a batch is interrupted: drop queued work, end busy-server retries.""" + self._stopping.set() + for future in futures: + future.cancel() def _handle_request_error( self, file_path: str, error: Exception, error_type: str = "Request" @@ -1595,172 +1614,180 @@ def process_batch( # we use ThreadPoolExecutor and not ProcessPoolExecutor because it is an I/O intensive process # A pool of one is still a pool: n < 1 would make ThreadPoolExecutor raise + self._stopping.clear() with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, n)) as executor: results = [] - for input_file in input_files: - # An entry is either a path or an in-memory document (a named - # stream, as fed by the archive/s3 streaming). Either way it goes - # by its name here, and process_pdf returns that same name, so - # the results below land on the same output file. - input_name = input_file if isinstance(input_file, str) else self._document_name(input_file) - # check if TEI file is already produced - filename = self._output_file_name(input_name, input_path, output) - if not force and os.path.isfile(filename): - self.logger.info( - f"{filename} already exists, skipping... (use --force to reprocess pdf input files)") - skipped_count += 1 - - # Check if JSON output is needed but JSON file doesn't exist - if json_output: - json_filename = filename.replace('.grobid.tei.xml', '.json') - # Expand ~ to home directory before checking file existence - json_filename_expanded = os.path.expanduser(json_filename) - if not os.path.isfile(json_filename_expanded): - self.logger.info(f"JSON file {json_filename} does not exist, generating JSON from existing TEI...") - try: - converter: Any = TEI2LossyJSONConverter() - json_data = converter.convert_tei_file(filename, stream=False) - - if json_data: - self._write_atomic( - json_filename_expanded, - json.dumps(json_data, indent=2, ensure_ascii=False)) - self.logger.debug(f"Successfully created JSON file: {json_filename_expanded}") - else: - self.logger.warning(f"Failed to convert TEI to JSON for {filename}") - except Exception as e: - self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}") - - # Check if Markdown output is needed but Markdown file doesn't exist - if markdown_output: - markdown_filename = filename.replace('.grobid.tei.xml', '.md') - # Expand ~ to home directory before checking file existence - markdown_filename_expanded = os.path.expanduser(markdown_filename) - if not os.path.isfile(markdown_filename_expanded): - self.logger.info(f"Markdown file {markdown_filename} does not exist, generating Markdown from existing TEI...") - try: - from .format.TEI2Markdown import TEI2MarkdownConverter - converter = TEI2MarkdownConverter() - markdown_data = converter.convert_tei_file(filename) - - if markdown_data: - self._write_atomic(markdown_filename_expanded, markdown_data) - self.logger.debug(f"Successfully created Markdown file: {markdown_filename_expanded}") - else: - self.logger.warning(f"Failed to convert TEI to Markdown for {filename}") - except Exception as e: - self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}") - - continue - - # Documents that already failed are usually going to fail again - # as long as nothing changed, so --skip-errors lets a re-run go - # straight past them. See issue #119. - if not force and skip_errors: - previous_errors = self._find_error_files(filename) - if previous_errors: + try: + for input_file in input_files: + # An entry is either a path or an in-memory document (a named + # stream, as fed by the archive/s3 streaming). Either way it goes + # by its name here, and process_pdf returns that same name, so + # the results below land on the same output file. + input_name = input_file if isinstance(input_file, str) else self._document_name(input_file) + # check if TEI file is already produced + filename = self._output_file_name(input_name, input_path, output) + if not force and os.path.isfile(filename): self.logger.info( - f"{input_name} previously failed ({os.path.basename(previous_errors[0])}), " - f"skipping... (use --force to retry it)") + f"{filename} already exists, skipping... (use --force to reprocess pdf input files)") skipped_count += 1 - continue - - selected_process: Any = self.process_pdf - if service == 'processCitationList': - selected_process = self.process_txt - if verbose: - self.logger.info(f"Adding {input_name} to the queue") + # Check if JSON output is needed but JSON file doesn't exist + if json_output: + json_filename = filename.replace('.grobid.tei.xml', '.json') + # Expand ~ to home directory before checking file existence + json_filename_expanded = os.path.expanduser(json_filename) + if not os.path.isfile(json_filename_expanded): + self.logger.info(f"JSON file {json_filename} does not exist, generating JSON from existing TEI...") + try: + converter: Any = TEI2LossyJSONConverter() + json_data = converter.convert_tei_file(filename, stream=False) + + if json_data: + self._write_atomic( + json_filename_expanded, + json.dumps(json_data, indent=2, ensure_ascii=False)) + self.logger.debug(f"Successfully created JSON file: {json_filename_expanded}") + else: + self.logger.warning(f"Failed to convert TEI to JSON for {filename}") + except Exception as e: + self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}") + + # Check if Markdown output is needed but Markdown file doesn't exist + if markdown_output: + markdown_filename = filename.replace('.grobid.tei.xml', '.md') + # Expand ~ to home directory before checking file existence + markdown_filename_expanded = os.path.expanduser(markdown_filename) + if not os.path.isfile(markdown_filename_expanded): + self.logger.info(f"Markdown file {markdown_filename} does not exist, generating Markdown from existing TEI...") + try: + from .format.TEI2Markdown import TEI2MarkdownConverter + converter = TEI2MarkdownConverter() + markdown_data = converter.convert_tei_file(filename) + + if markdown_data: + self._write_atomic(markdown_filename_expanded, markdown_data) + self.logger.debug(f"Successfully created Markdown file: {markdown_filename_expanded}") + else: + self.logger.warning(f"Failed to convert TEI to Markdown for {filename}") + except Exception as e: + self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}") - r = executor.submit( - selected_process, - service, - input_file, - generate_ids, - consolidate_header, - consolidate_citations, - include_raw_citations, - include_raw_affiliations, - tei_coordinates, - segment_sentences, - flavor, - -1, - -1) + continue - results.append(r) + # Documents that already failed are usually going to fail again + # as long as nothing changed, so --skip-errors lets a re-run go + # straight past them. See issue #119. + if not force and skip_errors: + previous_errors = self._find_error_files(filename) + if previous_errors: + self.logger.info( + f"{input_name} previously failed ({os.path.basename(previous_errors[0])}), " + f"skipping... (use --force to retry it)") + skipped_count += 1 + continue + + selected_process: Any = self.process_pdf + if service == 'processCitationList': + selected_process = self.process_txt - for r in concurrent.futures.as_completed(results): - input_file, status, text = r.result() - filename = self._output_file_name(input_file, input_path, output) + if verbose: + self.logger.info(f"Adding {input_name} to the queue") - if status != 200 or text is None: - self.logger.error(f"Processing of {input_file} failed with error {status}: {text}") - error_count += 1 - # writing error file with suffixed error code - error_filename = self._error_file_name(filename, status) - try: - self._write_atomic(error_filename, text if text is not None else "") - self.logger.info(f"Error details written to {error_filename}") - # A previous run may have failed with a different status code; - # keep only the marker of this run so the error state of the - # document stays unambiguous (and skippable). - self._remove_error_files(filename, keep=error_filename) - except OSError as e: - self.logger.error(f"Failed to write error file {filename}: {str(e)}") - else: - # writing TEI file - try: - self._write_atomic(filename, text) - self.logger.debug(f"Successfully wrote TEI file: {filename}") - # Counted once the TEI is on disk: a failed write - # leaves no output and must not be reported as done - processed_count += 1 + r = executor.submit( + selected_process, + service, + input_file, + generate_ids, + consolidate_header, + consolidate_citations, + include_raw_citations, + include_raw_affiliations, + tei_coordinates, + segment_sentences, + flavor, + -1, + -1) - # The document no longer is in error: drop the marker left by - # a previous failed run so it is not skipped by --skip-errors. - self._remove_error_files(filename) + results.append(r) + for r in concurrent.futures.as_completed(results): + input_file, status, text = r.result() + filename = self._output_file_name(input_file, input_path, output) - # Convert to JSON if requested - if json_output: - try: - converter = TEI2LossyJSONConverter() - json_data = converter.convert_tei_file(filename, stream=False) + if status != 200 or text is None: + self.logger.error(f"Processing of {input_file} failed with error {status}: {text}") + error_count += 1 + # writing error file with suffixed error code + error_filename = self._error_file_name(filename, status) + try: + self._write_atomic(error_filename, text if text is not None else "") + self.logger.info(f"Error details written to {error_filename}") + # A previous run may have failed with a different status code; + # keep only the marker of this run so the error state of the + # document stays unambiguous (and skippable). + self._remove_error_files(filename, keep=error_filename) + except OSError as e: + self.logger.error(f"Failed to write error file {filename}: {str(e)}") + else: + # writing TEI file + try: + self._write_atomic(filename, text) + self.logger.debug(f"Successfully wrote TEI file: {filename}") + # Counted once the TEI is on disk: a failed write + # leaves no output and must not be reported as done + processed_count += 1 + + # The document no longer is in error: drop the marker left by + # a previous failed run so it is not skipped by --skip-errors. + self._remove_error_files(filename) + + + # Convert to JSON if requested + if json_output: + try: + converter = TEI2LossyJSONConverter() + json_data = converter.convert_tei_file(filename, stream=False) - if json_data: - json_filename = filename.replace('.grobid.tei.xml', '.json') - # Always write JSON file when TEI is written (respects --force behavior) - json_filename_expanded = os.path.expanduser(json_filename) - self._write_atomic( - json_filename_expanded, - json.dumps(json_data, indent=2, ensure_ascii=False)) - self.logger.debug(f"Successfully wrote JSON file: {json_filename_expanded}") - else: - self.logger.warning(f"Failed to convert TEI to JSON for {filename}") - except Exception as e: - self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}") + if json_data: + json_filename = filename.replace('.grobid.tei.xml', '.json') + # Always write JSON file when TEI is written (respects --force behavior) + json_filename_expanded = os.path.expanduser(json_filename) + self._write_atomic( + json_filename_expanded, + json.dumps(json_data, indent=2, ensure_ascii=False)) + self.logger.debug(f"Successfully wrote JSON file: {json_filename_expanded}") + else: + self.logger.warning(f"Failed to convert TEI to JSON for {filename}") + except Exception as e: + self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}") - # Convert to Markdown if requested - if markdown_output: - try: - from .format.TEI2Markdown import TEI2MarkdownConverter - converter = TEI2MarkdownConverter() - markdown_data = converter.convert_tei_file(filename) - - if markdown_data is not None: - markdown_filename = filename.replace('.grobid.tei.xml', '.md') - # Always write Markdown file when TEI is written (respects --force behavior) - markdown_filename_expanded = os.path.expanduser(markdown_filename) - self._write_atomic(markdown_filename_expanded, markdown_data) - self.logger.debug(f"Successfully wrote Markdown file: {markdown_filename_expanded}") - else: - self.logger.warning(f"Failed to convert TEI to Markdown for {filename}") - except Exception as e: - self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}") + # Convert to Markdown if requested + if markdown_output: + try: + from .format.TEI2Markdown import TEI2MarkdownConverter + converter = TEI2MarkdownConverter() + markdown_data = converter.convert_tei_file(filename) + + if markdown_data is not None: + markdown_filename = filename.replace('.grobid.tei.xml', '.md') + # Always write Markdown file when TEI is written (respects --force behavior) + markdown_filename_expanded = os.path.expanduser(markdown_filename) + self._write_atomic(markdown_filename_expanded, markdown_data) + self.logger.debug(f"Successfully wrote Markdown file: {markdown_filename_expanded}") + else: + self.logger.warning(f"Failed to convert TEI to Markdown for {filename}") + except Exception as e: + self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}") - except OSError as e: - self.logger.error(f"Failed to write TEI XML file {filename}: {str(e)}") - error_count += 1 + except OSError as e: + self.logger.error(f"Failed to write TEI XML file {filename}: {str(e)}") + error_count += 1 + except BaseException: + # Ctrl-C (or any error) while results are awaited: the pool is + # about to be joined, so its threads must not keep retrying + self._stop_workers(results) + raise + # Calculate batch statistics batch_runtime = time.time() - batch_start_time @@ -1956,7 +1983,8 @@ def _send_pdf( if status != 503: return (name, status, res.text) - self._wait_for_busy_server(name) + if not self._wait_for_busy_server(name): + return (name, 503, self.BUSY_SERVER_INTERRUPTED) def process_documents( self, @@ -2003,27 +2031,33 @@ def process_documents( self._warn_on_engine_concurrency(max(1, n)) # A pool of one is still a pool: n < 1 would make ThreadPoolExecutor raise + self._stopping.clear() with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, n)) as executor: - futures = [ - executor.submit( - self._process_named_pdf, - service, - name, - document, - generate_ids, - consolidate_header, - consolidate_citations, - include_raw_citations, - include_raw_affiliations, - tei_coordinates, - segment_sentences, - flavor - ) - for name, document in items - ] - # Collected in submission order rather than with as_completed(): - # in-memory documents have no filenames to be matched back on later. - return [future.result() for future in futures] + futures: list = [] + try: + for name, document in items: + futures.append(executor.submit( + self._process_named_pdf, + service, + name, + document, + generate_ids, + consolidate_header, + consolidate_citations, + include_raw_citations, + include_raw_affiliations, + tei_coordinates, + segment_sentences, + flavor + )) + # Collected in submission order rather than with as_completed(): + # in-memory documents have no filenames to be matched back on later. + return [future.result() for future in futures] + except BaseException: + # Ctrl-C while results are awaited: the pool is about to be + # joined, so its threads must not keep retrying a busy server + self._stop_workers(futures) + raise def _name_documents(self, documents: Any) -> list: """Pair every document with the name it will be known by. @@ -2105,7 +2139,8 @@ def process_txt( if status != 503: return (txt_file, status, res.text) - self._wait_for_busy_server(txt_file) + if not self._wait_for_busy_server(txt_file): + return (txt_file, 503, self.BUSY_SERVER_INTERRUPTED) def main() -> None: diff --git a/tests/test_busy_retry.py b/tests/test_busy_retry.py index 719ab3a..0f2927a 100644 --- a/tests/test_busy_retry.py +++ b/tests/test_busy_retry.py @@ -44,7 +44,7 @@ def test_each_attempt_uses_a_fresh_closed_stream(self): client = _client() post, calls = _busy_then_ok(3) with patch.object(GrobidClient, 'post', side_effect=post): - with patch('grobid_client.grobid_client.time.sleep') as sleep: + with patch.object(client._stopping, 'wait', return_value=False) as sleep: name, status, text = client.process_pdf('processFulltextDocument', b'%PDF-bytes') assert (status, text) == (200, 'ok') assert sleep.call_count == 3 @@ -60,7 +60,7 @@ def test_a_file_on_disk_is_reopened_each_time(self): with open(pdf, 'wb') as fh: fh.write(b'%PDF-disk') with patch.object(GrobidClient, 'post', side_effect=post): - with patch('grobid_client.grobid_client.time.sleep'): + with patch.object(client._stopping, 'wait', return_value=False): _, status, _ = client.process_pdf('processFulltextDocument', pdf) assert status == 200 assert all(h.closed for h in calls) @@ -69,7 +69,7 @@ def test_a_long_busy_spell_does_not_recurse(self): client = _client() post, calls = _busy_then_ok(5000) with patch.object(GrobidClient, 'post', side_effect=post): - with patch('grobid_client.grobid_client.time.sleep'): + with patch.object(client._stopping, 'wait', return_value=False): _, status, _ = client.process_pdf('processFulltextDocument', b'%PDF') assert status == 200 assert len(calls) == 5001 @@ -85,7 +85,7 @@ def post(url, files=None, data=None, headers=None, timeout=None): return answer with patch.object(GrobidClient, 'post', side_effect=post): - with patch('grobid_client.grobid_client.time.sleep'): + with patch.object(client._stopping, 'wait', return_value=False): _, status, text = client.process_pdf('processFulltextDocument', b'%PDF') assert status == 408 and 'timeout' in text.lower() @@ -103,7 +103,7 @@ def test_503_then_ok(self): post, calls = _busy_then_ok(2) with tempfile.TemporaryDirectory() as d: with patch.object(GrobidClient, 'post', side_effect=post): - with patch('grobid_client.grobid_client.time.sleep') as sleep: + with patch.object(client._stopping, 'wait', return_value=False) as sleep: name, status, text = client.process_txt( 'processCitationList', self._refs(d), False, False, False, False, False, False, False) assert (status, text) == (200, 'ok') diff --git a/tests/test_grobid_client.py b/tests/test_grobid_client.py index e6fc31f..589dc4e 100644 --- a/tests/test_grobid_client.py +++ b/tests/test_grobid_client.py @@ -407,8 +407,8 @@ def test_process_pdf_server_busy_retry(self, mock_post): with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'): with patch('grobid_client.grobid_client.GrobidClient._configure_logging'): with patch('builtins.open', mock_open()): - with patch('time.sleep') as mock_sleep: - client = GrobidClient(check_server=False) + client = GrobidClient(check_server=False) + with patch.object(client._stopping, 'wait', return_value=False) as mock_sleep: client.logger = Mock() result = client.process_pdf( @@ -423,7 +423,7 @@ def test_process_pdf_server_busy_retry(self, mock_post): segment_sentences=False ) - # Should have called sleep due to 503 response + # Should have waited once due to the 503 response mock_sleep.assert_called_once() assert result[1] == 200 @@ -1635,7 +1635,7 @@ def test_retry_after_503_resends_the_whole_document(self): client = self._client() post, sent = self._post_spy(statuses=(503, 200)) - with patch('time.sleep'): + with patch.object(client._stopping, 'wait', return_value=False): with patch.object(GrobidClient, 'post', side_effect=post): result = client.process_pdf( 'processFulltextDocument', self._named(self.PDF, 'busy.pdf')) @@ -1653,7 +1653,7 @@ def test_retry_after_503_reopens_a_file_on_disk(self): with open(path, 'wb') as f: f.write(self.PDF) - with patch('time.sleep'): + with patch.object(client._stopping, 'wait', return_value=False): with patch.object(GrobidClient, 'post', side_effect=post): result = client.process_pdf('processFulltextDocument', path) diff --git a/tests/test_interrupt_busy_server.py b/tests/test_interrupt_busy_server.py new file mode 100644 index 0000000..e61c9b5 --- /dev/null +++ b/tests/test_interrupt_busy_server.py @@ -0,0 +1,150 @@ +""" +Ctrl-C while the server is busy. + +A document the server answers 503 to is re-posted after ``sleep_time`` for as +long as the server stays busy. That retry ran in the pool's worker threads, which +the interrupted main thread then waited for: the client printed "Interrupted" +and kept posting, so a permanently busy server needed a second Ctrl-C (or a kill). +""" +import http.server +import json +import signal +import subprocess +import sys +import threading +import time +from unittest.mock import Mock, patch + +import pytest + +from grobid_client.grobid_client import GrobidClient + + +def _client(sleep_time=60): + with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'): + with patch('grobid_client.grobid_client.GrobidClient._configure_logging'): + client = GrobidClient(check_server=False) + client.logger = Mock() + client.config['sleep_time'] = sleep_time + return client + + +def _always_busy(): + calls = [] + + def post(url, files=None, data=None, headers=None, timeout=None): + calls.append(url) + return Mock(text='busy'), 503 + + return post, calls + + +def test_a_stopped_client_does_not_wait_for_a_busy_server(): + client = _client() + client._stopping.set() + post, calls = _always_busy() + with patch.object(GrobidClient, 'post', side_effect=post): + start = time.time() + name, status, text = client.process_pdf('processFulltextDocument', b'%PDF') + assert time.time() - start < 5 + assert (status, text) == (503, GrobidClient.BUSY_SERVER_INTERRUPTED) + assert len(calls) == 1 + + +def test_interrupting_a_batch_ends_the_busy_retries(tmp_path): + """KeyboardInterrupt in the main thread must not leave workers retrying.""" + client = _client(sleep_time=60) + post, calls = _always_busy() + pdf = tmp_path / 'busy.pdf' + pdf.write_bytes(b'%PDF') + + def interrupted(futures): + deadline = time.time() + 10 + while not calls and time.time() < deadline: + time.sleep(0.01) + raise KeyboardInterrupt + + with patch.object(GrobidClient, 'post', side_effect=post): + with patch('grobid_client.grobid_client.concurrent.futures.as_completed', side_effect=interrupted): + start = time.time() + with pytest.raises(KeyboardInterrupt): + client.process_batch('processFulltextDocument', [str(pdf)], str(tmp_path), str(tmp_path / 'out'), + 2, False, False, False, False, False, False, False, False) + assert time.time() - start < 10 # not the 60 s sleep_time + assert len(calls) == 1 + assert not (tmp_path / 'out').exists() # an interrupted document leaves no error marker + + +def test_a_new_batch_after_an_interrupt_retries_again(tmp_path): + client = _client(sleep_time=0) + client._stopping.set() # left over from an interrupted batch + answers = iter([503, 200]) + + def post(url, files=None, data=None, headers=None, timeout=None): + return Mock(text=''), next(answers) + + pdf = tmp_path / 'p.pdf' + pdf.write_bytes(b'%PDF') + with patch.object(GrobidClient, 'post', side_effect=post): + processed, errors, skipped = client.process_batch( + 'processFulltextDocument', [str(pdf)], str(tmp_path), str(tmp_path / 'out'), + 1, False, False, False, False, False, False, False, False) + assert (processed, errors) == (1, 0) + + +class _Busy(http.server.BaseHTTPRequestHandler): + posts = 0 + + def log_message(self, *args): + pass + + def do_GET(self): + self.send_response(200) + self.send_header('Content-Length', '4') + self.end_headers() + self.wfile.write(b'true') + + def do_POST(self): + self.rfile.read(int(self.headers.get('Content-Length', 0))) + type(self).posts += 1 + self.send_response(503) + self.send_header('Content-Length', '4') + self.end_headers() + self.wfile.write(b'busy') + + +@pytest.mark.skipif(sys.platform == 'win32', reason='needs POSIX signals') +def test_cli_stops_on_the_first_ctrl_c_against_a_busy_server(tmp_path): + httpd = http.server.ThreadingHTTPServer(('127.0.0.1', 0), _Busy) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + try: + url = f'http://127.0.0.1:{httpd.server_address[1]}' + (tmp_path / 'in').mkdir() + for name in ('a.pdf', 'b.pdf', 'c.pdf'): + (tmp_path / 'in' / name).write_bytes(b'%PDF') + config = tmp_path / 'config.json' + config.write_text(json.dumps({'grobid_server': url, 'sleep_time': 30})) + proc = subprocess.Popen( + [sys.executable, '-c', 'from grobid_client.grobid_client import main; main()', + '--config', str(config), '--input', str(tmp_path / 'in'), '--output', str(tmp_path / 'out'), + '--n', '2', 'processFulltextDocument'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, cwd=str(tmp_path)) + deadline = time.time() + 30 + while _Busy.posts < 2 and time.time() < deadline: + time.sleep(0.05) + assert _Busy.posts >= 2, 'the client never reached the server' + proc.send_signal(signal.SIGINT) + start = time.time() + try: + log, _ = proc.communicate(timeout=15) + except subprocess.TimeoutExpired: + proc.kill() + pytest.fail('still retrying the busy server 15 s after Ctrl-C') + assert proc.returncode == 130, log + assert time.time() - start < 15 + assert 'run the same command again' in log + # the documents interrupted while waiting are not recorded as failures + assert not list((tmp_path / 'out').glob('*_503.txt')) + finally: + httpd.shutdown() + httpd.server_close() From 38f95aaa9ec37d4a25bb8d7e68edaa74b58ff0ce Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Tue, 10 Nov 2026 20:01:52 +0100 Subject: [PATCH 2/7] Accept a server URL written with a trailing slash The service URL was the server plus "/api/", so --server https://host/ asked for https://host//api/isalive. A Hugging Face Space answers that with 400, and the client reported a server that was up as unavailable. Trailing slashes are now stripped when the service URL is built; a path prefix (https://host/grobid/) is kept. --- grobid_client/grobid_client.py | 6 +++++- tests/test_server_url.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/test_server_url.py diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index 0c6397e..c7449f4 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -2085,7 +2085,11 @@ def _name_documents(self, documents: Any) -> list: return items def get_server_url(self, service: str) -> str: - return self.config['grobid_server'] + "/api/" + service + # "https://host/" names the same server as "https://host", + # but host//api/isalive is a different path: some servers (a Hugging + # Face Space among them) answer it with 400, and the client then + # reported a server that is up as unavailable + return str(self.config['grobid_server']).rstrip('/') + "/api/" + service def process_txt( self, diff --git a/tests/test_server_url.py b/tests/test_server_url.py new file mode 100644 index 0000000..08adba5 --- /dev/null +++ b/tests/test_server_url.py @@ -0,0 +1,28 @@ +"""The server URL may be written with a trailing slash.""" +from unittest.mock import Mock, patch + +import pytest + +from grobid_client.grobid_client import GrobidClient + + +@pytest.mark.parametrize('server', [ + 'http://localhost:8070', 'http://localhost:8070/', 'http://localhost:8070//', +]) +def test_trailing_slashes_do_not_reach_the_path(server): + client = GrobidClient(grobid_server=server, check_server=False) + assert client.get_server_url('isalive') == 'http://localhost:8070/api/isalive' + assert client.get_server_url('processFulltextDocument') == \ + 'http://localhost:8070/api/processFulltextDocument' + + +def test_a_path_prefix_is_kept(): + client = GrobidClient(grobid_server='https://example.org/grobid/', check_server=False) + assert client.get_server_url('isalive') == 'https://example.org/grobid/api/isalive' + + +def test_server_check_uses_the_normalised_url(): + response = Mock(status_code=200) + with patch('grobid_client.grobid_client.requests.get', return_value=response) as get: + GrobidClient(grobid_server='http://localhost:8070/', check_server=True) + assert get.call_args[0][0] == 'http://localhost:8070/api/isalive' From 4cb6f3d59f32a64e14afd0db64feca0b3eb2f242 Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Tue, 10 Nov 2026 20:03:11 +0100 Subject: [PATCH 3/7] Stop and suggest --no-sign-request when an s3 zip needs credentials Missing AWS credentials were reported and suggested a fix only when they surfaced while listing a prefix or glob. A single s3:// zip is not listed but opened, and the opening error was logged as "Could not open archive", followed by "No eligible files found", and the command exited 0. A missing-credentials error now stops the run from the archive too, so the command line reports it, suggests --no-sign-request and exits 1. Any other error opening an archive is still logged, and that archive is skipped. --- grobid_client/grobid_client.py | 13 ++++++++++++- tests/test_s3_no_sign_request.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index c7449f4..c1da730 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -1316,6 +1316,11 @@ def _process_archive_core( try: kind, archive, member_names = self._open_archive(archive_path) except Exception as e: + # Without credentials no s3:// archive of the run can be read: + # stop, so that the caller can say so (and suggest + # --no-sign-request) rather than report nothing to process + if _is_missing_credentials(e): + raise self.logger.error(f"Could not open archive {archive_path}: {str(e)}") return 0, 0, 0, 0 @@ -2147,6 +2152,12 @@ def process_txt( return (txt_file, 503, self.BUSY_SERVER_INTERRUPTED) +def _is_missing_credentials(error: BaseException) -> bool: + """Tell botocore's "no AWS credentials" errors apart, without importing botocore.""" + return any(cls.__name__ in ("NoCredentialsError", "PartialCredentialsError") + for cls in type(error).__mro__) + + def main() -> None: # Basic logging setup for initialization only # The actual logging configuration will be done by GrobidClient based on config.json @@ -2400,7 +2411,7 @@ def main() -> None: sys.exit(130) except Exception as e: logger.error(f"Processing failed: {str(e)}") - if type(e).__name__ == "NoCredentialsError" and not args.no_sign_request: + if _is_missing_credentials(e) and not args.no_sign_request: logger.error("To read a public s3:// bucket without AWS credentials, add --no-sign-request") sys.exit(1) diff --git a/tests/test_s3_no_sign_request.py b/tests/test_s3_no_sign_request.py index b44579e..14eeed7 100644 --- a/tests/test_s3_no_sign_request.py +++ b/tests/test_s3_no_sign_request.py @@ -122,3 +122,33 @@ def test_cli_suggests_the_option_on_missing_credentials(monkeypatch, tmp_path): gc_module.main() assert exit_info.value.code == 1 assert any("--no-sign-request" in message for message in logged) + + +def test_a_single_zip_without_credentials_stops_the_run(no_credentials, tmp_path): + # It used to be logged as "Could not open archive", followed by "No + # eligible files found", and the command exited 0 without the hint. + client = _client() + from botocore.exceptions import NoCredentialsError + with pytest.raises(NoCredentialsError): + client.process_paths("processFulltextDocument", [f"s3://{BUCKET}/corpus/zipfiles/0000.zip"], + output=str(tmp_path / "out")) + + +def test_cli_on_a_single_zip_without_credentials(no_credentials, monkeypatch, tmp_path, capsys): + monkeypatch.setattr("sys.argv", ["grobid_client", "--input", f"s3://{BUCKET}/corpus/zipfiles/0000.zip", + "--output", str(tmp_path / "out"), "processFulltextDocument"]) + with patch.object(GrobidClient, "_test_server_connection", return_value=(True, 200)): + with pytest.raises(SystemExit) as exit_info: + gc_module.main() + assert exit_info.value.code == 1 + assert "--no-sign-request" in capsys.readouterr().err + + +def test_other_archive_errors_are_still_logged_and_skipped(tmp_path): + client = _client() + broken = tmp_path / "broken.zip" + broken.write_bytes(b"not a zip") + assert client._process_archive_core( + "processFulltextDocument", str(broken), str(tmp_path / "out"), 1, False, False, False, + False, False, False, False, False, False, None, False, False) == (0, 0, 0, 0) + assert "Could not open archive" in client.logger.error.call_args[0][0] From fd76062bea6c7de9367d8e7546f1c877d9b536f2 Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Tue, 10 Nov 2026 20:07:05 +0100 Subject: [PATCH 4/7] Give each input of a run its own output file Two inputs of one run could map to the same output: paper.pdf and paper.PDF in one directory, two directories listed together that both hold x.pdf, or two archives that both hold doc.pdf. The second document overwrote the first, while the summary counted both as processed. For archives it was skipped as "already existed". Each output file is now claimed by the first input of the run that maps to it. A later input gets its extension or its archive's name added to the stem (paper.pdf.grobid.tei.xml, doc.batch1.grobid.tei.xml), with -2, -3... if that is taken too, and a warning names both inputs. Inputs are met in the same order on every run, so a rerun finds the same names. Archive entries are claimed before the resume check, so a renamed entry that is finished is skipped without being read. Inputs that do not collide keep their names. --- Readme.md | 3 + grobid_client/grobid_client.py | 94 ++++++++++++++++++--- tests/test_output_collisions.py | 139 ++++++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 9 deletions(-) create mode 100644 tests/test_output_collisions.py diff --git a/Readme.md b/Readme.md index ef80a29..64af89a 100644 --- a/Readme.md +++ b/Readme.md @@ -193,6 +193,9 @@ grobid_client --input "~/data/**/*.pdf" --output ~/results processFulltextDocu > - **Directories** are processed recursively and the output mirrors the input tree: `~/pdfs/2023/a.pdf` → > `~/results/2023/a.grobid.tei.xml` (subdirectories of `--output` are created as needed, and the JSON/Markdown > outputs and error files sit next to each TEI file). Archives keep their internal paths the same way. +> When two inputs of one run would get the same output (`paper.pdf` and `paper.PDF`, or two archives that both +> hold `doc.pdf`), the first one met keeps the name and the other gets its extension or its archive's name added +> (`paper.pdf.grobid.tei.xml`, `doc..grobid.tei.xml`); a warning names both. > - **Archives** (`.zip`, `.tar`, `.tar.gz`/`.tgz`, `.tar.bz2`/`.tbz2`) are streamed: eligible entries are read into > memory in chunks of `queue_size` and sent to GROBID straight from there — the archive is never fully decompressed and > nothing but the results in `--output` ever touches the disk. If `--output` is omitted, results go to a directory named diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index c1da730..5c820f1 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -145,6 +145,9 @@ def __init__( # Set when a run is being interrupted: the worker threads waiting to # re-post a document to a busy server give up instead of retrying self._stopping = threading.Event() + # Output file -> the input writing it in the current run (see + # _claim_output_file) + self._output_claims: dict = {} # Initialize config with defaults self.config = copy.deepcopy(self.DEFAULT_CONFIG) @@ -506,6 +509,46 @@ def _output_file_name( return str(filename) + def _claim_output_file(self, filename: str, source: str, suffix: str) -> str: + """Give an output file to one input of the run, and another name to the next. + + Two inputs of one run can map to the same output: paper.pdf and + paper.PDF in one directory, two directories or two archives listed + together that both hold paper.pdf. The second one used to overwrite + the first, or to be skipped as already done. The first input of the + run keeps the name; a later one gets ``suffix`` (its extension, or + the archive it comes from) added to the stem, e.g. + paper.PDF.grobid.tei.xml or doc.batch1.grobid.tei.xml, and -2, -3... + when that is taken too. Inputs are met in the same order on every + run, so a rerun finds the same names again. + """ + claims = self._output_claims + owner = claims.setdefault(os.path.abspath(filename), source) + if owner == source: + return filename + stem = self._output_stem(filename) + candidate = f"{stem}.{suffix or 'dup'}{self.TEI_SUFFIX}" + k = 2 + while claims.get(os.path.abspath(candidate), source) != source: + candidate = f"{stem}.{suffix or 'dup'}-{k}{self.TEI_SUFFIX}" + k += 1 + if os.path.abspath(candidate) not in claims: + claims[os.path.abspath(candidate)] = source + self.logger.warning( + f"{source} and {owner} would both be written to {filename}; " + f"writing {source} to {candidate}") + return candidate + + @staticmethod + def _input_source(input_file: Any, input_name: str, sources: Optional[dict]) -> Tuple[str, str]: + """Who an input is, for _claim_output_file: (identity, suffix for its stem).""" + if sources and input_name in sources: + return sources[input_name] + suffix = os.path.splitext(input_name)[1].lstrip(".") + if isinstance(input_file, str): + return os.path.abspath(input_name), suffix + return input_name, suffix + def _write_atomic(self, filename: str, text: str) -> None: """Write text to filename via a temp file in the same directory, then os.replace. @@ -670,6 +713,7 @@ def process_paths( of paths). Results from all inputs are aggregated into one summary. """ start_time = time.time() + self._output_claims = {} # Expanded once here: every path derived from it below (the exists # check, the error markers, the JSON/Markdown siblings) must agree @@ -1230,6 +1274,7 @@ def process_archive( ``output`` (``nested/y.pdf`` -> ``/nested/y.grobid.tei.xml``). """ start_time = time.time() + self._output_claims = {} if output is not None: output = os.path.expanduser(output) self._warn_on_consolidation_timeout(consolidate_citations) @@ -1254,7 +1299,8 @@ def process_archive( ) def _archive_member_is_done(self, member_name: str, output: str, force: bool, skip_errors: bool, - json_output: bool, markdown_output: bool) -> bool: + json_output: bool, markdown_output: bool, + source: Optional[Tuple[str, str]] = None) -> bool: """Tell, before reading it, whether an archive entry would be skipped. process_batch makes the same decision, but only once it holds the @@ -1263,12 +1309,16 @@ def _archive_member_is_done(self, member_name: str, output: str, force: bool, sk there but whose requested JSON or Markdown is missing is not done: process_batch regenerates those from the TEI. """ - if force: - return False name = self._safe_member_path("", member_name) if name is None: return False # _read_archive_member reports it filename = self._output_file_name(name, ".", output) + if source is not None: + # Claimed even when done or forced: an entry finished by an + # earlier run still owns its name in this one + filename = self._claim_output_file(filename, *source) + if force: + return False if os.path.isfile(filename): if json_output and not os.path.isfile(filename.replace(self.TEI_SUFFIX, '.json')): return False @@ -1346,12 +1396,20 @@ def _process_archive_core( # read straight from the archive into memory and posted from there. use_disk = service == 'processCitationList' + # Entries are told apart from same-named ones of another input of + # the run by the archive they come from + archive_label = self._archive_stem(os.path.basename(archive_path)) + + def member_source(member: str) -> Tuple[str, str]: + return f"{archive_path}::{member}", archive_label + # Entries a previous run finished are skipped up front, so that a # resumed run does not read (for s3, download) them again pending_members = [ name for name in eligible_members if not self._archive_member_is_done(name, output, force, skip_errors, - json_output, markdown_output) + json_output, markdown_output, + source=member_source(name)) ] skipped_files_count += total_files - len(pending_members) if skipped_files_count: @@ -1364,12 +1422,14 @@ def _process_archive_core( if not use_disk: documents = [] + sources = {} for member_name in chunk: if verbose: self.logger.info(f"Reading {member_name} from {archive_path}") document = self._read_archive_member(kind, archive, member_name) if document is not None: documents.append(document) + sources[document.name] = member_source(member_name) if not documents: continue @@ -1392,7 +1452,8 @@ def _process_archive_core( flavor, json_output, markdown_output, - skip_errors=skip_errors + skip_errors=skip_errors, + sources=sources ) processed_files_count += batch_processed errors_files_count += batch_errors @@ -1402,12 +1463,14 @@ def _process_archive_core( temp_dir = tempfile.mkdtemp(prefix="grobid_archive_") try: extracted_files = [] + extracted_sources = {} for member_name in chunk: if verbose: self.logger.info(f"Extracting {member_name} from {archive_path}") extracted = self._extract_archive_member(kind, archive, member_name, temp_dir) if extracted is not None: extracted_files.append(extracted) + extracted_sources[extracted] = member_source(member_name) if not extracted_files: continue @@ -1430,7 +1493,8 @@ def _process_archive_core( flavor, json_output, markdown_output, - skip_errors=skip_errors + skip_errors=skip_errors, + sources=extracted_sources ) processed_files_count += batch_processed errors_files_count += batch_errors @@ -1607,8 +1671,15 @@ def process_batch( flavor: Optional[str] = None, json_output: bool = False, markdown_output: bool = False, - skip_errors: bool = False + skip_errors: bool = False, + sources: Optional[dict] = None ) -> Tuple[int, int, int]: + """Process one batch of inputs, writing a TEI (or an error marker) for each. + + ``sources`` optionally maps an input's name to ``(identity, suffix)``, + for inputs whose name does not tell where they come from (archive + entries): see _claim_output_file. + """ batch_start_time = time.time() if verbose: self.logger.info(f"{len(input_files)} files to process in current batch") @@ -1622,6 +1693,8 @@ def process_batch( self._stopping.clear() with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, n)) as executor: results = [] + # Output file of every input sent, by the name process_pdf returns + planned = {} try: for input_file in input_files: # An entry is either a path or an in-memory document (a named @@ -1630,7 +1703,9 @@ def process_batch( # the results below land on the same output file. input_name = input_file if isinstance(input_file, str) else self._document_name(input_file) # check if TEI file is already produced - filename = self._output_file_name(input_name, input_path, output) + filename = self._claim_output_file( + self._output_file_name(input_name, input_path, output), + *self._input_source(input_file, input_name, sources)) if not force and os.path.isfile(filename): self.logger.info( f"{filename} already exists, skipping... (use --force to reprocess pdf input files)") @@ -1714,10 +1789,11 @@ def process_batch( -1) results.append(r) + planned[input_name] = filename for r in concurrent.futures.as_completed(results): input_file, status, text = r.result() - filename = self._output_file_name(input_file, input_path, output) + filename = planned.get(input_file) or self._output_file_name(input_file, input_path, output) if status != 200 or text is None: self.logger.error(f"Processing of {input_file} failed with error {status}: {text}") diff --git a/tests/test_output_collisions.py b/tests/test_output_collisions.py new file mode 100644 index 0000000..5007e88 --- /dev/null +++ b/tests/test_output_collisions.py @@ -0,0 +1,139 @@ +""" +Two inputs of one run that map to the same output file. + +paper.pdf and paper.PDF in one directory, or two archives listed together that +both hold doc.pdf, used to share one output: the second document overwrote the +first (and both were counted as processed), or was skipped as "already +existed". The first input of the run keeps the name, a later one is written +under its own: paper.PDF.grobid.tei.xml, doc..grobid.tei.xml. +""" +import os +import zipfile +from unittest.mock import Mock, patch + +from grobid_client.grobid_client import GrobidClient + + +def _client(): + with patch.object(GrobidClient, '_configure_logging'): + client = GrobidClient(check_server=False) + client.logger = Mock() + return client + + +class _Server: + """Answers every document with a TEI naming the bytes it was sent.""" + + def __init__(self): + self.posted = [] + + def post(self, url, files=None, data=None, headers=None, timeout=None): + if files: + content = files['input'][1].read().decode() + else: + content = '|'.join(data['citations']) + self.posted.append(content) + return Mock(text=f'{content}'), 200 + + +def _run(client, inputs, out, service='processFulltextDocument', **kwargs): + server = _Server() + with patch.object(client, 'post', side_effect=server.post): + client.process_paths(service, inputs, output=str(out), n=2, force=False, **kwargs) + return server.posted + + +def _outputs(out): + return {os.path.relpath(os.path.join(root, f), out): open(os.path.join(root, f)).read() + for root, _, files in os.walk(out) for f in files} + + +def _zip(path, entries): + with zipfile.ZipFile(path, 'w') as zf: + for name, content in entries.items(): + zf.writestr(name, content) + return str(path) + + +def test_same_stem_different_extension_case(tmp_path): + src = tmp_path / 'in' + src.mkdir() + (src / 'paper.pdf').write_bytes(b'lower') + (src / 'paper.PDF').write_bytes(b'upper') + client = _client() + _run(client, [str(src)], tmp_path / 'out') + outputs = _outputs(tmp_path / 'out') + assert sorted(outputs.values()) == ['lower', 'upper'] + assert 'paper.grobid.tei.xml' in outputs + assert len(outputs) == 2 + assert any('would both be written' in c[0][0] for c in client.logger.warning.call_args_list) + + +def test_two_archives_with_the_same_entry(tmp_path): + a = _zip(tmp_path / 'batch0.zip', {'doc.pdf': 'zero', 'only0.pdf': 'o0'}) + b = _zip(tmp_path / 'batch1.zip', {'doc.pdf': 'one', 'only1.pdf': 'o1'}) + out = tmp_path / 'out' + _run(_client(), [a, b], out) + assert _outputs(out) == { + 'doc.grobid.tei.xml': 'zero', + 'doc.batch1.grobid.tei.xml': 'one', + 'only0.grobid.tei.xml': 'o0', + 'only1.grobid.tei.xml': 'o1', + } + # a rerun finds every document done, the renamed one included, and + # skips them all before reading them + client = _client() + with patch.object(client, '_read_archive_member') as read: + assert _run(client, [a, b], out) == [] + read.assert_not_called() + + +def test_a_resumed_run_finishes_only_the_missing_renamed_entry(tmp_path): + a = _zip(tmp_path / 'batch0.zip', {'doc.pdf': 'zero'}) + b = _zip(tmp_path / 'batch1.zip', {'doc.pdf': 'one'}) + out = tmp_path / 'out' + _run(_client(), [a, b], out) + os.remove(out / 'doc.batch1.grobid.tei.xml') + assert _run(_client(), [a, b], out) == ['one'] + assert _outputs(out)['doc.batch1.grobid.tei.xml'] == 'one' + assert _outputs(out)['doc.grobid.tei.xml'] == 'zero' + + +def test_two_directories_with_the_same_file(tmp_path): + for name in ('dirA', 'dirB'): + (tmp_path / name).mkdir() + (tmp_path / name / 'x.pdf').write_bytes(name.encode()) + out = tmp_path / 'out' + _run(_client(), [str(tmp_path / 'dirA'), str(tmp_path / 'dirB')], out) + assert _outputs(out) == {'x.grobid.tei.xml': 'dirA', + 'x.pdf.grobid.tei.xml': 'dirB'} + + +def test_citation_lists_from_two_archives(tmp_path): + # these go through a temporary directory rather than memory + a = _zip(tmp_path / 'a.zip', {'refs.txt': 'ref a\n'}) + b = _zip(tmp_path / 'b.zip', {'refs.txt': 'ref b\n'}) + out = tmp_path / 'out' + _run(_client(), [a, b], out, service='processCitationList') + assert _outputs(out) == {'refs.grobid.tei.xml': 'ref a', + 'refs.b.grobid.tei.xml': 'ref b'} + + +def test_a_taken_new_name_gets_a_number(tmp_path): + for i, name in enumerate(('x.zip', 'sub/x.zip', 'other/x.zip')): + (tmp_path / name).parent.mkdir(exist_ok=True) + _zip(tmp_path / name, {'doc.pdf': str(i)}) + out = tmp_path / 'out' + _run(_client(), [str(tmp_path / 'x.zip'), str(tmp_path / 'sub/x.zip'), str(tmp_path / 'other/x.zip')], out) + assert _outputs(out) == {'doc.grobid.tei.xml': '0', + 'doc.x.grobid.tei.xml': '1', + 'doc.x-2.grobid.tei.xml': '2'} + + +def test_inputs_that_do_not_collide_keep_their_names(tmp_path): + (tmp_path / 'in' / 'sub').mkdir(parents=True) + (tmp_path / 'in' / 'a.pdf').write_bytes(b'a') + (tmp_path / 'in' / 'sub' / 'a.pdf').write_bytes(b'sub a') + out = tmp_path / 'out' + _run(_client(), [str(tmp_path / 'in')], out) + assert sorted(_outputs(out)) == ['a.grobid.tei.xml', os.path.join('sub', 'a.grobid.tei.xml')] From 6835fd4b48fd15d41839f79485aafb0d0cd84e55 Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Tue, 10 Nov 2026 20:07:52 +0100 Subject: [PATCH 5/7] ping(): report an unreachable server instead of raising ping() is documented as returning (is_up, status), but on a refused connection, a DNS failure or a timeout it let the constructor's ServerUnavailableException through. It now returns (False, 0) in that case. The constructor still raises when it checks the server. --- grobid_client/grobid_client.py | 9 +++++++-- tests/test_server_check.py | 13 +++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index 5c820f1..6fdf5c3 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -647,9 +647,14 @@ def _remove_error_files(self, tei_filename: str, keep: Optional[str] = None) -> def ping(self) -> Tuple[bool, int]: """ Check the Grobid service. Returns True if the service is up. - In addition, returns also the status code. + In addition, returns also the status code, which is 0 when the + server could not be reached at all (connection refused, DNS, timeout): + a server that is down is an answer to a ping, not an exception. """ - return self._test_server_connection() + try: + return self._test_server_connection() + except ServerUnavailableException: + return False, 0 def process( self, diff --git a/tests/test_server_check.py b/tests/test_server_check.py index a0afef5..0df8bd9 100644 --- a/tests/test_server_check.py +++ b/tests/test_server_check.py @@ -51,3 +51,16 @@ def test_cli_exits_on_a_misconfigured_server(tmp_path): main() assert exc.value.code == 1 pp.assert_not_called() + + +def test_ping_reports_an_unreachable_server_instead_of_raising(): + client = GrobidClient(check_server=False, grobid_server='http://127.0.0.1:9') + with patch('grobid_client.grobid_client.requests.get', + side_effect=requests.exceptions.ConnectionError('refused')): + assert client.ping() == (False, 0) + + +def test_ping_reports_a_bad_status(): + client = GrobidClient(check_server=False) + with _get(503): + assert client.ping() == (False, 503) From 6397a0d23fbebe13ce8e387c9f93a31833c48a43 Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Tue, 10 Nov 2026 20:08:45 +0100 Subject: [PATCH 6/7] Treat an empty 200 answer as an error A 200 with an empty body was written out as a 0-byte TEI file. Every later run then took it for a finished document, so the document was never retried, and --json produced nothing for it. process_pdf and process_txt now return such an answer as a 500 with an explanatory text. The CLI records it as _500.txt and retries it on the next run. --- grobid_client/grobid_client.py | 17 +++++++++-- tests/test_empty_answer.py | 52 ++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 tests/test_empty_answer.py diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index 6fdf5c3..e7c1bdf 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -241,6 +241,19 @@ def _wait_for_busy_server(self, file_path: str) -> bool: self.logger.warning(f"Server busy (503), retrying {file_path} after {self.config['sleep_time']} seconds") return not self._stopping.wait(self.config["sleep_time"]) + EMPTY_ANSWER = "The server answered 200 with an empty body" + + def _checked_answer(self, name: str, status: int, text: Optional[str]) -> Tuple[str, int, Optional[str]]: + """Turn a 200 without content into an error. + + Written out as is, it became an empty TEI file, which every later run + took for a finished document: the document was never retried. + """ + if status == 200 and not (text or "").strip(): + self.logger.error(f"Empty answer from the server for {name}") + return (name, 500, self.EMPTY_ANSWER) + return (name, status, text) + def _stop_workers(self, futures: list) -> None: """Called when a batch is interrupted: drop queued work, end busy-server retries.""" self._stopping.set() @@ -2068,7 +2081,7 @@ def _send_pdf( pdf_handle.close() if status != 503: - return (name, status, res.text) + return self._checked_answer(name, status, res.text) if not self._wait_for_busy_server(name): return (name, 503, self.BUSY_SERVER_INTERRUPTED) @@ -2228,7 +2241,7 @@ def process_txt( return self._handle_unexpected_error(txt_file, e) if status != 503: - return (txt_file, status, res.text) + return self._checked_answer(txt_file, status, res.text) if not self._wait_for_busy_server(txt_file): return (txt_file, 503, self.BUSY_SERVER_INTERRUPTED) diff --git a/tests/test_empty_answer.py b/tests/test_empty_answer.py new file mode 100644 index 0000000..4a74882 --- /dev/null +++ b/tests/test_empty_answer.py @@ -0,0 +1,52 @@ +"""A 200 without content is an error, not an empty TEI that blocks every rerun.""" +import os +from unittest.mock import Mock, patch + +import pytest + +from grobid_client.grobid_client import GrobidClient + + +def _client(): + with patch.object(GrobidClient, '_configure_logging'): + client = GrobidClient(check_server=False) + client.logger = Mock() + return client + + +@pytest.mark.parametrize('body', ['', ' \n']) +def test_process_pdf_reports_an_empty_answer(body): + client = _client() + with patch.object(client, 'post', return_value=(Mock(text=body), 200)): + name, status, text = client.process_pdf('processFulltextDocument', b'%PDF') + assert (status, text) == (500, GrobidClient.EMPTY_ANSWER) + + +def test_process_txt_reports_an_empty_answer(tmp_path): + refs = tmp_path / 'refs.txt' + refs.write_text('a ref\n') + client = _client() + with patch.object(client, 'post', return_value=(Mock(text=''), 200)): + _, status, _ = client.process_txt('processCitationList', str(refs), False, False, False, False, + False, False, False) + assert status == 500 + + +def test_an_empty_answer_leaves_a_marker_and_is_retried(tmp_path): + src = tmp_path / 'in' + src.mkdir() + (src / 'p.pdf').write_bytes(b'%PDF') + out = tmp_path / 'out' + answers = iter([(Mock(text=''), 200), (Mock(text='ok'), 200)]) + client = _client() + with patch.object(client, 'post', side_effect=lambda *a, **k: next(answers)): + client.process_paths('processFulltextDocument', [str(src)], output=str(out), force=False) + assert sorted(os.listdir(out)) == ['p_500.txt'] + client.process_paths('processFulltextDocument', [str(src)], output=str(out), force=False) + assert sorted(os.listdir(out)) == ['p.grobid.tei.xml'] + + +def test_a_real_answer_is_untouched(): + client = _client() + with patch.object(client, 'post', return_value=(Mock(text=''), 200)): + assert client.process_pdf('processFulltextDocument', b'%PDF')[1:] == (200, '') From 7b36f77656a57da328f2ad97404b8bdebc291001 Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Tue, 10 Nov 2026 20:10:01 +0100 Subject: [PATCH 7/7] Flush progress lines so a log file keeps them in order The "Found N file(s)" lines and the summary are printed to stdout and the logs go to stderr. Redirected to a file, stdout is block-buffered, so in `grobid_client ... > run.log 2>&1` the "Found" line landed after every error of the run. The prints now flush. --- grobid_client/grobid_client.py | 20 ++++++++++---------- tests/test_end_to_end.py | 17 +++++++++++++++++ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index e7c1bdf..279e713 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -801,7 +801,7 @@ def process_paths( # Local files gathered from directories and/or loose glob matches if fs_files: - print(f"Found {len(fs_files)} local file(s) to process") + print(f"Found {len(fs_files)} local file(s) to process", flush=True) for base_dir, group_files in fs_groups: bp, be, bs = self._run_file_batches( service, group_files, base_dir, output, n, @@ -1024,15 +1024,15 @@ def _print_processing_summary( docs_per_second = processed / runtime if runtime > 0 else 0 seconds_per_doc = runtime / processed if processed > 0 else 0 - print(f"Processing completed: {processed} out of {total} files processed") - print(f"Errors: {errors} out of {total} files processed") + print(f"Processing completed: {processed} out of {total} files processed", flush=True) + print(f"Errors: {errors} out of {total} files processed", flush=True) if skipped > 0: reason = "already existed or previously failed" if skip_errors else "already existed" - print(f"Skipped: {skipped} out of {total} files ({reason}, use --force to reprocess)") + print(f"Skipped: {skipped} out of {total} files ({reason}, use --force to reprocess)", flush=True) - print(f"⏱️ Total runtime: {runtime:.2f} seconds") - print(f"🚀 Speed: {docs_per_second:.2f} documents/second") - print(f" Throughput: {seconds_per_doc:.2f} seconds/document") + print(f"⏱️ Total runtime: {runtime:.2f} seconds", flush=True) + print(f"🚀 Speed: {docs_per_second:.2f} documents/second", flush=True) + print(f" Throughput: {seconds_per_doc:.2f} seconds/document", flush=True) def _run_file_batches( self, @@ -1407,7 +1407,7 @@ def _process_archive_core( self.logger.warning(f"No eligible files found in archive {archive_path}") return 0, 0, 0, 0 - print(f"Found {total_files} file(s) to process in {archive_path}") + print(f"Found {total_files} file(s) to process in {archive_path}", flush=True) # Citation lists go through process_txt, which reads from a path, so # they still take the extract-to-a-temp-dir route. Everything else is @@ -1564,7 +1564,7 @@ def _process_remote_files( output = "." queue_size = self._effective_queue_size(n) - print(f"Found {total} remote file(s) to process") + print(f"Found {total} remote file(s) to process", flush=True) # Objects are named relative to their common prefix so that the # output mirrors the listed "directory", as it does for local ones prefix = self._s3_common_prefix(uris) @@ -2510,7 +2510,7 @@ def main() -> None: sys.exit(1) runtime = round(time.time() - start_time, 3) - print(f"Processing completed in {runtime} seconds") + print(f"Processing completed in {runtime} seconds", flush=True) if __name__ == "__main__": diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index b9beedd..703b22a 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -160,3 +160,20 @@ def test_without_output_results_sit_next_to_the_inputs(server, pdf_tree, tmp_pat assert result.returncode == 0, result.stderr assert (pdf_tree / 'a' / 'x.grobid.tei.xml').exists() assert (pdf_tree / 'a' / 'deep' / 'y.grobid.tei.xml').exists() + + +def test_progress_lines_come_before_the_errors_in_a_log_file(server, pdf_tree, tmp_path): + # stdout and stderr into one file, as with `grobid_client ... > run.log 2>&1`: + # "Found N file(s)" used to sit in stdout's buffer and land at the very end + log = tmp_path / 'run.log' + with open(log, 'w') as fh: + subprocess.run( + [sys.executable, '-c', 'from grobid_client.grobid_client import main; main()', + '--server', server, '--input', str(pdf_tree), '--output', str(tmp_path / 'out'), + 'processFulltextDocument'], + stdout=fh, stderr=subprocess.STDOUT, cwd=tmp_path, timeout=120) + lines = log.read_text().splitlines() + found = next(i for i, line in enumerate(lines) if line.startswith('Found 5 local file(s)')) + error = next(i for i, line in enumerate(lines) if 'broken.pdf failed' in line) + summary = next(i for i, line in enumerate(lines) if line.startswith('Processing completed: 4 out of 5')) + assert found < error < summary