diff --git a/Readme.md b/Readme.md index ef80a29..64af89a 100644 --- a/Readme.md +++ b/Readme.md @@ -193,6 +193,9 @@ grobid_client --input "~/data/**/*.pdf" --output ~/results processFulltextDocu > - **Directories** are processed recursively and the output mirrors the input tree: `~/pdfs/2023/a.pdf` → > `~/results/2023/a.grobid.tei.xml` (subdirectories of `--output` are created as needed, and the JSON/Markdown > outputs and error files sit next to each TEI file). Archives keep their internal paths the same way. +> When two inputs of one run would get the same output (`paper.pdf` and `paper.PDF`, or two archives that both +> hold `doc.pdf`), the first one met keeps the name and the other gets its extension or its archive's name added +> (`paper.pdf.grobid.tei.xml`, `doc..grobid.tei.xml`); a warning names both. > - **Archives** (`.zip`, `.tar`, `.tar.gz`/`.tgz`, `.tar.bz2`/`.tbz2`) are streamed: eligible entries are read into > memory in chunks of `queue_size` and sent to GROBID straight from there — the archive is never fully decompressed and > nothing but the results in `--output` ever touches the disk. If `--output` is omitted, results go to a directory named diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index 2bd99fd..279e713 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -34,6 +34,7 @@ import shutil import tarfile import tempfile +import threading import zipfile from typing import Any, BinaryIO, Callable, List, Optional, Tuple, Union import copy @@ -141,6 +142,12 @@ def __init__( # --no-sign-request) instead of with the default credential chain self.s3_no_sign_request = s3_no_sign_request self._s3_client_instance: Any = None + # Set when a run is being interrupted: the worker threads waiting to + # re-post a document to a busy server give up instead of retrying + self._stopping = threading.Event() + # Output file -> the input writing it in the current run (see + # _claim_output_file) + self._output_claims: dict = {} # Initialize config with defaults self.config = copy.deepcopy(self.DEFAULT_CONFIG) @@ -220,10 +227,38 @@ def _warn_on_consolidation_timeout(self, consolidate_citations: bool) -> None: f"(2-3 minutes is recommended)." ) - def _wait_for_busy_server(self, file_path: str) -> None: - """Pause before re-posting a document the server answered 503 to.""" + BUSY_SERVER_INTERRUPTED = "Interrupted while waiting for the busy server" + + def _wait_for_busy_server(self, file_path: str) -> bool: + """Pause before re-posting a document the server answered 503 to. + + Returns False, at once, when the run is being interrupted: a server + that stays busy would otherwise keep the worker threads retrying, and + the process alive, long after Ctrl-C. + """ + if self._stopping.is_set(): + return False self.logger.warning(f"Server busy (503), retrying {file_path} after {self.config['sleep_time']} seconds") - time.sleep(self.config["sleep_time"]) + return not self._stopping.wait(self.config["sleep_time"]) + + EMPTY_ANSWER = "The server answered 200 with an empty body" + + def _checked_answer(self, name: str, status: int, text: Optional[str]) -> Tuple[str, int, Optional[str]]: + """Turn a 200 without content into an error. + + Written out as is, it became an empty TEI file, which every later run + took for a finished document: the document was never retried. + """ + if status == 200 and not (text or "").strip(): + self.logger.error(f"Empty answer from the server for {name}") + return (name, 500, self.EMPTY_ANSWER) + return (name, status, text) + + def _stop_workers(self, futures: list) -> None: + """Called when a batch is interrupted: drop queued work, end busy-server retries.""" + self._stopping.set() + for future in futures: + future.cancel() def _handle_request_error( self, file_path: str, error: Exception, error_type: str = "Request" @@ -487,6 +522,46 @@ def _output_file_name( return str(filename) + def _claim_output_file(self, filename: str, source: str, suffix: str) -> str: + """Give an output file to one input of the run, and another name to the next. + + Two inputs of one run can map to the same output: paper.pdf and + paper.PDF in one directory, two directories or two archives listed + together that both hold paper.pdf. The second one used to overwrite + the first, or to be skipped as already done. The first input of the + run keeps the name; a later one gets ``suffix`` (its extension, or + the archive it comes from) added to the stem, e.g. + paper.PDF.grobid.tei.xml or doc.batch1.grobid.tei.xml, and -2, -3... + when that is taken too. Inputs are met in the same order on every + run, so a rerun finds the same names again. + """ + claims = self._output_claims + owner = claims.setdefault(os.path.abspath(filename), source) + if owner == source: + return filename + stem = self._output_stem(filename) + candidate = f"{stem}.{suffix or 'dup'}{self.TEI_SUFFIX}" + k = 2 + while claims.get(os.path.abspath(candidate), source) != source: + candidate = f"{stem}.{suffix or 'dup'}-{k}{self.TEI_SUFFIX}" + k += 1 + if os.path.abspath(candidate) not in claims: + claims[os.path.abspath(candidate)] = source + self.logger.warning( + f"{source} and {owner} would both be written to {filename}; " + f"writing {source} to {candidate}") + return candidate + + @staticmethod + def _input_source(input_file: Any, input_name: str, sources: Optional[dict]) -> Tuple[str, str]: + """Who an input is, for _claim_output_file: (identity, suffix for its stem).""" + if sources and input_name in sources: + return sources[input_name] + suffix = os.path.splitext(input_name)[1].lstrip(".") + if isinstance(input_file, str): + return os.path.abspath(input_name), suffix + return input_name, suffix + def _write_atomic(self, filename: str, text: str) -> None: """Write text to filename via a temp file in the same directory, then os.replace. @@ -585,9 +660,14 @@ def _remove_error_files(self, tei_filename: str, keep: Optional[str] = None) -> def ping(self) -> Tuple[bool, int]: """ Check the Grobid service. Returns True if the service is up. - In addition, returns also the status code. + In addition, returns also the status code, which is 0 when the + server could not be reached at all (connection refused, DNS, timeout): + a server that is down is an answer to a ping, not an exception. """ - return self._test_server_connection() + try: + return self._test_server_connection() + except ServerUnavailableException: + return False, 0 def process( self, @@ -651,6 +731,7 @@ def process_paths( of paths). Results from all inputs are aggregated into one summary. """ start_time = time.time() + self._output_claims = {} # Expanded once here: every path derived from it below (the exists # check, the error markers, the JSON/Markdown siblings) must agree @@ -720,7 +801,7 @@ def process_paths( # Local files gathered from directories and/or loose glob matches if fs_files: - print(f"Found {len(fs_files)} local file(s) to process") + print(f"Found {len(fs_files)} local file(s) to process", flush=True) for base_dir, group_files in fs_groups: bp, be, bs = self._run_file_batches( service, group_files, base_dir, output, n, @@ -943,15 +1024,15 @@ def _print_processing_summary( docs_per_second = processed / runtime if runtime > 0 else 0 seconds_per_doc = runtime / processed if processed > 0 else 0 - print(f"Processing completed: {processed} out of {total} files processed") - print(f"Errors: {errors} out of {total} files processed") + print(f"Processing completed: {processed} out of {total} files processed", flush=True) + print(f"Errors: {errors} out of {total} files processed", flush=True) if skipped > 0: reason = "already existed or previously failed" if skip_errors else "already existed" - print(f"Skipped: {skipped} out of {total} files ({reason}, use --force to reprocess)") + print(f"Skipped: {skipped} out of {total} files ({reason}, use --force to reprocess)", flush=True) - print(f"⏱️ Total runtime: {runtime:.2f} seconds") - print(f"🚀 Speed: {docs_per_second:.2f} documents/second") - print(f" Throughput: {seconds_per_doc:.2f} seconds/document") + print(f"⏱️ Total runtime: {runtime:.2f} seconds", flush=True) + print(f"🚀 Speed: {docs_per_second:.2f} documents/second", flush=True) + print(f" Throughput: {seconds_per_doc:.2f} seconds/document", flush=True) def _run_file_batches( self, @@ -1211,6 +1292,7 @@ def process_archive( ``output`` (``nested/y.pdf`` -> ``/nested/y.grobid.tei.xml``). """ start_time = time.time() + self._output_claims = {} if output is not None: output = os.path.expanduser(output) self._warn_on_consolidation_timeout(consolidate_citations) @@ -1235,7 +1317,8 @@ def process_archive( ) def _archive_member_is_done(self, member_name: str, output: str, force: bool, skip_errors: bool, - json_output: bool, markdown_output: bool) -> bool: + json_output: bool, markdown_output: bool, + source: Optional[Tuple[str, str]] = None) -> bool: """Tell, before reading it, whether an archive entry would be skipped. process_batch makes the same decision, but only once it holds the @@ -1244,12 +1327,16 @@ def _archive_member_is_done(self, member_name: str, output: str, force: bool, sk there but whose requested JSON or Markdown is missing is not done: process_batch regenerates those from the TEI. """ - if force: - return False name = self._safe_member_path("", member_name) if name is None: return False # _read_archive_member reports it filename = self._output_file_name(name, ".", output) + if source is not None: + # Claimed even when done or forced: an entry finished by an + # earlier run still owns its name in this one + filename = self._claim_output_file(filename, *source) + if force: + return False if os.path.isfile(filename): if json_output and not os.path.isfile(filename.replace(self.TEI_SUFFIX, '.json')): return False @@ -1297,6 +1384,11 @@ def _process_archive_core( try: kind, archive, member_names = self._open_archive(archive_path) except Exception as e: + # Without credentials no s3:// archive of the run can be read: + # stop, so that the caller can say so (and suggest + # --no-sign-request) rather than report nothing to process + if _is_missing_credentials(e): + raise self.logger.error(f"Could not open archive {archive_path}: {str(e)}") return 0, 0, 0, 0 @@ -1315,19 +1407,27 @@ def _process_archive_core( self.logger.warning(f"No eligible files found in archive {archive_path}") return 0, 0, 0, 0 - print(f"Found {total_files} file(s) to process in {archive_path}") + print(f"Found {total_files} file(s) to process in {archive_path}", flush=True) # Citation lists go through process_txt, which reads from a path, so # they still take the extract-to-a-temp-dir route. Everything else is # read straight from the archive into memory and posted from there. use_disk = service == 'processCitationList' + # Entries are told apart from same-named ones of another input of + # the run by the archive they come from + archive_label = self._archive_stem(os.path.basename(archive_path)) + + def member_source(member: str) -> Tuple[str, str]: + return f"{archive_path}::{member}", archive_label + # Entries a previous run finished are skipped up front, so that a # resumed run does not read (for s3, download) them again pending_members = [ name for name in eligible_members if not self._archive_member_is_done(name, output, force, skip_errors, - json_output, markdown_output) + json_output, markdown_output, + source=member_source(name)) ] skipped_files_count += total_files - len(pending_members) if skipped_files_count: @@ -1340,12 +1440,14 @@ def _process_archive_core( if not use_disk: documents = [] + sources = {} for member_name in chunk: if verbose: self.logger.info(f"Reading {member_name} from {archive_path}") document = self._read_archive_member(kind, archive, member_name) if document is not None: documents.append(document) + sources[document.name] = member_source(member_name) if not documents: continue @@ -1368,7 +1470,8 @@ def _process_archive_core( flavor, json_output, markdown_output, - skip_errors=skip_errors + skip_errors=skip_errors, + sources=sources ) processed_files_count += batch_processed errors_files_count += batch_errors @@ -1378,12 +1481,14 @@ def _process_archive_core( temp_dir = tempfile.mkdtemp(prefix="grobid_archive_") try: extracted_files = [] + extracted_sources = {} for member_name in chunk: if verbose: self.logger.info(f"Extracting {member_name} from {archive_path}") extracted = self._extract_archive_member(kind, archive, member_name, temp_dir) if extracted is not None: extracted_files.append(extracted) + extracted_sources[extracted] = member_source(member_name) if not extracted_files: continue @@ -1406,7 +1511,8 @@ def _process_archive_core( flavor, json_output, markdown_output, - skip_errors=skip_errors + skip_errors=skip_errors, + sources=extracted_sources ) processed_files_count += batch_processed errors_files_count += batch_errors @@ -1458,7 +1564,7 @@ def _process_remote_files( output = "." queue_size = self._effective_queue_size(n) - print(f"Found {total} remote file(s) to process") + print(f"Found {total} remote file(s) to process", flush=True) # Objects are named relative to their common prefix so that the # output mirrors the listed "directory", as it does for local ones prefix = self._s3_common_prefix(uris) @@ -1583,8 +1689,15 @@ def process_batch( flavor: Optional[str] = None, json_output: bool = False, markdown_output: bool = False, - skip_errors: bool = False + skip_errors: bool = False, + sources: Optional[dict] = None ) -> Tuple[int, int, int]: + """Process one batch of inputs, writing a TEI (or an error marker) for each. + + ``sources`` optionally maps an input's name to ``(identity, suffix)``, + for inputs whose name does not tell where they come from (archive + entries): see _claim_output_file. + """ batch_start_time = time.time() if verbose: self.logger.info(f"{len(input_files)} files to process in current batch") @@ -1595,172 +1708,185 @@ def process_batch( # we use ThreadPoolExecutor and not ProcessPoolExecutor because it is an I/O intensive process # A pool of one is still a pool: n < 1 would make ThreadPoolExecutor raise + self._stopping.clear() with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, n)) as executor: results = [] - for input_file in input_files: - # An entry is either a path or an in-memory document (a named - # stream, as fed by the archive/s3 streaming). Either way it goes - # by its name here, and process_pdf returns that same name, so - # the results below land on the same output file. - input_name = input_file if isinstance(input_file, str) else self._document_name(input_file) - # check if TEI file is already produced - filename = self._output_file_name(input_name, input_path, output) - if not force and os.path.isfile(filename): - self.logger.info( - f"{filename} already exists, skipping... (use --force to reprocess pdf input files)") - skipped_count += 1 - - # Check if JSON output is needed but JSON file doesn't exist - if json_output: - json_filename = filename.replace('.grobid.tei.xml', '.json') - # Expand ~ to home directory before checking file existence - json_filename_expanded = os.path.expanduser(json_filename) - if not os.path.isfile(json_filename_expanded): - self.logger.info(f"JSON file {json_filename} does not exist, generating JSON from existing TEI...") - try: - converter: Any = TEI2LossyJSONConverter() - json_data = converter.convert_tei_file(filename, stream=False) - - if json_data: - self._write_atomic( - json_filename_expanded, - json.dumps(json_data, indent=2, ensure_ascii=False)) - self.logger.debug(f"Successfully created JSON file: {json_filename_expanded}") - else: - self.logger.warning(f"Failed to convert TEI to JSON for {filename}") - except Exception as e: - self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}") - - # Check if Markdown output is needed but Markdown file doesn't exist - if markdown_output: - markdown_filename = filename.replace('.grobid.tei.xml', '.md') - # Expand ~ to home directory before checking file existence - markdown_filename_expanded = os.path.expanduser(markdown_filename) - if not os.path.isfile(markdown_filename_expanded): - self.logger.info(f"Markdown file {markdown_filename} does not exist, generating Markdown from existing TEI...") - try: - from .format.TEI2Markdown import TEI2MarkdownConverter - converter = TEI2MarkdownConverter() - markdown_data = converter.convert_tei_file(filename) - - if markdown_data: - self._write_atomic(markdown_filename_expanded, markdown_data) - self.logger.debug(f"Successfully created Markdown file: {markdown_filename_expanded}") - else: - self.logger.warning(f"Failed to convert TEI to Markdown for {filename}") - except Exception as e: - self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}") - - continue - - # Documents that already failed are usually going to fail again - # as long as nothing changed, so --skip-errors lets a re-run go - # straight past them. See issue #119. - if not force and skip_errors: - previous_errors = self._find_error_files(filename) - if previous_errors: + # Output file of every input sent, by the name process_pdf returns + planned = {} + try: + for input_file in input_files: + # An entry is either a path or an in-memory document (a named + # stream, as fed by the archive/s3 streaming). Either way it goes + # by its name here, and process_pdf returns that same name, so + # the results below land on the same output file. + input_name = input_file if isinstance(input_file, str) else self._document_name(input_file) + # check if TEI file is already produced + filename = self._claim_output_file( + self._output_file_name(input_name, input_path, output), + *self._input_source(input_file, input_name, sources)) + if not force and os.path.isfile(filename): self.logger.info( - f"{input_name} previously failed ({os.path.basename(previous_errors[0])}), " - f"skipping... (use --force to retry it)") + f"{filename} already exists, skipping... (use --force to reprocess pdf input files)") skipped_count += 1 - continue - - selected_process: Any = self.process_pdf - if service == 'processCitationList': - selected_process = self.process_txt - if verbose: - self.logger.info(f"Adding {input_name} to the queue") + # Check if JSON output is needed but JSON file doesn't exist + if json_output: + json_filename = filename.replace('.grobid.tei.xml', '.json') + # Expand ~ to home directory before checking file existence + json_filename_expanded = os.path.expanduser(json_filename) + if not os.path.isfile(json_filename_expanded): + self.logger.info(f"JSON file {json_filename} does not exist, generating JSON from existing TEI...") + try: + converter: Any = TEI2LossyJSONConverter() + json_data = converter.convert_tei_file(filename, stream=False) + + if json_data: + self._write_atomic( + json_filename_expanded, + json.dumps(json_data, indent=2, ensure_ascii=False)) + self.logger.debug(f"Successfully created JSON file: {json_filename_expanded}") + else: + self.logger.warning(f"Failed to convert TEI to JSON for {filename}") + except Exception as e: + self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}") + + # Check if Markdown output is needed but Markdown file doesn't exist + if markdown_output: + markdown_filename = filename.replace('.grobid.tei.xml', '.md') + # Expand ~ to home directory before checking file existence + markdown_filename_expanded = os.path.expanduser(markdown_filename) + if not os.path.isfile(markdown_filename_expanded): + self.logger.info(f"Markdown file {markdown_filename} does not exist, generating Markdown from existing TEI...") + try: + from .format.TEI2Markdown import TEI2MarkdownConverter + converter = TEI2MarkdownConverter() + markdown_data = converter.convert_tei_file(filename) + + if markdown_data: + self._write_atomic(markdown_filename_expanded, markdown_data) + self.logger.debug(f"Successfully created Markdown file: {markdown_filename_expanded}") + else: + self.logger.warning(f"Failed to convert TEI to Markdown for {filename}") + except Exception as e: + self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}") - r = executor.submit( - selected_process, - service, - input_file, - generate_ids, - consolidate_header, - consolidate_citations, - include_raw_citations, - include_raw_affiliations, - tei_coordinates, - segment_sentences, - flavor, - -1, - -1) + continue - results.append(r) + # Documents that already failed are usually going to fail again + # as long as nothing changed, so --skip-errors lets a re-run go + # straight past them. See issue #119. + if not force and skip_errors: + previous_errors = self._find_error_files(filename) + if previous_errors: + self.logger.info( + f"{input_name} previously failed ({os.path.basename(previous_errors[0])}), " + f"skipping... (use --force to retry it)") + skipped_count += 1 + continue + + selected_process: Any = self.process_pdf + if service == 'processCitationList': + selected_process = self.process_txt - for r in concurrent.futures.as_completed(results): - input_file, status, text = r.result() - filename = self._output_file_name(input_file, input_path, output) + if verbose: + self.logger.info(f"Adding {input_name} to the queue") - if status != 200 or text is None: - self.logger.error(f"Processing of {input_file} failed with error {status}: {text}") - error_count += 1 - # writing error file with suffixed error code - error_filename = self._error_file_name(filename, status) - try: - self._write_atomic(error_filename, text if text is not None else "") - self.logger.info(f"Error details written to {error_filename}") - # A previous run may have failed with a different status code; - # keep only the marker of this run so the error state of the - # document stays unambiguous (and skippable). - self._remove_error_files(filename, keep=error_filename) - except OSError as e: - self.logger.error(f"Failed to write error file {filename}: {str(e)}") - else: - # writing TEI file - try: - self._write_atomic(filename, text) - self.logger.debug(f"Successfully wrote TEI file: {filename}") - # Counted once the TEI is on disk: a failed write - # leaves no output and must not be reported as done - processed_count += 1 + r = executor.submit( + selected_process, + service, + input_file, + generate_ids, + consolidate_header, + consolidate_citations, + include_raw_citations, + include_raw_affiliations, + tei_coordinates, + segment_sentences, + flavor, + -1, + -1) - # The document no longer is in error: drop the marker left by - # a previous failed run so it is not skipped by --skip-errors. - self._remove_error_files(filename) + results.append(r) + planned[input_name] = filename + for r in concurrent.futures.as_completed(results): + input_file, status, text = r.result() + filename = planned.get(input_file) or self._output_file_name(input_file, input_path, output) - # Convert to JSON if requested - if json_output: - try: - converter = TEI2LossyJSONConverter() - json_data = converter.convert_tei_file(filename, stream=False) + if status != 200 or text is None: + self.logger.error(f"Processing of {input_file} failed with error {status}: {text}") + error_count += 1 + # writing error file with suffixed error code + error_filename = self._error_file_name(filename, status) + try: + self._write_atomic(error_filename, text if text is not None else "") + self.logger.info(f"Error details written to {error_filename}") + # A previous run may have failed with a different status code; + # keep only the marker of this run so the error state of the + # document stays unambiguous (and skippable). + self._remove_error_files(filename, keep=error_filename) + except OSError as e: + self.logger.error(f"Failed to write error file {filename}: {str(e)}") + else: + # writing TEI file + try: + self._write_atomic(filename, text) + self.logger.debug(f"Successfully wrote TEI file: {filename}") + # Counted once the TEI is on disk: a failed write + # leaves no output and must not be reported as done + processed_count += 1 + + # The document no longer is in error: drop the marker left by + # a previous failed run so it is not skipped by --skip-errors. + self._remove_error_files(filename) + + + # Convert to JSON if requested + if json_output: + try: + converter = TEI2LossyJSONConverter() + json_data = converter.convert_tei_file(filename, stream=False) - if json_data: - json_filename = filename.replace('.grobid.tei.xml', '.json') - # Always write JSON file when TEI is written (respects --force behavior) - json_filename_expanded = os.path.expanduser(json_filename) - self._write_atomic( - json_filename_expanded, - json.dumps(json_data, indent=2, ensure_ascii=False)) - self.logger.debug(f"Successfully wrote JSON file: {json_filename_expanded}") - else: - self.logger.warning(f"Failed to convert TEI to JSON for {filename}") - except Exception as e: - self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}") + if json_data: + json_filename = filename.replace('.grobid.tei.xml', '.json') + # Always write JSON file when TEI is written (respects --force behavior) + json_filename_expanded = os.path.expanduser(json_filename) + self._write_atomic( + json_filename_expanded, + json.dumps(json_data, indent=2, ensure_ascii=False)) + self.logger.debug(f"Successfully wrote JSON file: {json_filename_expanded}") + else: + self.logger.warning(f"Failed to convert TEI to JSON for {filename}") + except Exception as e: + self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}") - # Convert to Markdown if requested - if markdown_output: - try: - from .format.TEI2Markdown import TEI2MarkdownConverter - converter = TEI2MarkdownConverter() - markdown_data = converter.convert_tei_file(filename) - - if markdown_data is not None: - markdown_filename = filename.replace('.grobid.tei.xml', '.md') - # Always write Markdown file when TEI is written (respects --force behavior) - markdown_filename_expanded = os.path.expanduser(markdown_filename) - self._write_atomic(markdown_filename_expanded, markdown_data) - self.logger.debug(f"Successfully wrote Markdown file: {markdown_filename_expanded}") - else: - self.logger.warning(f"Failed to convert TEI to Markdown for {filename}") - except Exception as e: - self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}") + # Convert to Markdown if requested + if markdown_output: + try: + from .format.TEI2Markdown import TEI2MarkdownConverter + converter = TEI2MarkdownConverter() + markdown_data = converter.convert_tei_file(filename) + + if markdown_data is not None: + markdown_filename = filename.replace('.grobid.tei.xml', '.md') + # Always write Markdown file when TEI is written (respects --force behavior) + markdown_filename_expanded = os.path.expanduser(markdown_filename) + self._write_atomic(markdown_filename_expanded, markdown_data) + self.logger.debug(f"Successfully wrote Markdown file: {markdown_filename_expanded}") + else: + self.logger.warning(f"Failed to convert TEI to Markdown for {filename}") + except Exception as e: + self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}") - except OSError as e: - self.logger.error(f"Failed to write TEI XML file {filename}: {str(e)}") - error_count += 1 + except OSError as e: + self.logger.error(f"Failed to write TEI XML file {filename}: {str(e)}") + error_count += 1 + except BaseException: + # Ctrl-C (or any error) while results are awaited: the pool is + # about to be joined, so its threads must not keep retrying + self._stop_workers(results) + raise + # Calculate batch statistics batch_runtime = time.time() - batch_start_time @@ -1955,8 +2081,9 @@ def _send_pdf( pdf_handle.close() if status != 503: - return (name, status, res.text) - self._wait_for_busy_server(name) + return self._checked_answer(name, status, res.text) + if not self._wait_for_busy_server(name): + return (name, 503, self.BUSY_SERVER_INTERRUPTED) def process_documents( self, @@ -2003,27 +2130,33 @@ def process_documents( self._warn_on_engine_concurrency(max(1, n)) # A pool of one is still a pool: n < 1 would make ThreadPoolExecutor raise + self._stopping.clear() with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, n)) as executor: - futures = [ - executor.submit( - self._process_named_pdf, - service, - name, - document, - generate_ids, - consolidate_header, - consolidate_citations, - include_raw_citations, - include_raw_affiliations, - tei_coordinates, - segment_sentences, - flavor - ) - for name, document in items - ] - # Collected in submission order rather than with as_completed(): - # in-memory documents have no filenames to be matched back on later. - return [future.result() for future in futures] + futures: list = [] + try: + for name, document in items: + futures.append(executor.submit( + self._process_named_pdf, + service, + name, + document, + generate_ids, + consolidate_header, + consolidate_citations, + include_raw_citations, + include_raw_affiliations, + tei_coordinates, + segment_sentences, + flavor + )) + # Collected in submission order rather than with as_completed(): + # in-memory documents have no filenames to be matched back on later. + return [future.result() for future in futures] + except BaseException: + # Ctrl-C while results are awaited: the pool is about to be + # joined, so its threads must not keep retrying a busy server + self._stop_workers(futures) + raise def _name_documents(self, documents: Any) -> list: """Pair every document with the name it will be known by. @@ -2051,7 +2184,11 @@ def _name_documents(self, documents: Any) -> list: return items def get_server_url(self, service: str) -> str: - return self.config['grobid_server'] + "/api/" + service + # "https://host/" names the same server as "https://host", + # but host//api/isalive is a different path: some servers (a Hugging + # Face Space among them) answer it with 400, and the client then + # reported a server that is up as unavailable + return str(self.config['grobid_server']).rstrip('/') + "/api/" + service def process_txt( self, @@ -2104,8 +2241,15 @@ def process_txt( return self._handle_unexpected_error(txt_file, e) if status != 503: - return (txt_file, status, res.text) - self._wait_for_busy_server(txt_file) + return self._checked_answer(txt_file, status, res.text) + if not self._wait_for_busy_server(txt_file): + return (txt_file, 503, self.BUSY_SERVER_INTERRUPTED) + + +def _is_missing_credentials(error: BaseException) -> bool: + """Tell botocore's "no AWS credentials" errors apart, without importing botocore.""" + return any(cls.__name__ in ("NoCredentialsError", "PartialCredentialsError") + for cls in type(error).__mro__) def main() -> None: @@ -2361,12 +2505,12 @@ def main() -> None: sys.exit(130) except Exception as e: logger.error(f"Processing failed: {str(e)}") - if type(e).__name__ == "NoCredentialsError" and not args.no_sign_request: + if _is_missing_credentials(e) and not args.no_sign_request: logger.error("To read a public s3:// bucket without AWS credentials, add --no-sign-request") sys.exit(1) runtime = round(time.time() - start_time, 3) - print(f"Processing completed in {runtime} seconds") + print(f"Processing completed in {runtime} seconds", flush=True) if __name__ == "__main__": diff --git a/tests/test_busy_retry.py b/tests/test_busy_retry.py index 719ab3a..0f2927a 100644 --- a/tests/test_busy_retry.py +++ b/tests/test_busy_retry.py @@ -44,7 +44,7 @@ def test_each_attempt_uses_a_fresh_closed_stream(self): client = _client() post, calls = _busy_then_ok(3) with patch.object(GrobidClient, 'post', side_effect=post): - with patch('grobid_client.grobid_client.time.sleep') as sleep: + with patch.object(client._stopping, 'wait', return_value=False) as sleep: name, status, text = client.process_pdf('processFulltextDocument', b'%PDF-bytes') assert (status, text) == (200, 'ok') assert sleep.call_count == 3 @@ -60,7 +60,7 @@ def test_a_file_on_disk_is_reopened_each_time(self): with open(pdf, 'wb') as fh: fh.write(b'%PDF-disk') with patch.object(GrobidClient, 'post', side_effect=post): - with patch('grobid_client.grobid_client.time.sleep'): + with patch.object(client._stopping, 'wait', return_value=False): _, status, _ = client.process_pdf('processFulltextDocument', pdf) assert status == 200 assert all(h.closed for h in calls) @@ -69,7 +69,7 @@ def test_a_long_busy_spell_does_not_recurse(self): client = _client() post, calls = _busy_then_ok(5000) with patch.object(GrobidClient, 'post', side_effect=post): - with patch('grobid_client.grobid_client.time.sleep'): + with patch.object(client._stopping, 'wait', return_value=False): _, status, _ = client.process_pdf('processFulltextDocument', b'%PDF') assert status == 200 assert len(calls) == 5001 @@ -85,7 +85,7 @@ def post(url, files=None, data=None, headers=None, timeout=None): return answer with patch.object(GrobidClient, 'post', side_effect=post): - with patch('grobid_client.grobid_client.time.sleep'): + with patch.object(client._stopping, 'wait', return_value=False): _, status, text = client.process_pdf('processFulltextDocument', b'%PDF') assert status == 408 and 'timeout' in text.lower() @@ -103,7 +103,7 @@ def test_503_then_ok(self): post, calls = _busy_then_ok(2) with tempfile.TemporaryDirectory() as d: with patch.object(GrobidClient, 'post', side_effect=post): - with patch('grobid_client.grobid_client.time.sleep') as sleep: + with patch.object(client._stopping, 'wait', return_value=False) as sleep: name, status, text = client.process_txt( 'processCitationList', self._refs(d), False, False, False, False, False, False, False) assert (status, text) == (200, 'ok') diff --git a/tests/test_empty_answer.py b/tests/test_empty_answer.py new file mode 100644 index 0000000..4a74882 --- /dev/null +++ b/tests/test_empty_answer.py @@ -0,0 +1,52 @@ +"""A 200 without content is an error, not an empty TEI that blocks every rerun.""" +import os +from unittest.mock import Mock, patch + +import pytest + +from grobid_client.grobid_client import GrobidClient + + +def _client(): + with patch.object(GrobidClient, '_configure_logging'): + client = GrobidClient(check_server=False) + client.logger = Mock() + return client + + +@pytest.mark.parametrize('body', ['', ' \n']) +def test_process_pdf_reports_an_empty_answer(body): + client = _client() + with patch.object(client, 'post', return_value=(Mock(text=body), 200)): + name, status, text = client.process_pdf('processFulltextDocument', b'%PDF') + assert (status, text) == (500, GrobidClient.EMPTY_ANSWER) + + +def test_process_txt_reports_an_empty_answer(tmp_path): + refs = tmp_path / 'refs.txt' + refs.write_text('a ref\n') + client = _client() + with patch.object(client, 'post', return_value=(Mock(text=''), 200)): + _, status, _ = client.process_txt('processCitationList', str(refs), False, False, False, False, + False, False, False) + assert status == 500 + + +def test_an_empty_answer_leaves_a_marker_and_is_retried(tmp_path): + src = tmp_path / 'in' + src.mkdir() + (src / 'p.pdf').write_bytes(b'%PDF') + out = tmp_path / 'out' + answers = iter([(Mock(text=''), 200), (Mock(text='ok'), 200)]) + client = _client() + with patch.object(client, 'post', side_effect=lambda *a, **k: next(answers)): + client.process_paths('processFulltextDocument', [str(src)], output=str(out), force=False) + assert sorted(os.listdir(out)) == ['p_500.txt'] + client.process_paths('processFulltextDocument', [str(src)], output=str(out), force=False) + assert sorted(os.listdir(out)) == ['p.grobid.tei.xml'] + + +def test_a_real_answer_is_untouched(): + client = _client() + with patch.object(client, 'post', return_value=(Mock(text=''), 200)): + assert client.process_pdf('processFulltextDocument', b'%PDF')[1:] == (200, '') diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index b9beedd..703b22a 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -160,3 +160,20 @@ def test_without_output_results_sit_next_to_the_inputs(server, pdf_tree, tmp_pat assert result.returncode == 0, result.stderr assert (pdf_tree / 'a' / 'x.grobid.tei.xml').exists() assert (pdf_tree / 'a' / 'deep' / 'y.grobid.tei.xml').exists() + + +def test_progress_lines_come_before_the_errors_in_a_log_file(server, pdf_tree, tmp_path): + # stdout and stderr into one file, as with `grobid_client ... > run.log 2>&1`: + # "Found N file(s)" used to sit in stdout's buffer and land at the very end + log = tmp_path / 'run.log' + with open(log, 'w') as fh: + subprocess.run( + [sys.executable, '-c', 'from grobid_client.grobid_client import main; main()', + '--server', server, '--input', str(pdf_tree), '--output', str(tmp_path / 'out'), + 'processFulltextDocument'], + stdout=fh, stderr=subprocess.STDOUT, cwd=tmp_path, timeout=120) + lines = log.read_text().splitlines() + found = next(i for i, line in enumerate(lines) if line.startswith('Found 5 local file(s)')) + error = next(i for i, line in enumerate(lines) if 'broken.pdf failed' in line) + summary = next(i for i, line in enumerate(lines) if line.startswith('Processing completed: 4 out of 5')) + assert found < error < summary diff --git a/tests/test_grobid_client.py b/tests/test_grobid_client.py index e6fc31f..589dc4e 100644 --- a/tests/test_grobid_client.py +++ b/tests/test_grobid_client.py @@ -407,8 +407,8 @@ def test_process_pdf_server_busy_retry(self, mock_post): with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'): with patch('grobid_client.grobid_client.GrobidClient._configure_logging'): with patch('builtins.open', mock_open()): - with patch('time.sleep') as mock_sleep: - client = GrobidClient(check_server=False) + client = GrobidClient(check_server=False) + with patch.object(client._stopping, 'wait', return_value=False) as mock_sleep: client.logger = Mock() result = client.process_pdf( @@ -423,7 +423,7 @@ def test_process_pdf_server_busy_retry(self, mock_post): segment_sentences=False ) - # Should have called sleep due to 503 response + # Should have waited once due to the 503 response mock_sleep.assert_called_once() assert result[1] == 200 @@ -1635,7 +1635,7 @@ def test_retry_after_503_resends_the_whole_document(self): client = self._client() post, sent = self._post_spy(statuses=(503, 200)) - with patch('time.sleep'): + with patch.object(client._stopping, 'wait', return_value=False): with patch.object(GrobidClient, 'post', side_effect=post): result = client.process_pdf( 'processFulltextDocument', self._named(self.PDF, 'busy.pdf')) @@ -1653,7 +1653,7 @@ def test_retry_after_503_reopens_a_file_on_disk(self): with open(path, 'wb') as f: f.write(self.PDF) - with patch('time.sleep'): + with patch.object(client._stopping, 'wait', return_value=False): with patch.object(GrobidClient, 'post', side_effect=post): result = client.process_pdf('processFulltextDocument', path) diff --git a/tests/test_interrupt_busy_server.py b/tests/test_interrupt_busy_server.py new file mode 100644 index 0000000..e61c9b5 --- /dev/null +++ b/tests/test_interrupt_busy_server.py @@ -0,0 +1,150 @@ +""" +Ctrl-C while the server is busy. + +A document the server answers 503 to is re-posted after ``sleep_time`` for as +long as the server stays busy. That retry ran in the pool's worker threads, which +the interrupted main thread then waited for: the client printed "Interrupted" +and kept posting, so a permanently busy server needed a second Ctrl-C (or a kill). +""" +import http.server +import json +import signal +import subprocess +import sys +import threading +import time +from unittest.mock import Mock, patch + +import pytest + +from grobid_client.grobid_client import GrobidClient + + +def _client(sleep_time=60): + with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'): + with patch('grobid_client.grobid_client.GrobidClient._configure_logging'): + client = GrobidClient(check_server=False) + client.logger = Mock() + client.config['sleep_time'] = sleep_time + return client + + +def _always_busy(): + calls = [] + + def post(url, files=None, data=None, headers=None, timeout=None): + calls.append(url) + return Mock(text='busy'), 503 + + return post, calls + + +def test_a_stopped_client_does_not_wait_for_a_busy_server(): + client = _client() + client._stopping.set() + post, calls = _always_busy() + with patch.object(GrobidClient, 'post', side_effect=post): + start = time.time() + name, status, text = client.process_pdf('processFulltextDocument', b'%PDF') + assert time.time() - start < 5 + assert (status, text) == (503, GrobidClient.BUSY_SERVER_INTERRUPTED) + assert len(calls) == 1 + + +def test_interrupting_a_batch_ends_the_busy_retries(tmp_path): + """KeyboardInterrupt in the main thread must not leave workers retrying.""" + client = _client(sleep_time=60) + post, calls = _always_busy() + pdf = tmp_path / 'busy.pdf' + pdf.write_bytes(b'%PDF') + + def interrupted(futures): + deadline = time.time() + 10 + while not calls and time.time() < deadline: + time.sleep(0.01) + raise KeyboardInterrupt + + with patch.object(GrobidClient, 'post', side_effect=post): + with patch('grobid_client.grobid_client.concurrent.futures.as_completed', side_effect=interrupted): + start = time.time() + with pytest.raises(KeyboardInterrupt): + client.process_batch('processFulltextDocument', [str(pdf)], str(tmp_path), str(tmp_path / 'out'), + 2, False, False, False, False, False, False, False, False) + assert time.time() - start < 10 # not the 60 s sleep_time + assert len(calls) == 1 + assert not (tmp_path / 'out').exists() # an interrupted document leaves no error marker + + +def test_a_new_batch_after_an_interrupt_retries_again(tmp_path): + client = _client(sleep_time=0) + client._stopping.set() # left over from an interrupted batch + answers = iter([503, 200]) + + def post(url, files=None, data=None, headers=None, timeout=None): + return Mock(text=''), next(answers) + + pdf = tmp_path / 'p.pdf' + pdf.write_bytes(b'%PDF') + with patch.object(GrobidClient, 'post', side_effect=post): + processed, errors, skipped = client.process_batch( + 'processFulltextDocument', [str(pdf)], str(tmp_path), str(tmp_path / 'out'), + 1, False, False, False, False, False, False, False, False) + assert (processed, errors) == (1, 0) + + +class _Busy(http.server.BaseHTTPRequestHandler): + posts = 0 + + def log_message(self, *args): + pass + + def do_GET(self): + self.send_response(200) + self.send_header('Content-Length', '4') + self.end_headers() + self.wfile.write(b'true') + + def do_POST(self): + self.rfile.read(int(self.headers.get('Content-Length', 0))) + type(self).posts += 1 + self.send_response(503) + self.send_header('Content-Length', '4') + self.end_headers() + self.wfile.write(b'busy') + + +@pytest.mark.skipif(sys.platform == 'win32', reason='needs POSIX signals') +def test_cli_stops_on_the_first_ctrl_c_against_a_busy_server(tmp_path): + httpd = http.server.ThreadingHTTPServer(('127.0.0.1', 0), _Busy) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + try: + url = f'http://127.0.0.1:{httpd.server_address[1]}' + (tmp_path / 'in').mkdir() + for name in ('a.pdf', 'b.pdf', 'c.pdf'): + (tmp_path / 'in' / name).write_bytes(b'%PDF') + config = tmp_path / 'config.json' + config.write_text(json.dumps({'grobid_server': url, 'sleep_time': 30})) + proc = subprocess.Popen( + [sys.executable, '-c', 'from grobid_client.grobid_client import main; main()', + '--config', str(config), '--input', str(tmp_path / 'in'), '--output', str(tmp_path / 'out'), + '--n', '2', 'processFulltextDocument'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, cwd=str(tmp_path)) + deadline = time.time() + 30 + while _Busy.posts < 2 and time.time() < deadline: + time.sleep(0.05) + assert _Busy.posts >= 2, 'the client never reached the server' + proc.send_signal(signal.SIGINT) + start = time.time() + try: + log, _ = proc.communicate(timeout=15) + except subprocess.TimeoutExpired: + proc.kill() + pytest.fail('still retrying the busy server 15 s after Ctrl-C') + assert proc.returncode == 130, log + assert time.time() - start < 15 + assert 'run the same command again' in log + # the documents interrupted while waiting are not recorded as failures + assert not list((tmp_path / 'out').glob('*_503.txt')) + finally: + httpd.shutdown() + httpd.server_close() diff --git a/tests/test_output_collisions.py b/tests/test_output_collisions.py new file mode 100644 index 0000000..5007e88 --- /dev/null +++ b/tests/test_output_collisions.py @@ -0,0 +1,139 @@ +""" +Two inputs of one run that map to the same output file. + +paper.pdf and paper.PDF in one directory, or two archives listed together that +both hold doc.pdf, used to share one output: the second document overwrote the +first (and both were counted as processed), or was skipped as "already +existed". The first input of the run keeps the name, a later one is written +under its own: paper.PDF.grobid.tei.xml, doc..grobid.tei.xml. +""" +import os +import zipfile +from unittest.mock import Mock, patch + +from grobid_client.grobid_client import GrobidClient + + +def _client(): + with patch.object(GrobidClient, '_configure_logging'): + client = GrobidClient(check_server=False) + client.logger = Mock() + return client + + +class _Server: + """Answers every document with a TEI naming the bytes it was sent.""" + + def __init__(self): + self.posted = [] + + def post(self, url, files=None, data=None, headers=None, timeout=None): + if files: + content = files['input'][1].read().decode() + else: + content = '|'.join(data['citations']) + self.posted.append(content) + return Mock(text=f'{content}'), 200 + + +def _run(client, inputs, out, service='processFulltextDocument', **kwargs): + server = _Server() + with patch.object(client, 'post', side_effect=server.post): + client.process_paths(service, inputs, output=str(out), n=2, force=False, **kwargs) + return server.posted + + +def _outputs(out): + return {os.path.relpath(os.path.join(root, f), out): open(os.path.join(root, f)).read() + for root, _, files in os.walk(out) for f in files} + + +def _zip(path, entries): + with zipfile.ZipFile(path, 'w') as zf: + for name, content in entries.items(): + zf.writestr(name, content) + return str(path) + + +def test_same_stem_different_extension_case(tmp_path): + src = tmp_path / 'in' + src.mkdir() + (src / 'paper.pdf').write_bytes(b'lower') + (src / 'paper.PDF').write_bytes(b'upper') + client = _client() + _run(client, [str(src)], tmp_path / 'out') + outputs = _outputs(tmp_path / 'out') + assert sorted(outputs.values()) == ['lower', 'upper'] + assert 'paper.grobid.tei.xml' in outputs + assert len(outputs) == 2 + assert any('would both be written' in c[0][0] for c in client.logger.warning.call_args_list) + + +def test_two_archives_with_the_same_entry(tmp_path): + a = _zip(tmp_path / 'batch0.zip', {'doc.pdf': 'zero', 'only0.pdf': 'o0'}) + b = _zip(tmp_path / 'batch1.zip', {'doc.pdf': 'one', 'only1.pdf': 'o1'}) + out = tmp_path / 'out' + _run(_client(), [a, b], out) + assert _outputs(out) == { + 'doc.grobid.tei.xml': 'zero', + 'doc.batch1.grobid.tei.xml': 'one', + 'only0.grobid.tei.xml': 'o0', + 'only1.grobid.tei.xml': 'o1', + } + # a rerun finds every document done, the renamed one included, and + # skips them all before reading them + client = _client() + with patch.object(client, '_read_archive_member') as read: + assert _run(client, [a, b], out) == [] + read.assert_not_called() + + +def test_a_resumed_run_finishes_only_the_missing_renamed_entry(tmp_path): + a = _zip(tmp_path / 'batch0.zip', {'doc.pdf': 'zero'}) + b = _zip(tmp_path / 'batch1.zip', {'doc.pdf': 'one'}) + out = tmp_path / 'out' + _run(_client(), [a, b], out) + os.remove(out / 'doc.batch1.grobid.tei.xml') + assert _run(_client(), [a, b], out) == ['one'] + assert _outputs(out)['doc.batch1.grobid.tei.xml'] == 'one' + assert _outputs(out)['doc.grobid.tei.xml'] == 'zero' + + +def test_two_directories_with_the_same_file(tmp_path): + for name in ('dirA', 'dirB'): + (tmp_path / name).mkdir() + (tmp_path / name / 'x.pdf').write_bytes(name.encode()) + out = tmp_path / 'out' + _run(_client(), [str(tmp_path / 'dirA'), str(tmp_path / 'dirB')], out) + assert _outputs(out) == {'x.grobid.tei.xml': 'dirA', + 'x.pdf.grobid.tei.xml': 'dirB'} + + +def test_citation_lists_from_two_archives(tmp_path): + # these go through a temporary directory rather than memory + a = _zip(tmp_path / 'a.zip', {'refs.txt': 'ref a\n'}) + b = _zip(tmp_path / 'b.zip', {'refs.txt': 'ref b\n'}) + out = tmp_path / 'out' + _run(_client(), [a, b], out, service='processCitationList') + assert _outputs(out) == {'refs.grobid.tei.xml': 'ref a', + 'refs.b.grobid.tei.xml': 'ref b'} + + +def test_a_taken_new_name_gets_a_number(tmp_path): + for i, name in enumerate(('x.zip', 'sub/x.zip', 'other/x.zip')): + (tmp_path / name).parent.mkdir(exist_ok=True) + _zip(tmp_path / name, {'doc.pdf': str(i)}) + out = tmp_path / 'out' + _run(_client(), [str(tmp_path / 'x.zip'), str(tmp_path / 'sub/x.zip'), str(tmp_path / 'other/x.zip')], out) + assert _outputs(out) == {'doc.grobid.tei.xml': '0', + 'doc.x.grobid.tei.xml': '1', + 'doc.x-2.grobid.tei.xml': '2'} + + +def test_inputs_that_do_not_collide_keep_their_names(tmp_path): + (tmp_path / 'in' / 'sub').mkdir(parents=True) + (tmp_path / 'in' / 'a.pdf').write_bytes(b'a') + (tmp_path / 'in' / 'sub' / 'a.pdf').write_bytes(b'sub a') + out = tmp_path / 'out' + _run(_client(), [str(tmp_path / 'in')], out) + assert sorted(_outputs(out)) == ['a.grobid.tei.xml', os.path.join('sub', 'a.grobid.tei.xml')] diff --git a/tests/test_s3_no_sign_request.py b/tests/test_s3_no_sign_request.py index b44579e..14eeed7 100644 --- a/tests/test_s3_no_sign_request.py +++ b/tests/test_s3_no_sign_request.py @@ -122,3 +122,33 @@ def test_cli_suggests_the_option_on_missing_credentials(monkeypatch, tmp_path): gc_module.main() assert exit_info.value.code == 1 assert any("--no-sign-request" in message for message in logged) + + +def test_a_single_zip_without_credentials_stops_the_run(no_credentials, tmp_path): + # It used to be logged as "Could not open archive", followed by "No + # eligible files found", and the command exited 0 without the hint. + client = _client() + from botocore.exceptions import NoCredentialsError + with pytest.raises(NoCredentialsError): + client.process_paths("processFulltextDocument", [f"s3://{BUCKET}/corpus/zipfiles/0000.zip"], + output=str(tmp_path / "out")) + + +def test_cli_on_a_single_zip_without_credentials(no_credentials, monkeypatch, tmp_path, capsys): + monkeypatch.setattr("sys.argv", ["grobid_client", "--input", f"s3://{BUCKET}/corpus/zipfiles/0000.zip", + "--output", str(tmp_path / "out"), "processFulltextDocument"]) + with patch.object(GrobidClient, "_test_server_connection", return_value=(True, 200)): + with pytest.raises(SystemExit) as exit_info: + gc_module.main() + assert exit_info.value.code == 1 + assert "--no-sign-request" in capsys.readouterr().err + + +def test_other_archive_errors_are_still_logged_and_skipped(tmp_path): + client = _client() + broken = tmp_path / "broken.zip" + broken.write_bytes(b"not a zip") + assert client._process_archive_core( + "processFulltextDocument", str(broken), str(tmp_path / "out"), 1, False, False, False, + False, False, False, False, False, False, None, False, False) == (0, 0, 0, 0) + assert "Could not open archive" in client.logger.error.call_args[0][0] diff --git a/tests/test_server_check.py b/tests/test_server_check.py index a0afef5..0df8bd9 100644 --- a/tests/test_server_check.py +++ b/tests/test_server_check.py @@ -51,3 +51,16 @@ def test_cli_exits_on_a_misconfigured_server(tmp_path): main() assert exc.value.code == 1 pp.assert_not_called() + + +def test_ping_reports_an_unreachable_server_instead_of_raising(): + client = GrobidClient(check_server=False, grobid_server='http://127.0.0.1:9') + with patch('grobid_client.grobid_client.requests.get', + side_effect=requests.exceptions.ConnectionError('refused')): + assert client.ping() == (False, 0) + + +def test_ping_reports_a_bad_status(): + client = GrobidClient(check_server=False) + with _get(503): + assert client.ping() == (False, 503) diff --git a/tests/test_server_url.py b/tests/test_server_url.py new file mode 100644 index 0000000..08adba5 --- /dev/null +++ b/tests/test_server_url.py @@ -0,0 +1,28 @@ +"""The server URL may be written with a trailing slash.""" +from unittest.mock import Mock, patch + +import pytest + +from grobid_client.grobid_client import GrobidClient + + +@pytest.mark.parametrize('server', [ + 'http://localhost:8070', 'http://localhost:8070/', 'http://localhost:8070//', +]) +def test_trailing_slashes_do_not_reach_the_path(server): + client = GrobidClient(grobid_server=server, check_server=False) + assert client.get_server_url('isalive') == 'http://localhost:8070/api/isalive' + assert client.get_server_url('processFulltextDocument') == \ + 'http://localhost:8070/api/processFulltextDocument' + + +def test_a_path_prefix_is_kept(): + client = GrobidClient(grobid_server='https://example.org/grobid/', check_server=False) + assert client.get_server_url('isalive') == 'https://example.org/grobid/api/isalive' + + +def test_server_check_uses_the_normalised_url(): + response = Mock(status_code=200) + with patch('grobid_client.grobid_client.requests.get', return_value=response) as get: + GrobidClient(grobid_server='http://localhost:8070/', check_server=True) + assert get.call_args[0][0] == 'http://localhost:8070/api/isalive'