diff --git a/grobid_client/format/TEI2LossyJSON.py b/grobid_client/format/TEI2LossyJSON.py index c7e2a9c..60375bb 100644 --- a/grobid_client/format/TEI2LossyJSON.py +++ b/grobid_client/format/TEI2LossyJSON.py @@ -18,15 +18,12 @@ import dateparser from bs4 import BeautifulSoup, Tag +from bs4.element import CData, PreformattedString -# Configure module-level logger +# Module-level logger. Handlers and levels are the application's business: +# configuring the root logger at import time would override the logging +# set up by whoever imports this module (the format CLIs included). logger: logging.Logger = logging.getLogger(__name__) -logger.propagate = False # Prevent propagation to avoid duplicate logs - -# Only configure basic logging if nothing is set up yet -if not logger.handlers and not logging.getLogger().handlers: - # Basic configuration if not already configured by the application - logging.basicConfig(level=logging.INFO) class TEI2LossyJSONConverter: @@ -147,7 +144,9 @@ def convert_tei_file(self, tei_file: Union[str, Path, BinaryIO], stream: bool = [ { "id": sentence.get("xml:id") if sentence.has_attr("xml:id") else id, - "text": sentence.text, + # the same whitespace normalisation the + # ref offsets are computed against + "text": _normalise_passage_text(sentence.get_text()), "coords": [ box_to_dict(coord.split(",")) for coord in sentence['coords'].split(";") @@ -162,7 +161,7 @@ def convert_tei_file(self, tei_file: Union[str, Path, BinaryIO], stream: bool = biblio_structure["abstract"] = [ { "id": id, - "text": paragraph.text, + "text": _normalise_passage_text(paragraph.get_text()), "coords": [ box_to_dict(coord.split(",")) for coord in paragraph['coords'].split(";") @@ -554,17 +553,13 @@ def _compile_citation_data(self, citation_data: dict[str, Any], contributors: li citation_data['urls'] = links def _validate_citation_content(self, citation_data: dict[str, Any]) -> bool: - """Validate that the citation contains meaningful information.""" - # Check for essential bibliographic elements - essential_elements = ['title', 'authors', 'journal', 'doi', 'isbn', 'issn', 'pmc', 'pmid'] - - # Check if any essential element is present - has_essential = any(citation_data.get(element) for element in essential_elements) + """Validate that the citation carries any information beyond its id. - # Check for fallback elements - has_fallback = any(citation_data.get(element) for element in ['raw_reference', 'reference_text']) - - return has_essential or has_fallback + A shortlist of "essential" fields used to decide this, which dropped + entries GROBID describes with an arXiv id, a URL, editors or a date + only, and left the in-text references pointing at them dangling. + """ + return any(value for key, value in citation_data.items() if key not in ('id', 'target')) def _extract_person_data(self, person_element: Tag) -> dict[str, Any] | None: """ @@ -852,7 +847,7 @@ def process_directory(self, directory: Union[str, Path], pattern: str = "*.tei.x # Use processes for CPU-bound parsing when many files are available workers = workers or min(32, (os.cpu_count() or 1)) with ProcessPoolExecutor(max_workers=workers) as ex: - futures = {ex.submit(_convert_file_worker, str(f)): f for f in files} + futures = {ex.submit(_convert_file_worker, str(f), self.validate_refs): f for f in files} for fut in as_completed(futures): f = futures[fut] try: @@ -863,15 +858,9 @@ def process_directory(self, directory: Union[str, Path], pattern: str = "*.tei.x yield {"path": f, "document": doc} -def _convert_file_worker(path: str) -> Any: - """Worker used by ProcessPoolExecutor. Imports inside function to avoid pickling issues.""" - from bs4 import BeautifulSoup - # Reuse existing top-level helpers from this module by importing here - with open(path, 'r') as f: - content = f.read() - soup = BeautifulSoup(content, 'xml') - converter = TEI2LossyJSONConverter() - return converter.convert_tei_file(path, stream=False) +def _convert_file_worker(path: str, validate_refs: bool = True) -> Any: + """Worker used by ProcessPoolExecutor: convert one file with the caller's settings.""" + return TEI2LossyJSONConverter(validate_refs=validate_refs).convert_tei_file(path, stream=False) def box_to_dict(coord_list: list[str]) -> dict[str, float]: @@ -983,6 +972,11 @@ def traverse(node, pos=0): if child_text is not None: parts.append(child_text) return "".join(parts), pos + elif isinstance(node, PreformattedString) and not isinstance(node, CData): + # Comments, processing instructions and declarations are not + # part of get_text(), which gives the passage its text: counting + # them here would shift every offset after them + return "", pos else: s = str(node) return s, pos + len(s) @@ -1018,17 +1012,21 @@ def traverse(node, pos=0): return final_refs +def _normalise_passage_text(text: str) -> str: + """Collapse whitespace runs and strip: the text that ref offsets index into. + + get_refs_with_offsets maps its offsets onto exactly this normalisation, so + every passage carrying refs must have its text built with it. + """ + if not text: + return "" + # DO NOT call html.unescape(text) here, to maintain parity and prevent offset corruption. + return re.sub(r'\s+', ' ', text.strip()) + + def get_formatted_passage(head_paragraph: str | None, head_section: str | None, paragraph_id: str, element: Tag) -> dict[str, Any]: """Format a passage (paragraph or sentence) with metadata and references.""" - # Import the clean_text method - def _clean_text_local(text: str) -> str: - if not text: - return "" - text = re.sub(r'\s+', ' ', text.strip()) - # DO NOT call html.unescape(text) here either, to maintain parity and prevent offset corruption. - return text - - text = _clean_text_local(element.get_text()) + text = _normalise_passage_text(element.get_text()) refs = get_refs_with_offsets(element) passage = { diff --git a/grobid_client/format/TEI2LossyJSON_cli.py b/grobid_client/format/TEI2LossyJSON_cli.py index 9f59875..69e81a1 100644 --- a/grobid_client/format/TEI2LossyJSON_cli.py +++ b/grobid_client/format/TEI2LossyJSON_cli.py @@ -64,13 +64,13 @@ def main() -> None: epilog=""" Examples: # Convert a single TEI file - python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml --output output.json + python -m grobid_client.format TEI2LossyJSON --input input.tei.xml --output output.json # Convert with verbose logging - python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml --output output.json --verbose + python -m grobid_client.format TEI2LossyJSON --input input.tei.xml --output output.json --verbose # Convert and output to stdout - python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml + python -m grobid_client.format TEI2LossyJSON --input input.tei.xml """ ) diff --git a/grobid_client/format/TEI2Markdown.py b/grobid_client/format/TEI2Markdown.py index 946453e..4151b12 100644 --- a/grobid_client/format/TEI2Markdown.py +++ b/grobid_client/format/TEI2Markdown.py @@ -20,11 +20,10 @@ import logging from datetime import datetime -# Configure module-level logger +# Module-level logger. Handlers and levels are the application's business: +# configuring the root logger at import time would override the logging +# set up by whoever imports this module (the format CLIs included). logger = logging.getLogger(__name__) -if not logger.handlers: - # Basic configuration if not already configured by the application - logging.basicConfig(level=logging.INFO) class TEI2MarkdownConverter: @@ -453,22 +452,29 @@ def _process_formula(self, formula_element: Tag) -> str: def _table_to_markdown(self, table_element: Tag) -> str: """Convert a table element to simple markdown.""" - markdown_lines = [] - - # Process table rows + rows = [] for row in table_element.find_all("row"): - cells = [] - for cell in row.find_all("cell"): - cell_text = cell.get_text().strip() - cells.append(cell_text) - + # A cell is one line of a table row: collapse its line breaks and + # escape the pipes that would otherwise end the cell early + cells = [re.sub(r"\s+", " ", cell.get_text()).strip().replace("|", "\\|") + for cell in row.find_all("cell")] if cells: - markdown_lines.append("| " + " | ".join(cells) + " |") - if len(markdown_lines) == 1: - # a separator after the first row makes it a Markdown table - markdown_lines.append("|" + "|".join([" --- "] * len(cells)) + "|") + rows.append(cells) + if not rows: + return "" + + # Every row gets as many cells as the widest one: renderers drop the + # cells of a row that go past the separator row's column count + width = max(len(cells) for cells in rows) + markdown_lines = [] + for cells in rows: + cells = cells + [""] * (width - len(cells)) + markdown_lines.append("| " + " | ".join(cells) + " |") + if len(markdown_lines) == 1: + # a separator after the first row makes it a Markdown table + markdown_lines.append("|" + "|".join([" --- "] * width) + "|") - return "\n".join(markdown_lines) if markdown_lines else "" + return "\n".join(markdown_lines) def _format_reference(self, bibl_struct: Tag, ref_num: int) -> str: """ @@ -619,10 +625,13 @@ def _process_series_section(self, series: Tag, bib_data: dict[str, Any]) -> None def _process_imprint_section(self, imprint: Tag, bib_data: dict[str, Any]) -> None: """Process the imprint section containing publication details.""" - # Extract publication date - date = imprint.find("date") + # Extract publication date. GROBID often writes it as an attribute + # only (), with no text. + date = imprint.find("date", attrs={"type": "published"}) or imprint.find("date") if date: - bib_data['year'] = self._extract_year(date.get_text().strip()) + date_value = date.get("when") or date.get_text().strip() + if date_value: + bib_data['year'] = self._extract_year(date_value) # Extract publication details from biblScope elements for bibl_scope in imprint.find_all("biblScope"): @@ -787,6 +796,9 @@ def _build_identifiers_and_links(self, ref_data: dict[str, Any]) -> list[str]: identifiers_and_links.append(f"{id_type.upper()}: {id_value}") elif id_type.lower() in ['isbn', 'issn']: identifiers_and_links.append(f"{id_type.upper()}: {id_value}") + elif id_type.lower() == 'arxiv': + arxiv_id = re.sub(r'^arxiv:\s*', '', id_value, flags=re.IGNORECASE) + identifiers_and_links.append(f"[arXiv:{arxiv_id}](https://arxiv.org/abs/{arxiv_id})") # Format URLs with display-friendly text. GROBID's for a DOI # is the same doi.org link already printed above: skip it. diff --git a/grobid_client/format/TEI2Markdown_cli.py b/grobid_client/format/TEI2Markdown_cli.py index 5161343..74ee3e0 100644 --- a/grobid_client/format/TEI2Markdown_cli.py +++ b/grobid_client/format/TEI2Markdown_cli.py @@ -63,13 +63,13 @@ def main() -> None: epilog=""" Examples: # Convert a single TEI file - python -m grobid_client.format.TEI2Markdown --input input.tei.xml --output output.md + python -m grobid_client.format TEI2Markdown --input input.tei.xml --output output.md # Convert with verbose logging - python -m grobid_client.format.TEI2Markdown --input input.tei.xml --output output.md --verbose + python -m grobid_client.format TEI2Markdown --input input.tei.xml --output output.md --verbose # Convert and output to stdout - python -m grobid_client.format.TEI2Markdown --input input.tei.xml + python -m grobid_client.format TEI2Markdown --input input.tei.xml """ ) diff --git a/grobid_client/format/__main__.py b/grobid_client/format/__main__.py index 4441b04..1979eea 100644 --- a/grobid_client/format/__main__.py +++ b/grobid_client/format/__main__.py @@ -5,15 +5,14 @@ """ from __future__ import annotations -import argparse import sys def main() -> None: """Main entry point that provides a menu for converter selection.""" - # Check if a converter was specified - if len(sys.argv) < 2: + # Without a converter, print the menu; asking for help gets it too + if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"): print("GROBID format converters - Choose a converter to run") print("\nUsage:") print(" python -m grobid_client.format [options]") @@ -26,7 +25,7 @@ def main() -> None: print("\nGet help for specific converter:") print(" python -m grobid_client.format TEI2LossyJSON --help") print(" python -m grobid_client.format TEI2Markdown --help") - sys.exit(1) + sys.exit(0 if len(sys.argv) >= 2 else 1) converter = sys.argv[1] @@ -41,8 +40,8 @@ def main() -> None: sys.argv = ["TEI2Markdown"] + sys.argv[2:] markdown_main() else: - print(f"Unknown converter: {converter}") - print("Available converters: TEI2LossyJSON, TEI2Markdown") + print(f"Unknown converter: {converter}", file=sys.stderr) + print("Available converters: TEI2LossyJSON, TEI2Markdown", file=sys.stderr) sys.exit(1) diff --git a/grobid_client/format/validate_json_refs.py b/grobid_client/format/validate_json_refs.py index e7f86a2..b7e8ce5 100755 --- a/grobid_client/format/validate_json_refs.py +++ b/grobid_client/format/validate_json_refs.py @@ -44,14 +44,16 @@ def __init__(self, verbose: bool = False) -> None: } def validate_directory(self, directory_path: str) -> Dict[str, Any]: - """Validate all JSON files in a directory or a single JSON file.""" + """Validate all JSON files under a directory (recursively) or a single JSON file.""" # Check if it's a single file if os.path.isfile(directory_path): if not directory_path.endswith('.json'): raise ValueError(f"File must be a JSON file: {directory_path}") json_files = [Path(directory_path)] elif os.path.isdir(directory_path): - json_files = list(Path(directory_path).glob("*.json")) + # The client mirrors the input tree, so the converted files of a + # recursive run sit in subdirectories of the output directory + json_files = sorted(Path(directory_path).rglob("*.json")) else: raise ValueError(f"Path does not exist: {directory_path}") @@ -96,11 +98,23 @@ def _validate_file(self, file_path: str) -> None: 'warnings': [] } + if not isinstance(data, dict): + error_msg = f"{file_path}: not a converted document (top-level JSON {type(data).__name__})" + self.results['errors'].append(error_msg) + self.results['invalid_files'] += 1 + if self.verbose: + print(f"❌ {error_msg}") + return + # Validate different parts of the JSON structure self._validate_body_text_refs(data, file_result) self._validate_abstract_refs(data, file_result) self._validate_other_sections(data, file_result) + # A file with a single bad reference is not a valid file + if file_result['invalid_refs'] > 0: + file_result['valid'] = False + # Update overall results self.results['total_refs'] += file_result['total_refs'] self.results['valid_refs'] += file_result['valid_refs'] @@ -141,11 +155,19 @@ def _validate_body_text_refs(self, data: Dict[str, Any], file_result: Dict[str, def _validate_abstract_refs(self, data: Dict[str, Any], file_result: Dict[str, Any]) -> None: """Validate references in abstract section.""" - if 'biblio' not in data or 'abstract' not in data['biblio']: - return - - for i, paragraph in enumerate(data['biblio']['abstract']): - if 'text' not in paragraph or 'refs' not in paragraph: + abstract = (data.get('biblio') or {}).get('abstract') or [] + + # At sentence level the abstract is a list of paragraphs, each a list + # of sentences; at paragraph level it is a flat list of paragraphs + passages = [] + for i, item in enumerate(abstract): + if isinstance(item, list): + passages.extend((f"biblio.abstract[{i}][{k}]", sentence) for k, sentence in enumerate(item)) + else: + passages.append((f"biblio.abstract[{i}]", item)) + + for location, paragraph in passages: + if not isinstance(paragraph, dict) or 'text' not in paragraph or 'refs' not in paragraph: continue text = paragraph['text'] @@ -153,7 +175,7 @@ def _validate_abstract_refs(self, data: Dict[str, Any], file_result: Dict[str, A file_result['total_refs'] += len(refs) for j, ref in enumerate(refs): - is_valid, error = self._validate_single_ref(text, ref, f"biblio.abstract[{i}].refs[{j}]") + is_valid, error = self._validate_single_ref(text, ref, f"{location}.refs[{j}]") if is_valid: file_result['valid_refs'] += 1 else: @@ -298,14 +320,14 @@ def save_json_report(self, output_path: str) -> None: json.dump(report_data, f, indent=2, ensure_ascii=False) -def main() -> None: +def main() -> int: """Main function.""" parser = argparse.ArgumentParser( description="Validate reference offsets in JSON files generated from TEI documents" ) parser.add_argument( "directory", - help="Directory containing JSON files to validate" + help="JSON file, or directory searched recursively for JSON files, to validate" ) parser.add_argument( "--verbose", "-v", diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index a75f485..2bd99fd 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -132,10 +132,15 @@ def __init__( timeout: Optional[int] = None, config_path: Optional[str] = None, check_server: bool = True, - verbose: bool = False + verbose: bool = False, + s3_no_sign_request: bool = False ) -> None: # Store verbose parameter for logging configuration self.verbose = verbose + # Read s3:// inputs anonymously (public buckets, like the AWS CLI's + # --no-sign-request) instead of with the default credential chain + self.s3_no_sign_request = s3_no_sign_request + self._s3_client_instance: Any = None # Initialize config with defaults self.config = copy.deepcopy(self.DEFAULT_CONFIG) @@ -867,13 +872,30 @@ def _import_boto3(self) -> Any: "Install it with: pip install grobid-client-python[s3]" ) from e + def _s3_client(self) -> Any: + """Return the boto3 S3 client shared by listing and reading. + + With ``s3_no_sign_request`` requests go out unsigned, which is how + public buckets are read without AWS credentials. boto3 clients are + thread-safe, so one client serves every worker. + """ + if self._s3_client_instance is None: + boto3 = self._import_boto3() + if self.s3_no_sign_request: + from botocore import UNSIGNED + from botocore.config import Config + self._s3_client_instance = boto3.client("s3", config=Config(signature_version=UNSIGNED)) + else: + self._s3_client_instance = boto3.client("s3") + return self._s3_client_instance + def _s3_open(self, uri: str) -> BinaryIO: """Open an S3 object as a seekable binary stream (HTTP range-streamed). The returned stream lets zipfile read only the central directory and the requested entries, so a remote zip is never fully downloaded. """ - return self._import_smart_open().open(uri, "rb") + return self._import_smart_open().open(uri, "rb", transport_params={"client": self._s3_client()}) def _resolve_s3_paths(self, uri: str) -> list: """Resolve an s3:// object/prefix/glob into a sorted list of object URIs. @@ -897,7 +919,7 @@ def _resolve_s3_paths(self, uri: str) -> list: else: return [uri] # a concrete object key - s3 = self._import_boto3().client("s3") + s3 = self._s3_client() keys = [] for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=prefix): for obj in page.get("Contents", []): @@ -1014,12 +1036,11 @@ def _is_eligible_input(self, filename: str, service: str) -> bool: if self.ERROR_FILE_RE.search(filename): return False return filename.endswith(".txt") or filename.endswith(".TXT") - if filename.endswith(".pdf") or filename.endswith(".PDF"): - return True - if service == 'processCitationPatentST36' and ( - filename.endswith(".xml") or filename.endswith(".XML")): - return True - return False + if service == 'processCitationPatentST36': + # ST.36 patent documents are XML; GROBID answers a PDF posted + # there with 204 No Content, which became an error marker + return filename.endswith(".xml") or filename.endswith(".XML") + return filename.endswith(".pdf") or filename.endswith(".PDF") def _looks_like_archive(self, path: str) -> bool: """Return True if the path/URI name has a known archive extension.""" @@ -1213,6 +1234,30 @@ def process_archive( processed, errors, skipped, total_files, runtime, skip_errors=skip_errors ) + def _archive_member_is_done(self, member_name: str, output: str, force: bool, skip_errors: bool, + json_output: bool, markdown_output: bool) -> bool: + """Tell, before reading it, whether an archive entry would be skipped. + + process_batch makes the same decision, but only once it holds the + entry's bytes: for an s3:// zip that means downloading every entry a + previous run already finished just to skip it. An entry whose TEI is + there but whose requested JSON or Markdown is missing is not done: + process_batch regenerates those from the TEI. + """ + if force: + return False + name = self._safe_member_path("", member_name) + if name is None: + return False # _read_archive_member reports it + filename = self._output_file_name(name, ".", output) + if os.path.isfile(filename): + if json_output and not os.path.isfile(filename.replace(self.TEI_SUFFIX, '.json')): + return False + if markdown_output and not os.path.isfile(filename.replace(self.TEI_SUFFIX, '.md')): + return False + return True + return skip_errors and bool(self._find_error_files(filename)) + def _process_archive_core( self, service: str, @@ -1277,8 +1322,21 @@ def _process_archive_core( # read straight from the archive into memory and posted from there. use_disk = service == 'processCitationList' - for chunk_start in range(0, total_files, queue_size): - chunk = eligible_members[chunk_start:chunk_start + queue_size] + # Entries a previous run finished are skipped up front, so that a + # resumed run does not read (for s3, download) them again + pending_members = [ + name for name in eligible_members + if not self._archive_member_is_done(name, output, force, skip_errors, + json_output, markdown_output) + ] + skipped_files_count += total_files - len(pending_members) + if skipped_files_count: + self.logger.info( + f"{skipped_files_count} file(s) in {archive_path} already processed, skipping " + f"(use --force to reprocess them)") + + for chunk_start in range(0, len(pending_members), queue_size): + chunk = pending_members[chunk_start:chunk_start + queue_size] if not use_disk: documents = [] @@ -2176,6 +2234,12 @@ def main() -> None: action="store_true", help="Convert TEI output to Markdown format", ) + parser.add_argument( + "--no-sign-request", + dest="no_sign_request", + action="store_true", + help="read s3:// inputs anonymously, without AWS credentials (for public buckets, as the AWS CLI option of the same name)", + ) args = parser.parse_args() @@ -2201,7 +2265,8 @@ def main() -> None: # Initialize GrobidClient which will configure logging based on config.json and verbose flag try: # Only pass grobid_server if it was explicitly provided (not the default) - client_kwargs = {'config_path': config_path, 'verbose': args.verbose} + client_kwargs = {'config_path': config_path, 'verbose': args.verbose, + 's3_no_sign_request': args.no_sign_request} if args.server is not None: # Only override if user specified a different server client_kwargs['grobid_server'] = args.server @@ -2260,6 +2325,13 @@ def main() -> None: logger.error("No input provided (use --input and/or --input-list)") sys.exit(1) + # A mistyped --input used to log a warning and exit 0, which a script + # cannot tell from a run that had nothing left to do + if (input_path is not None and not input_path.startswith("s3://") + and not glob.has_magic(input_path) and not os.path.exists(os.path.expanduser(input_path))): + logger.error(f"Input path does not exist: {input_path}") + sys.exit(1) + start_time = time.time() try: @@ -2282,8 +2354,15 @@ def main() -> None: markdown_output=markdown_output, skip_errors=skip_errors ) + except KeyboardInterrupt: + # Outputs are written atomically and finished documents are skipped + # on the next run, so an interrupted run loses only work in flight + logger.error("Interrupted. Finished documents are kept: run the same command again to resume.") + sys.exit(130) except Exception as e: logger.error(f"Processing failed: {str(e)}") + if type(e).__name__ == "NoCredentialsError" and not args.no_sign_request: + logger.error("To read a public s3:// bucket without AWS credentials, add --no-sign-request") sys.exit(1) runtime = round(time.time() - start_time, 3) diff --git a/tests/test_archive_resume.py b/tests/test_archive_resume.py new file mode 100644 index 0000000..1e90ab3 --- /dev/null +++ b/tests/test_archive_resume.py @@ -0,0 +1,131 @@ +""" +A resumed archive run must not read the entries it already finished. + +process_batch skips an entry whose TEI exists, but only once it holds the +entry's bytes, so a rerun read every finished entry again first. For an +s3:// zip that is a download: on a public 1.3 GB corpus zip a rerun after +330 finished documents spent over a minute fetching them again. +""" +import io +import os +import tarfile +import zipfile +from unittest.mock import Mock, patch + +import pytest + +from grobid_client.grobid_client import GrobidClient + +TEI = "" +NAMES = ["a.pdf", "sub/b.pdf", "sub/c.pdf", "d.pdf"] + + +def _client(): + with patch.object(GrobidClient, "_configure_logging"): + client = GrobidClient(check_server=False) + client.logger = Mock() + client.config["queue_size"] = 2 + return client + + +def _fake_post(url, files=None, data=None, headers=None, timeout=None): + response = Mock() + response.text = TEI + return response, 200 + + +@pytest.fixture(params=["zip", "tar"]) +def archive(request, tmp_path): + path = tmp_path / f"in.{request.param}" + if request.param == "zip": + with zipfile.ZipFile(path, "w") as zf: + for name in NAMES: + zf.writestr(name, b"%PDF " + name.encode()) + else: + with tarfile.open(path, "w") as tf: + for name in NAMES: + data = b"%PDF " + name.encode() + info = tarfile.TarInfo(name) + info.size = len(data) + tf.addfile(info, io.BytesIO(data)) + return str(path) + + +def _run(client, archive, out, **kwargs): + """Process the archive; return the entries read and the summary counts.""" + read = [] + real_read = client._read_archive_member + + def spy(kind, handle, member_name): + read.append(member_name) + return real_read(kind, handle, member_name) + + with patch.object(client, "post", side_effect=_fake_post), \ + patch.object(client, "_read_archive_member", side_effect=spy): + counts = client._process_archive_core( + "processFulltextDocument", archive, str(out), 2, False, False, False, False, False, + False, False, kwargs.get("force", False), False, None, + kwargs.get("json_output", False), False, skip_errors=kwargs.get("skip_errors", False)) + return sorted(read), counts + + +def test_rerun_reads_nothing(archive, tmp_path): + client, out = _client(), tmp_path / "out" + read, counts = _run(client, archive, out) + assert read == sorted(NAMES) + assert counts == (4, 4, 0, 0) + + read, counts = _run(client, archive, out) + assert read == [] + assert counts == (4, 0, 0, 4) + + +def test_only_missing_entries_are_read(archive, tmp_path): + client, out = _client(), tmp_path / "out" + _run(client, archive, out) + os.remove(out / "sub" / "b.grobid.tei.xml") + read, counts = _run(client, archive, out) + assert read == ["sub/b.pdf"] + assert counts == (4, 1, 0, 3) + + +def test_force_reads_everything(archive, tmp_path): + client, out = _client(), tmp_path / "out" + _run(client, archive, out) + read, _ = _run(client, archive, out, force=True) + assert read == sorted(NAMES) + + +def test_missing_json_is_still_regenerated(archive, tmp_path): + client, out = _client(), tmp_path / "out" + _run(client, archive, out, json_output=True) + os.remove(out / "d.json") + read, _ = _run(client, archive, out, json_output=True) + assert read == ["d.pdf"] + assert (out / "d.json").exists() + + +def test_previous_failure_is_not_read_with_skip_errors(archive, tmp_path): + client, out = _client(), tmp_path / "out" + _run(client, archive, out) + os.remove(out / "a.grobid.tei.xml") + (out / "a_500.txt").write_text("boom") + read, counts = _run(client, archive, out, skip_errors=True) + assert read == [] + assert counts == (4, 0, 0, 4) + read, _ = _run(client, archive, out) # without --skip_errors it is retried + assert read == ["a.pdf"] + + +def test_ctrl_c_exits_cleanly_with_a_resume_hint(monkeypatch, tmp_path): + from grobid_client import grobid_client as gc_module + logged = [] + fake_client = Mock() + fake_client.process_paths.side_effect = KeyboardInterrupt + fake_client.logger.error.side_effect = logged.append + monkeypatch.setattr(gc_module, "GrobidClient", Mock(return_value=fake_client)) + monkeypatch.setattr("sys.argv", ["grobid_client", "--input", str(tmp_path), "processFulltextDocument"]) + with pytest.raises(SystemExit) as exit_info: + gc_module.main() + assert exit_info.value.code == 130 + assert any("run the same command again" in message for message in logged) diff --git a/tests/test_cli_and_outputs.py b/tests/test_cli_and_outputs.py index 6afa867..05a8afc 100644 --- a/tests/test_cli_and_outputs.py +++ b/tests/test_cli_and_outputs.py @@ -99,7 +99,7 @@ def test_all_flags_are_forwarded(self): '--generate_ids', '--consolidate_header', '--consolidate_citations', '--include_raw_citations', '--include_raw_affiliations', '--force', '--skip_errors', '--tei_coordinates', '--segment_sentences', '--verbose', - '--flavor', 'article/light', '--json', '--markdown', + '--flavor', 'article/light', '--json', '--markdown', '--no-sign-request', 'processFulltextDocument', ]) pp.assert_called_once() @@ -115,6 +115,7 @@ def test_all_flags_are_forwarded(self): 'markdown_output': True, 'skip_errors': True, } assert constructed == {'config_path': 'my.json', 'verbose': True, + 's3_no_sign_request': True, 'grobid_server': 'http://srv:8070'} def test_defaults_when_no_flag_is_given(self): @@ -130,7 +131,8 @@ def test_defaults_when_no_flag_is_given(self): 'json_output', 'markdown_output', 'skip_errors'): assert kwargs[flag] is False, flag # the server is only overridden when given on the command line - assert constructed == {'config_path': None, 'verbose': False} + assert constructed == {'config_path': None, 'verbose': False, + 's3_no_sign_request': False} def test_camel_case_aliases(self): """--teiCoordinates, --generateIDs, --segmentSentences, --skip-errors.""" diff --git a/tests/test_cli_input_checks.py b/tests/test_cli_input_checks.py new file mode 100644 index 0000000..b738fc3 --- /dev/null +++ b/tests/test_cli_input_checks.py @@ -0,0 +1,70 @@ +""" +Command-line input checks. + +A mistyped --input logged "Input path does not exist" and exited 0, so a +script could not tell it from a run with nothing left to do. And +processCitationPatentST36 took PDFs as inputs as well as its ST.36 XML: +GROBID answers a PDF there with 204 No Content, which left an error +marker per PDF (seen against GROBID 0.9.2). +""" +import os +from unittest.mock import Mock + +import pytest + +from grobid_client import grobid_client as gc_module +from grobid_client.grobid_client import GrobidClient + + +def _run_main(monkeypatch, *argv): + client = Mock() + monkeypatch.setattr(gc_module, "GrobidClient", Mock(return_value=client)) + monkeypatch.setattr("sys.argv", ["grobid_client", *argv]) + code = None + try: + gc_module.main() + except SystemExit as exc: + code = exc.code + return code, client + + +def test_missing_input_exits_1(monkeypatch, tmp_path): + code, client = _run_main(monkeypatch, "--input", str(tmp_path / "typo"), "processFulltextDocument") + assert code == 1 + client.process_paths.assert_not_called() + assert any("does not exist" in str(c) for c in client.logger.error.call_args_list) + + +@pytest.mark.parametrize("make_input", [ + lambda d: str(d), # a directory + lambda d: str(d / "*.pdf"), # a glob, resolved later + lambda d: "s3://bucket/prefix/", # remote, resolved later + lambda d: "~/" + os.path.relpath(str(d), os.path.expanduser("~")), # ~ is expanded +]) +def test_existing_or_deferred_inputs_are_processed(monkeypatch, tmp_path, make_input): + home = tmp_path / "home" + monkeypatch.setenv("HOME", str(home)) + folder = home / "papers" + folder.mkdir(parents=True) + code, client = _run_main(monkeypatch, "--input", make_input(folder), "processFulltextDocument") + assert code is None + client.process_paths.assert_called_once() + + +@pytest.fixture +def client(): + return GrobidClient(check_server=False) + + +def test_st36_takes_only_xml(client): + assert client._is_eligible_input("patent.xml", "processCitationPatentST36") + assert client._is_eligible_input("PATENT.XML", "processCitationPatentST36") + assert not client._is_eligible_input("patent.pdf", "processCitationPatentST36") + + +@pytest.mark.parametrize("service", ["processFulltextDocument", "processHeaderDocument", "processReferences", + "processCitationPatentPDF"]) +def test_pdf_services_take_only_pdfs(client, service): + assert client._is_eligible_input("paper.pdf", service) + assert client._is_eligible_input("PAPER.PDF", service) + assert not client._is_eligible_input("patent.xml", service) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py new file mode 100644 index 0000000..b9beedd --- /dev/null +++ b/tests/test_end_to_end.py @@ -0,0 +1,162 @@ +""" +End to end: the grobid_client command against a local stand-in GROBID server. + +Every other client test mocks the HTTP layer. These run the real command +line in a subprocess against a small HTTP server that answers like GROBID +(isalive, then TEI for every PDF, a 500 for a PDF that asks for one), and +check what lands on disk: the input tree is mirrored, JSON and Markdown +sit next to the TEI, failures leave an error marker, archives behave like +directories, and a second run skips what is done. +""" +import http.server +import json +import os +import subprocess +import sys +import tarfile +import threading +import zipfile + +import pytest + +from tests.resources import TEST_DATA_PATH + +TEI = open(os.path.join(TEST_DATA_PATH, '0046d83a-edd6-4631-b57c-755cdcce8b7f.tei.xml'), 'rb').read() +PDF = b'%PDF-1.4 stand-in' +BROKEN = b'%PDF-1.4 FAIL500' + + +class _FakeGrobid(http.server.BaseHTTPRequestHandler): + posts = [] + + def log_message(self, *args): + pass + + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b'true') + + def do_POST(self): + body = self.rfile.read(int(self.headers.get('Content-Length', 0))) + type(self).posts.append(self.path) + if b'FAIL500' in body: + self.send_response(500) + self.end_headers() + self.wfile.write(b'boom') + return + self.send_response(200) + self.send_header('Content-Type', 'application/xml') + self.end_headers() + self.wfile.write(TEI) + + +@pytest.fixture(scope='module') +def server(): + httpd = http.server.ThreadingHTTPServer(('127.0.0.1', 0), _FakeGrobid) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + yield f'http://127.0.0.1:{httpd.server_address[1]}' + httpd.shutdown() + httpd.server_close() + + +@pytest.fixture +def pdf_tree(tmp_path): + root = tmp_path / 'in' + for rel, content in {'top.pdf': PDF, 'a/x.pdf': PDF, 'a/deep/y.pdf': PDF, + 'b/x.pdf': PDF, 'b/broken.pdf': BROKEN, 'b/notes.txt': b'skip me'}.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + return root + + +def run_client(server, *args, cwd): + _FakeGrobid.posts = [] + return subprocess.run( + [sys.executable, '-c', 'from grobid_client.grobid_client import main; main()', + '--server', server, *args], + capture_output=True, text=True, cwd=cwd, timeout=120) + + +def files_under(path): + return sorted(str(p.relative_to(path)) for p in path.rglob('*') if p.is_file()) + + +def test_directory_tree_is_mirrored_with_json_and_markdown(server, pdf_tree, tmp_path): + out = tmp_path / 'out' + result = run_client(server, '--input', str(pdf_tree), '--output', str(out), '--json', '--markdown', + '--n', '4', 'processFulltextDocument', cwd=tmp_path) + assert result.returncode == 0, result.stderr + expected = [] + for stem in ['top', 'a/x', 'a/deep/y', 'b/x']: + expected += [f'{stem}.grobid.tei.xml', f'{stem}.json', f'{stem}.md'] + assert files_under(out) == sorted(expected + ['b/broken_500.txt']) + assert (out / 'a' / 'x.grobid.tei.xml').read_bytes() == TEI + assert json.loads((out / 'a' / 'deep' / 'y.json').read_text())['body_text'] + assert (out / 'b' / 'x.md').read_text().startswith('# Multi-contact') + assert all(p == '/api/processFulltextDocument' for p in _FakeGrobid.posts) + + +def test_second_run_skips_done_files_and_restores_missing_json(server, pdf_tree, tmp_path): + out = tmp_path / 'out' + run_client(server, '--input', str(pdf_tree), '--output', str(out), '--json', + 'processFulltextDocument', cwd=tmp_path) + (out / 'a' / 'deep' / 'y.json').unlink() + + result = run_client(server, '--input', str(pdf_tree), '--output', str(out), '--json', + 'processFulltextDocument', cwd=tmp_path) + assert result.returncode == 0, result.stderr + # only the document that failed before is sent again + assert len(_FakeGrobid.posts) == 1 + assert (out / 'a' / 'deep' / 'y.json').exists() + + result = run_client(server, '--input', str(pdf_tree), '--output', str(out), '--skip_errors', + 'processFulltextDocument', cwd=tmp_path) + assert result.returncode == 0, result.stderr + assert _FakeGrobid.posts == [] + + +def test_force_reprocesses_everything(server, pdf_tree, tmp_path): + out = tmp_path / 'out' + run_client(server, '--input', str(pdf_tree), '--output', str(out), 'processFulltextDocument', cwd=tmp_path) + run_client(server, '--input', str(pdf_tree), '--output', str(out), '--force', + 'processFulltextDocument', cwd=tmp_path) + assert len(_FakeGrobid.posts) == 5 + + +@pytest.mark.parametrize('kind', ['zip', 'tar.gz']) +def test_archive_is_processed_like_a_directory(server, pdf_tree, tmp_path, kind): + archive = tmp_path / f'in.{kind}' + if kind == 'zip': + with zipfile.ZipFile(archive, 'w') as zf: + for path in pdf_tree.rglob('*'): + if path.is_file(): + zf.write(path, path.relative_to(pdf_tree)) + else: + with tarfile.open(archive, 'w:gz') as tf: + tf.add(pdf_tree, arcname='.') + out = tmp_path / 'out' + result = run_client(server, '--input', str(archive), '--output', str(out), 'processFulltextDocument', + cwd=tmp_path) + assert result.returncode == 0, result.stderr + assert files_under(out) == sorted(['top.grobid.tei.xml', 'a/x.grobid.tei.xml', 'a/deep/y.grobid.tei.xml', + 'b/x.grobid.tei.xml', 'b/broken_500.txt']) + + +def test_input_list_keeps_the_relative_layout(server, pdf_tree, tmp_path): + listing = tmp_path / 'list.txt' + listing.write_text('in/a/x.pdf\nin/b/x.pdf\n') + out = tmp_path / 'out' + result = run_client(server, '--input-list', str(listing), '--output', str(out), 'processFulltextDocument', + cwd=tmp_path) + assert result.returncode == 0, result.stderr + assert files_under(out) == ['a/x.grobid.tei.xml', 'b/x.grobid.tei.xml'] + + +def test_without_output_results_sit_next_to_the_inputs(server, pdf_tree, tmp_path): + result = run_client(server, '--input', str(pdf_tree / 'a'), 'processFulltextDocument', cwd=tmp_path) + assert result.returncode == 0, result.stderr + assert (pdf_tree / 'a' / 'x.grobid.tei.xml').exists() + assert (pdf_tree / 'a' / 'deep' / 'y.grobid.tei.xml').exists() diff --git a/tests/test_format_cli.py b/tests/test_format_cli.py new file mode 100644 index 0000000..3cc0973 --- /dev/null +++ b/tests/test_format_cli.py @@ -0,0 +1,104 @@ +""" +The format converter command lines: `python -m grobid_client.format `. + +These entry points had no tests. `--help` on the menu was reported as an +unknown converter, and the examples in each converter's --help ran +`python -m grobid_client.format.TEI2Markdown`, which imports the module +and exits 0 without converting anything. +""" +import json +import os +import re +import subprocess +import sys + +import pytest + +from tests.resources import TEST_DATA_PATH + +SAMPLE = os.path.join(TEST_DATA_PATH, '0046d83a-edd6-4631-b57c-755cdcce8b7f.tei.xml') +TITLE = 'Multi-contact functional electrical stimulation for hand opening' + + +def run(*args): + return subprocess.run([sys.executable, '-m', 'grobid_client.format', *args], + capture_output=True, text=True, timeout=120) + + +class TestMenu: + def test_without_arguments_prints_the_menu_and_fails(self): + out = run() + assert out.returncode == 1 + assert 'Available converters' in out.stdout + + @pytest.mark.parametrize('flag', ['--help', '-h']) + def test_help_prints_the_menu(self, flag): + out = run(flag) + assert out.returncode == 0 + assert 'Available converters' in out.stdout + assert 'Unknown converter' not in out.stdout + out.stderr + + def test_unknown_converter(self): + out = run('TEI2Html') + assert out.returncode == 1 + assert 'Unknown converter: TEI2Html' in out.stderr + + +class TestConverters: + def test_json_to_file(self, tmp_path): + target = tmp_path / 'nested' / 'out.json' + out = run('TEI2LossyJSON', '--input', SAMPLE, '--output', str(target)) + assert out.returncode == 0, out.stderr + document = json.loads(target.read_text(encoding='utf-8')) + assert document['biblio']['title'].startswith(TITLE) + assert document['body_text'] + + def test_json_to_stdout(self): + out = run('TEI2LossyJSON', '--input', SAMPLE) + assert out.returncode == 0, out.stderr + assert json.loads(out.stdout)['biblio']['title'].startswith(TITLE) + + def test_markdown_to_file(self, tmp_path): + target = tmp_path / 'out.md' + out = run('TEI2Markdown', '--input', SAMPLE, '--output', str(target)) + assert out.returncode == 0, out.stderr + assert target.read_text(encoding='utf-8').startswith(f'# {TITLE}') + + def test_markdown_to_stdout(self): + out = run('TEI2Markdown', '--input', SAMPLE) + assert out.returncode == 0, out.stderr + assert out.stdout.startswith(f'# {TITLE}') + + @pytest.mark.parametrize('converter', ['TEI2LossyJSON', 'TEI2Markdown']) + def test_missing_input(self, converter, tmp_path): + out = run(converter, '--input', str(tmp_path / 'nope.tei.xml')) + assert out.returncode == 1 + assert 'does not exist' in out.stderr + + @pytest.mark.parametrize('converter', ['TEI2LossyJSON', 'TEI2Markdown']) + def test_directory_input(self, converter, tmp_path): + out = run(converter, '--input', str(tmp_path)) + assert out.returncode == 1 + assert 'not a file' in out.stderr + + @pytest.mark.parametrize('converter', ['TEI2LossyJSON', 'TEI2Markdown']) + def test_not_tei_fails_without_writing(self, converter, tmp_path): + source = tmp_path / 'notes.xml' + source.write_text('hello world') + target = tmp_path / 'out' + out = run(converter, '--input', str(source), '--output', str(target)) + assert out.returncode == 1 + assert not target.exists() + + @pytest.mark.parametrize('converter', ['TEI2LossyJSON', 'TEI2Markdown']) + def test_help_examples_name_a_working_command(self, converter, tmp_path): + out = run(converter, '--help') + assert out.returncode == 0 + examples = re.findall(r'^\s*(python -m \S+(?: \S+)?) --input', out.stdout, re.MULTILINE) + assert examples + for example in set(examples): + module_args = example.split()[2:] # drop "python -m" + result = subprocess.run([sys.executable, '-m', *module_args, '--input', SAMPLE], + capture_output=True, text=True, timeout=120) + assert result.returncode == 0, example + assert TITLE in result.stdout, f"{example} printed nothing" diff --git a/tests/test_json_abstract_offsets.py b/tests/test_json_abstract_offsets.py new file mode 100644 index 0000000..058e04c --- /dev/null +++ b/tests/test_json_abstract_offsets.py @@ -0,0 +1,66 @@ +""" +Abstract passages must carry the text their reference offsets index into. + +Reference offsets are computed on whitespace-normalised text, as for body +passages, but the abstract's text was the raw paragraph text. An abstract +with a line break or a double space before a citation got offsets that +point a few characters off (seen on GROBID 0.9.2 output with the sdo/ietf +flavor). At sentence level the abstract is a list of lists, which +validate_json_refs skipped, so this was never reported. +""" +import io +import json + +import pytest + +from grobid_client.format.TEI2LossyJSON import TEI2LossyJSONConverter +from grobid_client.format.validate_json_refs import JSONReferenceValidator + +PARAGRAPH = ('

Networks of depth sixteen [41] to\n thirty [16].\n' + ' Other tasks [8,12] too.

') +SENTENCES = ('

Networks of depth sixteen [41] to\n thirty [16].' + ' Other tasks [8]\n too.

') + + +def _convert(abstract, body='

x

'): + tei = ('' + f'
{abstract}
' + f'{body}
') + return TEI2LossyJSONConverter().convert_tei_file(io.StringIO(tei)) + + +def _abstract_passages(document): + for item in document['biblio']['abstract']: + yield from (item if isinstance(item, list) else [item]) + + +@pytest.mark.parametrize("abstract, body", [ + (PARAGRAPH, '

x

'), + # more than

in the document: sentence level + (SENTENCES, '

abc

'), +]) +def test_abstract_ref_offsets_point_at_the_ref_text(abstract, body): + document = _convert(abstract, body) + passages = list(_abstract_passages(document)) + refs = [(p, r) for p in passages for r in p['refs']] + assert refs + for passage, ref in refs: + assert passage['text'] == ' '.join(passage['text'].split()) + assert passage['text'][ref['offset_start']:ref['offset_end']] == ref['text'] + + +def test_validator_checks_sentence_level_abstracts(tmp_path): + document = _convert(SENTENCES, '

abc

') + assert isinstance(document['biblio']['abstract'][0], list) + ref_count = sum(len(p['refs']) for p in _abstract_passages(document)) + good = tmp_path / 'good.json' + good.write_text(json.dumps(document)) + results = JSONReferenceValidator().validate_directory(str(good)) + assert results['total_refs'] == ref_count and results['invalid_refs'] == 0 + + document['biblio']['abstract'][0][0]['refs'][0]['offset_start'] += 1 + bad = tmp_path / 'bad.json' + bad.write_text(json.dumps(document)) + results = JSONReferenceValidator().validate_directory(str(bad)) + assert results['invalid_refs'] == 1 + assert 'biblio.abstract[0][0].refs[0]' in results['file_details'][0]['errors'][0] diff --git a/tests/test_json_process_directory.py b/tests/test_json_process_directory.py new file mode 100644 index 0000000..663eb98 --- /dev/null +++ b/tests/test_json_process_directory.py @@ -0,0 +1,79 @@ +""" +TEI2LossyJSONConverter.process_directory, serial and parallel. + +The parallel worker built a default converter, so validate_refs was +ignored, and it first read each file with open(path, 'r') -- the locale's +encoding, not UTF-8 -- only to throw the result away. Where the locale is +not UTF-8 (cp1252 on Windows) bytes such as those of a closing double +quote cannot be decoded, and every document holding one came back None. +""" +import builtins +import concurrent.futures +import os +import shutil + +import pytest + +from grobid_client.format import TEI2LossyJSON +from grobid_client.format.TEI2LossyJSON import TEI2LossyJSONConverter, _convert_file_worker +from tests.resources import TEST_DATA_PATH + +SAMPLES = [ + os.path.join(TEST_DATA_PATH, '0046d83a-edd6-4631-b57c-755cdcce8b7f.tei.xml'), + os.path.join(TEST_DATA_PATH, 'refs_offsets', '10.1038_s41598-023-32039-z.grobid.tei.xml'), +] + + +@pytest.fixture +def tei_dir(tmp_path): + (tmp_path / 'sub').mkdir() + shutil.copy(SAMPLES[0], tmp_path / 'a.tei.xml') + shutil.copy(SAMPLES[1], tmp_path / 'sub' / 'b.tei.xml') + (tmp_path / 'ignored.json').write_text('{}') + return tmp_path + + +def _texts(document): + return [p['text'] for p in document['body_text']] + + +def _by_name(items): + return {item['path'].name: item['document'] for item in items} + + +def test_serial_and_parallel_agree(tei_dir): + converter = TEI2LossyJSONConverter() + serial = _by_name(converter.process_directory(tei_dir, parallel=False)) + parallel = _by_name(converter.process_directory(tei_dir, parallel=True, workers=2)) + assert set(serial) == set(parallel) == {'a.tei.xml', 'b.tei.xml'} + for name in serial: + assert parallel[name] is not None + assert _texts(parallel[name]) == _texts(serial[name]) + + +def test_parallel_worker_gets_the_converter_settings(tei_dir, monkeypatch): + calls = [] + + def spy(path, validate_refs=True): + calls.append(validate_refs) + return {'body_text': []} + + # threads instead of processes, so the spy sees the calls + monkeypatch.setattr(TEI2LossyJSON, 'ProcessPoolExecutor', concurrent.futures.ThreadPoolExecutor) + monkeypatch.setattr(TEI2LossyJSON, '_convert_file_worker', spy) + list(TEI2LossyJSONConverter(validate_refs=False).process_directory(tei_dir, parallel=True, workers=2)) + assert calls == [False, False] + + +def test_worker_reads_files_as_utf8(monkeypatch): + real_open = builtins.open + text_opens = [] + + def recording_open(file, mode='r', *args, **kwargs): + if 'b' not in mode: + text_opens.append(kwargs.get('encoding') or (args[1] if len(args) > 1 else None)) + return real_open(file, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, 'open', recording_open) + assert _convert_file_worker(SAMPLES[0]) is not None + assert text_opens and all(encoding == 'utf-8' for encoding in text_opens) diff --git a/tests/test_json_reference_filter.py b/tests/test_json_reference_filter.py new file mode 100644 index 0000000..0f17923 --- /dev/null +++ b/tests/test_json_reference_filter.py @@ -0,0 +1,38 @@ +""" +A reference GROBID describes with anything at all must be kept in the JSON. + +Entries were kept only when they had one of a shortlist of fields (title, +authors, journal, a few identifiers). An entry with only an arXiv id, a URL, +editors or a date was dropped, so the in-text refs pointing at it had no +target in references[]. arXiv-only entries are common in ML papers. +""" +import io + +import pytest + +from grobid_client.format.TEI2LossyJSON import TEI2LossyJSONConverter + + +def _references(bibl_struct_body): + tei = ('' + '
' + f'{bibl_struct_body}' + '
') + return TEI2LossyJSONConverter().convert_tei_file(io.StringIO(tei))['references'] + + +@pytest.mark.parametrize("body, field, value", [ + ('arXiv:1706.03762', 'arxiv', 'arXiv:1706.03762'), + ('', 'url', 'https://example.org/x'), + ('Smith', + 'editors', 'Smith'), + ('ACM', 'publisher', 'ACM'), +]) +def test_entry_with_some_information_is_kept(body, field, value): + [ref] = _references(body) + assert ref['id'] == 'b0' + assert ref[field] == value + + +def test_empty_entry_is_still_dropped(): + assert _references('') == [] diff --git a/tests/test_json_xml_comments.py b/tests/test_json_xml_comments.py new file mode 100644 index 0000000..5205192 --- /dev/null +++ b/tests/test_json_xml_comments.py @@ -0,0 +1,52 @@ +""" +Reference offsets must survive markup that get_text() leaves out. + +The passage text comes from get_text(), which skips XML comments and +processing instructions, but the offsets were counted over str(node) for +every string node, comments included. A comment before a reference shifted +its offsets and the validation assert aborted the whole conversion. +""" +import io + +import pytest + +from grobid_client.format.TEI2LossyJSON import TEI2LossyJSONConverter + +BIBL = ('
' + 'T' + '
') + + +def _convert(paragraph): + tei = ('' + f'
H{paragraph}
{BIBL}
') + return TEI2LossyJSONConverter().convert_tei_file(io.StringIO(tei)) + + +@pytest.mark.parametrize("markup", [ + '', + '', +]) +def test_markup_outside_the_text_does_not_shift_offsets(markup): + doc = _convert(f'

Text {markup} before [1] after.

') + passage = doc['body_text'][0] + assert passage['text'] == 'Text before [1] after.' + [ref] = passage['refs'] + assert passage['text'][ref['offset_start']:ref['offset_end']] == '[1]' + + +def test_cdata_is_text_and_counts(): + doc = _convert('

A then [1].

') + passage = doc['body_text'][0] + assert 'x < y' in passage['text'] + [ref] = passage['refs'] + assert passage['text'][ref['offset_start']:ref['offset_end']] == '[1]' + + +def test_comment_inside_a_sentence(): + doc = _convert('

One [1].' + 'Two [1].' + 'Three.

') + for passage in doc['body_text']: + for ref in passage['refs']: + assert passage['text'][ref['offset_start']:ref['offset_end']] == ref['text'] diff --git a/tests/test_library_logging.py b/tests/test_library_logging.py new file mode 100644 index 0000000..2d2fa1f --- /dev/null +++ b/tests/test_library_logging.py @@ -0,0 +1,49 @@ +""" +Importing the package must leave the application's logging alone. + +Both converter modules called logging.basicConfig(level=INFO) at import +time. Importing grobid_client (which imports TEI2LossyJSON) installed a +root handler at INFO in any program using the library, and turned the +format CLIs' own setup_logging() into a no-op, since basicConfig does +nothing once the root logger has a handler. TEI2LossyJSON's logger also +did not propagate, so an application's handlers never saw its warnings. +""" +import io +import logging +import subprocess +import sys + +import pytest + +from grobid_client.format.TEI2LossyJSON import TEI2LossyJSONConverter +from grobid_client.format.TEI2Markdown import TEI2MarkdownConverter + + +@pytest.mark.parametrize("module", [ + "grobid_client.grobid_client", + "grobid_client.format.TEI2LossyJSON", + "grobid_client.format.TEI2Markdown", +]) +def test_import_does_not_configure_the_root_logger(module): + code = (f"import logging, {module}; r = logging.getLogger(); " + "print(len(r.handlers), logging.getLevelName(r.level))") + out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True) + assert out.stdout.split() == ["0", "WARNING"] + + +@pytest.mark.parametrize("converter", [TEI2LossyJSONConverter, TEI2MarkdownConverter]) +def test_converter_warnings_reach_the_application_handlers(converter, caplog): + with caplog.at_level(logging.WARNING): + assert converter().convert_tei_file(io.StringIO("")) is None + assert any("not well-formed" in r.getMessage() for r in caplog.records) + + +@pytest.mark.parametrize("name", ["TEI2LossyJSON", "TEI2Markdown"]) +def test_format_cli_applies_its_own_log_format(name, tmp_path): + empty = tmp_path / "empty.tei.xml" + empty.write_text("") + out = subprocess.run([sys.executable, "-m", "grobid_client.format", name, "--input", str(empty)], + capture_output=True, text=True) + assert out.returncode == 1 + # setup_logging()'s format: "