diff --git a/grobid_client/format/TEI2LossyJSON.py b/grobid_client/format/TEI2LossyJSON.py
index c7e2a9c..60375bb 100644
--- a/grobid_client/format/TEI2LossyJSON.py
+++ b/grobid_client/format/TEI2LossyJSON.py
@@ -18,15 +18,12 @@
import dateparser
from bs4 import BeautifulSoup, Tag
+from bs4.element import CData, PreformattedString
-# Configure module-level logger
+# Module-level logger. Handlers and levels are the application's business:
+# configuring the root logger at import time would override the logging
+# set up by whoever imports this module (the format CLIs included).
logger: logging.Logger = logging.getLogger(__name__)
-logger.propagate = False # Prevent propagation to avoid duplicate logs
-
-# Only configure basic logging if nothing is set up yet
-if not logger.handlers and not logging.getLogger().handlers:
- # Basic configuration if not already configured by the application
- logging.basicConfig(level=logging.INFO)
class TEI2LossyJSONConverter:
@@ -147,7 +144,9 @@ def convert_tei_file(self, tei_file: Union[str, Path, BinaryIO], stream: bool =
[
{
"id": sentence.get("xml:id") if sentence.has_attr("xml:id") else id,
- "text": sentence.text,
+ # the same whitespace normalisation the
+ # ref offsets are computed against
+ "text": _normalise_passage_text(sentence.get_text()),
"coords": [
box_to_dict(coord.split(","))
for coord in sentence['coords'].split(";")
@@ -162,7 +161,7 @@ def convert_tei_file(self, tei_file: Union[str, Path, BinaryIO], stream: bool =
biblio_structure["abstract"] = [
{
"id": id,
- "text": paragraph.text,
+ "text": _normalise_passage_text(paragraph.get_text()),
"coords": [
box_to_dict(coord.split(","))
for coord in paragraph['coords'].split(";")
@@ -554,17 +553,13 @@ def _compile_citation_data(self, citation_data: dict[str, Any], contributors: li
citation_data['urls'] = links
def _validate_citation_content(self, citation_data: dict[str, Any]) -> bool:
- """Validate that the citation contains meaningful information."""
- # Check for essential bibliographic elements
- essential_elements = ['title', 'authors', 'journal', 'doi', 'isbn', 'issn', 'pmc', 'pmid']
-
- # Check if any essential element is present
- has_essential = any(citation_data.get(element) for element in essential_elements)
+ """Validate that the citation carries any information beyond its id.
- # Check for fallback elements
- has_fallback = any(citation_data.get(element) for element in ['raw_reference', 'reference_text'])
-
- return has_essential or has_fallback
+ A shortlist of "essential" fields used to decide this, which dropped
+ entries GROBID describes with an arXiv id, a URL, editors or a date
+ only, and left the in-text references pointing at them dangling.
+ """
+ return any(value for key, value in citation_data.items() if key not in ('id', 'target'))
def _extract_person_data(self, person_element: Tag) -> dict[str, Any] | None:
"""
@@ -852,7 +847,7 @@ def process_directory(self, directory: Union[str, Path], pattern: str = "*.tei.x
# Use processes for CPU-bound parsing when many files are available
workers = workers or min(32, (os.cpu_count() or 1))
with ProcessPoolExecutor(max_workers=workers) as ex:
- futures = {ex.submit(_convert_file_worker, str(f)): f for f in files}
+ futures = {ex.submit(_convert_file_worker, str(f), self.validate_refs): f for f in files}
for fut in as_completed(futures):
f = futures[fut]
try:
@@ -863,15 +858,9 @@ def process_directory(self, directory: Union[str, Path], pattern: str = "*.tei.x
yield {"path": f, "document": doc}
-def _convert_file_worker(path: str) -> Any:
- """Worker used by ProcessPoolExecutor. Imports inside function to avoid pickling issues."""
- from bs4 import BeautifulSoup
- # Reuse existing top-level helpers from this module by importing here
- with open(path, 'r') as f:
- content = f.read()
- soup = BeautifulSoup(content, 'xml')
- converter = TEI2LossyJSONConverter()
- return converter.convert_tei_file(path, stream=False)
+def _convert_file_worker(path: str, validate_refs: bool = True) -> Any:
+ """Worker used by ProcessPoolExecutor: convert one file with the caller's settings."""
+ return TEI2LossyJSONConverter(validate_refs=validate_refs).convert_tei_file(path, stream=False)
def box_to_dict(coord_list: list[str]) -> dict[str, float]:
@@ -983,6 +972,11 @@ def traverse(node, pos=0):
if child_text is not None:
parts.append(child_text)
return "".join(parts), pos
+ elif isinstance(node, PreformattedString) and not isinstance(node, CData):
+ # Comments, processing instructions and declarations are not
+ # part of get_text(), which gives the passage its text: counting
+ # them here would shift every offset after them
+ return "", pos
else:
s = str(node)
return s, pos + len(s)
@@ -1018,17 +1012,21 @@ def traverse(node, pos=0):
return final_refs
+def _normalise_passage_text(text: str) -> str:
+ """Collapse whitespace runs and strip: the text that ref offsets index into.
+
+ get_refs_with_offsets maps its offsets onto exactly this normalisation, so
+ every passage carrying refs must have its text built with it.
+ """
+ if not text:
+ return ""
+ # DO NOT call html.unescape(text) here, to maintain parity and prevent offset corruption.
+ return re.sub(r'\s+', ' ', text.strip())
+
+
def get_formatted_passage(head_paragraph: str | None, head_section: str | None, paragraph_id: str, element: Tag) -> dict[str, Any]:
"""Format a passage (paragraph or sentence) with metadata and references."""
- # Import the clean_text method
- def _clean_text_local(text: str) -> str:
- if not text:
- return ""
- text = re.sub(r'\s+', ' ', text.strip())
- # DO NOT call html.unescape(text) here either, to maintain parity and prevent offset corruption.
- return text
-
- text = _clean_text_local(element.get_text())
+ text = _normalise_passage_text(element.get_text())
refs = get_refs_with_offsets(element)
passage = {
diff --git a/grobid_client/format/TEI2LossyJSON_cli.py b/grobid_client/format/TEI2LossyJSON_cli.py
index 9f59875..69e81a1 100644
--- a/grobid_client/format/TEI2LossyJSON_cli.py
+++ b/grobid_client/format/TEI2LossyJSON_cli.py
@@ -64,13 +64,13 @@ def main() -> None:
epilog="""
Examples:
# Convert a single TEI file
- python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml --output output.json
+ python -m grobid_client.format TEI2LossyJSON --input input.tei.xml --output output.json
# Convert with verbose logging
- python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml --output output.json --verbose
+ python -m grobid_client.format TEI2LossyJSON --input input.tei.xml --output output.json --verbose
# Convert and output to stdout
- python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml
+ python -m grobid_client.format TEI2LossyJSON --input input.tei.xml
"""
)
diff --git a/grobid_client/format/TEI2Markdown.py b/grobid_client/format/TEI2Markdown.py
index 946453e..4151b12 100644
--- a/grobid_client/format/TEI2Markdown.py
+++ b/grobid_client/format/TEI2Markdown.py
@@ -20,11 +20,10 @@
import logging
from datetime import datetime
-# Configure module-level logger
+# Module-level logger. Handlers and levels are the application's business:
+# configuring the root logger at import time would override the logging
+# set up by whoever imports this module (the format CLIs included).
logger = logging.getLogger(__name__)
-if not logger.handlers:
- # Basic configuration if not already configured by the application
- logging.basicConfig(level=logging.INFO)
class TEI2MarkdownConverter:
@@ -453,22 +452,29 @@ def _process_formula(self, formula_element: Tag) -> str:
def _table_to_markdown(self, table_element: Tag) -> str:
"""Convert a table element to simple markdown."""
- markdown_lines = []
-
- # Process table rows
+ rows = []
for row in table_element.find_all("row"):
- cells = []
- for cell in row.find_all("cell"):
- cell_text = cell.get_text().strip()
- cells.append(cell_text)
-
+ # A cell is one line of a table row: collapse its line breaks and
+ # escape the pipes that would otherwise end the cell early
+ cells = [re.sub(r"\s+", " ", cell.get_text()).strip().replace("|", "\\|")
+ for cell in row.find_all("cell")]
if cells:
- markdown_lines.append("| " + " | ".join(cells) + " |")
- if len(markdown_lines) == 1:
- # a separator after the first row makes it a Markdown table
- markdown_lines.append("|" + "|".join([" --- "] * len(cells)) + "|")
+ rows.append(cells)
+ if not rows:
+ return ""
+
+ # Every row gets as many cells as the widest one: renderers drop the
+ # cells of a row that go past the separator row's column count
+ width = max(len(cells) for cells in rows)
+ markdown_lines = []
+ for cells in rows:
+ cells = cells + [""] * (width - len(cells))
+ markdown_lines.append("| " + " | ".join(cells) + " |")
+ if len(markdown_lines) == 1:
+ # a separator after the first row makes it a Markdown table
+ markdown_lines.append("|" + "|".join([" --- "] * width) + "|")
- return "\n".join(markdown_lines) if markdown_lines else ""
+ return "\n".join(markdown_lines)
def _format_reference(self, bibl_struct: Tag, ref_num: int) -> str:
"""
@@ -619,10 +625,13 @@ def _process_series_section(self, series: Tag, bib_data: dict[str, Any]) -> None
def _process_imprint_section(self, imprint: Tag, bib_data: dict[str, Any]) -> None:
"""Process the imprint section containing publication details."""
- # Extract publication date
- date = imprint.find("date")
+ # Extract publication date. GROBID often writes it as an attribute
+ # only (), with no text.
+ date = imprint.find("date", attrs={"type": "published"}) or imprint.find("date")
if date:
- bib_data['year'] = self._extract_year(date.get_text().strip())
+ date_value = date.get("when") or date.get_text().strip()
+ if date_value:
+ bib_data['year'] = self._extract_year(date_value)
# Extract publication details from biblScope elements
for bibl_scope in imprint.find_all("biblScope"):
@@ -787,6 +796,9 @@ def _build_identifiers_and_links(self, ref_data: dict[str, Any]) -> list[str]:
identifiers_and_links.append(f"{id_type.upper()}: {id_value}")
elif id_type.lower() in ['isbn', 'issn']:
identifiers_and_links.append(f"{id_type.upper()}: {id_value}")
+ elif id_type.lower() == 'arxiv':
+ arxiv_id = re.sub(r'^arxiv:\s*', '', id_value, flags=re.IGNORECASE)
+ identifiers_and_links.append(f"[arXiv:{arxiv_id}](https://arxiv.org/abs/{arxiv_id})")
# Format URLs with display-friendly text. GROBID's for a DOI
# is the same doi.org link already printed above: skip it.
diff --git a/grobid_client/format/TEI2Markdown_cli.py b/grobid_client/format/TEI2Markdown_cli.py
index 5161343..74ee3e0 100644
--- a/grobid_client/format/TEI2Markdown_cli.py
+++ b/grobid_client/format/TEI2Markdown_cli.py
@@ -63,13 +63,13 @@ def main() -> None:
epilog="""
Examples:
# Convert a single TEI file
- python -m grobid_client.format.TEI2Markdown --input input.tei.xml --output output.md
+ python -m grobid_client.format TEI2Markdown --input input.tei.xml --output output.md
# Convert with verbose logging
- python -m grobid_client.format.TEI2Markdown --input input.tei.xml --output output.md --verbose
+ python -m grobid_client.format TEI2Markdown --input input.tei.xml --output output.md --verbose
# Convert and output to stdout
- python -m grobid_client.format.TEI2Markdown --input input.tei.xml
+ python -m grobid_client.format TEI2Markdown --input input.tei.xml
"""
)
diff --git a/grobid_client/format/__main__.py b/grobid_client/format/__main__.py
index 4441b04..1979eea 100644
--- a/grobid_client/format/__main__.py
+++ b/grobid_client/format/__main__.py
@@ -5,15 +5,14 @@
"""
from __future__ import annotations
-import argparse
import sys
def main() -> None:
"""Main entry point that provides a menu for converter selection."""
- # Check if a converter was specified
- if len(sys.argv) < 2:
+ # Without a converter, print the menu; asking for help gets it too
+ if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"):
print("GROBID format converters - Choose a converter to run")
print("\nUsage:")
print(" python -m grobid_client.format [options]")
@@ -26,7 +25,7 @@ def main() -> None:
print("\nGet help for specific converter:")
print(" python -m grobid_client.format TEI2LossyJSON --help")
print(" python -m grobid_client.format TEI2Markdown --help")
- sys.exit(1)
+ sys.exit(0 if len(sys.argv) >= 2 else 1)
converter = sys.argv[1]
@@ -41,8 +40,8 @@ def main() -> None:
sys.argv = ["TEI2Markdown"] + sys.argv[2:]
markdown_main()
else:
- print(f"Unknown converter: {converter}")
- print("Available converters: TEI2LossyJSON, TEI2Markdown")
+ print(f"Unknown converter: {converter}", file=sys.stderr)
+ print("Available converters: TEI2LossyJSON, TEI2Markdown", file=sys.stderr)
sys.exit(1)
diff --git a/grobid_client/format/validate_json_refs.py b/grobid_client/format/validate_json_refs.py
index e7f86a2..b7e8ce5 100755
--- a/grobid_client/format/validate_json_refs.py
+++ b/grobid_client/format/validate_json_refs.py
@@ -44,14 +44,16 @@ def __init__(self, verbose: bool = False) -> None:
}
def validate_directory(self, directory_path: str) -> Dict[str, Any]:
- """Validate all JSON files in a directory or a single JSON file."""
+ """Validate all JSON files under a directory (recursively) or a single JSON file."""
# Check if it's a single file
if os.path.isfile(directory_path):
if not directory_path.endswith('.json'):
raise ValueError(f"File must be a JSON file: {directory_path}")
json_files = [Path(directory_path)]
elif os.path.isdir(directory_path):
- json_files = list(Path(directory_path).glob("*.json"))
+ # The client mirrors the input tree, so the converted files of a
+ # recursive run sit in subdirectories of the output directory
+ json_files = sorted(Path(directory_path).rglob("*.json"))
else:
raise ValueError(f"Path does not exist: {directory_path}")
@@ -96,11 +98,23 @@ def _validate_file(self, file_path: str) -> None:
'warnings': []
}
+ if not isinstance(data, dict):
+ error_msg = f"{file_path}: not a converted document (top-level JSON {type(data).__name__})"
+ self.results['errors'].append(error_msg)
+ self.results['invalid_files'] += 1
+ if self.verbose:
+ print(f"❌ {error_msg}")
+ return
+
# Validate different parts of the JSON structure
self._validate_body_text_refs(data, file_result)
self._validate_abstract_refs(data, file_result)
self._validate_other_sections(data, file_result)
+ # A file with a single bad reference is not a valid file
+ if file_result['invalid_refs'] > 0:
+ file_result['valid'] = False
+
# Update overall results
self.results['total_refs'] += file_result['total_refs']
self.results['valid_refs'] += file_result['valid_refs']
@@ -141,11 +155,19 @@ def _validate_body_text_refs(self, data: Dict[str, Any], file_result: Dict[str,
def _validate_abstract_refs(self, data: Dict[str, Any], file_result: Dict[str, Any]) -> None:
"""Validate references in abstract section."""
- if 'biblio' not in data or 'abstract' not in data['biblio']:
- return
-
- for i, paragraph in enumerate(data['biblio']['abstract']):
- if 'text' not in paragraph or 'refs' not in paragraph:
+ abstract = (data.get('biblio') or {}).get('abstract') or []
+
+ # At sentence level the abstract is a list of paragraphs, each a list
+ # of sentences; at paragraph level it is a flat list of paragraphs
+ passages = []
+ for i, item in enumerate(abstract):
+ if isinstance(item, list):
+ passages.extend((f"biblio.abstract[{i}][{k}]", sentence) for k, sentence in enumerate(item))
+ else:
+ passages.append((f"biblio.abstract[{i}]", item))
+
+ for location, paragraph in passages:
+ if not isinstance(paragraph, dict) or 'text' not in paragraph or 'refs' not in paragraph:
continue
text = paragraph['text']
@@ -153,7 +175,7 @@ def _validate_abstract_refs(self, data: Dict[str, Any], file_result: Dict[str, A
file_result['total_refs'] += len(refs)
for j, ref in enumerate(refs):
- is_valid, error = self._validate_single_ref(text, ref, f"biblio.abstract[{i}].refs[{j}]")
+ is_valid, error = self._validate_single_ref(text, ref, f"{location}.refs[{j}]")
if is_valid:
file_result['valid_refs'] += 1
else:
@@ -298,14 +320,14 @@ def save_json_report(self, output_path: str) -> None:
json.dump(report_data, f, indent=2, ensure_ascii=False)
-def main() -> None:
+def main() -> int:
"""Main function."""
parser = argparse.ArgumentParser(
description="Validate reference offsets in JSON files generated from TEI documents"
)
parser.add_argument(
"directory",
- help="Directory containing JSON files to validate"
+ help="JSON file, or directory searched recursively for JSON files, to validate"
)
parser.add_argument(
"--verbose", "-v",
diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py
index a75f485..2bd99fd 100644
--- a/grobid_client/grobid_client.py
+++ b/grobid_client/grobid_client.py
@@ -132,10 +132,15 @@ def __init__(
timeout: Optional[int] = None,
config_path: Optional[str] = None,
check_server: bool = True,
- verbose: bool = False
+ verbose: bool = False,
+ s3_no_sign_request: bool = False
) -> None:
# Store verbose parameter for logging configuration
self.verbose = verbose
+ # Read s3:// inputs anonymously (public buckets, like the AWS CLI's
+ # --no-sign-request) instead of with the default credential chain
+ self.s3_no_sign_request = s3_no_sign_request
+ self._s3_client_instance: Any = None
# Initialize config with defaults
self.config = copy.deepcopy(self.DEFAULT_CONFIG)
@@ -867,13 +872,30 @@ def _import_boto3(self) -> Any:
"Install it with: pip install grobid-client-python[s3]"
) from e
+ def _s3_client(self) -> Any:
+ """Return the boto3 S3 client shared by listing and reading.
+
+ With ``s3_no_sign_request`` requests go out unsigned, which is how
+ public buckets are read without AWS credentials. boto3 clients are
+ thread-safe, so one client serves every worker.
+ """
+ if self._s3_client_instance is None:
+ boto3 = self._import_boto3()
+ if self.s3_no_sign_request:
+ from botocore import UNSIGNED
+ from botocore.config import Config
+ self._s3_client_instance = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+ else:
+ self._s3_client_instance = boto3.client("s3")
+ return self._s3_client_instance
+
def _s3_open(self, uri: str) -> BinaryIO:
"""Open an S3 object as a seekable binary stream (HTTP range-streamed).
The returned stream lets zipfile read only the central directory and the
requested entries, so a remote zip is never fully downloaded.
"""
- return self._import_smart_open().open(uri, "rb")
+ return self._import_smart_open().open(uri, "rb", transport_params={"client": self._s3_client()})
def _resolve_s3_paths(self, uri: str) -> list:
"""Resolve an s3:// object/prefix/glob into a sorted list of object URIs.
@@ -897,7 +919,7 @@ def _resolve_s3_paths(self, uri: str) -> list:
else:
return [uri] # a concrete object key
- s3 = self._import_boto3().client("s3")
+ s3 = self._s3_client()
keys = []
for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=prefix):
for obj in page.get("Contents", []):
@@ -1014,12 +1036,11 @@ def _is_eligible_input(self, filename: str, service: str) -> bool:
if self.ERROR_FILE_RE.search(filename):
return False
return filename.endswith(".txt") or filename.endswith(".TXT")
- if filename.endswith(".pdf") or filename.endswith(".PDF"):
- return True
- if service == 'processCitationPatentST36' and (
- filename.endswith(".xml") or filename.endswith(".XML")):
- return True
- return False
+ if service == 'processCitationPatentST36':
+ # ST.36 patent documents are XML; GROBID answers a PDF posted
+ # there with 204 No Content, which became an error marker
+ return filename.endswith(".xml") or filename.endswith(".XML")
+ return filename.endswith(".pdf") or filename.endswith(".PDF")
def _looks_like_archive(self, path: str) -> bool:
"""Return True if the path/URI name has a known archive extension."""
@@ -1213,6 +1234,30 @@ def process_archive(
processed, errors, skipped, total_files, runtime, skip_errors=skip_errors
)
+ def _archive_member_is_done(self, member_name: str, output: str, force: bool, skip_errors: bool,
+ json_output: bool, markdown_output: bool) -> bool:
+ """Tell, before reading it, whether an archive entry would be skipped.
+
+ process_batch makes the same decision, but only once it holds the
+ entry's bytes: for an s3:// zip that means downloading every entry a
+ previous run already finished just to skip it. An entry whose TEI is
+ there but whose requested JSON or Markdown is missing is not done:
+ process_batch regenerates those from the TEI.
+ """
+ if force:
+ return False
+ name = self._safe_member_path("", member_name)
+ if name is None:
+ return False # _read_archive_member reports it
+ filename = self._output_file_name(name, ".", output)
+ if os.path.isfile(filename):
+ if json_output and not os.path.isfile(filename.replace(self.TEI_SUFFIX, '.json')):
+ return False
+ if markdown_output and not os.path.isfile(filename.replace(self.TEI_SUFFIX, '.md')):
+ return False
+ return True
+ return skip_errors and bool(self._find_error_files(filename))
+
def _process_archive_core(
self,
service: str,
@@ -1277,8 +1322,21 @@ def _process_archive_core(
# read straight from the archive into memory and posted from there.
use_disk = service == 'processCitationList'
- for chunk_start in range(0, total_files, queue_size):
- chunk = eligible_members[chunk_start:chunk_start + queue_size]
+ # Entries a previous run finished are skipped up front, so that a
+ # resumed run does not read (for s3, download) them again
+ pending_members = [
+ name for name in eligible_members
+ if not self._archive_member_is_done(name, output, force, skip_errors,
+ json_output, markdown_output)
+ ]
+ skipped_files_count += total_files - len(pending_members)
+ if skipped_files_count:
+ self.logger.info(
+ f"{skipped_files_count} file(s) in {archive_path} already processed, skipping "
+ f"(use --force to reprocess them)")
+
+ for chunk_start in range(0, len(pending_members), queue_size):
+ chunk = pending_members[chunk_start:chunk_start + queue_size]
if not use_disk:
documents = []
@@ -2176,6 +2234,12 @@ def main() -> None:
action="store_true",
help="Convert TEI output to Markdown format",
)
+ parser.add_argument(
+ "--no-sign-request",
+ dest="no_sign_request",
+ action="store_true",
+ help="read s3:// inputs anonymously, without AWS credentials (for public buckets, as the AWS CLI option of the same name)",
+ )
args = parser.parse_args()
@@ -2201,7 +2265,8 @@ def main() -> None:
# Initialize GrobidClient which will configure logging based on config.json and verbose flag
try:
# Only pass grobid_server if it was explicitly provided (not the default)
- client_kwargs = {'config_path': config_path, 'verbose': args.verbose}
+ client_kwargs = {'config_path': config_path, 'verbose': args.verbose,
+ 's3_no_sign_request': args.no_sign_request}
if args.server is not None: # Only override if user specified a different server
client_kwargs['grobid_server'] = args.server
@@ -2260,6 +2325,13 @@ def main() -> None:
logger.error("No input provided (use --input and/or --input-list)")
sys.exit(1)
+ # A mistyped --input used to log a warning and exit 0, which a script
+ # cannot tell from a run that had nothing left to do
+ if (input_path is not None and not input_path.startswith("s3://")
+ and not glob.has_magic(input_path) and not os.path.exists(os.path.expanduser(input_path))):
+ logger.error(f"Input path does not exist: {input_path}")
+ sys.exit(1)
+
start_time = time.time()
try:
@@ -2282,8 +2354,15 @@ def main() -> None:
markdown_output=markdown_output,
skip_errors=skip_errors
)
+ except KeyboardInterrupt:
+ # Outputs are written atomically and finished documents are skipped
+ # on the next run, so an interrupted run loses only work in flight
+ logger.error("Interrupted. Finished documents are kept: run the same command again to resume.")
+ sys.exit(130)
except Exception as e:
logger.error(f"Processing failed: {str(e)}")
+ if type(e).__name__ == "NoCredentialsError" and not args.no_sign_request:
+ logger.error("To read a public s3:// bucket without AWS credentials, add --no-sign-request")
sys.exit(1)
runtime = round(time.time() - start_time, 3)
diff --git a/tests/test_archive_resume.py b/tests/test_archive_resume.py
new file mode 100644
index 0000000..1e90ab3
--- /dev/null
+++ b/tests/test_archive_resume.py
@@ -0,0 +1,131 @@
+"""
+A resumed archive run must not read the entries it already finished.
+
+process_batch skips an entry whose TEI exists, but only once it holds the
+entry's bytes, so a rerun read every finished entry again first. For an
+s3:// zip that is a download: on a public 1.3 GB corpus zip a rerun after
+330 finished documents spent over a minute fetching them again.
+"""
+import io
+import os
+import tarfile
+import zipfile
+from unittest.mock import Mock, patch
+
+import pytest
+
+from grobid_client.grobid_client import GrobidClient
+
+TEI = ""
+NAMES = ["a.pdf", "sub/b.pdf", "sub/c.pdf", "d.pdf"]
+
+
+def _client():
+ with patch.object(GrobidClient, "_configure_logging"):
+ client = GrobidClient(check_server=False)
+ client.logger = Mock()
+ client.config["queue_size"] = 2
+ return client
+
+
+def _fake_post(url, files=None, data=None, headers=None, timeout=None):
+ response = Mock()
+ response.text = TEI
+ return response, 200
+
+
+@pytest.fixture(params=["zip", "tar"])
+def archive(request, tmp_path):
+ path = tmp_path / f"in.{request.param}"
+ if request.param == "zip":
+ with zipfile.ZipFile(path, "w") as zf:
+ for name in NAMES:
+ zf.writestr(name, b"%PDF " + name.encode())
+ else:
+ with tarfile.open(path, "w") as tf:
+ for name in NAMES:
+ data = b"%PDF " + name.encode()
+ info = tarfile.TarInfo(name)
+ info.size = len(data)
+ tf.addfile(info, io.BytesIO(data))
+ return str(path)
+
+
+def _run(client, archive, out, **kwargs):
+ """Process the archive; return the entries read and the summary counts."""
+ read = []
+ real_read = client._read_archive_member
+
+ def spy(kind, handle, member_name):
+ read.append(member_name)
+ return real_read(kind, handle, member_name)
+
+ with patch.object(client, "post", side_effect=_fake_post), \
+ patch.object(client, "_read_archive_member", side_effect=spy):
+ counts = client._process_archive_core(
+ "processFulltextDocument", archive, str(out), 2, False, False, False, False, False,
+ False, False, kwargs.get("force", False), False, None,
+ kwargs.get("json_output", False), False, skip_errors=kwargs.get("skip_errors", False))
+ return sorted(read), counts
+
+
+def test_rerun_reads_nothing(archive, tmp_path):
+ client, out = _client(), tmp_path / "out"
+ read, counts = _run(client, archive, out)
+ assert read == sorted(NAMES)
+ assert counts == (4, 4, 0, 0)
+
+ read, counts = _run(client, archive, out)
+ assert read == []
+ assert counts == (4, 0, 0, 4)
+
+
+def test_only_missing_entries_are_read(archive, tmp_path):
+ client, out = _client(), tmp_path / "out"
+ _run(client, archive, out)
+ os.remove(out / "sub" / "b.grobid.tei.xml")
+ read, counts = _run(client, archive, out)
+ assert read == ["sub/b.pdf"]
+ assert counts == (4, 1, 0, 3)
+
+
+def test_force_reads_everything(archive, tmp_path):
+ client, out = _client(), tmp_path / "out"
+ _run(client, archive, out)
+ read, _ = _run(client, archive, out, force=True)
+ assert read == sorted(NAMES)
+
+
+def test_missing_json_is_still_regenerated(archive, tmp_path):
+ client, out = _client(), tmp_path / "out"
+ _run(client, archive, out, json_output=True)
+ os.remove(out / "d.json")
+ read, _ = _run(client, archive, out, json_output=True)
+ assert read == ["d.pdf"]
+ assert (out / "d.json").exists()
+
+
+def test_previous_failure_is_not_read_with_skip_errors(archive, tmp_path):
+ client, out = _client(), tmp_path / "out"
+ _run(client, archive, out)
+ os.remove(out / "a.grobid.tei.xml")
+ (out / "a_500.txt").write_text("boom")
+ read, counts = _run(client, archive, out, skip_errors=True)
+ assert read == []
+ assert counts == (4, 0, 0, 4)
+ read, _ = _run(client, archive, out) # without --skip_errors it is retried
+ assert read == ["a.pdf"]
+
+
+def test_ctrl_c_exits_cleanly_with_a_resume_hint(monkeypatch, tmp_path):
+ from grobid_client import grobid_client as gc_module
+ logged = []
+ fake_client = Mock()
+ fake_client.process_paths.side_effect = KeyboardInterrupt
+ fake_client.logger.error.side_effect = logged.append
+ monkeypatch.setattr(gc_module, "GrobidClient", Mock(return_value=fake_client))
+ monkeypatch.setattr("sys.argv", ["grobid_client", "--input", str(tmp_path), "processFulltextDocument"])
+ with pytest.raises(SystemExit) as exit_info:
+ gc_module.main()
+ assert exit_info.value.code == 130
+ assert any("run the same command again" in message for message in logged)
diff --git a/tests/test_cli_and_outputs.py b/tests/test_cli_and_outputs.py
index 6afa867..05a8afc 100644
--- a/tests/test_cli_and_outputs.py
+++ b/tests/test_cli_and_outputs.py
@@ -99,7 +99,7 @@ def test_all_flags_are_forwarded(self):
'--generate_ids', '--consolidate_header', '--consolidate_citations',
'--include_raw_citations', '--include_raw_affiliations', '--force',
'--skip_errors', '--tei_coordinates', '--segment_sentences', '--verbose',
- '--flavor', 'article/light', '--json', '--markdown',
+ '--flavor', 'article/light', '--json', '--markdown', '--no-sign-request',
'processFulltextDocument',
])
pp.assert_called_once()
@@ -115,6 +115,7 @@ def test_all_flags_are_forwarded(self):
'markdown_output': True, 'skip_errors': True,
}
assert constructed == {'config_path': 'my.json', 'verbose': True,
+ 's3_no_sign_request': True,
'grobid_server': 'http://srv:8070'}
def test_defaults_when_no_flag_is_given(self):
@@ -130,7 +131,8 @@ def test_defaults_when_no_flag_is_given(self):
'json_output', 'markdown_output', 'skip_errors'):
assert kwargs[flag] is False, flag
# the server is only overridden when given on the command line
- assert constructed == {'config_path': None, 'verbose': False}
+ assert constructed == {'config_path': None, 'verbose': False,
+ 's3_no_sign_request': False}
def test_camel_case_aliases(self):
"""--teiCoordinates, --generateIDs, --segmentSentences, --skip-errors."""
diff --git a/tests/test_cli_input_checks.py b/tests/test_cli_input_checks.py
new file mode 100644
index 0000000..b738fc3
--- /dev/null
+++ b/tests/test_cli_input_checks.py
@@ -0,0 +1,70 @@
+"""
+Command-line input checks.
+
+A mistyped --input logged "Input path does not exist" and exited 0, so a
+script could not tell it from a run with nothing left to do. And
+processCitationPatentST36 took PDFs as inputs as well as its ST.36 XML:
+GROBID answers a PDF there with 204 No Content, which left an error
+marker per PDF (seen against GROBID 0.9.2).
+"""
+import os
+from unittest.mock import Mock
+
+import pytest
+
+from grobid_client import grobid_client as gc_module
+from grobid_client.grobid_client import GrobidClient
+
+
+def _run_main(monkeypatch, *argv):
+ client = Mock()
+ monkeypatch.setattr(gc_module, "GrobidClient", Mock(return_value=client))
+ monkeypatch.setattr("sys.argv", ["grobid_client", *argv])
+ code = None
+ try:
+ gc_module.main()
+ except SystemExit as exc:
+ code = exc.code
+ return code, client
+
+
+def test_missing_input_exits_1(monkeypatch, tmp_path):
+ code, client = _run_main(monkeypatch, "--input", str(tmp_path / "typo"), "processFulltextDocument")
+ assert code == 1
+ client.process_paths.assert_not_called()
+ assert any("does not exist" in str(c) for c in client.logger.error.call_args_list)
+
+
+@pytest.mark.parametrize("make_input", [
+ lambda d: str(d), # a directory
+ lambda d: str(d / "*.pdf"), # a glob, resolved later
+ lambda d: "s3://bucket/prefix/", # remote, resolved later
+ lambda d: "~/" + os.path.relpath(str(d), os.path.expanduser("~")), # ~ is expanded
+])
+def test_existing_or_deferred_inputs_are_processed(monkeypatch, tmp_path, make_input):
+ home = tmp_path / "home"
+ monkeypatch.setenv("HOME", str(home))
+ folder = home / "papers"
+ folder.mkdir(parents=True)
+ code, client = _run_main(monkeypatch, "--input", make_input(folder), "processFulltextDocument")
+ assert code is None
+ client.process_paths.assert_called_once()
+
+
+@pytest.fixture
+def client():
+ return GrobidClient(check_server=False)
+
+
+def test_st36_takes_only_xml(client):
+ assert client._is_eligible_input("patent.xml", "processCitationPatentST36")
+ assert client._is_eligible_input("PATENT.XML", "processCitationPatentST36")
+ assert not client._is_eligible_input("patent.pdf", "processCitationPatentST36")
+
+
+@pytest.mark.parametrize("service", ["processFulltextDocument", "processHeaderDocument", "processReferences",
+ "processCitationPatentPDF"])
+def test_pdf_services_take_only_pdfs(client, service):
+ assert client._is_eligible_input("paper.pdf", service)
+ assert client._is_eligible_input("PAPER.PDF", service)
+ assert not client._is_eligible_input("patent.xml", service)
diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py
new file mode 100644
index 0000000..b9beedd
--- /dev/null
+++ b/tests/test_end_to_end.py
@@ -0,0 +1,162 @@
+"""
+End to end: the grobid_client command against a local stand-in GROBID server.
+
+Every other client test mocks the HTTP layer. These run the real command
+line in a subprocess against a small HTTP server that answers like GROBID
+(isalive, then TEI for every PDF, a 500 for a PDF that asks for one), and
+check what lands on disk: the input tree is mirrored, JSON and Markdown
+sit next to the TEI, failures leave an error marker, archives behave like
+directories, and a second run skips what is done.
+"""
+import http.server
+import json
+import os
+import subprocess
+import sys
+import tarfile
+import threading
+import zipfile
+
+import pytest
+
+from tests.resources import TEST_DATA_PATH
+
+TEI = open(os.path.join(TEST_DATA_PATH, '0046d83a-edd6-4631-b57c-755cdcce8b7f.tei.xml'), 'rb').read()
+PDF = b'%PDF-1.4 stand-in'
+BROKEN = b'%PDF-1.4 FAIL500'
+
+
+class _FakeGrobid(http.server.BaseHTTPRequestHandler):
+ posts = []
+
+ def log_message(self, *args):
+ pass
+
+ def do_GET(self):
+ self.send_response(200)
+ self.end_headers()
+ self.wfile.write(b'true')
+
+ def do_POST(self):
+ body = self.rfile.read(int(self.headers.get('Content-Length', 0)))
+ type(self).posts.append(self.path)
+ if b'FAIL500' in body:
+ self.send_response(500)
+ self.end_headers()
+ self.wfile.write(b'boom')
+ return
+ self.send_response(200)
+ self.send_header('Content-Type', 'application/xml')
+ self.end_headers()
+ self.wfile.write(TEI)
+
+
+@pytest.fixture(scope='module')
+def server():
+ httpd = http.server.ThreadingHTTPServer(('127.0.0.1', 0), _FakeGrobid)
+ thread = threading.Thread(target=httpd.serve_forever, daemon=True)
+ thread.start()
+ yield f'http://127.0.0.1:{httpd.server_address[1]}'
+ httpd.shutdown()
+ httpd.server_close()
+
+
+@pytest.fixture
+def pdf_tree(tmp_path):
+ root = tmp_path / 'in'
+ for rel, content in {'top.pdf': PDF, 'a/x.pdf': PDF, 'a/deep/y.pdf': PDF,
+ 'b/x.pdf': PDF, 'b/broken.pdf': BROKEN, 'b/notes.txt': b'skip me'}.items():
+ path = root / rel
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(content)
+ return root
+
+
+def run_client(server, *args, cwd):
+ _FakeGrobid.posts = []
+ return subprocess.run(
+ [sys.executable, '-c', 'from grobid_client.grobid_client import main; main()',
+ '--server', server, *args],
+ capture_output=True, text=True, cwd=cwd, timeout=120)
+
+
+def files_under(path):
+ return sorted(str(p.relative_to(path)) for p in path.rglob('*') if p.is_file())
+
+
+def test_directory_tree_is_mirrored_with_json_and_markdown(server, pdf_tree, tmp_path):
+ out = tmp_path / 'out'
+ result = run_client(server, '--input', str(pdf_tree), '--output', str(out), '--json', '--markdown',
+ '--n', '4', 'processFulltextDocument', cwd=tmp_path)
+ assert result.returncode == 0, result.stderr
+ expected = []
+ for stem in ['top', 'a/x', 'a/deep/y', 'b/x']:
+ expected += [f'{stem}.grobid.tei.xml', f'{stem}.json', f'{stem}.md']
+ assert files_under(out) == sorted(expected + ['b/broken_500.txt'])
+ assert (out / 'a' / 'x.grobid.tei.xml').read_bytes() == TEI
+ assert json.loads((out / 'a' / 'deep' / 'y.json').read_text())['body_text']
+ assert (out / 'b' / 'x.md').read_text().startswith('# Multi-contact')
+ assert all(p == '/api/processFulltextDocument' for p in _FakeGrobid.posts)
+
+
+def test_second_run_skips_done_files_and_restores_missing_json(server, pdf_tree, tmp_path):
+ out = tmp_path / 'out'
+ run_client(server, '--input', str(pdf_tree), '--output', str(out), '--json',
+ 'processFulltextDocument', cwd=tmp_path)
+ (out / 'a' / 'deep' / 'y.json').unlink()
+
+ result = run_client(server, '--input', str(pdf_tree), '--output', str(out), '--json',
+ 'processFulltextDocument', cwd=tmp_path)
+ assert result.returncode == 0, result.stderr
+ # only the document that failed before is sent again
+ assert len(_FakeGrobid.posts) == 1
+ assert (out / 'a' / 'deep' / 'y.json').exists()
+
+ result = run_client(server, '--input', str(pdf_tree), '--output', str(out), '--skip_errors',
+ 'processFulltextDocument', cwd=tmp_path)
+ assert result.returncode == 0, result.stderr
+ assert _FakeGrobid.posts == []
+
+
+def test_force_reprocesses_everything(server, pdf_tree, tmp_path):
+ out = tmp_path / 'out'
+ run_client(server, '--input', str(pdf_tree), '--output', str(out), 'processFulltextDocument', cwd=tmp_path)
+ run_client(server, '--input', str(pdf_tree), '--output', str(out), '--force',
+ 'processFulltextDocument', cwd=tmp_path)
+ assert len(_FakeGrobid.posts) == 5
+
+
+@pytest.mark.parametrize('kind', ['zip', 'tar.gz'])
+def test_archive_is_processed_like_a_directory(server, pdf_tree, tmp_path, kind):
+ archive = tmp_path / f'in.{kind}'
+ if kind == 'zip':
+ with zipfile.ZipFile(archive, 'w') as zf:
+ for path in pdf_tree.rglob('*'):
+ if path.is_file():
+ zf.write(path, path.relative_to(pdf_tree))
+ else:
+ with tarfile.open(archive, 'w:gz') as tf:
+ tf.add(pdf_tree, arcname='.')
+ out = tmp_path / 'out'
+ result = run_client(server, '--input', str(archive), '--output', str(out), 'processFulltextDocument',
+ cwd=tmp_path)
+ assert result.returncode == 0, result.stderr
+ assert files_under(out) == sorted(['top.grobid.tei.xml', 'a/x.grobid.tei.xml', 'a/deep/y.grobid.tei.xml',
+ 'b/x.grobid.tei.xml', 'b/broken_500.txt'])
+
+
+def test_input_list_keeps_the_relative_layout(server, pdf_tree, tmp_path):
+ listing = tmp_path / 'list.txt'
+ listing.write_text('in/a/x.pdf\nin/b/x.pdf\n')
+ out = tmp_path / 'out'
+ result = run_client(server, '--input-list', str(listing), '--output', str(out), 'processFulltextDocument',
+ cwd=tmp_path)
+ assert result.returncode == 0, result.stderr
+ assert files_under(out) == ['a/x.grobid.tei.xml', 'b/x.grobid.tei.xml']
+
+
+def test_without_output_results_sit_next_to_the_inputs(server, pdf_tree, tmp_path):
+ result = run_client(server, '--input', str(pdf_tree / 'a'), 'processFulltextDocument', cwd=tmp_path)
+ assert result.returncode == 0, result.stderr
+ assert (pdf_tree / 'a' / 'x.grobid.tei.xml').exists()
+ assert (pdf_tree / 'a' / 'deep' / 'y.grobid.tei.xml').exists()
diff --git a/tests/test_format_cli.py b/tests/test_format_cli.py
new file mode 100644
index 0000000..3cc0973
--- /dev/null
+++ b/tests/test_format_cli.py
@@ -0,0 +1,104 @@
+"""
+The format converter command lines: `python -m grobid_client.format `.
+
+These entry points had no tests. `--help` on the menu was reported as an
+unknown converter, and the examples in each converter's --help ran
+`python -m grobid_client.format.TEI2Markdown`, which imports the module
+and exits 0 without converting anything.
+"""
+import json
+import os
+import re
+import subprocess
+import sys
+
+import pytest
+
+from tests.resources import TEST_DATA_PATH
+
+SAMPLE = os.path.join(TEST_DATA_PATH, '0046d83a-edd6-4631-b57c-755cdcce8b7f.tei.xml')
+TITLE = 'Multi-contact functional electrical stimulation for hand opening'
+
+
+def run(*args):
+ return subprocess.run([sys.executable, '-m', 'grobid_client.format', *args],
+ capture_output=True, text=True, timeout=120)
+
+
+class TestMenu:
+ def test_without_arguments_prints_the_menu_and_fails(self):
+ out = run()
+ assert out.returncode == 1
+ assert 'Available converters' in out.stdout
+
+ @pytest.mark.parametrize('flag', ['--help', '-h'])
+ def test_help_prints_the_menu(self, flag):
+ out = run(flag)
+ assert out.returncode == 0
+ assert 'Available converters' in out.stdout
+ assert 'Unknown converter' not in out.stdout + out.stderr
+
+ def test_unknown_converter(self):
+ out = run('TEI2Html')
+ assert out.returncode == 1
+ assert 'Unknown converter: TEI2Html' in out.stderr
+
+
+class TestConverters:
+ def test_json_to_file(self, tmp_path):
+ target = tmp_path / 'nested' / 'out.json'
+ out = run('TEI2LossyJSON', '--input', SAMPLE, '--output', str(target))
+ assert out.returncode == 0, out.stderr
+ document = json.loads(target.read_text(encoding='utf-8'))
+ assert document['biblio']['title'].startswith(TITLE)
+ assert document['body_text']
+
+ def test_json_to_stdout(self):
+ out = run('TEI2LossyJSON', '--input', SAMPLE)
+ assert out.returncode == 0, out.stderr
+ assert json.loads(out.stdout)['biblio']['title'].startswith(TITLE)
+
+ def test_markdown_to_file(self, tmp_path):
+ target = tmp_path / 'out.md'
+ out = run('TEI2Markdown', '--input', SAMPLE, '--output', str(target))
+ assert out.returncode == 0, out.stderr
+ assert target.read_text(encoding='utf-8').startswith(f'# {TITLE}')
+
+ def test_markdown_to_stdout(self):
+ out = run('TEI2Markdown', '--input', SAMPLE)
+ assert out.returncode == 0, out.stderr
+ assert out.stdout.startswith(f'# {TITLE}')
+
+ @pytest.mark.parametrize('converter', ['TEI2LossyJSON', 'TEI2Markdown'])
+ def test_missing_input(self, converter, tmp_path):
+ out = run(converter, '--input', str(tmp_path / 'nope.tei.xml'))
+ assert out.returncode == 1
+ assert 'does not exist' in out.stderr
+
+ @pytest.mark.parametrize('converter', ['TEI2LossyJSON', 'TEI2Markdown'])
+ def test_directory_input(self, converter, tmp_path):
+ out = run(converter, '--input', str(tmp_path))
+ assert out.returncode == 1
+ assert 'not a file' in out.stderr
+
+ @pytest.mark.parametrize('converter', ['TEI2LossyJSON', 'TEI2Markdown'])
+ def test_not_tei_fails_without_writing(self, converter, tmp_path):
+ source = tmp_path / 'notes.xml'
+ source.write_text('hello world')
+ target = tmp_path / 'out'
+ out = run(converter, '--input', str(source), '--output', str(target))
+ assert out.returncode == 1
+ assert not target.exists()
+
+ @pytest.mark.parametrize('converter', ['TEI2LossyJSON', 'TEI2Markdown'])
+ def test_help_examples_name_a_working_command(self, converter, tmp_path):
+ out = run(converter, '--help')
+ assert out.returncode == 0
+ examples = re.findall(r'^\s*(python -m \S+(?: \S+)?) --input', out.stdout, re.MULTILINE)
+ assert examples
+ for example in set(examples):
+ module_args = example.split()[2:] # drop "python -m"
+ result = subprocess.run([sys.executable, '-m', *module_args, '--input', SAMPLE],
+ capture_output=True, text=True, timeout=120)
+ assert result.returncode == 0, example
+ assert TITLE in result.stdout, f"{example} printed nothing"
diff --git a/tests/test_json_abstract_offsets.py b/tests/test_json_abstract_offsets.py
new file mode 100644
index 0000000..058e04c
--- /dev/null
+++ b/tests/test_json_abstract_offsets.py
@@ -0,0 +1,66 @@
+"""
+Abstract passages must carry the text their reference offsets index into.
+
+Reference offsets are computed on whitespace-normalised text, as for body
+passages, but the abstract's text was the raw paragraph text. An abstract
+with a line break or a double space before a citation got offsets that
+point a few characters off (seen on GROBID 0.9.2 output with the sdo/ietf
+flavor). At sentence level the abstract is a list of lists, which
+validate_json_refs skipped, so this was never reported.
+"""
+import io
+import json
+
+import pytest
+
+from grobid_client.format.TEI2LossyJSON import TEI2LossyJSONConverter
+from grobid_client.format.validate_json_refs import JSONReferenceValidator
+
+PARAGRAPH = ('
Networks of depth sixteen [41] to\n thirty [16].\n'
+ ' Other tasks [8,12] too.
')
+SENTENCES = ('
Networks of depth sixteen [41] to\n thirty [16].'
+ ' Other tasks [8]\n too.
')
+
+
+def _convert(abstract, body='
x
'):
+ tei = (''
+ f'
{abstract}
'
+ f'{body}')
+ return TEI2LossyJSONConverter().convert_tei_file(io.StringIO(tei))
+
+
+def _abstract_passages(document):
+ for item in document['biblio']['abstract']:
+ yield from (item if isinstance(item, list) else [item])
+
+
+@pytest.mark.parametrize("abstract, body", [
+ (PARAGRAPH, '
x
'),
+ # more than
in the document: sentence level
+ (SENTENCES, '
abc
'),
+])
+def test_abstract_ref_offsets_point_at_the_ref_text(abstract, body):
+ document = _convert(abstract, body)
+ passages = list(_abstract_passages(document))
+ refs = [(p, r) for p in passages for r in p['refs']]
+ assert refs
+ for passage, ref in refs:
+ assert passage['text'] == ' '.join(passage['text'].split())
+ assert passage['text'][ref['offset_start']:ref['offset_end']] == ref['text']
+
+
+def test_validator_checks_sentence_level_abstracts(tmp_path):
+ document = _convert(SENTENCES, '
abc
')
+ assert isinstance(document['biblio']['abstract'][0], list)
+ ref_count = sum(len(p['refs']) for p in _abstract_passages(document))
+ good = tmp_path / 'good.json'
+ good.write_text(json.dumps(document))
+ results = JSONReferenceValidator().validate_directory(str(good))
+ assert results['total_refs'] == ref_count and results['invalid_refs'] == 0
+
+ document['biblio']['abstract'][0][0]['refs'][0]['offset_start'] += 1
+ bad = tmp_path / 'bad.json'
+ bad.write_text(json.dumps(document))
+ results = JSONReferenceValidator().validate_directory(str(bad))
+ assert results['invalid_refs'] == 1
+ assert 'biblio.abstract[0][0].refs[0]' in results['file_details'][0]['errors'][0]
diff --git a/tests/test_json_process_directory.py b/tests/test_json_process_directory.py
new file mode 100644
index 0000000..663eb98
--- /dev/null
+++ b/tests/test_json_process_directory.py
@@ -0,0 +1,79 @@
+"""
+TEI2LossyJSONConverter.process_directory, serial and parallel.
+
+The parallel worker built a default converter, so validate_refs was
+ignored, and it first read each file with open(path, 'r') -- the locale's
+encoding, not UTF-8 -- only to throw the result away. Where the locale is
+not UTF-8 (cp1252 on Windows) bytes such as those of a closing double
+quote cannot be decoded, and every document holding one came back None.
+"""
+import builtins
+import concurrent.futures
+import os
+import shutil
+
+import pytest
+
+from grobid_client.format import TEI2LossyJSON
+from grobid_client.format.TEI2LossyJSON import TEI2LossyJSONConverter, _convert_file_worker
+from tests.resources import TEST_DATA_PATH
+
+SAMPLES = [
+ os.path.join(TEST_DATA_PATH, '0046d83a-edd6-4631-b57c-755cdcce8b7f.tei.xml'),
+ os.path.join(TEST_DATA_PATH, 'refs_offsets', '10.1038_s41598-023-32039-z.grobid.tei.xml'),
+]
+
+
+@pytest.fixture
+def tei_dir(tmp_path):
+ (tmp_path / 'sub').mkdir()
+ shutil.copy(SAMPLES[0], tmp_path / 'a.tei.xml')
+ shutil.copy(SAMPLES[1], tmp_path / 'sub' / 'b.tei.xml')
+ (tmp_path / 'ignored.json').write_text('{}')
+ return tmp_path
+
+
+def _texts(document):
+ return [p['text'] for p in document['body_text']]
+
+
+def _by_name(items):
+ return {item['path'].name: item['document'] for item in items}
+
+
+def test_serial_and_parallel_agree(tei_dir):
+ converter = TEI2LossyJSONConverter()
+ serial = _by_name(converter.process_directory(tei_dir, parallel=False))
+ parallel = _by_name(converter.process_directory(tei_dir, parallel=True, workers=2))
+ assert set(serial) == set(parallel) == {'a.tei.xml', 'b.tei.xml'}
+ for name in serial:
+ assert parallel[name] is not None
+ assert _texts(parallel[name]) == _texts(serial[name])
+
+
+def test_parallel_worker_gets_the_converter_settings(tei_dir, monkeypatch):
+ calls = []
+
+ def spy(path, validate_refs=True):
+ calls.append(validate_refs)
+ return {'body_text': []}
+
+ # threads instead of processes, so the spy sees the calls
+ monkeypatch.setattr(TEI2LossyJSON, 'ProcessPoolExecutor', concurrent.futures.ThreadPoolExecutor)
+ monkeypatch.setattr(TEI2LossyJSON, '_convert_file_worker', spy)
+ list(TEI2LossyJSONConverter(validate_refs=False).process_directory(tei_dir, parallel=True, workers=2))
+ assert calls == [False, False]
+
+
+def test_worker_reads_files_as_utf8(monkeypatch):
+ real_open = builtins.open
+ text_opens = []
+
+ def recording_open(file, mode='r', *args, **kwargs):
+ if 'b' not in mode:
+ text_opens.append(kwargs.get('encoding') or (args[1] if len(args) > 1 else None))
+ return real_open(file, mode, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, 'open', recording_open)
+ assert _convert_file_worker(SAMPLES[0]) is not None
+ assert text_opens and all(encoding == 'utf-8' for encoding in text_opens)
diff --git a/tests/test_json_reference_filter.py b/tests/test_json_reference_filter.py
new file mode 100644
index 0000000..0f17923
--- /dev/null
+++ b/tests/test_json_reference_filter.py
@@ -0,0 +1,38 @@
+"""
+A reference GROBID describes with anything at all must be kept in the JSON.
+
+Entries were kept only when they had one of a shortlist of fields (title,
+authors, journal, a few identifiers). An entry with only an arXiv id, a URL,
+editors or a date was dropped, so the in-text refs pointing at it had no
+target in references[]. arXiv-only entries are common in ML papers.
+"""
+import io
+
+import pytest
+
+from grobid_client.format.TEI2LossyJSON import TEI2LossyJSONConverter
+
+
+def _references(bibl_struct_body):
+ tei = (''
+ '
'
+ f'{bibl_struct_body}'
+ '
')
+ return TEI2LossyJSONConverter().convert_tei_file(io.StringIO(tei))['references']
+
+
+@pytest.mark.parametrize("body, field, value", [
+ ('arXiv:1706.03762', 'arxiv', 'arXiv:1706.03762'),
+ ('', 'url', 'https://example.org/x'),
+ ('Smith',
+ 'editors', 'Smith'),
+ ('ACM', 'publisher', 'ACM'),
+])
+def test_entry_with_some_information_is_kept(body, field, value):
+ [ref] = _references(body)
+ assert ref['id'] == 'b0'
+ assert ref[field] == value
+
+
+def test_empty_entry_is_still_dropped():
+ assert _references('') == []
diff --git a/tests/test_json_xml_comments.py b/tests/test_json_xml_comments.py
new file mode 100644
index 0000000..5205192
--- /dev/null
+++ b/tests/test_json_xml_comments.py
@@ -0,0 +1,52 @@
+"""
+Reference offsets must survive markup that get_text() leaves out.
+
+The passage text comes from get_text(), which skips XML comments and
+processing instructions, but the offsets were counted over str(node) for
+every string node, comments included. A comment before a reference shifted
+its offsets and the validation assert aborted the whole conversion.
+"""
+import io
+
+import pytest
+
+from grobid_client.format.TEI2LossyJSON import TEI2LossyJSONConverter
+
+BIBL = ('