Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 36 additions & 38 deletions grobid_client/format/TEI2LossyJSON.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,12 @@

import dateparser
from bs4 import BeautifulSoup, Tag
from bs4.element import CData, PreformattedString

# Configure module-level logger
# Module-level logger. Handlers and levels are the application's business:
# configuring the root logger at import time would override the logging
# set up by whoever imports this module (the format CLIs included).
logger: logging.Logger = logging.getLogger(__name__)
logger.propagate = False # Prevent propagation to avoid duplicate logs

# Only configure basic logging if nothing is set up yet
if not logger.handlers and not logging.getLogger().handlers:
# Basic configuration if not already configured by the application
logging.basicConfig(level=logging.INFO)


class TEI2LossyJSONConverter:
Expand Down Expand Up @@ -147,7 +144,9 @@ def convert_tei_file(self, tei_file: Union[str, Path, BinaryIO], stream: bool =
[
{
"id": sentence.get("xml:id") if sentence.has_attr("xml:id") else id,
"text": sentence.text,
# the same whitespace normalisation the
# ref offsets are computed against
"text": _normalise_passage_text(sentence.get_text()),
"coords": [
box_to_dict(coord.split(","))
for coord in sentence['coords'].split(";")
Expand All @@ -162,7 +161,7 @@ def convert_tei_file(self, tei_file: Union[str, Path, BinaryIO], stream: bool =
biblio_structure["abstract"] = [
{
"id": id,
"text": paragraph.text,
"text": _normalise_passage_text(paragraph.get_text()),
"coords": [
box_to_dict(coord.split(","))
for coord in paragraph['coords'].split(";")
Expand Down Expand Up @@ -554,17 +553,13 @@ def _compile_citation_data(self, citation_data: dict[str, Any], contributors: li
citation_data['urls'] = links

def _validate_citation_content(self, citation_data: dict[str, Any]) -> bool:
"""Validate that the citation contains meaningful information."""
# Check for essential bibliographic elements
essential_elements = ['title', 'authors', 'journal', 'doi', 'isbn', 'issn', 'pmc', 'pmid']

# Check if any essential element is present
has_essential = any(citation_data.get(element) for element in essential_elements)
"""Validate that the citation carries any information beyond its id.

# Check for fallback elements
has_fallback = any(citation_data.get(element) for element in ['raw_reference', 'reference_text'])

return has_essential or has_fallback
A shortlist of "essential" fields used to decide this, which dropped
entries GROBID describes with an arXiv id, a URL, editors or a date
only, and left the in-text references pointing at them dangling.
"""
return any(value for key, value in citation_data.items() if key not in ('id', 'target'))

def _extract_person_data(self, person_element: Tag) -> dict[str, Any] | None:
"""
Expand Down Expand Up @@ -852,7 +847,7 @@ def process_directory(self, directory: Union[str, Path], pattern: str = "*.tei.x
# Use processes for CPU-bound parsing when many files are available
workers = workers or min(32, (os.cpu_count() or 1))
with ProcessPoolExecutor(max_workers=workers) as ex:
futures = {ex.submit(_convert_file_worker, str(f)): f for f in files}
futures = {ex.submit(_convert_file_worker, str(f), self.validate_refs): f for f in files}
for fut in as_completed(futures):
f = futures[fut]
try:
Expand All @@ -863,15 +858,9 @@ def process_directory(self, directory: Union[str, Path], pattern: str = "*.tei.x
yield {"path": f, "document": doc}


def _convert_file_worker(path: str) -> Any:
"""Worker used by ProcessPoolExecutor. Imports inside function to avoid pickling issues."""
from bs4 import BeautifulSoup
# Reuse existing top-level helpers from this module by importing here
with open(path, 'r') as f:
content = f.read()
soup = BeautifulSoup(content, 'xml')
converter = TEI2LossyJSONConverter()
return converter.convert_tei_file(path, stream=False)
def _convert_file_worker(path: str, validate_refs: bool = True) -> Any:
"""Worker used by ProcessPoolExecutor: convert one file with the caller's settings."""
return TEI2LossyJSONConverter(validate_refs=validate_refs).convert_tei_file(path, stream=False)


def box_to_dict(coord_list: list[str]) -> dict[str, float]:
Expand Down Expand Up @@ -983,6 +972,11 @@ def traverse(node, pos=0):
if child_text is not None:
parts.append(child_text)
return "".join(parts), pos
elif isinstance(node, PreformattedString) and not isinstance(node, CData):
# Comments, processing instructions and declarations are not
# part of get_text(), which gives the passage its text: counting
# them here would shift every offset after them
return "", pos
else:
s = str(node)
return s, pos + len(s)
Expand Down Expand Up @@ -1018,17 +1012,21 @@ def traverse(node, pos=0):
return final_refs


def _normalise_passage_text(text: str) -> str:
"""Collapse whitespace runs and strip: the text that ref offsets index into.

get_refs_with_offsets maps its offsets onto exactly this normalisation, so
every passage carrying refs must have its text built with it.
"""
if not text:
return ""
# DO NOT call html.unescape(text) here, to maintain parity and prevent offset corruption.
return re.sub(r'\s+', ' ', text.strip())


def get_formatted_passage(head_paragraph: str | None, head_section: str | None, paragraph_id: str, element: Tag) -> dict[str, Any]:
"""Format a passage (paragraph or sentence) with metadata and references."""
# Import the clean_text method
def _clean_text_local(text: str) -> str:
if not text:
return ""
text = re.sub(r'\s+', ' ', text.strip())
# DO NOT call html.unescape(text) here either, to maintain parity and prevent offset corruption.
return text

text = _clean_text_local(element.get_text())
text = _normalise_passage_text(element.get_text())
refs = get_refs_with_offsets(element)

passage = {
Expand Down
6 changes: 3 additions & 3 deletions grobid_client/format/TEI2LossyJSON_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,13 @@ def main() -> None:
epilog="""
Examples:
# Convert a single TEI file
python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml --output output.json
python -m grobid_client.format TEI2LossyJSON --input input.tei.xml --output output.json

# Convert with verbose logging
python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml --output output.json --verbose
python -m grobid_client.format TEI2LossyJSON --input input.tei.xml --output output.json --verbose

# Convert and output to stdout
python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml
python -m grobid_client.format TEI2LossyJSON --input input.tei.xml
"""
)

Expand Down
52 changes: 32 additions & 20 deletions grobid_client/format/TEI2Markdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,10 @@
import logging
from datetime import datetime

# Configure module-level logger
# Module-level logger. Handlers and levels are the application's business:
# configuring the root logger at import time would override the logging
# set up by whoever imports this module (the format CLIs included).
logger = logging.getLogger(__name__)
if not logger.handlers:
# Basic configuration if not already configured by the application
logging.basicConfig(level=logging.INFO)


class TEI2MarkdownConverter:
Expand Down Expand Up @@ -453,22 +452,29 @@ def _process_formula(self, formula_element: Tag) -> str:

def _table_to_markdown(self, table_element: Tag) -> str:
"""Convert a table element to simple markdown."""
markdown_lines = []

# Process table rows
rows = []
for row in table_element.find_all("row"):
cells = []
for cell in row.find_all("cell"):
cell_text = cell.get_text().strip()
cells.append(cell_text)

# A cell is one line of a table row: collapse its line breaks and
# escape the pipes that would otherwise end the cell early
cells = [re.sub(r"\s+", " ", cell.get_text()).strip().replace("|", "\\|")
for cell in row.find_all("cell")]
if cells:
markdown_lines.append("| " + " | ".join(cells) + " |")
if len(markdown_lines) == 1:
# a separator after the first row makes it a Markdown table
markdown_lines.append("|" + "|".join([" --- "] * len(cells)) + "|")
rows.append(cells)
if not rows:
return ""

# Every row gets as many cells as the widest one: renderers drop the
# cells of a row that go past the separator row's column count
width = max(len(cells) for cells in rows)
markdown_lines = []
for cells in rows:
cells = cells + [""] * (width - len(cells))
markdown_lines.append("| " + " | ".join(cells) + " |")
if len(markdown_lines) == 1:
# a separator after the first row makes it a Markdown table
markdown_lines.append("|" + "|".join([" --- "] * width) + "|")

return "\n".join(markdown_lines) if markdown_lines else ""
return "\n".join(markdown_lines)

def _format_reference(self, bibl_struct: Tag, ref_num: int) -> str:
"""
Expand Down Expand Up @@ -619,10 +625,13 @@ def _process_series_section(self, series: Tag, bib_data: dict[str, Any]) -> None

def _process_imprint_section(self, imprint: Tag, bib_data: dict[str, Any]) -> None:
"""Process the imprint section containing publication details."""
# Extract publication date
date = imprint.find("date")
# Extract publication date. GROBID often writes it as an attribute
# only (<date type="published" when="2019"/>), with no text.
date = imprint.find("date", attrs={"type": "published"}) or imprint.find("date")
if date:
bib_data['year'] = self._extract_year(date.get_text().strip())
date_value = date.get("when") or date.get_text().strip()
if date_value:
bib_data['year'] = self._extract_year(date_value)

# Extract publication details from biblScope elements
for bibl_scope in imprint.find_all("biblScope"):
Expand Down Expand Up @@ -787,6 +796,9 @@ def _build_identifiers_and_links(self, ref_data: dict[str, Any]) -> list[str]:
identifiers_and_links.append(f"{id_type.upper()}: {id_value}")
elif id_type.lower() in ['isbn', 'issn']:
identifiers_and_links.append(f"{id_type.upper()}: {id_value}")
elif id_type.lower() == 'arxiv':
arxiv_id = re.sub(r'^arxiv:\s*', '', id_value, flags=re.IGNORECASE)
identifiers_and_links.append(f"[arXiv:{arxiv_id}](https://arxiv.org/abs/{arxiv_id})")

# Format URLs with display-friendly text. GROBID's <ptr> for a DOI
# is the same doi.org link already printed above: skip it.
Expand Down
6 changes: 3 additions & 3 deletions grobid_client/format/TEI2Markdown_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,13 @@ def main() -> None:
epilog="""
Examples:
# Convert a single TEI file
python -m grobid_client.format.TEI2Markdown --input input.tei.xml --output output.md
python -m grobid_client.format TEI2Markdown --input input.tei.xml --output output.md

# Convert with verbose logging
python -m grobid_client.format.TEI2Markdown --input input.tei.xml --output output.md --verbose
python -m grobid_client.format TEI2Markdown --input input.tei.xml --output output.md --verbose

# Convert and output to stdout
python -m grobid_client.format.TEI2Markdown --input input.tei.xml
python -m grobid_client.format TEI2Markdown --input input.tei.xml
"""
)

Expand Down
11 changes: 5 additions & 6 deletions grobid_client/format/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,14 @@
"""
from __future__ import annotations

import argparse
import sys


def main() -> None:
"""Main entry point that provides a menu for converter selection."""

# Check if a converter was specified
if len(sys.argv) < 2:
# Without a converter, print the menu; asking for help gets it too
if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"):
print("GROBID format converters - Choose a converter to run")
print("\nUsage:")
print(" python -m grobid_client.format <converter> [options]")
Expand All @@ -26,7 +25,7 @@ def main() -> None:
print("\nGet help for specific converter:")
print(" python -m grobid_client.format TEI2LossyJSON --help")
print(" python -m grobid_client.format TEI2Markdown --help")
sys.exit(1)
sys.exit(0 if len(sys.argv) >= 2 else 1)

converter = sys.argv[1]

Expand All @@ -41,8 +40,8 @@ def main() -> None:
sys.argv = ["TEI2Markdown"] + sys.argv[2:]
markdown_main()
else:
print(f"Unknown converter: {converter}")
print("Available converters: TEI2LossyJSON, TEI2Markdown")
print(f"Unknown converter: {converter}", file=sys.stderr)
print("Available converters: TEI2LossyJSON, TEI2Markdown", file=sys.stderr)
sys.exit(1)


Expand Down
42 changes: 32 additions & 10 deletions grobid_client/format/validate_json_refs.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,16 @@ def __init__(self, verbose: bool = False) -> None:
}

def validate_directory(self, directory_path: str) -> Dict[str, Any]:
"""Validate all JSON files in a directory or a single JSON file."""
"""Validate all JSON files under a directory (recursively) or a single JSON file."""
# Check if it's a single file
if os.path.isfile(directory_path):
if not directory_path.endswith('.json'):
raise ValueError(f"File must be a JSON file: {directory_path}")
json_files = [Path(directory_path)]
elif os.path.isdir(directory_path):
json_files = list(Path(directory_path).glob("*.json"))
# The client mirrors the input tree, so the converted files of a
# recursive run sit in subdirectories of the output directory
json_files = sorted(Path(directory_path).rglob("*.json"))
else:
raise ValueError(f"Path does not exist: {directory_path}")

Expand Down Expand Up @@ -96,11 +98,23 @@ def _validate_file(self, file_path: str) -> None:
'warnings': []
}

if not isinstance(data, dict):
error_msg = f"{file_path}: not a converted document (top-level JSON {type(data).__name__})"
self.results['errors'].append(error_msg)
self.results['invalid_files'] += 1
if self.verbose:
print(f"❌ {error_msg}")
return

# Validate different parts of the JSON structure
self._validate_body_text_refs(data, file_result)
self._validate_abstract_refs(data, file_result)
self._validate_other_sections(data, file_result)

# A file with a single bad reference is not a valid file
if file_result['invalid_refs'] > 0:
file_result['valid'] = False

# Update overall results
self.results['total_refs'] += file_result['total_refs']
self.results['valid_refs'] += file_result['valid_refs']
Expand Down Expand Up @@ -141,19 +155,27 @@ def _validate_body_text_refs(self, data: Dict[str, Any], file_result: Dict[str,

def _validate_abstract_refs(self, data: Dict[str, Any], file_result: Dict[str, Any]) -> None:
"""Validate references in abstract section."""
if 'biblio' not in data or 'abstract' not in data['biblio']:
return

for i, paragraph in enumerate(data['biblio']['abstract']):
if 'text' not in paragraph or 'refs' not in paragraph:
abstract = (data.get('biblio') or {}).get('abstract') or []

# At sentence level the abstract is a list of paragraphs, each a list
# of sentences; at paragraph level it is a flat list of paragraphs
passages = []
for i, item in enumerate(abstract):
if isinstance(item, list):
passages.extend((f"biblio.abstract[{i}][{k}]", sentence) for k, sentence in enumerate(item))
else:
passages.append((f"biblio.abstract[{i}]", item))

for location, paragraph in passages:
if not isinstance(paragraph, dict) or 'text' not in paragraph or 'refs' not in paragraph:
continue

text = paragraph['text']
refs = paragraph.get('refs', [])
file_result['total_refs'] += len(refs)

for j, ref in enumerate(refs):
is_valid, error = self._validate_single_ref(text, ref, f"biblio.abstract[{i}].refs[{j}]")
is_valid, error = self._validate_single_ref(text, ref, f"{location}.refs[{j}]")
if is_valid:
file_result['valid_refs'] += 1
else:
Expand Down Expand Up @@ -298,14 +320,14 @@ def save_json_report(self, output_path: str) -> None:
json.dump(report_data, f, indent=2, ensure_ascii=False)


def main() -> None:
def main() -> int:
"""Main function."""
parser = argparse.ArgumentParser(
description="Validate reference offsets in JSON files generated from TEI documents"
)
parser.add_argument(
"directory",
help="Directory containing JSON files to validate"
help="JSON file, or directory searched recursively for JSON files, to validate"
)
parser.add_argument(
"--verbose", "-v",
Expand Down
Loading
Loading