From b305703b1212774bf5be66ef8240978a1fa8c448 Mon Sep 17 00:00:00 2001 From: Ricci Adams Date: Thu, 3 Sep 2026 08:49:47 -0700 Subject: [PATCH] Add updated importer script and documentation --- tools/import_meeting_notes.py | 499 ++++++++++++++++++++++++---------- tools/tools.md | 78 ++++++ 2 files changed, 428 insertions(+), 149 deletions(-) create mode 100644 tools/tools.md diff --git a/tools/import_meeting_notes.py b/tools/import_meeting_notes.py index 33f5490..97bbbc7 100644 --- a/tools/import_meeting_notes.py +++ b/tools/import_meeting_notes.py @@ -1,17 +1,52 @@ """ -Tool to import meeting notes into per-month .md files +Tool to automatically import meeting notes using the HackMD API. """ -# To use: -# 1) Open https://hackmd.io/@matplotlib/SyePADPcxx -# 2) Save the current meeting notes as a .md file (/path/to/import.md) -# 3) python tools/import_meeting_notes.py --into meeting_notes/2026 /path/to/import.md - import argparse +import json as _json import re +import urllib.request +from collections import defaultdict from dataclasses import dataclass +from datetime import date from pathlib import Path -from datetime import datetime, date + + +MEETING_DATA = [{ + "title": "Matplotlib Weekly Meeting", + "note_id": "SyePADPcxx", # https://hackmd.io/@matplotlib/SyePADPcxx + "archive_dir": "meeting_notes", + "prune_count": 4 # Keep last 4 meetings and save back to HackMD +}] + +HEADING_RE = re.compile(r"^#\s+(.+?)\s*$") + +HEADING_WITH_DATE_RE = re.compile(r""" + ^\# # 1st-level heading + (?:.*?) # Optional text before date + ( + \d{1,2}/\d{1,2}/\d{4} | # 08/25/1982 + \d{1,2}/\d{1,2}/\d{2} | # 08/25/82 + (?: # Group that handles "Aug 25, 1982" / "25th Aug" / etc. + (?:\d+\s*(?:nd|rd|st|th)?\s+)? # Match a day before the month + (?: # Month name + January|February|March|April|May|June| + July|August|September|October|November|December| + Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec + ) + (?:\s+\d+\s*(?:nd|rd|st|th)?)? # Match a day after the month + (?:[\s,]*)? # Optional whitespace or comma + (?:[\d]{4})? # Optional year + ) + ) + (?:\s+.*?)?$ # Optional text after date +""", re.VERBOSE | re.IGNORECASE) + +IMPORTER_START_RE = re.compile(r"^\s*\s*$") +IMPORTER_END_RE = re.compile(r"^\s*\s*$") + +WHITESPACE_OR_HORIZONTAL_RULE_RE = re.compile(r"^[\s\-_\*]*$") +USERNAME_LINK_RE = re.compile(r"\[(\@[\w\-_]+)\]\(\@[\w\-_]+\)") @dataclass @@ -19,78 +54,93 @@ class Meeting: date: date lines: list[str] - def trimmed_contents(self) -> str: - """Removes leading/trailing lines which contain""" - """only whitespace or horizontal rule characters.""" - lines = self.lines - pattern = re.compile(r"^[\s\-_\*]*$") - start = 0 - while start < len(lines) and pattern.match(lines[start]): - start += 1 +@dataclass +class Document: + header: str + footer: str | None + meetings: list[Meeting] - end = len(lines) - while end > start and pattern.match(lines[end - 1]): - end -= 1 - return "\n".join(lines[start:end]) +def get_repository_path(): + script_path = Path(__file__).resolve() + for parent in [script_path.parent, *script_path.parents]: + if (parent / ".git").exists(): + return parent -class InputFile: - def __init__(self, path: Path, year: int): - self.path = path - self.year = year - self.issues = [] - self.meetings_dict = {} + raise RuntimeError("Could not find git repository root") - with open(self.path, "r", encoding="utf-8") as f: - contents = f.read() - filename = self.path.stem - contents = re.sub(r"", "", contents, flags=re.DOTALL) - - meeting = None - HEADING_RE = re.compile(r"^#\s+(.+?)\s*$") - FIRST_HEADING_RE = re.compile(r"^#\s+Matplotlib Weekly Meeting", re.IGNORECASE) - - for i, line in enumerate(contents.splitlines()): - match = HEADING_RE.match(line) - if match: - date = parse_date(match.group(1), self.year) - if date: - meeting = Meeting(date, []) - self.meetings_dict[date] = meeting - elif not FIRST_HEADING_RE.match(line): - self.issues.append( - f"{filename}:{i + 1}: Found dateless heading: '{line}'" - ) - elif meeting is not None: - meeting.lines.append(line) - - -def parse_date(date_str: str, default_year: int) -> date: - """Parses a date string in various formats into a date""" - text = date_str.strip() - text = re.sub(r"(\d+)(st|nd|rd|th)\b", r"\1", text) - text = re.sub(r"\bSept\b", "Sep", text) - text = re.sub(r",\s+", " ", text) - - for value in (text, f"{text} {default_year}"): - for fmt in [ - "%B %d %Y", # August 5 2020 - "%b %d %Y", # Aug 5 2020 - "%d %B %Y", # 5 August 2020 - "%d %b %Y", # 5 Aug 2020 - ]: - try: - return datetime.strptime(value, fmt) - except ValueError: - pass +def write_contents(path: Path, contents: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") - return None + +def call_hackmd_api( + access_token: str, + url: str, + json: dict | None = None, + method: str | None = None +) -> dict: + request = urllib.request.Request( + url, + data=_json.dumps(json).encode("utf-8") if json is not None else None, + headers={ + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + }, + method=method + ) + + with urllib.request.urlopen(request) as response: + response_bytes = response.read() + encoding = response.headers.get_content_charset("utf-8") + response_text = response_bytes.decode(encoding) + result = _json.loads(response_text) if response_text else {} + + return result + + +def get_team_note_url(note_id: str) -> str: + return f"https://api.hackmd.io/v1/teams/matplotlib/notes/{note_id}" + + +def save_to_hackmd( + access_token: str, + note_id: str, + contents: str, + debug_path: Path | None = None +) -> None: + if (debug_path): + write_contents(debug_path / "output" / f"{note_id}.md", contents) + else: + call_hackmd_api( + access_token, + url=get_team_note_url(note_id), + json={"content": contents}, + method="PATCH" + ) + + +def load_from_hackmd( + access_token: str, + note_id: str, + debug_path: Path | None = None +) -> str: + if debug_path: + local_md_path = debug_path / "input" / f"{note_id}.md" + with open(local_md_path, "r", encoding="utf-8") as f: + contents = f.read() + return contents + else: + url = get_team_note_url(note_id) + response_dict = call_hackmd_api(access_token, url) + return response_dict["content"] def get_year_from_path(path: Path) -> int | None: + """Finds the year in a Path.""" year = None for part in path.parts: if part.isdigit() and len(part) == 4: @@ -98,108 +148,259 @@ def get_year_from_path(path: Path) -> int | None: return year -def process_file( - path: Path, +def get_date_from_heading(heading: str, default_year: int) -> date | None: + """Attempts to extract a date from a Level 1 Heading (#).""" + match = HEADING_WITH_DATE_RE.match(heading) + + if match: + text = match[1].strip() + text = re.sub(r"(\d+)(nd|rd|st|th)\b", r"\1", text) + text = re.sub(r"\bSept\b", "Sep", text) + text = re.sub(r",\s+", " ", text) + + for value in (text, f"{text} {default_year}"): + for fmt in [ + "%m/%d/%y", # 8/25/82 + "%m/%d/%Y", # 8/25/1982 + "%B %d %Y", # August 25 1982 + "%b %d %Y", # Aug 25 1982 + "%d %B %Y", # 25 August 1982 + "%d %b %Y", # 25 Aug 1982 + ]: + try: + return date.strptime(value, fmt) + except ValueError: + pass + + return None + + +def get_meetings_from_lines( + lines: list[str], year: int, - meetings_dict: dict[date, Meeting], - issues: list[str] -) -> None: - input_file = InputFile(path, year) - meetings_dict.update(input_file.meetings_dict) - issues.extend(input_file.issues) + filename: str +) -> list[Meeting]: + meetings = [] + meeting = None + for line in lines: + if match := HEADING_RE.match(line): + date = get_date_from_heading(line, year) -def get_meetings_by_month( - meetings_dict: dict[date, Meeting], - month: int -) -> None: - meetings = meetings_dict.values() - return sorted( - (meeting for meeting in meetings if meeting.date.month == month), - key=lambda meeting: meeting.date, - ) + if date is None: + raise ValueError( + f"{filename}: Could not get date from heading '{line}'" + ) + meeting = Meeting(date, []) + meetings.append(meeting) + elif meeting is not None: + meeting.lines.append(line) -def main(): - parser = argparse.ArgumentParser(description="Process meeting notes") - parser.add_argument("--into", type=Path, required=True, - help="Directory containing archived .md files") - parser.add_argument("files", type=Path, nargs="*", help="Files to import") + return meetings - args = parser.parse_args() - issues = [] - into_meetings_dict = {} - import_meetings_dict = {} - year = get_year_from_path(args.into) +def get_meetings_from_archive_path(path: Path) -> list[Meeting]: + """Reads an on-disk Markdown file and extracts a list of Meeting objects.""" + with open(path, "r", encoding="utf-8") as f: + lines = f.read().splitlines() + + if len(lines) == 0 or HEADING_RE.match(lines[0]) is None: + raise ValueError(f"{path.name}: Could not find first heading") + + year = get_year_from_path(path) + + return get_meetings_from_lines(lines[1:], year, path.name) + - if not args.into.is_dir(): - raise ValueError(f"Path is not a directory: {args.into}") +def get_document_from_hackmd_note(note_id: str, contents: str) -> Document: + """Parses a Markdown content string and creates a *Document*.""" + lines = contents.splitlines() - if year is None: - raise ValueError(f"No year found in path: {args.into}") + # Find comment + try: + start = next( + i for i, line in enumerate(lines) + if IMPORTER_START_RE.match(line) + ) + except StopIteration: + raise ValueError(f"{note_id}: Missing ") - for path in args.into.glob("*.md"): - process_file(path, year, into_meetings_dict, issues) + # Find comment + try: + end = next( + i for i, line in enumerate(lines) + if IMPORTER_END_RE.match(line) + ) + except StopIteration: + end = len(lines) + + today = date.today() + meetings = get_meetings_from_lines(lines[start+1:end], today.year, note_id) + for meeting in meetings: + if meeting.date > today: + if today.month in (1, 2) and meeting.date.month in (11, 12): + meeting.date = date( + today.year - 1, + meeting.date.month, + meeting.date.day + ) + else: + raise ValueError(f"{note_id}: Date in the future: {meeting.date}") + + return Document( + "\n".join(lines[:start + 1]), + "\n".join(lines[end:]), + meetings + ) + + +def get_meeting_string(meeting: Meeting) -> str: + """Returns a Markdown string containing the meeting's content.""" + """Leading/trailing whitespace lines and horizontal rule lines are removed.""" + """Hackmd username links are converted to `@github_username`.""" + lines = meeting.lines - for path in args.files: - if path.is_dir(): - for md_path in path.glob("*.md"): - process_file(md_path, year, import_meetings_dict, issues) - elif path.exists(): - if path.suffix == ".md": - process_file(path, year, import_meetings_dict, issues) - else: - raise ValueError(f"Could not read path: {path}") + start = 0 + while start < len(lines) and WHITESPACE_OR_HORIZONTAL_RULE_RE.match(lines[start]): + start += 1 - # Output meeting notes by month - meetings_dict = {} - meetings_dict.update(into_meetings_dict) - meetings_dict.update(import_meetings_dict) + end = len(lines) + while end > start and WHITESPACE_OR_HORIZONTAL_RULE_RE.match(lines[end - 1]): + end -= 1 - if ( - len(get_meetings_by_month(meetings_dict, 1)) > 0 and - len(get_meetings_by_month(meetings_dict, 12)) > 0 - ): - issues.append("Import file(s) contain both January and December meetings.") + result = "\n".join(lines[start:end]) - for month in range(1, 13): - meetings = get_meetings_by_month(meetings_dict, month) + # Convert '[@efiring](@QWhXj01mSwmTjk5kN1H_qQ)' to '@efiring' + result = USERNAME_LINK_RE.sub(r"\1", result) - if len(meetings) == 0: - continue + return result - month_date = date(year, month, 1) - file_name = month_date.strftime("%Y_%m_%b.md").lower() - output = [ - f"# Matplotlib Weekly Meeting: {month_date.strftime('%B %Y')}", +def get_meetings_string(meetings: list[Meeting]) -> str: + """Generates a content string for the list of meetings.""" + result = [] + + for meeting in meetings: + date = meeting.date + + result.extend(( + "---", "", - f"###### tags: `{year} dev call`", + f"# {date.strftime('%B')} {date.day}, {date.year}", + "", + get_meeting_string(meeting), "" - ] + )) - for meeting in meetings: - output.extend(( - "---", - "", - f"# {meeting.date.strftime('%B')} {meeting.date.day}, {year}", + return "\n".join(result) + + +def get_pruned_contents(document: Document, prune_count: int) -> str: + """Generates a content string containing *document*'s header, footer, and""" + """*prune_count* meetings (sorted in reverse-chronological order).""" + output = [] + + if document.header: + output.append(document.header) + + latest_meetings = sorted( + document.meetings, + key=lambda m: m.date, reverse=True + )[:prune_count] + + output.extend(("", get_meetings_string(latest_meetings))) + + if document.footer: + output.append(document.footer) + + return "\n".join(output) + + +def import_meetings( + archive_path: Path, + title: str, + in_meetings: list[Meeting] +) -> None: + year_month_re = re.compile(r"\d{4}_\d{2}") + meetings_by_month = defaultdict(list) + + def add_meeting(meeting: Meeting) -> None: + meetings_by_month[(meeting.date.year, meeting.date.month)].append(meeting) + + # Grab all existing meetings + for path in archive_path.rglob("*.md"): + if year_month_re.search(path.stem): + for meeting in get_meetings_from_archive_path(path): + add_meeting(meeting) + + # Add new meetings + for meeting in in_meetings: + add_meeting(meeting) + + if len(meetings_by_month) == 0: + return + + min_year = min(year for year, month in meetings_by_month) + max_year = max(year for year, month in meetings_by_month) + + for year in range(min_year, max_year + 1): + for month in range(1, 13): + meetings = sorted( + meetings_by_month[(year, month)], + key=lambda meeting: meeting.date + ) + + if len(meetings) == 0: + continue + + month_date = date(year, month, 1) + file_name = month_date.strftime("%Y_%m_%b.md").lower() + + write_contents(archive_path / f"{year}" / file_name, "\n".join([ + f"# {title}: {month_date.strftime('%B %Y')}", "", - meeting.trimmed_contents(), - "" - )) - - meetings_noun = "meeting" if len(meetings) == 1 else "meetings" - print(f"Wrote {len(meetings)} {meetings_noun} to '{args.into / file_name}'") - with open(args.into / file_name, "w", encoding="utf-8") as f: - f.write("\n".join(output)) - - if len(issues): - print("") - print("Issues:") - print("=" * 80) - for issue in issues: - print(issue) + get_meetings_string(meetings) + ])) + + +def main(): + parser = argparse.ArgumentParser(description="Import meeting notes from HackMD") + parser.add_argument("--debug-path", type=Path, + help="Use local directory instead of calling the HackMD API (for debug)") + parser.add_argument("--hackmd-access-token", + help="Access token for HackMD API") + + args = parser.parse_args() + debug_path = args.debug_path + repo_path = get_repository_path() + + access_token = None + if debug_path is None: + access_token = args.hackmd_access_token + if access_token is None: + token_path = repo_path / ".hackmd_access_token" + print(token_path) + if token_path.exists(): + access_token = token_path.read_text().strip() + + if access_token is None: + raise ValueError("A HackMD access token was not found") + + for meeting_data_dict in MEETING_DATA: + note_id = meeting_data_dict["note_id"] + title = meeting_data_dict["title"] + prune_count = meeting_data_dict.get("prune_count") + archive_path = repo_path / meeting_data_dict["archive_dir"] + + hackmd_note_content = load_from_hackmd(access_token, note_id, debug_path) + document = get_document_from_hackmd_note(note_id, hackmd_note_content) + import_meetings(archive_path, title, document.meetings) + + if prune_count is not None: + updated_content = get_pruned_contents(document, prune_count) + if updated_content != hackmd_note_content: + save_to_hackmd(access_token, note_id, updated_content, debug_path) if __name__ == "__main__": diff --git a/tools/tools.md b/tools/tools.md new file mode 100644 index 0000000..09f8b30 --- /dev/null +++ b/tools/tools.md @@ -0,0 +1,78 @@ +# Tools + +## import_meeting_notes.py + +### Configuration + +Each meeting type should have a "live" note on HackMD as well as a corresponding directory in this Git repository. Currently, the script only imports the Weekly Meetings. New Contributors and GSOC could be added in the future. + +Configuration is kept in `MEETING_DATA` at the top of the file: + +```python +MEETING_DATA = [{ + "title": "Matplotlib Weekly Meeting", + "note_id": "SyePADPcxx", # https://hackmd.io/@matplotlib/SyePADPcxx + "archive_dir": "meeting_notes", + "prune_count": 4 # Keep last 4 meetings and save back to HackMD +}] +``` + +| Key | Description | +|-|-| +| `title` | Used as the title/first-heading for each archived .md file. | +| `note_id` | The HackMD note ID for the "live" note. | +| `archive_dir` | The path in the git repository to use as the meeting note archive. Relative to the repository root. | +| `prune_count` | If present, the number of meetings to keep in the "live" note. | + +### HackMD API Key + +The [HackMD API token](https://hackmd.io/@docs/how-to-issue-an-api-token) can be provided in one of the following ways: + +- The `--hackmd-api-token` command-line argument. +- The `HACKMD_API_TOKEN` environment variable. +- A `.hackmd_api_token` file at the root level of the repository. + + +### HackMD Note Format + +The HackMD note should have the following format: + +```markdown +# First Heading + +Some header information about the meeting. + + + +# February 6, 2026 + +Notes for February 6 meeting. + +# January 30th + +Notes for January 30 meeting. + +…etc. + + + +An optional footer for the meeting. +``` + +- `` is important, it tells the script where to start looking for meetings. +- `` is optional. If missing, the script will assume that the end of the file is ``. +- Each meeting must have a date as a `#` heading. The exact format of the date is flexible. +- No other `#` headings may be present between `` and ``. +- If using `prune_count`, only the content between `` and `` will be modified when the note is pruned. + + +### Overview of Import Process + +1. The script iterates through all ".md" files in the `archive_dir` directory and extracts the notes for all existing meetings. +2. The script downloads the latest meeting notes from HackMD using the [public API](https://hackmd.io/@docs/Getting-Started-with-the-HackMD-API). Any meetings in the "live" document will overwrite archived meetings. +3. The script creates a `%Y_%m_%b.md` file in the corresponding `%Y` directory for each month with meetings. + + +### Pruning + +When `prune_count` is specified, the script will write an updated version of the "live" meeting notes back to HackMD with the latest `prune_count` meetings. To match historical usage, these will be arranged in reverse-chronological order (latest meeting on top).