Skip to content

overhaul pg_rst.py to dump proper *.rst files #17

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zaihuaji merged 2 commits into from
Mar 26, 2026
Merged

overhaul pg_rst.py to dump proper *.rst files #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8702464
overhaul pg_rst.py to dump proper *.rst files
Mar 26, 2026
c223d9d
save for claude fix
Mar 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "rda_python_miscs"
version = "2.0.5"
version = "2.0.6"
authors = [
{ name="Zaihua Ji", email="zji@ucar.edu" },
]
Expand Down
97 changes: 47 additions & 50 deletions src/rda_python_miscs/pg_rst.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
from rda_python_common.pg_file import PgFile
from rda_python_common.pg_util import PgUtil


class PgRST(PgFile, PgUtil):
"""Convert text-based program usage documents (.usg files) into
reStructuredText (.rst) files using RST template files.
Expand All @@ -48,7 +47,7 @@ class PgRST(PgFile, PgUtil):
Q0 = "'"
Q1 = "**" # RST bold open (was "<i><b>")
Q2 = "**" # RST bold close (was "</i></b>")

TLEVEL = 3 # max section level to be rendered (1, 2, or 3)
EMLIST = {
'dsarch' : 1,
'msarch' : 1,
Expand All @@ -60,7 +59,6 @@ class PgRST(PgFile, PgUtil):
'rcm' : 1,
'dcm' : 1,
}

SEARCH = "(Action|Info|Mode|Multi-Value|Single-Value)"

def __init__(self):
Expand Down Expand Up @@ -116,7 +114,6 @@ def __init__(self):
'DOCTIT' : "", # document name in upper case letters
'DOCLNK' : None,
}

self.LINKS = ['dsarch', 'dsupdt', 'dsrqst', 'dscheck']

#
Expand Down Expand Up @@ -150,7 +147,7 @@ def process_docs(self, docname, opts, alias):
self.change_local_directory(self.DOCS['DOCDIR'], self.LGWNEX)
self.pglog("Write rst document '{}' under {}".format(docname, self.DOCS['DOCDIR']), self.LOGWRN)

if op.exists("index.rst"): # write index file once
if op.exists(op.join(self.DOCS['DOCDIR'], "index.rst")): # write index file once
self.pglog("index.rst exists already, delete first if needs to be regenerated", self.LOGWRN)
else:
self.write_index(self.sections[0])
Expand All @@ -164,12 +161,8 @@ def process_docs(self, docname, opts, alias):
def parse_docs(self, docname):
"""Read *docname*.usg and populate ``sections``, ``options``, and ``examples``.

Lines beginning with ``#`` are treated as comments and skipped. Inline
trailing comments are also stripped. Angle-bracketed uppercase tokens
(e.g. ``<FILENAME>``) are temporarily escaped to ``&ltFILENAME&gt``
so they are not misidentified as option markers (``<:>``, ``<=>``,
``<!>``) later in processing. They are unescaped back to ``<FILENAME>``
in :meth:`replace_option_link` before appearing in RST output.
Lines beginning with ``#`` are treated as comments and skipped. In-line
trailing comments are also stripped.

Args:
docname (str): Short document name used to locate ``<ORIGIN>/<docname>.usg``.
Expand All @@ -190,14 +183,6 @@ def parse_docs(self, docname):
else:
line = line.rstrip() # remove trailing white spaces

# Temporarily escape <UPPERCASE> tokens so they are not confused
# with special markers like <:>, <=>, <!> used in option parsing.
while True:
ms = re.search(r'(<([A-Z/\-\.]+)>)', line)
if ms:
line = line.replace(ms.group(1), "&lt{}&gt".format(ms.group(2)))
else:
break
ms = re.match(r'^([\d\.]+)\s+(.+)$', line)
if ms: # start new section
section = self.record_section(section, option, example, ms.group(1), ms.group(2))
Expand Down Expand Up @@ -298,10 +283,16 @@ def record_example(self, option, example, ndesc=None):
dict | None: A new example dict when *ndesc* is given, else ``None``.
"""
if example:
ms = re.match(r'^(.*)\.\s*(.*)$', example['desc'])
lines = example['desc'].split('\n')
first_line = lines[0]
rest = '\n'.join(lines[1:]) if len(lines) > 1 else ''
ms = re.match(r'^(.*)\.\s*(.*)$', first_line)
if ms:
example['title'] = ms.group(1)
example['desc'] = ms.group(2)
example['desc'] = (ms.group(2) + '\n' + rest) if rest else ms.group(2)
else:
example['title'] = first_line
example['desc'] = rest
option['exmidxs'].append(len(self.examples)) # record example index in option
self.examples.append(example) # record example globally

Expand Down Expand Up @@ -394,7 +385,8 @@ def init_example(self, opt, desc):
Returns:
dict: New example dict with keys ``opt``, ``title``, and ``desc``.
"""
return {'opt' : opt, 'title' : "", 'desc' : desc.title() + "\n"}
desc = (desc[0].upper() + desc[1:]) if desc else desc
return {'opt' : opt, 'title' : "", 'desc' : desc + "\n"}

#
# write the entry file: index.rst
Expand Down Expand Up @@ -481,38 +473,47 @@ def template_to_rst(self, template, hash, extra=None):
#
# create rst content for table of contents
#
def create_toc(self):
"""Build and return the RST table-of-contents string.
def create_toc(self, csection=None):
"""Build and return the RST table-of-contents string of a given section.

Produces a nested bullet list of section links (indented by section
level) followed by a flat Appendix A list of all example links.

Returns:
str: RST-formatted TOC content ready for ``__TOC__`` substitution.
"""

content = ""
clevel = csection['level'] if csection else 0
csecid = csection['secid'] if csection else ""
depth = self.TLEVEL - clevel

# nested bullet list for all sections
for section in self.sections:
secid = section['secid']
indent = " " * (section['level'] - 1)
content += "{}- `{}. {} <section{}.rst>`_\n".format(
indent, secid, section['title'], secid)

content += "\n"

# appendix A: list of examples
content += "**Appendix A: List of Examples**\n\n"

level = section['level']
if csecid:
if not secid.startswith(csecid + "."): continue
elif level > (clevel+1):
continue
content += " section{}\n".format(secid)

if not content: return ""

content = f".. toctree::\n :maxdepth: {depth}\n :caption: Table of Contents\n{content}\n"
# appendix A: list of examples for the parent section and its subsections
appendix = ""
idx = 1 # used as example index
for example in self.examples:
opt = example['opt']
for exm in self.examples:
opt = exm['opt']
option = self.options[opt]
secid = option['secid']
content += "- `A.{}. {} Option -{} (-{}) <section{}.rst#e{}>`_\n".format(
idx, option['type'], opt, option['name'], secid, idx)
if not csecid or secid == csecid or secid.startswith(csecid + "."):
appendix += "- `A.{}. {} Option -{} (-{}) <{}_e{}>`_\n".format(
idx, option['type'], opt, option['name'], secid, idx)
idx += 1
Comment on lines +504 to 514

Copilot AI Mar 26, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In create_toc(), example is initialized as a string and then reused as the loop variable (for example in self.examples), so example += ... will fail at runtime. Also psecid is referenced but never defined. Please rename the accumulator (e.g., examples_content) and use the intended section id filter variable (likely csecid).

Copilot uses AI. Check for mistakes.
content += "\n"
if appendix:
content += "**Appendix A: List of Examples**\n\n" + appendix + "\n"

return content

Expand All @@ -537,6 +538,7 @@ def create_section(self, section):
for opt in section['opts']:
content += self.create_option(opt, secid)

content += self.create_toc(section) # add a local TOC for the section and its subsections
return content

#
Expand Down Expand Up @@ -612,7 +614,7 @@ def create_option_name(self, opt, option):
def create_example(self, exmidx, secid):
"""Build the RST content for a single example.

Emits a ``.. _e<N>:`` anchor, a bold ``EXAMPLE N. <title>`` heading,
Emits a ``.. _secid_e<N>:`` anchor, a bold ``EXAMPLE N. <title>`` heading,
and the example's body description.

Args:
Expand All @@ -624,7 +626,7 @@ def create_example(self, exmidx, secid):
"""
example = self.examples[exmidx]
exm = exmidx + 1
content = "\n.. _e{}:\n\n".format(exm)
content = "\n.. _{}_e{}:\n\n".format(secid, exm)
content += "**EXAMPLE {}. {}**\n\n".format(exm, example['title'])
content += self.create_description(example['desc'], secid, 2)

Expand Down Expand Up @@ -713,7 +715,7 @@ def replace_option_link(self, line, csecid, ptype=None, dtype=None):
if ptype == 2 and re.search(r'Mode Options*', opt) and dtype == 3:
link = "{}`{} <mode_>`_{}".format(pre, opt, after)
else:
link = "{}`{} <section{}_>`_{}".format(pre, opt, secid, after)
link = "{}`{} <section{}>`_{}".format(pre, opt, secid, after)

Copilot AI Mar 26, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This link target uses <section{}> whereas other internal references in this module use the named-target form <..._> (see the option-link branch just above). As written, this will be treated as a URI-like relative link rather than a reference to the .. _section<id>: target. Please make the target consistent (or switch to Sphinx :ref:/:doc: roles).

Suggested change
link = "{}`{} <section{}>`_{}".format(pre, opt, secid, after)
link = "{}`{} <section{}_>`_{}".format(pre, opt, secid, after)

Copilot uses AI. Check for mistakes.
line = line.replace(replace, link)

ms = re.search(r'(https*://\S+)(\.|\,)', line)
Expand All @@ -734,10 +736,6 @@ def replace_option_link(self, line, csecid, ptype=None, dtype=None):
link = self.Q1 + opt + self.Q2
line = line.replace(replace, link)

# Unescape <UPPERCASE> tokens that were temporarily escaped during
# parsing to avoid confusion with option markers (<:>, <=>, <!>).
line = line.replace('&lt', '<').replace('&gt', '>')

return line

#
Expand Down Expand Up @@ -879,7 +877,6 @@ def create_table(self, lines, cnt, secid):

Detects three sub-formats:

* Lines starting with ``- `` → RST numbered list (``#.``).
* Lines ending with ``=>`` → RST line block (``|``).
* Lines split on `` - `` (key-value pairs) → ``.. list-table::`` directive.
* All other lines split on 2+ spaces → RST simple table.
Expand All @@ -900,7 +897,7 @@ def create_table(self, lines, cnt, secid):
line = lines[i]
ms = re.match(r'^\s+-\s+(.*)', line)
if ms:
content += "#. " + self.replace_option_link(ms.group(1), secid, 1) + "\n"
content += "* " + self.replace_option_link(ms.group(1), secid, 1) + "\n"
else:
content += " " + self.replace_option_link(line, secid, 1) + "\n"
content += "\n"
Expand Down Expand Up @@ -1031,7 +1028,7 @@ def create_synopsis(self, lines, cnt, secid, dtype):
if ms:
content += "| {}{}{} {}\n".format(self.Q1, self.DOCS['DOCNAM'], self.Q2, ms.group(1))
else:
content += "| {}\n".format(line.strip())
content += "|{}\n".format(line)
content += "\n"

return content
Expand Down Expand Up @@ -1077,11 +1074,11 @@ def get_title_link(self, title):
title (str): Section title text to look up.

Returns:
str: RST `` `title <sectionN.rst>`_ `` link, or *title* if not found.
str: RST `` `title <sectionN>`_ `` link, or *title* if not found.
"""
for section in self.sections:
if title == section['title']:
return "`{} <section{}.rst>`_".format(title, section['secid'])
return "`{} <section{}>`_".format(title, section['secid'])

Comment on lines 1079 to 1082

Copilot AI Mar 26, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

get_title_link() now returns a link with target <sectionN> instead of the named-target form <sectionN_>. This likely won’t resolve to the .. _sectionN: label and may produce a broken relative URL. Please align this with the rest of the module’s internal link strategy (e.g., sectionN_ or Sphinx :ref:).

Copilot uses AI. Check for mistakes.
return title

Expand Down
11 changes: 2 additions & 9 deletions src/rda_python_miscs/rst_templates/index.rst.temp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
################################################################################
#
# Title : index.temp
# Title : index_rst.temp
# Author : Zaihua Ji, zji@ucar.edu
# Date : 03/17/2026
# Purpose : template file for help document index.rst (reStructuredText)
#
# Work File : $DSSHOME/lib/templates/index.temp
# Github : https://github.com/NCAR/rda-python-mics.git
Comment on lines +3 to +7

Copilot AI Mar 26, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The header comment’s GitHub URL appears to have a typo ("rda-python-mics" vs this repo/package name "rda_python_miscs" / "rda-python-miscs"). Also the "Title" comment says index_rst.temp but the actual filename is index.rst.temp. Please update these header metadata lines to match the real file/repo.

Copilot uses AI. Check for mistakes.
#
################################################################################

Expand All @@ -15,10 +14,4 @@
A GUIDE TO __TITLE__
============================

.. toctree::
:maxdepth: 2
:caption: Contents

section__SECID__

__TOC__
5 changes: 2 additions & 3 deletions src/rda_python_miscs/rst_templates/section.rst.temp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
################################################################################
#
# Title : section.temp
# Title : section_rst.temp
# Author : Zaihua Ji, zji@ucar.edu
# Date : 03/17/2026
# Purpose : template file for help document section.rst (reStructuredText)
#
# Work File : $DSSHOME/lib/templates/section.temp
# Github : https://github.com/NCAR/rda-python-mics.git
Comment on lines +3 to +7

Copilot AI Mar 26, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The header comment’s GitHub URL appears to have a typo ("rda-python-mics" vs this repo/package name "rda_python_miscs" / "rda-python-miscs"). Also the "Title" comment says section_rst.temp but the actual filename is section.rst.temp. Please update these header metadata lines to match the real file/repo.

Copilot uses AI. Check for mistakes.
#
################################################################################

Expand Down
Loading