From 87024641558b32df0a87cfdaf29d082c12f0ed97 Mon Sep 17 00:00:00 2001 From: zaihuaji Date: Thu, 26 Mar 2026 16:27:42 -0500 Subject: [PATCH 1/2] overhaul pg_rst.py to dump proper *.rst files --- pyproject.toml | 2 +- src/rda_python_miscs/pg_rst.py | 79 ++++++++----------- .../rst_templates/index.rst.temp | 11 +-- .../rst_templates/section.rst.temp | 5 +- 4 files changed, 40 insertions(+), 57 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index be749ca..cd8ded6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "rda_python_miscs" -version = "2.0.5" +version = "2.0.6" authors = [ { name="Zaihua Ji", email="zji@ucar.edu" }, ] diff --git a/src/rda_python_miscs/pg_rst.py b/src/rda_python_miscs/pg_rst.py index c683ead..f487cd4 100755 --- a/src/rda_python_miscs/pg_rst.py +++ b/src/rda_python_miscs/pg_rst.py @@ -21,7 +21,7 @@ from os import path as op from rda_python_common.pg_file import PgFile from rda_python_common.pg_util import PgUtil - +from posix import WIFCONTINUED class PgRST(PgFile, PgUtil): """Convert text-based program usage documents (.usg files) into @@ -48,7 +48,7 @@ class PgRST(PgFile, PgUtil): Q0 = "'" Q1 = "**" # RST bold open (was "") Q2 = "**" # RST bold close (was "") - + TLEVEL = 3 # max section level to be rendered (1, 2, or 3) EMLIST = { 'dsarch' : 1, 'msarch' : 1, @@ -60,7 +60,6 @@ class PgRST(PgFile, PgUtil): 'rcm' : 1, 'dcm' : 1, } - SEARCH = "(Action|Info|Mode|Multi-Value|Single-Value)" def __init__(self): @@ -116,7 +115,6 @@ def __init__(self): 'DOCTIT' : "", # document name in upper case letters 'DOCLNK' : None, } - self.LINKS = ['dsarch', 'dsupdt', 'dsrqst', 'dscheck'] # @@ -164,12 +162,8 @@ def process_docs(self, docname, opts, alias): def parse_docs(self, docname): """Read *docname*.usg and populate ``sections``, ``options``, and ``examples``. - Lines beginning with ``#`` are treated as comments and skipped. Inline - trailing comments are also stripped. Angle-bracketed uppercase tokens - (e.g. ````) are temporarily escaped to ``<FILENAME>`` - so they are not misidentified as option markers (``<:>``, ``<=>``, - ````) later in processing. They are unescaped back to ```` - in :meth:`replace_option_link` before appearing in RST output. + Lines beginning with ``#`` are treated as comments and skipped. In-line + trailing comments are also stripped. Args: docname (str): Short document name used to locate ``/.usg``. @@ -190,14 +184,6 @@ def parse_docs(self, docname): else: line = line.rstrip() # remove trailing white spaces - # Temporarily escape tokens so they are not confused - # with special markers like <:>, <=>, used in option parsing. - while True: - ms = re.search(r'(<([A-Z/\-\.]+)>)', line) - if ms: - line = line.replace(ms.group(1), "<{}>".format(ms.group(2))) - else: - break ms = re.match(r'^([\d\.]+)\s+(.+)$', line) if ms: # start new section section = self.record_section(section, option, example, ms.group(1), ms.group(2)) @@ -481,8 +467,8 @@ def template_to_rst(self, template, hash, extra=None): # # create rst content for table of contents # - def create_toc(self): - """Build and return the RST table-of-contents string. + def create_toc(self, csection=None): + """Build and return the RST table-of-contents string of a given section. Produces a nested bullet list of section links (indented by section level) followed by a flat Appendix A list of all example links. @@ -490,29 +476,38 @@ def create_toc(self): Returns: str: RST-formatted TOC content ready for ``__TOC__`` substitution. """ + content = "" + clevel = section['level'] if csection else 0 + csecid = section['secid'] if csection else "" + depth = self.TLEVEL - clevel # nested bullet list for all sections for section in self.sections: secid = section['secid'] - indent = " " * (section['level'] - 1) - content += "{}- `{}. {} `_\n".format( - indent, secid, section['title'], secid) - - content += "\n" - - # appendix A: list of examples - content += "**Appendix A: List of Examples**\n\n" - + level = section['level'] + if csecid: + if not secid.startswith(csecid + "."): continue + elif level > (clevel+1): + continue + content += " section{}\n".format(secid) + + if not content: return "" + + content = f".. toctree::\n :maxdepth: {depth}\n :caption: Table of Contents\n{content}\n" + # appendix A: list of examples for the parent section and its subsections + example = "" idx = 1 # used as example index for example in self.examples: opt = example['opt'] option = self.options[opt] secid = option['secid'] - content += "- `A.{}. {} Option -{} (-{}) `_\n".format( - idx, option['type'], opt, option['name'], secid, idx) + if not psecid or secid.startswith(psecid + "."): + example += "- `A.{}. {} Option -{} (-{}) <{}_e{}>`_\n".format( + idx, option['type'], opt, option['name'], secid, idx) idx += 1 - content += "\n" + if example: + content += "**Appendix A: List of Examples**\n\n" + example + "\n" return content @@ -537,6 +532,7 @@ def create_section(self, section): for opt in section['opts']: content += self.create_option(opt, secid) + content += create_toc(section) # add a local TOC for the section and its subsections return content # @@ -612,7 +608,7 @@ def create_option_name(self, opt, option): def create_example(self, exmidx, secid): """Build the RST content for a single example. - Emits a ``.. _e:`` anchor, a bold ``EXAMPLE N. `` heading, + Emits a ``.. _secid_e<N>:`` anchor, a bold ``EXAMPLE N. <title>`` heading, and the example's body description. Args: @@ -624,7 +620,7 @@ def create_example(self, exmidx, secid): """ example = self.examples[exmidx] exm = exmidx + 1 - content = "\n.. _e{}:\n\n".format(exm) + content = "\n.. _{}_e{}:\n\n".format(secid, exm) content += "**EXAMPLE {}. {}**\n\n".format(exm, example['title']) content += self.create_description(example['desc'], secid, 2) @@ -713,7 +709,7 @@ def replace_option_link(self, line, csecid, ptype=None, dtype=None): if ptype == 2 and re.search(r'Mode Options*', opt) and dtype == 3: link = "{}`{} <mode_>`_{}".format(pre, opt, after) else: - link = "{}`{} <section{}_>`_{}".format(pre, opt, secid, after) + link = "{}`{} <section{}>`_{}".format(pre, opt, secid, after) line = line.replace(replace, link) ms = re.search(r'(https*://\S+)(\.|\,)', line) @@ -734,10 +730,6 @@ def replace_option_link(self, line, csecid, ptype=None, dtype=None): link = self.Q1 + opt + self.Q2 line = line.replace(replace, link) - # Unescape <UPPERCASE> tokens that were temporarily escaped during - # parsing to avoid confusion with option markers (<:>, <=>, <!>). - line = line.replace('<', '<').replace('>', '>') - return line # @@ -879,7 +871,6 @@ def create_table(self, lines, cnt, secid): Detects three sub-formats: - * Lines starting with ``- `` → RST numbered list (``#.``). * Lines ending with ``=>`` → RST line block (``|``). * Lines split on `` - `` (key-value pairs) → ``.. list-table::`` directive. * All other lines split on 2+ spaces → RST simple table. @@ -900,7 +891,7 @@ def create_table(self, lines, cnt, secid): line = lines[i] ms = re.match(r'^\s+-\s+(.*)', line) if ms: - content += "#. " + self.replace_option_link(ms.group(1), secid, 1) + "\n" + content += "*. " + self.replace_option_link(ms.group(1), secid, 1) + "\n" else: content += " " + self.replace_option_link(line, secid, 1) + "\n" content += "\n" @@ -1031,7 +1022,7 @@ def create_synopsis(self, lines, cnt, secid, dtype): if ms: content += "| {}{}{} {}\n".format(self.Q1, self.DOCS['DOCNAM'], self.Q2, ms.group(1)) else: - content += "| {}\n".format(line.strip()) + content += "|{}\n".format(line) content += "\n" return content @@ -1077,11 +1068,11 @@ def get_title_link(self, title): title (str): Section title text to look up. Returns: - str: RST `` `title <sectionN.rst>`_ `` link, or *title* if not found. + str: RST `` `title <sectionN>`_ `` link, or *title* if not found. """ for section in self.sections: if title == section['title']: - return "`{} <section{}.rst>`_".format(title, section['secid']) + return "`{} <section{}>`_".format(title, section['secid']) return title diff --git a/src/rda_python_miscs/rst_templates/index.rst.temp b/src/rda_python_miscs/rst_templates/index.rst.temp index 9f6bcc2..b44f52a 100644 --- a/src/rda_python_miscs/rst_templates/index.rst.temp +++ b/src/rda_python_miscs/rst_templates/index.rst.temp @@ -1,11 +1,10 @@ ################################################################################ # -# Title : index.temp +# Title : index_rst.temp # Author : Zaihua Ji, zji@ucar.edu # Date : 03/17/2026 # Purpose : template file for help document index.rst (reStructuredText) -# -# Work File : $DSSHOME/lib/templates/index.temp +# Github : https://github.com/NCAR/rda-python-mics.git # ################################################################################ @@ -15,10 +14,4 @@ A GUIDE TO __TITLE__ ============================ -.. toctree:: - :maxdepth: 2 - :caption: Contents - - section__SECID__ - __TOC__ \ No newline at end of file diff --git a/src/rda_python_miscs/rst_templates/section.rst.temp b/src/rda_python_miscs/rst_templates/section.rst.temp index c80b7dd..22a3876 100644 --- a/src/rda_python_miscs/rst_templates/section.rst.temp +++ b/src/rda_python_miscs/rst_templates/section.rst.temp @@ -1,11 +1,10 @@ ################################################################################ # -# Title : section.temp +# Title : section_rst.temp # Author : Zaihua Ji, zji@ucar.edu # Date : 03/17/2026 # Purpose : template file for help document section.rst (reStructuredText) -# -# Work File : $DSSHOME/lib/templates/section.temp +# Github : https://github.com/NCAR/rda-python-mics.git # ################################################################################ From c223d9dec89d60df46e595ee2c600ffe06e4ecba Mon Sep 17 00:00:00 2001 From: zaihuaji <zaihuaji@gmail.com> Date: Thu, 26 Mar 2026 18:51:38 -0500 Subject: [PATCH 2/2] save for claude fix --- src/rda_python_miscs/pg_rst.py | 38 ++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/rda_python_miscs/pg_rst.py b/src/rda_python_miscs/pg_rst.py index f487cd4..c87de95 100755 --- a/src/rda_python_miscs/pg_rst.py +++ b/src/rda_python_miscs/pg_rst.py @@ -21,7 +21,6 @@ from os import path as op from rda_python_common.pg_file import PgFile from rda_python_common.pg_util import PgUtil -from posix import WIFCONTINUED class PgRST(PgFile, PgUtil): """Convert text-based program usage documents (.usg files) into @@ -148,7 +147,7 @@ def process_docs(self, docname, opts, alias): self.change_local_directory(self.DOCS['DOCDIR'], self.LGWNEX) self.pglog("Write rst document '{}' under {}".format(docname, self.DOCS['DOCDIR']), self.LOGWRN) - if op.exists("index.rst"): # write index file once + if op.exists(op.join(self.DOCS['DOCDIR'], "index.rst")): # write index file once self.pglog("index.rst exists already, delete first if needs to be regenerated", self.LOGWRN) else: self.write_index(self.sections[0]) @@ -284,10 +283,16 @@ def record_example(self, option, example, ndesc=None): dict | None: A new example dict when *ndesc* is given, else ``None``. """ if example: - ms = re.match(r'^(.*)\.\s*(.*)$', example['desc']) + lines = example['desc'].split('\n') + first_line = lines[0] + rest = '\n'.join(lines[1:]) if len(lines) > 1 else '' + ms = re.match(r'^(.*)\.\s*(.*)$', first_line) if ms: example['title'] = ms.group(1) - example['desc'] = ms.group(2) + example['desc'] = (ms.group(2) + '\n' + rest) if rest else ms.group(2) + else: + example['title'] = first_line + example['desc'] = rest option['exmidxs'].append(len(self.examples)) # record example index in option self.examples.append(example) # record example globally @@ -380,7 +385,8 @@ def init_example(self, opt, desc): Returns: dict: New example dict with keys ``opt``, ``title``, and ``desc``. """ - return {'opt' : opt, 'title' : "", 'desc' : desc.title() + "\n"} + desc = (desc[0].upper() + desc[1:]) if desc else desc + return {'opt' : opt, 'title' : "", 'desc' : desc + "\n"} # # write the entry file: index.rst @@ -478,8 +484,8 @@ def create_toc(self, csection=None): """ content = "" - clevel = section['level'] if csection else 0 - csecid = section['secid'] if csection else "" + clevel = csection['level'] if csection else 0 + csecid = csection['secid'] if csection else "" depth = self.TLEVEL - clevel # nested bullet list for all sections @@ -496,18 +502,18 @@ def create_toc(self, csection=None): content = f".. toctree::\n :maxdepth: {depth}\n :caption: Table of Contents\n{content}\n" # appendix A: list of examples for the parent section and its subsections - example = "" + appendix = "" idx = 1 # used as example index - for example in self.examples: - opt = example['opt'] + for exm in self.examples: + opt = exm['opt'] option = self.options[opt] secid = option['secid'] - if not psecid or secid.startswith(psecid + "."): - example += "- `A.{}. {} Option -{} (-{}) <{}_e{}>`_\n".format( + if not csecid or secid == csecid or secid.startswith(csecid + "."): + appendix += "- `A.{}. {} Option -{} (-{}) <{}_e{}>`_\n".format( idx, option['type'], opt, option['name'], secid, idx) idx += 1 - if example: - content += "**Appendix A: List of Examples**\n\n" + example + "\n" + if appendix: + content += "**Appendix A: List of Examples**\n\n" + appendix + "\n" return content @@ -532,7 +538,7 @@ def create_section(self, section): for opt in section['opts']: content += self.create_option(opt, secid) - content += create_toc(section) # add a local TOC for the section and its subsections + content += self.create_toc(section) # add a local TOC for the section and its subsections return content # @@ -891,7 +897,7 @@ def create_table(self, lines, cnt, secid): line = lines[i] ms = re.match(r'^\s+-\s+(.*)', line) if ms: - content += "*. " + self.replace_option_link(ms.group(1), secid, 1) + "\n" + content += "* " + self.replace_option_link(ms.group(1), secid, 1) + "\n" else: content += " " + self.replace_option_link(line, secid, 1) + "\n" content += "\n"