From 4aea6b4ea14ad1f2d9fb8531c8f61a3f23eef1d9 Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:03:38 +0200 Subject: [PATCH 1/2] docs: the spec extractor renders what the docstrings say Four defects in scripts/extract_required_docs.py, each surfacing as a wrong page in the generated spec (cubic, utcp-specification #66): - A prose line containing a colon that is not a 'name: description' pair was silently DROPPED: '(note: ...)' sentences, quoted URLs after '(e.g.', anything with a colon mid-sentence. Such a line is now ordinary text. - Any line ending in a colon became a title-cased section header, code spans included ('Inheritance is controlled by `inherit_env_vars`:' -> '**Inheritance Is Controlled By `Inherit_Env_Vars`**'; 'def tool1():' inside an example -> '**Def Tool1()**'). A header is now a known Google-style one or a short title of words, digits, spaces and hyphens. - A docstring with no section header at all was never flushed, so 62 REQUIRED docstrings across the spec rendered as '*No ... documentation available*' (UtcpClient, every auth serializer, the plugin loader, all socket methods, ...). The bogus headers above had been flushing some of them by accident, which the fix exposed. - Cross-reference links were inserted inside inline code spans, where Markdown shows them as literal brackets. Links now stop at code spans; the field-list placeholder backticks are unwrapped before the pass so fields keep their links. Also: a class whose own docstring is not REQUIRED but whose methods are now renders those methods (the index already counted them), and index links are POSIX paths on every platform. Two docstrings corrected on the way: the CLI template claimed tool_args are 'shell-quoted' - the mechanism is per-invocation environment variables, as the same docstring explains - and 'OAuth2Auth.cache_key' is now written so the class links and cache_key stays code. Regenerated against the published pages: 43 pages change; every removed line is a placeholder, a bogus header, a link-in-code-span or a mangled example fragment; 360 lines of previously invisible documentation come back. Co-Authored-By: Claude Fable 5.1 --- .../cli/src/utcp_cli/cli_call_template.py | 14 +- .../utcp_http/http_communication_protocol.py | 2 +- .../websocket_communication_protocol.py | 2 +- scripts/extract_required_docs.py | 188 +++++++++++------- 4 files changed, 123 insertions(+), 83 deletions(-) diff --git a/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.py b/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.py index 378f9dc..cf2e63f 100644 --- a/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.py +++ b/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.py @@ -147,8 +147,10 @@ class CliCallTemplate(CallTemplate): commands: A list of CommandStep objects defining the commands to execute in order. Each command can contain UTCP_ARG_argname_UTCP_END placeholders that will be replaced with values from tool_args during execution. - Placeholders are shell-quoted and therefore expand to exactly one - shell token (see class docstring). + Each placeholder becomes a shell-variable reference whose value + reaches the subprocess through a per-invocation environment + variable, so it expands to exactly one shell token and cannot be + reinterpreted as shell syntax (see class docstring). env_vars: A dictionary of environment variables to set for the command's execution context. Values can be static strings or placeholders for variables from the UTCP client's variable substitutor. Always @@ -238,9 +240,11 @@ class CliCallTemplate(CallTemplate): Security Considerations: - Commands are executed in a subprocess. Ensure that the commands specified are from a trusted source. - - `tool_args` values are shell-quoted on substitution, but the - *command template itself* is not — never assemble it from - untrusted input. + - `tool_args` values never touch the command text: they reach the + subprocess through per-invocation environment variables and the + shell expands them only after it has parsed the script. The + *command template itself* has no such protection — never assemble + it from untrusted input. - The host environment is restricted; secrets are not propagated unless explicitly named in `env_vars` or `inherit_env_vars`. - Commands should use the appropriate syntax for the target platform diff --git a/plugins/communication_protocols/http/src/utcp_http/http_communication_protocol.py b/plugins/communication_protocols/http/src/utcp_http/http_communication_protocol.py index 87990a1..f26433e 100644 --- a/plugins/communication_protocols/http/src/utcp_http/http_communication_protocol.py +++ b/plugins/communication_protocols/http/src/utcp_http/http_communication_protocol.py @@ -63,7 +63,7 @@ class HttpCommunicationProtocol(CommunicationProtocol): Attributes: _session: Optional aiohttp ClientSession for connection reuse. - _oauth_tokens: Cache of OAuth2 tokens keyed by the full credential configuration (``OAuth2Auth.cache_key``). + _oauth_tokens: Cache of OAuth2 tokens keyed by the full credential configuration (OAuth2Auth's ``cache_key``). _log: Logger function for debugging and error reporting. """ diff --git a/plugins/communication_protocols/websocket/src/utcp_websocket/websocket_communication_protocol.py b/plugins/communication_protocols/websocket/src/utcp_websocket/websocket_communication_protocol.py index 74ca2ee..38243b3 100644 --- a/plugins/communication_protocols/websocket/src/utcp_websocket/websocket_communication_protocol.py +++ b/plugins/communication_protocols/websocket/src/utcp_websocket/websocket_communication_protocol.py @@ -63,7 +63,7 @@ class WebSocketCommunicationProtocol(CommunicationProtocol): Attributes: _connections: Active WebSocket connections by provider key. _sessions: aiohttp ClientSessions for connection management. - _oauth_tokens: Cache of OAuth2 tokens keyed by the full credential configuration (``OAuth2Auth.cache_key``). + _oauth_tokens: Cache of OAuth2 tokens keyed by the full credential configuration (OAuth2Auth's ``cache_key``). """ def __init__(self, logger_func: Optional[Callable[[str], None]] = None): diff --git a/scripts/extract_required_docs.py b/scripts/extract_required_docs.py index af6bfbb..fe61b58 100644 --- a/scripts/extract_required_docs.py +++ b/scripts/extract_required_docs.py @@ -148,44 +148,49 @@ def process_section_content(content_lines): line = line.replace('{', '\\{').replace('}', '\\}') stripped = line.strip() - # Check if this looks like a parameter/item definition (name: description) + # Check if this looks like a parameter/item definition (name: description). + # A line with a colon that is NOT one -- prose such as "(note: ...)" or a + # quoted URL -- is ordinary text and falls through to the branches below; + # it must never be dropped. + param_match = None if ':' in stripped and not stripped.endswith(':'): colon_pos = stripped.find(':') - param_name = stripped[:colon_pos].strip() - param_desc = stripped[colon_pos + 1:].strip() - - # Check if param_name looks like a parameter (no spaces, reasonable length) - if ' ' not in param_name and len(param_name) <= 50 and param_name.replace('_', '').isalnum(): - # This is likely a parameter definition - processed.append(f"- **`{param_name}`**: {param_desc}") - - # Check for continuation lines (indented more than the parameter line) - base_indent = len(line) - len(line.lstrip()) - i += 1 - while i < len(content_lines): - next_line = content_lines[i] - next_stripped = next_line.strip() - next_indent = len(next_line) - len(next_line.lstrip()) if next_stripped else 0 - - # Check if we hit a code block - if next_stripped.startswith('```'): - break - - if not next_stripped: - # Empty line - add it and continue - processed.append('') - i += 1 - elif next_indent > base_indent: - # Continuation line - add with proper spacing - processed.append(f" {next_stripped}") - i += 1 - else: - # Not a continuation, back up and break - break - continue - + candidate = stripped[:colon_pos].strip() + # A parameter name has no spaces and is a plain identifier + if ' ' not in candidate and len(candidate) <= 50 and candidate.replace('_', '').isalnum(): + param_match = (candidate, stripped[colon_pos + 1:].strip()) + + if param_match is not None: + param_name, param_desc = param_match + processed.append(f"- **`{param_name}`**: {param_desc}") + + # Check for continuation lines (indented more than the parameter line) + base_indent = len(line) - len(line.lstrip()) + i += 1 + while i < len(content_lines): + next_line = content_lines[i] + next_stripped = next_line.strip() + next_indent = len(next_line) - len(next_line.lstrip()) if next_stripped else 0 + + # Check if we hit a code block + if next_stripped.startswith('```'): + break + + if not next_stripped: + # Empty line - add it and continue + processed.append('') + i += 1 + elif next_indent > base_indent: + # Continuation line - add with proper spacing + processed.append(f" {next_stripped}") + i += 1 + else: + # Not a continuation, back up and break + break + continue + # Check if line starts with a list marker - elif stripped.startswith(('- ', '* ', '+ ')): + if stripped.startswith(('- ', '* ', '+ ')): # This is already a markdown list item processed.append(stripped) elif stripped.startswith(('1. ', '2. ', '3. ', '4. ', '5. ', '6. ', '7. ', '8. ', '9. ')): @@ -204,9 +209,13 @@ def process_section_content(content_lines): # Parse the docstring line by line for line in lines: stripped_lower = line.strip().lower() - - # Check if this line is a section header - if stripped_lower in section_headers or stripped_lower.endswith(':'): + + # Check if this line is a section header: a known Google-style header, or a + # short title made of words only ("Security Considerations:"). A sentence + # that merely ends in a colon ("Inheritance is controlled by `x`:") is + # content -- treating it as a header would title-case it, code span included. + is_custom_header = re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9 \-]{0,40}:', line.strip()) is not None + if stripped_lower in section_headers or is_custom_header: # Save previous section if it exists if current_section: processed_content = process_section_content(current_section_content) @@ -231,6 +240,10 @@ def process_section_content(content_lines): if processed_content: result.append(f"\n**{current_section.title()}**\n") result.extend(processed_content) + else: + # No section header anywhere: the whole docstring is the preamble, + # which is otherwise only flushed when a header follows it. + result.extend(process_section_content(current_section_content)) # Clean up the result final_result = [] @@ -530,18 +543,46 @@ def add_cross_references_post_generation(self, text: str, current_output_file: s class_anchor = re.sub(r'[^\w\-_]', '-', class_name.lower()).strip('-') link = f"[{class_name}](./{relative_path_str}#{class_anchor})" - # Don't replace matches that are in code blocks + # Don't replace matches that are in code blocks or inline code spans lines = modified_text.split('\n') in_code_block = False for i, line in enumerate(lines): if line.strip().startswith('```'): in_code_block = not in_code_block elif not in_code_block: - lines[i] = re.sub(pattern, link, line) + lines[i] = self._sub_outside_inline_code(pattern, link, line) modified_text = '\n'.join(lines) - + return modified_text + + @staticmethod + def _sub_outside_inline_code(pattern: str, replacement: str, line: str) -> str: + """Substitute only outside `...` / ``...`` spans. + + Markdown renders a code span literally, so a link inserted inside one shows up + as raw brackets instead of a link. + """ + parts = re.split(r'(`+[^`]*`+)', line) + return ''.join(part if part.startswith('`') else re.sub(pattern, replacement, part) for part in parts) + def _render_methods(self, content: List[str], methods: List[DocEntry], file_path: str) -> None: + """Append a class's documented methods to ``content``.""" + content.extend(["#### Methods:", ""]) + for method in methods: + # Add cross-references to method signature + linked_signature = self.add_cross_references(method.signature, file_path) + docstrings = method.docstring if method.docstring else "*No method documentation available*" + content.extend( + [ + "
", + f"{linked_signature}", + "", + docstrings, + "
", + "", + ] + ) + def generate_module_markdown(self, file_path: str, file_data: Dict[str, List[DocEntry]]) -> str: """Generate markdown content for a single module/file.""" if not any(file_data.values()): @@ -632,34 +673,27 @@ def generate_module_markdown(self, file_path: str, file_data: Dict[str, List[Doc # Add methods for this class if class_entry.name in methods_by_class: - content.extend(["#### Methods:", ""]) - - for method in methods_by_class[class_entry.name]: - method_anchor = re.sub(r'[^\w\-_]', '-', f"{class_entry.name}-{method.name}".lower()).strip('-') - - # Add cross-references to method signature - linked_signature = self.add_cross_references(method.signature, file_path) - - docstrings = "" - - if method.docstring: - docstrings = method.docstring - else: - docstrings = "*No method documentation available*" - - content.extend( - [ - "
", - f"{linked_signature}", - "", - docstrings, - "
", - "", - ] - ) - + self._render_methods(content, methods_by_class[class_entry.name], file_path) + content.extend(["---", ""]) - + + # A class whose own docstring is not REQUIRED can still have REQUIRED + # methods. They are required documentation and the index counts them, + # so they are rendered under a bare class heading rather than lost. + documented_classes = {class_entry.name for class_entry in file_data['classes']} + for class_name, methods in methods_by_class.items(): + if class_name in documented_classes: + continue + class_anchor = re.sub(r'[^\w\-_]', '-', class_name.lower()).strip('-') + content.extend([ + f"### class {class_name} {{#{class_anchor}}}", + "", + "*No class documentation available*", + "", + ]) + self._render_methods(content, methods, file_path) + content.extend(["---", ""]) + # Add standalone functions if file_data['functions']: for func_entry in file_data['functions']: @@ -748,7 +782,7 @@ def generate_index_file(self, modules: Dict[str, Dict[str, List[DocEntry]]], out index_path = output_path / "index.md" target_path = Path(output_file_path) try: - relative_path = target_path.relative_to(output_path) + relative_path = target_path.relative_to(output_path).as_posix() link_path = f"./{relative_path}" except ValueError: # Fallback to simple filename if relative path calculation fails @@ -805,7 +839,7 @@ def generate_index_file(self, modules: Dict[str, Dict[str, List[DocEntry]]], out index_path = output_path / "index.md" target_path = Path(output_file_path) try: - relative_path = target_path.relative_to(output_path) + relative_path = target_path.relative_to(output_path).as_posix() link_path = f"./{relative_path}" except ValueError: # Fallback to simple filename if relative path calculation fails @@ -900,10 +934,11 @@ def generate_docs(self, output_dir: str) -> None: # Second pass: Add cross-references and write files for file_path, (content, output_file_path) in generated_files.items(): - # Post-process content to add proper cross-references - processed_content = self.add_cross_references_post_generation(content, str(output_file_path).replace('\\', '/')) - # Also process field references - lines = processed_content.split('\n') + # Field lines were emitted as "- `name: type`" -- the backticks are a + # placeholder (see format_field_with_references), not a code span. + # Unwrap them BEFORE cross-referencing: the cross-reference pass + # leaves real code spans alone, and these must receive links. + lines = content.split('\n') processed_lines = [] for line in lines: if line.strip().startswith('- `') and ':' in line: @@ -916,8 +951,9 @@ def generate_docs(self, output_dir: str) -> None: processed_lines.append(line) else: processed_lines.append(line) - - final_content = '\n'.join(processed_lines) + + # Post-process content to add proper cross-references + final_content = self.add_cross_references_post_generation('\n'.join(processed_lines), str(output_file_path).replace('\\', '/')) with open(output_file_path, 'w', encoding='utf-8') as f: f.write(final_content) From b33e4254994f54379436db8d8350eabeee7bf103 Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:01:59 +0200 Subject: [PATCH 2/2] docs: extractor heuristics stated as rules, not examples Three findings from cubic on #110, each a heuristic that was right on the cases in front of me and wrong one step over: - The inline-code splitter closed a span at the first backtick run, so ``a`b`` ended at the inner tick and class names still inside the span were linked. A span now closes only on a run of the same length. - A bare URL line ("https://example.com") still parsed as a definition: "https" is short, alphanumeric and has no space. A definition has a space after its colon; a URL has "//". That is the whole difference, so it is the rule. - The header heuristic (short, words only) admitted "Use the following:" and rejected "Return Values (Complex):" or anything over 41 chars. A custom header is a Title-Cased line ending in a colon: every word starts with a capital or a digit, no code span, any length. Sentence- case captions ("Basic command step:") now stay under their real "Examples" header instead of replacing it. Unit-checked on the boundary cases; regenerated and compared with the previous run -- the only movement is captions becoming paragraphs under the section header they belong to. Co-Authored-By: Claude Fable 5.1 --- scripts/extract_required_docs.py | 79 ++++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/scripts/extract_required_docs.py b/scripts/extract_required_docs.py index fe61b58..92cfa3c 100644 --- a/scripts/extract_required_docs.py +++ b/scripts/extract_required_docs.py @@ -156,8 +156,11 @@ def process_section_content(content_lines): if ':' in stripped and not stripped.endswith(':'): colon_pos = stripped.find(':') candidate = stripped[:colon_pos].strip() - # A parameter name has no spaces and is a plain identifier - if ' ' not in candidate and len(candidate) <= 50 and candidate.replace('_', '').isalnum(): + # A parameter name has no spaces and is a plain identifier, and a + # definition puts a space after its colon ("name: description") -- + # which is what separates it from a URL such as https://example.com + if (' ' not in candidate and len(candidate) <= 50 and candidate.replace('_', '').isalnum() + and stripped[colon_pos + 1] == ' '): param_match = (candidate, stripped[colon_pos + 1:].strip()) if param_match is not None: @@ -211,11 +214,11 @@ def process_section_content(content_lines): stripped_lower = line.strip().lower() # Check if this line is a section header: a known Google-style header, or a - # short title made of words only ("Security Considerations:"). A sentence - # that merely ends in a colon ("Inheritance is controlled by `x`:") is - # content -- treating it as a header would title-case it, code span included. - is_custom_header = re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9 \-]{0,40}:', line.strip()) is not None - if stripped_lower in section_headers or is_custom_header: + # Title-Cased line ending in a colon ("Security Considerations:", + # "Return Values (Complex):"). A sentence that merely ends in a colon + # ("Inheritance is controlled by `x`:", "Use the following:") is content -- + # treating it as a header would title-case it, code span included. + if stripped_lower in section_headers or self._is_custom_section_header(line.strip()): # Save previous section if it exists if current_section: processed_content = process_section_content(current_section_content) @@ -556,14 +559,70 @@ def add_cross_references_post_generation(self, text: str, current_output_file: s return modified_text @staticmethod - def _sub_outside_inline_code(pattern: str, replacement: str, line: str) -> str: + def _is_custom_section_header(stripped: str) -> bool: + """A custom section header is a Title-Cased line ending in a colon. + + Every word starts with a capital letter or a digit (leading punctuation such + as an opening parenthesis is skipped), and there is no code span. Length is + not a criterion: "Return Values (Complex):" and "Section 1/2:" are headers, + "Use the following:" and "def tool1():" are content. + """ + if not stripped.endswith(':') or '`' in stripped: + return False + words = stripped[:-1].split() + if not words: + return False + for word in words: + first = next((ch for ch in word if ch.isalnum()), None) + if first is None or not (first.isupper() or first.isdigit()): + return False + return True + + @staticmethod + def _split_inline_code(line: str) -> List[Tuple[str, bool]]: + """Split a line into (text, is_code) parts. + + A code span opened by a run of N backticks closes only on the next run of + exactly N backticks, so ``a`b`` is one span. An unclosed run is text. + """ + parts: List[Tuple[str, bool]] = [] + pos = 0 + text_start = 0 + while pos < len(line): + if line[pos] != '`': + pos += 1 + continue + run_end = pos + while run_end < len(line) and line[run_end] == '`': + run_end += 1 + fence = line[pos:run_end] + close = line.find(fence, run_end) + # The closing run must be exactly as long: skip longer runs + while close != -1 and close + len(fence) < len(line) and line[close + len(fence)] == '`': + skip = close + while skip < len(line) and line[skip] == '`': + skip += 1 + close = line.find(fence, skip) + if close == -1: + pos = run_end + continue + if text_start < pos: + parts.append((line[text_start:pos], False)) + span_end = close + len(fence) + parts.append((line[pos:span_end], True)) + pos = text_start = span_end + if text_start < len(line): + parts.append((line[text_start:], False)) + return parts + + @classmethod + def _sub_outside_inline_code(cls, pattern: str, replacement: str, line: str) -> str: """Substitute only outside `...` / ``...`` spans. Markdown renders a code span literally, so a link inserted inside one shows up as raw brackets instead of a link. """ - parts = re.split(r'(`+[^`]*`+)', line) - return ''.join(part if part.startswith('`') else re.sub(pattern, replacement, part) for part in parts) + return ''.join(text if is_code else re.sub(pattern, replacement, text) for text, is_code in cls._split_inline_code(line)) def _render_methods(self, content: List[str], methods: List[DocEntry], file_path: str) -> None: """Append a class's documented methods to ``content``."""