Add Nox autodoc install & build to Nox and Contributing Guide

Author: CAM-Gerlach

CONTRIBUTING.md

@@ -144,7 +144,7 @@ Regardless of the tool you use, make sure to remember to always activate your en
 
 #### Conda
 
-To create an environment with Conda (recommended), simply execute the following:
+To create an environment with Conda (recommended), execute the following:
 
 ```shell
 conda create -c conda-forge -n spyder-api-docs-env python
@@ -193,6 +193,13 @@ Or if using ``pip``, you can grab them with:
 python -m pip install -r requirements.txt
 ```
 
+If you plan to generate and build the API reference documentation extracted from Spyder's docstrings, you'll also need to install Spyder in development mode as well as its dev dependencies.
+To do so, you can run the ``install_dev_repos.py`` script in the ``spyder`` submodule:
+
+```shell
+python install_dev_repos.py
+```
+
 
 
 ## Installing and Using the Pre-Commit Hooks
@@ -247,7 +254,13 @@ To build the project using Nox, just run
 nox -s build
 ```
 
-and can then open the rendered output in your default web browser with
+or, to also extract, generate and build the API reference from Spyder's docstrings (expensive the first time), pass the ``-t autodoc`` argument to any build command:
+
+```shell
+nox -s build -- -t autodoc
+```
+
+and then open the rendered output in your default web browser with
 
 ```shell
 nox -s serve
@@ -256,7 +269,7 @@ nox -s serve
 Alternatively, to automatically rebuild the project when changes occur, you can invoke
 
 ```shell
-nox -s autobuild
+nox -s autorebuild
 ```
 
 You can also pass your own custom [Sphinx build options](https://www.sphinx-doc.org/en/master/man/sphinx-build.html) after a ``--`` separator, which are added to the default set.
@@ -266,6 +279,12 @@ For example, to rebuild just the install guide and FAQ in verbose mode with the
 nox -s build -- --verbose --builder dirhtml -- index.rst
 ```
 
+When changing build options (particularly autodoc), cleaning the generated files avoids spurious errors:
+
+```shell
+nox -s clean
+```
+
 
 ### Build manually
 
@@ -275,8 +294,21 @@ For manual installations, you can invoke Sphinx yourself with the appropriate op
 python -m sphinx -n -W --keep-going docs docs/_build/html
 ```
 
+or to also extract, generate and build the API reference from Spyder's docstrings (requires Spyder and its dependencies to be installed in your environment):
+
+```shell
+python -I -m sphinx -n --keep-going -t autodoc docs docs/_build/html
+```
+
 Then, navigate to the ``_build/html`` directory inside the ``spyder-docs`` repository and open ``index.html`` (the main page of the docs) in your preferred browser.
 
+When changing build options (particularly autodoc), cleaning the generated files first avoids spurious errors:
+
+```shell
+rm -r docs/_autosummary/
+rm -r docs/_build/
+```
+
 
 
 ## Contributing Changes

docs/conf.py

@@ -145,6 +145,9 @@
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
 
+# Warning suppression
+suppress_warnings = []
+
 
 # -- Options for HTMLHelp output ---------------------------------------
 
@@ -264,8 +267,13 @@
     # "qstylizer",
 ]
 
-# Generate autodoc stubs with summaries from code
-autosummary_generate = True
+# Generate autosummaries if the autodoc tag is passed
+if "autodoc" in tags:
+    autosummary_generate = True
+else:
+    autosummary_generate = False
+    suppress_warnings += ["autodoc", "autosummary", "toc.excluded"]
+    exclude_patterns += ["reference.rst"]
 
 
 # -- Additional Directives ---------------------------------------------------

noxfile.py

@@ -4,15 +4,16 @@
 import contextlib
 import logging
 import os
-import tempfile
 import shutil
 import sys
+import tempfile
 import webbrowser
 from pathlib import Path
 
 # Third party imports
 import nox  # pylint: disable=import-error
 import nox.logger  # pylint: disable=import-error
+import packaging.requirements
 
 
 # --- Global constants --- #
@@ -29,7 +30,7 @@
 REPO_URL_SSH = "git@github.com:{user}/{repo}.git"
 
 # Build config
-BUILD_INVOCATION = ("python", "-m", "sphinx")
+BUILD_INVOCATION = ("python", "-I", "-m", "sphinx")
 SOURCE_DIR = Path("docs").resolve()
 BUILD_DIR = Path("docs/_build").resolve()
 BUILD_OPTIONS = ("-n", "-W", "--keep-going")
@@ -56,12 +57,29 @@
 BASE_URL = "https://spyder-ide.github.io/spyder-api-docs/"
 
 # Other config
-CANARY_COMMAND = ("pre-commit", "--version")
+CANARY_COMMANDS = {
+    "doc": {
+        "cmd": ("pre-commit", "--version"),
+        "default": True,
+        "env": {},
+    },
+    "autodoc": {
+        "cmd": ("python", "-I", "-m", "spyder.app.start", "--help"),
+        "default": False,
+        "env": {"HOME": str(Path().home())},
+    },
+}
 IGNORE_REVS_FILE = ".git-blame-ignore-revs"
 PRE_COMMIT_VERSION_SPEC = ">=2.10.0,<4"
 
 # Custom config
 SCRIPT_DIR = Path("scripts").resolve()
+AUTOSUMMARY_DIR = SOURCE_DIR / "_autosummary"
+SPYDER_PATH = Path("spyder").resolve()
+DEPS_PATH = SPYDER_PATH / 'external-deps'
+
+# Post config
+DIRS_TO_CLEAN = [BUILD_DIR, AUTOSUMMARY_DIR]
 
 
 # ---- Helpers ---- #
@@ -145,6 +163,9 @@ def construct_sphinx_invocation(
     builder = builders[-1] if builders else builder
     build_dir = BUILD_DIR / builder if build_dir is None else build_dir
 
+    if "autodoc" in cli_options:
+        build_options = [item for item in build_options if item != "-W"]
+
     if CI:
         build_options = list(build_options) + ["--color"]
 
@@ -163,6 +184,41 @@ def construct_sphinx_invocation(
     return sphinx_invocation
 
 
+def list_spyder_dev_repos():
+    """List the development repos included as subrepos of Spyder."""
+    repos = []
+    for p in [SPYDER_PATH] + list(DEPS_PATH.iterdir()):
+        if (
+            p.name.startswith('.')
+            or not p.is_dir()
+            and not ((p / 'setup.py').exists() or (p / 'pyproject.toml').exists())
+        ):
+            continue
+
+        repos.append(p)
+    return repos
+
+
+def get_python_lsp_version():
+    """Get current version to pass it to setuptools-scm."""
+    req_file = SPYDER_PATH / 'requirements' / 'main.yml'
+    with open(req_file, 'r', encoding='UTF-8') as f:
+        for line in f:
+            if 'python-lsp-server' not in line:
+                continue
+            line = line.split('-')[-1]
+            specifiers = packaging.requirements.Requirement(line).specifier
+            break
+        else:
+            return "0.0.0"
+
+    for specifier in specifiers:
+        if "=" in specifier.operator:
+            return specifier.version
+    else:
+        return "0.0.0"
+
+
 # ---- Dispatch ---- #
 
 
@@ -176,16 +232,28 @@ def _execute(session):
             "Must pass a list of functions to execute as first posarg"
         )
 
+    canary_commands = {}
+    install_tags = set()
+    for arg, properties in CANARY_COMMANDS.items():
+        cmd = properties["cmd"]
+        if properties["default"] or arg in session.posargs:
+            canary_commands[arg] = cmd
+        env = properties["env"] if properties["env"] else None
+
     if not session.posargs or session.posargs[0] is not _install:
-        # pylint: disable=too-many-try-statements
-        try:
-            with set_log_level():
-                session.run(
-                    *CANARY_COMMAND, include_outer_env=False, silent=True
-                )
-        except nox.command.CommandFailed:
-            print("Installing dependencies in isolated environment...")
-            _install(session, use_posargs=False)
+        for arg, cmd in canary_commands.items():
+            # pylint: disable=too-many-try-statements
+            try:
+                with set_log_level():
+                    session.run(
+                        *cmd, env=env, include_outer_env=False, silent=True
+                    )
+            except nox.command.CommandFailed:
+                install_tags.add(arg)
+
+    if install_tags:
+        print("Installing dependencies in isolated environment...")
+        _install(session, use_posargs=False, install_tags=install_tags)
 
     if session.posargs:
         for task in session.posargs[0]:
@@ -195,13 +263,47 @@ def _execute(session):
 # ---- Install ---- #
 
 
-def _install(session, *, use_posargs=True):
-    """Execute the dependency installation."""
-    posargs = session.posargs[1:] if use_posargs else ()
+def _install_doc(session, posargs=()):
+    """Install the basic documentation and dev dependencies."""
     session.install(f"pre-commit{PRE_COMMIT_VERSION_SPEC}")
     session.install("-r", "requirements.txt", *posargs)
 
 
+def _install_autodoc(session, posargs=()):
+    """Install the dependencies to generate API autodocs."""
+    dev_repos = list_spyder_dev_repos()
+    for dev_repo in dev_repos:
+        env = None
+        if 'python-lsp-server' in str(dev_repo):
+            env = {**os.environ}
+            env.update(
+                {'SETUPTOOLS_SCM_PRETEND_VERSION': get_python_lsp_version()})
+        session.install("-e", dev_repo, *posargs, env=env)
+
+
+INSTALL_FUNCTIONS = {
+    "doc": _install_doc,
+    "autodoc": _install_autodoc,
+}
+
+
+def _install(session, *, use_posargs=True, install_tags=None):
+    """Execute the dependency installation."""
+    posargs = session.posargs[1:] if use_posargs else ()
+
+    install_tags = {"doc"} if install_tags is None else install_tags
+    for arg in CANARY_COMMANDS.keys():
+        if f"--{arg}" in session.posargs:
+            install_tags.add(arg)
+            if posargs:
+                posargs.remove(f"--{arg}")
+        elif arg in session.posargs:
+            install_tags.add(arg)
+
+    for tag in install_tags:
+        INSTALL_FUNCTIONS[tag](session, posargs)
+
+
 @nox.session
 def install(session):
     """Install the project's dependencies (passes through args to pip)."""
@@ -238,17 +340,21 @@ def run(session):
 
 def _clean(session):
     """Remove the build directory."""
-    print(f"Removing build directory {BUILD_DIR.as_posix()!r}")
     ignore_flag = "--ignore"
     should_ignore = ignore_flag in session.posargs
 
-    try:
-        shutil.rmtree(BUILD_DIR, ignore_errors=should_ignore)
-    except FileNotFoundError:
-        pass
-    except Exception:
-        print(f"\nError removing files; pass {ignore_flag!r} flag to ignore\n")
-        raise
+    for dir_to_clean in DIRS_TO_CLEAN:
+        if not dir_to_clean.exists():
+            continue
+        print(f"Removing generated directory {dir_to_clean.as_posix()!r}")
+        try:
+            shutil.rmtree(dir_to_clean, ignore_errors=should_ignore)
+        except FileNotFoundError:
+            pass
+        except Exception:
+            print(f"\nError removing files in {dir_to_clean.as_posix()!r}")
+            print(f"Pass {ignore_flag!r} flag to ignore\n")
+            raise
 
 
 @nox.session
@@ -471,15 +577,15 @@ def build(session):
     session.notify("_execute", posargs=([_build], *session.posargs))
 
 
-def _autobuild(session):
+def _autorebuild(session):
     """Use Sphinx-Autobuild to rebuild the project and open in browser."""
-    _autodocs(session)
+    _docs_autobuild(session)
 
 
 @nox.session
-def autobuild(session):
+def autorebuild(session):
     """Rebuild the project continuously as source files are changed."""
-    session.notify("_execute", posargs=([_autobuild], *session.posargs))
+    session.notify("_execute", posargs=([_autorebuild], *session.posargs))
 
 
 # --- Docs --- #
@@ -499,7 +605,7 @@ def docs(session):
     session.notify("_execute", posargs=([_docs], *session.posargs))
 
 
-def _autodocs(session):
+def _docs_autobuild(session):
     """Use Sphinx-Autobuild to rebuild the project and open in browser."""
     session.install("sphinx-autobuild")
 
@@ -518,10 +624,10 @@ def _autodocs(session):
         session.run(*sphinx_invocation)
 
 
-@nox.session
-def autodocs(session):
+@nox.session(name="docs-autobuild")
+def docs_autobuild(session):
     """Rebuild the docs continuously as source files are changed."""
-    session.notify("_execute", posargs=([_autodocs], *session.posargs))
+    session.notify("_execute", posargs=([_docs_autobuild], *session.posargs))
 
 
 def _build_languages(session):