Split docs requirements into separate group (#122)

On Python versions earlier than 3.10, flake8 and Sphinx are just not going to be compatible, and need to be in separate groups so both don't have to be installed at the same time or in the same environment. This also updates our ReadTheDocs setup to use their newer configuration file format. Overall, this follows the arrangement I set up for the Wayback package in edgi-govdata-archiving/wayback#91.

Author: Mr0grog

.circleci/config.yml

@@ -1,67 +1,91 @@
 version: 2.1
 
 parameters:
-  python-version:
+  primary-python-version:
     type: string
-    default: "3.7.13"
+    default: "3.10.7"
 
-jobs:
-  build:
-    working_directory: ~/web-monitoring-diff
-    docker:
-      - image: cimg/python:<< pipeline.parameters.python-version >>
+commands:
+  setup_pip:
+    description: "Set Up Dependencies"
+    parameters:
+      python-version:
+        type: string
+      install-docs:
+        description: Install docs dependencies
+        type: boolean
+        default: false
     steps:
-      - checkout
-      - run:
-          name: Install System Dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install \
-              libxml2-dev libxslt-dev libssl-dev openssl libcurl4-openssl-dev
-
       - restore_cache:
           keys:
-            - cache-v2-{{ arch }}-python<< pipeline.parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}
-            - cache-v2-{{ arch }}-python<< pipeline.parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-
-            - cache-v2-{{ arch }}-python<< pipeline.parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-
-            - cache-v2-{{ arch }}-python<< pipeline.parameters.python-version >>-{{ checksum "requirements.txt" }}-
-            - cache-v2-{{ arch }}-python<< pipeline.parameters.python-version >>-
+            - cache-v3-{{ arch }}-python<< parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}-{{ checksum "requirements-docs.txt" }}
+            - cache-v3-{{ arch }}-python<< parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}-
+            - cache-v3-{{ arch }}-python<< parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-
+            - cache-v3-{{ arch }}-python<< parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-
+            - cache-v3-{{ arch }}-python<< parameters.python-version >>-{{ checksum "requirements.txt" }}-
+            - cache-v3-{{ arch }}-python<< parameters.python-version >>-
 
-      # Bundle install dependencies
       - run:
           name: Install Dependencies
           command: |
-            python -m venv venv
-            . venv/bin/activate
-            pip install -e .[server,dev] --no-binary lxml
+            python -m venv ~/venv
+            . ~/venv/bin/activate
+            # Ensure pip is up-to-date
+            pip install --upgrade pip
+            pip install .[server,dev] --no-binary lxml
             pip install -r requirements-experimental.txt
 
-      # Store bundle cache
+      # Docs dependencies are only compatible on Python 3.10+, so only install
+      # them on demand.
+      - when:
+          condition: << parameters.install-docs >>
+          steps:
+            - run:
+                command: |
+                  . ~/venv/bin/activate
+                  pip install -r requirements-docs.txt
+
       - save_cache:
-          key: cache-v2-{{ arch }}-python<< pipeline.parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}
+          key: cache-v3-{{ arch }}-python<< parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}-{{ checksum "requirements-docs.txt" }}
           paths:
-            - venv
+            - ~/venv
 
+jobs:
+  test:
+    parameters:
+      python-version:
+        type: string
+    working_directory: ~/web-monitoring-diff
+    docker:
+      - image: cimg/python:<< parameters.python-version >>
+    steps:
+      - checkout
+      - setup_pip:
+          python-version: << parameters.python-version >>
       - run:
           name: Tests
           command: |
-            . venv/bin/activate
+            . ~/venv/bin/activate
             coverage run -m pytest -vv
       - run:
           name: Coverage
           command: |
-            . venv/bin/activate
+            . ~/venv/bin/activate
             coverage report -m
+
+  lint:
+    working_directory: ~/web-monitoring-diff
+    docker:
+      - image: cimg/python:<< pipeline.parameters.primary-python-version >>
+    steps:
+      - checkout
+      - setup_pip:
+          python-version: << pipeline.parameters.primary-python-version >>
       - run:
           name: Code linting
           command: |
-            . venv/bin/activate
+            . ~/venv/bin/activate
             flake8 .
-    #   - run:
-    #       name: Build docs
-    #       command: |
-    #         . venv/bin/activate
-    #         cd docs && make html
 
   build_docker:
     machine:
@@ -149,7 +173,14 @@ workflows:
       #
       # See more in:
       # https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag
-      - build:
+      - test:
+          filters:
+            tags:
+              only: /^v.*/
+          matrix:
+            parameters:
+              python-version: ["3.7", "3.8", "3.9", "3.10"]
+      - lint:
           filters:
             tags:
               only: /^v.*/
@@ -160,7 +191,8 @@ workflows:
       # Publishing only happens for tags starting with "v", and no branches.
       - publish_docker:
           requires:
-            - build
+            - test
+            - lint
             - build_docker
           filters:
             branches:

.readthedocs.yaml

@@ -0,0 +1,31 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+  apt_packages:
+    - libxml2-dev
+    - libxslt-dev
+    - libssl-dev
+    - openssl
+    - libcurl4-openssl-dev
+
+python:
+  install:
+    - method: pip
+      path: "."
+      extra_requirements:
+        - server
+        - docs
+    - requirements: "requirements-experimental.txt"
+
+sphinx:
+  configuration: docs/source/conf.py
+
+formats: all

Dockerfile

@@ -6,7 +6,7 @@
 # We separate them out so that the final `release` image can layer on top of
 # this one without needing compiler-related packages.
 ##
-FROM python:3.7.13-slim as base
+FROM python:3.10.7-slim as base
 LABEL maintainer="enviroDGI@gmail.com"
 
 RUN apt-get update && apt-get install -y --no-install-recommends \

README.md

@@ -189,14 +189,19 @@ First, make sure you have an appropriate Python version and the necessary system
 2. Perform an *editable* install of the package in the repo:
 
     ```sh
-    $ pip install -e . --no-binary lxml
+    $ pip install -e .[server,dev,docs] --no-binary lxml
     ```
 
-3. Install additional experimental and development dependencies:
+    NOTE: if you are using Python 3.9 or earlier you may not be able to install both the development and docs dependencies at the same time. Instead, just install the `dev` dependencies:
+
+    ```sh
+    $ pip install -e .[server,dev] --no-binary lxml
+    ```
+
+3. Install additional dependencies for experimental features:
 
     ```sh
     $ pip install -r requirements-experimental.txt
-    $ pip install -r requirements-dev.txt
     ```
 
 4. Make sure it works without errors by running a python interpreter and importing the package:

docs/requirements.txt

@@ -31,9 +31,15 @@
 # The full version, including alpha/beta/rc tags.
 release = web_monitoring_diff.__version__
 
-
 # -- General configuration ---------------------------------------------------
 
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = 'en'
+
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
@@ -55,6 +61,8 @@
 autosummary_generate = True
 numpydoc_show_class_members = False
 autodoc_mock_imports = ['html5_parser', 'pycurl', 'sentry']
+numpydoc_xref_param_type = True
+numpydoc_xref_ignore = {'type', 'optional', 'default'}
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -66,7 +74,7 @@
 
 # Set up link shortcuts
 extlinks = {
-    'issue': ('https://github.com/edgi-govdata-archiving/web-monitoring-diff/issues/%s', '#'),
+    'issue': ('https://github.com/edgi-govdata-archiving/web-monitoring-diff/issues/%s', 'Issue #%s'),
 }
 
 # Example configuration for intersphinx: refer to the Python standard library.

docs/source/conf.py

@@ -10,8 +10,3 @@
 coverage ~=6.5
 flake8 ~=5.0.4
 pytest ~=7.1.3
-ipython ~=7.34.0
-numpydoc ~=1.4
-sphinx ~=4.3.2
-sphinx-copybutton ~=0.5.0
-sphinx_rtd_theme ~=1.0.0

requirements-dev.txt

@@ -0,0 +1,16 @@
+# Tools for building documentation. Some of our docs tools conflict with dev
+# tools on Python < 10, so docs tools are listed here in order to make it
+# possible to skip installing them if not needed.
+#
+# Unlike most requirements.txt files, this is not a frozen list of exact
+# dependencies (a.k.a. a lock file). Instead, it specifies:
+# - Direct dependencies only.
+# - Package names and valid version *ranges*
+#
+# It only exists to keep the list of dependencies in a separate file from
+# setup.py.
+ipython ~=7.34.0
+numpydoc ~=1.4
+sphinx ~=5.2.3
+sphinx-copybutton ~=0.5.0
+sphinx_rtd_theme ~=1.0.0

requirements-docs.txt

@@ -77,5 +77,6 @@ def read_requirements(fname):
     extras_require={
         'server': read_requirements('requirements-server.txt'),
         'dev': read_requirements('requirements-dev.txt'),
+        'docs': read_requirements('requirements-docs.txt'),
     },
 )

setup.py

@@ -131,8 +131,8 @@ class PublicError(tornado.web.HTTPError):
 
     Parameters
     ----------
-    status_code : int, optional
-        Status code for the response. Defaults to `500`.
+    status_code : int, default: 500
+        Status code for the response.
     public_message : str, optional
         Textual description of the error. This will be publicly visible in
         production mode, unlike `log_message`.