fix: Handling of site connection issues during outage

bot/bot.py

@@ -1,19 +1,23 @@
 import asyncio
 import contextlib
+import types
 from sys import exception
 
 import aiohttp
 from discord.errors import Forbidden
+from discord.ext import commands
 from pydis_core import BotBase
+from pydis_core.utils import scheduling
+from pydis_core.utils._extensions import walk_extensions
 from pydis_core.utils.error_handling import handle_forbidden_from_block
 from sentry_sdk import new_scope, start_transaction
 
 from bot import constants, exts
 from bot.log import get_logger
+from bot.utils.startup_reporting import StartupFailureReporter
 
 log = get_logger("bot")
 
-
 class StartupError(Exception):
     """Exception class for startup errors."""
 
@@ -26,9 +30,13 @@ class Bot(BotBase):
     """A subclass of `pydis_core.BotBase` that implements bot-specific functions."""
 
     def __init__(self, *args, **kwargs):
-
         super().__init__(*args, **kwargs)
 
+        # Track extension load failures and tasks so we can report them after all attempts have completed
+        self.extension_load_failures: dict[str, BaseException] = {}
+        self._extension_load_tasks: dict[str, asyncio.Task] = {}
+        self._startup_failure_reporter = StartupFailureReporter()
+
     async def load_extension(self, name: str, *args, **kwargs) -> None:
         """Extend D.py's load_extension function to also record sentry performance stats."""
         with start_transaction(op="cog-load", name=name):
@@ -77,3 +85,53 @@ async def on_error(self, event: str, *args, **kwargs) -> None:
             scope.set_extra("kwargs", kwargs)
 
             log.exception(f"Unhandled exception in {event}.")
+
+    async def add_cog(self, cog: commands.Cog) -> None:
+        """
+        Add a cog to the bot with exception handling.
+
+        Override of `BotBase.add_cog` to capture and log any exceptions raised during cog loading,
+        including the extension name if available.
+        """
+        extension = cog.__module__
+
+        try:
+            await super().add_cog(cog)
+            log.info(f"Cog successfully loaded: {cog.qualified_name}")
+
+        except BaseException as e:
+            key = extension or f"(unknown)::{cog.qualified_name}"
+            self.extension_load_failures[key] = e
+
+            log.exception(
+                f"Failed during add_cog (extension={extension}, cog={cog.qualified_name})"
+            )
+            # Propagate error
+            raise
+
+    async def _load_extensions(self, module: types.ModuleType) -> None:
+        """Load extensions for the bot."""
+        await self.wait_until_guild_available()
+
+        self.all_extensions = walk_extensions(module)
+
+        async def _load_one(extension: str) -> None:
+            try:
+                await self.load_extension(extension)
+                log.info(f"Extension successfully loaded: {extension}")
+
+            except BaseException as e:
+                self.extension_load_failures[extension] = e
+                log.exception(f"Failed to load extension: {extension}")
+                raise
+
+        for extension in self.all_extensions:
+            task = scheduling.create_task(_load_one(extension))
+            self._extension_load_tasks[extension] = task
+
+        # Wait for all load tasks to complete so we can report any failures together
+        await asyncio.gather(*self._extension_load_tasks.values(), return_exceptions=True)
+
+        # Send a Discord message to moderators if any extensions failed to load
+        if self.extension_load_failures :
+            await self._startup_failure_reporter.notify(self, self.extension_load_failures)

bot/constants.py

@@ -462,6 +462,9 @@ class _URLs(_BaseURLs):
     connect_max_retries: int = 3
     connect_cooldown: int = 5
 
+    # Back-off in cog_load
+    connect_initial_backoff: int = 1
+
     site_logs_view: str = "https://pythondiscord.com/staff/bot/logs"
 
 

bot/exts/filtering/filtering.py

@@ -1,3 +1,4 @@
+import asyncio
 import datetime
 import io
 import json
@@ -24,7 +25,7 @@
 import bot.exts.filtering._ui.filter as filters_ui
 from bot import constants
 from bot.bot import Bot
-from bot.constants import BaseURLs, Channels, Guild, MODERATION_ROLES, Roles
+from bot.constants import BaseURLs, Channels, Guild, MODERATION_ROLES, Roles, URLs
 from bot.exts.backend.branding._repository import HEADERS, PARAMS
 from bot.exts.filtering._filter_context import Event, FilterContext
 from bot.exts.filtering._filter_lists import FilterList, ListType, ListTypeConverter, filter_list_types
@@ -55,6 +56,7 @@
 from bot.utils.channel import is_mod_channel
 from bot.utils.lock import lock_arg
 from bot.utils.message_cache import MessageCache
+from bot.utils.retry import is_retryable_api_error
 
 log = get_logger(__name__)
 
@@ -108,7 +110,31 @@ async def cog_load(self) -> None:
         await self.bot.wait_until_guild_available()
 
         log.trace("Loading filtering information from the database.")
-        raw_filter_lists = await self.bot.api_client.get("bot/filter/filter_lists")
+        for attempt in range(1, URLs.connect_max_retries + 1):
+            try:
+                raw_filter_lists = await self.bot.api_client.get("bot/filter/filter_lists")
+                break
+            except Exception as error:
+                is_retryable = is_retryable_api_error(error)
+                is_last_attempt = attempt == URLs.connect_max_retries
+
+                if not is_retryable:
+                    raise
+
+                if is_last_attempt:
+                    log.exception("Failed to load filtering data after %d attempts.", URLs.connect_max_retries)
+                    raise
+
+                backoff_seconds = URLs.connect_initial_backoff * (2 ** (attempt - 1))
+                log.warning(
+                    "Failed to load filtering data (attempt %d/%d). Retrying in %d second(s): %s",
+                    attempt,
+                    URLs.connect_max_retries,
+                    backoff_seconds,
+                    error
+                )
+                await asyncio.sleep(backoff_seconds)
+
         example_list = None
         for raw_filter_list in raw_filter_lists:
             loaded_list = self._load_raw_filter_list(raw_filter_list)

bot/exts/info/python_news.py

@@ -1,3 +1,4 @@
+import asyncio
 import re
 import typing as t
 from datetime import UTC, datetime, timedelta
@@ -12,7 +13,9 @@
 
 from bot import constants
 from bot.bot import Bot
+from bot.constants import URLs
 from bot.log import get_logger
+from bot.utils.retry import is_retryable_api_error
 from bot.utils.webhooks import send_webhook
 
 PEPS_RSS_URL = "https://peps.python.org/peps.rss"
@@ -46,19 +49,45 @@ def __init__(self, bot: Bot):
 
     async def cog_load(self) -> None:
         """Load all existing seen items from db and create any missing mailing lists."""
-        with sentry_sdk.start_span(description="Fetch mailing lists from site"):
-            response = await self.bot.api_client.get("bot/mailing-lists")
-
-        for mailing_list in response:
-            self.seen_items[mailing_list["name"]] = set(mailing_list["seen_items"])
-
-        with sentry_sdk.start_span(description="Update site with new mailing lists"):
-            for mailing_list in ("pep", *constants.PythonNews.mail_lists):
-                if mailing_list not in self.seen_items:
-                    await self.bot.api_client.post("bot/mailing-lists", json={"name": mailing_list})
-                    self.seen_items[mailing_list] = set()
-
-        self.fetch_new_media.start()
+        for attempt in range(1, URLs.connect_max_retries + 1):
+            try:
+                with sentry_sdk.start_span(description="Fetch mailing lists from site"):
+                    response = await self.bot.api_client.get("bot/mailing-lists")
+
+                # Rebuild state on each successful fetch (avoid partial state across retries)
+                self.seen_items = {}
+                for mailing_list in response:
+                    self.seen_items[mailing_list["name"]] = set(mailing_list["seen_items"])
+
+                with sentry_sdk.start_span(description="Update site with new mailing lists"):
+                    for mailing_list in ("pep", *constants.PythonNews.mail_lists):
+                        if mailing_list not in self.seen_items:
+                            await self.bot.api_client.post("bot/mailing-lists", json={"name": mailing_list})
+                            self.seen_items[mailing_list] = set()
+
+                self.fetch_new_media.start()
+                return
+
+            except Exception as error:
+                if not is_retryable_api_error(error):
+                    raise
+
+                if attempt == URLs.connect_max_retries:
+                    log.exception(
+                        "Failed to load PythonNews mailing lists after %d attempt(s).",
+                        URLs.connect_max_retries,
+                    )
+                    raise
+
+                backoff_seconds = URLs.connect_initial_backoff * (2 ** (attempt - 1))
+                log.warning(
+                    "Failed to load PythonNews mailing lists (attempt %d/%d). Retrying in %d second(s). Error: %s",
+                    attempt,
+                    URLs.connect_max_retries,
+                    backoff_seconds,
+                    error,
+                )
+                await asyncio.sleep(backoff_seconds)
 
     async def cog_unload(self) -> None:
         """Stop news posting tasks on cog unload."""

bot/exts/moderation/infraction/superstarify.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import random
 import textwrap
@@ -10,13 +11,15 @@
 
 from bot import constants
 from bot.bot import Bot
+from bot.constants import URLs
 from bot.converters import Duration, DurationOrExpiry
 from bot.decorators import ensure_future_timestamp
 from bot.exts.moderation.infraction import _utils
 from bot.exts.moderation.infraction._scheduler import InfractionScheduler
 from bot.log import get_logger
 from bot.utils import time
 from bot.utils.messages import format_user
+from bot.utils.retry import is_retryable_api_error
 
 log = get_logger(__name__)
 NICKNAME_POLICY_URL = "https://pythondiscord.com/pages/rules/#nickname-policy"
@@ -43,9 +46,7 @@ async def on_member_update(self, before: Member, after: Member) -> None:
             f"{after.display_name}. Checking if the user is in superstar-prison..."
         )
 
-        active_superstarifies = await self.bot.api_client.get(
-            "bot/infractions",
-            params={
+        active_superstarifies = await self._fetch_with_retries(params={
                 "active": "true",
                 "type": "superstar",
                 "user__id": str(before.id)
@@ -84,9 +85,7 @@ async def on_member_update(self, before: Member, after: Member) -> None:
     @Cog.listener()
     async def on_member_join(self, member: Member) -> None:
         """Reapply active superstar infractions for returning members."""
-        active_superstarifies = await self.bot.api_client.get(
-            "bot/infractions",
-            params={
+        active_superstarifies = await self._fetch_with_retries(params={
                 "active": "true",
                 "type": "superstar",
                 "user__id": member.id
@@ -238,6 +237,22 @@ async def cog_check(self, ctx: Context) -> bool:
         """Only allow moderators to invoke the commands in this cog."""
         return await has_any_role(*constants.MODERATION_ROLES).predicate(ctx)
 
+    async def _fetch_with_retries(self,
+        retries: int = URLs.connect_max_retries,
+        params: dict[str, str] | None = None) -> list[dict]:
+        """Fetch infractions from the API with retries and exponential backoff."""
+        if retries < 1:
+            raise ValueError("retries must be at least 1")
+
+        for attempt in range(1, retries + 1):
+            try:
+                return await self.bot.api_client.get("bot/infractions", params=params)
+            except Exception as e:
+                if attempt == retries or not is_retryable_api_error(e):
+                    raise
+                await asyncio.sleep(URLs.connect_initial_backoff * (2 ** (attempt - 1)))
+        return None
+
 
 async def setup(bot: Bot) -> None:
     """Load the Superstarify cog."""

bot/exts/utils/reminders.py

@@ -1,3 +1,4 @@
+import asyncio
 import random
 import textwrap
 import typing as t
@@ -23,6 +24,7 @@
     POSITIVE_REPLIES,
     Roles,
     STAFF_AND_COMMUNITY_ROLES,
+    URLs,
 )
 from bot.converters import Duration, UnambiguousUser
 from bot.errors import LockedResourceError
@@ -224,13 +226,25 @@ async def cog_unload(self) -> None:
     async def cog_load(self) -> None:
         """Get all current reminders from the API and reschedule them."""
         await self.bot.wait_until_guild_available()
-        response = await self.bot.api_client.get(
-            "bot/reminders",
-            params={"active": "true"}
-        )
-
+        # retry fetching reminders with exponential backoff
+        for attempt in range(1, URLs.connect_max_retries + 1):
+            try:
+                # response either throws, or is a list of reminders (possibly empty)
+                response = await self.bot.api_client.get(
+                    "bot/reminders",
+                    params={"active": "true"}
+                )
+                break
+            except Exception as e:
+                if not self._check_error_is_retriable(e):
+                    log.error(f"Failed to load reminders due to non-retryable error: {e}")
+                    raise
+                log.warning(f"Attempt {attempt} - Failed to fetch reminders from the API: {e}")
+                if attempt == URLs.connect_max_retries:
+                    log.error("Max retry attempts reached. Failed to load reminders.")
+                    raise
+            await asyncio.sleep(URLs.connect_initial_backoff * (2 ** (attempt - 1)))  # Exponential backoff
         now = datetime.now(UTC)
-
         for reminder in response:
             is_valid, *_ = self.ensure_valid_reminder(reminder)
             if not is_valid:
@@ -244,6 +258,13 @@ async def cog_load(self) -> None:
             else:
                 self.schedule_reminder(reminder)
 
+    def _check_error_is_retriable(self, error: Exception) -> bool:
+        """Return whether loading filter lists failed due to some temporary error, thus retrying could help."""
+        if isinstance(error, ResponseCodeError):
+            return error.status in (408, 429) or error.status >= 500
+
+        return isinstance(error, (TimeoutError, OSError))
+
     def ensure_valid_reminder(self, reminder: dict) -> tuple[bool, discord.TextChannel]:
         """Ensure reminder channel can be fetched otherwise delete the reminder."""
         channel = self.bot.get_channel(reminder["channel_id"])

bot/utils/retry.py

@@ -0,0 +1,9 @@
+from pydis_core.site_api import ResponseCodeError
+
+
+def is_retryable_api_error(error: Exception) -> bool:
+    """Return whether an API error is temporary and worth retrying."""
+    if isinstance(error, ResponseCodeError):
+        return error.status in (408, 429) or error.status >= 500
+
+    return isinstance(error, (TimeoutError, OSError))