diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e3ee6d64..1214a486a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,21 +64,7 @@ The server will load environment variables from the `.env` file in the project r ### Healthchecks Reporting -Set `HEALTHCHECKS_URL` to a Healthchecks success ping URL to enable operational -reports. After a one-minute startup allowance, the server sends a POST every -five minutes, including a short diagnostic body. Configure the check with a -five-minute period and ten minutes of grace. - -A successful database query and an operational job executor produce a success -ping. An initializing, restarting, or livelock-paused executor, or a failed -database query, produces a `/fail` ping. Idle and busy executors are both healthy; -an individual failed package job does not mark the service unhealthy. Reports -time out after ten seconds and retry on the next scheduled report indefinitely. - -This is an internal operational check, not a probe of public DNS, TLS, or nginx, -and does not detect every possible stalled job or external dependency failure. -`/api/v1/status` remains a simple HTTP liveness check. Ping URLs are secrets: do -not commit them or expose them in logs. +Set `HEALTHCHECKS_URL` to a Healthchecks success ping URL to enable operational reports. After a one-minute startup allowance, the server sends a POST every five minutes, including a short diagnostic body. This expects a simple schedule with five-minute internals and ten minutes of grace. ## Quick Start: Running Integration Tests diff --git a/app/src/App/Server/Healthcheck.purs b/app/src/App/Server/Healthcheck.purs index c239ea814..93ddf2685 100644 --- a/app/src/App/Server/Healthcheck.purs +++ b/app/src/App/Server/Healthcheck.purs @@ -3,7 +3,6 @@ module Registry.App.Server.Healthcheck (run, report) where import Registry.App.Prelude import Control.Parallel as Parallel -import Data.DateTime (diff) import Data.Time.Duration (Milliseconds(..)) import Effect.Aff as Aff import Effect.Class.Console as Console @@ -12,15 +11,14 @@ import Fetch as Fetch import Registry.App.Effect.Db as Db import Registry.App.Server.Env (ServerEnv, runEffects) import Registry.App.Server.Env as Env +import Registry.Foreign.Console as Console.Foreign --- | Give startup one minute, then report on a five-minute cadence. Failures to --- | deliver a ping never stop the reporter; the next scheduled ping retries. +-- | Give startup one minute, then wait five minutes after a delivered report. +-- | Failed deliveries retry after one minute, indefinitely. run :: ServerEnv -> String -> Aff Unit run env url = Aff.delay (Milliseconds 60_000.0) *> loop where - interval = Milliseconds 300_000.0 loop = do - start <- nowUTC database <- runEffects env $ void Db.selectNextPublishJob executor <- liftEffect $ Ref.read env.executorStatus let @@ -32,11 +30,12 @@ run env url = Aff.delay (Milliseconds 60_000.0) *> loop Env.Paused -> Left "Executor paused after repeated job resets" Env.Restarting -> Left "Executor restarting after unexpected exit; see server logs" result <- Aff.attempt $ report url health - for_ (either Just (const Nothing) result) \_ -> - Console.warn "Healthcheck report failed; retrying next scheduled report" - end <- nowUTC - let Milliseconds elapsed = end `diff` start - Aff.delay $ Milliseconds $ max 0.0 (unwrap interval - elapsed) + case result of + Left error -> do + Console.warn "Healthcheck report failed; retrying in one minute:" + liftEffect $ Console.Foreign.warnError error + Aff.delay $ Milliseconds 60_000.0 + Right _ -> Aff.delay $ Milliseconds 300_000.0 loop -- | Bound requests and use abortable fetch so an unreachable monitoring service diff --git a/foreign/src/Foreign/Console.js b/foreign/src/Foreign/Console.js new file mode 100644 index 000000000..070bd7d6c --- /dev/null +++ b/foreign/src/Foreign/Console.js @@ -0,0 +1,3 @@ +import { inspect } from "node:util"; + +export const warnError = (error) => () => console.warn(inspect(error, { depth: null })); diff --git a/foreign/src/Foreign/Console.purs b/foreign/src/Foreign/Console.purs new file mode 100644 index 000000000..1a5c242e1 --- /dev/null +++ b/foreign/src/Foreign/Console.purs @@ -0,0 +1,8 @@ +module Registry.Foreign.Console (warnError) where + +import Prelude + +import Effect (Effect) +import Effect.Exception (Error) + +foreign import warnError :: Error -> Effect Unit