fly.toml

# fly.toml app configuration file generated for SourceBox Sentry Command
#
# See https://fly.io/docs/reference/configuration/ for information about settings

app = "sentinel-command"
primary_region = "sjc"

[build]

[env]
  FRONTEND_URL = "https://app.sentinel-command.com"
  # DATABASE_URL is deliberately NOT here any more. It points at Postgres
  # now, which makes it a credential, so it lives in `fly secrets` —
  # `fly.toml` is committed to a public repository.
  #
  # Self-hosted installs are unaffected: they get the SQLite default from
  # app/core/config.py and the data layer branches on the URL scheme.
  #
  # Segment-cache ceiling MUST stay well under [[vm]].memory_mb below —
  # above physical RAM, the kernel OOM-killer fires (killing every
  # org's streams at once) long before the cache's own eviction can.
  # 384 MiB on the 1 GiB machine leaves ~600 MiB for Python
  # (~25 concurrent live cameras). Scale this and memory_mb TOGETHER.
  # The database no longer shares this machine's RAM for page cache,
  # so there is a little more headroom here than there used to be.
  SEGMENT_CACHE_MAX_TOTAL_BYTES = "402653184"

# This app runs TWO process groups from ONE image. Fly has no per-group
# image, only per-group command — which is why the Sentinel AI agent
# resolves against backend/pyproject.toml rather than carrying its own
# dependency set.
#
#   app   — the web tier. Owns the volume and the HLS segment cache,
#           always-on (min_machines_running = 1 below).
#   agent — the Sentinel AI worker. No volume, scales to zero, woken by
#           the HMAC-signed wakeup webhook over .flycast.
#
# The agent is a separate PROCESS GROUP rather than a thread inside the
# web app on purpose: a run holds base64 frames for up to 270s, and the
# segment cache is already budgeted at 384 MiB of this machine's 1 GiB
# (see SEGMENT_CACHE_MAX_TOTAL_BYTES above). Sharing one machine is how
# the OOM killer ends up taking every org's streams down at once.
#
# Defining [processes] overrides the Dockerfile CMD for BOTH groups, so
# the web command below must stay in sync with that CMD.
[processes]
  app = "/app/.venv/bin/uvicorn app.main:app --host 0.0.0.0 --port 8000 --workers 1 --timeout-keep-alive 65 --forwarded-allow-ips=* --no-access-log"
  agent = "/app/.venv/bin/python -m app.sentinel_agent"

# Still required after the Postgres migration — /data holds HLS segment
# working files and the local backup directory. It is no longer where
# the database lives, which also means health_probes' disk check now
# measures segment storage rather than database growth.
#
# `processes` is REQUIRED here now. Without it the mount applies to every
# process group, and the agent machine fails to boot fighting the web
# machine for the volume's single attachment slot.
[[mounts]]
  source = "sentinel_data"
  destination = "/data"
  processes = ["app"]

[deploy]
  # CI uses build-image-then-`fly machine update` (see
  # .github/workflows/deploy.yml) which doesn't honour this strategy
  # — it goes straight to the in-place machine API.  This block
  # only matters if someone runs plain `fly deploy` manually.
  #
  # `immediate` was forced by SQLite-on-a-single-volume: rolling tries
  # to provision a parallel machine and errored on the volume's single
  # attachment slot. With the database on Postgres that constraint is
  # gone, so this now uses the default rolling strategy.
  #
  # Note the volume above still pins us to one machine for HLS segment
  # state, so a deploy is still a brief restart. Genuine zero-downtime
  # needs a second machine — which is now *possible* where it wasn't
  # before, rather than something this line alone delivers.

[http_service]
  internal_port = 8000
  force_https = true
  auto_stop_machines = "off"
  auto_start_machines = true
  min_machines_running = 1
  processes = ["app"]

  [[http_service.checks]]
    grace_period = "30s"
    interval = "10s"
    timeout = "5s"
    method = "GET"
    path = "/api/health"

# Internal-only service for the agent process group. Not exposed
# publicly: it is reachable at http://sentinel-command.flycast:8080 over
# 6PN, which is where SENTINEL_AGENT_WEBHOOK_URL points.
#
# Requires a private IPv6 on the app: `fly ips allocate-v6 --private`.
#
# NOT scaled to zero, despite that being this worker's obvious shape, and
# despite it being how the agent ran as its own app. Fly's proxy waits
# only ~8s for a machine it auto-started to bind its port, and this
# process cannot reliably beat that: Python plus the MCP SDK plus Sentry
# took ~7s before LiteLLM and ~10s after — so an auto-started machine got
# declared unreachable and Command Center's wakeup came back
# RemoteDisconnected. Measured, twice, on 2026-09-09.
#
# The 7s figure means this was always marginal and happened to fit; the
# LiteLLM move only exposed it. Deferring litellm's import cut boot from
# 16s to 10s, which was necessary but not sufficient, and trimming
# further imports would be tuning against a proxy limit we do not
# control and cannot test except in production.
#
# So the agent stays warm. One shared-cpu-1x/512MB machine is ~$2/month,
# which is noise against the LLM spend of a single run, and it removes
# cold starts from the wakeup path entirely rather than racing them.
[[services]]
  internal_port = 8080
  protocol = "tcp"
  auto_stop_machines = "off"
  auto_start_machines = true
  min_machines_running = 1
  processes = ["agent"]

  [[services.ports]]
    port = 8080

[[vm]]
  cpu_kind = "shared"
  cpus = 1
  memory_mb = 1024
  processes = ["app"]

# The agent needs less than the web tier: no segment cache, no SPA, one
# run at a time. 512 MiB comfortably holds the Python process plus the
# base64 frames of a single investigation, and it only bills while a run
# is actually in flight.
[[vm]]
  cpu_kind = "shared"
  cpus = 1
  memory_mb = 512
  processes = ["agent"]