diff --git a/services/mail-system/mail-ingest/app.py b/services/mail-system/mail-ingest/app.py index 493f1bda..8fd38efb 100644 --- a/services/mail-system/mail-ingest/app.py +++ b/services/mail-system/mail-ingest/app.py @@ -4,6 +4,7 @@ import smtplib import re import json +import ssl import time import uuid import logging @@ -25,6 +26,12 @@ MAILSERVER_HOST = os.getenv("MAILSERVER_HOST", "mailserver") MAILSERVER_PORT = int(os.getenv("MAILSERVER_PORT", "25")) MAILSERVER_HELO_DOMAIN = os.getenv("MAILSERVER_HELO_DOMAIN", "mail-ingest.local") +# When set, log in after STARTTLS. The mailserver then accepts the relay on its +# mail-ingest listener without re-checking the original sender's SPF. +MAILSERVER_SMTP_USER = os.getenv("MAILSERVER_SMTP_USER", "").strip() +MAILSERVER_SMTP_PASSWORD = os.getenv("MAILSERVER_SMTP_PASSWORD", "") +# The mailserver's certificate carries its public name, not the container name. +MAILSERVER_TLS_SERVER_NAME = os.getenv("MAILSERVER_TLS_SERVER_NAME", "").strip() or MAILSERVER_HOST SMTP_TIMEOUT = float(os.getenv("SMTP_TIMEOUT_SECONDS", "15")) SMTP_RETRY_ATTEMPTS = int(os.getenv("SMTP_RETRY_ATTEMPTS", "2")) SMTP_RETRY_DELAY = float(os.getenv("SMTP_RETRY_DELAY_SECONDS", "2")) @@ -216,6 +223,17 @@ def build_fallback_message( return message.as_bytes() +def login(smtp: smtplib.SMTP) -> None: + """STARTTLS, verifying the mailserver's certificate, then SASL login.""" + + # smtplib verifies the certificate against the host it connected to, which + # is the container name. Check it against the mailserver's public name. + smtp._host = MAILSERVER_TLS_SERVER_NAME + smtp.starttls(context=ssl.create_default_context()) + smtp.ehlo(MAILSERVER_HELO_DOMAIN) + smtp.login(MAILSERVER_SMTP_USER, MAILSERVER_SMTP_PASSWORD) + + def smtp_forward(envelope_from: str, envelope_to: str, raw_mime: bytes) -> None: """ Blocking SMTP send into docker-mailserver. Retries a few times to avoid transient drops. @@ -230,12 +248,22 @@ def smtp_forward(envelope_from: str, envelope_to: str, raw_mime: bytes) -> None: try: with smtplib.SMTP(MAILSERVER_HOST, MAILSERVER_PORT, timeout=SMTP_TIMEOUT) as smtp: smtp.ehlo(MAILSERVER_HELO_DOMAIN) - # internal, no TLS/auth needed + if MAILSERVER_SMTP_USER: + login(smtp) smtp.mail(envelope_from) + refused = {} for rcpt in recipients: - smtp.rcpt(rcpt) + code, resp = smtp.rcpt(rcpt) + if code not in (250, 251): + refused[rcpt] = (code, resp) + if len(refused) == len(recipients): + raise smtplib.SMTPRecipientsRefused(refused) + if refused: + logger.error("smtp_recipients_partly_refused %s", {"refused": repr(refused)}) smtp.data(raw_mime) return + except smtplib.SMTPRecipientsRefused: + raise except Exception as exc: # noqa: PERF203 - we want to surface all SMTP/network issues last_exc = exc logger.warning( diff --git a/services/mail-system/mail-ingest/docker-compose.yml b/services/mail-system/mail-ingest/docker-compose.yml index 93696053..23f1b59b 100644 --- a/services/mail-system/mail-ingest/docker-compose.yml +++ b/services/mail-system/mail-ingest/docker-compose.yml @@ -10,7 +10,11 @@ services: environment: - MAILGUN_WEBHOOK_SIGNING_KEY=${MAILGUN_WEBHOOK_SIGNING_KEY} - MAILSERVER_HOST=mailserver - - MAILSERVER_PORT=25 + # Authenticated, local-delivery-only listener; see mailserver/config/user-patches.sh + - MAILSERVER_PORT=10587 + - MAILSERVER_SMTP_USER=mail-ingest@${BASE_DOMAIN} + - MAILSERVER_SMTP_PASSWORD=${MAIL_INGEST_SMTP_PASSWORD:?MAIL_INGEST_SMTP_PASSWORD is unset; run corekit up mailserver first} + - MAILSERVER_TLS_SERVER_NAME=${MAILSERVER_WEBMAIL_HOSTNAME} - MAILSERVER_HELO_DOMAIN=mail-ingest.${BASE_DOMAIN} expose: - '3000' diff --git a/services/mail-system/mail-ingest/test_app.py b/services/mail-system/mail-ingest/test_app.py index 121eceef..7d2e6e8b 100644 --- a/services/mail-system/mail-ingest/test_app.py +++ b/services/mail-system/mail-ingest/test_app.py @@ -1,8 +1,10 @@ import hashlib import hmac import json +import smtplib import time import unittest +from unittest import mock from email import policy from email.parser import BytesParser @@ -241,3 +243,102 @@ def test_non_numeric_timestamp_is_not_fresh(self): if __name__ == "__main__": unittest.main() + + +class FakeSMTP: + """Records what smtp_forward does to the mailserver connection.""" + + instances: list["FakeSMTP"] = [] + rcpt_codes: dict[str, int] = {} + starttls_error: Exception | None = None + + def __init__(self, host, port, timeout=None): + self._host = host + self.calls: list[tuple] = [] + FakeSMTP.instances.append(self) + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def ehlo(self, name): + self.calls.append(("ehlo", name)) + + def starttls(self, context=None): + if FakeSMTP.starttls_error: + raise FakeSMTP.starttls_error + self.calls.append(("starttls", self._host, context.check_hostname)) + + def login(self, user, password): + self.calls.append(("login", user)) + + def mail(self, sender): + self.calls.append(("mail", sender)) + + def rcpt(self, rcpt): + code = FakeSMTP.rcpt_codes.get(rcpt, 250) + self.calls.append(("rcpt", rcpt)) + return code, b"5.7.1 refused" if code >= 400 else b"2.1.5 Ok" + + def data(self, raw): + self.calls.append(("data",)) + + +class SmtpForwardTests(unittest.TestCase): + def setUp(self): + FakeSMTP.instances = [] + FakeSMTP.rcpt_codes = {} + FakeSMTP.starttls_error = None + patches = [ + mock.patch.object(app_module.smtplib, "SMTP", FakeSMTP), + mock.patch.object(app_module, "SMTP_RETRY_ATTEMPTS", 1), + mock.patch.object(app_module, "MAILSERVER_SMTP_USER", "mail-ingest@example.com"), + mock.patch.object(app_module, "MAILSERVER_SMTP_PASSWORD", "pw"), + mock.patch.object(app_module, "MAILSERVER_TLS_SERVER_NAME", "mail.example.com"), + ] + for p in patches: + p.start() + self.addCleanup(p.stop) + + def calls(self): + return [c[0] for c in FakeSMTP.instances[-1].calls] + + def test_logs_in_after_starttls_verified_against_the_public_name(self): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertEqual( + self.calls(), ["ehlo", "starttls", "ehlo", "login", "mail", "rcpt", "data"] + ) + self.assertEqual( + FakeSMTP.instances[-1].calls[1], ("starttls", "mail.example.com", True) + ) + + def test_never_logs_in_when_starttls_is_unavailable(self): + FakeSMTP.starttls_error = smtplib.SMTPNotSupportedError("no STARTTLS") + with self.assertRaises(smtplib.SMTPNotSupportedError): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertNotIn("login", self.calls()) + + def test_without_a_login_configured_it_relays_unauthenticated(self): + with mock.patch.object(app_module, "MAILSERVER_SMTP_USER", ""): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertEqual(self.calls(), ["ehlo", "mail", "rcpt", "data"]) + + def test_refused_recipient_raises_instead_of_reaching_data(self): + FakeSMTP.rcpt_codes = {"tom@example.com": 550} + with self.assertRaises(smtplib.SMTPRecipientsRefused): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertNotIn("data", self.calls()) + + def test_partly_refused_recipients_still_deliver_to_the_rest(self): + FakeSMTP.rcpt_codes = {"bad@example.com": 550} + with self.assertLogs("mail-ingest", level="ERROR"): + app_module.smtp_forward("a@hotmail.com", "tom@example.com, bad@example.com", b"raw") + self.assertIn("data", self.calls()) + + def test_refused_recipient_is_reported_to_mailgun_as_422(self): + FakeSMTP.rcpt_codes = {"info@example.com": 550} + with mock.patch.object(app_module, "MAILGUN_WEBHOOK_SIGNING_KEY", SIGNING_KEY): + response = TestClient(app).post("/mailgun/incoming", data=signed_form(token="token-rcpt")) + self.assertEqual(response.status_code, 422) diff --git a/services/mail-system/mailserver/README.md b/services/mail-system/mailserver/README.md index 29190bec..3b23c68d 100644 --- a/services/mail-system/mailserver/README.md +++ b/services/mail-system/mailserver/README.md @@ -85,6 +85,29 @@ docker exec mailserver setup email list - Authentication: noreply@yourdomain.com - Password: auto-generated (see `.env`) +### Inbound Mail via Mailgun (mail-ingest) + +MX points at Mailgun. Mailgun checks SPF and DKIM at the edge, then posts each +message to `mail-ingest`, which relays it to this container over SMTP. + +That relay does **not** use port 25. On port 25 Postfix re-checks the original +sender's SPF against mail-ingest's Docker address, which fails for every sender +that publishes `-all`. Instead, `config/user-patches.sh` adds a listener on port +`10587` (not published on the host) that: + +- requires STARTTLS and a login as `mail-ingest@`, and nobody else +- accepts local recipients only, so it cannot relay even when authenticated +- skips the SPF policy check +- hides the login from Rspamd, OpenDKIM and OpenDMARC, so the message is still + scanned and DMARC-checked as inbound mail + +`secrets.sh` generates `MAIL_INGEST_SMTP_PASSWORD`, and `startup.sh` creates the +account. The account is barred from submission (587/465), so its credential +cannot send mail out. + +Do not set `PERMIT_DOCKER` to `network` or `connected-networks` to make inbound +work: that lets any container on the network relay mail out through Mailgun. + ### n8n Integration Setup **Create SMTP Credentials in n8n:** diff --git a/services/mail-system/mailserver/config/user-patches.sh b/services/mail-system/mailserver/config/user-patches.sh new file mode 100755 index 00000000..cc0b4fe6 --- /dev/null +++ b/services/mail-system/mailserver/config/user-patches.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# user-patches.sh - Run by docker-mailserver inside the container at startup, +# after its own configuration and before Postfix starts. +# prepare.sh copies this file into data/dms/config/. A container restart runs +# it again over the already-patched files, so every step must be idempotent. + +set -euo pipefail + +# ── mail-ingest listener ── +# mail-ingest relays mail that Mailgun has already accepted, after Mailgun's +# own SPF/DKIM checks at the edge. On port 25 Postfix re-checks the original +# sender's SPF against mail-ingest's Docker address, which fails for every +# sender publishing -all. +# +# This listener is for mail-ingest alone: +# - requires STARTTLS and a SASL login as MAIL_INGEST_SMTP_USER, nobody else +# - accepts local recipients only, so it cannot relay even when authenticated +# - skips the SPF policy service +# - withholds the login from milters, so Rspamd, OpenDKIM and OpenDMARC +# treat the message as inbound rather than as mail sent by a local user +# The port is not published on the host. +# MAIL_INGEST_SMTP_USER is also barred from submission (587/465), so its +# credential cannot be used to send mail out. + +INGEST_PORT="${MAIL_INGEST_SMTP_PORT:-10587}" +INGEST_USER="${MAIL_INGEST_SMTP_USER:-}" + +if [[ -z "$INGEST_USER" ]]; then + echo "user-patches: MAIL_INGEST_SMTP_USER is unset; mail-ingest listener not configured" + exit 0 +fi + +printf '%s OK\n' "$INGEST_USER" > /etc/postfix/mail-ingest-login +printf '%s REJECT this account only delivers inbound mail\n' "$INGEST_USER" > /etc/postfix/mail-ingest-deny + +# Keep the mailbox quota check that port 25 applies, when quotas are enabled. +recipient_restrictions="reject_unauth_destination" +if postconf -h smtpd_recipient_restrictions | grep -q 'inet:localhost:65265'; then + recipient_restrictions+=", check_policy_service inet:localhost:65265" +fi + +postconf -e \ + "mail_ingest_client_restrictions = check_sasl_access texthash:/etc/postfix/mail-ingest-login, reject" \ + "mail_ingest_relay_restrictions = reject_unauth_destination, permit" \ + "mail_ingest_recipient_restrictions = ${recipient_restrictions}" \ + "mail_ingest_milter_mail_macros = i {mail_addr} {mail_host} {mail_mailer}" + +postconf -M "${INGEST_PORT}/inet=${INGEST_PORT} inet n - n - - smtpd" +postconf -P \ + "${INGEST_PORT}/inet/syslog_name=postfix/mail-ingest" \ + "${INGEST_PORT}/inet/smtpd_tls_security_level=encrypt" \ + "${INGEST_PORT}/inet/smtpd_tls_auth_only=yes" \ + "${INGEST_PORT}/inet/smtpd_sasl_auth_enable=yes" \ + "${INGEST_PORT}/inet/smtpd_sasl_type=dovecot" \ + "${INGEST_PORT}/inet/smtpd_sasl_authenticated_header=no" \ + "${INGEST_PORT}/inet/smtpd_client_restrictions=\$mail_ingest_client_restrictions" \ + "${INGEST_PORT}/inet/smtpd_relay_restrictions=\$mail_ingest_relay_restrictions" \ + "${INGEST_PORT}/inet/smtpd_recipient_restrictions=\$mail_ingest_recipient_restrictions" \ + "${INGEST_PORT}/inet/milter_mail_macros=\$mail_ingest_milter_mail_macros" + +mua_sender="$(postconf -h mua_sender_restrictions)" +if [[ "$mua_sender" != *mail-ingest-deny* ]]; then + postconf -e "mua_sender_restrictions = check_sasl_access texthash:/etc/postfix/mail-ingest-deny, ${mua_sender}" +fi + +echo "user-patches: mail-ingest listener on ${INGEST_PORT} for ${INGEST_USER}" diff --git a/services/mail-system/mailserver/docker-compose.yml b/services/mail-system/mailserver/docker-compose.yml index c785982f..a171dcbc 100644 --- a/services/mail-system/mailserver/docker-compose.yml +++ b/services/mail-system/mailserver/docker-compose.yml @@ -60,6 +60,7 @@ services: - RELAY_PASSWORD=${MAILGUN_SMTP_PASSWORD} - RELAY_TLS_LEVEL=encrypt - ACCOUNT_PROVISIONER=FILE + - MAIL_INGEST_SMTP_USER=mail-ingest@${BASE_DOMAIN} - ENABLE_UPDATE_CHECK=0 cap_add: - NET_ADMIN diff --git a/services/mail-system/mailserver/prepare.sh b/services/mail-system/mailserver/prepare.sh index a772bdc1..d10cf903 100755 --- a/services/mail-system/mailserver/prepare.sh +++ b/services/mail-system/mailserver/prepare.sh @@ -20,3 +20,12 @@ F2B_DST="${SCRIPT_DIR}/data/dms/config/fail2ban-jail.cf" if [ -f "$F2B_SRC" ]; then cp -v "$F2B_SRC" "$F2B_DST" fi + +# ── User patches (mail-ingest listener) ── +PATCHES_SRC="${SCRIPT_DIR}/config/user-patches.sh" +PATCHES_DST="${SCRIPT_DIR}/data/dms/config/user-patches.sh" + +if [ -f "$PATCHES_SRC" ]; then + cp -v "$PATCHES_SRC" "$PATCHES_DST" + chmod 755 "$PATCHES_DST" +fi diff --git a/services/mail-system/mailserver/secrets.sh b/services/mail-system/mailserver/secrets.sh index d775dbe7..00d4c267 100644 --- a/services/mail-system/mailserver/secrets.sh +++ b/services/mail-system/mailserver/secrets.sh @@ -14,6 +14,7 @@ declare -A SECRETS=( ["SMTP_PASS"]="password:16" ["MAIL_NOREPLY_PASSWORD"]="password:32" ["MAIL_INFO_PASSWORD"]="password:32" + ["MAIL_INGEST_SMTP_PASSWORD"]="password:32" ) ############################################ diff --git a/services/mail-system/mailserver/startup.sh b/services/mail-system/mailserver/startup.sh index f2879bfc..e2c58a18 100755 --- a/services/mail-system/mailserver/startup.sh +++ b/services/mail-system/mailserver/startup.sh @@ -35,5 +35,8 @@ ensure_account() { ensure_account "noreply@${BASE_DOMAIN}" "${MAIL_NOREPLY_PASSWORD}" ensure_account "info@${BASE_DOMAIN}" "${MAIL_INFO_PASSWORD}" +# mail-ingest logs in on its own listener to deliver Mailgun's inbound mail +# (see config/user-patches.sh). It is barred from submission. +ensure_account "mail-ingest@${BASE_DOMAIN}" "${MAIL_INGEST_SMTP_PASSWORD}" log_success "Mail accounts provisioned"