From 5a8ab885c6d97c147483eedd64b53321b78e7a59 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 19 Nov 2026 01:59:47 +0000 Subject: [PATCH] mail-ingest: relay on an authenticated, local-only listener so SPF stops rejecting inbound Since PERMIT_DOCKER=none, mail-ingest is no longer in mynetworks, so Postfix runs the SPF policy on every message it relays. That checks the original sender's SPF against mail-ingest's Docker address, so every sender that publishes -all has been rejected with 550 5.7.23 (hotmail, Amazon SES, Postmark, dribbble). ~all senders were delivered, which is why it went unnoticed. Mailgun has already checked SPF and DKIM by the time we see the message. mailserver gets a second smtpd on 10587, added by a tracked user-patches.sh: STARTTLS and a login as mail-ingest@ only, local recipients only, no SPF policy, and the login hidden from the milters so Rspamd, OpenDKIM and OpenDMARC still treat the message as inbound. The same login is refused on 587/465, so a leaked credential cannot send out. The password is generated by secrets.sh and the account created by startup.sh, like the other mail accounts. mail-ingest logs in on that port, verifying the certificate against the mailserver's public name. It now also checks RCPT replies: a refused recipient used to surface only as "no valid recipients" at DATA, and with several recipients a refused one was silently dropped. --- services/mail-system/mail-ingest/app.py | 32 +++++- .../mail-ingest/docker-compose.yml | 6 +- services/mail-system/mail-ingest/test_app.py | 101 ++++++++++++++++++ services/mail-system/mailserver/README.md | 23 ++++ .../mailserver/config/user-patches.sh | 66 ++++++++++++ .../mail-system/mailserver/docker-compose.yml | 1 + services/mail-system/mailserver/prepare.sh | 9 ++ services/mail-system/mailserver/secrets.sh | 1 + services/mail-system/mailserver/startup.sh | 3 + 9 files changed, 239 insertions(+), 3 deletions(-) create mode 100755 services/mail-system/mailserver/config/user-patches.sh diff --git a/services/mail-system/mail-ingest/app.py b/services/mail-system/mail-ingest/app.py index 493f1bda..8fd38efb 100644 --- a/services/mail-system/mail-ingest/app.py +++ b/services/mail-system/mail-ingest/app.py @@ -4,6 +4,7 @@ import smtplib import re import json +import ssl import time import uuid import logging @@ -25,6 +26,12 @@ MAILSERVER_HOST = os.getenv("MAILSERVER_HOST", "mailserver") MAILSERVER_PORT = int(os.getenv("MAILSERVER_PORT", "25")) MAILSERVER_HELO_DOMAIN = os.getenv("MAILSERVER_HELO_DOMAIN", "mail-ingest.local") +# When set, log in after STARTTLS. The mailserver then accepts the relay on its +# mail-ingest listener without re-checking the original sender's SPF. +MAILSERVER_SMTP_USER = os.getenv("MAILSERVER_SMTP_USER", "").strip() +MAILSERVER_SMTP_PASSWORD = os.getenv("MAILSERVER_SMTP_PASSWORD", "") +# The mailserver's certificate carries its public name, not the container name. +MAILSERVER_TLS_SERVER_NAME = os.getenv("MAILSERVER_TLS_SERVER_NAME", "").strip() or MAILSERVER_HOST SMTP_TIMEOUT = float(os.getenv("SMTP_TIMEOUT_SECONDS", "15")) SMTP_RETRY_ATTEMPTS = int(os.getenv("SMTP_RETRY_ATTEMPTS", "2")) SMTP_RETRY_DELAY = float(os.getenv("SMTP_RETRY_DELAY_SECONDS", "2")) @@ -216,6 +223,17 @@ def build_fallback_message( return message.as_bytes() +def login(smtp: smtplib.SMTP) -> None: + """STARTTLS, verifying the mailserver's certificate, then SASL login.""" + + # smtplib verifies the certificate against the host it connected to, which + # is the container name. Check it against the mailserver's public name. + smtp._host = MAILSERVER_TLS_SERVER_NAME + smtp.starttls(context=ssl.create_default_context()) + smtp.ehlo(MAILSERVER_HELO_DOMAIN) + smtp.login(MAILSERVER_SMTP_USER, MAILSERVER_SMTP_PASSWORD) + + def smtp_forward(envelope_from: str, envelope_to: str, raw_mime: bytes) -> None: """ Blocking SMTP send into docker-mailserver. Retries a few times to avoid transient drops. @@ -230,12 +248,22 @@ def smtp_forward(envelope_from: str, envelope_to: str, raw_mime: bytes) -> None: try: with smtplib.SMTP(MAILSERVER_HOST, MAILSERVER_PORT, timeout=SMTP_TIMEOUT) as smtp: smtp.ehlo(MAILSERVER_HELO_DOMAIN) - # internal, no TLS/auth needed + if MAILSERVER_SMTP_USER: + login(smtp) smtp.mail(envelope_from) + refused = {} for rcpt in recipients: - smtp.rcpt(rcpt) + code, resp = smtp.rcpt(rcpt) + if code not in (250, 251): + refused[rcpt] = (code, resp) + if len(refused) == len(recipients): + raise smtplib.SMTPRecipientsRefused(refused) + if refused: + logger.error("smtp_recipients_partly_refused %s", {"refused": repr(refused)}) smtp.data(raw_mime) return + except smtplib.SMTPRecipientsRefused: + raise except Exception as exc: # noqa: PERF203 - we want to surface all SMTP/network issues last_exc = exc logger.warning( diff --git a/services/mail-system/mail-ingest/docker-compose.yml b/services/mail-system/mail-ingest/docker-compose.yml index 93696053..23f1b59b 100644 --- a/services/mail-system/mail-ingest/docker-compose.yml +++ b/services/mail-system/mail-ingest/docker-compose.yml @@ -10,7 +10,11 @@ services: environment: - MAILGUN_WEBHOOK_SIGNING_KEY=${MAILGUN_WEBHOOK_SIGNING_KEY} - MAILSERVER_HOST=mailserver - - MAILSERVER_PORT=25 + # Authenticated, local-delivery-only listener; see mailserver/config/user-patches.sh + - MAILSERVER_PORT=10587 + - MAILSERVER_SMTP_USER=mail-ingest@${BASE_DOMAIN} + - MAILSERVER_SMTP_PASSWORD=${MAIL_INGEST_SMTP_PASSWORD:?MAIL_INGEST_SMTP_PASSWORD is unset; run corekit up mailserver first} + - MAILSERVER_TLS_SERVER_NAME=${MAILSERVER_WEBMAIL_HOSTNAME} - MAILSERVER_HELO_DOMAIN=mail-ingest.${BASE_DOMAIN} expose: - '3000' diff --git a/services/mail-system/mail-ingest/test_app.py b/services/mail-system/mail-ingest/test_app.py index 121eceef..7d2e6e8b 100644 --- a/services/mail-system/mail-ingest/test_app.py +++ b/services/mail-system/mail-ingest/test_app.py @@ -1,8 +1,10 @@ import hashlib import hmac import json +import smtplib import time import unittest +from unittest import mock from email import policy from email.parser import BytesParser @@ -241,3 +243,102 @@ def test_non_numeric_timestamp_is_not_fresh(self): if __name__ == "__main__": unittest.main() + + +class FakeSMTP: + """Records what smtp_forward does to the mailserver connection.""" + + instances: list["FakeSMTP"] = [] + rcpt_codes: dict[str, int] = {} + starttls_error: Exception | None = None + + def __init__(self, host, port, timeout=None): + self._host = host + self.calls: list[tuple] = [] + FakeSMTP.instances.append(self) + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def ehlo(self, name): + self.calls.append(("ehlo", name)) + + def starttls(self, context=None): + if FakeSMTP.starttls_error: + raise FakeSMTP.starttls_error + self.calls.append(("starttls", self._host, context.check_hostname)) + + def login(self, user, password): + self.calls.append(("login", user)) + + def mail(self, sender): + self.calls.append(("mail", sender)) + + def rcpt(self, rcpt): + code = FakeSMTP.rcpt_codes.get(rcpt, 250) + self.calls.append(("rcpt", rcpt)) + return code, b"5.7.1 refused" if code >= 400 else b"2.1.5 Ok" + + def data(self, raw): + self.calls.append(("data",)) + + +class SmtpForwardTests(unittest.TestCase): + def setUp(self): + FakeSMTP.instances = [] + FakeSMTP.rcpt_codes = {} + FakeSMTP.starttls_error = None + patches = [ + mock.patch.object(app_module.smtplib, "SMTP", FakeSMTP), + mock.patch.object(app_module, "SMTP_RETRY_ATTEMPTS", 1), + mock.patch.object(app_module, "MAILSERVER_SMTP_USER", "mail-ingest@example.com"), + mock.patch.object(app_module, "MAILSERVER_SMTP_PASSWORD", "pw"), + mock.patch.object(app_module, "MAILSERVER_TLS_SERVER_NAME", "mail.example.com"), + ] + for p in patches: + p.start() + self.addCleanup(p.stop) + + def calls(self): + return [c[0] for c in FakeSMTP.instances[-1].calls] + + def test_logs_in_after_starttls_verified_against_the_public_name(self): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertEqual( + self.calls(), ["ehlo", "starttls", "ehlo", "login", "mail", "rcpt", "data"] + ) + self.assertEqual( + FakeSMTP.instances[-1].calls[1], ("starttls", "mail.example.com", True) + ) + + def test_never_logs_in_when_starttls_is_unavailable(self): + FakeSMTP.starttls_error = smtplib.SMTPNotSupportedError("no STARTTLS") + with self.assertRaises(smtplib.SMTPNotSupportedError): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertNotIn("login", self.calls()) + + def test_without_a_login_configured_it_relays_unauthenticated(self): + with mock.patch.object(app_module, "MAILSERVER_SMTP_USER", ""): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertEqual(self.calls(), ["ehlo", "mail", "rcpt", "data"]) + + def test_refused_recipient_raises_instead_of_reaching_data(self): + FakeSMTP.rcpt_codes = {"tom@example.com": 550} + with self.assertRaises(smtplib.SMTPRecipientsRefused): + app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw") + self.assertNotIn("data", self.calls()) + + def test_partly_refused_recipients_still_deliver_to_the_rest(self): + FakeSMTP.rcpt_codes = {"bad@example.com": 550} + with self.assertLogs("mail-ingest", level="ERROR"): + app_module.smtp_forward("a@hotmail.com", "tom@example.com, bad@example.com", b"raw") + self.assertIn("data", self.calls()) + + def test_refused_recipient_is_reported_to_mailgun_as_422(self): + FakeSMTP.rcpt_codes = {"info@example.com": 550} + with mock.patch.object(app_module, "MAILGUN_WEBHOOK_SIGNING_KEY", SIGNING_KEY): + response = TestClient(app).post("/mailgun/incoming", data=signed_form(token="token-rcpt")) + self.assertEqual(response.status_code, 422) diff --git a/services/mail-system/mailserver/README.md b/services/mail-system/mailserver/README.md index 29190bec..3b23c68d 100644 --- a/services/mail-system/mailserver/README.md +++ b/services/mail-system/mailserver/README.md @@ -85,6 +85,29 @@ docker exec mailserver setup email list - Authentication: noreply@yourdomain.com - Password: auto-generated (see `.env`) +### Inbound Mail via Mailgun (mail-ingest) + +MX points at Mailgun. Mailgun checks SPF and DKIM at the edge, then posts each +message to `mail-ingest`, which relays it to this container over SMTP. + +That relay does **not** use port 25. On port 25 Postfix re-checks the original +sender's SPF against mail-ingest's Docker address, which fails for every sender +that publishes `-all`. Instead, `config/user-patches.sh` adds a listener on port +`10587` (not published on the host) that: + +- requires STARTTLS and a login as `mail-ingest@`, and nobody else +- accepts local recipients only, so it cannot relay even when authenticated +- skips the SPF policy check +- hides the login from Rspamd, OpenDKIM and OpenDMARC, so the message is still + scanned and DMARC-checked as inbound mail + +`secrets.sh` generates `MAIL_INGEST_SMTP_PASSWORD`, and `startup.sh` creates the +account. The account is barred from submission (587/465), so its credential +cannot send mail out. + +Do not set `PERMIT_DOCKER` to `network` or `connected-networks` to make inbound +work: that lets any container on the network relay mail out through Mailgun. + ### n8n Integration Setup **Create SMTP Credentials in n8n:** diff --git a/services/mail-system/mailserver/config/user-patches.sh b/services/mail-system/mailserver/config/user-patches.sh new file mode 100755 index 00000000..cc0b4fe6 --- /dev/null +++ b/services/mail-system/mailserver/config/user-patches.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# user-patches.sh - Run by docker-mailserver inside the container at startup, +# after its own configuration and before Postfix starts. +# prepare.sh copies this file into data/dms/config/. A container restart runs +# it again over the already-patched files, so every step must be idempotent. + +set -euo pipefail + +# ── mail-ingest listener ── +# mail-ingest relays mail that Mailgun has already accepted, after Mailgun's +# own SPF/DKIM checks at the edge. On port 25 Postfix re-checks the original +# sender's SPF against mail-ingest's Docker address, which fails for every +# sender publishing -all. +# +# This listener is for mail-ingest alone: +# - requires STARTTLS and a SASL login as MAIL_INGEST_SMTP_USER, nobody else +# - accepts local recipients only, so it cannot relay even when authenticated +# - skips the SPF policy service +# - withholds the login from milters, so Rspamd, OpenDKIM and OpenDMARC +# treat the message as inbound rather than as mail sent by a local user +# The port is not published on the host. +# MAIL_INGEST_SMTP_USER is also barred from submission (587/465), so its +# credential cannot be used to send mail out. + +INGEST_PORT="${MAIL_INGEST_SMTP_PORT:-10587}" +INGEST_USER="${MAIL_INGEST_SMTP_USER:-}" + +if [[ -z "$INGEST_USER" ]]; then + echo "user-patches: MAIL_INGEST_SMTP_USER is unset; mail-ingest listener not configured" + exit 0 +fi + +printf '%s OK\n' "$INGEST_USER" > /etc/postfix/mail-ingest-login +printf '%s REJECT this account only delivers inbound mail\n' "$INGEST_USER" > /etc/postfix/mail-ingest-deny + +# Keep the mailbox quota check that port 25 applies, when quotas are enabled. +recipient_restrictions="reject_unauth_destination" +if postconf -h smtpd_recipient_restrictions | grep -q 'inet:localhost:65265'; then + recipient_restrictions+=", check_policy_service inet:localhost:65265" +fi + +postconf -e \ + "mail_ingest_client_restrictions = check_sasl_access texthash:/etc/postfix/mail-ingest-login, reject" \ + "mail_ingest_relay_restrictions = reject_unauth_destination, permit" \ + "mail_ingest_recipient_restrictions = ${recipient_restrictions}" \ + "mail_ingest_milter_mail_macros = i {mail_addr} {mail_host} {mail_mailer}" + +postconf -M "${INGEST_PORT}/inet=${INGEST_PORT} inet n - n - - smtpd" +postconf -P \ + "${INGEST_PORT}/inet/syslog_name=postfix/mail-ingest" \ + "${INGEST_PORT}/inet/smtpd_tls_security_level=encrypt" \ + "${INGEST_PORT}/inet/smtpd_tls_auth_only=yes" \ + "${INGEST_PORT}/inet/smtpd_sasl_auth_enable=yes" \ + "${INGEST_PORT}/inet/smtpd_sasl_type=dovecot" \ + "${INGEST_PORT}/inet/smtpd_sasl_authenticated_header=no" \ + "${INGEST_PORT}/inet/smtpd_client_restrictions=\$mail_ingest_client_restrictions" \ + "${INGEST_PORT}/inet/smtpd_relay_restrictions=\$mail_ingest_relay_restrictions" \ + "${INGEST_PORT}/inet/smtpd_recipient_restrictions=\$mail_ingest_recipient_restrictions" \ + "${INGEST_PORT}/inet/milter_mail_macros=\$mail_ingest_milter_mail_macros" + +mua_sender="$(postconf -h mua_sender_restrictions)" +if [[ "$mua_sender" != *mail-ingest-deny* ]]; then + postconf -e "mua_sender_restrictions = check_sasl_access texthash:/etc/postfix/mail-ingest-deny, ${mua_sender}" +fi + +echo "user-patches: mail-ingest listener on ${INGEST_PORT} for ${INGEST_USER}" diff --git a/services/mail-system/mailserver/docker-compose.yml b/services/mail-system/mailserver/docker-compose.yml index c785982f..a171dcbc 100644 --- a/services/mail-system/mailserver/docker-compose.yml +++ b/services/mail-system/mailserver/docker-compose.yml @@ -60,6 +60,7 @@ services: - RELAY_PASSWORD=${MAILGUN_SMTP_PASSWORD} - RELAY_TLS_LEVEL=encrypt - ACCOUNT_PROVISIONER=FILE + - MAIL_INGEST_SMTP_USER=mail-ingest@${BASE_DOMAIN} - ENABLE_UPDATE_CHECK=0 cap_add: - NET_ADMIN diff --git a/services/mail-system/mailserver/prepare.sh b/services/mail-system/mailserver/prepare.sh index a772bdc1..d10cf903 100755 --- a/services/mail-system/mailserver/prepare.sh +++ b/services/mail-system/mailserver/prepare.sh @@ -20,3 +20,12 @@ F2B_DST="${SCRIPT_DIR}/data/dms/config/fail2ban-jail.cf" if [ -f "$F2B_SRC" ]; then cp -v "$F2B_SRC" "$F2B_DST" fi + +# ── User patches (mail-ingest listener) ── +PATCHES_SRC="${SCRIPT_DIR}/config/user-patches.sh" +PATCHES_DST="${SCRIPT_DIR}/data/dms/config/user-patches.sh" + +if [ -f "$PATCHES_SRC" ]; then + cp -v "$PATCHES_SRC" "$PATCHES_DST" + chmod 755 "$PATCHES_DST" +fi diff --git a/services/mail-system/mailserver/secrets.sh b/services/mail-system/mailserver/secrets.sh index d775dbe7..00d4c267 100644 --- a/services/mail-system/mailserver/secrets.sh +++ b/services/mail-system/mailserver/secrets.sh @@ -14,6 +14,7 @@ declare -A SECRETS=( ["SMTP_PASS"]="password:16" ["MAIL_NOREPLY_PASSWORD"]="password:32" ["MAIL_INFO_PASSWORD"]="password:32" + ["MAIL_INGEST_SMTP_PASSWORD"]="password:32" ) ############################################ diff --git a/services/mail-system/mailserver/startup.sh b/services/mail-system/mailserver/startup.sh index f2879bfc..e2c58a18 100755 --- a/services/mail-system/mailserver/startup.sh +++ b/services/mail-system/mailserver/startup.sh @@ -35,5 +35,8 @@ ensure_account() { ensure_account "noreply@${BASE_DOMAIN}" "${MAIL_NOREPLY_PASSWORD}" ensure_account "info@${BASE_DOMAIN}" "${MAIL_INFO_PASSWORD}" +# mail-ingest logs in on its own listener to deliver Mailgun's inbound mail +# (see config/user-patches.sh). It is barred from submission. +ensure_account "mail-ingest@${BASE_DOMAIN}" "${MAIL_INGEST_SMTP_PASSWORD}" log_success "Mail accounts provisioned"