Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 30 additions & 2 deletions services/mail-system/mail-ingest/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import smtplib
import re
import json
import ssl
import time
import uuid
import logging
Expand All @@ -25,6 +26,12 @@
MAILSERVER_HOST = os.getenv("MAILSERVER_HOST", "mailserver")
MAILSERVER_PORT = int(os.getenv("MAILSERVER_PORT", "25"))
MAILSERVER_HELO_DOMAIN = os.getenv("MAILSERVER_HELO_DOMAIN", "mail-ingest.local")
# When set, log in after STARTTLS. The mailserver then accepts the relay on its
# mail-ingest listener without re-checking the original sender's SPF.
MAILSERVER_SMTP_USER = os.getenv("MAILSERVER_SMTP_USER", "").strip()
MAILSERVER_SMTP_PASSWORD = os.getenv("MAILSERVER_SMTP_PASSWORD", "")
# The mailserver's certificate carries its public name, not the container name.
MAILSERVER_TLS_SERVER_NAME = os.getenv("MAILSERVER_TLS_SERVER_NAME", "").strip() or MAILSERVER_HOST
SMTP_TIMEOUT = float(os.getenv("SMTP_TIMEOUT_SECONDS", "15"))
SMTP_RETRY_ATTEMPTS = int(os.getenv("SMTP_RETRY_ATTEMPTS", "2"))
SMTP_RETRY_DELAY = float(os.getenv("SMTP_RETRY_DELAY_SECONDS", "2"))
Expand Down Expand Up @@ -216,6 +223,17 @@ def build_fallback_message(
return message.as_bytes()


def login(smtp: smtplib.SMTP) -> None:
"""STARTTLS, verifying the mailserver's certificate, then SASL login."""

# smtplib verifies the certificate against the host it connected to, which
# is the container name. Check it against the mailserver's public name.
smtp._host = MAILSERVER_TLS_SERVER_NAME
smtp.starttls(context=ssl.create_default_context())
smtp.ehlo(MAILSERVER_HELO_DOMAIN)
smtp.login(MAILSERVER_SMTP_USER, MAILSERVER_SMTP_PASSWORD)


def smtp_forward(envelope_from: str, envelope_to: str, raw_mime: bytes) -> None:
"""
Blocking SMTP send into docker-mailserver. Retries a few times to avoid transient drops.
Expand All @@ -230,12 +248,22 @@ def smtp_forward(envelope_from: str, envelope_to: str, raw_mime: bytes) -> None:
try:
with smtplib.SMTP(MAILSERVER_HOST, MAILSERVER_PORT, timeout=SMTP_TIMEOUT) as smtp:
smtp.ehlo(MAILSERVER_HELO_DOMAIN)
# internal, no TLS/auth needed
if MAILSERVER_SMTP_USER:
login(smtp)
smtp.mail(envelope_from)
refused = {}
for rcpt in recipients:
smtp.rcpt(rcpt)
code, resp = smtp.rcpt(rcpt)
if code not in (250, 251):
refused[rcpt] = (code, resp)
if len(refused) == len(recipients):
raise smtplib.SMTPRecipientsRefused(refused)
if refused:
logger.error("smtp_recipients_partly_refused %s", {"refused": repr(refused)})
smtp.data(raw_mime)
return
except smtplib.SMTPRecipientsRefused:
raise
except Exception as exc: # noqa: PERF203 - we want to surface all SMTP/network issues
last_exc = exc
logger.warning(
Expand Down
6 changes: 5 additions & 1 deletion services/mail-system/mail-ingest/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,11 @@ services:
environment:
- MAILGUN_WEBHOOK_SIGNING_KEY=${MAILGUN_WEBHOOK_SIGNING_KEY}
- MAILSERVER_HOST=mailserver
- MAILSERVER_PORT=25
# Authenticated, local-delivery-only listener; see mailserver/config/user-patches.sh
- MAILSERVER_PORT=10587
- MAILSERVER_SMTP_USER=mail-ingest@${BASE_DOMAIN}
- MAILSERVER_SMTP_PASSWORD=${MAIL_INGEST_SMTP_PASSWORD:?MAIL_INGEST_SMTP_PASSWORD is unset; run corekit up mailserver first}
- MAILSERVER_TLS_SERVER_NAME=${MAILSERVER_WEBMAIL_HOSTNAME}
- MAILSERVER_HELO_DOMAIN=mail-ingest.${BASE_DOMAIN}
expose:
- '3000'
Expand Down
101 changes: 101 additions & 0 deletions services/mail-system/mail-ingest/test_app.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import hashlib
import hmac
import json
import smtplib
import time
import unittest
from unittest import mock
from email import policy
from email.parser import BytesParser

Expand Down Expand Up @@ -241,3 +243,102 @@ def test_non_numeric_timestamp_is_not_fresh(self):

if __name__ == "__main__":
unittest.main()


class FakeSMTP:
"""Records what smtp_forward does to the mailserver connection."""

instances: list["FakeSMTP"] = []
rcpt_codes: dict[str, int] = {}
starttls_error: Exception | None = None

def __init__(self, host, port, timeout=None):
self._host = host
self.calls: list[tuple] = []
FakeSMTP.instances.append(self)

def __enter__(self):
return self

def __exit__(self, *exc):
return False

def ehlo(self, name):
self.calls.append(("ehlo", name))

def starttls(self, context=None):
if FakeSMTP.starttls_error:
raise FakeSMTP.starttls_error
self.calls.append(("starttls", self._host, context.check_hostname))

def login(self, user, password):
self.calls.append(("login", user))

def mail(self, sender):
self.calls.append(("mail", sender))

def rcpt(self, rcpt):
code = FakeSMTP.rcpt_codes.get(rcpt, 250)
self.calls.append(("rcpt", rcpt))
return code, b"5.7.1 refused" if code >= 400 else b"2.1.5 Ok"

def data(self, raw):
self.calls.append(("data",))


class SmtpForwardTests(unittest.TestCase):
def setUp(self):
FakeSMTP.instances = []
FakeSMTP.rcpt_codes = {}
FakeSMTP.starttls_error = None
patches = [
mock.patch.object(app_module.smtplib, "SMTP", FakeSMTP),
mock.patch.object(app_module, "SMTP_RETRY_ATTEMPTS", 1),
mock.patch.object(app_module, "MAILSERVER_SMTP_USER", "mail-ingest@example.com"),
mock.patch.object(app_module, "MAILSERVER_SMTP_PASSWORD", "pw"),
mock.patch.object(app_module, "MAILSERVER_TLS_SERVER_NAME", "mail.example.com"),
]
for p in patches:
p.start()
self.addCleanup(p.stop)

def calls(self):
return [c[0] for c in FakeSMTP.instances[-1].calls]

def test_logs_in_after_starttls_verified_against_the_public_name(self):
app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw")
self.assertEqual(
self.calls(), ["ehlo", "starttls", "ehlo", "login", "mail", "rcpt", "data"]
)
self.assertEqual(
FakeSMTP.instances[-1].calls[1], ("starttls", "mail.example.com", True)
)

def test_never_logs_in_when_starttls_is_unavailable(self):
FakeSMTP.starttls_error = smtplib.SMTPNotSupportedError("no STARTTLS")
with self.assertRaises(smtplib.SMTPNotSupportedError):
app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw")
self.assertNotIn("login", self.calls())

def test_without_a_login_configured_it_relays_unauthenticated(self):
with mock.patch.object(app_module, "MAILSERVER_SMTP_USER", ""):
app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw")
self.assertEqual(self.calls(), ["ehlo", "mail", "rcpt", "data"])

def test_refused_recipient_raises_instead_of_reaching_data(self):
FakeSMTP.rcpt_codes = {"tom@example.com": 550}
with self.assertRaises(smtplib.SMTPRecipientsRefused):
app_module.smtp_forward("a@hotmail.com", "tom@example.com", b"raw")
self.assertNotIn("data", self.calls())

def test_partly_refused_recipients_still_deliver_to_the_rest(self):
FakeSMTP.rcpt_codes = {"bad@example.com": 550}
with self.assertLogs("mail-ingest", level="ERROR"):
app_module.smtp_forward("a@hotmail.com", "tom@example.com, bad@example.com", b"raw")
self.assertIn("data", self.calls())

def test_refused_recipient_is_reported_to_mailgun_as_422(self):
FakeSMTP.rcpt_codes = {"info@example.com": 550}
with mock.patch.object(app_module, "MAILGUN_WEBHOOK_SIGNING_KEY", SIGNING_KEY):
response = TestClient(app).post("/mailgun/incoming", data=signed_form(token="token-rcpt"))
self.assertEqual(response.status_code, 422)
23 changes: 23 additions & 0 deletions services/mail-system/mailserver/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,29 @@ docker exec mailserver setup email list
- Authentication: noreply@yourdomain.com
- Password: auto-generated (see `.env`)

### Inbound Mail via Mailgun (mail-ingest)

MX points at Mailgun. Mailgun checks SPF and DKIM at the edge, then posts each
message to `mail-ingest`, which relays it to this container over SMTP.

That relay does **not** use port 25. On port 25 Postfix re-checks the original
sender's SPF against mail-ingest's Docker address, which fails for every sender
that publishes `-all`. Instead, `config/user-patches.sh` adds a listener on port
`10587` (not published on the host) that:

- requires STARTTLS and a login as `mail-ingest@<yourdomain>`, and nobody else
- accepts local recipients only, so it cannot relay even when authenticated
- skips the SPF policy check
- hides the login from Rspamd, OpenDKIM and OpenDMARC, so the message is still
scanned and DMARC-checked as inbound mail

`secrets.sh` generates `MAIL_INGEST_SMTP_PASSWORD`, and `startup.sh` creates the
account. The account is barred from submission (587/465), so its credential
cannot send mail out.

Do not set `PERMIT_DOCKER` to `network` or `connected-networks` to make inbound
work: that lets any container on the network relay mail out through Mailgun.

### n8n Integration Setup

**Create SMTP Credentials in n8n:**
Expand Down
66 changes: 66 additions & 0 deletions services/mail-system/mailserver/config/user-patches.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#!/bin/bash
# user-patches.sh - Run by docker-mailserver inside the container at startup,
# after its own configuration and before Postfix starts.
# prepare.sh copies this file into data/dms/config/. A container restart runs
# it again over the already-patched files, so every step must be idempotent.

set -euo pipefail

# ── mail-ingest listener ──
# mail-ingest relays mail that Mailgun has already accepted, after Mailgun's
# own SPF/DKIM checks at the edge. On port 25 Postfix re-checks the original
# sender's SPF against mail-ingest's Docker address, which fails for every
# sender publishing -all.
#
# This listener is for mail-ingest alone:
# - requires STARTTLS and a SASL login as MAIL_INGEST_SMTP_USER, nobody else
# - accepts local recipients only, so it cannot relay even when authenticated
# - skips the SPF policy service
# - withholds the login from milters, so Rspamd, OpenDKIM and OpenDMARC
# treat the message as inbound rather than as mail sent by a local user
# The port is not published on the host.
# MAIL_INGEST_SMTP_USER is also barred from submission (587/465), so its
# credential cannot be used to send mail out.

INGEST_PORT="${MAIL_INGEST_SMTP_PORT:-10587}"
INGEST_USER="${MAIL_INGEST_SMTP_USER:-}"

if [[ -z "$INGEST_USER" ]]; then
echo "user-patches: MAIL_INGEST_SMTP_USER is unset; mail-ingest listener not configured"
exit 0
fi

printf '%s OK\n' "$INGEST_USER" > /etc/postfix/mail-ingest-login
printf '%s REJECT this account only delivers inbound mail\n' "$INGEST_USER" > /etc/postfix/mail-ingest-deny

# Keep the mailbox quota check that port 25 applies, when quotas are enabled.
recipient_restrictions="reject_unauth_destination"
if postconf -h smtpd_recipient_restrictions | grep -q 'inet:localhost:65265'; then
recipient_restrictions+=", check_policy_service inet:localhost:65265"
fi

postconf -e \
"mail_ingest_client_restrictions = check_sasl_access texthash:/etc/postfix/mail-ingest-login, reject" \
"mail_ingest_relay_restrictions = reject_unauth_destination, permit" \
"mail_ingest_recipient_restrictions = ${recipient_restrictions}" \
"mail_ingest_milter_mail_macros = i {mail_addr} {mail_host} {mail_mailer}"

postconf -M "${INGEST_PORT}/inet=${INGEST_PORT} inet n - n - - smtpd"
postconf -P \
"${INGEST_PORT}/inet/syslog_name=postfix/mail-ingest" \
"${INGEST_PORT}/inet/smtpd_tls_security_level=encrypt" \
"${INGEST_PORT}/inet/smtpd_tls_auth_only=yes" \
"${INGEST_PORT}/inet/smtpd_sasl_auth_enable=yes" \
"${INGEST_PORT}/inet/smtpd_sasl_type=dovecot" \
"${INGEST_PORT}/inet/smtpd_sasl_authenticated_header=no" \
"${INGEST_PORT}/inet/smtpd_client_restrictions=\$mail_ingest_client_restrictions" \
"${INGEST_PORT}/inet/smtpd_relay_restrictions=\$mail_ingest_relay_restrictions" \
"${INGEST_PORT}/inet/smtpd_recipient_restrictions=\$mail_ingest_recipient_restrictions" \
"${INGEST_PORT}/inet/milter_mail_macros=\$mail_ingest_milter_mail_macros"

mua_sender="$(postconf -h mua_sender_restrictions)"
if [[ "$mua_sender" != *mail-ingest-deny* ]]; then
postconf -e "mua_sender_restrictions = check_sasl_access texthash:/etc/postfix/mail-ingest-deny, ${mua_sender}"
fi

echo "user-patches: mail-ingest listener on ${INGEST_PORT} for ${INGEST_USER}"
1 change: 1 addition & 0 deletions services/mail-system/mailserver/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ services:
- RELAY_PASSWORD=${MAILGUN_SMTP_PASSWORD}
- RELAY_TLS_LEVEL=encrypt
- ACCOUNT_PROVISIONER=FILE
- MAIL_INGEST_SMTP_USER=mail-ingest@${BASE_DOMAIN}
- ENABLE_UPDATE_CHECK=0
cap_add:
- NET_ADMIN
Expand Down
9 changes: 9 additions & 0 deletions services/mail-system/mailserver/prepare.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,12 @@ F2B_DST="${SCRIPT_DIR}/data/dms/config/fail2ban-jail.cf"
if [ -f "$F2B_SRC" ]; then
cp -v "$F2B_SRC" "$F2B_DST"
fi

# ── User patches (mail-ingest listener) ──
PATCHES_SRC="${SCRIPT_DIR}/config/user-patches.sh"
PATCHES_DST="${SCRIPT_DIR}/data/dms/config/user-patches.sh"

if [ -f "$PATCHES_SRC" ]; then
cp -v "$PATCHES_SRC" "$PATCHES_DST"
chmod 755 "$PATCHES_DST"
fi
1 change: 1 addition & 0 deletions services/mail-system/mailserver/secrets.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ declare -A SECRETS=(
["SMTP_PASS"]="password:16"
["MAIL_NOREPLY_PASSWORD"]="password:32"
["MAIL_INFO_PASSWORD"]="password:32"
["MAIL_INGEST_SMTP_PASSWORD"]="password:32"
)

############################################
Expand Down
3 changes: 3 additions & 0 deletions services/mail-system/mailserver/startup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,8 @@ ensure_account() {

ensure_account "noreply@${BASE_DOMAIN}" "${MAIL_NOREPLY_PASSWORD}"
ensure_account "info@${BASE_DOMAIN}" "${MAIL_INFO_PASSWORD}"
# mail-ingest logs in on its own listener to deliver Mailgun's inbound mail
# (see config/user-patches.sh). It is barred from submission.
ensure_account "mail-ingest@${BASE_DOMAIN}" "${MAIL_INGEST_SMTP_PASSWORD}"

log_success "Mail accounts provisioned"