From 5b6966e014845fe70d4d513da2f6c41825dee508 Mon Sep 17 00:00:00 2001 From: Kyryl R Date: Wed, 28 Oct 2026 10:27:15 +0200 Subject: [PATCH] Harden nginx ingress admission and secret-safe logging Use emergency-only request error logging, bound header time, and group IPv6 source limits by prefix. Add an opt-in stream front that counts incomplete-header connections, with loopback-only plain HTTP by default. Size nginx workers and document deployment checks and topology limits. --- deployment/README.md | 206 ++++++++++++++++-- deployment/configs/compose.env.example | 18 +- .../configs/nginx/default.conf.template | 33 ++- .../nginx/direct/default.conf.template | 81 +++++++ .../nginx/direct/edge.conf.stream-template | 35 +++ deployment/configs/nginx/direct/nginx.conf | 40 ++++ deployment/configs/nginx/nginx.conf | 36 +++ deployment/docker-compose.direct.yml | 27 +++ deployment/docker-compose.yml | 3 + 9 files changed, 446 insertions(+), 33 deletions(-) create mode 100644 deployment/configs/nginx/direct/default.conf.template create mode 100644 deployment/configs/nginx/direct/edge.conf.stream-template create mode 100644 deployment/configs/nginx/direct/nginx.conf create mode 100644 deployment/configs/nginx/nginx.conf create mode 100644 deployment/docker-compose.direct.yml diff --git a/deployment/README.md b/deployment/README.md index f5cb393..fe78a04 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -9,16 +9,25 @@ This stack runs one relay instance behind an nginx reverse proxy: the relay and forwards everything else to it. TLS is not terminated here. Put a load balancer, CDN or ingress that speaks -HTTPS/WSS in front of the published port. +HTTPS/WSS in front of the published port, or, with nothing in front, add the +`docker-compose.direct.yml` override for local testing or after configuring TLS +in this stack (see [Per-source limits](#per-source-limits)). Do not expose the +direct override's plain HTTP port to the Internet: connection tokens travel in +the URL. ## Layout - `Dockerfile.relay`: image build, from the repository root context - `docker-compose.yml`: the two services above +- `docker-compose.direct.yml`: override for when nothing sits in front of the + stack; adds a per-source connection cap at accept - `configs/compose.env.example`: the single env file for the stack - `configs/relay/*.template`: rendered into the relay's `configuration/` at container start, then read by the binary +- `configs/nginx/nginx.conf`: nginx main config (worker pool, log defaults) - `configs/nginx/default.conf.template`: rendered by the nginx image entrypoint +- `configs/nginx/direct/`: the main config, http template and stream front + that `docker-compose.direct.yml` mounts instead - `scripts/render-relay-config.sh`: the relay container's entrypoint ## Quick start @@ -27,11 +36,22 @@ From this directory: ```bash cp ./configs/compose.env.example ./configs/compose.env -# edit APPLICATION_AUDIENCE and set TRUSTED_PROXY_CIDR (127.0.0.1/32 when -# nothing sits in front, otherwise the terminator range), then: +# edit APPLICATION_AUDIENCE and set TRUSTED_PROXY_CIDR, then either, behind a +# TLS terminator or load balancer (TRUSTED_PROXY_CIDR = its range): docker compose --env-file ./configs/compose.env up --build -d +# or, for local-only testing with nothing in front +# (TRUSTED_PROXY_CIDR=127.0.0.1/32; the override binds loopback): +docker compose -f docker-compose.yml -f docker-compose.direct.yml \ + --env-file ./configs/compose.env up --build -d ``` +With the override, pass the same `-f` flags to every later `docker compose` +command (or export `COMPOSE_FILE=docker-compose.yml:docker-compose.direct.yml`); +an `up` without them puts the default nginx configuration back. +The override requires Docker Compose 2.24.4 or newer for `!override` and replaces +the base port mapping with a loopback-only one. For a public direct deployment, +configure TLS in this nginx first and deliberately change that mapping. + Check it: ```bash @@ -52,10 +72,11 @@ Set these in `configs/compose.env`: accepts the wrong audience would either reject every client or, worse, accept tokens minted for another relay. - `TRUSTED_PROXY_CIDR`: the proxy range allowed to assert a client address - through `X-Forwarded-For`. Use `127.0.0.1/32` when nothing sits in front of - this stack, or the load balancer / CDN range when TLS terminates upstream. - The stack refuses to start without it, because a range that does not match - the terminator silently collapses the per-source limits into one bucket. + through `X-Forwarded-For`. Use the load balancer / CDN range when TLS + terminates upstream, or `127.0.0.1/32` when nothing sits in front of this + stack (the direct override then takes the address from its own front). The + stack refuses to start without it, because a range that does not match the + terminator silently collapses the per-source limits into one bucket. - `RELAY_PUBLIC_PORT`: the host port nginx publishes. - `RELAY_PORT`, `HEALTH_PORT`: in-container ports. - `MAX_TOKEN_LIFETIME_SECONDS`: longest remaining token lifetime accepted. @@ -68,6 +89,9 @@ LIMITS__MAX_CONNECTIONS=20000 SESSION__GRACE_SECONDS=30 ``` +Raising `LIMITS__MAX_CONNECTIONS` also needs nginx's `worker_connections` +raised; see [Capacity](#capacity). + The effective configuration is written to `/app/configuration/` in the container, so it can be inspected with `docker compose --env-file ./configs/compose.env exec relay cat configuration/base.yaml`. @@ -88,23 +112,161 @@ healthcheck polls `/ready` inside the relay container. The relay and nginx both speak plain HTTP here. Terminate TLS in front of the published port and forward to it, preserving the `Upgrade` and `Connection` -headers. If you would rather terminate in this stack, mount certificates into -the nginx service, add a `listen 443 ssl` server block to -`configs/nginx/default.conf.template`, and publish `443`. - -Connection tokens travel in the URL query string, so they appear in the access -logs of every proxy in the path. The template shipped here logs the path only -(`$uri`, never the query) and keeps the nginx error log at `crit`, because -upstream failures and rate-limit rejections embed the full request line. Any -upstream terminator, load balancer or CDN must strip the query from its access +headers. If you would rather terminate in this stack, nothing sits in front of +nginx, so use `docker-compose.direct.yml`: mount the certificates into the +nginx service, add `ssl` and the certificate directives to the unix `listen` in +`configs/nginx/direct/default.conf.template`, and set `RELAY_PUBLIC_PORT=443`. +The stream front then caps connections per source before the TLS handshake. + +## Logs + +Connection tokens and topics travel in the URL query string, so they reach the +logs of every proxy in the path that records the request line. The +configuration shipped here keeps them out of nginx's logs: + +- The access log records the path only (`$uri`, never the query). The main + config turns the stock http-level access log off, because its `main` format + writes the full request line. +- The http server's error log is at `emerg`. nginx appends the full request + line to every error-log message raised while it handles a request, whatever + the level: upstream failures and rate-limit rejections at `error`, worker + pool and file-descriptor exhaustion at `alert`. `crit` would still pass the + `alert` lines; only `emerg` withholds them. Those events therefore show only + as statuses in the access log (502, 503, 500, 408), and a worker pool that is + already full when a client connects leaves no line at all, except at the + direct override's stream front. The main config makes `emerg` the default + for any server block you add; keep it there. An `emerg` message raised while + handling a request, in practice a failed memory allocation, would still carry + the request line. +- With the direct override, the stream front logs refused connections at + `warn` with the client address only; it never sees a request line. + +Any upstream terminator, load balancer or CDN must strip the query from its own logs too. Keep `MAX_TOKEN_LIFETIME_SECONDS` short regardless. -The per-source limits (`PER_IP_*`) are keyed on the client address. When a load -balancer, CDN or another proxy sits in front, that address is the terminator's -unless `TRUSTED_PROXY_CIDR` names its range: real_ip then resolves the client -from `X-Forwarded-For`, and forwarded headers from any other peer are ignored, -so the limits cannot be spoofed. The stack requires the variable for exactly -that reason; set it to `127.0.0.1/32` only when nothing sits in front of nginx. +Log retention is left to the host. Both containers write client-driven lines +to stdout: nginx one access-log line per request, `limit_req` rejections +included, and the relay its own records. The compose file sets no `logging:` +options, so the host Docker daemon's driver decides how much is kept: +`json-file` without `max-size` keeps one ever-growing file per container, while +`local` rotates by default. Check it on the host: + +```bash +docker info --format '{{.LoggingDriver}}' +docker inspect --format '{{.Name}} {{json .HostConfig.LogConfig}}' \ + $(docker compose --env-file ./configs/compose.env ps -q) +``` + +If that shows `json-file` without `max-size`, set `log-opts` in the daemon's +`daemon.json`, or give both services a `logging:` section in an override file, +for example `logging: {driver: json-file, options: {max-size: 10m, max-file: '3'}}`. +Under an orchestrator, its own log rotation applies instead. + +## Per-source limits + +The per-source limits (`PER_IP_*`) are keyed on the client address: the whole +IPv4 address, or the /64 prefix of an IPv6 address, because one subscriber or +LAN normally holds a whole /64 and could otherwise take a fresh budget with +every address in it. Clients behind one IPv4 NAT, or on one shared IPv6 /64, +share a budget; raise the limits if that is normal for your users. +IPv4-mapped IPv6 addresses (`::ffff:a.b.c.d`) keep the whole address. + +When a load balancer, CDN or another proxy sits in front, that address is the +terminator's unless `TRUSTED_PROXY_CIDR` names its range: real_ip then resolves +the client from `X-Forwarded-For`, and forwarded headers from any other peer +are ignored, so the limits cannot be spoofed. The stack requires the variable +for exactly that reason. + +`limit_conn` and `limit_req` count a connection only once its request header +is complete. A client that opens connections and never finishes a header is +counted against no source, and enough of them fill nginx's worker pool, after +which nginx resets every new client. `client_header_timeout 10s` releases each +such connection after 10 seconds, but does not cap how many one source holds. +That cap has to apply where connections are accepted, which depends on what +sits in front: + +- **A front** (TLS terminator, load balancer, CDN): run `docker-compose.yml` + alone. The front must buffer each complete request header before forwarding + it and enforce its own per-client connection limit, and the published port + must be reachable only from the front. This stack cannot check either. An L4 + load balancer, TLS passthrough, or a front that streams partial headers does + not meet the requirement. +- **Nothing in front**: add `docker-compose.direct.yml`. It puts an nginx + `stream {}` front on the published port that caps connections per client + address at accept (`PER_IP_CONNECTIONS`, same key as above), before any HTTP + byte is read, and hands each connection to the http server over a unix + socket with the PROXY protocol, which carries the client address. Do not use + it behind a front: every client would share the front's address. + +nginx listens on IPv4 only inside the container. If Docker forwards IPv6 +clients through its userland proxy (for example when the port is published on +IPv6 but the compose network is IPv4-only), nginx sees the proxy's address for +all of them, so they share one budget. Check the address nginx logs for an +IPv6 client before relying on the per-source limits for IPv6. + +## Capacity + +nginx's connection pool (`worker_connections`) is per worker process, and +every proxied WebSocket holds two entries in the worker that accepted it, the +client side and the relay side; through the direct override's front it holds +four. With the image's stock `worker_connections 1024`, a worker fills at about +511 tunnels, far below the relay's `limits.max_connections` (10000 by default). +A full worker resets new clients at accept or answers 500 at the upstream +connect, and the stack still reports healthy, because the compose healthcheck +probes the relay directly. + +`configs/nginx/nginx.conf` and `configs/nginx/direct/nginx.conf` therefore +replace the image's main config with a pool of at least 2 × (direct: 4 ×) +`LIMITS__MAX_CONNECTIONS` plus headroom per worker: 24576 and 49152 for the +default 10000. The relay's limit then binds even if every tunnel lands on one +worker. When you raise `LIMITS__MAX_CONNECTIONS`, raise `worker_connections` +and `worker_rlimit_nofile` in the main config you run in proportion. nginx +allocates each worker's pool when the worker starts, so larger values cost +memory per worker even when idle; check the nginx container's memory after a +change. + +Whether the stock pool was the binding limit on a given host depends on its +worker count (`worker_processes auto` starts one per CPU visible to the +container), which this repository does not record. To see the effective values +and whether the descriptor limit could be raised: + +```bash +docker compose --env-file ./configs/compose.env exec nginx sh -c \ + 'nproc; nginx -T 2>/dev/null | grep -E "^[[:space:]]*(worker_processes|worker_connections|worker_rlimit_nofile)"' +docker compose --env-file ./configs/compose.env logs nginx | grep setrlimit +``` + +A `setrlimit(RLIMIT_NOFILE, ...) failed` line means the container's hard +descriptor limit is below `worker_rlimit_nofile`; raise it with +`ulimits: {nofile: ...}` on the nginx service. + +## Verification status + +This stack has not been deployed, and there is no live ingress or production +client to check it against. Both Compose modes passed local structural rendering +with `--no-env-resolution`, and both nginx variants passed `nginx -t` after the +pinned image rendered their templates. A local direct-mode probe with +`PER_IP_CONNECTIONS=1` held one incomplete header and confirmed that a second +connection from the same source was refused. IPv6 forwarding, real clients, +and production TLS remain unverified. Before relying on a deployment: + +```bash +# Both variants render (TRUSTED_PROXY_CIDR set in compose.env): +docker compose --env-file ./configs/compose.env config >/dev/null +docker compose -f docker-compose.yml -f docker-compose.direct.yml \ + --env-file ./configs/compose.env config >/dev/null +# After `up`, nginx accepts the rendered configuration; with the direct +# override, the stream front must be the listener on port 80: +docker compose --env-file ./configs/compose.env exec nginx nginx -T +``` + +The direct override relies on the nginx image entrypoint rendering +`*.stream-template` files into `/etc/nginx/stream-conf.d`. If it did not, the +stream block would be empty and the published port would refuse connections +rather than bypass the cap. Then confirm with real clients that the access log +shows client addresses (and, for IPv6, one key per /64), that connections over +`PER_IP_CONNECTIONS` from one source are refused, including ones that never +finish their request header, and that no nginx log line contains `auth=`. ## Scaling diff --git a/deployment/configs/compose.env.example b/deployment/configs/compose.env.example index 670ef1a..f9d829d 100644 --- a/deployment/configs/compose.env.example +++ b/deployment/configs/compose.env.example @@ -1,14 +1,15 @@ # Copy to ./configs/compose.env and edit. This is the only file the stack # needs; values are rendered into the relay configuration at container start. # Any further relay setting can be added here as SECTION__KEY=value, for -# example LIMITS__MAX_SESSIONS=1000. +# example LIMITS__MAX_SESSIONS=1000. Raising LIMITS__MAX_CONNECTIONS also +# needs worker_connections raised in the nginx main config (README: Capacity). # Audience the relay accepts in connection tokens. Required: clients sign # their tokens with this `aud`, and the relay refuses to start without it. APPLICATION_AUDIENCE=wss://relay.example.com -# Port published by nginx. TLS terminates in front of this stack, so this is -# normally behind a load balancer or CDN that speaks HTTPS/WSS. +# Port published by nginx. Normally behind a load balancer or CDN that speaks +# HTTPS/WSS; with nothing in front, run docker-compose.direct.yml as well. RELAY_PUBLIC_PORT=8080 # Ports inside the relay container: the WebSocket listener and the probes. @@ -19,13 +20,16 @@ HEALTH_PORT=8081 # Longest remaining lifetime accepted in a connection token, in seconds. MAX_TOKEN_LIFETIME_SECONDS=600 -# Per-source connection and request limits at the proxy. +# Per-source connection and request limits at the proxy, keyed on the client's +# IPv4 address or IPv6 /64. They count only complete requests unless +# docker-compose.direct.yml also caps connections at accept (README). PER_IP_CONNECTIONS=64 PER_IP_RATE=5 PER_IP_BURST=20 # Required: the range allowed to assert a client address through -# X-Forwarded-For (real_ip). Uncomment and set it to 127.0.0.1/32 when nothing -# sits in front of this stack, or to the load balancer / CDN range when TLS -# terminates upstream. +# X-Forwarded-For (real_ip). Uncomment and set it to the load balancer / CDN +# range when TLS terminates upstream, or to 127.0.0.1/32 when nothing sits in +# front of this stack (docker-compose.direct.yml then takes the client address +# from its own front instead). #TRUSTED_PROXY_CIDR=127.0.0.1/32 diff --git a/deployment/configs/nginx/default.conf.template b/deployment/configs/nginx/default.conf.template index 837959b..87f128b 100644 --- a/deployment/configs/nginx/default.conf.template +++ b/deployment/configs/nginx/default.conf.template @@ -1,8 +1,14 @@ # Rendered into /etc/nginx/conf.d/default.conf by the nginx image entrypoint, # which substitutes the environment variables that are set. # -# The stock image includes this file inside its own `http {}` block, so it +# configs/nginx/nginx.conf includes this file inside its `http {}` block, so it # holds http-level directives only (no `events` or `http` wrapper). +# +# limit_conn and limit_req below count a connection only once its request +# header is complete. Whatever sits in front of this stack must buffer complete +# request headers and enforce its own per-client connection limit. With nothing +# in front, use docker-compose.direct.yml instead, which caps connections per +# client address at accept. log_format relay '$remote_addr - $time_iso8601 "$request_method $uri" $status $body_bytes_sent "$http_user_agent"'; @@ -10,8 +16,17 @@ set_real_ip_from ${TRUSTED_PROXY_CIDR}; real_ip_header X-Forwarded-For; real_ip_recursive on; -limit_conn_zone $binary_remote_addr zone=perip:10m; -limit_req_zone $binary_remote_addr zone=perip_rate:10m rate=${PER_IP_RATE}r/s; +# Per-source key: the whole IPv4 address, or the /64 prefix of an IPv6 address, +# since one subscriber or LAN normally holds a whole /64. IPv4-mapped IPv6 +# addresses keep the whole address so that they do not all share one key. +map $binary_remote_addr $relay_limit_key { + "~(?s)^\x00{10}\xff\xff.{4}\z" $binary_remote_addr; + "~(?s)^(?.{8}).{8}\z" $relay_v6_prefix; + default $binary_remote_addr; +} + +limit_conn_zone $relay_limit_key zone=perip:10m; +limit_req_zone $relay_limit_key zone=perip_rate:10m rate=${PER_IP_RATE}r/s; # WebSocket connections are upgraded, plain requests are closed normally. map $http_upgrade $connection_upgrade { @@ -23,8 +38,18 @@ server { listen 80; server_name _; + # Bound how long a connection may take to send its request header (nginx + # default: 60 s, counted from accept). This shortens how long a stalled + # connection holds a worker pool entry; it does not cap how many one + # source holds. + client_header_timeout 10s; + access_log /dev/stdout relay; - error_log /dev/stderr crit; + # nginx appends the request line, which carries the connection token and + # topic, to every error-log message raised while handling a request, at + # every level: upstream failures and rate-limit rejections at error, pool + # and descriptor exhaustion at alert. Only emerg withholds it. + error_log /dev/stderr emerg; location = /health { proxy_pass http://relay:${HEALTH_PORT}; diff --git a/deployment/configs/nginx/direct/default.conf.template b/deployment/configs/nginx/direct/default.conf.template new file mode 100644 index 0000000..f783be3 --- /dev/null +++ b/deployment/configs/nginx/direct/default.conf.template @@ -0,0 +1,81 @@ +# Rendered into /etc/nginx/conf.d/default.conf by the nginx image entrypoint +# when docker-compose.direct.yml is in use (nothing in front of this stack). +# Clients reach this server only through the stream front in +# edge.conf.stream-template, which caps connections per client address at +# accept and passes the client address in a PROXY protocol header over a local +# unix socket. + +log_format relay '$remote_addr - $time_iso8601 "$request_method $uri" $status $body_bytes_sent "$http_user_agent"'; + +# The client address comes from the front's PROXY protocol header only; +# X-Forwarded-For from clients is never trusted here. +set_real_ip_from unix:; +real_ip_header proxy_protocol; + +# Per-source key: the whole IPv4 address, or the /64 prefix of an IPv6 address, +# since one subscriber or LAN normally holds a whole /64. IPv4-mapped IPv6 +# addresses keep the whole address so that they do not all share one key. +map $binary_remote_addr $relay_limit_key { + "~(?s)^\x00{10}\xff\xff.{4}\z" $binary_remote_addr; + "~(?s)^(?.{8}).{8}\z" $relay_v6_prefix; + default $binary_remote_addr; +} + +limit_conn_zone $relay_limit_key zone=perip:10m; +limit_req_zone $relay_limit_key zone=perip_rate:10m rate=${PER_IP_RATE}r/s; + +# WebSocket connections are upgraded, plain requests are closed normally. +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +server { + listen unix:/run/nginx-edge/http.sock proxy_protocol; + server_name _; + + # Bound how long a connection may take to send its request header (nginx + # default: 60 s, counted from accept). The front closes the client side + # when this server closes its side. + client_header_timeout 10s; + + access_log /dev/stdout relay; + # nginx appends the request line, which carries the connection token and + # topic, to every error-log message raised while handling a request, at + # every level: upstream failures and rate-limit rejections at error, pool + # and descriptor exhaustion at alert. Only emerg withholds it. + error_log /dev/stderr emerg; + + location = /health { + proxy_pass http://relay:${HEALTH_PORT}; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location / { + # The front already caps connections per source at accept; these + # still pace requests and bound concurrent requests per source. + limit_conn perip ${PER_IP_CONNECTIONS}; + limit_req zone=perip_rate burst=${PER_IP_BURST} nodelay; + + proxy_pass http://relay:${RELAY_PORT}; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + # Nothing trusted sits in front, so a client-supplied chain is + # replaced rather than extended. + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + + # Relay frames are interactive; do not buffer them, and keep the + # tunnel open well past the relay's 30 second keepalive pings. + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + } +} diff --git a/deployment/configs/nginx/direct/edge.conf.stream-template b/deployment/configs/nginx/direct/edge.conf.stream-template new file mode 100644 index 0000000..d3e6451 --- /dev/null +++ b/deployment/configs/nginx/direct/edge.conf.stream-template @@ -0,0 +1,35 @@ +# Rendered into /etc/nginx/stream-conf.d/edge.conf by the nginx image +# entrypoint when docker-compose.direct.yml is in use. The entrypoint strips +# only the ".stream-template" suffix, so keep ".conf" in this file name: +# direct/nginx.conf includes stream-conf.d/*.conf, and a file rendered without +# it is never loaded, which leaves the published port with no listener. +# +# Caps connections per client address at accept, before nginx reads any HTTP +# byte. limit_conn in the http server counts a connection only once its request +# header is complete, so a client that never finishes its header would +# otherwise be counted nowhere. Do not use this behind a load balancer or CDN: +# every client would share its address. + +# Same key as the http server: the whole IPv4 address, or the /64 prefix of an +# IPv6 address; IPv4-mapped IPv6 addresses keep the whole address. +map $binary_remote_addr $relay_limit_key { + "~(?s)^\x00{10}\xff\xff.{4}\z" $binary_remote_addr; + "~(?s)^(?.{8}).{8}\z" $relay_v6_prefix; + default $binary_remote_addr; +} + +limit_conn_zone $relay_limit_key zone=tcp_perip:10m; +# Refusals are logged with the client address only; the front never sees a +# request line. +limit_conn_log_level warn; + +server { + listen 80; + limit_conn tcp_perip ${PER_IP_CONNECTIONS}; + + proxy_pass unix:/run/nginx-edge/http.sock; + proxy_protocol on; + # Tunnels carry relay pings every 30 s; keep well above that, as the http + # server's proxy_read_timeout does. + proxy_timeout 3600s; +} diff --git a/deployment/configs/nginx/direct/nginx.conf b/deployment/configs/nginx/direct/nginx.conf new file mode 100644 index 0000000..5f6ddc2 --- /dev/null +++ b/deployment/configs/nginx/direct/nginx.conf @@ -0,0 +1,40 @@ +# Main nginx configuration for docker-compose.direct.yml (nothing in front of +# this stack): configs/nginx/nginx.conf plus a stream{} block for the front in +# edge.conf.stream-template. The image entrypoint renders *.stream-template +# files only into a stream block the main config already includes; it cannot +# add one to this read-only mount. +user nginx; +worker_processes auto; + +# Through the front a proxied WebSocket holds four pool entries (client, front +# to unix socket, http side, relay side), all of which may sit in one worker. +# Keep worker_connections at least 4 x LIMITS__MAX_CONNECTIONS (10000 by +# default) plus headroom, and raise both values with it. +worker_rlimit_nofile 65536; + +error_log /var/log/nginx/error.log notice; +pid /run/nginx.pid; + +events { + worker_connections 49152; +} + +stream { + include /etc/nginx/stream-conf.d/*.conf; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Defaults for any server block: no access log (the stock `main` format + # writes the full request line, connection token included) and an error + # log that withholds request lines (see default.conf.template). + access_log off; + error_log /dev/stderr emerg; + + sendfile on; + keepalive_timeout 65; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/deployment/configs/nginx/nginx.conf b/deployment/configs/nginx/nginx.conf new file mode 100644 index 0000000..43c5f29 --- /dev/null +++ b/deployment/configs/nginx/nginx.conf @@ -0,0 +1,36 @@ +# Main nginx configuration for docker-compose.yml (a TLS terminator or load +# balancer in front of this stack). It replaces the image's stock nginx.conf to +# size the worker connection pool for the relay and to keep the stock access +# log, which records the query string, off. docker-compose.direct.yml mounts +# configs/nginx/direct/nginx.conf instead. +user nginx; +worker_processes auto; + +# A proxied WebSocket holds two pool entries (client side and relay side), both +# of which may sit in one worker. Keep worker_connections at least +# 2 x LIMITS__MAX_CONNECTIONS (10000 by default) plus headroom, and raise both +# values with it. +worker_rlimit_nofile 32768; + +error_log /var/log/nginx/error.log notice; +pid /run/nginx.pid; + +events { + worker_connections 24576; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Defaults for any server block: no access log (the stock `main` format + # writes the full request line, connection token included) and an error + # log that withholds request lines (see default.conf.template). + access_log off; + error_log /dev/stderr emerg; + + sendfile on; + keepalive_timeout 65; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/deployment/docker-compose.direct.yml b/deployment/docker-compose.direct.yml new file mode 100644 index 0000000..3386f77 --- /dev/null +++ b/deployment/docker-compose.direct.yml @@ -0,0 +1,27 @@ +# Override for when nothing sits in front of this stack (clients reach the +# published port directly). Adds an nginx stream{} front on the published port +# that caps connections per client address at accept, before nginx reads any +# HTTP byte: +# +# docker compose -f docker-compose.yml -f docker-compose.direct.yml \ +# --env-file ./configs/compose.env up --build -d +# +# Do not use it behind a load balancer, CDN or other proxy: every client would +# share the proxy's address. That front must buffer complete request headers +# and enforce its own per-client connection limit instead (see README.md). +services: + nginx: + # The shipped direct variant speaks plain HTTP. Expose it on loopback for + # local checks; a public bind requires TLS configured in this nginx first. + ports: !override + - '127.0.0.1:${RELAY_PUBLIC_PORT}:80' + # Each entry replaces the base file's mount with the same target. + volumes: + - ./configs/nginx/direct/nginx.conf:/etc/nginx/nginx.conf:ro + - ./configs/nginx/direct/default.conf.template:/etc/nginx/templates/default.conf.template:ro + - ./configs/nginx/direct/edge.conf.stream-template:/etc/nginx/templates/edge.conf.stream-template:ro + # Holds the unix socket between the front and the http server. nginx does + # not remove a leftover socket file before binding, so a socket left by an + # unclean stop would make the next start fail; a tmpfs starts empty. + tmpfs: + - /run/nginx-edge diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 036a1ba..b31f284 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -31,6 +31,9 @@ services: PER_IP_BURST: ${PER_IP_BURST:-20} TRUSTED_PROXY_CIDR: ${TRUSTED_PROXY_CIDR:?set TRUSTED_PROXY_CIDR to the trusted proxy range} volumes: + # The main config sizes nginx's worker connection pool for the relay + # and keeps the stock access log, which records the query, off. + - ./configs/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - ./configs/nginx/default.conf.template:/etc/nginx/templates/default.conf.template:ro ports: - '${RELAY_PUBLIC_PORT}:80'