perf(gateway): reverse proxy throughput optimizations

.github/workflows/gateway-proxy-tests.yml

@@ -0,0 +1,81 @@
+# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Gateway proxy tests
+
+# The gateway's proxy data path has two opt-in optimisations (`tcp_splice`,
+# `ktls`) whose behaviour depends on kernel capabilities and on a per-connection
+# gate. Unit tests cover the relay functions; this runs a real gateway process
+# and asserts on what actually reaches the wire.
+on:
+  push:
+    branches: [ master, next, dev-* ]
+    paths:
+      - 'dstack/gateway/**'
+      - 'dstack/vendor/ktls/**'
+      - '.github/workflows/gateway-proxy-tests.yml'
+  pull_request:
+    branches: [ master, next, dev-* ]
+    paths:
+      - 'dstack/gateway/**'
+      - 'dstack/vendor/ktls/**'
+      - '.github/workflows/gateway-proxy-tests.yml'
+
+permissions:
+  contents: read
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  proxy-integration:
+    runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }}
+    # Each of the ~25 arms restarts the gateway, and the idle-timeout arms wait
+    # out a real timeout, so this is minutes rather than seconds.
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@1.92.0
+
+      - name: Cache cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            dstack/target
+          key: gateway-proxy-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }}
+          restore-keys: gateway-proxy-${{ runner.os }}-
+
+      - name: Build the gateway
+        working-directory: dstack
+        run: cargo build --release -p dstack-gateway
+
+      - name: Record kernel capabilities
+        # The suite adapts to what the kernel offers, so the log needs to say
+        # what it had: a run that skipped kTLS looks the same as one that
+        # covered it otherwise.
+        run: |
+          echo "kernel: $(uname -r)"
+          sudo modprobe tls 2>&1 || echo "no TLS ULP available"
+          echo "tls module loaded: $(lsmod | grep -c '^tls ' || true)"
+          grep -B2 -A3 'gcm(aes)' /proc/crypto | grep -E '^(driver|priority)' \
+            | paste - - | sort -u || true
+
+      - name: Proxy integration tests
+        working-directory: dstack/gateway/test-run
+        env:
+          GATEWAY_BIN: ${{ github.workspace }}/dstack/target/release/dstack-gateway
+        run: ./test_proxy.sh
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-proxy-test-logs
+          path: /tmp/dstack-gw-proxy-test.*/logs/
+          if-no-files-found: ignore
+          retention-days: 7

REUSE.toml

@@ -181,6 +181,13 @@ SPDX-FileCopyrightText = "NONE"
 SPDX-License-Identifier = "Apache-2.0"
 precedence = "override"
 
+# ktls 6.0.2 plus rustls/ktls#70; see dstack/vendor/README.md.
+[[annotations]]
+path = "dstack/vendor/ktls/**"
+SPDX-FileCopyrightText = "Copyright (c) 2022 Amos Wenger <amos@bearcove.net>"
+SPDX-License-Identifier = "MIT OR Apache-2.0"
+precedence = "override"
+
 # Generated files
 
 [[annotations]]

dstack/Cargo.toml

@@ -71,6 +71,11 @@ members = [
     "crates/build-info",
     "crates/mock-attestation",
 ]
+# Vendored third-party crates are path dependencies but deliberately not members:
+# `--all-features` applies to members, and ktls declares its `ring` and
+# `aws_lc_rs` features mutually exclusive, so membership makes
+# `cargo test --all-features` fail to compile.
+exclude = ["vendor/ktls"]
 resolver = "2"
 
 [workspace.dependencies]
@@ -141,6 +146,9 @@ tracing-subscriber = { version = "0.3.20", features = ["env-filter"] }
 safe-write = "0.1.2"
 rustix = { version = "0.38", features = ["fs"] }
 nix = "0.29.0"
+# Vendored: 6.0.2 does not build for musl. See vendor/README.md.
+ktls = { path = "vendor/ktls" }
+socket2 = { version = "0.5", features = ["all"] }
 sd-notify = "0.4.5"
 listenfd = "1.0"
 jemallocator = "0.5.4"

dstack/gateway/Cargo.toml

@@ -67,7 +67,10 @@ dstack-api-auth.workspace = true
 cached-cell.workspace = true
 
 [target.'cfg(unix)'.dependencies]
-nix = { workspace = true, features = ["resource"] }
+nix = { workspace = true, features = ["resource", "fs", "socket", "zerocopy"] }
+ktls.workspace = true
+libc.workspace = true
+socket2.workspace = true
 
 [[bin]]
 name = "gen_debug_key"
@@ -76,3 +79,6 @@ path = "src/gen_debug_key.rs"
 [dev-dependencies]
 insta.workspace = true
 tempfile.workspace = true
+# `test-util` gives the idle-watchdog tests a paused clock, so they assert on
+# the window without waiting for it in wall-clock time.
+tokio = { workspace = true, features = ["test-util"] }

dstack/gateway/gateway.toml

@@ -77,18 +77,44 @@ tls_versions = ["1.2"]
 listen_addr = "0.0.0.0"
 listen_port = 8443
 agent_port = 8090
-buffer_size = 8192
+buffer_size = 65536
 # number of hosts to try to connect to
 connect_top_n = 3
 localhost_enabled = false
 app_address_ns_prefix = "_dstack-app-address"
 app_address_ns_compat = true
 workers = 32
+# One runtime + SO_REUSEPORT listener per worker (thread-per-core). Linux only.
+thread_per_core = true
+# Hand new connections to a less loaded core when SO_REUSEPORT skews them.
+connection_rebalance = true
 external_port = 443
 # Maximum concurrent connections per app. 0 means unlimited.
 max_connections_per_app = 2000
 # Whether to read PROXY protocol from inbound connections (e.g. from Cloudflare).
 inbound_pp_enabled = false
+# splice(2) zero-copy relaying for the TLS-passthrough path (Linux only), and
+# kernel TLS offload for the terminate path, are both configured by an optional
+# section. Omitting the section disables the optimisation; an empty section
+# engages it from the first byte; `after_bytes` / `after_duration` are two
+# independent gates and whichever fires first engages it.
+#
+# [core.proxy.tcp_splice]
+# # Bulk transfers trip this within milliseconds.
+# after_bytes = 65536
+# # Long-lived low-rate streams (e.g. LLM token streaming) never accumulate
+# # bytes fast enough for the gate above, so they need this one instead.
+# after_duration = "5s"
+# # Park a relay's pipe in the pool while it waits for the next chunk. A spliced
+# # connection otherwise pins 4 descriptors for its whole lifetime, so bursty
+# # traffic (a token every 25 ms) holds them >99% idle.
+# release_idle_pipes = true
+#
+# kTLS hands session keys to the kernel; see config.rs for the security note.
+# Gated offload also requires [core.proxy.tcp_splice].
+#
+# [core.proxy.ktls]
+# after_bytes = 65536
 
 [core.proxy.port_policy_fetch]
 # Background lazy-fetch of port_policy from legacy CVM agents.

dstack/gateway/rpc/proto/gateway_rpc.proto

@@ -104,6 +104,28 @@ message StatusResponse {
   repeated GatewayNodeInfo nodes = 6;
   // Peer uuid
   bytes uuid = 7;
+  // What the data path is actually doing with kTLS and splice.
+  ProxyAccelStatus accel = 8;
+}
+
+// Effective state of the data-path acceleration options.
+//
+// The configured value does not say what is running: the startup probe can turn
+// kTLS off on a kernel without the TLS ULP, and both options engage per
+// connection only once their gate fires. These are reported so an operator can
+// tell the difference without reading the startup log.
+message ProxyAccelStatus {
+  // Effective kTLS mode: "off", "immediate", "after 64 KiB or 5s", or
+  // "disabled (kernel has no TLS ULP)" when the startup probe cleared it.
+  string ktls_mode = 1;
+  // Effective splice mode, same encoding minus the probe case.
+  string splice_mode = 2;
+  // Connections handed to the kernel's TLS ULP since start.
+  uint64 ktls_offloaded = 3;
+  // Connections whose handover to the kernel failed.
+  uint64 ktls_offload_failed = 4;
+  // Connections that entered a zero-copy splice relay, on either path.
+  uint64 splice_engaged = 5;
 }
 
 // HostInfo is the information of a host.

dstack/gateway/src/admin_service.rs

@@ -31,7 +31,7 @@ use crate::{
     kv::{DnsCredential, DnsProvider, NodeStatus, PortFlags, PortPolicy, ZtDomainConfig},
     main_service::Proxy,
     models::PortPolicyView,
-    proxy::NUM_CONNECTIONS,
+    proxy::{stats::accel_status, NUM_CONNECTIONS},
 };
 
 pub struct AdminRpcHandler {
@@ -74,6 +74,9 @@ impl AdminRpcHandler {
             nodes: state.get_all_nodes(),
             hosts,
             num_connections: NUM_CONNECTIONS.load(Ordering::Relaxed),
+            // Reads the post-probe config, so this is what the data path is
+            // running rather than what the file asked for.
+            accel: Some(accel_status(&state.config.proxy)),
         })
     }
 }