Merge pull request #327 from cipherstash/refactor-frontend-context

Refactor frontend context

Author: tobyhede

.tool-versions

@@ -32,6 +32,8 @@
 
 - Configuration errors:
   - [Missing or invalid TLS configuration](#config-missing-or-invalid-tls)
+  - [Network configuration change requires restart](#config-network-change-requires-restart)
+
 <!-- ---------------------------------------------------------------------------------------------------- -->
 
 <!-- ---------------------------------------------------------------------------------------------------- -->
@@ -651,3 +653,35 @@ Check that the certificate and private key are valid.
 
 
 <!-- ---------------------------------------------------------------------------------------------------- -->
+
+
+## Network configuration change requires restart <a id='config-network-change-requires-restart'></a>
+
+A configuration reload was attempted with network-level changes that require a full restart.
+
+### Error message
+
+```
+Network configuration change requires restart
+```
+
+### Notes
+
+When receiving a SIGHUP signal, CipherStash Proxy attempts to reload application-level configuration without disrupting active connections. However, certain network-related configuration changes require stopping and restarting the proxy service to take effect.
+
+The following settings require a restart when changed:
+- `server.host` - The host address the proxy listens on
+- `server.port` - The port the proxy listens on
+- `server.require_tls` - TLS requirement setting
+- `server.worker_threads` - Number of worker threads
+- `tls` - Any TLS certificate or key configuration
+
+### How to fix
+
+1. Stop the CipherStash Proxy service
+2. Update the configuration as needed
+3. Restart the CipherStash Proxy service
+
+Application-level configuration changes (database, auth, encrypt, log, prometheus, development) can be reloaded without restart using SIGHUP.
+
+<!-- ---------------------------------------------------------------------------------------------------- -->

Cargo.lock

@@ -311,6 +311,8 @@ echo
 mise --env tcp run postgres:setup
 mise --env tls run postgres:setup
 
+mise run test:integration:showcase
+
 echo
 echo '###############################################'
 echo '# Test: Prometheus'
@@ -379,10 +381,8 @@ echo '###############################################'
 echo '# Test: Showcase'
 echo '###############################################'
 echo
-mise --env tls run proxy:up proxy-tls --extra-args "--detach --wait"
-mise --env tls run test:wait_for_postgres_to_quack --port 6432 --max-retries 20 --tls
-RUST_BACKTRACE=full cargo run -p showcase
-mise --env tls run proxy:down
+
+mise run test:integration:showcase
 
 echo
 echo '###############################################'
@@ -646,6 +646,15 @@ else
 fi
 """
 
+[tasks."test:integration:showcase"]
+description = "Run Showcase integration test"
+run = """
+mise --env tls run proxy:up proxy-tls --extra-args "--detach --wait"
+mise --env tls run test:wait_for_postgres_to_quack --port 6432 --max-retries 20 --tls
+RUST_BACKTRACE=full cargo run -p showcase
+mise --env tls run proxy:down
+"""
+
 [tasks.release]
 description = "Publish release artifacts"
 depends = ["release:docker"]

docs/errors.md

@@ -4,6 +4,7 @@ version = "2.0.0"
 edition = "2021"
 
 [dependencies]
+async-trait = "0.1"
 aws-lc-rs = "1.13.3"
 bigdecimal = { version = "0.4.6", features = ["serde-json"] }
 arc-swap = "1.7.1"

mise.toml

@@ -87,6 +87,21 @@ impl DatabaseConfig {
         })?;
         Ok(name)
     }
+
+    #[cfg(test)]
+    pub fn for_testing() -> Self {
+        Self {
+            host: Self::default_host(),
+            port: Self::default_port(),
+            name: "test".to_string(),
+            username: "test".to_string(),
+            password: Protected::new("test".to_string()),
+            connection_timeout: None,
+            with_tls_verification: false,
+            config_reload_interval: Self::default_config_reload_interval(),
+            schema_reload_interval: Self::default_schema_reload_interval(),
+        }
+    }
 }
 
 ///

packages/cipherstash-proxy/Cargo.toml

@@ -302,6 +302,29 @@ impl TandemConfig {
 
         DEFAULT_THREAD_STACK_SIZE
     }
+
+    #[cfg(test)]
+    pub fn for_testing() -> Self {
+        Self {
+            server: ServerConfig::default(),
+            database: DatabaseConfig::for_testing(),
+            auth: AuthConfig {
+                workspace_crn: "crn:ap-southeast-2.aws:IJGECSCWKREECNBS".parse().unwrap(),
+                client_access_key: "test".to_string(),
+            },
+            encrypt: EncryptConfig {
+                client_id: "test".to_string(),
+                client_key: "test".to_string(),
+                default_keyset_id: Some(
+                    Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(),
+                ),
+            },
+            tls: None,
+            log: LogConfig::default(),
+            prometheus: PrometheusConfig::default(),
+            development: None,
+        }
+    }
 }
 
 impl PrometheusConfig {

packages/cipherstash-proxy/src/config/database.rs

@@ -10,7 +10,7 @@ use crate::{error::TlsConfigError, log::CONFIG};
 /// Server TLS Configuration
 /// This is listener/inbound connection config
 ///
-#[derive(Clone, Debug, Deserialize)]
+#[derive(Clone, Debug, Deserialize, PartialEq)]
 #[serde(untagged)]
 pub enum TlsConfig {
     Pem {

packages/cipherstash-proxy/src/config/tandem.rs

@@ -176,6 +176,9 @@ pub enum ConfigError {
     #[error("Expected an Encrypt configuration table")]
     MissingEncryptConfigTable,
 
+    #[error("Network configuration change requires restart For help visit {}#config-network-change-requires-restart", ERROR_DOC_BASE_URL)]
+    NetworkConfigurationChangeRequiresRestart,
+
     #[error(transparent)]
     Parse(#[from] serde_json::Error),
 

packages/cipherstash-proxy/src/config/tls.rs

@@ -1,12 +1,11 @@
 use cipherstash_proxy::config::TandemConfig;
 use cipherstash_proxy::connect::{self, AsyncStream};
-use cipherstash_proxy::error::Error;
+use cipherstash_proxy::error::{ConfigError, Error};
 use cipherstash_proxy::prometheus::CLIENTS_ACTIVE_CONNECTIONS;
 use cipherstash_proxy::proxy::Proxy;
 use cipherstash_proxy::{cli, log, postgresql as pg, prometheus, tls, Args};
 use clap::Parser;
 use metrics::gauge;
-use tokio::net::TcpListener;
 use tokio::signal::unix::{signal, SignalKind};
 use tokio_util::task::TaskTracker;
 use tracing::{error, info, warn};
@@ -55,7 +54,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
         let mut proxy = init(config).await;
 
-        let mut listener = connect::bind_with_retry(&proxy.config.server).await;
+        let listener = connect::bind_with_retry(&proxy.config.server).await;
         let tracker = TaskTracker::new();
 
         let mut client_id = 0;
@@ -81,26 +80,24 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 break;
             },
             _ = sighup() => {
-                info!(msg = "Received SIGHUP. Reloading configuration");
-                (listener, proxy) = reload_config(listener, &args, proxy).await;
-                info!(msg = "Reloaded configuration");
+                info!(msg = "Received SIGHUP. Reloading application configuration");
+                proxy = reload_application_config(&proxy.config, &args).await.unwrap_or(proxy);
             },
             _ = sigterm() => {
                 info!(msg = "Received SIGTERM");
                 break;
             },
             Ok(client_stream) = AsyncStream::accept(&listener) => {
 
-                    let proxy = proxy.clone();
-
                     client_id += 1;
 
+                    let context = proxy.context(client_id);
+
                     tracker.spawn(async move {
-                        let proxy = proxy.clone();
 
                         gauge!(CLIENTS_ACTIVE_CONNECTIONS).increment(1);
 
-                        match pg::handler(client_stream, proxy, client_id).await {
+                        match pg::handler(client_stream,context).await {
                             Ok(_) => (),
                             Err(err) => {
 
@@ -261,25 +258,35 @@ async fn sighup() -> std::io::Result<()> {
     Ok(())
 }
 
-async fn reload_config(listener: TcpListener, args: &Args, proxy: Proxy) -> (TcpListener, Proxy) {
+fn has_network_config_changed(current: &TandemConfig, new: &TandemConfig) -> bool {
+    current.server.host != new.server.host
+        || current.server.port != new.server.port
+        || current.server.require_tls != new.server.require_tls
+        || current.server.worker_threads != new.server.worker_threads
+        || current.tls != new.tls
+}
+
+async fn reload_application_config(config: &TandemConfig, args: &Args) -> Result<Proxy, Error> {
     let new_config = match TandemConfig::load(args) {
         Ok(config) => config,
         Err(err) => {
             warn!(
                 msg = "Configuration could not be reloaded: {}",
                 error = err.to_string()
             );
-            return (listener, proxy);
+            return Err(err);
         }
     };
 
-    let new_proxy = init(new_config).await;
+    // Check for network config changes that require restart
+    if has_network_config_changed(config, &new_config) {
+        let err = ConfigError::NetworkConfigurationChangeRequiresRestart;
+        warn!(msg = err.to_string());
 
-    // Explicit drop needed here to free the network resources before binding if using the same address & port
-    std::mem::drop(listener);
+        return Err(err.into());
+    }
 
-    (
-        connect::bind_with_retry(&new_proxy.config.server).await,
-        new_proxy,
-    )
+    info!(msg = "Configuration reloaded");
+    let proxy = init(new_config).await;
+    Ok(proxy)
 }