feat: automatic mTLS certificate regeneration and retry mechanism

This adds support for automatically recovering from SSL handshake errors when certificates expired.
When an SSL error occurs, the plugin will now attempt to execute a configured external command
to refresh certificates. If successful, the SSL context is reloaded and the failed request is
transparently retried. This improves reliability in environments with short-lived or frequently
rotating certificates.

Netflix requested this, they don't have a reliable mechanism to detect and refresh the certificates
before any major disruption in Coder Toolbox.

Author: fioan89

src/main/kotlin/com/coder/toolbox/CoderRemoteProvider.kt

@@ -414,14 +414,14 @@ class CoderRemoteProvider(
      * Auto-login only on first the firs run if there is a url & token configured or the auth
      * should be done via certificates.
      */
-    private fun shouldDoAutoSetup(): Boolean = firstRun && (canAutoLogin() || !settings.requireTokenAuth)
+    private fun shouldDoAutoSetup(): Boolean = firstRun && (canAutoLogin() || !settings.requiresTokenAuth)
 
     fun canAutoLogin(): Boolean = !context.secrets.tokenFor(context.deploymentUrl).isNullOrBlank()
 
     private fun onConnect(client: CoderRestClient, cli: CoderCLIManager) {
         // Store the URL and token for use next time.
         context.settingsStore.updateLastUsedUrl(client.url)
-        if (context.settingsStore.requireTokenAuth) {
+        if (context.settingsStore.requiresTokenAuth) {
             context.secrets.storeTokenFor(client.url, client.token ?: "")
             context.logger.info("Deployment URL and token were stored and will be available for automatic connection")
         } else {

src/main/kotlin/com/coder/toolbox/cli/CoderCLIManager.kt

@@ -19,6 +19,7 @@ import com.coder.toolbox.sdk.v2.models.Workspace
 import com.coder.toolbox.sdk.v2.models.WorkspaceAgent
 import com.coder.toolbox.settings.SignatureFallbackStrategy.ALLOW
 import com.coder.toolbox.util.InvalidVersionException
+import com.coder.toolbox.util.ReloadableTlsContext
 import com.coder.toolbox.util.SemVer
 import com.coder.toolbox.util.escape
 import com.coder.toolbox.util.escapeSubcommand
@@ -153,7 +154,8 @@ class CoderCLIManager(
         }
         val okHttpClient = CoderHttpClientBuilder.build(
             context,
-            interceptors
+            interceptors,
+            ReloadableTlsContext(context.settingsStore.readOnly().tls)
         )
 
         val retrofit = Retrofit.Builder()

src/main/kotlin/com/coder/toolbox/sdk/CoderHttpClientBuilder.kt

@@ -2,24 +2,19 @@ package com.coder.toolbox.sdk
 
 import com.coder.toolbox.CoderToolboxContext
 import com.coder.toolbox.util.CoderHostnameVerifier
-import com.coder.toolbox.util.coderSocketFactory
-import com.coder.toolbox.util.coderTrustManagers
+import com.coder.toolbox.util.ReloadableTlsContext
 import com.jetbrains.toolbox.api.remoteDev.connection.ProxyAuth
 import okhttp3.Credentials
 import okhttp3.Interceptor
 import okhttp3.OkHttpClient
-import javax.net.ssl.X509TrustManager
 
 object CoderHttpClientBuilder {
     fun build(
         context: CoderToolboxContext,
-        interceptors: List<Interceptor>
+        interceptors: List<Interceptor>,
+        tlsContext: ReloadableTlsContext
     ): OkHttpClient {
-        val settings = context.settingsStore.readOnly()
-
-        val socketFactory = coderSocketFactory(settings.tls)
-        val trustManagers = coderTrustManagers(settings.tls.caPath)
-        var builder = OkHttpClient.Builder()
+        val builder = OkHttpClient.Builder()
 
         context.proxySettings.getProxy()?.let { proxy ->
             context.logger.info("proxy: $proxy")
@@ -43,8 +38,8 @@ object CoderHttpClientBuilder {
                 .build()
         }
 
-        builder.sslSocketFactory(socketFactory, trustManagers[0] as X509TrustManager)
-            .hostnameVerifier(CoderHostnameVerifier(settings.tls.altHostname))
+        builder.sslSocketFactory(tlsContext.sslSocketFactory, tlsContext.trustManager)
+            .hostnameVerifier(CoderHostnameVerifier(context.settingsStore.tls.altHostname))
             .retryOnConnectionFailure(true)
 
         interceptors.forEach { interceptor ->

src/main/kotlin/com/coder/toolbox/sdk/CoderRestClient.kt

@@ -7,6 +7,7 @@ import com.coder.toolbox.sdk.convertors.LoggingConverterFactory
 import com.coder.toolbox.sdk.convertors.OSConverter
 import com.coder.toolbox.sdk.convertors.UUIDConverter
 import com.coder.toolbox.sdk.ex.APIResponseException
+import com.coder.toolbox.sdk.interceptors.CertificateRefreshInterceptor
 import com.coder.toolbox.sdk.interceptors.Interceptors
 import com.coder.toolbox.sdk.v2.CoderV2RestFacade
 import com.coder.toolbox.sdk.v2.models.ApiErrorResponse
@@ -20,6 +21,7 @@ import com.coder.toolbox.sdk.v2.models.WorkspaceBuild
 import com.coder.toolbox.sdk.v2.models.WorkspaceBuildReason
 import com.coder.toolbox.sdk.v2.models.WorkspaceResource
 import com.coder.toolbox.sdk.v2.models.WorkspaceTransition
+import com.coder.toolbox.util.ReloadableTlsContext
 import com.squareup.moshi.Moshi
 import okhttp3.OkHttpClient
 import retrofit2.Response
@@ -40,6 +42,7 @@ open class CoderRestClient(
     val token: String?,
     private val pluginVersion: String = "development",
 ) {
+    private lateinit var tlsContext: ReloadableTlsContext
     private lateinit var moshi: Moshi
     private lateinit var httpClient: OkHttpClient
     private lateinit var retroRestClient: CoderV2RestFacade
@@ -60,12 +63,17 @@ open class CoderRestClient(
                 .add(OSConverter())
                 .add(UUIDConverter())
                 .build()
+
+        tlsContext = ReloadableTlsContext(context.settingsStore.readOnly().tls)
+
         val interceptors = buildList {
-            if (context.settingsStore.requireTokenAuth) {
+            if (context.settingsStore.requiresTokenAuth) {
                 if (token.isNullOrBlank()) {
                     throw IllegalStateException("Token is required for $url deployment")
                 }
                 add(Interceptors.tokenAuth(token))
+            } else if (context.settingsStore.requiresMTlsAuth && context.settingsStore.tls.certRefreshCommand?.isNotBlank() == true) {
+                add(CertificateRefreshInterceptor(context, tlsContext))
             }
             add((Interceptors.userAgent(pluginVersion)))
             add(Interceptors.externalHeaders(context, url))
@@ -74,7 +82,8 @@ open class CoderRestClient(
 
         httpClient = CoderHttpClientBuilder.build(
             context,
-            interceptors
+            interceptors,
+            tlsContext
         )
 
         retroRestClient =

src/main/kotlin/com/coder/toolbox/sdk/interceptors/CertificateRefreshInterceptor.kt

@@ -0,0 +1,53 @@
+package com.coder.toolbox.sdk.interceptors
+
+import com.coder.toolbox.CoderToolboxContext
+import com.coder.toolbox.util.ReloadableTlsContext
+import okhttp3.Interceptor
+import okhttp3.Response
+import org.zeroturnaround.exec.ProcessExecutor
+import javax.net.ssl.SSLHandshakeException
+import javax.net.ssl.SSLPeerUnverifiedException
+
+class CertificateRefreshInterceptor(
+    private val context: CoderToolboxContext,
+    private val tlsContext: ReloadableTlsContext
+) : Interceptor {
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val request = chain.request()
+        try {
+            return chain.proceed(request)
+        } catch (e: Exception) {
+            if ((e is SSLHandshakeException || e is SSLPeerUnverifiedException) && (e.message?.contains("certificate_expired") == true)) {
+                val command = context.settingsStore.tls.certRefreshCommand
+                if (command.isNullOrBlank()) {
+                    throw IllegalStateException(
+                        "Certificate expiration interceptor was set but the refresh command was removed in the meantime",
+                        e
+                    )
+                }
+
+                context.logger.info("SSL handshake exception encountered: certificates expired. Running certificate refresh command: $command")
+                try {
+                    val result = ProcessExecutor()
+                        .command(command.split(" ").toList())
+                        .exitValueNormal()
+                        .readOutput(true)
+                        .execute()
+                    context.logger.info("`$command`: ${result.outputUTF8()}")
+
+                    if (result.exitValue == 0) {
+                        context.logger.info("Certificate refresh command executed successfully. Reloading SSL certificates.")
+                        tlsContext.reload()
+                        // Retry the request
+                        return chain.proceed(request)
+                    } else {
+                        context.logger.error("Certificate refresh command failed with exit code ${result.exitValue}")
+                    }
+                } catch (ex: Exception) {
+                    context.logger.error(ex, "Failed to execute certificate refresh command")
+                }
+            }
+            throw e
+        }
+    }
+}

src/main/kotlin/com/coder/toolbox/settings/ReadOnlyCoderSettings.kt

@@ -114,7 +114,12 @@ interface ReadOnlyCoderSettings {
     /**
      * Whether login should be done with a token
      */
-    val requireTokenAuth: Boolean
+    val requiresTokenAuth: Boolean
+
+    /**
+     * Whether the authentication is done with certificates.
+     */
+    val requiresMTlsAuth: Boolean
 
     /**
      * Whether to add --disable-autostart to the proxy command.  This works
@@ -216,6 +221,12 @@ interface ReadOnlyTLSSettings {
      * Coder service does not match the hostname in the TLS certificate.
      */
     val altHostname: String?
+
+    /**
+     * Command to run when certificates expire and SSLHandshakeException
+     * is raised with `Received fatal alert: certificate_expired` as message
+     */
+    val certRefreshCommand: String?
 }
 
 enum class SignatureFallbackStrategy {

src/main/kotlin/com/coder/toolbox/store/CoderSettingsStore.kt

@@ -32,7 +32,8 @@ class CoderSettingsStore(
         override val certPath: String?,
         override val keyPath: String?,
         override val caPath: String?,
-        override val altHostname: String?
+        override val altHostname: String?,
+        override val certRefreshCommand: String?
     ) : ReadOnlyTLSSettings
 
     // Properties implementation
@@ -62,9 +63,11 @@ class CoderSettingsStore(
             certPath = store[TLS_CERT_PATH],
             keyPath = store[TLS_KEY_PATH],
             caPath = store[TLS_CA_PATH],
-            altHostname = store[TLS_ALTERNATE_HOSTNAME]
+            altHostname = store[TLS_ALTERNATE_HOSTNAME],
+            certRefreshCommand = store[TLS_CERT_REFRESH_COMMAND]
         )
-    override val requireTokenAuth: Boolean get() = tls.certPath.isNullOrBlank() || tls.keyPath.isNullOrBlank()
+    override val requiresTokenAuth: Boolean get() = tls.certPath.isNullOrBlank() || tls.keyPath.isNullOrBlank()
+    override val requiresMTlsAuth: Boolean get() = tls.certPath?.isNotBlank() == true && tls.keyPath?.isNotBlank() == true
     override val disableAutostart: Boolean
         get() = store[DISABLE_AUTOSTART]?.toBooleanStrictOrNull() ?: (getOS() == OS.MAC)
     override val isSshWildcardConfigEnabled: Boolean

src/main/kotlin/com/coder/toolbox/store/StoreKeys.kt

@@ -36,6 +36,8 @@ internal const val TLS_CA_PATH = "tlsCAPath"
 
 internal const val TLS_ALTERNATE_HOSTNAME = "tlsAlternateHostname"
 
+internal const val TLS_CERT_REFRESH_COMMAND = "tlsCertRefreshCommand"
+
 internal const val DISABLE_AUTOSTART = "disableAutostart"
 
 internal const val ENABLE_SSH_WILDCARD_CONFIG = "enableSshWildcardConfig"

src/main/kotlin/com/coder/toolbox/util/CoderProtocolHandler.kt

@@ -70,7 +70,7 @@ open class CoderProtocolHandler(
 
         context.logger.info("Handling $uri...")
         val deploymentURL = resolveDeploymentUrl(params) ?: return
-        val token = if (!context.settingsStore.requireTokenAuth) null else resolveToken(params) ?: return
+        val token = if (!context.settingsStore.requiresTokenAuth) null else resolveToken(params) ?: return
         val workspaceName = resolveWorkspaceName(params) ?: return
 
         suspend fun onConnect(

src/main/kotlin/com/coder/toolbox/util/TLS.kt

@@ -280,3 +280,77 @@ class MergedSystemTrustManger(private val otherTrustManager: X509TrustManager) :
     override fun getAcceptedIssuers(): Array<X509Certificate> =
         otherTrustManager.acceptedIssuers + systemTrustManager.acceptedIssuers
 }
+
+class ReloadableX509TrustManager(
+    private val caPath: String?,
+) : X509TrustManager {
+    @Volatile
+    private var delegate: X509TrustManager = loadTrustManager()
+
+    private fun loadTrustManager(): X509TrustManager {
+        val trustManagers = coderTrustManagers(caPath)
+        return trustManagers.first { it is X509TrustManager } as X509TrustManager
+    }
+
+    fun reload() {
+        delegate = loadTrustManager()
+    }
+
+    override fun checkClientTrusted(chain: Array<out X509Certificate>?, authType: String?) {
+        delegate.checkClientTrusted(chain, authType)
+    }
+
+    override fun checkServerTrusted(chain: Array<out X509Certificate>?, authType: String?) {
+        delegate.checkServerTrusted(chain, authType)
+    }
+
+    override fun getAcceptedIssuers(): Array<X509Certificate> {
+        return delegate.acceptedIssuers
+    }
+}
+
+class ReloadableSSLSocketFactory(
+    private val settings: ReadOnlyTLSSettings,
+) : SSLSocketFactory() {
+    @Volatile
+    private var delegate: SSLSocketFactory = loadSocketFactory()
+
+    private fun loadSocketFactory(): SSLSocketFactory {
+        return coderSocketFactory(settings)
+    }
+
+    fun reload() {
+        delegate = loadSocketFactory()
+    }
+
+    override fun getDefaultCipherSuites(): Array<String> = delegate.defaultCipherSuites
+
+    override fun getSupportedCipherSuites(): Array<String> = delegate.supportedCipherSuites
+
+    override fun createSocket(): Socket = delegate.createSocket()
+
+    override fun createSocket(s: Socket?, host: String?, port: Int, autoClose: Boolean): Socket =
+        delegate.createSocket(s, host, port, autoClose)
+
+    override fun createSocket(host: String?, port: Int): Socket = delegate.createSocket(host, port)
+
+    override fun createSocket(host: String?, port: Int, localHost: InetAddress?, localPort: Int): Socket =
+        delegate.createSocket(host, port, localHost, localPort)
+
+    override fun createSocket(host: InetAddress?, port: Int): Socket = delegate.createSocket(host, port)
+
+    override fun createSocket(address: InetAddress?, port: Int, localAddress: InetAddress?, localPort: Int): Socket =
+        delegate.createSocket(address, port, localAddress, localPort)
+}
+
+class ReloadableTlsContext(
+    settings: ReadOnlyTLSSettings
+) {
+    val sslSocketFactory = ReloadableSSLSocketFactory(settings)
+    val trustManager = ReloadableX509TrustManager(settings.caPath)
+
+    fun reload() {
+        sslSocketFactory.reload()
+        trustManager.reload()
+    }
+}