From bc896b19fe0ca4de4ac8eed1f50946afcbb62bf4 Mon Sep 17 00:00:00 2001 From: rotilho Date: Sun, 8 Nov 2026 19:58:27 +0100 Subject: [PATCH 1/5] Replace peer Ktor transport with Reactor Netty --- NETWORK.md | 38 +- README.md | 4 +- build.gradle.kts | 6 + .../cash/atto/node/JvmAddressResolverGroup.kt | 61 ++ .../atto/node/network/HandshakeCallback.kt | 71 +- .../node/network/HandshakePayloadLimit.kt | 62 +- .../atto/node/network/NetworkProcessor.kt | 486 +++++++------- .../node/network/NodeConnectionManager.kt | 21 +- .../cash/atto/node/network/PeerNettyClient.kt | 224 +++++++ .../cash/atto/node/network/PeerNettyServer.kt | 135 ++++ .../cash/atto/node/network/PeerSession.kt | 22 + .../atto/node/network/ReactorPeerSession.kt | 189 ++++++ .../network/HandshakeCallbackServiceTest.kt | 24 +- .../network/InboundPeerAuthenticationTest.kt | 198 ++++++ .../atto/node/network/NetworkProcessorTest.kt | 260 +++++++- .../node/network/NodeConnectionManagerTest.kt | 46 +- .../atto/node/network/PeerNettyClientTest.kt | 604 ++++++++++++++++++ .../atto/node/network/PeerNettyServerTest.kt | 223 +++++++ .../atto/node/network/PeerNettyTlsTest.kt | 313 +++++++++ .../node/network/ReactorPeerSessionTest.kt | 97 +++ 20 files changed, 2637 insertions(+), 447 deletions(-) create mode 100644 src/main/kotlin/cash/atto/node/JvmAddressResolverGroup.kt create mode 100644 src/main/kotlin/cash/atto/node/network/PeerNettyClient.kt create mode 100644 src/main/kotlin/cash/atto/node/network/PeerNettyServer.kt create mode 100644 src/main/kotlin/cash/atto/node/network/PeerSession.kt create mode 100644 src/main/kotlin/cash/atto/node/network/ReactorPeerSession.kt create mode 100644 src/test/kotlin/cash/atto/node/network/InboundPeerAuthenticationTest.kt create mode 100644 src/test/kotlin/cash/atto/node/network/PeerNettyClientTest.kt create mode 100644 src/test/kotlin/cash/atto/node/network/PeerNettyServerTest.kt create mode 100644 src/test/kotlin/cash/atto/node/network/PeerNettyTlsTest.kt create mode 100644 src/test/kotlin/cash/atto/node/network/ReactorPeerSessionTest.kt diff --git a/NETWORK.md b/NETWORK.md index 5a5474ee..6b63b5d7 100644 --- a/NETWORK.md +++ b/NETWORK.md @@ -4,9 +4,30 @@ The networking subsystem authenticates peers in [NetworkProcessor][processor], m [NodeConnectionManager][connections], and counts traffic in [Guardian][guardian]. These owners communicate through direct queries and asynchronous network-message and lifecycle events. +## Transport Ownership + +The [peer server][peer-server] uses Reactor Netty directly on `websocket.port` (default `8082`). It owns its I/O event loops +and virtual-thread application dispatcher. Spring WebFlux controllers remain on `server.port` (default `8080`), with management +on `8081` by default. + +[PeerNettyClient][peer-client] owns the outgoing Reactor Netty WebSocket and HTTP callback clients. They use private event +loops, a virtual-thread application dispatcher, separate connection providers and an asynchronous bridge to the existing JVM +DNS resolver. Neither peer transport uses Spring's shared Reactor resources. Request hostnames remain available for HTTP Host, +TLS SNI and certificate hostname verification. +Callback POSTs do not follow redirects or retry automatically; WebSocket redirects retain the existing secure-downgrade +restriction and request limit. + +[PeerSession][peer-session] exposes complete binary messages, ordered enqueue, graceful close and cancellation to the +connection manager. The [Reactor session][peer-reactor-session] owns transport framing and cleanup for inbound and outbound +connections, including enabling frame aggregation synchronously in the upgrade callback, before application dispatch. +Both directions retain a 300-byte receive limit, +including cumulative fragmented messages, and bounded incoming/outgoing queues with close-on-overflow. These transport limits +belong to the session contract. +Text messages are ignored after admission to the incoming queue; ping/pong and close frames remain transport concerns. + ## Peer Authentication -`NetworkProcessor` starts an embedded Ktor CIO server with two endpoints: +`NetworkProcessor` starts its private peer server with two endpoints: - **`POST /handshakes`** accepts a `CounterChallengeResponse`. Guardian admits the remote address, the processor consumes a one-time challenge, checks the expected public URI, genesis hash, signature and counter-challenge prefix, then signs @@ -14,11 +35,15 @@ queries and asynchronous network-message and lifecycle events. - **`WS /`** accepts a WebSocket with `Atto-Public-Uri` and `Atto-Http-Challenge` headers. Guardian admits the remote address, the processor validates the headers and challenge prefix, then posts a counter-challenge to the peer's `/handshakes` endpoint. It checks the returned URI and signature before handing the session to `NodeConnectionManager.manage()`. + Authentication owns its attempt reservation: failure or cancellation removes that attempt conditionally by identity; + success retains the existing five-second expiry window. An inbound HTTP handshake has a five-second preparation deadline covering admission, body reading, verification and signing. A separate semaphore limits concurrent HTTP handshakes to 64. The [callback service][callback] applies a five-second deadline covering URI validation, request construction/signing and the callback exchange. Both handshake payload directions have bounded body readers in [HandshakePayloadLimit][payload-limit]. Cancellation propagates to ongoing handshake work. +Early HTTP rejection closes the connection after writing its status, so unread request content cannot leave a reusable +connection stalled. WebSocket authentication runs after upgrade; a rejected attempt terminates without registering a peer. ### Outbound Connections @@ -64,7 +89,7 @@ Consumers query this owner instead of reconstructing membership from connection ### Session Lifecycle -`manage()` registers the connection and consumes binary frames from its incoming channel. It deserializes each binary +`manage()` registers the connection and consumes complete binary messages from its peer session. It deserializes each binary message through [NetworkSerializer][serializer], which also checks protocol validity. Invalid messages remove the connection. Valid messages become `InboundNetworkMessage` events with the peer URI, socket address and `WEBSOCKET` source. @@ -80,7 +105,10 @@ side effect; its connectivity check prevents a delayed disconnect from releasing Gap sessions identify peers by URI and retain their existing expiry; this is not connection-generation isolation. `clear()` removes connections and waits for already scheduled cleanup jobs. `stop()` clears the map and cancels the manager's -scope. `NetworkProcessor.stop()` separately closes its client, cancels outbound work and stops its embedded server. +scope. `NetworkProcessor.stop()` stops admission, cancels outbound work, closes the server and client transports, then clears +managed connections. Closing the transports first releases stalled writes before manager cleanup waits for disconnect jobs. +`PeerNettyClient` cancels active operations and closes sockets before disposing its connection providers, resolver work and +event loops. Transport closure is idempotent. Neither shutdown path disposes Spring's HTTP resources. ### Sending and Keepalives @@ -130,6 +158,10 @@ transport, feature, traffic-accounting or metrics listener to finish. | `NodeUnbanned` | `PeerBans` expiry review | No current consumers. | [processor]: src/main/kotlin/cash/atto/node/network/NetworkProcessor.kt +[peer-server]: src/main/kotlin/cash/atto/node/network/PeerNettyServer.kt +[peer-client]: src/main/kotlin/cash/atto/node/network/PeerNettyClient.kt +[peer-session]: src/main/kotlin/cash/atto/node/network/PeerSession.kt +[peer-reactor-session]: src/main/kotlin/cash/atto/node/network/ReactorPeerSession.kt [connections]: src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt [connected-peers]: src/main/kotlin/cash/atto/node/network/ConnectedPeers.kt [guardian]: src/main/kotlin/cash/atto/node/network/guardian/Guardian.kt diff --git a/README.md b/README.md index 96416a73..a512c93f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Atto Node is the network-facing service for the **Atto** cryptocurrency. It expo submitting/querying blocks (transactions), maintains account state in a MySQL database, connects to other nodes over WebSockets, and participates in consensus by producing and relaying votes. -This repository is a **Kotlin + Spring Boot (WebFlux)** application with a small embedded **Ktor WebSocket/HTTP server** +This repository is a **Kotlin + Spring Boot (WebFlux)** application with a dedicated **Reactor Netty WebSocket/HTTP server** for node-to-node communication. [Website](https://atto.cash/) | [Docs](https://atto.cash/docs/integration) | [Commons](https://github.com/attocash/commons) @@ -115,7 +115,7 @@ From `application.yaml`: - **8080**: public REST API (Spring WebFlux) - **8081**: management/metrics (Actuator + Prometheus) -- **8082**: node-to-node WebSocket server (Ktor) +- **8082**: node-to-node WebSocket server (Reactor Netty) Swagger UI is configured at: diff --git a/build.gradle.kts b/build.gradle.kts index 1902ba45..a37c11e2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -137,6 +137,12 @@ dependencies { tasks.withType { useJUnitPlatform() maxHeapSize = "4g" + systemProperty( + "atto.test.runtime-classpath", + sourceSets.test + .get() + .runtimeClasspath.asPath, + ) } tasks.named("check") { diff --git a/src/main/kotlin/cash/atto/node/JvmAddressResolverGroup.kt b/src/main/kotlin/cash/atto/node/JvmAddressResolverGroup.kt new file mode 100644 index 00000000..f054ff85 --- /dev/null +++ b/src/main/kotlin/cash/atto/node/JvmAddressResolverGroup.kt @@ -0,0 +1,61 @@ +package cash.atto.node + +import io.netty.resolver.AbstractAddressResolver +import io.netty.resolver.AddressResolver +import io.netty.resolver.AddressResolverGroup +import io.netty.util.concurrent.EventExecutor +import io.netty.util.concurrent.Promise +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import java.net.InetAddress +import java.net.InetSocketAddress + +/** Preserve JVM/system DNS semantics on the transport owner's workers, outside Netty I/O event loops. */ +internal class JvmAddressResolverGroup( + private val dispatcher: CoroutineDispatcher, +) : AddressResolverGroup() { + override fun newResolver(executor: EventExecutor): AddressResolver = + object : AbstractAddressResolver(executor, InetSocketAddress::class.java) { + private val scope = CoroutineScope(SupervisorJob() + dispatcher) + + override fun doIsResolved(address: InetSocketAddress): Boolean = !address.isUnresolved + + override fun doResolve( + address: InetSocketAddress, + promise: Promise, + ) { + resolve(promise) { InetSocketAddress(InetAddress.getByName(address.hostString), address.port) } + } + + override fun doResolveAll( + address: InetSocketAddress, + promise: Promise>, + ) { + resolve(promise) { + InetAddress.getAllByName(address.hostString).map { InetSocketAddress(it, address.port) }.toMutableList() + } + } + + private fun resolve( + promise: Promise, + lookup: () -> T, + ) { + val job = + scope.launch { + try { + promise.trySuccess(lookup()) + } catch (exception: Exception) { + promise.tryFailure(exception) + } + } + promise.addListener { if (it.isCancelled) job.cancel() } + } + + override fun close() { + scope.cancel() + } + } +} diff --git a/src/main/kotlin/cash/atto/node/network/HandshakeCallback.kt b/src/main/kotlin/cash/atto/node/network/HandshakeCallback.kt index 05071aa3..d493d950 100644 --- a/src/main/kotlin/cash/atto/node/network/HandshakeCallback.kt +++ b/src/main/kotlin/cash/atto/node/network/HandshakeCallback.kt @@ -6,17 +6,7 @@ import cash.atto.commons.AttoInstantAsStringSerializer import cash.atto.commons.AttoSignature import cash.atto.protocol.AttoNode import io.github.oshai.kotlinlogging.KotlinLogging -import io.ktor.client.HttpClient -import io.ktor.client.plugins.HttpTimeout -import io.ktor.client.request.preparePost -import io.ktor.client.request.setBody -import io.ktor.client.statement.bodyAsChannel -import io.ktor.http.ContentType -import io.ktor.http.HttpHeaders -import io.ktor.http.HttpStatusCode -import io.ktor.http.contentType -import io.ktor.serialization.kotlinx.json.json -import jakarta.annotation.PreDestroy +import io.netty.handler.codec.http.HttpResponseStatus import kotlinx.coroutines.withTimeout import kotlinx.serialization.Serializable import org.springframework.stereotype.Component @@ -39,77 +29,28 @@ class HandshakeCallbackService( val validation = peerUriValidator.validate(publicUri) if (validation is PeerUriValidationResult.Rejected) { callbackLogger.trace { "Rejected handshake callback to $publicUri from $remoteHost: ${validation.reason}" } - return@withTimeout HandshakeCallbackResult.Rejected(HttpStatusCode.BadRequest) + return@withTimeout HandshakeCallbackResult.Rejected(HttpResponseStatus.BAD_REQUEST) } - callbackClient.post(publicUri.toHandshakeHttpUri(), requestFactory()) + callbackClient.postHandshake(publicUri.toHandshakeHttpUri(), requestFactory()) } } interface HandshakeCallbackClient { - suspend fun post( + suspend fun postHandshake( handshakeUri: URI, request: CounterChallengeResponse, ): HandshakeCallbackResult.Completed } -@Component -class KtorHandshakeCallbackClient : HandshakeCallbackClient { - private val httpClient = - HttpClient(io.ktor.client.engine.cio.CIO) { - install( - io - .ktor - .client - .plugins - .contentnegotiation - .ContentNegotiation, - ) { - json() - } - - install(HttpTimeout) { - requestTimeoutMillis = NetworkProcessor.CONNECTION_TIMEOUT_IN_SECONDS.seconds.inWholeMilliseconds - } - } - - override suspend fun post( - handshakeUri: URI, - request: CounterChallengeResponse, - ): HandshakeCallbackResult.Completed = - httpClient - .preparePost(handshakeUri.toString()) { - contentType(ContentType.Application.Json) - setBody(request) - }.execute { response -> - HandshakeCallbackResult.Completed( - status = response.status, - response = - if (response.status.value in 200..299) { - response.bodyAsChannel().receiveHandshakePayload( - MAX_CHALLENGE_RESPONSE_SIZE_BYTES, - response.headers[HttpHeaders.ContentLength]?.toLongOrNull(), - ) - } else { - null - }, - ) - } - - @PreDestroy - fun close() { - httpClient.close() - } -} - sealed interface HandshakeCallbackResult { data class Completed( - val status: HttpStatusCode, + val status: HttpResponseStatus, val response: ChallengeResponse?, ) : HandshakeCallbackResult data class Rejected( - val status: HttpStatusCode, + val status: HttpResponseStatus, ) : HandshakeCallbackResult } diff --git a/src/main/kotlin/cash/atto/node/network/HandshakePayloadLimit.kt b/src/main/kotlin/cash/atto/node/network/HandshakePayloadLimit.kt index e65e6f17..ccd61a48 100644 --- a/src/main/kotlin/cash/atto/node/network/HandshakePayloadLimit.kt +++ b/src/main/kotlin/cash/atto/node/network/HandshakePayloadLimit.kt @@ -1,10 +1,10 @@ package cash.atto.node.network -import io.ktor.serialization.kotlinx.json.DefaultJson -import io.ktor.utils.io.ByteReadChannel -import io.ktor.utils.io.readAvailable +import kotlinx.coroutines.reactor.awaitSingle import kotlinx.serialization.decodeFromString -import java.nio.charset.StandardCharsets +import kotlinx.serialization.json.Json +import reactor.core.publisher.Mono +import reactor.netty.ByteBufFlux internal const val MAX_PUBLIC_URI_SIZE_BYTES = 266 internal const val MAX_COUNTER_CHALLENGE_RESPONSE_SIZE_BYTES = 2_400 @@ -14,36 +14,34 @@ internal class HandshakePayloadTooLargeException( maxSize: Int, ) : RuntimeException("Handshake payload exceeds $maxSize bytes") -internal suspend inline fun ByteReadChannel.receiveHandshakePayload( - maxSize: Int, - declaredSize: Long?, -): T { - if (declaredSize != null && declaredSize > maxSize) { - val exception = HandshakePayloadTooLargeException(maxSize) - cancel(exception) - throw exception +// Preserve the JSON settings used by peers before the transport migration. +internal val peerJson = + Json { + encodeDefaults = true + isLenient = true + allowSpecialFloatingPointValues = true + allowStructuredMapKeys = true } - return DefaultJson.decodeFromString(readHandshakePayload(maxSize)) -} - -internal suspend fun ByteReadChannel.readHandshakePayload(maxSize: Int): String { - val payload = ByteArray(maxSize + 1) - var size = 0 +internal suspend inline fun ByteBufFlux.receiveHandshakePayload( + maxSize: Int, + declaredSize: Long?, +): T = peerJson.decodeFromString(readHandshakePayload(maxSize, declaredSize).awaitSingle()) - while (size < payload.size) { - val read = readAvailable(payload, size, payload.size - size) - if (read == -1) { - break +internal fun ByteBufFlux.readHandshakePayload( + maxSize: Int, + declaredSize: Long?, +): Mono = + Mono.defer { + if (declaredSize != null && declaredSize > maxSize) { + return@defer Mono.error(HandshakePayloadTooLargeException(maxSize)) } - size += read + val payload = ByteArray(maxSize + 1) + var size = 0 + handle { buffer, sink -> + val count = minOf(buffer.readableBytes(), payload.size - size) + buffer.getBytes(buffer.readerIndex(), payload, size, count) + size += count + if (size > maxSize) sink.error(HandshakePayloadTooLargeException(maxSize)) + }.then(Mono.fromCallable { payload.decodeToString(0, size) }) } - - if (size > maxSize) { - val exception = HandshakePayloadTooLargeException(maxSize) - cancel(exception) - throw exception - } - - return String(payload, 0, size, StandardCharsets.UTF_8) -} diff --git a/src/main/kotlin/cash/atto/node/network/NetworkProcessor.kt b/src/main/kotlin/cash/atto/node/network/NetworkProcessor.kt index 0c27b8d0..fef05d24 100644 --- a/src/main/kotlin/cash/atto/node/network/NetworkProcessor.kt +++ b/src/main/kotlin/cash/atto/node/network/NetworkProcessor.kt @@ -11,59 +11,47 @@ import cash.atto.node.network.guardian.InboundConnectionDecision import cash.atto.node.transaction.Transaction import cash.atto.protocol.AttoKeepAlive import cash.atto.protocol.AttoNode -import cash.atto.protocol.AttoTransactionStreamRequest import com.github.benmanes.caffeine.cache.Caffeine import com.github.benmanes.caffeine.cache.Scheduler import io.github.oshai.kotlinlogging.KotlinLogging -import io.ktor.client.HttpClient -import io.ktor.client.request.header -import io.ktor.http.HttpStatusCode -import io.ktor.http.isSuccess -import io.ktor.serialization.kotlinx.json.json -import io.ktor.server.application.ApplicationCall -import io.ktor.server.application.install -import io.ktor.server.engine.embeddedServer -import io.ktor.server.plugins.origin -import io.ktor.server.request.contentLength -import io.ktor.server.request.receiveChannel -import io.ktor.server.response.respond -import io.ktor.server.routing.post -import io.ktor.server.routing.routing -import io.ktor.server.websocket.DefaultWebSocketServerSession -import io.ktor.server.websocket.webSocket -import io.ktor.utils.io.ByteReadChannel -import io.ktor.websocket.ChannelOverflow +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpResponseStatus import jakarta.annotation.PreDestroy import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.cancel +import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.delay import kotlinx.coroutines.ensureActive import kotlinx.coroutines.job import kotlinx.coroutines.launch +import kotlinx.coroutines.reactor.awaitSingleOrNull +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.serialization.encodeToString import org.springframework.context.event.EventListener import org.springframework.core.env.Environment import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Component -import java.net.InetSocketAddress +import reactor.core.publisher.Mono +import reactor.netty.http.server.HttpServerRequest +import reactor.netty.http.server.HttpServerResponse import java.net.URI import java.time.Duration import java.util.concurrent.Executors import java.util.concurrent.Semaphore +import java.util.concurrent.atomic.AtomicBoolean import kotlin.time.Duration.Companion.seconds -import io.ktor.client.plugins.websocket.webSocket as clientWebSocket private sealed interface InboundHandshakePreparation { - data object Responded : InboundHandshakePreparation - data class Rejected( - val status: HttpStatusCode, + val status: HttpResponseStatus, + val closeConnection: Boolean = false, ) : InboundHandshakePreparation data class Accepted( @@ -73,6 +61,11 @@ private sealed interface InboundHandshakePreparation { ) : InboundHandshakePreparation } +private data class InboundPeerHeaders( + val publicUri: URI, + val challenge: String, +) + @Component class NetworkProcessor( private val genesisTransaction: Transaction, @@ -85,11 +78,11 @@ class NetworkProcessor( private val dnsResolver: NetworkDnsResolver, private val connectionManager: NodeConnectionManager, private val guardian: Guardian, + private val peerClient: PeerNettyClient, ) : CacheSupport { private val logger = KotlinLogging.logger {} companion object { - const val MAX_MESSAGE_SIZE = 300 const val PUBLIC_URI_HEADER = "Atto-Public-Uri" const val CHALLENGE_HEADER = "Atto-Http-Challenge" const val CONNECTION_TIMEOUT_IN_SECONDS = 5L @@ -97,33 +90,12 @@ class NetworkProcessor( private const val MAX_CONCURRENT_OUTBOUND_CONNECTIONS = 64 } - private val websocketClient = - HttpClient(io.ktor.client.engine.cio.CIO) { - install( - io - .ktor - .client - .plugins - .websocket - .WebSockets, - ) { - maxFrameSize = MAX_MESSAGE_SIZE.toLong() - channels { - incoming = - bounded( - capacity = AttoTransactionStreamRequest.MAX_TRANSACTIONS.toInt(), - onOverflow = ChannelOverflow.CLOSE, - ) - outgoing = - bounded( - capacity = AttoTransactionStreamRequest.MAX_TRANSACTIONS.toInt(), - onOverflow = ChannelOverflow.CLOSE, - ) - } - } - } - - private val scope = CoroutineScope(Executors.newVirtualThreadPerTaskExecutor().asCoroutineDispatcher() + SupervisorJob()) + private val dispatcher = + Executors + .newThreadPerTaskExecutor(Thread.ofVirtual().name("atto-peer-connect-work-", 0).factory()) + .asCoroutineDispatcher() + private val stopped = AtomicBoolean() + private val scope = CoroutineScope(dispatcher + SupervisorJob()) private val handshakePermits = Semaphore(MAX_CONCURRENT_HANDSHAKES) private val outboundConnectionPermits = Semaphore(MAX_CONCURRENT_OUTBOUND_CONNECTIONS) @@ -137,113 +109,80 @@ class NetworkProcessor( .asMap() private val server = - embeddedServer(io.ktor.server.cio.CIO, port = environment.getRequiredProperty("websocket.port", Int::class.java)) { - install( - io - .ktor - .server - .websocket - .WebSockets, - ) { - maxFrameSize = MAX_MESSAGE_SIZE.toLong() - channels { - incoming = - bounded( - capacity = AttoTransactionStreamRequest.MAX_TRANSACTIONS.toInt(), - onOverflow = ChannelOverflow.CLOSE, - ) - outgoing = - bounded( - capacity = AttoTransactionStreamRequest.MAX_TRANSACTIONS.toInt(), - onOverflow = ChannelOverflow.CLOSE, - ) - } - } - install( - io - .ktor - .server - .plugins - .contentnegotiation - .ContentNegotiation, - ) { - json() - } - routing { - post("/handshakes") { - call.handleInboundHandshake() - } - webSocket(path = "/") { - handleInboundWebSocket() - } - } - }.start(wait = false) - - private suspend fun ApplicationCall.handleInboundHandshake() { - val channel = receiveChannel() + PeerNettyServer( + environment.getRequiredProperty("websocket.port", Int::class.java), + ::handleInboundHandshake, + ::handleInboundWebSocket, + ) + + private suspend fun handleInboundHandshake( + request: HttpServerRequest, + response: HttpServerResponse, + ) { if (!handshakePermits.tryAcquire()) { - channel.cancel(null) - respond(HttpStatusCode.TooManyRequests) + response.keepAlive(false).respond(HttpResponseStatus.TOO_MANY_REQUESTS) return } try { val preparation = withTimeout(CONNECTION_TIMEOUT_IN_SECONDS.seconds) { - prepareInboundHandshake(channel) + prepareInboundHandshake(request) } when (preparation) { - InboundHandshakePreparation.Responded -> Unit - is InboundHandshakePreparation.Rejected -> respond(preparation.status) - is InboundHandshakePreparation.Accepted -> finishInboundHandshake(preparation) + is InboundHandshakePreparation.Rejected -> { + if (preparation.closeConnection) response.keepAlive(false) + response.respond(preparation.status) + } + + is InboundHandshakePreparation.Accepted -> { + response.finishInboundHandshake(preparation) + } } } catch (e: TimeoutCancellationException) { - channel.cancel(e) - respond(HttpStatusCode.RequestTimeout) + response.keepAlive(false).respond(HttpResponseStatus.REQUEST_TIMEOUT) } catch (e: CancellationException) { - channel.cancel(e) throw e } catch (e: HandshakePayloadTooLargeException) { - logger.trace(e) { "Oversized handshake from ${request.origin.remoteHost}" } - respond(HttpStatusCode.PayloadTooLarge) + logger.trace(e) { "Oversized handshake from ${request.remoteHost()}" } + response.keepAlive(false).respond(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE) } catch (e: Exception) { - channel.cancel(e) - logger.trace(e) { "Exception during handshake with ${request.origin.remoteHost}" } - respond(HttpStatusCode.InternalServerError) + logger.trace(e) { "Exception during handshake with ${request.remoteHost()}" } + response.keepAlive(false).respond(HttpResponseStatus.INTERNAL_SERVER_ERROR) } finally { handshakePermits.release() } } - private suspend fun ApplicationCall.prepareInboundHandshake(channel: ByteReadChannel): InboundHandshakePreparation { - val remoteHost = request.origin.remoteHost - if (!acceptInboundConnection(remoteHost, "handshake")) { - channel.cancel(null) - return InboundHandshakePreparation.Responded + private suspend fun prepareInboundHandshake(request: HttpServerRequest): InboundHandshakePreparation { + val remoteHost = request.remoteHost() + val rejection = inboundConnectionRejection(remoteHost, "handshake") + if (rejection != null) { + return InboundHandshakePreparation.Rejected(rejection, closeConnection = true) } val counterResponse = - channel.receiveHandshakePayload( + request.receive().receiveHandshakePayload( MAX_COUNTER_CHALLENGE_RESPONSE_SIZE_BYTES, - request.contentLength(), + request.requestHeaders().get(HttpHeaderNames.CONTENT_LENGTH)?.toLongOrNull(), ) val challenge = counterResponse.challenge val publicUri = ChallengeStore.remove(challenge) if (publicUri == null) { logger.trace { "Received invalid challenge request from $remoteHost $counterResponse" } - return InboundHandshakePreparation.Rejected(HttpStatusCode.BadRequest) + return InboundHandshakePreparation.Rejected(HttpResponseStatus.BAD_REQUEST) } val node = counterResponse.node if (node.publicUri != publicUri) { logger.trace { "Node publicUri ${node.publicUri} doesn't match expected $publicUri from $remoteHost" } - return InboundHandshakePreparation.Rejected(HttpStatusCode.BadRequest) + return InboundHandshakePreparation.Rejected(HttpResponseStatus.BAD_REQUEST) } if (counterResponse.genesis != genesisTransaction.hash) { logger.trace { "Received mismatched genesis hash from $publicUri $remoteHost $counterResponse" } - return InboundHandshakePreparation.Rejected(HttpStatusCode.BadRequest) + return InboundHandshakePreparation.Rejected(HttpResponseStatus.BAD_REQUEST) } val counterTimestamp = counterResponse.timestamp @@ -258,7 +197,7 @@ class NetworkProcessor( val signature = counterResponse.signature if (!signature.isValid(node.publicKey, hash)) { logger.trace { "Received invalid signature from server $remoteHost $counterResponse" } - return InboundHandshakePreparation.Rejected(HttpStatusCode.BadRequest) + return InboundHandshakePreparation.Rejected(HttpResponseStatus.BAD_REQUEST) } val counterChallenge = counterResponse.counterChallenge @@ -267,7 +206,7 @@ class NetworkProcessor( "Received invalid challenge prefix request from $publicUri $remoteHost " + "$counterResponse" } - return InboundHandshakePreparation.Rejected(HttpStatusCode.BadRequest) + return InboundHandshakePreparation.Rejected(HttpResponseStatus.BAD_REQUEST) } logger.trace { "Challenge $challenge validated successfully" } @@ -283,160 +222,151 @@ class NetworkProcessor( return InboundHandshakePreparation.Accepted(publicUri, node, response) } - private suspend fun ApplicationCall.finishInboundHandshake(handshake: InboundHandshakePreparation.Accepted) { + private suspend fun HttpServerResponse.finishInboundHandshake(handshake: InboundHandshakePreparation.Accepted) { currentCoroutineContext().ensureActive() val connectionAttempt = connectingMap[handshake.publicUri] if (connectionAttempt == null || !connectionAttempt.authenticate(handshake.node)) { logger.trace { "Received valid handshake but connection already expired" } - respond(HttpStatusCode.InternalServerError) + respond(HttpResponseStatus.INTERNAL_SERVER_ERROR) return } - respond(HttpStatusCode.OK, handshake.response) + status(HttpResponseStatus.OK) + .header(HttpHeaderNames.CONTENT_TYPE, "application/json") + .sendString(Mono.just(peerJson.encodeToString(handshake.response))) + .then() + .awaitSingleOrNull() } - private suspend fun DefaultWebSocketServerSession.handleInboundWebSocket() { + private suspend fun handleInboundWebSocket( + request: HttpServerRequest, + session: PeerSession, + ) { try { - val remoteHost = call.request.origin.remoteHost + val remoteHost = request.remoteHost() logger.trace { "New websocket connection attempt from $remoteHost" } + if (inboundConnectionRejection(remoteHost, "websocket") != null) return - if (!call.acceptInboundConnection(remoteHost, "websocket")) return - - val publicUriHeader = call.request.headers[PUBLIC_URI_HEADER] - val challengeHeader = call.request.headers[CHALLENGE_HEADER] - if (publicUriHeader == null || challengeHeader == null) { - logger.trace { "Missing required headers from $remoteHost" } - call.respond(HttpStatusCode.BadRequest) - return - } - - val publicUri = publicUriHeader.toUriOrNull() - if (publicUri == null) { - logger.trace { "Invalid public URI header '$publicUriHeader' from $remoteHost" } - call.respond(HttpStatusCode.BadRequest) - return - } + val headers = readInboundPeerHeaders(request, remoteHost) ?: return + val node = authenticateInboundPeer(remoteHost, headers) ?: return + connectionManager.manage(node, requireNotNull(request.remoteAddress()), session) + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + logger.trace(e) { "Exception during handshake with ${request.remoteHost()}" } + } + } - val scheme = publicUri.scheme - if (scheme != "ws" && scheme != "wss") { - logger.trace { "Invalid URI scheme '$scheme' from $remoteHost" } - call.respond(HttpStatusCode.BadRequest) - return - } + private fun readInboundPeerHeaders( + request: HttpServerRequest, + remoteHost: String, + ): InboundPeerHeaders? { + val publicUriHeader = request.requestHeaders().get(PUBLIC_URI_HEADER) + val challengeHeader = request.requestHeaders().get(CHALLENGE_HEADER) + if (publicUriHeader == null || challengeHeader == null) { + logger.trace { "Missing required headers from $remoteHost" } + return null + } - if (publicUri == thisNode.publicUri) { - logger.trace { "Can't connect as a server to $publicUri. This uri is this node." } - call.respond(HttpStatusCode.BadRequest) - return - } + val publicUri = publicUriHeader.toUriOrNull() + if (publicUri == null) { + logger.trace { "Invalid public URI header '$publicUriHeader' from $remoteHost" } + return null + } + if (publicUri.scheme != "ws" && publicUri.scheme != "wss") { + logger.trace { "Invalid URI scheme '${publicUri.scheme}' from $remoteHost" } + return null + } + if (publicUri == thisNode.publicUri) { + logger.trace { "Can't connect as a server to $publicUri. This uri is this node." } + return null + } - logger.trace { "Headers received: publicUri=$publicUri, challenge=$challengeHeader" } - if (!challengeHeader.isChallengePrefixValid()) { - logger.trace { "Received invalid challenge prefix request from $publicUri $remoteHost" } - call.respond(HttpStatusCode.BadRequest) - return - } + logger.trace { "Headers received: publicUri=$publicUri, challenge=$challengeHeader" } + if (!challengeHeader.isChallengePrefixValid()) { + logger.trace { "Received invalid challenge prefix request from $publicUri $remoteHost" } + return null + } + return InboundPeerHeaders(publicUri, challengeHeader) + } - val connectionAttempt = OutboundConnectionAttempt() - if (connectingMap.putIfAbsent(publicUri, connectionAttempt) != null) { - logger.trace { "Can't connect as a server to $publicUri. Connection attempt in progress." } - call.respond(HttpStatusCode.BadRequest) - return - } + private suspend fun authenticateInboundPeer( + remoteHost: String, + headers: InboundPeerHeaders, + ): AttoNode? { + val publicUri = headers.publicUri + val connectionAttempt = OutboundConnectionAttempt() + if (connectingMap.putIfAbsent(publicUri, connectionAttempt) != null) { + logger.trace { "Can't connect as a server to $publicUri. Connection attempt in progress." } + return null + } + var authenticatedNode: AttoNode? = null + try { val timestamp = AttoInstant.now() var counterChallenge: String? = null val callbackResult = - postHandshakeCallback(remoteHost, publicUri, connectionAttempt) { + handshakeCallbackService.post(remoteHost, publicUri) { val generatedCounterChallenge = ChallengeStore.generate(publicUri) counterChallenge = generatedCounterChallenge CounterChallengeResponse( - challengeHeader, + headers.challenge, genesisTransaction.hash, thisNode, timestamp, - signer.sign(AttoChallenge(challengeHeader.fromHexToByteArray()), timestamp), + signer.sign(AttoChallenge(headers.challenge.fromHexToByteArray()), timestamp), generatedCounterChallenge, ) } - - if (callbackResult is HandshakeCallbackResult.Rejected) { - connectingMap.remove(publicUri, connectionAttempt) - call.respond(callbackResult.status) - return - } - - val result = callbackResult as HandshakeCallbackResult.Completed - logger.trace { "Challenge response status from ${publicUri.toHandshakeHttpUri()}: ${result.status}" } - if (!result.status.isSuccess()) { - connectingMap.remove(publicUri, connectionAttempt) - logger.trace { "Received invalid ${result.status.value} challenge status from $publicUri $remoteHost" } - call.respond(HttpStatusCode.BadRequest) - return - } - - val response = result.response - if (response == null) { - connectingMap.remove(publicUri, connectionAttempt) - logger.trace { "Received empty challenge response from $publicUri $remoteHost" } - call.respond(HttpStatusCode.BadRequest) - return - } - - val expectedCounterChallenge = counterChallenge - if (expectedCounterChallenge == null || ChallengeStore.remove(expectedCounterChallenge) == null) { - connectingMap.remove(publicUri, connectionAttempt) - logger.trace { "Received invalid challenge response from $publicUri $remoteHost $response" } - call.respond(HttpStatusCode.BadRequest) - return - } - - val node = response.node - if (node.publicUri != publicUri) { - connectingMap.remove(publicUri, connectionAttempt) - logger.trace { "Node publicUri ${node.publicUri} doesn't match header $publicUri from $remoteHost" } - call.respond(HttpStatusCode.BadRequest) - return - } - - val counterHash = - AttoHash.hash( - 64, - node.publicKey.value, - expectedCounterChallenge.fromHexToByteArray(), - response.timestamp.toByteArray(), - ) - - val signature = response.signature - if (!signature.isValid(node.publicKey, counterHash)) { - connectingMap.remove(publicUri, connectionAttempt) - logger.trace { "Received invalid signature from client $remoteHost $response" } - call.respond(HttpStatusCode.BadRequest) - return - } - - val connectionSocketAddress = InetSocketAddress(call.request.origin.remoteHost, call.request.origin.remotePort) - connectionManager.manage(node, connectionSocketAddress, this) - } catch (e: CancellationException) { - throw e - } catch (e: Exception) { - logger.trace(e) { "Exception during handshake with ${call.request.origin.remoteHost}" } - call.respond(HttpStatusCode.InternalServerError) + val result = callbackResult as? HandshakeCallbackResult.Completed ?: return null + authenticatedNode = validateInboundCallback(remoteHost, publicUri, counterChallenge, result) + return authenticatedNode + } finally { + // Failed attempts release admission; successful attempts retain the existing expiry window. + if (authenticatedNode == null) connectingMap.remove(publicUri, connectionAttempt) } } - private suspend fun postHandshakeCallback( + private suspend fun validateInboundCallback( remoteHost: String, publicUri: URI, - connectionAttempt: OutboundConnectionAttempt, - requestFactory: suspend () -> CounterChallengeResponse, - ): HandshakeCallbackResult = - try { - handshakeCallbackService.post(remoteHost, publicUri, requestFactory) - } catch (e: Throwable) { - connectingMap.remove(publicUri, connectionAttempt) - throw e + counterChallenge: String?, + result: HandshakeCallbackResult.Completed, + ): AttoNode? { + logger.trace { "Challenge response status from ${publicUri.toHandshakeHttpUri()}: ${result.status}" } + if (result.status.code() !in 200..299) { + logger.trace { "Received invalid ${result.status.code()} challenge status from $publicUri $remoteHost" } + return null + } + + val response = result.response + if (response == null) { + logger.trace { "Received empty challenge response from $publicUri $remoteHost" } + return null + } + if (counterChallenge == null || ChallengeStore.remove(counterChallenge) == null) { + logger.trace { "Received invalid challenge response from $publicUri $remoteHost $response" } + return null + } + + val node = response.node + if (node.publicUri != publicUri) { + logger.trace { "Node publicUri ${node.publicUri} doesn't match header $publicUri from $remoteHost" } + return null } + val counterHash = + AttoHash.hash( + 64, + node.publicKey.value, + counterChallenge.fromHexToByteArray(), + response.timestamp.toByteArray(), + ) + if (!response.signature.isValid(node.publicKey, counterHash)) { + logger.trace { "Received invalid signature from client $remoteHost $response" } + return null + } + return node + } override fun clear() { connectingMap.clear() @@ -444,11 +374,33 @@ class NetworkProcessor( @PreDestroy fun stop() { + if (!stopped.compareAndSet(false, true)) return logger.info { "Network Processor is stopping..." } + server.stopAccepting() clear() - websocketClient.close() - scope.cancel() - server.stop() + try { + runBlocking { + withTimeoutOrNull(5.seconds) { scope.coroutineContext.job.cancelAndJoin() } + } + } finally { + try { + closeTransports() + } finally { + try { + connectionManager.clear() + } finally { + dispatcher.close() + } + } + } + } + + private fun closeTransports() { + try { + server.close() + } finally { + peerClient.close() + } } @Scheduled(fixedRate = 1_000) @@ -471,7 +423,7 @@ class NetworkProcessor( } private suspend fun connectAsync(publicUri: URI) { - if (publicUri == thisNode.publicUri) { + if (stopped.get() || publicUri == thisNode.publicUri) { return } @@ -503,28 +455,32 @@ class NetworkProcessor( } } - private suspend fun ApplicationCall.acceptInboundConnection( + private suspend fun inboundConnectionRejection( remoteHost: String, requestType: String, - ): Boolean = + ): HttpResponseStatus? = when (guardian.requestInboundConnection(dnsResolver.getByName(remoteHost))) { InboundConnectionDecision.Accepted -> { - true + null } InboundConnectionDecision.Banned -> { logger.trace { "Rejected $requestType from banned address $remoteHost" } - respond(HttpStatusCode.Forbidden) - false + HttpResponseStatus.FORBIDDEN } InboundConnectionDecision.RateLimited -> { logger.trace { "Rejected $requestType from rate limited address $remoteHost" } - respond(HttpStatusCode.TooManyRequests) - false + HttpResponseStatus.TOO_MANY_REQUESTS } } + private suspend fun HttpServerResponse.respond(status: HttpResponseStatus) { + status(status).send().awaitSingleOrNull() + } + + private fun HttpServerRequest.remoteHost(): String = requireNotNull(remoteAddress()).address.hostAddress + private suspend fun connection( publicUri: URI, connectionAttempt: OutboundConnectionAttempt, @@ -555,28 +511,22 @@ class NetworkProcessor( logger.trace { "Connecting to $publicUri" } - websocketClient.clientWebSocket( - urlString = publicUri.toString(), - request = { - header(PUBLIC_URI_HEADER, thisNode.publicUri.toString()) - header(CHALLENGE_HEADER, ChallengeStore.generate(publicUri)) - }, - ) { - val connectionSocketAddress = - InetSocketAddress( - call.request.url.host, - call.request.url.port, - ) - + peerClient.webSocket( + publicUri, + mapOf( + PUBLIC_URI_HEADER to thisNode.publicUri.toString(), + CHALLENGE_HEADER to ChallengeStore.generate(publicUri), + ), + ) { session, connectionSocketAddress -> val node = connectionAttempt.awaitNode() if (node.publicUri != publicUri) { logger.trace { "Node publicUri ${node.publicUri} doesn't match expected $publicUri" } - return@clientWebSocket + return@webSocket } timeoutJob.cancel() - connectionManager.manage(node, connectionSocketAddress, this) + connectionManager.manage(node, connectionSocketAddress, session) } } finally { timeoutJob.cancel() diff --git a/src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt b/src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt index 9f04a633..b9242db3 100644 --- a/src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt +++ b/src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt @@ -10,10 +10,6 @@ import cash.atto.protocol.AttoNode import com.github.benmanes.caffeine.cache.Caffeine import com.github.benmanes.caffeine.cache.Scheduler import io.github.oshai.kotlinlogging.KotlinLogging -import io.ktor.websocket.Frame -import io.ktor.websocket.WebSocketSession -import io.ktor.websocket.close -import io.ktor.websocket.readBytes import jakarta.annotation.PreDestroy import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope @@ -21,9 +17,6 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.cancel import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.consumeAsFlow -import kotlinx.coroutines.flow.filterIsInstance -import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onStart import kotlinx.coroutines.joinAll @@ -47,7 +40,8 @@ class NodeConnectionManager( private val logger = KotlinLogging.logger {} private val supervisorJob = SupervisorJob() - private val scope = CoroutineScope(Executors.newVirtualThreadPerTaskExecutor().asCoroutineDispatcher() + supervisorJob) + private val dispatcher = Executors.newVirtualThreadPerTaskExecutor().asCoroutineDispatcher() + private val scope = CoroutineScope(dispatcher + supervisorJob) private val connectionMap = Caffeine @@ -91,6 +85,7 @@ class NodeConnectionManager( logger.info { "Node Connection Manager is stopping..." } clear() scope.cancel() + dispatcher.close() } override fun snapshot(): List = connectionMap.values.map { ConnectedPeer(it.node, it.connectionInetSocketAddress) } @@ -100,7 +95,7 @@ class NodeConnectionManager( suspend fun manage( node: AttoNode, connectionSocketAddress: InetSocketAddress, - session: WebSocketSession, + session: PeerSession, ) { val publicUri = node.publicUri val connection = NodeConnection(node, connectionSocketAddress, session) @@ -211,17 +206,15 @@ class NodeConnectionManager( private inner class NodeConnection( val node: AttoNode, val connectionInetSocketAddress: InetSocketAddress, - val session: WebSocketSession, + val session: PeerSession, ) { fun incomingFlow(): Flow = session .incoming - .consumeAsFlow() - .filterIsInstance() .onStart { logger.info { "Connected to ${node.publicUri} ${node.publicKey}" } } .onCompletion { cause -> logger.info(cause?.takeUnless { it is CancellationException }) { "Disconnected from ${node.publicUri}" } - }.map { it.readBytes() } + } suspend fun disconnect() { try { @@ -233,7 +226,7 @@ class NodeConnectionManager( } suspend fun send(message: ByteArray) { - session.outgoing.send(Frame.Binary(true, message)) + session.send(message) } } } diff --git a/src/main/kotlin/cash/atto/node/network/PeerNettyClient.kt b/src/main/kotlin/cash/atto/node/network/PeerNettyClient.kt new file mode 100644 index 00000000..7bd29dfe --- /dev/null +++ b/src/main/kotlin/cash/atto/node/network/PeerNettyClient.kt @@ -0,0 +1,224 @@ +package cash.atto.node.network + +import cash.atto.node.JvmAddressResolverGroup +import io.netty.buffer.Unpooled +import io.netty.handler.codec.http.HttpHeaderNames +import jakarta.annotation.PreDestroy +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.reactor.awaitSingle +import kotlinx.coroutines.reactor.awaitSingleOrNull +import kotlinx.coroutines.reactor.mono +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import org.springframework.stereotype.Component +import reactor.core.publisher.Mono +import reactor.netty.ByteBufFlux +import reactor.netty.Connection +import reactor.netty.http.HttpProtocol +import reactor.netty.http.client.HttpClient +import reactor.netty.http.client.HttpClientInfos +import reactor.netty.http.client.HttpClientResponse +import reactor.netty.http.client.WebsocketClientSpec +import reactor.netty.resources.ConnectionProvider +import reactor.netty.resources.LoopResources +import java.net.InetSocketAddress +import java.net.URI +import java.time.Duration +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicReference +import kotlin.time.Duration.Companion.seconds + +/** Private resources keep peer traffic independent of Spring's HTTP server and signer clients. */ +@Component +class PeerNettyClient( + private val dnsResolver: NetworkDnsResolver, +) : HandshakeCallbackClient { + private val lifecycle = Any() + private var closed = false + private val jobs = mutableSetOf() + private val connections = mutableSetOf() + private val dispatcher = + Executors.newThreadPerTaskExecutor(Thread.ofVirtual().name("atto-peer-client-work-", 0).factory()).asCoroutineDispatcher() + private val loops = LoopResources.create("atto-peer-client-io", 2, true) + private val websocketConnections = provider("atto-peer-websocket") + private val callbackConnections = provider("atto-peer-callback") + private val resolver = JvmAddressResolverGroup(dispatcher) + private val websocketClient = client(websocketConnections) + private val callbackClient = client(callbackConnections).keepAlive(false).followRedirect(false) + private val websocketSpec = + WebsocketClientSpec + .builder() + .maxFramePayloadLength(PeerSession.MAX_MESSAGE_SIZE) + .compress(false) + .handlePing(false) + .build() + + suspend fun webSocket( + publicUri: URI, + headers: Map, + block: suspend (PeerSession, InetSocketAddress) -> Unit, + ) = operation { + val context = currentCoroutineContext().minusKey(Job) + websocketClient + .headers { target -> headers.forEach { (name, value) -> target.set(name, value) } } + .followRedirect { request, response -> + val location = response.responseHeaders().get(HttpHeaderNames.LOCATION) + if (response.status().code() !in REDIRECT_STATUSES || location == null || request.redirectedFrom().size >= 19) { + false + } else { + val source = URI(requireNotNull(request.resourceUrl())) + val target = source.resolve(location) + target.scheme.lowercase() in SUPPORTED_SCHEMES && !(source.isSecure() && !target.isSecure()) + } + }.websocket(websocketSpec) + .uri(publicUri) + .handle { inbound, outbound -> + var connection: Connection? = null + inbound.withConnection { connection = it } + val session = ReactorPeerSession(requireNotNull(connection), inbound, outbound) + mono(context) { + operation { + session.run { peer -> + val effectiveUri = URI(requireNotNull((inbound as HttpClientInfos).resourceUrl())) + val port = effectiveUri.port.takeIf { it >= 0 } ?: if (effectiveUri.isSecure()) 443 else 80 + val address = InetSocketAddress(dnsResolver.getByName(effectiveUri.host), port) + block(peer, address) + } + } + } + }.then() + .awaitSingleOrNull() + Unit + } + + override suspend fun postHandshake( + handshakeUri: URI, + request: CounterChallengeResponse, + ): HandshakeCallbackResult.Completed = + operation { + withTimeout(NetworkProcessor.CONNECTION_TIMEOUT_IN_SECONDS.seconds) { + val payload = peerJson.encodeToString(request).encodeToByteArray() + val connection = AtomicReference() + try { + callbackClient + .doOnRequest { _, current -> connection.set(current) } + .headers { + it.set(HttpHeaderNames.CONTENT_TYPE, "application/json") + it.setInt(HttpHeaderNames.CONTENT_LENGTH, payload.size) + }.post() + .uri(handshakeUri) + .send(Mono.fromSupplier { Unpooled.wrappedBuffer(payload) }) + .response { response, body -> + receiveHandshakeResponse(response, body) + }.single() + .awaitSingle() + } finally { + // Cancellation may precede the response handler when a chunk is only partially received. + connection.get()?.dispose() + } + } + } + + private fun receiveHandshakeResponse( + response: HttpClientResponse, + body: ByteBufFlux, + ): Mono { + val status = response.status() + if (status.code() !in 200..299) { + return Mono.just(HandshakeCallbackResult.Completed(status, null)) + } + val limit = MAX_CHALLENGE_RESPONSE_SIZE_BYTES + val declaredSize = response.responseHeaders().get(HttpHeaderNames.CONTENT_LENGTH)?.toLongOrNull() + return body + .readHandshakePayload(limit, declaredSize) + .map { HandshakeCallbackResult.Completed(status, peerJson.decodeFromString(it)) } + } + + private fun provider(name: String): ConnectionProvider = + ConnectionProvider + .builder(name) + .maxConnections(1000) + .pendingAcquireMaxCount(1000) + .pendingAcquireTimeout(Duration.ofSeconds(NetworkProcessor.CONNECTION_TIMEOUT_IN_SECONDS)) + .build() + + private fun client(provider: ConnectionProvider): HttpClient = + HttpClient + .create(provider) + .runOn(loops) + .resolver(resolver) + .protocol(HttpProtocol.HTTP11) + .compress(false) + .disableRetry(true) + .doOnConnected { connection -> + val accepted = + synchronized(lifecycle) { + if (closed) false else connections.add(connection) + } + if (accepted) { + connection.onDispose().doFinally { synchronized(lifecycle) { connections.remove(connection) } }.subscribe() + } else { + connection.dispose() + } + } + + private suspend fun operation(block: suspend CoroutineScope.() -> T): T = + withContext(dispatcher) { + coroutineScope { + val job = currentCoroutineContext()[Job]!! + synchronized(lifecycle) { + check(!closed) { "Peer transport is closed" } + jobs.add(job) + } + try { + block() + } finally { + synchronized(lifecycle) { jobs.remove(job) } + } + } + } + + @PreDestroy + fun close() { + val snapshot = + synchronized(lifecycle) { + if (closed) return + closed = true + jobs.toList() to connections.toList() + } + snapshot.first.forEach { it.cancel() } + snapshot.second.forEach { it.dispose() } + runBlocking { + withTimeoutOrNull(5.seconds) { + snapshot.first.joinAll() + snapshot.second.forEach { it.onDispose().awaitSingleOrNull() } + } + } + try { + Mono.`when`(websocketConnections.disposeLater(), callbackConnections.disposeLater()).block(Duration.ofSeconds(5)) + } finally { + resolver.close() + try { + loops.disposeLater(Duration.ZERO, Duration.ofSeconds(5)).block(Duration.ofSeconds(6)) + } finally { + dispatcher.close() + } + } + } + + private fun URI.isSecure(): Boolean = scheme.equals("https", true) || scheme.equals("wss", true) + + private companion object { + val REDIRECT_STATUSES = setOf(301, 302, 303, 307, 308) + val SUPPORTED_SCHEMES = setOf("http", "https", "ws", "wss") + } +} diff --git a/src/main/kotlin/cash/atto/node/network/PeerNettyServer.kt b/src/main/kotlin/cash/atto/node/network/PeerNettyServer.kt new file mode 100644 index 00000000..23882cf4 --- /dev/null +++ b/src/main/kotlin/cash/atto/node/network/PeerNettyServer.kt @@ -0,0 +1,135 @@ +package cash.atto.node.network + +import kotlinx.coroutines.Job +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.reactor.awaitSingleOrNull +import kotlinx.coroutines.reactor.mono +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull +import reactor.core.publisher.Mono +import reactor.netty.Connection +import reactor.netty.DisposableServer +import reactor.netty.http.HttpProtocol +import reactor.netty.http.server.HttpServer +import reactor.netty.http.server.HttpServerRequest +import reactor.netty.http.server.HttpServerResponse +import reactor.netty.http.server.WebsocketServerSpec +import reactor.netty.resources.LoopResources +import java.time.Duration +import java.util.concurrent.Executors +import kotlin.time.Duration.Companion.seconds + +/** Owns the peer listener, application workers and connections independently of Spring HTTP resources. */ +internal class PeerNettyServer( + port: Int, + private val handshake: suspend (HttpServerRequest, HttpServerResponse) -> Unit, + private val webSocket: suspend (HttpServerRequest, PeerSession) -> Unit, +) : AutoCloseable { + private val lifecycle = Any() + private var accepting = true + private var closed = false + private val jobs = mutableSetOf() + private val connections = mutableSetOf() + private val loops = LoopResources.create("atto-peer-server-io", 2, true) + private val dispatcher = + Executors + .newThreadPerTaskExecutor(Thread.ofVirtual().name("atto-peer-server-work-", 0).factory()) + .asCoroutineDispatcher() + private val websocketSpec = + WebsocketServerSpec + .builder() + .maxFramePayloadLength(PeerSession.MAX_MESSAGE_SIZE) + .compress(false) + .handlePing(false) + .build() + private val server: DisposableServer = + try { + HttpServer + .create() + .host("0.0.0.0") + .port(port) + .runOn(loops) + .protocol(HttpProtocol.HTTP11) + .compress(false) + .doOnConnection { connection -> + val accepted = + synchronized(lifecycle) { + if (!accepting) false else connections.add(connection) + } + if (accepted) { + connection.onDispose().doFinally { synchronized(lifecycle) { connections.remove(connection) } }.subscribe() + } else { + connection.dispose() + } + }.route { routes -> + routes.post("/handshakes") { request, response -> application { handshake(request, response) } } + routes.ws( + "/", + { inbound, outbound -> + var connection: Connection? = null + inbound.withConnection { connection = it } + val session = ReactorPeerSession(requireNotNull(connection), inbound, outbound) + application { + session.run { peer -> + webSocket(inbound as HttpServerRequest, peer) + } + } + }, + websocketSpec, + ) + }.bindNow(Duration.ofSeconds(5)) + } catch (exception: Throwable) { + dispatcher.close() + loops.disposeLater(Duration.ZERO, Duration.ofSeconds(5)).block(Duration.ofSeconds(6)) + throw exception + } + + private fun application(block: suspend () -> Unit): Mono = + mono(dispatcher) { + val job = currentCoroutineContext()[Job]!! + synchronized(lifecycle) { + check(accepting) { "Peer server is stopping" } + jobs.add(job) + } + try { + block() + } finally { + synchronized(lifecycle) { jobs.remove(job) } + } + }.then() + + fun stopAccepting() { + synchronized(lifecycle) { accepting = false } + server.dispose() + } + + override fun close() { + val snapshot = + synchronized(lifecycle) { + if (closed) return + accepting = false + closed = true + jobs.toList() to connections.toList() + } + server.dispose() + snapshot.first.forEach { it.cancel() } + snapshot.second.forEach { it.dispose() } + try { + runBlocking { + withTimeoutOrNull(5.seconds) { + snapshot.first.joinAll() + snapshot.second.forEach { it.onDispose().awaitSingleOrNull() } + server.onDispose().awaitSingleOrNull() + } + } + } finally { + try { + loops.disposeLater(Duration.ZERO, Duration.ofSeconds(5)).block(Duration.ofSeconds(6)) + } finally { + dispatcher.close() + } + } + } +} diff --git a/src/main/kotlin/cash/atto/node/network/PeerSession.kt b/src/main/kotlin/cash/atto/node/network/PeerSession.kt new file mode 100644 index 00000000..90514cf3 --- /dev/null +++ b/src/main/kotlin/cash/atto/node/network/PeerSession.kt @@ -0,0 +1,22 @@ +package cash.atto.node.network + +import kotlinx.coroutines.flow.Flow + +/** A single peer connection. Transport adapters own framing, bounded queues and socket cleanup. */ +interface PeerSession { + companion object { + const val MAX_MESSAGE_SIZE = 300 + const val QUEUE_CAPACITY = 1_000 + } + + /** Complete binary messages, consumed once by the connection owner. */ + val incoming: Flow + + /** Enqueue a binary message in send order; completion does not acknowledge peer delivery. */ + suspend fun send(message: ByteArray) + + /** Initiate graceful shutdown without waiting for the remote close acknowledgement. */ + suspend fun close() + + fun cancel() +} diff --git a/src/main/kotlin/cash/atto/node/network/ReactorPeerSession.kt b/src/main/kotlin/cash/atto/node/network/ReactorPeerSession.kt new file mode 100644 index 00000000..32b270f7 --- /dev/null +++ b/src/main/kotlin/cash/atto/node/network/ReactorPeerSession.kt @@ -0,0 +1,189 @@ +package cash.atto.node.network + +import io.netty.buffer.Unpooled +import io.netty.handler.codec.http.websocketx.BinaryWebSocketFrame +import io.netty.handler.codec.http.websocketx.TextWebSocketFrame +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.reactor.awaitSingleOrNull +import kotlinx.coroutines.withTimeoutOrNull +import reactor.core.Disposable +import reactor.core.publisher.Mono +import reactor.netty.Connection +import reactor.netty.http.websocket.WebsocketInbound +import reactor.netty.http.websocket.WebsocketOutbound +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.time.Duration.Companion.seconds + +/** Owns both bounded handoff queues and the single ordered writer for one connection. */ +internal class ReactorPeerSession( + private val connection: Connection, + private val inbound: WebsocketInbound, + private val outbound: WebsocketOutbound, +) : PeerSession { + init { + // Install framing during upgrade, before application dispatch can queue raw fragments. + inbound.aggregateFrames(PeerSession.MAX_MESSAGE_SIZE) + } + + private class IncomingMessage( + val binary: ByteArray?, + ) + + private val incomingMessages = Channel(PeerSession.QUEUE_CAPACITY) + + private sealed interface OutgoingMessage { + class Binary( + val bytes: ByteArray, + ) : OutgoingMessage + + data object Close : OutgoingMessage + } + + private val outgoingMessages = Channel(PeerSession.QUEUE_CAPACITY) + private val sendLock = Any() + private var closing = false + private val closeRequested = CompletableDeferred() + private val closeFlushed = CompletableDeferred() + private val collected = AtomicBoolean() + + override val incoming: Flow = + flow { + check(collected.compareAndSet(false, true)) { "Peer messages support one collector" } + for (message in incomingMessages) { + message.binary?.let { emit(it) } + } + } + + override suspend fun send(message: ByteArray) { + val accepted = + synchronized(sendLock) { + !closing && outgoingMessages.trySend(OutgoingMessage.Binary(message.copyOf())).isSuccess + } + if (!accepted) { + cancel() + throw CancellationException("Peer outgoing queue is full or closed") + } + } + + override suspend fun close() { + val accepted = + synchronized(sendLock) { + if (closing) { + true + } else { + closing = true + outgoingMessages.trySend(OutgoingMessage.Close).isSuccess + } + } + if (!accepted) { + cancel() + throw CancellationException("Peer outgoing queue is full or closed") + } + closeRequested.complete(Unit) + } + + override fun cancel() { + synchronized(sendLock) { closing = true } + closeFlushed.completeExceptionally(CancellationException("Peer connection closed")) + incomingMessages.cancel() + outgoingMessages.cancel() + connection.dispose() + } + + suspend fun run(block: suspend (PeerSession) -> Unit) = + coroutineScope { + val closeTimeout = + launch { + closeRequested.await() + val flushed = + withTimeoutOrNull(5.seconds) { + closeFlushed.await() + true + } + if (flushed != true) { + this@ReactorPeerSession.cancel() + } + } + val receiver = receiveMessages(this) + val writer = launch { writeMessages() } + try { + block(this@ReactorPeerSession) + } finally { + try { + if (currentCoroutineContext().isActive) { + close() + withTimeoutOrNull(5.seconds) { closeFlushed.await() } + } + } finally { + receiver.dispose() + closeTimeout.cancel() + writer.cancel() + this@ReactorPeerSession.cancel() + } + } + } + + private fun receiveMessages(sessionScope: CoroutineScope): Disposable = + inbound + .receiveFrames() + .subscribe( + { frame -> + if (frame is BinaryWebSocketFrame || frame is TextWebSocketFrame) { + val message = + IncomingMessage( + if (frame is BinaryWebSocketFrame) { + ByteArray(frame.content().readableBytes()).also { + frame.content().getBytes(frame.content().readerIndex(), it) + } + } else { + null + }, + ) + // Text occupies the same bounded queue until the collector discards it. + if (incomingMessages.trySend(message).isFailure) { + cancel() + sessionScope.cancel("Peer incoming queue is full or closed") + } + } + }, + { error -> + incomingMessages.close(error) + sessionScope.cancel("Peer receive failed", error) + }, + { + incomingMessages.close() + }, + ) + + private suspend fun writeMessages() { + for (message in outgoingMessages) { + when (message) { + is OutgoingMessage.Binary -> { + outbound.send(Mono.fromSupplier { Unpooled.wrappedBuffer(message.bytes) }).then().awaitSingleOrNull() + } + + OutgoingMessage.Close -> { + try { + outbound.sendClose(1000, "").awaitSingleOrNull() + } catch (_: Exception) { + // A remote close can race our final close frame after its messages were drained. + connection.dispose() + } finally { + closeFlushed.complete(Unit) + } + break + } + } + } + } +} diff --git a/src/test/kotlin/cash/atto/node/network/HandshakeCallbackServiceTest.kt b/src/test/kotlin/cash/atto/node/network/HandshakeCallbackServiceTest.kt index 770ec50e..0013a8d5 100644 --- a/src/test/kotlin/cash/atto/node/network/HandshakeCallbackServiceTest.kt +++ b/src/test/kotlin/cash/atto/node/network/HandshakeCallbackServiceTest.kt @@ -1,9 +1,9 @@ package cash.atto.node.network -import io.ktor.http.HttpStatusCode import io.mockk.coEvery import io.mockk.coVerify import io.mockk.mockk +import io.netty.handler.codec.http.HttpResponseStatus import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope @@ -63,13 +63,13 @@ class HandshakeCallbackServiceTest { // Then assertInstanceOf(HandshakeCallbackResult.Rejected::class.java, result) - assertEquals(HttpStatusCode.BadRequest, (result as HandshakeCallbackResult.Rejected).status) + assertEquals(HttpResponseStatus.BAD_REQUEST, (result as HandshakeCallbackResult.Rejected).status) } // Then assertFalse(requestBuilt) coVerify(exactly = unsafeUris.size) { dnsResolver.getAllByName(any()) } - coVerify(exactly = 0) { callbackClient.post(any(), any()) } + coVerify(exactly = 0) { callbackClient.postHandshake(any(), any()) } } @Test @@ -81,8 +81,8 @@ class HandshakeCallbackServiceTest { val service = callbackService(safeCallbackProperties(), callbackClient) coEvery { - callbackClient.post(URI("http://93.184.216.34:7070/handshakes"), request) - } returns HandshakeCallbackResult.Completed(HttpStatusCode.OK, null) + callbackClient.postHandshake(URI("http://93.184.216.34:7070/handshakes"), request) + } returns HandshakeCallbackResult.Completed(HttpResponseStatus.OK, null) // When val result = @@ -93,7 +93,7 @@ class HandshakeCallbackServiceTest { // Then assertInstanceOf(HandshakeCallbackResult.Completed::class.java, result) coVerify(exactly = 1) { - callbackClient.post(URI("http://93.184.216.34:7070/handshakes"), request) + callbackClient.postHandshake(URI("http://93.184.216.34:7070/handshakes"), request) } } @@ -103,10 +103,10 @@ class HandshakeCallbackServiceTest { // Given val callbackClient = mockk() val request = mockk() - val result = HandshakeCallbackResult.Completed(HttpStatusCode.OK, null) + val result = HandshakeCallbackResult.Completed(HttpResponseStatus.OK, null) val service = callbackService(safeCallbackProperties(), callbackClient) var factoryCancelled = false - coEvery { callbackClient.post(any(), any()) } returns result + coEvery { callbackClient.postHandshake(any(), any()) } returns result // When val stalledCall = @@ -120,7 +120,7 @@ class HandshakeCallbackServiceTest { // Then assertInstanceOf(TimeoutCancellationException::class.java, stalledCall.await().exceptionOrNull()) assertTrue(factoryCancelled) - coVerify(exactly = 0) { callbackClient.post(any(), any()) } + coVerify(exactly = 0) { callbackClient.postHandshake(any(), any()) } // When var nextFactoryCalled = false @@ -133,7 +133,7 @@ class HandshakeCallbackServiceTest { // Then assertSame(result, nextResult) assertTrue(nextFactoryCalled) - coVerify(exactly = 1) { callbackClient.post(any(), request) } + coVerify(exactly = 1) { callbackClient.postHandshake(any(), request) } } @Test @@ -145,7 +145,7 @@ class HandshakeCallbackServiceTest { val service = callbackService(safeCallbackProperties(), callbackClient) var callbackStarted = false var callbackCancelled = false - coEvery { callbackClient.post(any(), request) } coAnswers { + coEvery { callbackClient.postHandshake(any(), request) } coAnswers { callbackStarted = true awaitCancellationWithCleanup { callbackCancelled = true } } @@ -204,7 +204,7 @@ class HandshakeCallbackServiceTest { assertInstanceOf(CancellationException::class.java, cancellation) assertEquals(parentCancellation.message, cancellation.message) assertTrue(request.isCancelled) - coVerify(exactly = 0) { callbackClient.post(any(), any()) } + coVerify(exactly = 0) { callbackClient.postHandshake(any(), any()) } } private fun callbackService( diff --git a/src/test/kotlin/cash/atto/node/network/InboundPeerAuthenticationTest.kt b/src/test/kotlin/cash/atto/node/network/InboundPeerAuthenticationTest.kt new file mode 100644 index 00000000..5ef35e21 --- /dev/null +++ b/src/test/kotlin/cash/atto/node/network/InboundPeerAuthenticationTest.kt @@ -0,0 +1,198 @@ +package cash.atto.node.network + +import cash.atto.commons.AttoAlgorithm +import cash.atto.commons.AttoChallenge +import cash.atto.commons.AttoHash +import cash.atto.commons.AttoInstant +import cash.atto.commons.AttoNetwork +import cash.atto.commons.AttoPrivateKey +import cash.atto.commons.AttoPublicKey +import cash.atto.commons.AttoSignature +import cash.atto.commons.fromHexToByteArray +import cash.atto.commons.toHex +import cash.atto.commons.toSignerBlocking +import cash.atto.node.network.guardian.Guardian +import cash.atto.node.network.guardian.InboundConnectionDecision +import cash.atto.node.transaction.Transaction +import cash.atto.protocol.AttoNode +import cash.atto.protocol.NodeFeature +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import io.netty.handler.codec.http.HttpResponseStatus +import kotlinx.coroutines.reactor.awaitSingleOrNull +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.EnumSource +import org.springframework.mock.env.MockEnvironment +import reactor.netty.http.client.HttpClient +import reactor.netty.resources.LoopResources +import java.io.IOException +import java.net.ServerSocket +import java.net.URI +import java.security.SecureRandom +import java.time.Duration +import java.util.concurrent.ConcurrentLinkedQueue +import kotlin.time.Duration.Companion.seconds + +class InboundPeerAuthenticationTest { + @ParameterizedTest + @EnumSource(CallbackFailure::class) + fun `failed inbound authentication closes the socket and permits an immediate authenticated retry`(failure: CallbackFailure): Unit = + runBlocking { + // Given + RunningServer().use { server -> + server.failCallback(failure) + + withTimeout(4.seconds) { + // When + server.connectAndAwaitClose() + + // Then + coVerify(exactly = 1) { server.callback.post(any(), server.peerNode.publicUri, any()) } + coVerify(exactly = 0) { server.connections.manage(any(), any(), any()) } + + // When: retry the same URI before the five-second reservation expires. + server.acceptCallback() + server.connectAndAwaitClose() + + // Then + coVerify(exactly = 2) { server.callback.post(any(), server.peerNode.publicUri, any()) } + coVerify(exactly = 1) { server.connections.manage(server.peerNode, any(), any()) } + + // When: an authenticated URI retains its reservation until the existing expiry. + server.connectAndAwaitClose() + + // Then + coVerify(exactly = 2) { server.callback.post(any(), server.peerNode.publicUri, any()) } + coVerify(exactly = 1) { server.connections.manage(any(), any(), any()) } + } + } + } + + enum class CallbackFailure { + REJECTED_URI, + CONNECTION_FAILURE, + INVALID_SIGNATURE, + } + + private class RunningServer : AutoCloseable { + private val port = freePort() + private val signer = AttoPrivateKey.generate().toSignerBlocking() + private val peerSigner = AttoPrivateKey.generate().toSignerBlocking() + private val localNode = node(signer.publicKey, URI("ws://127.0.0.1:$port")) + private val genesis = AttoHash(ByteArray(32) { 3 }) + private val counterChallenges = ConcurrentLinkedQueue() + private val resolver = NetworkDnsResolver() + private val peerClient = PeerNettyClient(resolver) + private val clientLoops = LoopResources.create("inbound-peer-auth-test", 1, true) + private val client = HttpClient.newConnection().runOn(clientLoops).disableRetry(true) + val peerNode = node(peerSigner.publicKey, URI("ws://127.0.0.1:${freePort()}")) + val callback = mockk() + val connections = mockk(relaxed = true) + private val processor = + NetworkProcessor( + genesisTransaction = mockk { every { hash } returns genesis }, + thisNode = localNode, + signer = signer, + environment = MockEnvironment().withProperty("websocket.port", port.toString()), + networkProperties = NetworkProperties(), + peerUriValidator = mockk(), + handshakeCallbackService = callback, + dnsResolver = resolver, + connectionManager = connections, + guardian = + mockk { + every { requestInboundConnection(any()) } returns InboundConnectionDecision.Accepted + }, + peerClient = peerClient, + ) + + fun failCallback(failure: CallbackFailure) { + coEvery { callback.post(any(), peerNode.publicUri, any()) } coAnswers { + if (failure == CallbackFailure.REJECTED_URI) { + HandshakeCallbackResult.Rejected(HttpResponseStatus.BAD_REQUEST) + } else { + val request = thirdArg CounterChallengeResponse>().invoke() + counterChallenges.add(request.counterChallenge) + if (failure == CallbackFailure.CONNECTION_FAILURE) { + throw IOException("Peer closed the callback connection") + } + HandshakeCallbackResult.Completed( + HttpResponseStatus.OK, + ChallengeResponse(peerNode, AttoInstant.now(), AttoSignature(ByteArray(64))), + ) + } + } + } + + fun acceptCallback() { + coEvery { callback.post(any(), peerNode.publicUri, any()) } coAnswers { + val request = thirdArg CounterChallengeResponse>().invoke() + counterChallenges.add(request.counterChallenge) + assertEquals(genesis, request.genesis) + assertEquals(localNode, request.node) + val timestamp = AttoInstant.now() + HandshakeCallbackResult.Completed( + HttpResponseStatus.OK, + ChallengeResponse( + peerNode, + timestamp, + peerSigner.sign(AttoChallenge(request.counterChallenge.fromHexToByteArray()), timestamp), + ), + ) + } + } + + suspend fun connectAndAwaitClose() { + val random = ByteArray(ChallengeStore.CHALLENGE_SIZE).also { SecureRandom().nextBytes(it) } + val challenge = (localNode.publicUri.toString().toByteArray() + random).toHex() + client + .headers { + it.set(NetworkProcessor.PUBLIC_URI_HEADER, peerNode.publicUri.toString()) + it.set(NetworkProcessor.CHALLENGE_HEADER, challenge) + }.websocket() + .uri("ws://127.0.0.1:$port/") + .handle { inbound, _ -> inbound.receive().then() } + .then() + .awaitSingleOrNull() + } + + override fun close() { + try { + processor.stop() + } finally { + try { + peerClient.close() + resolver.close() + } finally { + try { + clientLoops.disposeLater(Duration.ZERO, Duration.ofSeconds(5)).block(Duration.ofSeconds(6)) + } finally { + counterChallenges.forEach(ChallengeStore::remove) + } + } + } + } + + companion object { + private fun freePort(): Int = ServerSocket(0).use { it.localPort } + + private fun node( + publicKey: AttoPublicKey, + publicUri: URI, + ): AttoNode = + AttoNode( + network = AttoNetwork.LOCAL, + protocolVersion = AttoNode.CURRENT_PROTOCOL_VERSION, + algorithm = AttoAlgorithm.V1, + publicKey = publicKey, + publicUri = publicUri, + features = setOf(NodeFeature.VOTING), + ) + } + } +} diff --git a/src/test/kotlin/cash/atto/node/network/NetworkProcessorTest.kt b/src/test/kotlin/cash/atto/node/network/NetworkProcessorTest.kt index 57cb0f91..1aab54f7 100644 --- a/src/test/kotlin/cash/atto/node/network/NetworkProcessorTest.kt +++ b/src/test/kotlin/cash/atto/node/network/NetworkProcessorTest.kt @@ -17,16 +17,6 @@ import cash.atto.node.network.guardian.InboundConnectionDecision import cash.atto.node.transaction.Transaction import cash.atto.protocol.AttoNode import cash.atto.protocol.NodeFeature -import io.ktor.client.HttpClient -import io.ktor.client.engine.cio.CIO -import io.ktor.client.plugins.contentnegotiation.ContentNegotiation -import io.ktor.client.request.post -import io.ktor.client.request.setBody -import io.ktor.client.statement.HttpResponse -import io.ktor.http.ContentType -import io.ktor.http.HttpStatusCode -import io.ktor.http.contentType -import io.ktor.serialization.kotlinx.json.json import io.mockk.coEvery import io.mockk.coVerify import io.mockk.every @@ -37,19 +27,124 @@ import kotlinx.coroutines.awaitCancellation import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.delay import kotlinx.coroutines.isActive +import kotlinx.coroutines.reactor.awaitSingle import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeout +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import org.springframework.http.HttpStatusCode +import org.springframework.http.MediaType +import org.springframework.http.client.reactive.ReactorClientHttpConnector import org.springframework.mock.env.MockEnvironment +import org.springframework.web.reactive.function.client.WebClient +import reactor.netty.http.client.HttpClient +import reactor.netty.resources.LoopResources +import java.io.InputStream import java.net.InetAddress import java.net.ServerSocket +import java.net.Socket +import java.net.SocketException import java.net.URI +import java.time.Duration import java.util.concurrent.atomic.AtomicInteger import kotlin.time.Duration.Companion.seconds class NetworkProcessorTest { + private val json = Json { encodeDefaults = true } + + @Test + fun `accepts 2400 byte handshake bodies and closes fixed and chunked oversized request sockets`() { + // Given + RunningServer().use { server -> + val encoded = json.encodeToString(server.request) + val atLimit = encoded + " ".repeat(MAX_COUNTER_CHALLENGE_RESPONSE_SIZE_BYTES - encoded.length) + for (chunked in listOf(false, true)) { + // When + val acceptedStatus = + server.socket().use { socket -> + socket.writeRequest(atLimit, chunked) + socket.getInputStream().readResponseStatus() + } + + // Then: a complete body at the limit reaches challenge authentication. + assertEquals(400, acceptedStatus, "At-limit chunked=$chunked body must reach authentication") + + // When + server.socket().use { socket -> + socket.writeRequest(" ".repeat(MAX_COUNTER_CHALLENGE_RESPONSE_SIZE_BYTES + 1), chunked) + val oversizedStatus = socket.getInputStream().readResponseStatus() + + // Then: a keep-alive connection cannot be reused after body cancellation. + assertEquals(413, oversizedStatus, "Oversized chunked=$chunked body must be rejected") + socket.assertClosed() + } + } + } + } + + @Test + fun `times out incomplete inbound body and closes its socket`() { + // Given + RunningServer().use { server -> + server.socket().use { socket -> + socket.writeAscii( + "POST /handshakes HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\n" + + "Content-Length: 10\r\nConnection: keep-alive\r\n\r\n{", + ) + + // When + val status = socket.getInputStream().readResponseStatus() + + // Then + assertEquals(408, status) + socket.assertClosed() + } + } + } + + @Test + fun `rejects banned and rate limited inbound requests before waiting for their body`() { + // Given + for ((decision, expectedStatus) in listOf(InboundConnectionDecision.Banned to 403, InboundConnectionDecision.RateLimited to 429)) { + RunningServer(decision).use { server -> + server.socket().use { socket -> + socket.soTimeout = 2_000 + socket.writeAscii( + "POST /handshakes HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\n" + + "Content-Length: 2400\r\nConnection: keep-alive\r\n\r\n", + ) + + // When + val status = socket.getInputStream().readResponseStatus() + + // Then + assertEquals(expectedStatus, status) + socket.assertClosed() + } + } + } + } + + @Test + fun `preserves malformed JSON failure status and closes cancelled request socket`() { + // Given + RunningServer().use { server -> + server.socket().use { socket -> + socket.writeRequest("{", chunked = false) + + // When + val status = socket.getInputStream().readResponseStatus() + + // Then + assertEquals(500, status) + socket.assertClosed() + } + } + } + @Test fun `times out inbound HTTP handshake signing and cancels the signer`() = runBlocking { @@ -73,6 +168,7 @@ class NetworkProcessorTest { } val dnsResolver = mockk() coEvery { dnsResolver.getByName(any()) } returns InetAddress.getByName("127.0.0.1") + val peerClient = PeerNettyClient(dnsResolver) val processor = NetworkProcessor( @@ -84,18 +180,19 @@ class NetworkProcessorTest { peerUriValidator = mockk(), handshakeCallbackService = mockk(), dnsResolver = dnsResolver, + peerClient = peerClient, connectionManager = mockk(relaxed = true), guardian = mockk { every { requestInboundConnection(any()) } returns InboundConnectionDecision.Accepted }, ) + val clientLoops = LoopResources.create("handshake-test-http", 1, true) val client = - HttpClient(CIO) { - install(ContentNegotiation) { - json() - } - } + WebClient + .builder() + .clientConnector(ReactorClientHttpConnector(HttpClient.newConnection().runOn(clientLoops).disableRetry(true))) + .build() var challenge: String? = null try { @@ -132,7 +229,7 @@ class NetworkProcessorTest { withTimeout(1.seconds) { signingStopped.await() } // Then - assertEquals(HttpStatusCode.RequestTimeout, response.status) + assertEquals(408, response.value()) assertEquals(1, signerInvocations.get()) assertTrue(signingAttempts.get() > 1) @@ -143,10 +240,14 @@ class NetworkProcessorTest { } // Then - assertEquals(HttpStatusCode.BadRequest, rejectedResponse.status) + assertEquals(400, rejectedResponse.value()) coVerify(exactly = 1) { signer.sign(any(), any()) } } finally { - cleanup(client, processor, challenge) + try { + cleanup(clientLoops, processor, challenge) + } finally { + peerClient.close() + } } } @@ -188,22 +289,24 @@ class NetworkProcessorTest { awaitCancellation() } - private suspend fun HttpClient.postHandshake( + private suspend fun WebClient.postHandshake( port: Int, request: CounterChallengeResponse, - ): HttpResponse = - post("http://127.0.0.1:$port/handshakes") { - contentType(ContentType.Application.Json) - setBody(request) - } + ): HttpStatusCode = + post() + .uri("http://127.0.0.1:$port/handshakes") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(json.encodeToString(request)) + .exchangeToMono { response -> response.releaseBody().thenReturn(response.statusCode()) } + .awaitSingle() private fun cleanup( - client: HttpClient, + clientLoops: LoopResources, processor: NetworkProcessor, challenge: String?, ) { try { - client.close() + clientLoops.disposeLater().block(Duration.ofSeconds(5)) } finally { stopProcessorAndRemoveChallenge(processor, challenge) } @@ -219,4 +322,109 @@ class NetworkProcessorTest { challenge?.let(ChallengeStore::remove) } } + + private inner class RunningServer( + decision: InboundConnectionDecision = InboundConnectionDecision.Accepted, + ) : AutoCloseable { + private val port = freePort() + private val resolver = NetworkDnsResolver() + private val peerClient = PeerNettyClient(resolver) + private val signer = AttoPrivateKey.generate().toSignerBlocking() + private val localNode = node(signer.publicKey, URI("ws://127.0.0.1:$port")) + private val genesis = AttoHash(ByteArray(32) { 3 }) + private val processor = + NetworkProcessor( + genesisTransaction = mockk { every { hash } returns genesis }, + thisNode = localNode, + signer = signer, + environment = MockEnvironment().withProperty("websocket.port", port.toString()), + networkProperties = NetworkProperties(), + peerUriValidator = mockk(), + handshakeCallbackService = mockk(), + dnsResolver = resolver, + peerClient = peerClient, + connectionManager = mockk(relaxed = true), + guardian = + mockk { + every { requestInboundConnection(any()) } returns decision + }, + ) + val request = CounterChallengeResponse("00", genesis, localNode, AttoInstant.now(), AttoSignature(ByteArray(64)), "00") + + fun socket(): Socket = Socket("127.0.0.1", port).apply { soTimeout = 10_000 } + + override fun close() { + try { + processor.stop() + } finally { + peerClient.close() + resolver.close() + } + } + } + + private fun Socket.writeRequest( + body: String, + chunked: Boolean, + ) { + val framing = + if (chunked) { + "Transfer-Encoding: chunked\r\n\r\n${body.length.toString(16)}\r\n$body\r\n0\r\n\r\n" + } else { + "Content-Length: ${body.length}\r\n\r\n$body" + } + writeAscii( + "POST /handshakes HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nConnection: keep-alive\r\n$framing", + ) + } + + private fun Socket.writeAscii(text: String) { + getOutputStream().write(text.toByteArray(Charsets.US_ASCII)) + getOutputStream().flush() + } + + private fun Socket.assertClosed() { + try { + assertEquals(-1, getInputStream().read(), "Cancelled request socket must close before another request can reuse it") + } catch (_: SocketException) { + // Reset and EOF both prevent a client from reusing this connection. + } + } + + private fun InputStream.readResponseStatus(): Int { + val status = readAsciiLine().split(' ')[1].toInt() + val headers = mutableMapOf() + while (true) { + val line = readAsciiLine() + if (line.isEmpty()) break + headers[line.substringBefore(':').lowercase()] = line.substringAfter(':').trim() + } + if (headers["transfer-encoding"] == "chunked") { + while (true) { + val size = readAsciiLine().substringBefore(';').toInt(16) + if (size == 0) { + check(readAsciiLine().isEmpty()) + break + } + check(readNBytes(size).size == size) + check(readAsciiLine().isEmpty()) + } + } else { + val size = headers["content-length"]?.toInt() ?: 0 + check(readNBytes(size).size == size) + } + return status + } + + private fun InputStream.readAsciiLine(): String { + val bytes = ArrayList() + while (true) { + val value = read() + check(value >= 0) { "Unexpected EOF reading peer HTTP response" } + if (value == '\n'.code) break + bytes.add(value.toByte()) + check(bytes.size < 16_384) + } + return bytes.toByteArray().decodeToString().removeSuffix("\r") + } } diff --git a/src/test/kotlin/cash/atto/node/network/NodeConnectionManagerTest.kt b/src/test/kotlin/cash/atto/node/network/NodeConnectionManagerTest.kt index 6d80e4ca..4e1f6bab 100644 --- a/src/test/kotlin/cash/atto/node/network/NodeConnectionManagerTest.kt +++ b/src/test/kotlin/cash/atto/node/network/NodeConnectionManagerTest.kt @@ -7,16 +7,12 @@ import cash.atto.node.EventPublisher import cash.atto.protocol.AttoKeepAlive import cash.atto.protocol.AttoNode import cash.atto.protocol.NodeFeature -import io.ktor.websocket.Frame -import io.ktor.websocket.WebSocketExtension -import io.ktor.websocket.WebSocketSession import io.mockk.mockk import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.cancel import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.consumeAsFlow import kotlinx.coroutines.launch import kotlinx.coroutines.test.runCurrent import kotlinx.coroutines.test.runTest @@ -26,7 +22,6 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import java.net.InetSocketAddress import java.net.URI -import kotlin.coroutines.CoroutineContext @OptIn(ExperimentalCoroutinesApi::class) class NodeConnectionManagerTest { @@ -52,7 +47,7 @@ class NodeConnectionManagerTest { // When val next = launch(start = CoroutineStart.UNDISPATCHED) { manager.manage(replacement, nextSocket, nextSession) } - firstSession.incoming.close() + firstSession.messages.close() first.join() // Then @@ -60,7 +55,7 @@ class NodeConnectionManagerTest { assertEquals(listOf(ConnectedPeer(original, firstSocket)), initialSnapshot) // When - nextSession.incoming.close() + nextSession.messages.close() next.join() // Then @@ -86,7 +81,7 @@ class NodeConnectionManagerTest { val nextSession = ChannelSession() try { val first = launch(start = CoroutineStart.UNDISPATCHED) { manager.manage(original, firstSocket, firstSession) } - firstSession.incoming.send(Frame.Binary(true, NetworkSerializer.serialize(AttoKeepAlive()))) + firstSession.messages.send(NetworkSerializer.serialize(AttoKeepAlive())) // When: replace before the scheduled old frame collector resumes. val next = launch(start = CoroutineStart.UNDISPATCHED) { manager.manage(replacement, nextSocket, nextSession) } @@ -96,8 +91,8 @@ class NodeConnectionManagerTest { assertEquals(listOf(ConnectedPeer(replacement, nextSocket)), manager.snapshot()) // When - firstSession.incoming.close() - nextSession.incoming.close() + firstSession.messages.close() + nextSession.messages.close() first.join() next.join() } finally { @@ -126,7 +121,7 @@ class NodeConnectionManagerTest { // Then assertFalse(manager.isConnected(peer.publicUri)) assertTrue(manager.snapshot().isEmpty()) - assertTrue(session.outgoing.tryReceive().getOrNull() is Frame.Close) + assertTrue(session.closeInitiated) } finally { session.cancel() manager.stop() @@ -141,19 +136,20 @@ class NodeConnectionManagerTest { ) // Sending Close starts graceful shutdown; frames already received can still be consumed until the peer closes. - private class ChannelSession : WebSocketSession { - override val coroutineContext: CoroutineContext = SupervisorJob() - override var masking: Boolean = false - override var maxFrameSize: Long = Long.MAX_VALUE - override val incoming = Channel(Channel.UNLIMITED) - override val outgoing = Channel(Channel.UNLIMITED) - override val extensions: List> = emptyList() - - override suspend fun flush() = Unit - - @Deprecated("Use cancel() instead.", level = DeprecationLevel.ERROR) - override fun terminate() { - cancel() + private class ChannelSession : PeerSession { + val messages = Channel(Channel.UNLIMITED) + override val incoming = messages.consumeAsFlow() + var closeInitiated = false + private set + + override suspend fun send(message: ByteArray) = Unit + + override suspend fun close() { + closeInitiated = true + } + + override fun cancel() { + messages.cancel() } } diff --git a/src/test/kotlin/cash/atto/node/network/PeerNettyClientTest.kt b/src/test/kotlin/cash/atto/node/network/PeerNettyClientTest.kt new file mode 100644 index 00000000..7902cf08 --- /dev/null +++ b/src/test/kotlin/cash/atto/node/network/PeerNettyClientTest.kt @@ -0,0 +1,604 @@ +package cash.atto.node.network + +import cash.atto.commons.AttoAlgorithm +import cash.atto.commons.AttoHash +import cash.atto.commons.AttoInstant +import cash.atto.commons.AttoNetwork +import cash.atto.commons.AttoPrivateKey +import cash.atto.commons.AttoSignature +import cash.atto.commons.toSignerBlocking +import cash.atto.protocol.AttoNode +import cash.atto.protocol.AttoTransactionStreamRequest +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.selects.select +import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.io.InputStream +import java.net.InetAddress +import java.net.InetSocketAddress +import java.net.ServerSocket +import java.net.Socket +import java.net.SocketException +import java.net.URI +import java.security.MessageDigest +import java.util.Base64 +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import kotlin.time.Duration.Companion.seconds + +class PeerNettyClientTest { + @Test + fun `drains queued binary after remote normal close before collector starts`() = + runBlocking { + // Given + val expected = byteArrayOf(4, 2) + val applicationStarted = CountDownLatch(1) + val remoteClosed = CompletableDeferred() + val releaseCollector = CompletableDeferred() + RawPeerServer { socket, headers -> + socket.upgradeWebSocket(headers) + check(applicationStarted.await(5, TimeUnit.SECONDS)) + socket.writeFrame(2, expected) + socket.writeFrame(8, byteArrayOf(3, -24)) + try { + while (true) { + val frame = socket.readFrame() ?: break + if (frame.opcode == 8) break + } + } catch (_: SocketException) { + // The completed close handshake may already have disposed the client socket. + } + remoteClosed.complete(Unit) + }.use { server -> + withClient { client -> + val messages = mutableListOf() + val connection = + async { + client.webSocket(server.websocketUri, emptyMap()) { session, _ -> + applicationStarted.countDown() + releaseCollector.await() + session.incoming.toList(messages) + } + } + withTimeout(5.seconds) { remoteClosed.await() } + + // When + releaseCollector.complete(Unit) + withTimeout(5.seconds) { connection.await() } + + // Then + assertEquals(1, messages.size) + assertArrayEquals(expected, messages.single()) + } + } + } + + @Test + fun `flushes accepted binary messages in order before normal close`() = + runBlocking { + // Given + val expected = listOf(byteArrayOf(1), byteArrayOf(2), byteArrayOf(3)) + val received = CompletableDeferred, Int>>() + RawPeerServer { socket, headers -> + socket.upgradeWebSocket(headers) + val messages = mutableListOf() + while (true) { + val frame = checkNotNull(socket.readFrame()) + if (frame.opcode == 2) messages.add(frame.payload) + if (frame.opcode == 8) { + check(frame.payload.size >= 2) + val code = ((frame.payload[0].toInt() and 0xff) shl 8) or (frame.payload[1].toInt() and 0xff) + received.complete(messages to code) + try { + socket.writeFrame(8, frame.payload) + } catch (_: SocketException) { + // Local graceful close need not wait for this remote acknowledgement. + } + break + } + } + }.use { server -> + withClient { client -> + // When + withTimeout(5.seconds) { + client.webSocket(server.websocketUri, emptyMap()) { session, _ -> + expected.forEach { session.send(it) } + session.close() + } + } + val (messages, code) = withTimeout(5.seconds) { received.await() } + + // Then + assertEquals(1000, code) + assertEquals(expected.size, messages.size) + expected.zip(messages).forEach { (sent, actual) -> assertArrayEquals(sent, actual) } + } + } + } + + @Test + fun `closes overflowing inbound text queue while application collector is inactive`() = + runBlocking { + // Given + val applicationStarted = CountDownLatch(1) + val remoteClosed = CompletableDeferred() + val capacity = AttoTransactionStreamRequest.MAX_TRANSACTIONS.toInt() + RawPeerServer { socket, headers -> + socket.upgradeWebSocket(headers) + check(applicationStarted.await(5, TimeUnit.SECONDS)) + try { + repeat(capacity + 1) { socket.writeFrame(1, byteArrayOf(65)) } + while (true) { + val frame = socket.readFrame() ?: break + if (frame.opcode == 8) break + } + } catch (_: SocketException) { + // A reset while writing or reading also proves the overflow closed this socket. + } + remoteClosed.complete(Unit) + }.use { server -> + withClient { client -> + // When + val result = + withTimeout(5.seconds) { + runCatching { + client.webSocket(server.websocketUri, emptyMap()) { _, _ -> + applicationStarted.countDown() + awaitCancellation() + } + } + } + + // Then + assertTrue(result.isFailure) + withTimeout(5.seconds) { remoteClosed.await() } + } + } + } + + @Test + fun `aggregates fragmented binary through 300 bytes and ignores control and text`() = + runBlocking { + // Given + val expected = ByteArray(300) { it.toByte() } + RawPeerServer { socket, headers -> + socket.upgradeWebSocket(headers) + socket.writeFrame(9, byteArrayOf(1)) + socket.writeFrame(10, byteArrayOf(2)) + socket.writeFrame(1, "ignored text".toByteArray()) + socket.writeFrame(2, expected.copyOfRange(0, 150), final = false) + socket.writeFrame(0, expected.copyOfRange(150, 300)) + var frame = socket.readFrame() + while (frame != null && frame.opcode != 8) frame = socket.readFrame() + }.use { server -> + withClient { client -> + // When + withTimeout(5.seconds) { + client.webSocket(server.websocketUri, emptyMap()) { session, _ -> + // Then + assertArrayEquals(expected, session.incoming.first()) + } + } + } + } + } + + @Test + fun `rejects binary larger than 300 bytes for single and fragmented messages`() = + runBlocking { + // Given + for (fragmented in listOf(false, true)) { + val closed = CompletableDeferred() + val messages = mutableListOf() + RawPeerServer { socket, headers -> + socket.upgradeWebSocket(headers) + if (fragmented) { + socket.writeFrame(2, ByteArray(150), final = false) + socket.writeFrame(0, ByteArray(151)) + } else { + socket.writeFrame(2, ByteArray(301)) + } + try { + while (true) { + val frame = socket.readFrame() ?: break + if (frame.opcode == 8) break + } + } catch (_: SocketException) { + // Rejecting an oversized frame may reset the socket instead of completing a close handshake. + } finally { + closed.complete(Unit) + } + }.use { server -> + withClient { client -> + // When + withTimeout(5.seconds) { + runCatching { + client.webSocket(server.websocketUri, emptyMap()) { session, _ -> + session.incoming.toList(messages) + } + } + closed.await() + } + + // Then + assertTrue(messages.isEmpty(), "Oversized fragmented=$fragmented message must not be delivered") + } + } + } + } + + @Test + fun `preserves handshake callback rejection statuses without following redirect`() = + runBlocking { + // Given + val request = handshakeRequest() + RawPeerServer { socket, _ -> socket.writeHttp(200, "unused") }.use { redirectTarget -> + withClient { client -> + for (status in listOf(302, 400, 429, 500)) { + RawPeerServer { socket, _ -> + socket.writeHttp(status, "ignored", "Location: ${redirectTarget.httpUri}\r\n") + }.use { rejection -> + // When + val rejected = withTimeout(5.seconds) { client.postHandshake(rejection.httpUri, request) } + + // Then + assertEquals(status, rejected.status.code()) + assertEquals(null, rejected.response) + } + } + assertEquals(0, redirectTarget.requestCount.get()) + } + } + } + + @Test + fun `rejects declared and chunked handshake callback responses above 700 bytes`() = + runBlocking { + // Given + val request = handshakeRequest() + for (chunked in listOf(false, true)) { + RawPeerServer { socket, _ -> + if (chunked) { + socket.writeAscii("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2bd\r\n${" ".repeat(701)}\r\n0\r\n\r\n") + } else { + socket.writeHttp(200, " ".repeat(701)) + } + }.use { server -> + withClient { client -> + // When + val result = withTimeout(5.seconds) { runCatching { client.postHandshake(server.httpUri, request) } } + + // Then + assertTrue( + result.exceptionOrNull().hasCause(), + "Expected callback payload limit for chunked=$chunked, got ${result.exceptionOrNull()}", + ) + } + } + } + } + + @Test + fun `rejects malformed successful callback and permits a later callback`() = + runBlocking { + // Given + val request = handshakeRequest() + val response = ChallengeResponse(request.node, request.timestamp, request.signature) + RawPeerServer { socket, _ -> + socket.writeHttp(200, "{") + }.use { malformed -> + RawPeerServer { socket, _ -> + socket.writeHttp(200, Json.encodeToString(ChallengeResponse.serializer(), response)) + }.use { valid -> + withClient { client -> + // When + val result = withTimeout(5.seconds) { runCatching { client.postHandshake(malformed.httpUri, request) } } + val nextResult = withTimeout(5.seconds) { client.postHandshake(valid.httpUri, request) } + + // Then + assertTrue(result.isFailure) + assertEquals(response, nextResult.response) + } + } + } + } + + @Test + fun `cancelling stalled declared and chunked callback bodies closes the socket`() = + runBlocking { + // Given + val request = handshakeRequest() + for (chunked in listOf(false, true)) { + val bodyStarted = CompletableDeferred() + val disconnected = CompletableDeferred() + RawPeerServer { socket, _ -> + if (chunked) { + socket.writeAscii("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\na\r\n{") + } else { + socket.writeAscii("HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n{") + } + bodyStarted.complete(Unit) + try { + check(socket.getInputStream().read() == -1) + } catch (_: SocketException) { + // Either EOF or reset proves that cancellation released the request socket. + } + disconnected.complete(Unit) + }.use { server -> + withClient { client -> + val callback = async { client.postHandshake(server.httpUri, request) } + assertTrue( + withTimeoutOrNull(5.seconds) { + server.await(bodyStarted) + true + } == true, + "Raw peer must start the stalled callback response, chunked=$chunked", + ) + + // When + callback.cancelAndJoin() + + // Then + assertTrue( + withTimeoutOrNull(5.seconds) { + server.await(disconnected) + true + } == true, + "Cancelled callback must close the stalled response socket, chunked=$chunked", + ) + assertTrue(callback.isCancelled) + } + } + } + } + + @Test + fun `follows websocket redirect and exposes effective socket after upgrade`() = + runBlocking { + // Given + val expected = byteArrayOf(3, 0, 7) + var currentAddress: InetSocketAddress? = null + RawPeerServer { socket, headers -> + socket.upgradeWebSocket(headers) + socket.writeFrame(2, expected) + socket.readFrame() + }.use { target -> + RawPeerServer { socket, _ -> + socket.writeHttp(307, "", "Location: ${target.websocketUri}\r\n") + }.use { redirect -> + withClient { client -> + // When + withTimeout(5.seconds) { + client.webSocket(redirect.websocketUri, emptyMap()) { session, address -> + currentAddress = address + assertArrayEquals(expected, session.incoming.first()) + } + } + + // Then + assertEquals(target.port, currentAddress?.port) + assertEquals(InetAddress.getByName("127.0.0.1"), currentAddress?.address) + assertEquals(1, redirect.requestCount.get()) + assertEquals(1, target.requestCount.get()) + } + } + } + } + + private suspend fun withClient(block: suspend (PeerNettyClient) -> Unit) { + val resolver = NetworkDnsResolver() + val client = PeerNettyClient(resolver) + try { + block(client) + } finally { + client.close() + resolver.close() + } + } + + private fun handshakeRequest(): CounterChallengeResponse { + val node = + AttoNode( + network = AttoNetwork.LOCAL, + protocolVersion = AttoNode.CURRENT_PROTOCOL_VERSION, + algorithm = AttoAlgorithm.V1, + publicKey = AttoPrivateKey.generate().toSignerBlocking().publicKey, + publicUri = URI("ws://127.0.0.1:7070"), + features = emptySet(), + ) + return CounterChallengeResponse("00", AttoHash(ByteArray(32)), node, AttoInstant.now(), AttoSignature(ByteArray(64)), "00") + } + + private inline fun Throwable?.hasCause(): Boolean = generateSequence(this) { it.cause }.any { it is T } + + private class RawPeerServer( + handler: (Socket, Map) -> Unit, + ) : AutoCloseable { + private val server = ServerSocket(0, 50, InetAddress.getByName("127.0.0.1")) + private val executor = Executors.newVirtualThreadPerTaskExecutor() + private val sockets = ConcurrentHashMap.newKeySet() + private val closing = AtomicBoolean() + private val failures = ConcurrentLinkedQueue() + private val firstFailure = CompletableDeferred() + + val port: Int = server.localPort + val httpUri: URI = URI("http://127.0.0.1:$port/handshakes") + val websocketUri: URI = URI("ws://127.0.0.1:$port") + val requestCount = AtomicInteger() + + init { + executor.submit { + while (!server.isClosed) { + val socket = + try { + server.accept() + } catch (_: SocketException) { + break + } + sockets.add(socket) + executor.submit { + socket.use { + try { + socket.soTimeout = 10_000 + val headers = socket.getInputStream().readHttpRequest() + requestCount.incrementAndGet() + handler(socket, headers) + } catch (failure: Throwable) { + if (!closing.get()) { + failures.add(failure) + firstFailure.complete(failure) + } + } finally { + sockets.remove(socket) + } + } + } + } + } + } + + suspend fun await(signal: Deferred): T = + select { + signal.onAwait { it } + firstFailure.onAwait { throw AssertionError("Raw peer fixture failed", it) } + } + + override fun close() { + closing.set(true) + server.close() + sockets.forEach { it.close() } + executor.shutdownNow() + check(executor.awaitTermination(5, TimeUnit.SECONDS)) { "Raw peer fixture workers did not stop" } + val failure = failures.poll() + if (failure != null) { + val error = AssertionError("Raw peer fixture failed", failure) + failures.forEach(error::addSuppressed) + throw error + } + } + } + + private data class WireFrame( + val opcode: Int, + val payload: ByteArray, + ) + + companion object { + private fun InputStream.readHttpRequest(): Map { + readAsciiLine() + val headers = mutableMapOf() + while (true) { + val line = readAsciiLine() + if (line.isEmpty()) break + headers[line.substringBefore(':').lowercase()] = line.substringAfter(':').trim() + } + if (headers["transfer-encoding"]?.contains("chunked") == true) { + while (true) { + val size = readAsciiLine().substringBefore(';').toInt(16) + if (size == 0) { + check(readAsciiLine().isEmpty()) + break + } + check(readNBytes(size).size == size) + check(readAsciiLine().isEmpty()) + } + } else { + val size = headers["content-length"]?.toInt() ?: 0 + check(readNBytes(size).size == size) + } + return headers + } + + private fun InputStream.readAsciiLine(): String { + val bytes = ArrayList() + while (true) { + val value = read() + check(value != -1) { "Unexpected EOF reading HTTP headers" } + if (value == '\n'.code) break + bytes.add(value.toByte()) + check(bytes.size < 16_384) + } + return bytes.toByteArray().decodeToString().removeSuffix("\r") + } + + private fun Socket.writeAscii(text: String) { + getOutputStream().write(text.toByteArray(Charsets.US_ASCII)) + getOutputStream().flush() + } + + private fun Socket.writeHttp( + status: Int, + body: String, + extraHeaders: String = "", + ) { + writeAscii( + "HTTP/1.1 $status Test\r\nContent-Type: application/json\r\nContent-Length: ${body.toByteArray().size}\r\n$extraHeaders\r\n$body", + ) + } + + private fun Socket.upgradeWebSocket(headers: Map) { + val key = checkNotNull(headers["sec-websocket-key"]) + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + val accept = Base64.getEncoder().encodeToString(MessageDigest.getInstance("SHA-1").digest(key.toByteArray())) + writeAscii( + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: $accept\r\n\r\n", + ) + } + + private fun Socket.writeFrame( + opcode: Int, + payload: ByteArray, + final: Boolean = true, + ) { + val output = getOutputStream() + output.write(opcode or if (final) 0x80 else 0) + if (payload.size < 126) { + output.write(payload.size) + } else { + output.write(126) + output.write(payload.size ushr 8) + output.write(payload.size and 0xff) + } + output.write(payload) + output.flush() + } + + private fun Socket.readFrame(): WireFrame? { + val input = getInputStream() + val first = input.read() + if (first == -1) return null + val second = input.read() + check(second >= 0) + val masked = second and 0x80 != 0 + val length = + when (val shortLength = second and 0x7f) { + 126 -> (input.read() shl 8) or input.read() + 127 -> error("Unexpected oversized client frame") + else -> shortLength + } + val mask = if (masked) input.readNBytes(4) else byteArrayOf() + val payload = input.readNBytes(length) + check(payload.size == length) + if (masked) { + payload.indices.forEach { payload[it] = (payload[it].toInt() xor mask[it % 4].toInt()).toByte() } + } + return WireFrame(first and 0x0f, payload) + } + } +} diff --git a/src/test/kotlin/cash/atto/node/network/PeerNettyServerTest.kt b/src/test/kotlin/cash/atto/node/network/PeerNettyServerTest.kt new file mode 100644 index 00000000..d725f0cd --- /dev/null +++ b/src/test/kotlin/cash/atto/node/network/PeerNettyServerTest.kt @@ -0,0 +1,223 @@ +package cash.atto.node.network + +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.handler.codec.http.websocketx.BinaryWebSocketFrame +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.reactor.mono +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import reactor.netty.Connection +import reactor.netty.NettyPipeline +import reactor.netty.http.server.HttpServer +import reactor.netty.http.server.WebsocketServerSpec +import java.net.ServerSocket +import java.net.URI +import java.net.http.HttpClient +import java.net.http.WebSocket +import java.nio.ByteBuffer +import java.time.Duration +import java.util.concurrent.CompletableFuture +import java.util.concurrent.CompletionStage +import java.util.concurrent.TimeUnit +import kotlin.time.Duration.Companion.seconds + +class PeerNettyServerTest { + @Test + fun `installs aggregation before delayed session run receives fragmented frames`(): Unit = + runBlocking { + for (size in listOf(300, 301)) { + // Given + val expected = ByteArray(size) { it.toByte() } + val startApplication = CompletableDeferred() + val frameQueued = CompletableDeferred>() + val received = CompletableDeferred() + val listener = ClosingListener() + val server = + HttpServer + .create() + .host("127.0.0.1") + .port(0) + .route { routes -> + routes.ws( + "/", + { inbound, outbound -> + var connection: Connection? = null + inbound.withConnection { connection = it } + val transport = requireNotNull(connection) + val session = ReactorPeerSession(transport, inbound, outbound) + transport.channel().pipeline().addBefore( + NettyPipeline.ReactiveBridge, + "queued-binary-observer", + object : ChannelInboundHandlerAdapter() { + override fun channelRead( + context: ChannelHandlerContext, + message: Any, + ) { + val binary = + if (message is BinaryWebSocketFrame) { + message.content().readableBytes() to message.isFinalFragment + } else { + null + } + context.fireChannelRead(message) + binary?.let { frameQueued.complete(it) } + } + }, + ) + mono { + startApplication.await() + session.run { received.complete(it.incoming.first()) } + }.then() + }, + WebsocketServerSpec + .builder() + .maxFramePayloadLength(PeerSession.MAX_MESSAGE_SIZE) + .handlePing(false) + .build(), + ) + }.bindNow(Duration.ofSeconds(5)) + try { + HttpClient.newHttpClient().use { client -> + val socket = client.connect(server.port(), listener) + try { + // When: fragments reach the transport before the application subscribes. + socket.sendBinary(ByteBuffer.wrap(expected.copyOfRange(0, 150)), false).get(5, TimeUnit.SECONDS) + socket.sendBinary(ByteBuffer.wrap(expected.copyOfRange(150, size)), true).get(5, TimeUnit.SECONDS) + if (size == 300) { + // Observe the same frame dispatch; a later Ping can stall without an inbound subscriber. + assertEquals(300 to true, withTimeout(5.seconds) { frameQueued.await() }) + } else { + withTimeout(5.seconds) { listener.closed.await() } + } + + // Then + assertFalse(received.isCompleted) + startApplication.complete(Unit) + if (size == 300) { + assertArrayEquals(expected, withTimeout(5.seconds) { received.await() }) + } else { + assertFalse(received.isCompleted, "Oversized queued fragments must not reach the application") + } + } finally { + socket.abort() + } + } + } finally { + startApplication.complete(Unit) + server.disposeNow(Duration.ofSeconds(5)) + } + } + } + + @Test + fun `accepts fragmented 300 byte binary with interleaved controls and ignores text`(): Unit = + runBlocking { + // Given + val expected = ByteArray(300) { it.toByte() } + val received = CompletableDeferred() + val port = ServerSocket(0).use { it.localPort } + PeerNettyServer( + port, + handshake = { _, _ -> }, + webSocket = { _, session -> received.complete(session.incoming.first()) }, + ).use { + HttpClient.newHttpClient().use { client -> + val socket = client.connect(port, ClosingListener()) + try { + // When + socket.sendText("ignored text", true).get(5, TimeUnit.SECONDS) + socket.sendBinary(ByteBuffer.wrap(expected.copyOfRange(0, 150)), false).get(5, TimeUnit.SECONDS) + socket.sendPing(ByteBuffer.wrap(byteArrayOf(1))).get(5, TimeUnit.SECONDS) + socket.sendPong(ByteBuffer.wrap(byteArrayOf(2))).get(5, TimeUnit.SECONDS) + socket.sendBinary(ByteBuffer.wrap(expected.copyOfRange(150, 300)), true).get(5, TimeUnit.SECONDS) + + // Then + assertArrayEquals(expected, withTimeout(5.seconds) { received.await() }) + } finally { + socket.abort() + } + } + } + } + + @Test + fun `closes single and fragmented binary above 300 bytes without delivering it`(): Unit = + runBlocking { + for (fragmented in listOf(false, true)) { + // Given + val messages = mutableListOf() + val applicationStopped = CompletableDeferred() + val listener = ClosingListener() + val port = ServerSocket(0).use { it.localPort } + PeerNettyServer( + port, + handshake = { _, _ -> }, + webSocket = { _, session -> + try { + session.incoming.toList(messages) + } finally { + applicationStopped.complete(Unit) + } + }, + ).use { + HttpClient.newHttpClient().use { client -> + val socket = client.connect(port, listener) + try { + // When + if (fragmented) { + socket.sendBinary(ByteBuffer.wrap(ByteArray(150)), false).get(5, TimeUnit.SECONDS) + socket.sendBinary(ByteBuffer.wrap(ByteArray(151)), true).get(5, TimeUnit.SECONDS) + } else { + socket.sendBinary(ByteBuffer.wrap(ByteArray(301)), true).get(5, TimeUnit.SECONDS) + } + withTimeout(5.seconds) { + listener.closed.await() + applicationStopped.await() + } + + // Then + assertTrue(messages.isEmpty(), "Oversized fragmented=$fragmented message must not be delivered") + } finally { + socket.abort() + } + } + } + } + } + + private fun HttpClient.connect( + port: Int, + listener: WebSocket.Listener, + ): WebSocket = + newWebSocketBuilder() + .buildAsync(URI("ws://127.0.0.1:$port/"), listener) + .get(5, TimeUnit.SECONDS) + + private class ClosingListener : WebSocket.Listener { + val closed = CompletableDeferred() + + override fun onClose( + webSocket: WebSocket, + statusCode: Int, + reason: String, + ): CompletionStage<*> { + closed.complete(Unit) + return CompletableFuture.completedFuture(null) + } + + override fun onError( + webSocket: WebSocket, + error: Throwable, + ) { + closed.complete(Unit) + } + } +} diff --git a/src/test/kotlin/cash/atto/node/network/PeerNettyTlsTest.kt b/src/test/kotlin/cash/atto/node/network/PeerNettyTlsTest.kt new file mode 100644 index 00000000..76cd413d --- /dev/null +++ b/src/test/kotlin/cash/atto/node/network/PeerNettyTlsTest.kt @@ -0,0 +1,313 @@ +package cash.atto.node.network + +import cash.atto.commons.AttoAlgorithm +import cash.atto.commons.AttoHash +import cash.atto.commons.AttoInstant +import cash.atto.commons.AttoNetwork +import cash.atto.commons.AttoPrivateKey +import cash.atto.commons.AttoSignature +import cash.atto.commons.toSignerBlocking +import cash.atto.protocol.AttoNode +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.InputStream +import java.net.InetAddress +import java.net.Socket +import java.net.SocketException +import java.net.URI +import java.nio.file.Path +import java.security.KeyStore +import java.security.MessageDigest +import java.util.Base64 +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import javax.net.ssl.KeyManagerFactory +import javax.net.ssl.SSLContext +import javax.net.ssl.SSLException +import javax.net.ssl.SSLHandshakeException +import javax.net.ssl.SSLServerSocket +import kotlin.io.path.inputStream +import kotlin.io.path.readText +import kotlin.time.Duration.Companion.seconds + +class PeerNettyTlsTest { + @TempDir + lateinit var directory: Path + + @Test + fun `WSS and HTTPS verify certificates and hostname using the JVM trust store`() { + // Given + val keytool = Path.of(System.getProperty("java.home"), "bin", "keytool").toString() + for (name in listOf("trusted", "untrusted")) { + command( + name, + listOf( + keytool, + "-genkeypair", + "-alias", + name, + "-keyalg", + "RSA", + "-keysize", + "2048", + "-dname", + "CN=localhost", + "-ext", + "SAN=dns:localhost", + "-validity", + "2", + "-keystore", + directory.resolve("$name.p12").toString(), + "-storetype", + "PKCS12", + "-storepass", + "atto-test", + "-keypass", + "atto-test", + "-noprompt", + ), + ) + } + command( + "export", + listOf( + keytool, + "-exportcert", + "-alias", + "trusted", + "-keystore", + directory.resolve("trusted.p12").toString(), + "-storepass", + "atto-test", + "-file", + directory.resolve("trusted.cer").toString(), + ), + ) + command( + "import", + listOf( + keytool, + "-importcert", + "-alias", + "trusted", + "-keystore", + directory.resolve("trust.p12").toString(), + "-storetype", + "PKCS12", + "-storepass", + "atto-test", + "-file", + directory.resolve("trusted.cer").toString(), + "-noprompt", + ), + ) + val classpath = + checkNotNull(System.getProperty("atto.test.runtime-classpath")) { + "The isolated TLS probe needs the test runtime classpath supplied by Gradle" + } + assertTrue(classpath.isNotEmpty(), "The isolated TLS probe needs the actual test runtime classpath") + + // When: use a fresh JVM so Reactor's cached default TLS context sees this trust store before initialization. + val output = + command( + "probe", + listOf( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-Djavax.net.ssl.trustStore=${directory.resolve("trust.p12")}", + "-Djavax.net.ssl.trustStorePassword=atto-test", + "-Djavax.net.ssl.trustStoreType=PKCS12", + "-cp", + classpath, + PeerNettyTlsProbe::class.java.name, + directory.toString(), + ), + ) + + // Then + assertTrue(output.contains("Verified WSS and HTTPS trust and hostname checks"), output) + } + + private fun command( + name: String, + arguments: List, + ): String { + val log = directory.resolve("$name.log") + val process = ProcessBuilder(arguments).redirectErrorStream(true).redirectOutput(log.toFile()).start() + if (!process.waitFor(45, TimeUnit.SECONDS)) { + process.destroyForcibly().waitFor(5, TimeUnit.SECONDS) + error("TLS test command $name timed out: ${log.readText()}") + } + val output = log.readText() + assertEquals(0, process.exitValue(), "TLS test command $name failed: $output") + return output + } +} + +/** Executed in an isolated JVM; no trust settings or SSL contexts are changed in the Gradle test worker. */ +object PeerNettyTlsProbe { + private val json = Json { encodeDefaults = true } + + @JvmStatic + fun main(arguments: Array) = + runBlocking { + val directory = Path.of(arguments.single()) + val node = + AttoNode( + network = AttoNetwork.LOCAL, + protocolVersion = AttoNode.CURRENT_PROTOCOL_VERSION, + algorithm = AttoAlgorithm.V1, + publicKey = AttoPrivateKey.generate().toSignerBlocking().publicKey, + publicUri = URI("wss://localhost:7070"), + features = emptySet(), + ) + val request = + CounterChallengeResponse("00", AttoHash(ByteArray(32)), node, AttoInstant.now(), AttoSignature(ByteArray(64)), "00") + val response = ChallengeResponse(node, request.timestamp, request.signature) + val resolver = NetworkDnsResolver() + val client = PeerNettyClient(resolver) + try { + val cases = + listOf( + Triple("trusted", "localhost", true), + Triple("trusted", "127.0.0.1", false), + Triple("untrusted", "localhost", false), + ) + for ((store, host, accepted) in cases) { + TlsPeer(directory.resolve("$store.p12"), json.encodeToString(response)).use { server -> + val websocket = + withTimeout(10.seconds) { + runCatching { + client.webSocket(URI("wss://$host:${server.port}/"), emptyMap()) { session, address -> + check(address.port == server.port) + check(session.incoming.first().contentEquals(byteArrayOf(42))) + } + } + } + val callback = + withTimeout(10.seconds) { + runCatching { client.postHandshake(URI("https://$host:${server.port}/handshakes"), request) } + } + check(websocket.isSuccess == accepted) { "WSS $store/$host: ${websocket.exceptionOrNull()}" } + check(callback.isSuccess == accepted) { "HTTPS $store/$host: ${callback.exceptionOrNull()}" } + if (accepted) { + check(callback.getOrThrow().response == response) + } else { + check(websocket.exceptionOrNull().isTlsRejection()) { "WSS failed outside TLS verification: $websocket" } + check(callback.exceptionOrNull().isTlsRejection()) { "HTTPS failed outside TLS verification: $callback" } + } + } + } + } finally { + client.close() + resolver.close() + } + println("Verified WSS and HTTPS trust and hostname checks") + } + + private fun Throwable?.isTlsRejection(): Boolean = generateSequence(this) { it.cause }.any { it is SSLHandshakeException } + + private class TlsPeer( + store: Path, + private val callbackBody: String, + ) : AutoCloseable { + private val executor = Executors.newVirtualThreadPerTaskExecutor() + private val sockets = ConcurrentHashMap.newKeySet() + private val server: SSLServerSocket + val port: Int get() = server.localPort + + init { + val keyStore = KeyStore.getInstance("PKCS12").apply { store.inputStream().use { load(it, "atto-test".toCharArray()) } } + val managers = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()).apply { init(keyStore, "atto-test".toCharArray()) } + val context = SSLContext.getInstance("TLS").apply { init(managers.keyManagers, null, null) } + server = context.serverSocketFactory.createServerSocket(0, 50, InetAddress.getByName("127.0.0.1")) as SSLServerSocket + executor.submit { + while (!server.isClosed) { + val socket = + try { + server.accept() + } catch (_: SocketException) { + break + } + sockets.add(socket) + executor.submit { + socket.use { + try { + socket.soTimeout = 10_000 + respond(socket) + } catch (_: SSLException) { + // Untrusted and mismatched certificate cases terminate during TLS negotiation. + } catch (_: SocketException) { + // Client shutdown may reset a successfully upgraded connection. + } finally { + sockets.remove(socket) + } + } + } + } + } + } + + private fun respond(socket: Socket) { + val input = socket.getInputStream() + val request = input.readLine() + val headers = mutableMapOf() + while (true) { + val line = input.readLine() + if (line.isEmpty()) break + headers[line.substringBefore(':').lowercase()] = line.substringAfter(':').trim() + } + val output = socket.getOutputStream() + if (request.startsWith("POST ")) { + val length = checkNotNull(headers["content-length"]).toInt() + check(input.readNBytes(length).size == length) + val bytes = callbackBody.encodeToByteArray() + output.write( + ( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + + "Content-Length: ${bytes.size}\r\nConnection: close\r\n\r\n" + ).toByteArray(), + ) + output.write(bytes) + } else { + val key = checkNotNull(headers["sec-websocket-key"]) + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + val accept = Base64.getEncoder().encodeToString(MessageDigest.getInstance("SHA-1").digest(key.toByteArray())) + output.write( + ( + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\n" + + "Connection: Upgrade\r\nSec-WebSocket-Accept: $accept\r\n\r\n" + ).toByteArray(), + ) + output.write(byteArrayOf(0x82.toByte(), 1, 42)) + } + output.flush() + if (!request.startsWith("POST ")) input.read() + } + + override fun close() { + server.close() + sockets.forEach { it.close() } + executor.shutdownNow() + } + } + + private fun InputStream.readLine(): String { + val bytes = ArrayList() + while (true) { + val value = read() + check(value >= 0) { "Unexpected EOF reading TLS peer request" } + if (value == 10) break + bytes.add(value.toByte()) + check(bytes.size < 16_384) + } + return bytes.toByteArray().decodeToString().removeSuffix("\r") + } +} diff --git a/src/test/kotlin/cash/atto/node/network/ReactorPeerSessionTest.kt b/src/test/kotlin/cash/atto/node/network/ReactorPeerSessionTest.kt new file mode 100644 index 00000000..a4ccdc00 --- /dev/null +++ b/src/test/kotlin/cash/atto/node/network/ReactorPeerSessionTest.kt @@ -0,0 +1,97 @@ +package cash.atto.node.network + +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.async +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import reactor.netty.Connection +import reactor.netty.NettyOutbound +import reactor.netty.http.websocket.WebsocketInbound +import reactor.netty.http.websocket.WebsocketOutbound + +@OptIn(ExperimentalCoroutinesApi::class) +class ReactorPeerSessionTest { + @Test + fun `close returns while writer is blocked and disposes socket after five seconds`() = + runTest { + // Given + val writeStarted = CompletableDeferred() + val closeReturned = CompletableDeferred() + val releaseApplication = CompletableDeferred() + val connection = mockk(relaxed = true) + val inbound = mockk() + every { inbound.aggregateFrames(PeerSession.MAX_MESSAGE_SIZE) } returns inbound + every { inbound.receiveFrames() } returns Flux.never() + val blockedWrite = mockk() + every { blockedWrite.then() } returns Mono.never().doOnSubscribe { writeStarted.complete(Unit) } + val outbound = mockk() + every { outbound.send(any()) } returns blockedWrite + every { outbound.sendClose(any(), any()) } returns Mono.empty() + val session = ReactorPeerSession(connection, inbound, outbound) + val managed = + async { + runCatching { + session.run { peer -> + peer.send(byteArrayOf(1)) + writeStarted.await() + peer.close() + closeReturned.complete(Unit) + releaseApplication.await() + } + } + } + + try { + // When + runCurrent() + + // Then: graceful close is queued behind the accepted write without blocking its caller. + assertTrue(writeStarted.isCompleted) + assertTrue(closeReturned.isCompleted) + assertFalse(managed.isCompleted) + assertEquals(0L, testScheduler.currentTime) + verify(exactly = 0) { outbound.sendClose(any(), any()) } + verify(exactly = 0) { connection.dispose() } + + // When + advanceTimeBy(4_999) + runCurrent() + + // Then + verify(exactly = 0) { connection.dispose() } + + // When + advanceTimeBy(1) + runCurrent() + + // Then + assertEquals(5_000L, testScheduler.currentTime) + verify(exactly = 1) { connection.dispose() } + verify(exactly = 0) { outbound.sendClose(any(), any()) } + + // When + releaseApplication.complete(Unit) + runCurrent() + + // Then: inspect the worker outcome so an unexpected failure cannot disappear in a child coroutine. + assertInstanceOf(CancellationException::class.java, managed.await().exceptionOrNull()) + } finally { + releaseApplication.complete(Unit) + managed.cancelAndJoin() + } + } +} From 159e63181de04ff2edaf56f0d2ff0bec584d6e25 Mon Sep 17 00:00:00 2001 From: rotilho Date: Sun, 8 Nov 2026 19:59:00 +0100 Subject: [PATCH 2/5] Replace remote signer transport with Reactor Netty --- NETWORK.md | 16 + README.md | 5 +- build.gradle.kts | 12 - .../atto/node/signature/MeteredAttoSigner.kt | 9 +- .../atto/node/signature/NettyRemoteSigner.kt | 164 ++++++++++ .../signature/RemoteSignerHttpTransport.kt | 160 +++++++++ .../node/signature/SignatureConfiguration.kt | 5 +- .../node/network/PeerHttpIsolationTest.kt | 148 +++++++++ .../node/signature/NettyRemoteSignerTest.kt | 308 ++++++++++++++++++ .../cash/atto/node/signature/SignerTest.kt | 147 +++++---- 10 files changed, 895 insertions(+), 79 deletions(-) create mode 100644 src/main/kotlin/cash/atto/node/signature/NettyRemoteSigner.kt create mode 100644 src/main/kotlin/cash/atto/node/signature/RemoteSignerHttpTransport.kt create mode 100644 src/test/kotlin/cash/atto/node/network/PeerHttpIsolationTest.kt create mode 100644 src/test/kotlin/cash/atto/node/signature/NettyRemoteSignerTest.kt diff --git a/NETWORK.md b/NETWORK.md index 6b63b5d7..c8c25d77 100644 --- a/NETWORK.md +++ b/NETWORK.md @@ -45,6 +45,20 @@ body readers in [HandshakePayloadLimit][payload-limit]. Cancellation propagates Early HTTP rejection closes the connection after writing its status, so unread request content cannot leave a reusable connection stalled. WebSocket authentication runs after upgrade; a rejected attempt terminates without registering a peer. +### Remote Signing + +When `atto.signer.backend=REMOTE`, [NettyRemoteSigner][remote-signer] calls the existing signer's `public-keys`, `blocks`, +`votes` and `challenges` endpoints. It preserves the configured authorization token, lazy public-key cache, signature checks, +one-second retry delay and socket timeouts (five seconds for the public key, one second for signing). HTTP 408 and 429 are +retried, as are transport and decoding failures; other HTTP errors terminate the operation. Cancellation stops retrying. + +The signer owns JSON, retries and signature validation. Its [HTTP transport][signer-transport] owns connection and operation +admission, shutdown, and separate `atto-signer-io` event loops, connection provider and `atto-signer-work` virtual-thread dispatcher. +The whole signing operation runs on that dispatcher, including codecs, retry waits and signature validation. +Peer server/client I/O uses separate `atto-peer-server-io` and `atto-peer-client-io` groups. Each transport closes only its owned +resources. Virtual-thread executors have separate lifecycles but share the JVM's carrier scheduler. Local signing and signer +metrics retain their existing interfaces. The node runtime does not depend on Ktor. + ### Outbound Connections `bootstrap()` runs every second and tries the configured default nodes. An inbound `AttoKeepAlive` can also supply a neighbour @@ -162,6 +176,8 @@ transport, feature, traffic-accounting or metrics listener to finish. [peer-client]: src/main/kotlin/cash/atto/node/network/PeerNettyClient.kt [peer-session]: src/main/kotlin/cash/atto/node/network/PeerSession.kt [peer-reactor-session]: src/main/kotlin/cash/atto/node/network/ReactorPeerSession.kt +[remote-signer]: src/main/kotlin/cash/atto/node/signature/NettyRemoteSigner.kt +[signer-transport]: src/main/kotlin/cash/atto/node/signature/RemoteSignerHttpTransport.kt [connections]: src/main/kotlin/cash/atto/node/network/NodeConnectionManager.kt [connected-peers]: src/main/kotlin/cash/atto/node/network/ConnectedPeers.kt [guardian]: src/main/kotlin/cash/atto/node/network/guardian/Guardian.kt diff --git a/README.md b/README.md index a512c93f..03005689 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,9 @@ Atto Node is the network-facing service for the **Atto** cryptocurrency. It expo submitting/querying blocks (transactions), maintains account state in a MySQL database, connects to other nodes over WebSockets, and participates in consensus by producing and relaying votes. -This repository is a **Kotlin + Spring Boot (WebFlux)** application with a dedicated **Reactor Netty WebSocket/HTTP server** -for node-to-node communication. +This repository is a **Kotlin + Spring Boot (WebFlux)** application with dedicated **Reactor Netty WebSocket/HTTP transports** +for node-to-node communication and remote signing. Peer server, peer client and signer I/O resources are isolated from each +other and from the public HTTP API. See [Network Layer](NETWORK.md) for transport ownership and authentication. [Website](https://atto.cash/) | [Docs](https://atto.cash/docs/integration) | [Commons](https://github.com/attocash/commons) diff --git a/build.gradle.kts b/build.gradle.kts index a37c11e2..7b7386d6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -56,12 +56,10 @@ dependencies { val commonsVersion = "8.0.0" val cucumberVersion = "7.34.8" val springdocVersion = "3.1.1" - val ktorVersion = "3.6.0" implementation("cash.atto:commons-core:$commonsVersion") implementation("cash.atto:commons-node:$commonsVersion") implementation("cash.atto:commons-worker:$commonsVersion") - implementation("cash.atto:commons-signer-remote:$commonsVersion") implementation("cash.atto:commons-spring-boot-starter:$commonsVersion") implementation(platform(org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES)) @@ -76,16 +74,6 @@ dependencies { implementation("org.springframework.boot:spring-boot-starter-webflux") annotationProcessor("org.springframework.boot:spring-boot-configuration-processor") - implementation("io.ktor:ktor-client-cio:$ktorVersion") - implementation("io.ktor:ktor-server-cio:$ktorVersion") - implementation("io.ktor:ktor-serialization-kotlinx-json:$ktorVersion") - - implementation("io.ktor:ktor-server-websockets:$ktorVersion") - implementation("io.ktor:ktor-server-content-negotiation:$ktorVersion") - - implementation("io.ktor:ktor-client-websockets:$ktorVersion") - implementation("io.ktor:ktor-client-content-negotiation:$ktorVersion") - implementation("org.springdoc:springdoc-openapi-starter-webflux-ui:$springdocVersion") implementation("com.github.ben-manes.caffeine:caffeine") diff --git a/src/main/kotlin/cash/atto/node/signature/MeteredAttoSigner.kt b/src/main/kotlin/cash/atto/node/signature/MeteredAttoSigner.kt index e089a7fa..1a247066 100644 --- a/src/main/kotlin/cash/atto/node/signature/MeteredAttoSigner.kt +++ b/src/main/kotlin/cash/atto/node/signature/MeteredAttoSigner.kt @@ -12,6 +12,7 @@ import cash.atto.commons.AttoSigner import cash.atto.commons.AttoVote import io.micrometer.core.instrument.MeterRegistry import io.micrometer.core.instrument.Timer +import jakarta.annotation.PreDestroy import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit import kotlin.coroutines.cancellation.CancellationException @@ -19,7 +20,8 @@ import kotlin.coroutines.cancellation.CancellationException internal class MeteredAttoSigner( internal val delegate: AttoSigner, private val meterRegistry: MeterRegistry, -) : AttoSigner { +) : AttoSigner, + AutoCloseable { private val timers = ConcurrentHashMap() init { @@ -89,6 +91,11 @@ internal class MeteredAttoSigner( } } + @PreDestroy + override fun close() { + (delegate as? AutoCloseable)?.close() + } + private fun timer( operation: String, outcome: String, diff --git a/src/main/kotlin/cash/atto/node/signature/NettyRemoteSigner.kt b/src/main/kotlin/cash/atto/node/signature/NettyRemoteSigner.kt new file mode 100644 index 00000000..2fd1c2a7 --- /dev/null +++ b/src/main/kotlin/cash/atto/node/signature/NettyRemoteSigner.kt @@ -0,0 +1,164 @@ +package cash.atto.node.signature + +import cash.atto.commons.AttoAddress +import cash.atto.commons.AttoAlgorithm +import cash.atto.commons.AttoBlock +import cash.atto.commons.AttoChallenge +import cash.atto.commons.AttoHash +import cash.atto.commons.AttoInstant +import cash.atto.commons.AttoPublicKey +import cash.atto.commons.AttoSignature +import cash.atto.commons.AttoSigner +import cash.atto.commons.AttoVote +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlin.time.Duration.Companion.seconds + +/** Remote signing has its own I/O loops and virtual threads, including retries and JSON/cryptographic work. */ +class NettyRemoteSigner( + url: String, + token: String, +) : AttoSigner, + AutoCloseable { + private val logger = KotlinLogging.logger {} + private val json = Json { ignoreUnknownKeys = true } + private val transport = RemoteSignerHttpTransport(url, token) + + override val algorithm: AttoAlgorithm = AttoAlgorithm.V1 + override val publicKey: AttoPublicKey by lazy { + runBlocking { + transport.operation { + retrying("get publicKey") { request("public-keys", null, 5).publicKey } + } + } + } + override val address: AttoAddress by lazy { AttoAddress(algorithm, publicKey) } + + override suspend fun sign(hash: AttoHash): AttoSignature = throw NotImplementedError("Remote signer doesn't accept direct hash signing") + + override suspend fun signMessage(message: ByteArray): AttoSignature = transport.operation { super.signMessage(message) } + + override suspend fun sign(block: AttoBlock): AttoSignature = + transport.operation { + checkPublicKey(block.publicKey) + val signature = + retrying("sign $block") { + request("blocks", json.encodeToString(BlockSignatureRequest(block)), 1).signature + } + if (!signature.isValid(block.publicKey, block.hash)) throw RemoteSignerInvalidSignatureException("block") + signature + } + + override suspend fun sign(vote: AttoVote): AttoSignature = + transport.operation { + checkPublicKey(vote.publicKey) + val signature = + retrying("sign $vote") { + request("votes", json.encodeToString(VoteSignatureRequest(vote)), 1).signature + } + if (!signature.isValid(vote.publicKey, vote.hash)) throw RemoteSignerInvalidSignatureException("vote") + signature + } + + override suspend fun sign( + challenge: AttoChallenge, + timestamp: AttoInstant, + ): AttoSignature = + transport.operation { + val signature = + retrying("sign $challenge") { + val payload = json.encodeToString(ChallengeSignatureRequest(challenge, timestamp)) + request("challenges", payload, 1).signature + } + if (!signature.isValid(publicKey, challenge, timestamp)) throw RemoteSignerInvalidSignatureException("challenge") + signature + } + + private suspend inline fun request( + path: String, + payload: String?, + timeoutSeconds: Long, + ): T { + val response = transport.request(path, payload?.encodeToByteArray(), timeoutSeconds) + if (response.status >= 300) throw RemoteSignerHttpException(response.status) + return json.decodeFromString(requireNotNull(response.body).decodeToString()) + } + + private suspend fun retrying( + operation: String, + action: suspend () -> T, + ): T { + while (true) { + currentCoroutineContext().ensureActive() + try { + return action() + } catch (exception: CancellationException) { + throw exception + } catch (exception: Exception) { + generateSequence( + exception as Throwable, + ) { it.cause }.filterIsInstance().firstOrNull()?.let { throw it } + val status = + generateSequence( + exception as Throwable, + ) { it.cause }.filterIsInstance().firstOrNull()?.status + // Commons 8 treats server errors as terminal too; only 408 and 429 are retried. + if (status != null && status >= 400 && status != 408 && status != 429) { + throw RemoteSignerTerminalException(operation, status, exception) + } + logger.warn(exception) { "Failed to $operation. Retrying in 1s" } + delay(1.seconds) + } + } + } + + override fun close() { + transport.close() + } + + @Serializable + private data class PublicKeyResponse( + val publicKey: AttoPublicKey, + ) + + @Serializable + private data class BlockSignatureRequest( + val target: AttoBlock, + ) + + @Serializable + private data class VoteSignatureRequest( + val target: AttoVote, + ) + + @Serializable + private data class ChallengeSignatureRequest( + val target: AttoChallenge, + val timestamp: AttoInstant, + ) + + @Serializable + private data class SignatureResponse( + val signature: AttoSignature, + ) + + private class RemoteSignerHttpException( + val status: Int, + ) : IllegalStateException("Remote signer returned HTTP $status") +} + +class RemoteSignerTerminalException( + operation: String, + val status: Int, + cause: Throwable, +) : IllegalStateException("Remote signer failed to $operation with terminal status $status", cause) + +class RemoteSignerInvalidSignatureException( + targetType: String, +) : IllegalStateException("Remote signer returned invalid $targetType signature") diff --git a/src/main/kotlin/cash/atto/node/signature/RemoteSignerHttpTransport.kt b/src/main/kotlin/cash/atto/node/signature/RemoteSignerHttpTransport.kt new file mode 100644 index 00000000..b6f12535 --- /dev/null +++ b/src/main/kotlin/cash/atto/node/signature/RemoteSignerHttpTransport.kt @@ -0,0 +1,160 @@ +package cash.atto.node.signature + +import cash.atto.node.JvmAddressResolverGroup +import io.netty.buffer.Unpooled +import io.netty.channel.ChannelOption +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpMethod +import kotlinx.coroutines.Job +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.reactor.awaitSingle +import kotlinx.coroutines.reactor.awaitSingleOrNull +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import reactor.core.publisher.Mono +import reactor.netty.Connection +import reactor.netty.http.HttpProtocol +import reactor.netty.http.client.HttpClient +import reactor.netty.resources.ConnectionProvider +import reactor.netty.resources.LoopResources +import java.net.URI +import java.time.Duration +import java.util.concurrent.Executors +import kotlin.time.Duration.Companion.seconds + +internal class RemoteSignerHttpTransport( + url: String, + token: String, +) : AutoCloseable { + private val baseUrl = url.trimEnd('/') + private val dispatcher = + Executors.newThreadPerTaskExecutor(Thread.ofVirtual().name("atto-signer-work-", 0).factory()).asCoroutineDispatcher() + private val loops = LoopResources.create("atto-signer-io", 2, true) + private val provider = + ConnectionProvider + .builder( + "atto-signer", + ).maxConnections(100) + .pendingAcquireMaxCount(1000) + .maxIdleTime(Duration.ofSeconds(5)) + .build() + private val lifecycle = Any() + private var closed = false + private val jobs = mutableSetOf() + private val connections = mutableSetOf() + private val resolver = JvmAddressResolverGroup(dispatcher) + private val client = + HttpClient + .create(provider) + .runOn(loops) + .resolver(resolver) + .protocol(HttpProtocol.HTTP11) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 5000) + .disableRetry(true) + .compress(false) + .headers { + it.set(HttpHeaderNames.AUTHORIZATION, token) + it.set(HttpHeaderNames.CONTENT_TYPE, "application/json") + it.set(HttpHeaderNames.ACCEPT, "application/json") + }.followRedirect { request, response -> + val location = response.responseHeaders().get(HttpHeaderNames.LOCATION) + if (request.method() != HttpMethod.GET || + response.status().code() !in REDIRECT_STATUSES || + location == null || + request.redirectedFrom().size >= 19 + ) { + false + } else { + val source = URI(requireNotNull(request.resourceUrl())) + val target = source.resolve(location) + target.scheme in setOf("http", "https") && !(source.scheme == "https" && target.scheme != "https") + } + }.doOnConnected { connection -> + val accepted = synchronized(lifecycle) { if (closed) false else connections.add(connection) } + if (accepted) { + connection.onDispose().doFinally { synchronized(lifecycle) { connections.remove(connection) } }.subscribe() + } else { + connection.dispose() + } + } + + /** Covers retries, codecs and signature validation as well as their HTTP requests. */ + suspend fun operation(action: suspend () -> T): T = + coroutineScope { + val job = currentCoroutineContext()[Job]!! + synchronized(lifecycle) { + check(!closed) { "Remote signer is closed" } + jobs.add(job) + } + try { + withContext(dispatcher) { action() } + } finally { + synchronized(lifecycle) { jobs.remove(job) } + } + } + + suspend fun request( + path: String, + payload: ByteArray?, + timeoutSeconds: Long, + ): Response { + val configured = client.responseTimeout(Duration.ofSeconds(timeoutSeconds)) + val request = + if (payload == null) { + configured.get().uri("$baseUrl/$path") + } else { + configured.post().uri("$baseUrl/$path").send(Mono.fromSupplier { Unpooled.wrappedBuffer(payload) }) + } + return request + .response { response, content -> + val status = response.status().code() + if (status >= 300) { + // Return the status immediately; an error body may never finish. + Mono.just(Response(status, null)) + } else { + content.aggregate().asByteArray().map { Response(status, it) } + } + }.single() + .awaitSingle() + } + + override fun close() { + val snapshot = + synchronized(lifecycle) { + if (closed) return + closed = true + jobs.toList() to connections.toList() + } + snapshot.first.forEach { it.cancel() } + snapshot.second.forEach { it.dispose() } + try { + runBlocking { + withTimeoutOrNull(5.seconds) { + snapshot.first.joinAll() + snapshot.second.forEach { it.onDispose().awaitSingleOrNull() } + } + } + provider.disposeLater().block(Duration.ofSeconds(5)) + } finally { + resolver.close() + try { + loops.disposeLater(Duration.ZERO, Duration.ofSeconds(5)).block(Duration.ofSeconds(6)) + } finally { + dispatcher.close() + } + } + } + + class Response( + val status: Int, + val body: ByteArray?, + ) + + private companion object { + val REDIRECT_STATUSES = setOf(301, 302, 303, 307, 308) + } +} diff --git a/src/main/kotlin/cash/atto/node/signature/SignatureConfiguration.kt b/src/main/kotlin/cash/atto/node/signature/SignatureConfiguration.kt index 0cb8369b..e4b0d79b 100644 --- a/src/main/kotlin/cash/atto/node/signature/SignatureConfiguration.kt +++ b/src/main/kotlin/cash/atto/node/signature/SignatureConfiguration.kt @@ -3,7 +3,6 @@ package cash.atto.node.signature import cash.atto.commons.AttoPrivateKey import cash.atto.commons.AttoSigner import cash.atto.commons.fromHexToByteArray -import cash.atto.commons.node.remote import cash.atto.commons.toHex import io.github.oshai.kotlinlogging.KotlinLogging import io.micrometer.core.instrument.MeterRegistry @@ -22,9 +21,7 @@ class SignatureConfiguration { ): AttoSigner { val signer = if (signerProperties.backend == SignerProperties.Backend.REMOTE) { - AttoSigner.remote(signerProperties.remoteUrl!!) { - mapOf("Authorization" to signerProperties.token!!) - } + NettyRemoteSigner(signerProperties.remoteUrl!!, signerProperties.token!!) } else { val privateKey = if (!signerProperties.key.isNullOrEmpty()) { diff --git a/src/test/kotlin/cash/atto/node/network/PeerHttpIsolationTest.kt b/src/test/kotlin/cash/atto/node/network/PeerHttpIsolationTest.kt new file mode 100644 index 00000000..11c767f9 --- /dev/null +++ b/src/test/kotlin/cash/atto/node/network/PeerHttpIsolationTest.kt @@ -0,0 +1,148 @@ +package cash.atto.node.network + +import cash.atto.commons.AttoAlgorithm +import cash.atto.commons.AttoChallenge +import cash.atto.commons.AttoHash +import cash.atto.commons.AttoInstant +import cash.atto.commons.AttoNetwork +import cash.atto.commons.AttoPrivateKey +import cash.atto.commons.AttoSignature +import cash.atto.commons.toSignerBlocking +import cash.atto.node.network.guardian.Guardian +import cash.atto.node.network.guardian.InboundConnectionDecision +import cash.atto.node.signature.NettyRemoteSigner +import cash.atto.node.signature.SignerTest.MockRemoteSigner +import cash.atto.node.time.InstantController +import cash.atto.node.transaction.Transaction +import cash.atto.protocol.AttoNode +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.springframework.boot.reactor.netty.NettyReactiveWebServerFactory +import org.springframework.boot.test.context.TestConfiguration +import org.springframework.context.annotation.AnnotationConfigApplicationContext +import org.springframework.context.annotation.Import +import org.springframework.mock.env.MockEnvironment +import org.springframework.web.reactive.config.EnableWebFlux +import org.springframework.web.server.adapter.WebHttpHandlerBuilder +import reactor.netty.http.HttpResources +import java.net.ServerSocket +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.time.Duration + +class PeerHttpIsolationTest { + @Test + fun `stopping peer listeners and clients leaves the Spring controller operational`() = + runBlocking { + // Given + val context = AnnotationConfigApplicationContext(HttpConfiguration::class.java) + val httpServer = NettyReactiveWebServerFactory(0).getWebServer(WebHttpHandlerBuilder.applicationContext(context).build()) + val dnsResolver = NetworkDnsResolver() + val peerClient = PeerNettyClient(dnsResolver) + val referenceSigner = MockRemoteSigner() + referenceSigner.start() + val remoteSigner = NettyRemoteSigner("http://127.0.0.1:${referenceSigner.port}", "isolation-test") + val signer = AttoPrivateKey.generate().toSignerBlocking() + val peerPort = ServerSocket(0).use { it.localPort } + val node = + AttoNode( + network = AttoNetwork.LOCAL, + protocolVersion = AttoNode.CURRENT_PROTOCOL_VERSION, + algorithm = AttoAlgorithm.V1, + publicKey = signer.publicKey, + publicUri = URI("ws://127.0.0.1:$peerPort"), + features = emptySet(), + ) + val connectionManager = NodeConnectionManager(node, mockk(relaxed = true), mockk(relaxed = true)) + val genesisHash = AttoHash(ByteArray(32) { 1 }) + val processor = + NetworkProcessor( + genesisTransaction = mockk { every { hash } returns genesisHash }, + thisNode = node, + signer = signer, + environment = MockEnvironment().withProperty("websocket.port", peerPort.toString()), + networkProperties = NetworkProperties(), + peerUriValidator = mockk(), + handshakeCallbackService = mockk(), + dnsResolver = dnsResolver, + connectionManager = connectionManager, + guardian = + mockk { + every { requestInboundConnection(any()) } returns InboundConnectionDecision.Accepted + }, + peerClient = peerClient, + ) + val client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build() + httpServer.start() + val apiUri = URI("http://127.0.0.1:${httpServer.port}") + val sharedResources = HttpResources.get() + try { + assertEquals(200, client.get(apiUri.resolve("/instants/2026-11-07T00:00:00Z"))) + assertEquals(404, client.get(URI("http://127.0.0.1:$peerPort/instants/2026-11-07T00:00:00Z"))) + val callback = + CounterChallengeResponse( + challenge = "00", + genesis = genesisHash, + node = node, + timestamp = AttoInstant.now(), + signature = AttoSignature(ByteArray(64)), + counterChallenge = "00", + ) + // Initialize the actual peer client against the separate HTTP server. + assertEquals(404, peerClient.postHandshake(apiUri.resolve("/handshakes"), callback).status.code()) + assertEquals(referenceSigner.signer.publicKey, remoteSigner.publicKey) + val ioThreads = Thread.getAllStackTraces().keys.filter { it.isAlive } + for (prefix in listOf("atto-peer-server-io", "atto-peer-client-io", "atto-signer-io")) { + val owned = ioThreads.filter { it.name.startsWith(prefix) } + assertFalse(owned.isEmpty(), "Expected active private I/O threads for $prefix") + assertFalse(owned.any { it.isVirtual }, "Netty socket I/O must stay on its private event loops") + } + + // When + processor.stop() + peerClient.close() + + // Then + assertFalse(sharedResources.isDisposed) + val challenge = AttoChallenge.generate() + val timestamp = AttoInstant.now() + assertTrue(remoteSigner.sign(challenge, timestamp).isValid(remoteSigner.publicKey, challenge, timestamp)) + assertEquals(200, client.get(apiUri.resolve("/instants/2026-11-07T00:00:00Z"))) + assertEquals(404, client.get(apiUri.resolve("/handshakes"))) + remoteSigner.close() + assertEquals(200, client.get(apiUri.resolve("/instants/2026-11-07T00:00:00Z"))) + } finally { + processor.stop() + peerClient.close() + connectionManager.stop() + dnsResolver.close() + remoteSigner.close() + referenceSigner.stop() + client.close() + httpServer.stop() + context.close() + } + } + + private fun HttpClient.get(uri: URI): Int = + send( + HttpRequest + .newBuilder(uri) + .timeout(Duration.ofSeconds(5)) + .GET() + .build(), + HttpResponse.BodyHandlers.discarding(), + ).statusCode() + + @TestConfiguration(proxyBeanMethods = false) + @EnableWebFlux + @Import(InstantController::class) + class HttpConfiguration +} diff --git a/src/test/kotlin/cash/atto/node/signature/NettyRemoteSignerTest.kt b/src/test/kotlin/cash/atto/node/signature/NettyRemoteSignerTest.kt new file mode 100644 index 00000000..6d97baea --- /dev/null +++ b/src/test/kotlin/cash/atto/node/signature/NettyRemoteSignerTest.kt @@ -0,0 +1,308 @@ +package cash.atto.node.signature + +import cash.atto.commons.AttoChallenge +import cash.atto.commons.AttoHash +import cash.atto.commons.AttoInstant +import cash.atto.commons.AttoPrivateKey +import cash.atto.commons.AttoSignature +import cash.atto.commons.toSignerBlocking +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.async +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.reactivestreams.Publisher +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import reactor.netty.http.server.HttpServer +import reactor.netty.http.server.HttpServerRequest +import reactor.netty.http.server.HttpServerResponse +import reactor.netty.resources.LoopResources +import java.time.Duration +import java.util.concurrent.atomic.AtomicInteger +import kotlin.time.Duration.Companion.seconds + +class NettyRemoteSignerTest { + private val local = AttoPrivateKey.generate().toSignerBlocking() + private val keyJson = Json.encodeToString(SignerTest.PublicKeyResponse(local.publicKey)) + + @Test + fun `public key is lazy cached accepts unknown fields and forwards exact authorization`() { + // Given + val requests = AtomicInteger() + SignerServer { request, response -> + requests.incrementAndGet() + assertEquals("opaque token", request.requestHeaders().get("Authorization")) + assertEquals("application/json", request.requestHeaders().get("Accept")) + assertEquals("/prefix/public-keys", request.uri()) + response.sendString(Mono.just(keyJson.dropLast(1) + ",\"future\":true}")) + }.use { server -> + NettyRemoteSigner("${server.url}/prefix/", "opaque token").use { signer -> + assertEquals(0, requests.get()) + + // When + val first = signer.publicKey + val second = signer.publicKey + + // Then + assertEquals(local.publicKey, first) + assertEquals(first, second) + assertEquals(1, requests.get()) + assertTrue(signer.address.publicKey == first) + } + } + } + + @Test + fun `retries timeout throttling and malformed JSON then caches the successful public key`() { + // Given + val requests = AtomicInteger() + SignerServer { _, response -> + when (requests.incrementAndGet()) { + 1 -> response.status(408).send() + 2 -> response.status(429).send() + 3 -> response.sendString(Mono.just("{")) + else -> response.sendString(Mono.just(keyJson)) + } + }.use { server -> + NettyRemoteSigner(server.url, "test").use { signer -> + // When + val key = signer.publicKey + + // Then + assertEquals(local.publicKey, key) + assertEquals(4, requests.get()) + } + } + } + + @Test + fun `client and server HTTP errors are terminal except 408 and 429`() { + // Given + for (status in listOf(400, 401, 403, 404, 500, 503)) { + val requests = AtomicInteger() + SignerServer { _, response -> + requests.incrementAndGet() + response.status(status).send() + }.use { server -> + NettyRemoteSigner(server.url, "test").use { signer -> + // When + val error = assertThrows(RemoteSignerTerminalException::class.java) { signer.publicKey } + + // Then + assertEquals(status, error.status) + assertEquals(1, requests.get()) + } + } + } + } + + @Test + fun `terminal HTTP status closes an unfinished error body without waiting or retrying`(): Unit = + runBlocking { + // Given + val requests = AtomicInteger() + val disconnected = CompletableDeferred() + SignerServer { request, response -> + requests.incrementAndGet() + request.withConnection { it.onDispose().doFinally { disconnected.complete(Unit) }.subscribe() } + response.status(401).sendString(Flux.concat(Mono.just("{"), Flux.never())) + }.use { server -> + NettyRemoteSigner(server.url, "test").use { signer -> + // When + val result = runCatching { withTimeout(5.seconds) { signer.sign(AttoChallenge.generate(), AttoInstant.now()) } } + + // Then + val error = result.exceptionOrNull() + assertTrue(error is RemoteSignerTerminalException, "$result") + assertEquals(401, (error as RemoteSignerTerminalException).status) + assertEquals(1, requests.get()) + withTimeout(5.seconds) { disconnected.await() } + } + } + } + + @Test + fun `rejects invalid signatures without retrying the accepted signature response`() = + runBlocking { + // Given + val signatures = AtomicInteger() + SignerServer { request, response -> + if (request.uri() == "/public-keys") { + response.sendString(Mono.just(keyJson)) + } else { + signatures.incrementAndGet() + response.sendString(Mono.just(Json.encodeToString(SignerTest.SignatureResponse(AttoSignature(ByteArray(64)))))) + } + }.use { server -> + NettyRemoteSigner(server.url, "test").use { signer -> + // When + val result = runCatching { withTimeout(5.seconds) { signer.sign(AttoChallenge.generate(), AttoInstant.now()) } } + + // Then + assertTrue(result.exceptionOrNull() is RemoteSignerInvalidSignatureException, "$result") + assertEquals(1, signatures.get()) + } + } + } + + @Test + fun `successful signing releases connections across more requests than the pool capacity`() = + runBlocking { + // Given + val challenge = AttoChallenge.generate() + val timestamp = AttoInstant.now() + val signature = local.sign(challenge, timestamp) + val signatureJson = Json.encodeToString(SignerTest.SignatureResponse(signature)) + val requests = AtomicInteger() + SignerServer { request, response -> + requests.incrementAndGet() + val body = if (request.uri() == "/public-keys") keyJson else signatureJson + request.receive().then(response.sendString(Mono.just(body)).then()) + }.use { server -> + NettyRemoteSigner(server.url, "test").use { signer -> + // When + withTimeout(15.seconds) { + repeat(128) { assertEquals(signature, signer.sign(challenge, timestamp)) } + } + + // Then + assertEquals(129, requests.get()) + } + } + } + + @Test + fun `read inactivity timeout closes the stalled response and retries signing`() = + runBlocking { + // Given + val challenge = AttoChallenge.generate() + val timestamp = AttoInstant.now() + val signature = local.sign(challenge, timestamp) + val signatureJson = Json.encodeToString(SignerTest.SignatureResponse(signature)) + val requests = AtomicInteger() + val disconnected = CompletableDeferred() + SignerServer { request, response -> + if (request.uri() == "/public-keys") { + response.sendString(Mono.just(keyJson)) + } else if (requests.incrementAndGet() == 1) { + request.withConnection { it.onDispose().doFinally { disconnected.complete(Unit) }.subscribe() } + response.sendString(Flux.concat(Mono.just("{"), Flux.never())) + } else { + response.sendString(Mono.just(signatureJson)) + } + }.use { server -> + NettyRemoteSigner(server.url, "test").use { signer -> + // When + val result = withTimeout(8.seconds) { signer.sign(challenge, timestamp) } + + // Then + assertEquals(signature, result) + withTimeout(5.seconds) { disconnected.await() } + assertEquals(2, requests.get()) + } + } + } + + @Test + fun `cancellation closes a stalled HTTP body and does not retry`() = + runBlocking { + // Given + val started = CompletableDeferred() + val disconnected = CompletableDeferred() + val requests = AtomicInteger() + SignerServer { request, response -> + requests.incrementAndGet() + request.withConnection { it.onDispose().doFinally { disconnected.complete(Unit) }.subscribe() } + response.sendString(Flux.concat(Mono.just("{").doOnNext { started.complete(Unit) }, Flux.never())) + }.use { server -> + NettyRemoteSigner(server.url, "test").use { signer -> + val signing = async { signer.sign(AttoChallenge.generate(), AttoInstant.now()) } + withTimeout(5.seconds) { started.await() } + + // When + signing.cancelAndJoin() + + // Then + withTimeout(5.seconds) { disconnected.await() } + assertTrue(signing.isCancelled) + assertEquals(1, requests.get()) + } + } + } + + @Test + fun `direct hashes and signed messages retain the unsupported remote API contract`() = + runBlocking { + // Given + val requests = AtomicInteger() + SignerServer { _, response -> + requests.incrementAndGet() + response.sendString(Mono.just(keyJson)) + }.use { server -> + NettyRemoteSigner(server.url, "test").use { signer -> + // When + val direct = runCatching { signer.sign(AttoHash(ByteArray(32))) } + val message = runCatching { signer.signMessage(byteArrayOf(1)) } + + // Then + assertTrue(direct.exceptionOrNull() is NotImplementedError) + assertTrue(message.exceptionOrNull() is NotImplementedError) + assertEquals(1, requests.get()) + } + } + } + + @Test + fun `closing metered signer cancels active signing and closes its owned transport`(): Unit = + runBlocking { + // Given + val started = CompletableDeferred() + SignerServer { _, response -> + response.sendString(Flux.concat(Mono.just("{").doOnNext { started.complete(Unit) }, Flux.never())) + }.use { server -> + val remote = NettyRemoteSigner(server.url, "test") + val signer = MeteredAttoSigner(remote, SimpleMeterRegistry()) + val signing = async { signer.sign(AttoChallenge.generate(), AttoInstant.now()) } + withTimeout(5.seconds) { started.await() } + + // When + signer.close() + signer.close() + withTimeout(5.seconds) { signing.join() } + + // Then + assertTrue(signing.isCancelled) + assertFalse(Thread.getAllStackTraces().keys.any { it.isAlive && it.name.startsWith("atto-signer-io") }) + assertThrows(IllegalStateException::class.java) { remote.publicKey } + } + } + + private class SignerServer( + handler: (HttpServerRequest, HttpServerResponse) -> Publisher, + ) : AutoCloseable { + private val loops = LoopResources.create("signer-test-server", 1, true) + private val server = + HttpServer + .create() + .host("127.0.0.1") + .port(0) + .runOn(loops) + .handle(handler) + .bindNow() + val url = "http://127.0.0.1:${server.port()}" + + override fun close() { + server.disposeNow() + loops.disposeLater(Duration.ZERO, Duration.ofSeconds(5)).block(Duration.ofSeconds(6)) + } + } +} diff --git a/src/test/kotlin/cash/atto/node/signature/SignerTest.kt b/src/test/kotlin/cash/atto/node/signature/SignerTest.kt index b11675ae..083270ed 100644 --- a/src/test/kotlin/cash/atto/node/signature/SignerTest.kt +++ b/src/test/kotlin/cash/atto/node/signature/SignerTest.kt @@ -12,37 +12,31 @@ import cash.atto.commons.AttoPublicKey import cash.atto.commons.AttoReceivable import cash.atto.commons.AttoReceiveBlock import cash.atto.commons.AttoSignature -import cash.atto.commons.AttoSigner import cash.atto.commons.AttoVersion import cash.atto.commons.AttoVote -import cash.atto.commons.node.remote import cash.atto.commons.toAttoAmount import cash.atto.commons.toAttoHeight import cash.atto.commons.toAttoVersion import cash.atto.commons.toSignerBlocking -import io.ktor.serialization.kotlinx.json.json -import io.ktor.server.application.install -import io.ktor.server.cio.CIO -import io.ktor.server.engine.embeddedServer -import io.ktor.server.plugins.contentnegotiation.ContentNegotiation -import io.ktor.server.request.receive -import io.ktor.server.response.respond -import io.ktor.server.routing.get -import io.ktor.server.routing.post -import io.ktor.server.routing.routing +import kotlinx.coroutines.reactor.mono import kotlinx.coroutines.runBlocking import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test -import java.net.ServerSocket +import reactor.core.publisher.Mono +import reactor.netty.DisposableServer +import reactor.netty.http.server.HttpServer +import reactor.netty.resources.LoopResources +import java.time.Duration import kotlin.random.Random class SignerTest { companion object { - private val signerPort = randomPort() - private val signerServer = MocktRemoteSigner(signerPort) + private val signerServer = MockRemoteSigner() @JvmStatic @BeforeAll @@ -55,17 +49,14 @@ class SignerTest { fun stop() { signerServer.stop() } - - private fun randomPort(): Int = - ServerSocket(0).use { - it.localPort - } } - val signer = - AttoSigner.remote("http://localhost:$signerPort") { - mapOf("Authorization" to "mock_test") - } + val signer = NettyRemoteSigner("http://localhost:${signerServer.port}", "mock_test") + + @AfterEach + fun closeSigner() { + signer.close() + } @Test fun `should sign block`(): Unit = @@ -145,49 +136,85 @@ class SignerTest { amount = 1000UL.toAttoAmount(), ) - class MocktRemoteSigner( - port: Int, - ) { + class MockRemoteSigner { val signer = AttoPrivateKey.generate().toSignerBlocking() - - val server = - embeddedServer(CIO, port = port) { - install(ContentNegotiation) { - json() - } - - routing { - get("/public-keys") { - val response = PublicKeyResponse(signer.publicKey) - call.respond(response) - } - - post("/blocks") { - val request = call.request.call.receive() - val signature = signer.sign(request.target) - call.respond(SignatureResponse(signature)) - } - - post("/votes") { - val request = call.request.call.receive() - val signature = signer.sign(request.target) - call.respond(SignatureResponse(signature)) - } - - post("/challenges") { - val request = call.request.call.receive() - val signature = signer.sign(request.target, request.timestamp) - call.respond(SignatureResponse(signature)) - } - } + private val loops = LoopResources.create("signer-contract-test-server", 1, true) + private val timeout = Duration.ofSeconds(5) + private val json = + Json { + encodeDefaults = true + isLenient = true + allowSpecialFloatingPointValues = true + allowStructuredMapKeys = true + prettyPrint = false + useArrayPolymorphism = false } + private lateinit var server: DisposableServer + + val port: Int + get() = server.port() fun start() { - server.start() + try { + server = + HttpServer + .create() + .host("localhost") + .port(0) + .runOn(loops) + .route { routes -> + routes.get("/public-keys") { _, response -> + response.header("Content-Type", "application/json").sendString( + Mono.fromSupplier { json.encodeToString(PublicKeyResponse(signer.publicKey)) }, + ) + } + routes.post("/blocks") { request, response -> + val body = + request.receive().aggregate().asString().flatMap { payload -> + mono { + val target = json.decodeFromString(payload).target + json.encodeToString(SignatureResponse(signer.sign(target))) + } + } + response.header("Content-Type", "application/json").sendString(body) + } + routes.post("/votes") { request, response -> + val body = + request.receive().aggregate().asString().flatMap { payload -> + mono { + val target = json.decodeFromString(payload).target + json.encodeToString(SignatureResponse(signer.sign(target))) + } + } + response.header("Content-Type", "application/json").sendString(body) + } + routes.post("/challenges") { request, response -> + val body = + request.receive().aggregate().asString().flatMap { payload -> + mono { + val target = json.decodeFromString(payload) + json.encodeToString(SignatureResponse(signer.sign(target.target, target.timestamp))) + } + } + response.header("Content-Type", "application/json").sendString(body) + } + }.bindNow(timeout) + } catch (exception: Exception) { + disposeLoops() + throw exception + } } fun stop() { - server.stop() + try { + server.disposeNow(timeout) + } finally { + disposeLoops() + } + } + + private fun disposeLoops() { + loops.disposeLater(Duration.ZERO, timeout).block(timeout.plusSeconds(1)) } } From 247e0bb9f5cfd01aa4a51817dcc27f25cde3e6d6 Mon Sep 17 00:00:00 2001 From: rotilho Date: Sun, 8 Nov 2026 19:59:27 +0100 Subject: [PATCH 3/5] Register node info metadata for native images --- src/main/kotlin/cash/atto/node/NodeInfoContributor.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/kotlin/cash/atto/node/NodeInfoContributor.kt b/src/main/kotlin/cash/atto/node/NodeInfoContributor.kt index 053fde8e..7b0df353 100644 --- a/src/main/kotlin/cash/atto/node/NodeInfoContributor.kt +++ b/src/main/kotlin/cash/atto/node/NodeInfoContributor.kt @@ -2,11 +2,14 @@ package cash.atto.node import cash.atto.protocol.AttoNode import kotlinx.coroutines.ExperimentalCoroutinesApi +import org.springframework.aot.hint.annotation.RegisterReflectionForBinding import org.springframework.boot.actuate.info.Info import org.springframework.boot.actuate.info.InfoContributor import org.springframework.stereotype.Component @ExperimentalCoroutinesApi +// Info's untyped details map hides this DTO from automatic response-type hints. +@RegisterReflectionForBinding(AttoNode::class) @Component class NodeInfoContributor( val thisNode: AttoNode, From cbe93aaef9d6d60ea3b899b7d2cc7acd0f3eafb0 Mon Sep 17 00:00:00 2001 From: rotilho Date: Sun, 8 Nov 2026 19:59:47 +0100 Subject: [PATCH 4/5] Verify compatibility against released node neighbours --- NETWORK.md | 3 +- README.md | 5 + build.gradle.kts | 3 + .../cash/atto/node/CucumberConfiguration.kt | 3 +- src/test/kotlin/cash/atto/node/Neighbour.kt | 2 + src/test/kotlin/cash/atto/node/NodeHolder.kt | 34 +++-- .../cash/atto/node/NodeStepDefinition.kt | 140 ++++++++++++++++-- .../node/network/peer/PeerStepDefinition.kt | 136 +++++++++++++++-- .../transaction/TransactionStepDefinition.kt | 4 +- .../UncheckedTransactionStepDefinition.kt | 13 +- src/test/resources/features/peering.feature | 18 ++- 11 files changed, 314 insertions(+), 47 deletions(-) diff --git a/NETWORK.md b/NETWORK.md index c8c25d77..ab8e5b15 100644 --- a/NETWORK.md +++ b/NETWORK.md @@ -57,7 +57,8 @@ admission, shutdown, and separate `atto-signer-io` event loops, connection provi The whole signing operation runs on that dispatcher, including codecs, retry waits and signature validation. Peer server/client I/O uses separate `atto-peer-server-io` and `atto-peer-client-io` groups. Each transport closes only its owned resources. Virtual-thread executors have separate lifecycles but share the JVM's carrier scheduler. Local signing and signer -metrics retain their existing interfaces. The node runtime does not depend on Ktor. +metrics retain their existing interfaces. The node runtime does not depend on Ktor. Cucumber's `live` neighbour mode checks +compatibility with the actual released node image, including connections initiated in either direction and reconnection. ### Outbound Connections diff --git a/README.md b/README.md index 03005689..cd684796 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,11 @@ On startup, the node will: - start the REST API - start the node-to-node WebSocket server +Run `./gradlew test` to test with neighbours built from the current checkout. For compatibility with the released node, run +`./gradlew test -Datto.test.neighbour-mode=live`. This mode pulls `ghcr.io/attocash/node:live` once, reuses the resolved image +throughout the run, and logs its digest and release metadata. It requires a local Linux container engine with host networking. +Released neighbours use their production APIs and scheduled bootstrap processing; test-only controls remain on classpath nodes. + ## Configuration Configuration is split across `application.yaml` and profile overrides such as: diff --git a/build.gradle.kts b/build.gradle.kts index 7b7386d6..f730465f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -125,6 +125,9 @@ dependencies { tasks.withType { useJUnitPlatform() maxHeapSize = "4g" + val neighbourMode = providers.systemProperty("atto.test.neighbour-mode").orElse("classpath").get() + require(neighbourMode in setOf("classpath", "live")) { "Unsupported neighbour mode: $neighbourMode" } + systemProperty("atto.test.neighbour-mode", neighbourMode) systemProperty( "atto.test.runtime-classpath", sourceSets.test diff --git a/src/test/kotlin/cash/atto/node/CucumberConfiguration.kt b/src/test/kotlin/cash/atto/node/CucumberConfiguration.kt index f6620392..39a6bc7f 100644 --- a/src/test/kotlin/cash/atto/node/CucumberConfiguration.kt +++ b/src/test/kotlin/cash/atto/node/CucumberConfiguration.kt @@ -69,6 +69,7 @@ class CucumberConfiguration( @After fun after() { -// DebugProbes.dumpCoroutines() + networkProperties.defaultNodes.clear() + NodeHolder.clear(except = context) } } diff --git a/src/test/kotlin/cash/atto/node/Neighbour.kt b/src/test/kotlin/cash/atto/node/Neighbour.kt index 03d4fe1e..8aa0edd7 100644 --- a/src/test/kotlin/cash/atto/node/Neighbour.kt +++ b/src/test/kotlin/cash/atto/node/Neighbour.kt @@ -3,4 +3,6 @@ package cash.atto.node data class Neighbour( val websocketPort: UShort, val httpPort: UShort, + val released: Boolean = false, + val managementPort: UShort = httpPort, ) diff --git a/src/test/kotlin/cash/atto/node/NodeHolder.kt b/src/test/kotlin/cash/atto/node/NodeHolder.kt index cdf6ec67..eec56934 100644 --- a/src/test/kotlin/cash/atto/node/NodeHolder.kt +++ b/src/test/kotlin/cash/atto/node/NodeHolder.kt @@ -21,27 +21,33 @@ object NodeHolder { fun clear(except: Closeable) { val toClose = nodes.filter { it.context != except } - toClose.forEach { it.context.close() } + val failures = mutableListOf() + toClose.forEach { node -> + runCatching { node.context.close() }.onFailure { failures.add(it) } + } toClose.forEach { node -> val classLoader = node.classLoader as? URLClassLoader ?: return@forEach - val remaining = waitForThreads(classLoader) - - if (remaining.isEmpty()) { - classLoader.close() - return@forEach - } + runCatching { closeClassLoader(classLoader) }.onFailure { failures.add(it) } + } - Thread - .getAllStackTraces() - .keys - .filter { it.contextClassLoader == classLoader && it.isAlive && it.isDaemon } - .forEach { it.contextClassLoader = null } + nodes.removeAll(toClose.toSet()) + if (failures.isNotEmpty()) { + val failure = IllegalStateException("Failed to close ${failures.size} neighbour resources") + failures.forEach { failure.addSuppressed(it) } + throw failure + } + } - logger.warn { "Leaving classloader open with ${remaining.size} threads still alive: ${remaining.map { it.name }}" } + private fun closeClassLoader(classLoader: URLClassLoader) { + val remaining = waitForThreads(classLoader) + if (remaining.isEmpty()) { + classLoader.close() + return } - nodes.removeAll(toClose.toSet()) + remaining.filter { it.isDaemon }.forEach { it.contextClassLoader = null } + logger.warn { "Leaving classloader open with ${remaining.size} threads still alive: ${remaining.map { it.name }}" } } private fun waitForThreads(classLoader: ClassLoader): List { diff --git a/src/test/kotlin/cash/atto/node/NodeStepDefinition.kt b/src/test/kotlin/cash/atto/node/NodeStepDefinition.kt index eb387d65..bc813983 100644 --- a/src/test/kotlin/cash/atto/node/NodeStepDefinition.kt +++ b/src/test/kotlin/cash/atto/node/NodeStepDefinition.kt @@ -8,19 +8,25 @@ import cash.atto.commons.toSignerBlocking import cash.atto.node.network.NetworkProperties import cash.atto.node.transaction.Transaction import io.cucumber.java.en.Given +import io.github.oshai.kotlinlogging.KotlinLogging import io.r2dbc.spi.Option import org.springframework.beans.factory.annotation.Value import org.springframework.boot.builder.SpringApplicationBuilder import org.springframework.boot.r2dbc.autoconfigure.R2dbcConnectionDetails import org.springframework.core.io.DefaultResourceLoader import org.springframework.r2dbc.core.DatabaseClient +import org.testcontainers.DockerClientFactory import org.testcontainers.containers.GenericContainer import org.testcontainers.images.PullPolicy +import org.testcontainers.images.RemoteDockerImage import org.testcontainers.utility.DockerImageName import java.io.Closeable import java.io.File +import java.net.HttpURLConnection import java.net.ServerSocket +import java.net.URI import java.net.URLClassLoader +import java.time.Duration import java.util.concurrent.FutureTask class NodeStepDefinition( @@ -30,12 +36,22 @@ class NodeStepDefinition( private val databaseClient: DatabaseClient, @Value("\${atto.test.neighbour-mode:classpath}") private val neighbourMode: String, ) { + private val logger = KotlinLogging.logger {} + @Given("^the neighbour node (\\w+)$") - fun startNeighbour(shortId: String) { + fun startNeighbour(shortId: String) = startNeighbour(shortId, emptyList()) + + fun startNeighbour( + shortId: String, + defaultNodes: List, + ) { + require(neighbourMode == CLASSPATH_MODE || neighbourMode == LIVE_MODE) { "Unsupported neighbour mode: $neighbourMode" } + logger.info { "Starting neighbour $shortId with backend $neighbourMode" } val nodeName = "Node $shortId" - val ports = randomPorts() + val ports = randomPorts(3) val websocketPort = ports[0] val httpPort = ports[1] + val managementPort = ports[2] val sql = "DROP DATABASE IF EXISTS $shortId; CREATE DATABASE $shortId" databaseClient @@ -57,7 +73,10 @@ class NodeStepDefinition( "--spring.application.name=neighbour-atto-node-$shortId", "--server.port=$httpPort", "--NODE_NAME=$nodeName", - "--management.server.port=", + "--management.server.port=$managementPort", + "--management.endpoints.web.base-path=/", + "--management.endpoints.web.exposure.include=health,info,metrics,prometheus", + "--management.endpoint.health.probes.enabled=true", "--atto.test.mysql-container.enabled=false", "--spring.r2dbc.url=r2dbc:$driver://$host:$port/$shortId", "--spring.r2dbc.username=$user", @@ -66,21 +85,29 @@ class NodeStepDefinition( "--spring.flyway.user=$user", "--spring.flyway.password=$password", "--atto.node.public-uri=ws://localhost:$websocketPort", + "--atto.node.network=LOCAL", + "--atto.node.force-voter=true", + "--atto.node.force-historical=true", + "--atto.network.loopback-blocked=false", + "--atto.vote.weight.minimal-confirmation-weight=9000000000000000000", "--websocket.port=$websocketPort", "--atto.signer.key=${privateKey.value.toHex()}", "--atto.transaction.genesis=${transaction.toAttoTransaction().toBuffer().toHex()}", - ) + ) + defaultNodes.mapIndexed { index, uri -> "--atto.network.default-nodes[$index]=$uri" } when (neighbourMode) { CLASSPATH_MODE -> startClasspathNode(nodeName, args) - LIVE_MODE -> startLiveNode(args) + LIVE_MODE -> startLiveNode(shortId, managementPort, args) else -> error("Unsupported neighbour mode: $neighbourMode") } PropertyHolder.add(shortId, privateKey.toSignerBlocking()) PropertyHolder.add(shortId, privateKey.toPublicKeyBlocking()) PropertyHolder.add(shortId, AttoAlgorithm.V1) - PropertyHolder.add(shortId, Neighbour(websocketPort, httpPort)) + PropertyHolder.add( + shortId, + Neighbour(websocketPort, httpPort, released = neighbourMode == LIVE_MODE, managementPort = managementPort), + ) } @Given("is a default node") @@ -110,17 +137,71 @@ class NodeStepDefinition( neighbourThread.contextClassLoader = classLoader neighbourThread.name = nodeName neighbourThread.start() - futureTask.get() + try { + futureTask.get() + } catch (exception: Exception) { + runCatching { classLoader.close() }.onFailure { exception.addSuppressed(it) } + throw exception + } } - private fun startLiveNode(args: Array) { - val container = GenericContainer(DockerImageName.parse(LIVE_NODE_IMAGE)) - container.withImagePullPolicy(PullPolicy.alwaysPull()) + private fun startLiveNode( + shortId: String, + managementPort: UShort, + args: Array, + ) { + val image = releasedImage + val logFile = File("build/reports/released-neighbours/$shortId-${System.nanoTime()}.log") + logFile.parentFile.mkdirs() + logFile.writeText("${image.description}\n") + logger.info { "Starting released neighbour $shortId: ${image.description}; logs=${logFile.absolutePath}" } + + val container = GenericContainer(image.name) container.withNetworkMode("host") container.withEnv("SPRING_PROFILES_ACTIVE", "default,json") + container.withEnv("MANAGEMENT_SERVER_PORT", managementPort.toString()) container.withCommand(*args) - container.start() - NodeHolder.add(Closeable { container.stop() }, null) + container.withLogConsumer { frame -> logFile.appendText(frame.utf8String) } + val cleanup = Closeable { container.stop() } + NodeHolder.add(cleanup, null) + try { + container.start() + waitForReadiness(container, managementPort) + } catch (exception: Exception) { + runCatching { cleanup.close() }.onFailure { exception.addSuppressed(it) } + throw IllegalStateException( + "Released neighbour $shortId failed to start (${image.description}); logs=${logFile.absolutePath}", + exception, + ) + } + } + + private fun waitForReadiness( + container: GenericContainer<*>, + managementPort: UShort, + ) { + val deadline = System.nanoTime() + Duration.ofMinutes(3).toNanos() + val endpoint = URI("http://localhost:$managementPort/health/readiness").toURL() + var lastFailure: Throwable? = null + while (System.nanoTime() < deadline) { + check(container.isRunning) { "Released neighbour exited before becoming ready" } + try { + val connection = endpoint.openConnection() as HttpURLConnection + try { + connection.connectTimeout = 1_000 + connection.readTimeout = 1_000 + if (connection.responseCode == 200) { + return + } + } finally { + connection.disconnect() + } + } catch (exception: Exception) { + lastFailure = exception + } + Thread.sleep(100) + } + throw IllegalStateException("Released neighbour did not become ready at $endpoint within 180s", lastFailure) } private fun randomPorts(count: Int = 2): List { @@ -133,20 +214,51 @@ class NodeStepDefinition( private fun createClassLoader(): URLClassLoader { val urlList = System - .getProperty("java.class.path") + .getProperty("atto.test.runtime-classpath", System.getProperty("java.class.path")) .split(File.pathSeparator) .asSequence() .map { if (it.endsWith(".jar")) it else "$it/" } - .map { if (it.startsWith("/")) it else "/$it" } .map { File(it).toURI().toURL() } .toList() val urlArray = Array(urlList.size) { urlList[it] } return URLClassLoader(urlArray, ClassLoader.getSystemClassLoader()) } + private data class ReleasedImage( + val name: DockerImageName, + val description: String, + ) + private companion object { const val CLASSPATH_MODE = "classpath" const val LIVE_MODE = "live" const val LIVE_NODE_IMAGE = "ghcr.io/attocash/node:live" + + val releasedImage: ReleasedImage by lazy { + val reference = + RemoteDockerImage(DockerImageName.parse(LIVE_NODE_IMAGE)) + .withImagePullPolicy(PullPolicy.alwaysPull()) + .get() + val image = + DockerClientFactory + .instance() + .client() + .inspectImageCmd(reference) + .exec() + val digest = + image.repoDigests.orEmpty().firstOrNull { it.startsWith("ghcr.io/attocash/node@") } + ?: error("Pulled $LIVE_NODE_IMAGE has no repository digest") + val version = + image.config + ?.env + .orEmpty() + .firstOrNull { it.startsWith("APPLICATION_VERSION=") } + val labels = image.config?.labels.orEmpty() + ReleasedImage( + DockerImageName.parse(digest), + "digest=$digest imageId=${image.id} $version " + + "version=${labels["org.opencontainers.image.version"]} revision=${labels["org.opencontainers.image.revision"]}", + ) + } } } diff --git a/src/test/kotlin/cash/atto/node/network/peer/PeerStepDefinition.kt b/src/test/kotlin/cash/atto/node/network/peer/PeerStepDefinition.kt index 643c4a0d..b2f5f740 100644 --- a/src/test/kotlin/cash/atto/node/network/peer/PeerStepDefinition.kt +++ b/src/test/kotlin/cash/atto/node/network/peer/PeerStepDefinition.kt @@ -5,23 +5,73 @@ import cash.atto.node.Neighbour import cash.atto.node.NodeStepDefinition import cash.atto.node.PropertyHolder import cash.atto.node.Waiter.waitUntilNonNull +import cash.atto.node.Waiter.waitUntilTrue import cash.atto.node.network.NetworkProcessor +import cash.atto.node.network.NetworkProperties +import cash.atto.node.network.NodeConnectionManager +import cash.atto.node.network.NodeDisconnected +import io.cucumber.java.After import io.cucumber.java.en.Given import io.cucumber.java.en.Then import io.cucumber.java.en.When import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.coroutines.runBlocking +import org.springframework.context.ApplicationListener +import org.springframework.context.ConfigurableApplicationContext +import org.springframework.context.PayloadApplicationEvent import org.springframework.web.reactive.function.client.WebClient import org.springframework.web.reactive.function.client.bodyToMono import java.time.Duration +import java.util.concurrent.ConcurrentLinkedQueue class PeerStepDefinition( private val nodeStepDefinition: NodeStepDefinition, private val networkProcessor: NetworkProcessor, private val webClient: WebClient, + private val connectionManager: NodeConnectionManager, + private val networkProperties: NetworkProperties, + private val context: ConfigurableApplicationContext, ) { private val logger = KotlinLogging.logger {} private val bootstrapPeerTimeoutInSeconds = 180L + private val disconnections = ConcurrentLinkedQueue() + private val disconnectListener = + object : ApplicationListener> { + override fun supportsAsyncExecution(): Boolean = false + + override fun onApplicationEvent(event: PayloadApplicationEvent) { + disconnections.add(event.payload) + } + } + + @After + fun stopObservingDisconnections() { + context.removeApplicationListener(disconnectListener) + } + + @Given("the peer {word} connected by {word}") + fun startPeer( + shortId: String, + initiator: String, + ) { + context.addApplicationListener(disconnectListener) + when (initiator) { + "THIS" -> { + startPeer(shortId) + } + + shortId -> { + val thisNeighbour = PropertyHolder[Neighbour::class.java, "THIS"] + nodeStepDefinition.startNeighbour(shortId, listOf("ws://localhost:${thisNeighbour.websocketPort}")) + checkPeer("THIS", shortId) + checkPeer(shortId, "THIS") + } + + else -> { + error("Unknown connection initiator: $initiator") + } + } + } @Given("^the peer (\\w+)$") fun startPeer(shortId: String) { @@ -47,28 +97,96 @@ class PeerStepDefinition( sourceNodeShortId: String, peerNodeShortId: String, ) { - val sourceNeighbour = PropertyHolder[Neighbour::class.java, sourceNodeShortId] val peerPublicKey = PropertyHolder[AttoPublicKey::class.java, peerNodeShortId] - logger.info { "Checking if $sourceNeighbour is connected to $peerPublicKey..." } + logger.info { "Checking if $sourceNodeShortId is connected to $peerPublicKey..." } waitUntilNonNull(bootstrapPeerTimeoutInSeconds) { - findPeer(sourceNeighbour, peerPublicKey) + findPeer(sourceNodeShortId, peerPublicKey) } } + @Then("peer {word} stays connected beyond the handshake deadline") + fun checkStableConnection(shortId: String) { + val publicKey = PropertyHolder[AttoPublicKey::class.java, shortId] + val disconnectsBefore = disconnectCount(publicKey) + val deadline = System.nanoTime() + Duration.ofMillis(5_200).toNanos() + + // Both nodes automatically reconnect; an eventual peer check alone can conceal a five-second session expiry. + while (System.nanoTime() < deadline) { + check(publicKey in connectionManager.connectedPublicKeys) { "Peer $shortId disconnected before the deadline" } + check(disconnectCount(publicKey) == disconnectsBefore) { "Peer $shortId reconnected during the stability interval" } + Thread.sleep(100) + } + check(publicKey in connectionManager.connectedPublicKeys) { "Peer $shortId disconnected before the deadline" } + check(disconnectCount(publicKey) == disconnectsBefore) { "Peer $shortId reconnected during the stability interval" } + checkPeer(shortId, "THIS") + } + + @When("peer {word} reconnects after its connection is closed") + fun reconnect(shortId: String) { + val neighbour = PropertyHolder[Neighbour::class.java, shortId] + val publicKey = PropertyHolder[AttoPublicKey::class.java, shortId] + val publicUri = "ws://localhost:${neighbour.websocketPort}" + val disconnectsBefore = disconnectCount(publicKey) + val wasDefault = networkProperties.defaultNodes.remove(publicUri) + + try { + connectionManager.clear() + waitUntilTrue(bootstrapPeerTimeoutInSeconds) { disconnectCount(publicKey) > disconnectsBefore } + } finally { + if (wasDefault) networkProperties.defaultNodes.add(publicUri) + } + + if (wasDefault) startDefaultHandshake() + checkPeer("THIS", shortId) + checkPeer(shortId, "THIS") + check(connectionManager.connectionCount == 1) { "Expected exactly one re-established peer" } + } + + private fun disconnectCount(publicKey: AttoPublicKey): Int = disconnections.count { it.node.publicKey == publicKey } + private fun findPeer( - sourceNeighbour: Neighbour, + sourceShortId: String, peerPublicKey: AttoPublicKey, - ): AttoPublicKey? = - runCatching { + ): AttoPublicKey? { + if (sourceShortId == "THIS") return peerPublicKey.takeIf { it in connectionManager.connectedPublicKeys } + + val sourceNeighbour = PropertyHolder[Neighbour::class.java, sourceShortId] + if (sourceNeighbour.released) { + require(peerPublicKey == PropertyHolder[AttoPublicKey::class.java, "THIS"]) { + "Released peer observation supports connections to THIS only" + } + val sourcePublicKey = PropertyHolder[AttoPublicKey::class.java, sourceShortId] + if (sourcePublicKey !in connectionManager.connectedPublicKeys) return null + + // Released native images can lack AttoNode reflection metadata for /info. + // The local manager identifies the authenticated peer; the remote gauge confirms active peer registration. + return runCatching { + val metric = + webClient + .get() + .uri("http://localhost:${sourceNeighbour.managementPort}/metrics/network.peers.active") + .retrieve() + .bodyToMono>() + .block(Duration.ofSeconds(1)) + val measurements = metric?.get("measurements") as? List<*> + peerPublicKey.takeIf { + measurements?.any { measurement -> + val value = measurement as? Map<*, *> + value?.get("statistic") == "VALUE" && (value["value"] as? Number)?.toDouble()?.let { it >= 1.0 } == true + } == true + } + }.getOrNull() + } + return runCatching { webClient .get() .uri("http://localhost:${sourceNeighbour.httpPort}/nodes/peers") .retrieve() .bodyToMono>() - .flatMapIterable { it } - .filter { it == peerPublicKey } - .blockFirst(Duration.ofSeconds(1)) + .block(Duration.ofSeconds(1)) + ?.firstOrNull { it == peerPublicKey } }.getOrNull() + } } diff --git a/src/test/kotlin/cash/atto/node/transaction/TransactionStepDefinition.kt b/src/test/kotlin/cash/atto/node/transaction/TransactionStepDefinition.kt index d77ee01f..bd5e65a8 100644 --- a/src/test/kotlin/cash/atto/node/transaction/TransactionStepDefinition.kt +++ b/src/test/kotlin/cash/atto/node/transaction/TransactionStepDefinition.kt @@ -59,7 +59,7 @@ class TransactionStepDefinition( logger.info { "Publishing $sendTransaction" } - publish("THIS", sendTransaction) + publish(senderShortId, sendTransaction) PropertyHolder.add(transactionShortId, sendTransaction) } @@ -99,7 +99,7 @@ class TransactionStepDefinition( logger.info { "Publishing $transaction" } - publish("THIS", transaction) + publish(receiverShortId, transaction) PropertyHolder.add(transactionShortId, transaction) } diff --git a/src/test/kotlin/cash/atto/node/unchecked/UncheckedTransactionStepDefinition.kt b/src/test/kotlin/cash/atto/node/unchecked/UncheckedTransactionStepDefinition.kt index 83284360..9e3f91d3 100644 --- a/src/test/kotlin/cash/atto/node/unchecked/UncheckedTransactionStepDefinition.kt +++ b/src/test/kotlin/cash/atto/node/unchecked/UncheckedTransactionStepDefinition.kt @@ -181,6 +181,9 @@ class UncheckedTransactionStepDefinition( expectedHash: AttoHash, ): String = runCatching { + if (neighbour.released) { + return@runCatching "released scheduler, hasTransaction=${hasTransaction(neighbour, expectedHash)}" + } val uncheckedTransactions = findUncheckedTransactions(neighbour) val uncheckedHashes = uncheckedTransactions.map { it.hash } "hasTransaction=${hasTransaction(neighbour, expectedHash)}, " + @@ -199,6 +202,10 @@ class UncheckedTransactionStepDefinition( neighbour: Neighbour, path: String, ) { + // Released nodes run the production schedulers and do not contain test controllers. + if (neighbour.released) { + return + } webClient .post() .uri("http://localhost:${neighbour.httpPort}$path") @@ -208,11 +215,13 @@ class UncheckedTransactionStepDefinition( .block(Duration.ofSeconds(6)) } - private fun findUncheckedTransactions(neighbour: Neighbour): List = - webClient + private fun findUncheckedTransactions(neighbour: Neighbour): List { + require(!neighbour.released) { "Unchecked state is available only on classpath test nodes" } + return webClient .get() .uri("http://localhost:${neighbour.httpPort}/unchecked-transactions") .retrieve() .bodyToMono>() .block()!! + } } diff --git a/src/test/resources/features/peering.feature b/src/test/resources/features/peering.feature index 37a7ad01..14691e21 100644 --- a/src/test/resources/features/peering.feature +++ b/src/test/resources/features/peering.feature @@ -1,10 +1,20 @@ Feature: Peering - Scenario: After start up should peer with default node - Given the neighbour node A - And is a default node + Scenario Outline: Peers stay connected and exchange transactions after reconnecting + Given the peer A connected by + Then peer A stays connected beyond the handshake deadline - When default handshake starts + When send transaction 1 from THIS account to A account + Then transaction 1 is confirmed + When peer A reconnects after its connection is closed Then THIS node is A node peer + And A node is THIS node peer + When receive transaction 2 from 1 send transaction to A account + Then transaction 2 is confirmed + + Examples: + | initiator | + | THIS | + | A | From ce029ede086310585c8a1bc966fd0c97a0b32acd Mon Sep 17 00:00:00 2001 From: rotilho Date: Sun, 8 Nov 2026 20:00:00 +0100 Subject: [PATCH 5/5] Run released node compatibility checks in CI --- .github/workflows/compatibility.yaml | 41 ++++++++++++++++++++++++++++ .github/workflows/pipeline.yaml | 3 ++ .github/workflows/pull-request.yaml | 8 ------ .github/workflows/test.yaml | 7 +---- README.md | 5 ++++ 5 files changed, 50 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/compatibility.yaml diff --git a/.github/workflows/compatibility.yaml b/.github/workflows/compatibility.yaml new file mode 100644 index 00000000..85b1938e --- /dev/null +++ b/.github/workflows/compatibility.yaml @@ -0,0 +1,41 @@ +# Owned by node; management-terraform supplies the separate classpath test workflow. +name: Compatibility + +on: + pull_request: + workflow_call: + +permissions: + contents: read + +jobs: + compatibility: + name: Released node compatibility + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + + - uses: gradle/actions/wrapper-validation@v6 + + - uses: graalvm/setup-graalvm@v1 + with: + distribution: 'graalvm' + version: '25i-latest' + native-image-musl: true + cache: 'gradle' + + - uses: gradle/actions/setup-gradle@v6 + + - name: Test against the released node + run: ./gradlew test --tests cash.atto.node.CucumberTest --no-daemon --info --fail-fast -Datto.test.neighbour-mode=live + + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: released-node-compatibility-reports + path: | + build/reports/tests/test + build/test-results/test + build/reports/Cucumber.html + build/reports/released-neighbours diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 5263ee41..b01ac3a0 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -11,12 +11,15 @@ jobs: uses: ./.github/workflows/test.yaml with: agent_enabled: true + compatibility: + uses: ./.github/workflows/compatibility.yaml build: permissions: contents: write packages: write needs: - test + - compatibility uses: ./.github/workflows/build.yaml release-tag: permissions: diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 721b7b21..0d568baf 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -7,12 +7,4 @@ on: jobs: test: - strategy: - fail-fast: false - matrix: - neighbour-mode: - - classpath - - live uses: ./.github/workflows/test.yaml - with: - neighbour_mode: ${{ matrix['neighbour-mode'] }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3c579200..427e5e3d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,11 +8,6 @@ on: required: false default: false type: boolean - neighbour_mode: - description: "Neighbour node implementation used by integration tests" - required: false - default: "classpath" - type: string jobs: test: @@ -31,7 +26,7 @@ jobs: - uses: gradle/actions/setup-gradle@v6 - - run: ./gradlew ${{ inputs.agent_enabled && '-Pagent' || '' }} test --no-daemon --info --fail-fast -Datto.test.neighbour-mode=${{ inputs.neighbour_mode }} + - run: ./gradlew ${{ inputs.agent_enabled && '-Pagent' || '' }} test --no-daemon --info --fail-fast - uses: actions/upload-artifact@v7 if: inputs.agent_enabled diff --git a/README.md b/README.md index cd684796..83cdadac 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,11 @@ Run `./gradlew test` to test with neighbours built from the current checkout. Fo throughout the run, and logs its digest and release metadata. It requires a local Linux container engine with host networking. Released neighbours use their production APIs and scheduled bootstrap processing; test-only controls remain on classpath nodes. +CI runs the full suite through the Terraform-managed classpath workflow. The node-owned +[compatibility workflow](.github/workflows/compatibility.yaml) runs Cucumber against `:live` for pull requests and is also +called by the [pipeline](.github/workflows/pipeline.yaml). Builds require both checks; native-image metadata comes from the +classpath run. Keep released-node checks in the node-owned workflow so Terraform can regenerate the shared workflows. + ## Configuration Configuration is split across `application.yaml` and profile overrides such as: