From e4abc2503ad23a579771010ae760ce2e9a3c7080 Mon Sep 17 00:00:00 2001 From: Jakub Balhar Date: Thu, 9 Jul 2026 13:18:33 +0200 Subject: [PATCH 1/9] refactor: move isTcpStackRestarted/isRecycledClass to static, add tcpStackAwareSocketChannel config (#4776) Move isRecycledClass and isTcpStackRestarted from FixedServerSocketChannel instance methods to package-private static methods on TomcatAcceptFixConfig. Add apiml.tcpStackAwareSocketChannel.enabled config property (default true). Update call sites in FixedServerSocketChannel.accept() to use static methods. Signed-off-by: Jakub Balhar --- .../product/web/TomcatAcceptFixConfig.java | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java index 3bd7a1c8c8..f4a8e52c32 100644 --- a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java +++ b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java @@ -51,6 +51,9 @@ public class TomcatAcceptFixConfig { @Value("${server.tomcat.retryRebindTimeoutSecs:10}") int retryRebindTimeoutSecs; + @Value("${apiml.tcpStackAwareSocketChannel.enabled:true}") + boolean tcpStackAwareSocketChannelEnabled; + private static final Field ENDPOINT_FIELD; private static final Field NIO_SOCKET_FIELD; @@ -143,6 +146,27 @@ public void stopping() { running.set(false); } + static boolean isRecycledClass(Throwable t) { + return "com.ibm.net.NetworkRecycledException".equals(t.getClass().getName()); + } + + static boolean isTcpStackRestarted(Throwable t) { + if ((t.getMessage() != null) && t.getMessage().contains("EDC5122I")) { + return true; + } + + if (isRecycledClass(t)) { + return true; + } + + Throwable cause = t.getCause(); + if ((cause != null) && (cause != t)) { + return isTcpStackRestarted(cause); + } + + return false; + } + /** * Socket implementation wrapper to handle rebinding on TCP Stack restart */ @@ -249,25 +273,8 @@ private synchronized void rebind(int stateBefore) throws IOException { } } - boolean isRecycledClass(Throwable t) { - return NETWORK_RECYCLED_EXCEPTION_CLASS.equals(t.getClass().getName()); - } - boolean isTcpStackRestarted(Throwable t) { - if ((t.getMessage() != null) && t.getMessage().contains("EDC5122I")) { - return true; - } - - if (isRecycledClass(t)) { - return true; - } - - Throwable cause = t.getCause(); - if ((cause != null) && (cause != t)) { - return isTcpStackRestarted(cause); - } - - return false; + return TomcatAcceptFixConfig.isTcpStackRestarted(t); } public SocketChannel accept() throws IOException { @@ -276,7 +283,7 @@ public SocketChannel accept() throws IOException { try { return socket.accept(); } catch (IOException ioe) { - if (isTcpStackRestarted(ioe)) { + if (TomcatAcceptFixConfig.isTcpStackRestarted(ioe)) { // the fix solve just one issue about stopped TCP/IP stack log.debug("The TCP/IP stack was probably restarted. The socket of Tomcat will rebind."); rebind(stateBefore); From 2e8ad5c19d6ab59b59477f0abe9d0bf10f05e827 Mon Sep 17 00:00:00 2001 From: Jakub Balhar Date: Thu, 9 Jul 2026 13:22:55 +0200 Subject: [PATCH 2/9] feat: add TcpStackAwareSocketChannel wrapper for z/OS TCP/IP recovery (#4776) Create ExcludedSocketOps interface (20 methods) and TcpStackAwareSocketChannel inner class extending SocketChannel with @Delegate. Intercepts read/write ops to detect EDC5122I/NetworkRecycledException, closes dead socket, re-throws. Wire into FixedServerSocketChannel.accept() guarded by config property apiml.tcpStackAwareSocketChannel.enabled (default true). Signed-off-by: Jakub Balhar --- .../product/web/TomcatAcceptFixConfig.java | 140 +++++++++++++++++- 1 file changed, 138 insertions(+), 2 deletions(-) diff --git a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java index f4a8e52c32..176dc2b205 100644 --- a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java +++ b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java @@ -30,6 +30,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; import java.net.SocketAddress; +import java.nio.ByteBuffer; import java.nio.channels.*; import java.nio.channels.spi.AbstractSelectableChannel; import java.nio.channels.spi.SelectorProvider; @@ -281,18 +282,25 @@ public SocketChannel accept() throws IOException { // obtain current state of rebinding to detection parallel actions final int stateBefore = state.get(); try { - return socket.accept(); + return wrapIfEnabled(socket.accept()); } catch (IOException ioe) { if (TomcatAcceptFixConfig.isTcpStackRestarted(ioe)) { // the fix solve just one issue about stopped TCP/IP stack log.debug("The TCP/IP stack was probably restarted. The socket of Tomcat will rebind."); rebind(stateBefore); - return socket.accept(); + return wrapIfEnabled(socket.accept()); } throw ioe; } } + private SocketChannel wrapIfEnabled(SocketChannel socketChannel) { + if (tcpStackAwareSocketChannelEnabled) { + return new TcpStackAwareSocketChannel(socketChannel); + } + return socketChannel; + } + } /** @@ -317,4 +325,132 @@ private interface Overridden { } + /** + * The list of methods excluded from Lombok @Delegate for TcpStackAwareSocketChannel. + * First 14 are final methods on SocketChannel; last 6 are manually overridden. + */ + private interface ExcludedSocketOps { + + SelectorProvider provider(); + boolean isRegistered(); + SelectionKey keyFor(Selector sel); + SelectionKey register(Selector sel, int ops, Object att); + SelectionKey register(Selector sel, int ops) throws ClosedChannelException; + boolean isBlocking(); + Object blockingLock(); + SelectableChannel configureBlocking(boolean block) throws IOException; + void implCloseChannel() throws IOException; + void close() throws IOException; + boolean isOpen(); + int validOps(); + long read(ByteBuffer[] dsts) throws IOException; + long write(ByteBuffer[] srcs) throws IOException; + int read(ByteBuffer dst) throws IOException; + long read(ByteBuffer[] dsts, int offset, int length) throws IOException; + int write(ByteBuffer src) throws IOException; + long write(ByteBuffer[] srcs, int offset, int length) throws IOException; + void implCloseSelectableChannel() throws IOException; + void implConfigureBlocking(boolean block) throws IOException; + + } + + /** + * SocketChannel wrapper that detects z/OS TCP/IP stack restarts during I/O operations. + * When EDC5122I or NetworkRecycledException is caught on read/write, the underlying + * socket is closed and the exception is re-thrown so Tomcat can discard the connection. + */ + class TcpStackAwareSocketChannel extends SocketChannel { + + @Delegate(excludes = ExcludedSocketOps.class) + private final SocketChannel delegate; + + TcpStackAwareSocketChannel(SocketChannel delegate) { + super(delegate.provider()); + this.delegate = delegate; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + try { + return delegate.read(dst); + } catch (IOException e) { + if (TomcatAcceptFixConfig.isTcpStackRestarted(e)) { + log.debug("TCP/IP stack restart detected during read on client socket; closing connection", e); + safeClose(delegate); + } + throw e; + } + } + + @Override + public long read(ByteBuffer[] dsts, int offset, int length) throws IOException { + try { + return delegate.read(dsts, offset, length); + } catch (IOException e) { + if (TomcatAcceptFixConfig.isTcpStackRestarted(e)) { + log.debug("TCP/IP stack restart detected during scatter read on client socket; closing connection", e); + safeClose(delegate); + } + throw e; + } + } + + @Override + public int write(ByteBuffer src) throws IOException { + try { + return delegate.write(src); + } catch (IOException e) { + if (TomcatAcceptFixConfig.isTcpStackRestarted(e)) { + log.debug("TCP/IP stack restart detected during write on client socket; closing connection", e); + safeClose(delegate); + } + throw e; + } + } + + @Override + public long write(ByteBuffer[] srcs, int offset, int length) throws IOException { + try { + return delegate.write(srcs, offset, length); + } catch (IOException e) { + if (TomcatAcceptFixConfig.isTcpStackRestarted(e)) { + log.debug("TCP/IP stack restart detected during scatter write on client socket; closing connection", e); + safeClose(delegate); + } + throw e; + } + } + + @Override + protected void implCloseSelectableChannel() throws IOException { + try { + IMPL_CLOSE_SELECTABGLE_CHANNEL_HANLE.invoke(delegate); + } catch (IOException | RuntimeException e) { + throw e; + } catch (Throwable t) { + throw new IllegalStateException(t); + } + } + + @Override + protected void implConfigureBlocking(boolean block) throws IOException { + try { + IMPL_CONFIGURE_BLOCKING.invoke(delegate, block); + } catch (IOException | RuntimeException e) { + throw e; + } catch (Throwable t) { + throw new IllegalStateException(t); + } + } + + private static void safeClose(SocketChannel ch) { + try { + ch.close(); + } catch (IOException ignored) { + // best-effort close + } + } + + } + } From 79e9f1dec2ebdcb51373dcf47e335dd7500df0ed Mon Sep 17 00:00:00 2001 From: Jakub Balhar Date: Thu, 9 Jul 2026 13:32:03 +0200 Subject: [PATCH 3/9] test: add TcpStackAwareSocketChannel unit tests (#4776) Add comprehensive tests for TcpStackAwareSocketChannel wrapper: - Normal read/write passthrough - EDC5122I detection on read/write/scatter/gather (close + rethrow) - NetworkRecycledException detection via MockedStatic - Non-EDC5122I IOException passthrough (no close) - safeClose robustness (IOException on close ignored) - configureBlocking/isBlocking delegation - Integration test with real SocketChannel - Config disabled returns raw SocketChannel Update TcpStackRestartHandling tests to use static methods. All 76 tests pass. Signed-off-by: Jakub Balhar --- .../web/TomcatAcceptFixConfigTest.java | 290 +++++++++++++++++- 1 file changed, 273 insertions(+), 17 deletions(-) diff --git a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java index ab44dbad99..87598977d6 100644 --- a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java +++ b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java @@ -23,6 +23,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.springframework.boot.web.embedded.tomcat.TomcatConnectorCustomizer; import java.io.IOException; @@ -30,6 +31,7 @@ import java.net.ServerSocket; import java.net.SocketAddress; import java.net.SocketOption; +import java.nio.ByteBuffer; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.nio.channels.spi.SelectorProvider; @@ -38,6 +40,7 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; class TomcatAcceptFixConfigTest { @@ -230,19 +233,16 @@ public void implConfigureBlocking(boolean block) { @Nested class TcpStackRestartHandling { - ServerSocketChannel serverSocket = new TestServerSocketChannel(mock(SelectorProvider.class)); - TomcatAcceptFixConfig.FixedServerSocketChannel channel = new TomcatAcceptFixConfig().new FixedServerSocketChannel(serverSocket, null, null); - @Test void givenExceptionWithTheMessage_whenHandle_thenReturnTrue() { - assertTrue(channel.isTcpStackRestarted(new RuntimeException("EDC5122I TCP Stack restarted"))); + assertTrue(TomcatAcceptFixConfig.isTcpStackRestarted(new RuntimeException("EDC5122I TCP Stack restarted"))); } @Test void givenExceptionWithCyclicCause_whenHandle_thenReturnFalse() { Exception e = spy(new RuntimeException("Error")); doReturn(e).when(e).getCause(); - assertFalse(channel.isTcpStackRestarted(e)); + assertFalse(TomcatAcceptFixConfig.isTcpStackRestarted(e)); } @Test @@ -250,21 +250,277 @@ void givenExceptionWithTheMessageAsCause_whenHandle_thenReturnTrue() { Exception e = new RuntimeException("EDC5122I TCP Stack restarted"); e = new RuntimeException("Wrapper1", e); e = new RuntimeException("Wrapper2", e); - assertTrue(channel.isTcpStackRestarted(e)); + assertTrue(TomcatAcceptFixConfig.isTcpStackRestarted(e)); + } + + @Test + void givenNetworkRecycledException_whenIsRecycledClass_thenReturnTrue() { + // Verify that isRecycledClass matches the exact class name + // We can't instantiate com.ibm.net.NetworkRecycledException directly, + // but we verify that isTcpStackRestarted calls isRecycledClass via cause chain + try (MockedStatic mocked = mockStatic(TomcatAcceptFixConfig.class, CALLS_REAL_METHODS)) { + mocked.when(() -> TomcatAcceptFixConfig.isRecycledClass(any())).thenReturn(true); + + Exception e = new IllegalArgumentException("Tested exception"); + e = new RuntimeException("Wrapper", e); + assertTrue(TomcatAcceptFixConfig.isTcpStackRestarted(e)); + } + } + + @Test + void givenNonMatchingException_whenIsRecycledClass_thenReturnFalse() { + assertFalse(TomcatAcceptFixConfig.isRecycledClass(new RuntimeException("not a match"))); + } + + } + + @Nested + class TcpStackAwareSocketChannelTests { + + private SocketChannel mockDelegate; + + @BeforeEach + void setUp() { + mockDelegate = mock(SocketChannel.class); + SelectorProvider provider = mock(SelectorProvider.class); + doReturn(provider).when(mockDelegate).provider(); } @Test - void givenExceptionWithSpecificClassName_whenHandle_thenReturnTrue() { - TomcatAcceptFixConfig.FixedServerSocketChannel channel = new TomcatAcceptFixConfig().new FixedServerSocketChannel(serverSocket, null, null) { - @Override - boolean isRecycledClass(Throwable t) { - return "java.lang.IllegalArgumentException".equals(t.getClass().getName()); - } - }; + void read_shouldPassthroughToDelegate() throws IOException { + ByteBuffer buf = ByteBuffer.allocate(10); + doReturn(5).when(mockDelegate).read(buf); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); + + assertEquals(5, channel.read(buf)); + verify(mockDelegate).read(buf); + verify(mockDelegate, never()).close(); + } + + @Test + void write_shouldPassthroughToDelegate() throws IOException { + ByteBuffer buf = ByteBuffer.allocate(10); + doReturn(5).when(mockDelegate).write(buf); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); + + assertEquals(5, channel.write(buf)); + verify(mockDelegate).write(buf); + verify(mockDelegate, never()).close(); + } + + @Test + void read_shouldCloseAndRethrowOnEdc5122I() throws IOException { + ByteBuffer buf = ByteBuffer.allocate(10); + IOException ioe = new IOException("EDC5122I TCP/IP stack was restarted"); + doThrow(ioe).when(mockDelegate).read(buf); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); + + IOException thrown = assertThrows(IOException.class, () -> channel.read(buf)); + assertSame(ioe, thrown); + verify(mockDelegate).close(); + } + + @Test + void write_shouldCloseAndRethrowOnEdc5122I() throws IOException { + ByteBuffer buf = ByteBuffer.allocate(10); + IOException ioe = new IOException("EDC5122I TCP/IP stack was restarted"); + doThrow(ioe).when(mockDelegate).write(buf); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); + + IOException thrown = assertThrows(IOException.class, () -> channel.write(buf)); + assertSame(ioe, thrown); + verify(mockDelegate).close(); + } + + @Test + void read_shouldNotCloseOnNonEdc5122I() throws IOException { + ByteBuffer buf = ByteBuffer.allocate(10); + IOException ioe = new IOException("Some other I/O error"); + doThrow(ioe).when(mockDelegate).read(buf); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); + + IOException thrown = assertThrows(IOException.class, () -> channel.read(buf)); + assertSame(ioe, thrown); + verify(mockDelegate, never()).close(); + } + + @Test + void scatterRead_shouldPassthroughToDelegate() throws IOException { + ByteBuffer[] bufs = new ByteBuffer[]{ByteBuffer.allocate(10)}; + doReturn(5L).when(mockDelegate).read(bufs, 0, 1); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); + + assertEquals(5L, channel.read(bufs, 0, 1)); + verify(mockDelegate).read(bufs, 0, 1); + } - Exception e = new IllegalArgumentException("Tested exception"); - e = new RuntimeException("Wrapper", e); - assertTrue(channel.isTcpStackRestarted(e)); + @Test + void scatterWrite_shouldPassthroughToDelegate() throws IOException { + ByteBuffer[] bufs = new ByteBuffer[]{ByteBuffer.allocate(10)}; + doReturn(5L).when(mockDelegate).write(bufs, 0, 1); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); + + assertEquals(5L, channel.write(bufs, 0, 1)); + verify(mockDelegate).write(bufs, 0, 1); + } + + @Test + void scatterRead_shouldCloseAndRethrowOnEdc5122I() throws IOException { + ByteBuffer[] bufs = new ByteBuffer[]{ByteBuffer.allocate(10)}; + IOException ioe = new IOException("EDC5122I scatter read failed"); + doThrow(ioe).when(mockDelegate).read(bufs, 0, 1); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); + + IOException thrown = assertThrows(IOException.class, () -> channel.read(bufs, 0, 1)); + assertSame(ioe, thrown); + verify(mockDelegate).close(); + } + + @Test + void scatterWrite_shouldCloseAndRethrowOnEdc5122I() throws IOException { + ByteBuffer[] bufs = new ByteBuffer[]{ByteBuffer.allocate(10)}; + IOException ioe = new IOException("EDC5122I scatter write failed"); + doThrow(ioe).when(mockDelegate).write(bufs, 0, 1); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); + + IOException thrown = assertThrows(IOException.class, () -> channel.write(bufs, 0, 1)); + assertSame(ioe, thrown); + verify(mockDelegate).close(); + } + + @Test + void read_shouldCloseAndRethrowOnNetworkRecycledException() throws IOException { + ByteBuffer buf = ByteBuffer.allocate(10); + IOException ioe = new IOException("Connection reset"); + doThrow(ioe).when(mockDelegate).read(buf); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); + + // Mock isRecycledClass to return true for this exception + try (MockedStatic mocked = mockStatic(TomcatAcceptFixConfig.class, CALLS_REAL_METHODS)) { + mocked.when(() -> TomcatAcceptFixConfig.isRecycledClass(any())).thenReturn(true); + + IOException thrown = assertThrows(IOException.class, () -> channel.read(buf)); + assertSame(ioe, thrown); + verify(mockDelegate).close(); + } + } + + @Test + void implCloseSelectableChannel_shouldDelegateCorrectly() throws IOException { + // Verify that the MethodHandle is invoked on the delegate + // Pattern matches the FixedServerSocketChannel tests + SocketChannel delegate = mock(SocketChannel.class); + SelectorProvider provider = mock(SelectorProvider.class); + doReturn(provider).when(delegate).provider(); + doThrow(new IOException("close failed")).when(delegate).close(); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(delegate); + + // implCloseSelectableChannel uses MethodHandle which we can't mock; + // test via safeClose on EDC5122I read: verifies delegate.close() is called + ByteBuffer buf = ByteBuffer.allocate(10); + doThrow(new IOException("EDC5122I")).when(delegate).read(buf); + + assertThrows(IOException.class, () -> channel.read(buf)); + verify(delegate).close(); + } + + @Test + void implConfigureBlocking_shouldDelegateCorrectly() throws IOException { + SocketChannel realChannel = SocketChannel.open(); + try { + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(realChannel); + + assertTrue(channel.isBlocking()); + // configureBlocking is final and calls implConfigureBlocking + updates wrapper state + channel.configureBlocking(false); + assertFalse(channel.isBlocking()); + } finally { + realChannel.close(); + } + } + + @Test + void integrationTest_registerWithSelector() throws IOException { + SocketChannel realChannel = SocketChannel.open(); + try { + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(realChannel); + channel.configureBlocking(false); + + // Verify non-blocking mode is set correctly + assertFalse(channel.isBlocking()); + // Verify validOps delegates correctly + assertTrue(channel.validOps() > 0); + } finally { + realChannel.close(); + } + } + + @Test + void safeClose_shouldNotThrowOnIoException() throws IOException { + doThrow(new IOException("close failed")).when(mockDelegate).close(); + + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); + + ByteBuffer buf = ByteBuffer.allocate(10); + IOException ioe = new IOException("EDC5122I"); + doThrow(ioe).when(mockDelegate).read(buf); + + // Should re-throw read IOE, not close IOE + IOException thrown = assertThrows(IOException.class, () -> channel.read(buf)); + assertSame(ioe, thrown); + verify(mockDelegate).close(); // close was attempted + } + + @Test + void configDisabled_returnsRawSocketChannel() throws Exception { + TomcatAcceptFixConfig config = new TomcatAcceptFixConfig(); + config.retryRebindTimeoutSecs = 0; + config.tcpStackAwareSocketChannelEnabled = false; + + SelectorProvider provider = mock(SelectorProvider.class); + SocketChannel rawSocket = mock(SocketChannel.class); + TestServerSocketChannel testServerSocket = spy(new TestServerSocketChannel(provider)); + TestEndpoint testEp = spy(new TestEndpoint(testServerSocket)); + TestProtocol testProto = spy(new TestProtocol(testEp)); + Connector conn = new Connector(testProto); + + TomcatConnectorCustomizer customizer = config.tomcatAcceptorFix(); + customizer.customize(conn); + Lifecycle lifecycle = mock(Lifecycle.class); + doReturn(LifecycleState.STARTED).when(lifecycle).getState(); + LifecycleEvent event = new LifecycleEvent(lifecycle, "type", "data"); + Stream.of(conn.findLifecycleListeners()).forEach(l -> l.lifecycleEvent(event)); + + doReturn(rawSocket).when(testServerSocket).accept(); + + SocketChannel result = testEp.serverSocketAccept(); + // When config is disabled, the raw socket should be returned directly (not wrapped) + assertSame(rawSocket, result); } } @@ -363,4 +619,4 @@ public SocketAddress getLocalAddress() { } -} \ No newline at end of file +} From df64bd32e153ee6a873dc5e22801d33a90a5a195 Mon Sep 17 00:00:00 2001 From: Jakub Balhar Date: Fri, 10 Jul 2026 09:12:15 +0200 Subject: [PATCH 4/9] fix: disable TcpStackAwareSocketChannel by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change default from true to false. The wrapper breaks Tomcat socket acceptance on non-z/OS platforms (Linux CI runners), causing tests to hang with Connection refused until BuildAndTest times out at 35 minutes. The feature only has value on z/OS where TCP/IP stack restarts can occur — z/OS deployments must explicitly set apiml.tcpStackAwareSocketChannel.enabled=true. Signed-off-by: Jakub Balhar --- .../java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java index 176dc2b205..89f31b20f5 100644 --- a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java +++ b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java @@ -52,7 +52,7 @@ public class TomcatAcceptFixConfig { @Value("${server.tomcat.retryRebindTimeoutSecs:10}") int retryRebindTimeoutSecs; - @Value("${apiml.tcpStackAwareSocketChannel.enabled:true}") + @Value("${apiml.tcpStackAwareSocketChannel.enabled:false}") boolean tcpStackAwareSocketChannelEnabled; private static final Field ENDPOINT_FIELD; From bb5f63ecb2011a84d6bba355e3a74e9d4e242601 Mon Sep 17 00:00:00 2001 From: Jakub Balhar Date: Mon, 24 Aug 2026 14:02:53 +0200 Subject: [PATCH 5/9] fix: address TcpStackAwareSocketChannel review feedback Signed-off-by: Jakub Balhar --- .../product/web/TomcatAcceptFixConfig.java | 27 ++++---- .../web/TomcatAcceptFixConfigTest.java | 62 ++++++++++++------- 2 files changed, 50 insertions(+), 39 deletions(-) diff --git a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java index 89f31b20f5..fd642b001f 100644 --- a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java +++ b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java @@ -58,8 +58,9 @@ public class TomcatAcceptFixConfig { private static final Field ENDPOINT_FIELD; private static final Field NIO_SOCKET_FIELD; - private static final MethodHandle IMPL_CLOSE_SELECTABGLE_CHANNEL_HANLE; // NOSONAR + private static final MethodHandle IMPL_CLOSE_SELECTABLE_CHANNEL_HANDLE; // NOSONAR private static final MethodHandle IMPL_CONFIGURE_BLOCKING; // NOSONAR + private static final String NETWORK_RECYCLED_EXCEPTION_CLASS = "com.ibm.net.NetworkRecycledException"; /** * To mitigate parallel treatment of socket rebinding @@ -73,7 +74,7 @@ public class TomcatAcceptFixConfig { Method implCloseSelectableChannel = AbstractSelectableChannel.class.getDeclaredMethod("implCloseSelectableChannel"); implCloseSelectableChannel.setAccessible(true); // NOSONAR - IMPL_CLOSE_SELECTABGLE_CHANNEL_HANLE = MethodHandles.lookup().unreflect(implCloseSelectableChannel); + IMPL_CLOSE_SELECTABLE_CHANNEL_HANDLE = MethodHandles.lookup().unreflect(implCloseSelectableChannel); Method implConfigureBlocking = AbstractSelectableChannel.class.getDeclaredMethod("implConfigureBlocking", boolean.class); implConfigureBlocking.setAccessible(true); // NOSONAR @@ -148,7 +149,7 @@ public void stopping() { } static boolean isRecycledClass(Throwable t) { - return "com.ibm.net.NetworkRecycledException".equals(t.getClass().getName()); + return NETWORK_RECYCLED_EXCEPTION_CLASS.equals(t.getClass().getName()); } static boolean isTcpStackRestarted(Throwable t) { @@ -173,7 +174,6 @@ static boolean isTcpStackRestarted(Throwable t) { */ class FixedServerSocketChannel extends ServerSocketChannel { - private static final String NETWORK_RECYCLED_EXCEPTION_CLASS = "com.ibm.net.NetworkRecycledException"; /** * Wrapper server socket inside @@ -206,7 +206,7 @@ class FixedServerSocketChannel extends ServerSocketChannel { @Override protected void implCloseSelectableChannel() throws IOException { try { - IMPL_CLOSE_SELECTABGLE_CHANNEL_HANLE.invoke(socket); + IMPL_CLOSE_SELECTABLE_CHANNEL_HANDLE.invoke(socket); } catch (IOException | RuntimeException e) { throw e; } catch (Throwable t) { @@ -327,7 +327,8 @@ private interface Overridden { /** * The list of methods excluded from Lombok @Delegate for TcpStackAwareSocketChannel. - * First 14 are final methods on SocketChannel; last 6 are manually overridden. + * It contains methods inherited from SocketChannel that must not be delegated and + * read/write/close-related methods that are wrapped explicitly in this class. */ private interface ExcludedSocketOps { @@ -423,13 +424,9 @@ public long write(ByteBuffer[] srcs, int offset, int length) throws IOException @Override protected void implCloseSelectableChannel() throws IOException { - try { - IMPL_CLOSE_SELECTABGLE_CHANNEL_HANLE.invoke(delegate); - } catch (IOException | RuntimeException e) { - throw e; - } catch (Throwable t) { - throw new IllegalStateException(t); - } + // SocketChannel.close() is final; it reaches this override through + // AbstractInterruptibleChannel.close(). + delegate.close(); } @Override @@ -446,8 +443,8 @@ protected void implConfigureBlocking(boolean block) throws IOException { private static void safeClose(SocketChannel ch) { try { ch.close(); - } catch (IOException ignored) { - // best-effort close + } catch (IOException e) { + log.trace("Best-effort close of client socket failed", e); } } diff --git a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java index 87598977d6..c1ecf87a43 100644 --- a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java +++ b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java @@ -426,30 +426,21 @@ void read_shouldCloseAndRethrowOnNetworkRecycledException() throws IOException { } @Test - void implCloseSelectableChannel_shouldDelegateCorrectly() throws IOException { - // Verify that the MethodHandle is invoked on the delegate - // Pattern matches the FixedServerSocketChannel tests - SocketChannel delegate = mock(SocketChannel.class); - SelectorProvider provider = mock(SelectorProvider.class); - doReturn(provider).when(delegate).provider(); - doThrow(new IOException("close failed")).when(delegate).close(); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(delegate); + void close_shouldCloseDelegate() throws IOException { + try (SocketChannel delegate = SocketChannel.open()) { + TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = + new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(delegate); - // implCloseSelectableChannel uses MethodHandle which we can't mock; - // test via safeClose on EDC5122I read: verifies delegate.close() is called - ByteBuffer buf = ByteBuffer.allocate(10); - doThrow(new IOException("EDC5122I")).when(delegate).read(buf); + channel.close(); - assertThrows(IOException.class, () -> channel.read(buf)); - verify(delegate).close(); + assertFalse(channel.isOpen()); + assertFalse(delegate.isOpen()); + } } @Test void implConfigureBlocking_shouldDelegateCorrectly() throws IOException { - SocketChannel realChannel = SocketChannel.open(); - try { + try (SocketChannel realChannel = SocketChannel.open()) { TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(realChannel); @@ -457,15 +448,12 @@ void implConfigureBlocking_shouldDelegateCorrectly() throws IOException { // configureBlocking is final and calls implConfigureBlocking + updates wrapper state channel.configureBlocking(false); assertFalse(channel.isBlocking()); - } finally { - realChannel.close(); } } @Test void integrationTest_registerWithSelector() throws IOException { - SocketChannel realChannel = SocketChannel.open(); - try { + try (SocketChannel realChannel = SocketChannel.open()) { TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(realChannel); channel.configureBlocking(false); @@ -474,8 +462,6 @@ void integrationTest_registerWithSelector() throws IOException { assertFalse(channel.isBlocking()); // Verify validOps delegates correctly assertTrue(channel.validOps() > 0); - } finally { - realChannel.close(); } } @@ -523,6 +509,34 @@ void configDisabled_returnsRawSocketChannel() throws Exception { assertSame(rawSocket, result); } + @Test + void configEnabled_wrapsAcceptedSocketChannel() throws Exception { + TomcatAcceptFixConfig config = new TomcatAcceptFixConfig(); + config.retryRebindTimeoutSecs = 0; + config.tcpStackAwareSocketChannelEnabled = true; + + SelectorProvider provider = mock(SelectorProvider.class); + SocketChannel rawSocket = mock(SocketChannel.class); + doReturn(provider).when(rawSocket).provider(); + TestServerSocketChannel testServerSocket = spy(new TestServerSocketChannel(provider)); + TestEndpoint testEp = spy(new TestEndpoint(testServerSocket)); + TestProtocol testProto = spy(new TestProtocol(testEp)); + Connector conn = new Connector(testProto); + + TomcatConnectorCustomizer customizer = config.tomcatAcceptorFix(); + customizer.customize(conn); + Lifecycle lifecycle = mock(Lifecycle.class); + doReturn(LifecycleState.STARTED).when(lifecycle).getState(); + LifecycleEvent event = new LifecycleEvent(lifecycle, "type", "data"); + Stream.of(conn.findLifecycleListeners()).forEach(l -> l.lifecycleEvent(event)); + + doReturn(rawSocket).when(testServerSocket).accept(); + + SocketChannel result = testEp.serverSocketAccept(); + + assertInstanceOf(TomcatAcceptFixConfig.TcpStackAwareSocketChannel.class, result); + } + } private static class TestEndpoint extends NioEndpoint { From 42d7d588f8825d0a75b0f65d86a1b0b80a4aa0bc Mon Sep 17 00:00:00 2001 From: Jakub Balhar Date: Wed, 2 Sep 2026 07:46:33 +0200 Subject: [PATCH 6/9] fix: close stale Tomcat connections after TCP/IP restart Signed-off-by: Jakub Balhar --- .../product/web/TomcatAcceptFixConfig.java | 159 ++-------- .../web/TomcatAcceptFixConfigTest.java | 271 +----------------- 2 files changed, 25 insertions(+), 405 deletions(-) diff --git a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java index fd642b001f..ab8201c6cd 100644 --- a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java +++ b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java @@ -18,6 +18,7 @@ import org.apache.coyote.ProtocolHandler; import org.apache.tomcat.util.net.AbstractEndpoint; import org.apache.tomcat.util.net.NioEndpoint; +import org.apache.tomcat.util.net.SocketWrapperBase; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.web.embedded.tomcat.TomcatConnectorCustomizer; import org.springframework.context.annotation.Bean; @@ -30,7 +31,6 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; import java.net.SocketAddress; -import java.nio.ByteBuffer; import java.nio.channels.*; import java.nio.channels.spi.AbstractSelectableChannel; import java.nio.channels.spi.SelectorProvider; @@ -52,9 +52,6 @@ public class TomcatAcceptFixConfig { @Value("${server.tomcat.retryRebindTimeoutSecs:10}") int retryRebindTimeoutSecs; - @Value("${apiml.tcpStackAwareSocketChannel.enabled:false}") - boolean tcpStackAwareSocketChannelEnabled; - private static final Field ENDPOINT_FIELD; private static final Field NIO_SOCKET_FIELD; @@ -247,6 +244,20 @@ private void bindWithWait() throws IOException, InterruptedException { } } + private void closeConnectionsAfterTcpStackRestart() { + SocketWrapperBase[] connections = abstractEndpoint.getConnections().toArray(SocketWrapperBase[]::new); + int closed = 0; + for (SocketWrapperBase connection : connections) { + try { + connection.close(); + closed++; + } catch (RuntimeException e) { + log.debug("Unable to close a stale client connection after TCP/IP stack restart", e); + } + } + log.info("Closed {} client connection(s) after TCP/IP stack restart", closed); + } + /** * Rebind the server socket. The action could be done just by one thread. Other treats are waiting to be finish * by first one. @@ -258,6 +269,8 @@ private void bindWithWait() throws IOException, InterruptedException { private synchronized void rebind(int stateBefore) throws IOException { if (state.compareAndSet(stateBefore, stateBefore + 1)) { try { + closeConnectionsAfterTcpStackRestart(); + // socket must be closed before new binding socket.close(); @@ -282,29 +295,22 @@ public SocketChannel accept() throws IOException { // obtain current state of rebinding to detection parallel actions final int stateBefore = state.get(); try { - return wrapIfEnabled(socket.accept()); + return socket.accept(); } catch (IOException ioe) { if (TomcatAcceptFixConfig.isTcpStackRestarted(ioe)) { // the fix solve just one issue about stopped TCP/IP stack log.debug("The TCP/IP stack was probably restarted. The socket of Tomcat will rebind."); rebind(stateBefore); - return wrapIfEnabled(socket.accept()); + return socket.accept(); } throw ioe; } } - private SocketChannel wrapIfEnabled(SocketChannel socketChannel) { - if (tcpStackAwareSocketChannelEnabled) { - return new TcpStackAwareSocketChannel(socketChannel); - } - return socketChannel; - } - } /** - * The list of final methods, which cannot be delegated. See {@link FixedServerSocketChannel#socket} + * Methods final on {@link ServerSocketChannel} that Lombok must not delegate. */ private interface Overridden { @@ -325,129 +331,4 @@ private interface Overridden { } - /** - * The list of methods excluded from Lombok @Delegate for TcpStackAwareSocketChannel. - * It contains methods inherited from SocketChannel that must not be delegated and - * read/write/close-related methods that are wrapped explicitly in this class. - */ - private interface ExcludedSocketOps { - - SelectorProvider provider(); - boolean isRegistered(); - SelectionKey keyFor(Selector sel); - SelectionKey register(Selector sel, int ops, Object att); - SelectionKey register(Selector sel, int ops) throws ClosedChannelException; - boolean isBlocking(); - Object blockingLock(); - SelectableChannel configureBlocking(boolean block) throws IOException; - void implCloseChannel() throws IOException; - void close() throws IOException; - boolean isOpen(); - int validOps(); - long read(ByteBuffer[] dsts) throws IOException; - long write(ByteBuffer[] srcs) throws IOException; - int read(ByteBuffer dst) throws IOException; - long read(ByteBuffer[] dsts, int offset, int length) throws IOException; - int write(ByteBuffer src) throws IOException; - long write(ByteBuffer[] srcs, int offset, int length) throws IOException; - void implCloseSelectableChannel() throws IOException; - void implConfigureBlocking(boolean block) throws IOException; - - } - - /** - * SocketChannel wrapper that detects z/OS TCP/IP stack restarts during I/O operations. - * When EDC5122I or NetworkRecycledException is caught on read/write, the underlying - * socket is closed and the exception is re-thrown so Tomcat can discard the connection. - */ - class TcpStackAwareSocketChannel extends SocketChannel { - - @Delegate(excludes = ExcludedSocketOps.class) - private final SocketChannel delegate; - - TcpStackAwareSocketChannel(SocketChannel delegate) { - super(delegate.provider()); - this.delegate = delegate; - } - - @Override - public int read(ByteBuffer dst) throws IOException { - try { - return delegate.read(dst); - } catch (IOException e) { - if (TomcatAcceptFixConfig.isTcpStackRestarted(e)) { - log.debug("TCP/IP stack restart detected during read on client socket; closing connection", e); - safeClose(delegate); - } - throw e; - } - } - - @Override - public long read(ByteBuffer[] dsts, int offset, int length) throws IOException { - try { - return delegate.read(dsts, offset, length); - } catch (IOException e) { - if (TomcatAcceptFixConfig.isTcpStackRestarted(e)) { - log.debug("TCP/IP stack restart detected during scatter read on client socket; closing connection", e); - safeClose(delegate); - } - throw e; - } - } - - @Override - public int write(ByteBuffer src) throws IOException { - try { - return delegate.write(src); - } catch (IOException e) { - if (TomcatAcceptFixConfig.isTcpStackRestarted(e)) { - log.debug("TCP/IP stack restart detected during write on client socket; closing connection", e); - safeClose(delegate); - } - throw e; - } - } - - @Override - public long write(ByteBuffer[] srcs, int offset, int length) throws IOException { - try { - return delegate.write(srcs, offset, length); - } catch (IOException e) { - if (TomcatAcceptFixConfig.isTcpStackRestarted(e)) { - log.debug("TCP/IP stack restart detected during scatter write on client socket; closing connection", e); - safeClose(delegate); - } - throw e; - } - } - - @Override - protected void implCloseSelectableChannel() throws IOException { - // SocketChannel.close() is final; it reaches this override through - // AbstractInterruptibleChannel.close(). - delegate.close(); - } - - @Override - protected void implConfigureBlocking(boolean block) throws IOException { - try { - IMPL_CONFIGURE_BLOCKING.invoke(delegate, block); - } catch (IOException | RuntimeException e) { - throw e; - } catch (Throwable t) { - throw new IllegalStateException(t); - } - } - - private static void safeClose(SocketChannel ch) { - try { - ch.close(); - } catch (IOException e) { - log.trace("Best-effort close of client socket failed", e); - } - } - - } - } diff --git a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java index c1ecf87a43..f239068e44 100644 --- a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java +++ b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java @@ -20,6 +20,7 @@ import org.apache.coyote.http11.Http11NioProtocol; import org.apache.tomcat.util.net.AbstractEndpoint; import org.apache.tomcat.util.net.NioEndpoint; +import org.apache.tomcat.util.net.SocketWrapperBase; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -31,7 +32,6 @@ import java.net.ServerSocket; import java.net.SocketAddress; import java.net.SocketOption; -import java.nio.ByteBuffer; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.nio.channels.spi.SelectorProvider; @@ -93,8 +93,10 @@ void givenCustomizedConnector_whenTcpipIsReady_thenReturnAccepted() throws Excep } @Test - void givenCustomizedConnector_whenTcpipIsRestarted_thenRebind() throws Exception { + void givenCustomizedConnectorWithOpenConnections_whenTcpipIsRestarted_thenCloseConnectionsAndRebind() throws Exception { AtomicInteger counter = new AtomicInteger(0); + SocketWrapperBase staleConnection = mock(SocketWrapperBase.class); + doReturn(Set.of(staleConnection)).when(testEndpoint).getConnections(); doAnswer(invocation -> { if (counter.getAndIncrement() == 0) { throw new IOException("EDC5122I"); @@ -104,6 +106,7 @@ void givenCustomizedConnector_whenTcpipIsRestarted_thenRebind() throws Exception }).when(serverSocket).accept(); assertSame(socketChannel, testEndpoint.serverSocketAccept()); + verify(staleConnection).close(); verify(serverSocket, times(1)).implCloseSelectableChannel(); verify(testEndpoint, times(1)).bind(); } @@ -274,270 +277,6 @@ void givenNonMatchingException_whenIsRecycledClass_thenReturnFalse() { } - @Nested - class TcpStackAwareSocketChannelTests { - - private SocketChannel mockDelegate; - - @BeforeEach - void setUp() { - mockDelegate = mock(SocketChannel.class); - SelectorProvider provider = mock(SelectorProvider.class); - doReturn(provider).when(mockDelegate).provider(); - } - - @Test - void read_shouldPassthroughToDelegate() throws IOException { - ByteBuffer buf = ByteBuffer.allocate(10); - doReturn(5).when(mockDelegate).read(buf); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); - - assertEquals(5, channel.read(buf)); - verify(mockDelegate).read(buf); - verify(mockDelegate, never()).close(); - } - - @Test - void write_shouldPassthroughToDelegate() throws IOException { - ByteBuffer buf = ByteBuffer.allocate(10); - doReturn(5).when(mockDelegate).write(buf); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); - - assertEquals(5, channel.write(buf)); - verify(mockDelegate).write(buf); - verify(mockDelegate, never()).close(); - } - - @Test - void read_shouldCloseAndRethrowOnEdc5122I() throws IOException { - ByteBuffer buf = ByteBuffer.allocate(10); - IOException ioe = new IOException("EDC5122I TCP/IP stack was restarted"); - doThrow(ioe).when(mockDelegate).read(buf); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); - - IOException thrown = assertThrows(IOException.class, () -> channel.read(buf)); - assertSame(ioe, thrown); - verify(mockDelegate).close(); - } - - @Test - void write_shouldCloseAndRethrowOnEdc5122I() throws IOException { - ByteBuffer buf = ByteBuffer.allocate(10); - IOException ioe = new IOException("EDC5122I TCP/IP stack was restarted"); - doThrow(ioe).when(mockDelegate).write(buf); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); - - IOException thrown = assertThrows(IOException.class, () -> channel.write(buf)); - assertSame(ioe, thrown); - verify(mockDelegate).close(); - } - - @Test - void read_shouldNotCloseOnNonEdc5122I() throws IOException { - ByteBuffer buf = ByteBuffer.allocate(10); - IOException ioe = new IOException("Some other I/O error"); - doThrow(ioe).when(mockDelegate).read(buf); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); - - IOException thrown = assertThrows(IOException.class, () -> channel.read(buf)); - assertSame(ioe, thrown); - verify(mockDelegate, never()).close(); - } - - @Test - void scatterRead_shouldPassthroughToDelegate() throws IOException { - ByteBuffer[] bufs = new ByteBuffer[]{ByteBuffer.allocate(10)}; - doReturn(5L).when(mockDelegate).read(bufs, 0, 1); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); - - assertEquals(5L, channel.read(bufs, 0, 1)); - verify(mockDelegate).read(bufs, 0, 1); - } - - @Test - void scatterWrite_shouldPassthroughToDelegate() throws IOException { - ByteBuffer[] bufs = new ByteBuffer[]{ByteBuffer.allocate(10)}; - doReturn(5L).when(mockDelegate).write(bufs, 0, 1); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); - - assertEquals(5L, channel.write(bufs, 0, 1)); - verify(mockDelegate).write(bufs, 0, 1); - } - - @Test - void scatterRead_shouldCloseAndRethrowOnEdc5122I() throws IOException { - ByteBuffer[] bufs = new ByteBuffer[]{ByteBuffer.allocate(10)}; - IOException ioe = new IOException("EDC5122I scatter read failed"); - doThrow(ioe).when(mockDelegate).read(bufs, 0, 1); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); - - IOException thrown = assertThrows(IOException.class, () -> channel.read(bufs, 0, 1)); - assertSame(ioe, thrown); - verify(mockDelegate).close(); - } - - @Test - void scatterWrite_shouldCloseAndRethrowOnEdc5122I() throws IOException { - ByteBuffer[] bufs = new ByteBuffer[]{ByteBuffer.allocate(10)}; - IOException ioe = new IOException("EDC5122I scatter write failed"); - doThrow(ioe).when(mockDelegate).write(bufs, 0, 1); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); - - IOException thrown = assertThrows(IOException.class, () -> channel.write(bufs, 0, 1)); - assertSame(ioe, thrown); - verify(mockDelegate).close(); - } - - @Test - void read_shouldCloseAndRethrowOnNetworkRecycledException() throws IOException { - ByteBuffer buf = ByteBuffer.allocate(10); - IOException ioe = new IOException("Connection reset"); - doThrow(ioe).when(mockDelegate).read(buf); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); - - // Mock isRecycledClass to return true for this exception - try (MockedStatic mocked = mockStatic(TomcatAcceptFixConfig.class, CALLS_REAL_METHODS)) { - mocked.when(() -> TomcatAcceptFixConfig.isRecycledClass(any())).thenReturn(true); - - IOException thrown = assertThrows(IOException.class, () -> channel.read(buf)); - assertSame(ioe, thrown); - verify(mockDelegate).close(); - } - } - - @Test - void close_shouldCloseDelegate() throws IOException { - try (SocketChannel delegate = SocketChannel.open()) { - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(delegate); - - channel.close(); - - assertFalse(channel.isOpen()); - assertFalse(delegate.isOpen()); - } - } - - @Test - void implConfigureBlocking_shouldDelegateCorrectly() throws IOException { - try (SocketChannel realChannel = SocketChannel.open()) { - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(realChannel); - - assertTrue(channel.isBlocking()); - // configureBlocking is final and calls implConfigureBlocking + updates wrapper state - channel.configureBlocking(false); - assertFalse(channel.isBlocking()); - } - } - - @Test - void integrationTest_registerWithSelector() throws IOException { - try (SocketChannel realChannel = SocketChannel.open()) { - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(realChannel); - channel.configureBlocking(false); - - // Verify non-blocking mode is set correctly - assertFalse(channel.isBlocking()); - // Verify validOps delegates correctly - assertTrue(channel.validOps() > 0); - } - } - - @Test - void safeClose_shouldNotThrowOnIoException() throws IOException { - doThrow(new IOException("close failed")).when(mockDelegate).close(); - - TomcatAcceptFixConfig.TcpStackAwareSocketChannel channel = - new TomcatAcceptFixConfig().new TcpStackAwareSocketChannel(mockDelegate); - - ByteBuffer buf = ByteBuffer.allocate(10); - IOException ioe = new IOException("EDC5122I"); - doThrow(ioe).when(mockDelegate).read(buf); - - // Should re-throw read IOE, not close IOE - IOException thrown = assertThrows(IOException.class, () -> channel.read(buf)); - assertSame(ioe, thrown); - verify(mockDelegate).close(); // close was attempted - } - - @Test - void configDisabled_returnsRawSocketChannel() throws Exception { - TomcatAcceptFixConfig config = new TomcatAcceptFixConfig(); - config.retryRebindTimeoutSecs = 0; - config.tcpStackAwareSocketChannelEnabled = false; - - SelectorProvider provider = mock(SelectorProvider.class); - SocketChannel rawSocket = mock(SocketChannel.class); - TestServerSocketChannel testServerSocket = spy(new TestServerSocketChannel(provider)); - TestEndpoint testEp = spy(new TestEndpoint(testServerSocket)); - TestProtocol testProto = spy(new TestProtocol(testEp)); - Connector conn = new Connector(testProto); - - TomcatConnectorCustomizer customizer = config.tomcatAcceptorFix(); - customizer.customize(conn); - Lifecycle lifecycle = mock(Lifecycle.class); - doReturn(LifecycleState.STARTED).when(lifecycle).getState(); - LifecycleEvent event = new LifecycleEvent(lifecycle, "type", "data"); - Stream.of(conn.findLifecycleListeners()).forEach(l -> l.lifecycleEvent(event)); - - doReturn(rawSocket).when(testServerSocket).accept(); - - SocketChannel result = testEp.serverSocketAccept(); - // When config is disabled, the raw socket should be returned directly (not wrapped) - assertSame(rawSocket, result); - } - - @Test - void configEnabled_wrapsAcceptedSocketChannel() throws Exception { - TomcatAcceptFixConfig config = new TomcatAcceptFixConfig(); - config.retryRebindTimeoutSecs = 0; - config.tcpStackAwareSocketChannelEnabled = true; - - SelectorProvider provider = mock(SelectorProvider.class); - SocketChannel rawSocket = mock(SocketChannel.class); - doReturn(provider).when(rawSocket).provider(); - TestServerSocketChannel testServerSocket = spy(new TestServerSocketChannel(provider)); - TestEndpoint testEp = spy(new TestEndpoint(testServerSocket)); - TestProtocol testProto = spy(new TestProtocol(testEp)); - Connector conn = new Connector(testProto); - - TomcatConnectorCustomizer customizer = config.tomcatAcceptorFix(); - customizer.customize(conn); - Lifecycle lifecycle = mock(Lifecycle.class); - doReturn(LifecycleState.STARTED).when(lifecycle).getState(); - LifecycleEvent event = new LifecycleEvent(lifecycle, "type", "data"); - Stream.of(conn.findLifecycleListeners()).forEach(l -> l.lifecycleEvent(event)); - - doReturn(rawSocket).when(testServerSocket).accept(); - - SocketChannel result = testEp.serverSocketAccept(); - - assertInstanceOf(TomcatAcceptFixConfig.TcpStackAwareSocketChannel.class, result); - } - - } private static class TestEndpoint extends NioEndpoint { From 2ae3ff5188b35fefe920147168c525702e947354 Mon Sep 17 00:00:00 2001 From: Jakub Balhar Date: Tue, 8 Sep 2026 16:15:12 +0200 Subject: [PATCH 7/9] fix: address Tomcat restart recovery review feedback Signed-off-by: Jakub Balhar --- .../product/web/TomcatAcceptFixConfig.java | 11 +++----- .../web/TomcatAcceptFixConfigTest.java | 25 ++++++++++++++++--- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java index ab8201c6cd..ce25f7714f 100644 --- a/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java +++ b/apiml-tomcat-common/src/main/java/org/zowe/apiml/product/web/TomcatAcceptFixConfig.java @@ -245,17 +245,18 @@ private void bindWithWait() throws IOException, InterruptedException { } private void closeConnectionsAfterTcpStackRestart() { - SocketWrapperBase[] connections = abstractEndpoint.getConnections().toArray(SocketWrapperBase[]::new); int closed = 0; - for (SocketWrapperBase connection : connections) { + int failed = 0; + for (SocketWrapperBase connection : abstractEndpoint.getConnections()) { try { connection.close(); closed++; } catch (RuntimeException e) { + failed++; log.debug("Unable to close a stale client connection after TCP/IP stack restart", e); } } - log.info("Closed {} client connection(s) after TCP/IP stack restart", closed); + log.info("Closed {} client connection(s) after TCP/IP stack restart; {} could not be closed", closed, failed); } /** @@ -287,10 +288,6 @@ private synchronized void rebind(int stateBefore) throws IOException { } } - boolean isTcpStackRestarted(Throwable t) { - return TomcatAcceptFixConfig.isTcpStackRestarted(t); - } - public SocketChannel accept() throws IOException { // obtain current state of rebinding to detection parallel actions final int stateBefore = state.get(); diff --git a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java index f239068e44..e018bf50bf 100644 --- a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java +++ b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java @@ -106,7 +106,28 @@ void givenCustomizedConnectorWithOpenConnections_whenTcpipIsRestarted_thenCloseC }).when(serverSocket).accept(); assertSame(socketChannel, testEndpoint.serverSocketAccept()); - verify(staleConnection).close(); + verify(staleConnection, times(1)).close(); + verify(serverSocket, times(1)).implCloseSelectableChannel(); + verify(testEndpoint, times(1)).bind(); + } + + @Test + void givenOneConnectionCannotBeClosed_whenTcpipIsRestarted_thenCloseRemainingConnectionsAndRebind() throws Exception { + AtomicInteger counter = new AtomicInteger(0); + SocketWrapperBase failingConnection = mock(SocketWrapperBase.class); + SocketWrapperBase staleConnection = mock(SocketWrapperBase.class); + doReturn(Set.of(failingConnection, staleConnection)).when(testEndpoint).getConnections(); + doThrow(new IllegalStateException("already closed")).when(failingConnection).close(); + doAnswer(invocation -> { + if (counter.getAndIncrement() == 0) { + throw new IOException("EDC5122I"); + } + return socketChannel; + }).when(serverSocket).accept(); + + assertSame(socketChannel, testEndpoint.serverSocketAccept()); + verify(failingConnection, times(1)).close(); + verify(staleConnection, times(1)).close(); verify(serverSocket, times(1)).implCloseSelectableChannel(); verify(testEndpoint, times(1)).bind(); } @@ -276,8 +297,6 @@ void givenNonMatchingException_whenIsRecycledClass_thenReturnFalse() { } } - - private static class TestEndpoint extends NioEndpoint { public TestEndpoint(ServerSocketChannel serverSocket) { From 7373abe2d10b4e56491d76720c15088df9b68a2a Mon Sep 17 00:00:00 2001 From: Jakub Balhar Date: Tue, 8 Sep 2026 16:17:45 +0200 Subject: [PATCH 8/9] test: cover recycled exception detection Signed-off-by: Jakub Balhar --- .../com/ibm/net/NetworkRecycledException.java | 14 ++++++++++++++ .../product/web/TomcatAcceptFixConfigTest.java | 16 +++++----------- 2 files changed, 19 insertions(+), 11 deletions(-) create mode 100644 apiml-tomcat-common/src/test/java/com/ibm/net/NetworkRecycledException.java diff --git a/apiml-tomcat-common/src/test/java/com/ibm/net/NetworkRecycledException.java b/apiml-tomcat-common/src/test/java/com/ibm/net/NetworkRecycledException.java new file mode 100644 index 0000000000..8da1831a7e --- /dev/null +++ b/apiml-tomcat-common/src/test/java/com/ibm/net/NetworkRecycledException.java @@ -0,0 +1,14 @@ +/* + * This program and the accompanying materials are made available under the terms of the + * Eclipse Public License v2.0 which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * SPDX-License-Identifier: EPL-2.0 + * + * Copyright Contributors to the Zowe Project. + */ + +package com.ibm.net; + +public class NetworkRecycledException extends RuntimeException { +} diff --git a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java index e018bf50bf..b469f11a6d 100644 --- a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java +++ b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; +import com.ibm.net.NetworkRecycledException; import org.springframework.boot.web.embedded.tomcat.TomcatConnectorCustomizer; import java.io.IOException; @@ -279,16 +279,10 @@ void givenExceptionWithTheMessageAsCause_whenHandle_thenReturnTrue() { @Test void givenNetworkRecycledException_whenIsRecycledClass_thenReturnTrue() { - // Verify that isRecycledClass matches the exact class name - // We can't instantiate com.ibm.net.NetworkRecycledException directly, - // but we verify that isTcpStackRestarted calls isRecycledClass via cause chain - try (MockedStatic mocked = mockStatic(TomcatAcceptFixConfig.class, CALLS_REAL_METHODS)) { - mocked.when(() -> TomcatAcceptFixConfig.isRecycledClass(any())).thenReturn(true); - - Exception e = new IllegalArgumentException("Tested exception"); - e = new RuntimeException("Wrapper", e); - assertTrue(TomcatAcceptFixConfig.isTcpStackRestarted(e)); - } + Exception e = new NetworkRecycledException(); + e = new RuntimeException("Wrapper", e); + + assertTrue(TomcatAcceptFixConfig.isTcpStackRestarted(e)); } @Test From 6baac6e47738b1c10fe9e6f4db712d4410819f48 Mon Sep 17 00:00:00 2001 From: Jakub Balhar Date: Mon, 14 Sep 2026 10:40:04 +0200 Subject: [PATCH 9/9] fix: drop unused Mockito matcher import :apiml-tomcat-common:checkstyleTest fails with Unused import - org.mockito.ArgumentMatchers.any. [UnusedImports] which fails the build in every job that runs check - BuildAndTest, Register and InfinispanJGroupStabilityTest - and leaves the whole integration suite skipped behind them. Signed-off-by: Jakub Balhar --- .../org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java index b469f11a6d..0615cbf7f3 100644 --- a/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java +++ b/apiml-tomcat-common/src/test/java/org/zowe/apiml/product/web/TomcatAcceptFixConfigTest.java @@ -40,7 +40,6 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; class TomcatAcceptFixConfigTest {