From 2935fa9781eaabee23daf1c8a8e6573ab0d29673 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sat, 21 Feb 2026 12:59:22 +0530 Subject: [PATCH 1/8] feat(health,version): add health and version endponts --- .../controller/health/HealthController.java | 86 +++ .../controller/version/VersionController.java | 6 + .../admin/service/health/HealthService.java | 510 ++++++++++++++++++ .../utils/JwtUserIdValidationFilter.java | 4 +- 4 files changed, 605 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/iemr/admin/controller/health/HealthController.java create mode 100644 src/main/java/com/iemr/admin/service/health/HealthService.java diff --git a/src/main/java/com/iemr/admin/controller/health/HealthController.java b/src/main/java/com/iemr/admin/controller/health/HealthController.java new file mode 100644 index 0000000..d1a9e80 --- /dev/null +++ b/src/main/java/com/iemr/admin/controller/health/HealthController.java @@ -0,0 +1,86 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.admin.controller.health; + +import java.time.Instant; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.iemr.admin.service.health.HealthService; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; + +@RestController +@RequestMapping("/health") +@Tag(name = "Health Check", description = "APIs for checking infrastructure health status") +public class HealthController { + + private static final Logger logger = LoggerFactory.getLogger(HealthController.class); + + private final HealthService healthService; + + public HealthController(HealthService healthService) { + this.healthService = healthService; + } + + @GetMapping + @Operation(summary = "Check infrastructure health", + description = "Returns the health status of MySQL, Redis, and other configured services") + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "Services are UP or DEGRADED (operational with warnings)"), + @ApiResponse(responseCode = "503", description = "One or more critical services are DOWN") + }) + public ResponseEntity> checkHealth() { + logger.debug("Health check endpoint called"); + + try { + Map healthStatus = healthService.checkHealth(); + String overallStatus = (String) healthStatus.get("status"); + + HttpStatus httpStatus = "DOWN".equals(overallStatus) ? HttpStatus.SERVICE_UNAVAILABLE : HttpStatus.OK; + + logger.debug("Health check completed with status: {}", overallStatus); + return new ResponseEntity<>(healthStatus, httpStatus); + + } catch (Exception e) { + logger.error("Unexpected error during health check", e); + + Map errorResponse = Map.of( + "status", "DOWN", + "timestamp", Instant.now().toString() + ); + + return new ResponseEntity<>(errorResponse, HttpStatus.SERVICE_UNAVAILABLE); + } + } +} + diff --git a/src/main/java/com/iemr/admin/controller/version/VersionController.java b/src/main/java/com/iemr/admin/controller/version/VersionController.java index fa8f0a6..4928354 100644 --- a/src/main/java/com/iemr/admin/controller/version/VersionController.java +++ b/src/main/java/com/iemr/admin/controller/version/VersionController.java @@ -19,6 +19,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ + package com.iemr.admin.controller.version; import java.io.BufferedReader; @@ -36,6 +37,11 @@ import com.iemr.admin.utils.response.OutputResponse; import io.swagger.v3.oas.annotations.Operation; +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.Properties; +import org.springframework.http.MediaType; @RestController diff --git a/src/main/java/com/iemr/admin/service/health/HealthService.java b/src/main/java/com/iemr/admin/service/health/HealthService.java new file mode 100644 index 0000000..f80d846 --- /dev/null +++ b/src/main/java/com/iemr/admin/service/health/HealthService.java @@ -0,0 +1,510 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +package com.iemr.admin.service.health; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; +import jakarta.annotation.PreDestroy; +import javax.sql.DataSource; +import com.zaxxer.hikari.HikariDataSource; +import com.zaxxer.hikari.HikariPoolMXBean; +import java.lang.management.ManagementFactory; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Service; + +@Service +public class HealthService { + + private static final Logger logger = LoggerFactory.getLogger(HealthService.class); + + private static final String STATUS_KEY = "status"; + private static final String STATUS_UP = "UP"; + private static final String STATUS_DOWN = "DOWN"; + private static final String STATUS_DEGRADED = "DEGRADED"; + private static final String SEVERITY_KEY = "severity"; + private static final String SEVERITY_OK = "OK"; + private static final String SEVERITY_WARNING = "WARNING"; + private static final String SEVERITY_CRITICAL = "CRITICAL"; + private static final String ERROR_KEY = "error"; + private static final String MESSAGE_KEY = "message"; + private static final String RESPONSE_TIME_KEY = "responseTimeMs"; + private static final long MYSQL_TIMEOUT_SECONDS = 3; + private static final long REDIS_TIMEOUT_SECONDS = 3; + + private static final long ADVANCED_CHECKS_THROTTLE_SECONDS = 30; + private static final long RESPONSE_TIME_THRESHOLD_MS = 2000; + + private static final String DIAGNOSTIC_LOCK_WAIT = "MYSQL_LOCK_WAIT"; + private static final String DIAGNOSTIC_DEADLOCK = "MYSQL_DEADLOCK"; + private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; + private static final String DIAGNOSTIC_POOL_EXHAUSTED = "MYSQL_POOL_EXHAUSTED"; + private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; + + private final DataSource dataSource; + private final RedisTemplate redisTemplate; + private final ExecutorService executorService; + + private volatile long lastAdvancedCheckTime = 0; + private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; + private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); + + private volatile boolean deadlockCheckDisabled = false; + + private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true; + + public HealthService(DataSource dataSource, + @Autowired(required = false) RedisTemplate redisTemplate) { + this.dataSource = dataSource; + this.redisTemplate = redisTemplate; + this.executorService = Executors.newFixedThreadPool(2); + } + + @PreDestroy + public void shutdown() { + if (executorService != null && !executorService.isShutdown()) { + try { + executorService.shutdown(); + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { + executorService.shutdownNow(); + logger.warn("ExecutorService did not terminate gracefully"); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); + Thread.currentThread().interrupt(); + logger.warn("ExecutorService shutdown interrupted", e); + } + } + } + + public Map checkHealth() { + Map response = new LinkedHashMap<>(); + response.put("timestamp", Instant.now().toString()); + + Map mysqlStatus = new ConcurrentHashMap<>(); + Map redisStatus = new ConcurrentHashMap<>(); + + Future mysqlFuture = executorService.submit( + () -> performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealthSync)); + Future redisFuture = executorService.submit( + () -> performHealthCheck("Redis", redisStatus, this::checkRedisHealthSync)); + + long maxTimeout = Math.max(MYSQL_TIMEOUT_SECONDS, REDIS_TIMEOUT_SECONDS) + 1; + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(maxTimeout); + try { + mysqlFuture.get(maxTimeout, TimeUnit.SECONDS); + long remainingNs = deadlineNs - System.nanoTime(); + if (remainingNs > 0) { + redisFuture.get(remainingNs, TimeUnit.NANOSECONDS); + } else { + redisFuture.cancel(true); + } + } catch (TimeoutException e) { + logger.warn("Health check aggregate timeout after {} seconds", maxTimeout); + mysqlFuture.cancel(true); + redisFuture.cancel(true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("Health check was interrupted"); + mysqlFuture.cancel(true); + redisFuture.cancel(true); + } catch (Exception e) { + logger.warn("Health check execution error: {}", e.getMessage()); + } + + ensurePopulated(mysqlStatus, "MySQL"); + ensurePopulated(redisStatus, "Redis"); + + Map> components = new LinkedHashMap<>(); + components.put("mysql", mysqlStatus); + components.put("redis", redisStatus); + + response.put("components", components); + + String overallStatus = computeOverallStatus(components); + response.put(STATUS_KEY, overallStatus); + + return response; + } + + private void ensurePopulated(Map status, String componentName) { + if (!status.containsKey(STATUS_KEY)) { + status.put(STATUS_KEY, STATUS_DOWN); + status.put(SEVERITY_KEY, SEVERITY_CRITICAL); + status.put(ERROR_KEY, componentName + " health check did not complete in time"); + } + } + + private HealthCheckResult checkMySQLHealthSync() { + try (Connection connection = dataSource.getConnection(); + PreparedStatement stmt = connection.prepareStatement("SELECT 1 as health_check")) { + + stmt.setQueryTimeout((int) MYSQL_TIMEOUT_SECONDS); + + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + boolean isDegraded = performAdvancedMySQLChecksWithThrottle(connection); + return new HealthCheckResult(true, null, isDegraded); + } + } + + return new HealthCheckResult(false, "No result from health check query", false); + + } catch (Exception e) { + logger.warn("MySQL health check failed: {}", e.getMessage(), e); + return new HealthCheckResult(false, "MySQL connection failed", false); + } + } + + private HealthCheckResult checkRedisHealthSync() { + if (redisTemplate == null) { + return new HealthCheckResult(true, "Redis not configured — skipped", false); + } + + try { + String pong = redisTemplate.execute((org.springframework.data.redis.core.RedisCallback) (connection) -> connection.ping()); + + if ("PONG".equals(pong)) { + return new HealthCheckResult(true, null, false); + } + + return new HealthCheckResult(false, "Redis PING failed", false); + + } catch (Exception e) { + logger.warn("Redis health check failed: {}", e.getMessage(), e); + return new HealthCheckResult(false, "Redis connection failed", false); + } + } + + private Map performHealthCheck(String componentName, + Map status, + Supplier checker) { + long startTime = System.currentTimeMillis(); + + try { + HealthCheckResult result = checker.get(); + long responseTime = System.currentTimeMillis() - startTime; + + String componentStatus; + if (!result.isHealthy) { + componentStatus = STATUS_DOWN; + } else if (result.isDegraded) { + componentStatus = STATUS_DEGRADED; + } else { + componentStatus = STATUS_UP; + } + status.put(STATUS_KEY, componentStatus); + + status.put(RESPONSE_TIME_KEY, responseTime); + + String severity = determineSeverity(result.isHealthy, responseTime, result.isDegraded); + status.put(SEVERITY_KEY, severity); + + if (result.error != null) { + String fieldKey = result.isHealthy ? MESSAGE_KEY : ERROR_KEY; + status.put(fieldKey, result.error); + } + + return status; + + } catch (Exception e) { + long responseTime = System.currentTimeMillis() - startTime; + logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); + + status.put(STATUS_KEY, STATUS_DOWN); + status.put(RESPONSE_TIME_KEY, responseTime); + status.put(SEVERITY_KEY, SEVERITY_CRITICAL); + status.put(ERROR_KEY, "Health check failed with an unexpected error"); + + return status; + } + } + + private String determineSeverity(boolean isHealthy, long responseTimeMs, boolean isDegraded) { + if (!isHealthy) { + return SEVERITY_CRITICAL; + } + + if (isDegraded) { + return SEVERITY_WARNING; + } + + if (responseTimeMs > RESPONSE_TIME_THRESHOLD_MS) { + return SEVERITY_WARNING; + } + + return SEVERITY_OK; + } + + private String computeOverallStatus(Map> components) { + boolean hasCritical = false; + boolean hasDegraded = false; + + for (Map componentStatus : components.values()) { + String status = (String) componentStatus.get(STATUS_KEY); + String severity = (String) componentStatus.get(SEVERITY_KEY); + + if (STATUS_DOWN.equals(status) || SEVERITY_CRITICAL.equals(severity)) { + hasCritical = true; + } + + if (STATUS_DEGRADED.equals(status)) { + hasDegraded = true; + } + + if (SEVERITY_WARNING.equals(severity)) { + hasDegraded = true; + } + } + + if (hasCritical) { + return STATUS_DOWN; + } + + if (hasDegraded) { + return STATUS_DEGRADED; + } + + return STATUS_UP; + } + + private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { + if (!ADVANCED_HEALTH_CHECKS_ENABLED) { + return false; + } + + long currentTime = System.currentTimeMillis(); + + advancedCheckLock.readLock().lock(); + try { + if (cachedAdvancedCheckResult != null && + (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { + return cachedAdvancedCheckResult.isDegraded; + } + } finally { + advancedCheckLock.readLock().unlock(); + } + + advancedCheckLock.writeLock().lock(); + try { + if (cachedAdvancedCheckResult != null && + (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { + return cachedAdvancedCheckResult.isDegraded; + } + + AdvancedCheckResult result = performAdvancedMySQLChecks(connection); + + lastAdvancedCheckTime = currentTime; + cachedAdvancedCheckResult = result; + + return result.isDegraded; + } finally { + advancedCheckLock.writeLock().unlock(); + } + } + + private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { + try { + boolean hasIssues = false; + + if (hasLockWaits(connection)) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_LOCK_WAIT); + hasIssues = true; + } + + if (hasDeadlocks(connection)) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_DEADLOCK); + hasIssues = true; + } + + if (hasSlowQueries(connection)) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_SLOW_QUERIES); + hasIssues = true; + } + + if (hasConnectionPoolExhaustion()) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_POOL_EXHAUSTED); + hasIssues = true; + } + + return new AdvancedCheckResult(hasIssues); + } catch (Exception e) { + logger.debug("Advanced MySQL checks encountered exception, marking degraded"); + return new AdvancedCheckResult(true); + } + } + + private boolean hasLockWaits(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE (state = 'Waiting for table metadata lock' " + + " OR state = 'Waiting for row lock' " + + " OR state = 'Waiting for lock') " + + "AND user = USER()")) { + stmt.setQueryTimeout(2); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + int lockCount = rs.getInt(1); + return lockCount > 0; + } + } + } catch (Exception e) { + logger.debug("Could not check for lock waits"); + } + return false; + } + + private boolean hasDeadlocks(Connection connection) { + if (deadlockCheckDisabled) { + return false; + } + + try (PreparedStatement stmt = connection.prepareStatement("SHOW ENGINE INNODB STATUS")) { + stmt.setQueryTimeout(2); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + String innodbStatus = rs.getString(3); + return innodbStatus != null && innodbStatus.contains("LATEST DETECTED DEADLOCK"); + } + } + } catch (java.sql.SQLException e) { + if (e.getErrorCode() == 1142 || e.getErrorCode() == 1227) { + deadlockCheckDisabled = true; + logger.warn("Deadlock check disabled: Insufficient privileges"); + } else { + logger.debug("Could not check for deadlocks"); + } + } catch (Exception e) { + logger.debug("Could not check for deadlocks"); + } + return false; + } + + private boolean hasSlowQueries(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE command != 'Sleep' AND time > ? AND user NOT IN ('event_scheduler', 'system user')")) { + stmt.setQueryTimeout(2); + stmt.setInt(1, 10); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + int slowQueryCount = rs.getInt(1); + return slowQueryCount > 3; + } + } + } catch (Exception e) { + logger.debug("Could not check for slow queries"); + } + return false; + } + + private boolean hasConnectionPoolExhaustion() { + if (dataSource instanceof HikariDataSource hikariDataSource) { + try { + HikariPoolMXBean poolMXBean = hikariDataSource.getHikariPoolMXBean(); + + if (poolMXBean != null) { + int activeConnections = poolMXBean.getActiveConnections(); + int maxPoolSize = hikariDataSource.getMaximumPoolSize(); + + int threshold = (int) (maxPoolSize * 0.8); + return activeConnections > threshold; + } + } catch (Exception e) { + logger.debug("Could not retrieve HikariCP pool metrics"); + } + } + + return checkPoolMetricsViaJMX(); + } + + private boolean checkPoolMetricsViaJMX() { + try { + MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName("com.zaxxer.hikari:type=Pool (*)"); + var mBeans = mBeanServer.queryMBeans(objectName, null); + + for (var mBean : mBeans) { + if (evaluatePoolMetrics(mBeanServer, mBean.getObjectName())) { + return true; + } + } + } catch (Exception e) { + logger.debug("Could not access HikariCP pool metrics via JMX"); + } + + logger.debug("Pool exhaustion check disabled: HikariCP metrics unavailable"); + return false; + } + + private boolean evaluatePoolMetrics(MBeanServer mBeanServer, ObjectName objectName) { + try { + Integer activeConnections = (Integer) mBeanServer.getAttribute(objectName, "ActiveConnections"); + Integer maximumPoolSize = (Integer) mBeanServer.getAttribute(objectName, "MaximumPoolSize"); + + if (activeConnections != null && maximumPoolSize != null) { + int threshold = (int) (maximumPoolSize * 0.8); + return activeConnections > threshold; + } + } catch (Exception e) { + // Continue to next MBean + } + return false; + } + + private static class AdvancedCheckResult { + final boolean isDegraded; + + AdvancedCheckResult(boolean isDegraded) { + this.isDegraded = isDegraded; + } + } + + private static class HealthCheckResult { + final boolean isHealthy; + final String error; + final boolean isDegraded; + + HealthCheckResult(boolean isHealthy, String error, boolean isDegraded) { + this.isHealthy = isHealthy; + this.error = error; + this.isDegraded = isDegraded; + } + } +} diff --git a/src/main/java/com/iemr/admin/utils/JwtUserIdValidationFilter.java b/src/main/java/com/iemr/admin/utils/JwtUserIdValidationFilter.java index e6c072d..8cca838 100644 --- a/src/main/java/com/iemr/admin/utils/JwtUserIdValidationFilter.java +++ b/src/main/java/com/iemr/admin/utils/JwtUserIdValidationFilter.java @@ -112,7 +112,9 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo || path.startsWith(contextPath + "/swagger-ui") || path.startsWith(contextPath + "/v3/api-docs") || path.startsWith(contextPath + "/user/refreshToken") - || path.startsWith(contextPath + "/public")) { + || path.startsWith(contextPath + "/public") + || path.equals(contextPath + "/health") + || path.equals(contextPath + "/version")) { logger.info("Skipping filter for path: " + path); filterChain.doFilter(servletRequest, servletResponse); return; From 9c2a2c44a9f9c0f9c9bd719713189b077e1ced37 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sat, 21 Feb 2026 13:16:54 +0530 Subject: [PATCH 2/8] fix(health): add constant and remove duplicates --- .../com/iemr/admin/utils/JwtUserIdValidationFilter.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/admin/utils/JwtUserIdValidationFilter.java b/src/main/java/com/iemr/admin/utils/JwtUserIdValidationFilter.java index 8cca838..bd65d43 100644 --- a/src/main/java/com/iemr/admin/utils/JwtUserIdValidationFilter.java +++ b/src/main/java/com/iemr/admin/utils/JwtUserIdValidationFilter.java @@ -20,6 +20,9 @@ public class JwtUserIdValidationFilter implements Filter { + private static final String HEALTH_ENDPOINT = "/health"; + private static final String VERSION_ENDPOINT = "/version"; + private final JwtAuthenticationUtil jwtAuthenticationUtil; private final Logger logger = LoggerFactory.getLogger(this.getClass().getName()); private final String allowedOrigins; @@ -113,8 +116,8 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo || path.startsWith(contextPath + "/v3/api-docs") || path.startsWith(contextPath + "/user/refreshToken") || path.startsWith(contextPath + "/public") - || path.equals(contextPath + "/health") - || path.equals(contextPath + "/version")) { + || path.equals(contextPath + HEALTH_ENDPOINT) + || path.equals(contextPath + VERSION_ENDPOINT)) { logger.info("Skipping filter for path: " + path); filterChain.doFilter(servletRequest, servletResponse); return; From 4f6f3819e15cacb8584e99a39909472c3773e9c7 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sat, 21 Feb 2026 13:25:56 +0530 Subject: [PATCH 3/8] fix(health): avoid permanent DEGRADED from historical deadlocks --- .../admin/service/health/HealthService.java | 53 +++++-------------- 1 file changed, 13 insertions(+), 40 deletions(-) diff --git a/src/main/java/com/iemr/admin/service/health/HealthService.java b/src/main/java/com/iemr/admin/service/health/HealthService.java index f80d846..02aa60b 100644 --- a/src/main/java/com/iemr/admin/service/health/HealthService.java +++ b/src/main/java/com/iemr/admin/service/health/HealthService.java @@ -85,8 +85,6 @@ public class HealthService { private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); - private volatile boolean deadlockCheckDisabled = false; - private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true; public HealthService(DataSource dataSource, @@ -179,7 +177,7 @@ private HealthCheckResult checkMySQLHealthSync() { try (ResultSet rs = stmt.executeQuery()) { if (rs.next()) { - boolean isDegraded = performAdvancedMySQLChecksWithThrottle(connection); + boolean isDegraded = performAdvancedMySQLChecksWithThrottle(); return new HealthCheckResult(true, null, isDegraded); } } @@ -304,7 +302,7 @@ private String computeOverallStatus(Map> components) return STATUS_UP; } - private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { + private boolean performAdvancedMySQLChecksWithThrottle() { if (!ADVANCED_HEALTH_CHECKS_ENABLED) { return false; } @@ -328,12 +326,17 @@ private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { return cachedAdvancedCheckResult.isDegraded; } - AdvancedCheckResult result = performAdvancedMySQLChecks(connection); - - lastAdvancedCheckTime = currentTime; - cachedAdvancedCheckResult = result; - - return result.isDegraded; + // Acquire a fresh connection for advanced checks to avoid pool exhaustion + try (Connection connection = dataSource.getConnection()) { + AdvancedCheckResult result = performAdvancedMySQLChecks(connection); + lastAdvancedCheckTime = currentTime; + cachedAdvancedCheckResult = result; + return result.isDegraded; + } + } catch (Exception e) { + logger.debug("Failed to get connection for advanced checks: {}", e.getMessage()); + // Return cached result or false if no cache + return cachedAdvancedCheckResult != null ? cachedAdvancedCheckResult.isDegraded : false; } finally { advancedCheckLock.writeLock().unlock(); } @@ -348,10 +351,6 @@ private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { hasIssues = true; } - if (hasDeadlocks(connection)) { - logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_DEADLOCK); - hasIssues = true; - } if (hasSlowQueries(connection)) { logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_SLOW_QUERIES); @@ -390,32 +389,6 @@ private boolean hasLockWaits(Connection connection) { return false; } - private boolean hasDeadlocks(Connection connection) { - if (deadlockCheckDisabled) { - return false; - } - - try (PreparedStatement stmt = connection.prepareStatement("SHOW ENGINE INNODB STATUS")) { - stmt.setQueryTimeout(2); - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - String innodbStatus = rs.getString(3); - return innodbStatus != null && innodbStatus.contains("LATEST DETECTED DEADLOCK"); - } - } - } catch (java.sql.SQLException e) { - if (e.getErrorCode() == 1142 || e.getErrorCode() == 1227) { - deadlockCheckDisabled = true; - logger.warn("Deadlock check disabled: Insufficient privileges"); - } else { - logger.debug("Could not check for deadlocks"); - } - } catch (Exception e) { - logger.debug("Could not check for deadlocks"); - } - return false; - } - private boolean hasSlowQueries(Connection connection) { try (PreparedStatement stmt = connection.prepareStatement( "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + From 6460f1b3ceb81e7d107df7776b9a1a86442f17c8 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sat, 21 Feb 2026 13:28:46 +0530 Subject: [PATCH 4/8] fix(health): Removed the unnecessary boolean literal --- src/main/java/com/iemr/admin/service/health/HealthService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/admin/service/health/HealthService.java b/src/main/java/com/iemr/admin/service/health/HealthService.java index 02aa60b..f651143 100644 --- a/src/main/java/com/iemr/admin/service/health/HealthService.java +++ b/src/main/java/com/iemr/admin/service/health/HealthService.java @@ -336,7 +336,7 @@ private boolean performAdvancedMySQLChecksWithThrottle() { } catch (Exception e) { logger.debug("Failed to get connection for advanced checks: {}", e.getMessage()); // Return cached result or false if no cache - return cachedAdvancedCheckResult != null ? cachedAdvancedCheckResult.isDegraded : false; + return cachedAdvancedCheckResult != null && cachedAdvancedCheckResult.isDegraded; } finally { advancedCheckLock.writeLock().unlock(); } From 8d893b36942afd0bece4d33cbf9897ef70483e66 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sat, 21 Feb 2026 14:09:43 +0530 Subject: [PATCH 5/8] fix(health): Fixed the broken lock-wait detection --- src/main/java/com/iemr/admin/service/health/HealthService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/admin/service/health/HealthService.java b/src/main/java/com/iemr/admin/service/health/HealthService.java index f651143..1586997 100644 --- a/src/main/java/com/iemr/admin/service/health/HealthService.java +++ b/src/main/java/com/iemr/admin/service/health/HealthService.java @@ -375,7 +375,7 @@ private boolean hasLockWaits(Connection connection) { "WHERE (state = 'Waiting for table metadata lock' " + " OR state = 'Waiting for row lock' " + " OR state = 'Waiting for lock') " + - "AND user = USER()")) { + "AND user = SUBSTRING_INDEX(USER(), '@', 1)")) { stmt.setQueryTimeout(2); try (ResultSet rs = stmt.executeQuery()) { if (rs.next()) { From ff5deab1396aa0dbe181ec171f877f660060670c Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sun, 22 Feb 2026 21:37:14 +0530 Subject: [PATCH 6/8] fix(health): avoid blocking DB I/O under write lock and restore interrupt flag --- .../admin/service/health/HealthService.java | 123 +++++++++++------- 1 file changed, 79 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/iemr/admin/service/health/HealthService.java b/src/main/java/com/iemr/admin/service/health/HealthService.java index 1586997..89bc923 100644 --- a/src/main/java/com/iemr/admin/service/health/HealthService.java +++ b/src/main/java/com/iemr/admin/service/health/HealthService.java @@ -29,11 +29,13 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.ExecutionException; import java.util.function.Supplier; import jakarta.annotation.PreDestroy; import javax.sql.DataSource; @@ -72,7 +74,6 @@ public class HealthService { private static final long RESPONSE_TIME_THRESHOLD_MS = 2000; private static final String DIAGNOSTIC_LOCK_WAIT = "MYSQL_LOCK_WAIT"; - private static final String DIAGNOSTIC_DEADLOCK = "MYSQL_DEADLOCK"; private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; private static final String DIAGNOSTIC_POOL_EXHAUSTED = "MYSQL_POOL_EXHAUSTED"; private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; @@ -84,6 +85,7 @@ public class HealthService { private volatile long lastAdvancedCheckTime = 0; private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); + private final AtomicBoolean advancedCheckInProgress = new AtomicBoolean(false); private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true; @@ -91,7 +93,7 @@ public HealthService(DataSource dataSource, @Autowired(required = false) RedisTemplate redisTemplate) { this.dataSource = dataSource; this.redisTemplate = redisTemplate; - this.executorService = Executors.newFixedThreadPool(2); + this.executorService = Executors.newFixedThreadPool(6); } @PreDestroy @@ -118,32 +120,8 @@ public Map checkHealth() { Map mysqlStatus = new ConcurrentHashMap<>(); Map redisStatus = new ConcurrentHashMap<>(); - Future mysqlFuture = executorService.submit( - () -> performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealthSync)); - Future redisFuture = executorService.submit( - () -> performHealthCheck("Redis", redisStatus, this::checkRedisHealthSync)); - - long maxTimeout = Math.max(MYSQL_TIMEOUT_SECONDS, REDIS_TIMEOUT_SECONDS) + 1; - long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(maxTimeout); - try { - mysqlFuture.get(maxTimeout, TimeUnit.SECONDS); - long remainingNs = deadlineNs - System.nanoTime(); - if (remainingNs > 0) { - redisFuture.get(remainingNs, TimeUnit.NANOSECONDS); - } else { - redisFuture.cancel(true); - } - } catch (TimeoutException e) { - logger.warn("Health check aggregate timeout after {} seconds", maxTimeout); - mysqlFuture.cancel(true); - redisFuture.cancel(true); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.warn("Health check was interrupted"); - mysqlFuture.cancel(true); - redisFuture.cancel(true); - } catch (Exception e) { - logger.warn("Health check execution error: {}", e.getMessage()); + if (!executorService.isShutdown()) { + performHealthChecks(mysqlStatus, redisStatus); } ensurePopulated(mysqlStatus, "MySQL"); @@ -154,13 +132,56 @@ public Map checkHealth() { components.put("redis", redisStatus); response.put("components", components); - - String overallStatus = computeOverallStatus(components); - response.put(STATUS_KEY, overallStatus); + response.put(STATUS_KEY, computeOverallStatus(components)); return response; } + private void performHealthChecks(Map mysqlStatus, Map redisStatus) { + Future mysqlFuture = null; + Future redisFuture = null; + try { + mysqlFuture = executorService.submit( + () -> performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealthSync)); + redisFuture = executorService.submit( + () -> performHealthCheck("Redis", redisStatus, this::checkRedisHealthSync)); + + awaitHealthChecks(mysqlFuture, redisFuture); + } catch (TimeoutException e) { + logger.warn("Health check aggregate timeout after {} seconds", getMaxTimeout()); + cancelFutures(mysqlFuture, redisFuture); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("Health check was interrupted"); + cancelFutures(mysqlFuture, redisFuture); + } catch (Exception e) { + logger.warn("Health check execution error: {}", e.getMessage()); + } + } + + private void awaitHealthChecks(Future mysqlFuture, Future redisFuture) throws TimeoutException, InterruptedException, ExecutionException { + long maxTimeout = getMaxTimeout(); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(maxTimeout); + + mysqlFuture.get(maxTimeout, TimeUnit.SECONDS); + long remainingNs = deadlineNs - System.nanoTime(); + + if (remainingNs > 0) { + redisFuture.get(remainingNs, TimeUnit.NANOSECONDS); + } else { + redisFuture.cancel(true); + } + } + + private long getMaxTimeout() { + return Math.max(MYSQL_TIMEOUT_SECONDS, REDIS_TIMEOUT_SECONDS) + 1; + } + + private void cancelFutures(Future mysqlFuture, Future redisFuture) { + if (mysqlFuture != null) mysqlFuture.cancel(true); + if (redisFuture != null) redisFuture.cancel(true); + } + private void ensurePopulated(Map status, String componentName) { if (!status.containsKey(STATUS_KEY)) { status.put(STATUS_KEY, STATUS_DOWN); @@ -319,26 +340,40 @@ private boolean performAdvancedMySQLChecksWithThrottle() { advancedCheckLock.readLock().unlock(); } - advancedCheckLock.writeLock().lock(); + // Only one thread may submit; others fall back to the (stale) cache + if (!advancedCheckInProgress.compareAndSet(false, true)) { + advancedCheckLock.readLock().lock(); + try { + return cachedAdvancedCheckResult != null && cachedAdvancedCheckResult.isDegraded; + } finally { + advancedCheckLock.readLock().unlock(); + } + } + try { - if (cachedAdvancedCheckResult != null && - (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { - return cachedAdvancedCheckResult.isDegraded; + // Perform DB I/O outside the write lock to avoid lock contention + AdvancedCheckResult result; + try (Connection connection = dataSource.getConnection()) { + result = performAdvancedMySQLChecks(connection); + } catch (Exception e) { + if (e.getCause() instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + logger.debug("Failed to get connection for advanced checks: {}", e.getMessage()); + result = new AdvancedCheckResult(false); } - // Acquire a fresh connection for advanced checks to avoid pool exhaustion - try (Connection connection = dataSource.getConnection()) { - AdvancedCheckResult result = performAdvancedMySQLChecks(connection); + // Re-acquire write lock only to update the cache atomically + advancedCheckLock.writeLock().lock(); + try { lastAdvancedCheckTime = currentTime; cachedAdvancedCheckResult = result; return result.isDegraded; + } finally { + advancedCheckLock.writeLock().unlock(); } - } catch (Exception e) { - logger.debug("Failed to get connection for advanced checks: {}", e.getMessage()); - // Return cached result or false if no cache - return cachedAdvancedCheckResult != null && cachedAdvancedCheckResult.isDegraded; } finally { - advancedCheckLock.writeLock().unlock(); + advancedCheckInProgress.set(false); } } @@ -392,7 +427,7 @@ private boolean hasLockWaits(Connection connection) { private boolean hasSlowQueries(Connection connection) { try (PreparedStatement stmt = connection.prepareStatement( "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + - "WHERE command != 'Sleep' AND time > ? AND user NOT IN ('event_scheduler', 'system user')")) { + "WHERE command != 'Sleep' AND time > ? AND user = SUBSTRING_INDEX(USER(), '@', 1)")) { stmt.setQueryTimeout(2); stmt.setInt(1, 10); try (ResultSet rs = stmt.executeQuery()) { From 6f29f4d23145f26282d476f89555cacb707a0977 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sun, 22 Feb 2026 21:47:46 +0530 Subject: [PATCH 7/8] fix(health): add cancelFutures in healthservice --- src/main/java/com/iemr/admin/service/health/HealthService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/iemr/admin/service/health/HealthService.java b/src/main/java/com/iemr/admin/service/health/HealthService.java index 89bc923..10c9037 100644 --- a/src/main/java/com/iemr/admin/service/health/HealthService.java +++ b/src/main/java/com/iemr/admin/service/health/HealthService.java @@ -156,6 +156,7 @@ private void performHealthChecks(Map mysqlStatus, Map Date: Tue, 24 Feb 2026 11:10:35 +0530 Subject: [PATCH 8/8] fix(health): close basic DB connection before advanced checks and remove shared-map race --- .../admin/service/health/HealthService.java | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/iemr/admin/service/health/HealthService.java b/src/main/java/com/iemr/admin/service/health/HealthService.java index 10c9037..e6a0432 100644 --- a/src/main/java/com/iemr/admin/service/health/HealthService.java +++ b/src/main/java/com/iemr/admin/service/health/HealthService.java @@ -48,9 +48,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; +import java.util.concurrent.atomic.AtomicInteger; + @Service public class HealthService { @@ -87,13 +90,23 @@ public class HealthService { private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); private final AtomicBoolean advancedCheckInProgress = new AtomicBoolean(false); - private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true; + @Value("${health.advanced.checks.enabled:true}") + private boolean advancedHealthChecksEnabled; public HealthService(DataSource dataSource, @Autowired(required = false) RedisTemplate redisTemplate) { this.dataSource = dataSource; this.redisTemplate = redisTemplate; - this.executorService = Executors.newFixedThreadPool(6); + this.executorService = Executors.newFixedThreadPool(2, new java.util.concurrent.ThreadFactory() { + private final AtomicInteger threadCounter = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r, "health-check-" + threadCounter.incrementAndGet()); + t.setDaemon(true); + return t; + } + }); } @PreDestroy @@ -184,11 +197,10 @@ private void cancelFutures(Future mysqlFuture, Future redisFuture) { } private void ensurePopulated(Map status, String componentName) { - if (!status.containsKey(STATUS_KEY)) { - status.put(STATUS_KEY, STATUS_DOWN); - status.put(SEVERITY_KEY, SEVERITY_CRITICAL); - status.put(ERROR_KEY, componentName + " health check did not complete in time"); - } + + status.putIfAbsent(STATUS_KEY, STATUS_DOWN); + status.putIfAbsent(SEVERITY_KEY, SEVERITY_CRITICAL); + status.putIfAbsent(ERROR_KEY, componentName + " health check did not complete in time"); } private HealthCheckResult checkMySQLHealthSync() { @@ -199,17 +211,19 @@ private HealthCheckResult checkMySQLHealthSync() { try (ResultSet rs = stmt.executeQuery()) { if (rs.next()) { - boolean isDegraded = performAdvancedMySQLChecksWithThrottle(); - return new HealthCheckResult(true, null, isDegraded); + // Connection is auto-closed by try-with-resources here + // Advanced checks will open a separate connection if needed } } - return new HealthCheckResult(false, "No result from health check query", false); - } catch (Exception e) { logger.warn("MySQL health check failed: {}", e.getMessage(), e); return new HealthCheckResult(false, "MySQL connection failed", false); } + + + boolean isDegraded = performAdvancedMySQLChecksWithThrottle(); + return new HealthCheckResult(true, null, isDegraded); } private HealthCheckResult checkRedisHealthSync() { @@ -325,7 +339,7 @@ private String computeOverallStatus(Map> components) } private boolean performAdvancedMySQLChecksWithThrottle() { - if (!ADVANCED_HEALTH_CHECKS_ENABLED) { + if (!advancedHealthChecksEnabled) { return false; }