diff --git a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java index d0b9eec793..16e8fada28 100644 --- a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java +++ b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java @@ -198,6 +198,13 @@ public final class Messages { public static final String THREADS_FOR_FILE_STORAGE_UPLOAD_0 = "Threads for file storage upload: {0}"; public static final String DELETED_ORPHANED_MTA_DESCRIPTORS_COUNT = "Deleted orphaned mta descriptors count: {0}"; public static final String IS_HEALTH_CHECK_ENABLED = "Is health check enabled: {0}"; + public static final String OPERATION_RATE_LIMITING_ENABLED = "Operation rate limiting enabled: {0}"; + public static final String MAX_ACTIVE_OPERATIONS_PER_SPACE = "Max active operations per space: {0}"; + public static final String MAX_ACTIVE_OPERATIONS_PER_USER = "Max active operations per user: {0}"; + public static final String OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY = "Operation rate limit per space capacity: {0}"; + public static final String OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR = "Operation rate limit per space refill per hour: {0}"; + public static final String OPERATION_RATE_LIMIT_PER_USER_CAPACITY = "Operation rate limit per user capacity: {0}"; + public static final String OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR = "Operation rate limit per user refill per hour: {0}"; // Debug messages public static final String DEPLOYMENT_DESCRIPTOR = "Deployment descriptor: {0}"; diff --git a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java index 143f7972e5..d460e88a9a 100644 --- a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java +++ b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java @@ -100,6 +100,13 @@ public class ApplicationConfiguration { static final String CFG_THREADS_FOR_FILE_UPLOAD_TO_CONTROLLER = "THREADS_FOR_FILE_UPLOAD_TO_CONTROLLER"; static final String CFG_THREADS_FOR_FILE_STORAGE_UPLOAD = "THREADS_FOR_FILE_STORAGE_UPLOAD"; static final String CFG_IS_HEALTH_CHECK_ENABLED = "IS_HEALTH_CHECK_ENABLED"; + static final String CFG_OPERATION_RATE_LIMITING_ENABLED = "OPERATION_RATE_LIMITING_ENABLED"; + static final String CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE = "MAX_ACTIVE_OPERATIONS_PER_SPACE"; + static final String CFG_MAX_ACTIVE_OPERATIONS_PER_USER = "MAX_ACTIVE_OPERATIONS_PER_USER"; + static final String CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY = "OP_RATE_LIMIT_PER_SPACE_CAPACITY"; + static final String CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR = "OP_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR"; + static final String CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY = "OP_RATE_LIMIT_PER_USER_CAPACITY"; + static final String CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR = "OP_RATE_LIMIT_PER_USER_REFILL_PER_HOUR"; private static final List VCAP_APPLICATION_URIS_KEYS = List.of("full_application_uris", "application_uris", "uris"); @@ -158,6 +165,13 @@ public class ApplicationConfiguration { public static final int DEFAULT_THREADS_FOR_FILE_UPLOAD_TO_CONTROLLER = 6; public static final int DEFAULT_THREADS_FOR_FILE_STORAGE_UPLOAD = 7; public static final boolean DEFAULT_IS_HEALTH_CHECK_ENABLED = false; + public static final boolean DEFAULT_OPERATION_RATE_LIMITING_ENABLED = false; + public static final int DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE = 500; + public static final int DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER = 200; + public static final int DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY = 300; + public static final int DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR = 800; + public static final int DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY = 150; + public static final int DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR = 300; protected final Environment environment; @@ -217,6 +231,13 @@ public class ApplicationConfiguration { private Integer threadsForFileStorageUpload; private Boolean isHealthCheckEnabled; private Set objectStoreRegions; + private Boolean operationRateLimitingEnabled; + private Integer maxActiveOperationsPerSpace; + private Integer maxActiveOperationsPerUser; + private Integer operationRateLimitPerSpaceCapacity; + private Integer operationRateLimitPerSpaceRefillPerHour; + private Integer operationRateLimitPerUserCapacity; + private Integer operationRateLimitPerUserRefillPerHour; public ApplicationConfiguration() { this(new Environment()); @@ -286,7 +307,10 @@ private Set getNotSensitiveConfigVariables() { CFG_FLOWABLE_JOB_EXECUTOR_QUEUE_CAPACITY, CFG_CONTROLLER_CLIENT_CONNECTION_POOL_SIZE, CFG_CONTROLLER_CLIENT_THREAD_POOL_SIZE, CFG_CONTROLLER_CLIENT_RESPONSE_TIMEOUT, CFG_DB_TRANSACTION_TIMEOUT_IN_SECONDS, - CFG_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS, CFG_SERVICE_HANDLING_MAX_PARALLEL_THREADS); + CFG_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS, CFG_SERVICE_HANDLING_MAX_PARALLEL_THREADS, + CFG_OPERATION_RATE_LIMITING_ENABLED, CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE, CFG_MAX_ACTIVE_OPERATIONS_PER_USER, + CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY, CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR, + CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY, CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR); } public URL getControllerUrl() { @@ -668,6 +692,55 @@ public boolean isHealthCheckEnabled() { return isHealthCheckEnabled; } + public boolean isOperationRateLimitingEnabled() { + if (operationRateLimitingEnabled == null) { + operationRateLimitingEnabled = isOperationRateLimitingEnabledThroughEnvironment(); + } + return operationRateLimitingEnabled; + } + + public Integer getMaxActiveOperationsPerSpace() { + if (maxActiveOperationsPerSpace == null) { + maxActiveOperationsPerSpace = getMaxActiveOperationsPerSpaceFromEnvironment(); + } + return maxActiveOperationsPerSpace; + } + + public Integer getMaxActiveOperationsPerUser() { + if (maxActiveOperationsPerUser == null) { + maxActiveOperationsPerUser = getMaxActiveOperationsPerUserFromEnvironment(); + } + return maxActiveOperationsPerUser; + } + + public Integer getOperationRateLimitPerSpaceCapacity() { + if (operationRateLimitPerSpaceCapacity == null) { + operationRateLimitPerSpaceCapacity = getOperationRateLimitPerSpaceCapacityFromEnvironment(); + } + return operationRateLimitPerSpaceCapacity; + } + + public Integer getOperationRateLimitPerSpaceRefillPerHour() { + if (operationRateLimitPerSpaceRefillPerHour == null) { + operationRateLimitPerSpaceRefillPerHour = getOperationRateLimitPerSpaceRefillPerHourFromEnvironment(); + } + return operationRateLimitPerSpaceRefillPerHour; + } + + public Integer getOperationRateLimitPerUserCapacity() { + if (operationRateLimitPerUserCapacity == null) { + operationRateLimitPerUserCapacity = getOperationRateLimitPerUserCapacityFromEnvironment(); + } + return operationRateLimitPerUserCapacity; + } + + public Integer getOperationRateLimitPerUserRefillPerHour() { + if (operationRateLimitPerUserRefillPerHour == null) { + operationRateLimitPerUserRefillPerHour = getOperationRateLimitPerUserRefillPerHourFromEnvironment(); + } + return operationRateLimitPerUserRefillPerHour; + } + private URL getControllerUrlFromEnvironment() { String controllerUrlString = environment.getString("CF_API"); if (StringUtils.isEmpty(controllerUrlString)) { @@ -1098,6 +1171,54 @@ public boolean isHealthCheckEnabledFromEnvironment() { return value; } + private Boolean isOperationRateLimitingEnabledThroughEnvironment() { + Boolean value = environment.getBoolean(CFG_OPERATION_RATE_LIMITING_ENABLED, DEFAULT_OPERATION_RATE_LIMITING_ENABLED); + logEnvironmentVariable(CFG_OPERATION_RATE_LIMITING_ENABLED, Messages.OPERATION_RATE_LIMITING_ENABLED, value); + return value; + } + + private Integer getMaxActiveOperationsPerSpaceFromEnvironment() { + Integer value = environment.getPositiveInteger(CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE, DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE); + logEnvironmentVariable(CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE, Messages.MAX_ACTIVE_OPERATIONS_PER_SPACE, value); + return value; + } + + private Integer getMaxActiveOperationsPerUserFromEnvironment() { + Integer value = environment.getPositiveInteger(CFG_MAX_ACTIVE_OPERATIONS_PER_USER, DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER); + logEnvironmentVariable(CFG_MAX_ACTIVE_OPERATIONS_PER_USER, Messages.MAX_ACTIVE_OPERATIONS_PER_USER, value); + return value; + } + + private Integer getOperationRateLimitPerSpaceCapacityFromEnvironment() { + Integer value = environment.getPositiveInteger(CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY, + DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY); + logEnvironmentVariable(CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY, Messages.OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY, value); + return value; + } + + private Integer getOperationRateLimitPerSpaceRefillPerHourFromEnvironment() { + Integer value = environment.getPositiveInteger(CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR, + DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR); + logEnvironmentVariable(CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR, + Messages.OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR, value); + return value; + } + + private Integer getOperationRateLimitPerUserCapacityFromEnvironment() { + Integer value = environment.getPositiveInteger(CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY, + DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY); + logEnvironmentVariable(CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY, Messages.OPERATION_RATE_LIMIT_PER_USER_CAPACITY, value); + return value; + } + + private Integer getOperationRateLimitPerUserRefillPerHourFromEnvironment() { + Integer value = environment.getPositiveInteger(CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR, + DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR); + logEnvironmentVariable(CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR, + Messages.OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR, value); + return value; + } + public Boolean isInternalEnvironment() { return environment.getBoolean(SAP_INTERNAL_DELIVERY, DEFAULT_SAP_INTERNAL_DELIVERY); } diff --git a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfigurationTest.java b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfigurationTest.java index e968387926..af8e25b17a 100644 --- a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfigurationTest.java +++ b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfigurationTest.java @@ -22,7 +22,6 @@ import org.junit.jupiter.params.provider.ValueSource; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -88,8 +87,8 @@ void testGetSpaceGuidWithNull() { @ParameterizedTest @ValueSource(strings = { "", "invalid", "{}" }) void testGetSpaceGuidReturnsDefault(String envValue) { - Mockito.when(environment.getString(ApplicationConfiguration.CFG_VCAP_APPLICATION)) - .thenReturn(envValue); + when(environment.getString(ApplicationConfiguration.CFG_VCAP_APPLICATION)) + .thenReturn(envValue); assertEquals(ApplicationConfiguration.DEFAULT_SPACE_GUID, configuration.getSpaceGuid()); } @@ -127,55 +126,55 @@ void testGetPlatformNoPlatformInEnvironment() { @Test void testGetMaxUploadSize() { - Mockito.when(environment.getLong(ApplicationConfiguration.CFG_MAX_UPLOAD_SIZE, ApplicationConfiguration.DEFAULT_MAX_UPLOAD_SIZE)) - .thenReturn(ApplicationConfiguration.DEFAULT_MAX_UPLOAD_SIZE); + when(environment.getLong(ApplicationConfiguration.CFG_MAX_UPLOAD_SIZE, ApplicationConfiguration.DEFAULT_MAX_UPLOAD_SIZE)) + .thenReturn(ApplicationConfiguration.DEFAULT_MAX_UPLOAD_SIZE); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_MAX_UPLOAD_SIZE, configuration.getMaxUploadSize()); } @Test void testGetMaxMtaDescriptorSize() { - Mockito.when(environment.getLong(ApplicationConfiguration.CFG_MAX_MTA_DESCRIPTOR_SIZE, - ApplicationConfiguration.DEFAULT_MAX_MTA_DESCRIPTOR_SIZE)) - .thenReturn(ApplicationConfiguration.DEFAULT_MAX_MTA_DESCRIPTOR_SIZE); + when(environment.getLong(ApplicationConfiguration.CFG_MAX_MTA_DESCRIPTOR_SIZE, + ApplicationConfiguration.DEFAULT_MAX_MTA_DESCRIPTOR_SIZE)) + .thenReturn(ApplicationConfiguration.DEFAULT_MAX_MTA_DESCRIPTOR_SIZE); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_MAX_MTA_DESCRIPTOR_SIZE, configuration.getMaxMtaDescriptorSize()); } @Test void testGetMaxManifestFileSize() { - Mockito.when(environment.getLong(ApplicationConfiguration.CFG_MAX_MANIFEST_SIZE, - ApplicationConfiguration.DEFAULT_MAX_MANIFEST_SIZE)) - .thenReturn(ApplicationConfiguration.DEFAULT_MAX_MANIFEST_SIZE); + when(environment.getLong(ApplicationConfiguration.CFG_MAX_MANIFEST_SIZE, + ApplicationConfiguration.DEFAULT_MAX_MANIFEST_SIZE)) + .thenReturn(ApplicationConfiguration.DEFAULT_MAX_MANIFEST_SIZE); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_MAX_MANIFEST_SIZE, configuration.getMaxManifestSize()); } @Test void testGetMaxResourceFileSize() { - Mockito.when(environment.getLong(ApplicationConfiguration.CFG_MAX_RESOURCE_FILE_SIZE, - ApplicationConfiguration.DEFAULT_MAX_RESOURCE_FILE_SIZE)) - .thenReturn(ApplicationConfiguration.DEFAULT_MAX_RESOURCE_FILE_SIZE); + when(environment.getLong(ApplicationConfiguration.CFG_MAX_RESOURCE_FILE_SIZE, + ApplicationConfiguration.DEFAULT_MAX_RESOURCE_FILE_SIZE)) + .thenReturn(ApplicationConfiguration.DEFAULT_MAX_RESOURCE_FILE_SIZE); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_MAX_RESOURCE_FILE_SIZE, configuration.getMaxResourceFileSize()); } @Test void testGetCronExpressionForOldData() { - Mockito.when(environment.getString(ApplicationConfiguration.CFG_CRON_EXPRESSION_FOR_OLD_DATA)) - .thenReturn(ApplicationConfiguration.DEFAULT_CRON_EXPRESSION_FOR_OLD_DATA); + when(environment.getString(ApplicationConfiguration.CFG_CRON_EXPRESSION_FOR_OLD_DATA)) + .thenReturn(ApplicationConfiguration.DEFAULT_CRON_EXPRESSION_FOR_OLD_DATA); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_CRON_EXPRESSION_FOR_OLD_DATA, configuration.getCronExpressionForOldData()); } @Test void testGetExecutionTimeForFinishedProcesses() { - Mockito.when(environment.getString(ApplicationConfiguration.CFG_EXECUTION_TIME_FOR_FINISHED_PROCESSES)) - .thenReturn(ApplicationConfiguration.DEFAULT_EXECUTION_TIME_FOR_FINISHED_PROCESSES); + when(environment.getString(ApplicationConfiguration.CFG_EXECUTION_TIME_FOR_FINISHED_PROCESSES)) + .thenReturn(ApplicationConfiguration.DEFAULT_EXECUTION_TIME_FOR_FINISHED_PROCESSES); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_EXECUTION_TIME_FOR_FINISHED_PROCESSES, configuration.getExecutionTimeForFinishedProcesses()); } @Test void testGetMaxTtlForOldDataFromEnvironment() { - Mockito.when(environment.getLong(ApplicationConfiguration.CFG_MAX_TTL_FOR_OLD_DATA, - ApplicationConfiguration.DEFAULT_MAX_TTL_FOR_OLD_DATA)) - .thenReturn(ApplicationConfiguration.DEFAULT_MAX_TTL_FOR_OLD_DATA); + when(environment.getLong(ApplicationConfiguration.CFG_MAX_TTL_FOR_OLD_DATA, + ApplicationConfiguration.DEFAULT_MAX_TTL_FOR_OLD_DATA)) + .thenReturn(ApplicationConfiguration.DEFAULT_MAX_TTL_FOR_OLD_DATA); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_MAX_TTL_FOR_OLD_DATA, configuration.getMaxTtlForOldData()); } @@ -207,104 +206,104 @@ void testGetDeployServiceUrlUrlIsNotSet() { @Test void testIsBasicAuthEnabled() { - Mockito.when(environment.getBoolean(ApplicationConfiguration.CFG_BASIC_AUTH_ENABLED, - ApplicationConfiguration.DEFAULT_BASIC_AUTH_ENABLED)) - .thenReturn(ApplicationConfiguration.DEFAULT_BASIC_AUTH_ENABLED); + when(environment.getBoolean(ApplicationConfiguration.CFG_BASIC_AUTH_ENABLED, + ApplicationConfiguration.DEFAULT_BASIC_AUTH_ENABLED)) + .thenReturn(ApplicationConfiguration.DEFAULT_BASIC_AUTH_ENABLED); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_BASIC_AUTH_ENABLED, configuration.isBasicAuthEnabled()); } @Test void testGetGlobalAuditorUser() { String globalAuditorUser = "globalAuditorUserName"; - Mockito.when(environment.getString(ApplicationConfiguration.CFG_GLOBAL_AUDITOR_USER)) - .thenReturn(globalAuditorUser); + when(environment.getString(ApplicationConfiguration.CFG_GLOBAL_AUDITOR_USER)) + .thenReturn(globalAuditorUser); Assertions.assertEquals(globalAuditorUser, configuration.getGlobalAuditorUser()); } @Test void testGetGlobalAuditorPasswordFromEnvironment() { String globalAuditorPassword = "globalAuditorUserPassword"; - Mockito.when(environment.getString(ApplicationConfiguration.CFG_GLOBAL_AUDITOR_PASSWORD)) - .thenReturn(globalAuditorPassword); + when(environment.getString(ApplicationConfiguration.CFG_GLOBAL_AUDITOR_PASSWORD)) + .thenReturn(globalAuditorPassword); Assertions.assertEquals(globalAuditorPassword, configuration.getGlobalAuditorPassword()); } @Test void testGetDbConnectionThreadsFromEnvironment() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_DB_CONNECTION_THREADS, - ApplicationConfiguration.DEFAULT_DB_CONNECTION_THREADS)) - .thenReturn(ApplicationConfiguration.DEFAULT_DB_CONNECTION_THREADS); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_DB_CONNECTION_THREADS, + ApplicationConfiguration.DEFAULT_DB_CONNECTION_THREADS)) + .thenReturn(ApplicationConfiguration.DEFAULT_DB_CONNECTION_THREADS); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_DB_CONNECTION_THREADS, configuration.getDbConnectionThreads()); } @Test void testGetStepPollingIntervalInSeconds() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_STEP_POLLING_INTERVAL_IN_SECONDS, - ApplicationConfiguration.DEFAULT_STEP_POLLING_INTERVAL_IN_SECONDS)) - .thenReturn(ApplicationConfiguration.DEFAULT_STEP_POLLING_INTERVAL_IN_SECONDS); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_STEP_POLLING_INTERVAL_IN_SECONDS, + ApplicationConfiguration.DEFAULT_STEP_POLLING_INTERVAL_IN_SECONDS)) + .thenReturn(ApplicationConfiguration.DEFAULT_STEP_POLLING_INTERVAL_IN_SECONDS); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_STEP_POLLING_INTERVAL_IN_SECONDS, configuration.getStepPollingIntervalInSeconds()); } @Test void testShouldSkipSslValidation() { - Mockito.when(environment.getBoolean(ApplicationConfiguration.CFG_SKIP_SSL_VALIDATION, - ApplicationConfiguration.DEFAULT_SKIP_SSL_VALIDATION)) - .thenReturn(ApplicationConfiguration.DEFAULT_SKIP_SSL_VALIDATION); + when(environment.getBoolean(ApplicationConfiguration.CFG_SKIP_SSL_VALIDATION, + ApplicationConfiguration.DEFAULT_SKIP_SSL_VALIDATION)) + .thenReturn(ApplicationConfiguration.DEFAULT_SKIP_SSL_VALIDATION); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_SKIP_SSL_VALIDATION, configuration.shouldSkipSslValidation()); } @Test void testGetVersionFromEnvironment() { - Mockito.when(environment.getString(ApplicationConfiguration.CFG_VERSION, ApplicationConfiguration.DEFAULT_VERSION)) - .thenReturn(ApplicationConfiguration.DEFAULT_VERSION); + when(environment.getString(ApplicationConfiguration.CFG_VERSION, ApplicationConfiguration.DEFAULT_VERSION)) + .thenReturn(ApplicationConfiguration.DEFAULT_VERSION); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_VERSION, configuration.getVersion()); } @Test void testGetChangeLogLockPollRate() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CHANGE_LOG_LOCK_POLL_RATE, - ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_POLL_RATE)) - .thenReturn(ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_POLL_RATE); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CHANGE_LOG_LOCK_POLL_RATE, + ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_POLL_RATE)) + .thenReturn(ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_POLL_RATE); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_POLL_RATE, configuration.getChangeLogLockPollRate()); } @Test void testGetControllerClientSslHandshakeTimeout() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CONTROLLER_CLIENT_SSL_HANDSHAKE_TIMEOUT_IN_SECONDS, - ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_SSL_HANDSHAKE_TIMEOUT_IN_SECONDS)) - .thenReturn(120); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CONTROLLER_CLIENT_SSL_HANDSHAKE_TIMEOUT_IN_SECONDS, + ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_SSL_HANDSHAKE_TIMEOUT_IN_SECONDS)) + .thenReturn(120); assertEquals(Duration.ofSeconds(120), configuration.getControllerClientSslHandshakeTimeout()); } @Test void testGetControllerClientConnectTimeout() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CONTROLLER_CLIENT_CONNECT_TIMEOUT_IN_SECONDS, - ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_CONNECT_TIMEOUT_IN_SECONDS)) - .thenReturn(10); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CONTROLLER_CLIENT_CONNECT_TIMEOUT_IN_SECONDS, + ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_CONNECT_TIMEOUT_IN_SECONDS)) + .thenReturn(10); assertEquals(Duration.ofSeconds(10), configuration.getControllerClientConnectTimeout()); } void testGetChangeLogLockDuration() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CHANGE_LOG_LOCK_DURATION, - ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_DURATION)) - .thenReturn(ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_DURATION); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CHANGE_LOG_LOCK_DURATION, + ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_DURATION)) + .thenReturn(ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_DURATION); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_DURATION, configuration.getChangeLogLockDuration()); } @Test void testGetChangeLogLockAttempts() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CHANGE_LOG_LOCK_ATTEMPTS, - ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_ATTEMPTS)) - .thenReturn(ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_ATTEMPTS); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CHANGE_LOG_LOCK_ATTEMPTS, + ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_ATTEMPTS)) + .thenReturn(ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_ATTEMPTS); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_CHANGE_LOG_LOCK_ATTEMPTS, configuration.getChangeLogLockAttempts()); } @Test void testGetGlobalConfigSpace() { String globalConfigSpace = "globalConfigSpace"; - Mockito.when(environment.getString(ApplicationConfiguration.CFG_GLOBAL_CONFIG_SPACE)) - .thenReturn(globalConfigSpace); + when(environment.getString(ApplicationConfiguration.CFG_GLOBAL_CONFIG_SPACE)) + .thenReturn(globalConfigSpace); Assertions.assertEquals(globalConfigSpace, configuration.getGlobalConfigSpace()); } @@ -314,15 +313,15 @@ void testGetHealthCheckConfigurationFromEnvironment() { String healthCheckMtaId = "healthCheckMtaId"; String healthCheckUserName = "healthCheckUserId"; int healthCheckTimeRange = 10; - Mockito.when(environment.getString(ApplicationConfiguration.CFG_HEALTH_CHECK_SPACE_GUID)) - .thenReturn(healthCheckSpaceId); - Mockito.when(environment.getString(ApplicationConfiguration.CFG_HEALTH_CHECK_MTA_ID)) - .thenReturn(healthCheckMtaId); - Mockito.when(environment.getString(ApplicationConfiguration.CFG_HEALTH_CHECK_USER)) - .thenReturn(healthCheckUserName); - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_HEALTH_CHECK_TIME_RANGE, - ApplicationConfiguration.DEFAULT_HEALTH_CHECK_TIME_RANGE)) - .thenReturn(healthCheckTimeRange); + when(environment.getString(ApplicationConfiguration.CFG_HEALTH_CHECK_SPACE_GUID)) + .thenReturn(healthCheckSpaceId); + when(environment.getString(ApplicationConfiguration.CFG_HEALTH_CHECK_MTA_ID)) + .thenReturn(healthCheckMtaId); + when(environment.getString(ApplicationConfiguration.CFG_HEALTH_CHECK_USER)) + .thenReturn(healthCheckUserName); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_HEALTH_CHECK_TIME_RANGE, + ApplicationConfiguration.DEFAULT_HEALTH_CHECK_TIME_RANGE)) + .thenReturn(healthCheckTimeRange); HealthCheckConfiguration healthCheckConfiguration = configuration.getHealthCheckConfiguration(); Assertions.assertEquals(healthCheckSpaceId, healthCheckConfiguration.getSpaceId()); Assertions.assertEquals(healthCheckMtaId, healthCheckConfiguration.getMtaId()); @@ -339,95 +338,95 @@ void testGetApplicationGuid() { @Test void testGetApplicationInstanceIndex() { Integer instanceIndex = 1; - Mockito.when(environment.getInteger(ApplicationConfiguration.CFG_CF_INSTANCE_INDEX)) - .thenReturn(instanceIndex); + when(environment.getInteger(ApplicationConfiguration.CFG_CF_INSTANCE_INDEX)) + .thenReturn(instanceIndex); Assertions.assertEquals(instanceIndex, configuration.getApplicationInstanceIndex()); } @Test void testGetFlowableJobExecutorCoreThreads() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_FLOWABLE_JOB_EXECUTOR_CORE_THREADS, - ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_CORE_THREADS)) - .thenReturn(ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_CORE_THREADS); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_FLOWABLE_JOB_EXECUTOR_CORE_THREADS, + ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_CORE_THREADS)) + .thenReturn(ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_CORE_THREADS); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_CORE_THREADS, configuration.getFlowableJobExecutorCoreThreads()); } @Test void testGetFlowableJobExecutorMaxThreads() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_FLOWABLE_JOB_EXECUTOR_MAX_THREADS, - ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_MAX_THREADS)) - .thenReturn(ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_MAX_THREADS); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_FLOWABLE_JOB_EXECUTOR_MAX_THREADS, + ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_MAX_THREADS)) + .thenReturn(ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_MAX_THREADS); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_MAX_THREADS, configuration.getFlowableJobExecutorMaxThreads()); } @Test void testGetFlowableJobExecutorQueueCapacity() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_FLOWABLE_JOB_EXECUTOR_QUEUE_CAPACITY, - ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_QUEUE_CAPACITY)) - .thenReturn(ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_QUEUE_CAPACITY); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_FLOWABLE_JOB_EXECUTOR_QUEUE_CAPACITY, + ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_QUEUE_CAPACITY)) + .thenReturn(ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_QUEUE_CAPACITY); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_FLOWABLE_JOB_EXECUTOR_QUEUE_CAPACITY, configuration.getFlowableJobExecutorQueueCapacity()); } @Test void testGetFssCacheUpdateTimeoutMinutes() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_FSS_CACHE_UPDATE_TIMEOUT_MINUTES, - ApplicationConfiguration.DEFAULT_FSS_CACHE_UPDATE_TIMEOUT_MINUTES)) - .thenReturn(ApplicationConfiguration.DEFAULT_FSS_CACHE_UPDATE_TIMEOUT_MINUTES); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_FSS_CACHE_UPDATE_TIMEOUT_MINUTES, + ApplicationConfiguration.DEFAULT_FSS_CACHE_UPDATE_TIMEOUT_MINUTES)) + .thenReturn(ApplicationConfiguration.DEFAULT_FSS_CACHE_UPDATE_TIMEOUT_MINUTES); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_FSS_CACHE_UPDATE_TIMEOUT_MINUTES, configuration.getFssCacheUpdateTimeoutMinutes()); } @Test void testGetSpaceDeveloperCacheExpirationInSeconds() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_SPACE_DEVELOPER_CACHE_TIME_IN_SECONDS, - ApplicationConfiguration.DEFAULT_SPACE_DEVELOPER_CACHE_TIME_IN_SECONDS)) - .thenReturn(ApplicationConfiguration.DEFAULT_SPACE_DEVELOPER_CACHE_TIME_IN_SECONDS); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_SPACE_DEVELOPER_CACHE_TIME_IN_SECONDS, + ApplicationConfiguration.DEFAULT_SPACE_DEVELOPER_CACHE_TIME_IN_SECONDS)) + .thenReturn(ApplicationConfiguration.DEFAULT_SPACE_DEVELOPER_CACHE_TIME_IN_SECONDS); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_SPACE_DEVELOPER_CACHE_TIME_IN_SECONDS, configuration.getSpaceDeveloperCacheExpirationInSeconds()); } @Test void testGetControllerClientConnectionPoolSize() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CONTROLLER_CLIENT_CONNECTION_POOL_SIZE, - ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_CONNECTION_POOL_SIZE)) - .thenReturn(ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_CONNECTION_POOL_SIZE); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CONTROLLER_CLIENT_CONNECTION_POOL_SIZE, + ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_CONNECTION_POOL_SIZE)) + .thenReturn(ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_CONNECTION_POOL_SIZE); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_CONNECTION_POOL_SIZE, configuration.getControllerClientConnectionPoolSize()); } @Test void testGetControllerClientThreadPoolSize() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CONTROLLER_CLIENT_THREAD_POOL_SIZE, - ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_THREAD_POOL_SIZE)) - .thenReturn(ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_THREAD_POOL_SIZE); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_CONTROLLER_CLIENT_THREAD_POOL_SIZE, + ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_THREAD_POOL_SIZE)) + .thenReturn(ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_THREAD_POOL_SIZE); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_CONTROLLER_CLIENT_THREAD_POOL_SIZE, configuration.getControllerClientThreadPoolSize()); } @Test void testGetSnakeyamlMaxAliasesForCollections() { - Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS, - ApplicationConfiguration.DEFAULT_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS)) - .thenReturn(ApplicationConfiguration.DEFAULT_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS); + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS, + ApplicationConfiguration.DEFAULT_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS)) + .thenReturn(ApplicationConfiguration.DEFAULT_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS); Assertions.assertEquals(ApplicationConfiguration.DEFAULT_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS, configuration.getSnakeyamlMaxAliasesForCollections()); } @Test void testIsInternalEnvironment() { - Mockito.when(environment.getBoolean(ApplicationConfiguration.SAP_INTERNAL_DELIVERY, - ApplicationConfiguration.DEFAULT_SAP_INTERNAL_DELIVERY)) - .thenReturn(true); + when(environment.getBoolean(ApplicationConfiguration.SAP_INTERNAL_DELIVERY, + ApplicationConfiguration.DEFAULT_SAP_INTERNAL_DELIVERY)) + .thenReturn(true); assertTrue(configuration.isInternalEnvironment()); } @Test void testGetCloudComponentsInvalidJson() { - Mockito.when(environment.getString(ApplicationConfiguration.SUPPORT_COMPONENTS)) - .thenReturn("Invalid json"); + when(environment.getString(ApplicationConfiguration.SUPPORT_COMPONENTS)) + .thenReturn("Invalid json"); Map cloudComponents = configuration.getCloudComponents(); Assertions.assertEquals(0, cloudComponents.size()); } @@ -442,34 +441,159 @@ void testGetCloudComponentsValidJson() { @Test void testGetInternalSupportChannel() { String internalSupportChannel = "internal-support-channel"; - Mockito.when(environment.getString(ApplicationConfiguration.INTERNAL_SUPPORT_CHANNEL)) - .thenReturn(internalSupportChannel); + when(environment.getString(ApplicationConfiguration.INTERNAL_SUPPORT_CHANNEL)) + .thenReturn(internalSupportChannel); Assertions.assertEquals(internalSupportChannel, configuration.getInternalSupportChannel()); } @Test void testGetCertificateCN() { String certificateCN = "cert-cn"; - Mockito.when(environment.getString(ApplicationConfiguration.CFG_CERTIFICATE_CN)) - .thenReturn(certificateCN); + when(environment.getString(ApplicationConfiguration.CFG_CERTIFICATE_CN)) + .thenReturn(certificateCN); Assertions.assertEquals(certificateCN, configuration.getCertificateCN()); } @Test void testGetSpringSchedulerTaskExecutorThreads() { int executorThreads = 2; - Mockito.when(environment.getInteger(ApplicationConfiguration.CFG_SPRING_SCHEDULER_TASK_EXECUTOR_THREADS, - ApplicationConfiguration.DEFAULT_SPRING_SCHEDULER_TASK_EXECUTOR_THREADS)) - .thenReturn(executorThreads); + when(environment.getInteger(ApplicationConfiguration.CFG_SPRING_SCHEDULER_TASK_EXECUTOR_THREADS, + ApplicationConfiguration.DEFAULT_SPRING_SCHEDULER_TASK_EXECUTOR_THREADS)) + .thenReturn(executorThreads); Assertions.assertEquals(executorThreads, configuration.getSpringSchedulerTaskExecutorThreads()); } + @Test + void testIsOperationRateLimitingEnabled() { + when(environment.getBoolean(ApplicationConfiguration.CFG_OPERATION_RATE_LIMITING_ENABLED, + ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMITING_ENABLED)) + .thenReturn(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMITING_ENABLED); + assertEquals(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMITING_ENABLED, + configuration.isOperationRateLimitingEnabled()); + } + + @Test + void testIsOperationRateLimitingEnabledWithCustomValue() { + when(environment.getBoolean(ApplicationConfiguration.CFG_OPERATION_RATE_LIMITING_ENABLED, + ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMITING_ENABLED)) + .thenReturn(true); + assertTrue(configuration.isOperationRateLimitingEnabled()); + } + + @Test + void testGetMaxActiveOperationsPerSpace() { + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE, + ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE)) + .thenReturn(ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE); + assertEquals(ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE, + configuration.getMaxActiveOperationsPerSpace()); + } + + @Test + void testGetMaxActiveOperationsPerSpaceWithCustomValue() { + int customValue = 750; + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE, + ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE)) + .thenReturn(customValue); + assertEquals(customValue, configuration.getMaxActiveOperationsPerSpace()); + } + + @Test + void testGetMaxActiveOperationsPerUser() { + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_MAX_ACTIVE_OPERATIONS_PER_USER, + ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER)) + .thenReturn(ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER); + assertEquals(ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER, + configuration.getMaxActiveOperationsPerUser()); + } + + @Test + void testGetMaxActiveOperationsPerUserWithCustomValue() { + int customValue = 250; + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_MAX_ACTIVE_OPERATIONS_PER_USER, + ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER)) + .thenReturn(customValue); + assertEquals(customValue, configuration.getMaxActiveOperationsPerUser()); + } + + @Test + void testGetOperationRateLimitPerSpaceCapacity() { + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY, + ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY)) + .thenReturn(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY); + assertEquals(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY, + configuration.getOperationRateLimitPerSpaceCapacity()); + } + + @Test + void testGetOperationRateLimitPerSpaceCapacityWithCustomValue() { + int customValue = 400; + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY, + ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY)) + .thenReturn(customValue); + assertEquals(customValue, configuration.getOperationRateLimitPerSpaceCapacity()); + } + + @Test + void testGetOperationRateLimitPerSpaceRefillPerHour() { + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR, + ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR)) + .thenReturn(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR); + assertEquals(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR, + configuration.getOperationRateLimitPerSpaceRefillPerHour()); + } + + @Test + void testGetOperationRateLimitPerSpaceRefillPerHourWithCustomValue() { + int customValue = 1000; + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR, + ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR)) + .thenReturn(customValue); + assertEquals(customValue, configuration.getOperationRateLimitPerSpaceRefillPerHour()); + } + + @Test + void testGetOperationRateLimitPerUserCapacity() { + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY, + ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY)) + .thenReturn(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY); + assertEquals(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY, + configuration.getOperationRateLimitPerUserCapacity()); + } + + @Test + void testGetOperationRateLimitPerUserCapacityWithCustomValue() { + int customValue = 200; + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY, + ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY)) + .thenReturn(customValue); + assertEquals(customValue, configuration.getOperationRateLimitPerUserCapacity()); + } + + @Test + void testGetOperationRateLimitPerUserRefillPerHour() { + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR, + ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR)) + .thenReturn(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR); + assertEquals(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR, + configuration.getOperationRateLimitPerUserRefillPerHour()); + } + + @Test + void testGetOperationRateLimitPerUserRefillPerHourWithCustomValue() { + int customValue = 500; + when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR, + ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR)) + .thenReturn(customValue); + assertEquals(customValue, configuration.getOperationRateLimitPerUserRefillPerHour()); + } + @Test void testGetFilteredEnv() { Map filteredEnvironment = new HashMap<>(); filteredEnvironment.put(ApplicationConfiguration.CFG_MAX_MTA_DESCRIPTOR_SIZE, "1024"); - Mockito.when(environment.getAllVariables()) - .thenReturn(filteredEnvironment); + when(environment.getAllVariables()) + .thenReturn(filteredEnvironment); Map filteredEnv = configuration.getNotSensitiveVariables(); assertTrue(filteredEnv.containsKey(ApplicationConfiguration.CFG_MAX_MTA_DESCRIPTOR_SIZE)); } @@ -481,16 +605,16 @@ void testIsOnStartFilesWithoutContentCleanerEnabledThroughEnvironment() { @Test void testIsOnStartFilesWithoutContentCleanerEnabledThroughEnvironmentWithEnv() { - Mockito.when(environment.getBoolean(ApplicationConfiguration.CFG_ENABLE_ON_START_FILES_WITHOUT_CONTENT_CLEANER, - ApplicationConfiguration.DEFAULT_ENABLE_ON_START_FILES_WITHOUT_CONTENT_CLEANER)) - .thenReturn(true); + when(environment.getBoolean(ApplicationConfiguration.CFG_ENABLE_ON_START_FILES_WITHOUT_CONTENT_CLEANER, + ApplicationConfiguration.DEFAULT_ENABLE_ON_START_FILES_WITHOUT_CONTENT_CLEANER)) + .thenReturn(true); assertTrue(configuration.isOnStartFilesWithoutContentCleanerEnabled()); } @Test void testLoad() { - Mockito.when(environment.getString(ApplicationConfiguration.OBJECTSTORE_REGIONS, Strings.EMPTY)) - .thenReturn(Strings.EMPTY); + when(environment.getString(ApplicationConfiguration.OBJECTSTORE_REGIONS, Strings.EMPTY)) + .thenReturn(Strings.EMPTY); Map vcapApplication = injectFileInEnvironment(VCAP_APPLICATION, ApplicationConfiguration.CFG_VCAP_APPLICATION); configuration.load(); Assertions.assertEquals(vcapApplication.get("cf_api"), configuration.getControllerUrl() @@ -499,8 +623,8 @@ void testLoad() { private Map injectFileInEnvironment(String filename, String envVariable) { String vcapApplicationJson = TestUtil.getResourceAsString(filename, getClass()); - Mockito.when(environment.getString(envVariable)) - .thenReturn(vcapApplicationJson); + when(environment.getString(envVariable)) + .thenReturn(vcapApplicationJson); return JsonUtil.convertJsonToMap(vcapApplicationJson); } diff --git a/multiapps-controller-persistence/src/main/resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-2.52.0-persistence.xml b/multiapps-controller-persistence/src/main/resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-2.52.0-persistence.xml new file mode 100644 index 0000000000..08f20ca848 --- /dev/null +++ b/multiapps-controller-persistence/src/main/resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-2.52.0-persistence.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/multiapps-controller-persistence/src/main/resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog.xml b/multiapps-controller-persistence/src/main/resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog.xml index 0592d75f42..de059351b2 100644 --- a/multiapps-controller-persistence/src/main/resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog.xml +++ b/multiapps-controller-persistence/src/main/resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog.xml @@ -47,5 +47,7 @@ - + + diff --git a/multiapps-controller-process/pom.xml b/multiapps-controller-process/pom.xml index f33f5c6386..10495633af 100644 --- a/multiapps-controller-process/pom.xml +++ b/multiapps-controller-process/pom.xml @@ -110,5 +110,9 @@ org.cloudfoundry.multiapps multiapps-controller-shutdown-client + + com.bucket4j + bucket4j_jdk17-postgresql + \ No newline at end of file diff --git a/multiapps-controller-process/src/main/java/module-info.java b/multiapps-controller-process/src/main/java/module-info.java index 6b87de4bd9..6a67ce6fab 100644 --- a/multiapps-controller-process/src/main/java/module-info.java +++ b/multiapps-controller-process/src/main/java/module-info.java @@ -67,5 +67,7 @@ requires static java.compiler; requires static org.immutables.value; requires org.cloudfoundry.multiapps.controller.shutdown.client; + requires io.github.bucket4j.core; + requires io.github.bucket4j.postgresql; } \ No newline at end of file diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java index ac8279c2f8..86ef2d9b56 100755 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java @@ -105,6 +105,7 @@ public class Messages { public static final String ERROR_DURING_INCREMENTAL_INSTANCE_UPDATE_OF_MODULE_0 = "Error during incremental instance update of module \"{0}\""; public static final String ERROR_DURING_POLL_OF_INCREMENTAL_INSTANCE_UPDATE_OF_MODULE_0 = "Error during poll of incremental instance update of module \"{0}\""; public static final String FAILED_TO_GET_CLOUD_LOGGING_SERVICE_KEY = "Failed to get Cloud Logging service key"; + public static final String COULD_NOT_CLEAN_UP_EXPIRED_OPERATION_RATE_LIMIT_BUCKETS = "Could not clean up expired operation rate limit buckets"; // Process step errors public static final String ERROR_VALIDATING_PARAMS = "Error validating parameters"; @@ -358,6 +359,8 @@ public class Messages { public static final String DELETING_BACKUP_DESCRIPTORS_WITH_MTA_ID_0_SPACE_1_NAMESPACE_2_AND_SKIP_VERSIONS_3 = "Deleting backup descriptors with mta id \"{0}\" in space \"{1}\" namespace \"{2}\" and skip the following mta versions \"{3}\""; public static final String EXISTING_APPS_TO_BACKUP = "Existing apps to backup: {0}"; public static final String TASK_0_ON_APPLICATION_1_IS_STILL_2 = "Task \"{0}\" on application \"{1}\" is still \"{2}\""; + public static final String STARTING_CLEAN_UP_OF_EXPIRED_OPERATION_RATE_LIMIT_BUCKETS = "Starting clean up of expired operation rate limit buckets..."; + public static final String DELETED_EXPIRED_OPERATION_RATE_LIMIT_BUCKETS_0 = "Deleted {0} expired operation rate limit buckets"; // Progress messages public static final String OPERATION_ID = "Operation ID: {0}"; diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/jobs/OperationRateLimitBucketCleaner.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/jobs/OperationRateLimitBucketCleaner.java new file mode 100644 index 0000000000..7ce526176d --- /dev/null +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/jobs/OperationRateLimitBucketCleaner.java @@ -0,0 +1,65 @@ +package org.cloudfoundry.multiapps.controller.process.jobs; + +import java.text.MessageFormat; +import java.util.concurrent.TimeUnit; + +import jakarta.inject.Inject; +import jakarta.inject.Named; + +import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.process.Messages; +import org.cloudfoundry.multiapps.controller.process.util.BucketStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; + +/** + * Periodically deletes expired rows from the operation rate limit bucket table. bucket4j populates each row's expiry but never removes + * expired rows on its own, so without this sweeper the table grows without bound as new spaces and users start operations. + */ +@Named +public class OperationRateLimitBucketCleaner { + + private static final Logger LOGGER = LoggerFactory.getLogger(OperationRateLimitBucketCleaner.class); + private static final int SELECTED_INSTANCE_FOR_CLEAN_UP = 0; + private static final int DELETE_BATCH_SIZE = 1000; + private static final int MAX_ITERATIONS = 10000; + + private final ApplicationConfiguration applicationConfiguration; + private final BucketStore bucketStore; + + @Inject + public OperationRateLimitBucketCleaner(ApplicationConfiguration applicationConfiguration, BucketStore bucketStore) { + this.applicationConfiguration = applicationConfiguration; + this.bucketStore = bucketStore; + } + + @Scheduled(fixedRate = 1, timeUnit = TimeUnit.HOURS) + public void cleanUpExpiredBuckets() { + if (!applicationConfiguration.isOperationRateLimitingEnabled()) { + return; + } + if (applicationConfiguration.getApplicationInstanceIndex() != SELECTED_INSTANCE_FOR_CLEAN_UP) { + return; + } + LOGGER.info(Messages.STARTING_CLEAN_UP_OF_EXPIRED_OPERATION_RATE_LIMIT_BUCKETS); + try { + int totalDeleted = deleteExpiredBucketsInBatches(); + LOGGER.info(MessageFormat.format(Messages.DELETED_EXPIRED_OPERATION_RATE_LIMIT_BUCKETS_0, totalDeleted)); + } catch (Exception e) { + LOGGER.error(Messages.COULD_NOT_CLEAN_UP_EXPIRED_OPERATION_RATE_LIMIT_BUCKETS, e); + } + } + + private int deleteExpiredBucketsInBatches() { + int totalDeleted = 0; + for (int iteration = 0; iteration < MAX_ITERATIONS; iteration++) { + int deleted = bucketStore.removeExpiredEntries(DELETE_BATCH_SIZE); + totalDeleted += deleted; + if (deleted < DELETE_BATCH_SIZE) { + break; + } + } + return totalDeleted; + } +} diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/BucketStore.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/BucketStore.java new file mode 100644 index 0000000000..56d1961a39 --- /dev/null +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/BucketStore.java @@ -0,0 +1,15 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import io.github.bucket4j.Bucket; +import io.github.bucket4j.BucketConfiguration; + +/** + * Resolves distributed token buckets by key. Abstracting this behind an interface keeps the concrete bucket4j proxy manager (and its backing + * data store) out of the rate limiter, so unit tests can supply mocked buckets without touching a real database. + */ +public interface BucketStore { + + Bucket getBucket(long key, BucketConfiguration configuration); + + int removeExpiredEntries(int batchSize); +} diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/PostgresBucketStore.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/PostgresBucketStore.java new file mode 100644 index 0000000000..4ece62969a --- /dev/null +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/PostgresBucketStore.java @@ -0,0 +1,46 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import java.time.Duration; + +import javax.sql.DataSource; + +import jakarta.inject.Inject; +import jakarta.inject.Named; + +import io.github.bucket4j.Bucket; +import io.github.bucket4j.BucketConfiguration; +import io.github.bucket4j.distributed.ExpirationAfterWriteStrategy; +import io.github.bucket4j.distributed.proxy.ProxyManager; +import io.github.bucket4j.postgresql.Bucket4jPostgreSQL; +import io.github.bucket4j.postgresql.PostgreSQLSelectForUpdateBasedProxyManager; + +/** + * {@link BucketStore} backed by a PostgreSQL {@link ProxyManager} that uses SELECT ... FOR UPDATE row locking to coordinate token + * consumption across all controller instances sharing the database. + */ +@Named +public class PostgresBucketStore implements BucketStore { + + private static final String BUCKET_TABLE_NAME = "operation_rate_limit_bucket"; + private static final Duration BUCKET_TIME_TO_LIVE = Duration.ofHours(1); + + private final PostgreSQLSelectForUpdateBasedProxyManager proxyManager; + + @Inject + public PostgresBucketStore(DataSource dataSource) { + this.proxyManager = Bucket4jPostgreSQL.selectForUpdateBasedBuilder(dataSource) + .table(BUCKET_TABLE_NAME) + .expirationAfterWrite(ExpirationAfterWriteStrategy.basedOnTimeForRefillingBucketUpToMax(BUCKET_TIME_TO_LIVE)) + .build(); + } + + @Override + public Bucket getBucket(long key, BucketConfiguration configuration) { + return proxyManager.getProxy(key, () -> configuration); + } + + @Override + public int removeExpiredEntries(int batchSize) { + return proxyManager.removeExpired(batchSize); + } +} diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/jobs/OperationRateLimitBucketCleanerTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/jobs/OperationRateLimitBucketCleanerTest.java new file mode 100644 index 0000000000..cad08ea837 --- /dev/null +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/jobs/OperationRateLimitBucketCleanerTest.java @@ -0,0 +1,88 @@ +package org.cloudfoundry.multiapps.controller.process.jobs; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.process.util.BucketStore; + +class OperationRateLimitBucketCleanerTest { + + private static final int SELECTED_INSTANCE = 0; + private static final int OTHER_INSTANCE = 3; + private static final int BATCH_SIZE = 1000; + + @Mock + private ApplicationConfiguration applicationConfiguration; + @Mock + private BucketStore bucketStore; + @InjectMocks + private OperationRateLimitBucketCleaner cleaner; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + } + + @Test + void testDoesNothingWhenRateLimitingDisabled() { + when(applicationConfiguration.isOperationRateLimitingEnabled()).thenReturn(false); + + cleaner.cleanUpExpiredBuckets(); + + verifyNoInteractions(bucketStore); + } + + @Test + void testDoesNothingWhenNotSelectedInstance() { + when(applicationConfiguration.isOperationRateLimitingEnabled()).thenReturn(true); + when(applicationConfiguration.getApplicationInstanceIndex()).thenReturn(OTHER_INSTANCE); + + cleaner.cleanUpExpiredBuckets(); + + verifyNoInteractions(bucketStore); + } + + @Test + void testDeletesInASingleBatchWhenFewerThanBatchSizeExpired() { + enableCleaningOnSelectedInstance(); + when(bucketStore.removeExpiredEntries(BATCH_SIZE)).thenReturn(0); + + cleaner.cleanUpExpiredBuckets(); + + verify(bucketStore, times(1)).removeExpiredEntries(BATCH_SIZE); + } + + @Test + void testKeepsDeletingUntilBatchNotFull() { + enableCleaningOnSelectedInstance(); + when(bucketStore.removeExpiredEntries(BATCH_SIZE)).thenReturn(BATCH_SIZE, 30); + + cleaner.cleanUpExpiredBuckets(); + + verify(bucketStore, times(2)).removeExpiredEntries(BATCH_SIZE); + } + + @Test + void testSwallowsExceptionFromBucketStore() { + enableCleaningOnSelectedInstance(); + when(bucketStore.removeExpiredEntries(BATCH_SIZE)).thenThrow(new RuntimeException("boom")); + + assertDoesNotThrow(() -> cleaner.cleanUpExpiredBuckets()); + } + + private void enableCleaningOnSelectedInstance() { + when(applicationConfiguration.isOperationRateLimitingEnabled()).thenReturn(true); + when(applicationConfiguration.getApplicationInstanceIndex()).thenReturn(SELECTED_INSTANCE); + } +} diff --git a/multiapps-controller-web/pom.xml b/multiapps-controller-web/pom.xml index 1c377426ed..5e180abdcc 100644 --- a/multiapps-controller-web/pom.xml +++ b/multiapps-controller-web/pom.xml @@ -151,6 +151,10 @@ io.github.resilience4j resilience4j-ratelimiter + + com.bucket4j + bucket4j_jdk17-postgresql + org.apache.jclouds.common googlecloud diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java index 2be0acf27a..8875564c41 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java @@ -24,6 +24,9 @@ public final class Messages { public static final String MISSING_PROPERTIES_FOR_CREATING_THE_SPECIFIC_PROVIDER = "Missing properties for creating the specific provider!"; public static final String DEPLOY_FROM_URL_WRONG_CREDENTIALS_FOR_JOB_WITH_ID = "Credentials to {0} are wrong. Make sure that they are correct. Job id: {1}"; public static final String JOB_NOT_UPDATED_FOR_0_SECONDS = "Job not updated for {0} seconds"; + public static final String TOO_MANY_ACTIVE_OPERATIONS_IN_SPACE = "Too many active operations in space"; + public static final String TOO_MANY_ACTIVE_OPERATIONS_FOR_USER = "Too many active operations for user"; + public static final String OPERATION_RATE_LIMIT_EXCEEDED = "Operation rate limit exceeded"; public static final String FAILED_TO_CREATE_BLOB_STORE_CONTEXT = "Failed to create BlobStoreContext"; @@ -87,6 +90,8 @@ public final class Messages { public static final String ASYNC_UPLOAD_JOB_EXISTS = "Async upload job for URL {} exists: {}"; public static final String CREATING_ASYNC_UPLOAD_JOB = "Creating async upload job for URL {} with ID: {}"; public static final String ASYNC_UPLOAD_JOB_REJECTED = "Async upload job with space guid: {}, namespace: {}, URL: {} rejected."; + public static final String OPERATION_START_RATE_LIMITED = "Start of operation in space {} rejected due to rate limiting: {}"; + public static final String OPERATION_START_RATE_LIMITED_STRUCTURED = "Operation start rejected: user=\"{0}\" spaceGuid=\"{1}\" reason=\"{2}\""; public static final String STARTING_DOWNLOAD_OF_MTAR_WITH_JOB_ID = "Starting download of MTAR from remote endpoint: {}. Job id: {}"; public static final String UPLOADED_MTAR_FROM_REMOTE_ENDPOINT_AND_JOB_ID = "Uploaded MTAR from remote endpoint {}. Job id: {} in {} ms"; public static final String ASYNC_UPLOAD_JOB_FINISHED = "Async upload job {} finished"; diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImpl.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImpl.java index f0752061fc..eb92cd01da 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImpl.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImpl.java @@ -56,10 +56,13 @@ import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; import org.cloudfoundry.multiapps.controller.web.monitoring.ApiUsageLogger; +import org.cloudfoundry.multiapps.controller.web.util.OperationRateLimitExceededException; +import org.cloudfoundry.multiapps.controller.web.util.OperationRateLimiter; import org.cloudfoundry.multiapps.controller.web.util.SecurityContextUtil; import org.flowable.engine.runtime.ProcessInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.server.ResponseStatusException; @@ -84,6 +87,7 @@ public class OperationsApiServiceImpl implements OperationsApiService { private final OperationsApiServiceAuditLog operationsApiServiceAuditLog; private final ApiUsageLogger apiUsageLogger; private final HttpServletRequest httpServletRequest; + private final OperationRateLimiter operationRateLimiter; @Inject public OperationsApiServiceImpl(CloudControllerClientFactory clientFactory, TokenService tokenService, @@ -93,7 +97,8 @@ public OperationsApiServiceImpl(CloudControllerClientFactory clientFactory, Toke OperationsHelper operationsHelper, ProgressMessageService progressMessageService, ProcessActionRegistry processActionRegistry, OperationsApiServiceAuditLog operationsApiServiceAuditLog, - ApiUsageLogger apiUsageLogger, HttpServletRequest httpServletRequest) { + ApiUsageLogger apiUsageLogger, HttpServletRequest httpServletRequest, + OperationRateLimiter operationRateLimiter) { this.clientFactory = clientFactory; this.tokenService = tokenService; this.operationService = operationService; @@ -106,6 +111,7 @@ public OperationsApiServiceImpl(CloudControllerClientFactory clientFactory, Toke this.operationsApiServiceAuditLog = operationsApiServiceAuditLog; this.apiUsageLogger = apiUsageLogger; this.httpServletRequest = httpServletRequest; + this.operationRateLimiter = operationRateLimiter; } @Override @@ -174,6 +180,11 @@ public ResponseEntity startOperation(String spaceGuid, Operation oper operation.getNamespace(), httpServletRequest); operationsApiServiceAuditLog.logStartOperation(SecurityContextUtil.getUsername(), spaceGuid, operation); UserInfo authenticatedUser = getAuthenticatedUser(); + try { + operationRateLimiter.checkStartAllowed(authenticatedUser.getName(), spaceGuid); + } catch (OperationRateLimitExceededException e) { + return buildRateLimitExceededResponse(spaceGuid, e); + } String processDefinitionKey = operationsHelper.getProcessDefinitionKey(operation); Set predefinedParameters = operationMetadataMapper.getOperationMetadata(operation.getProcessType()) .getParameters(); @@ -188,6 +199,13 @@ public ResponseEntity startOperation(String spaceGuid, Operation oper .build(); } + private ResponseEntity buildRateLimitExceededResponse(String spaceGuid, OperationRateLimitExceededException e) { + LOGGER.debug(Messages.OPERATION_START_RATE_LIMITED, spaceGuid, e.getMessage()); + return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS) + .header(HttpHeaders.RETRY_AFTER, String.valueOf(e.getRetryAfterSeconds())) + .build(); + } + protected void logStartOperation(String processInstanceId, UserInfo authenticatedUser) { LOGGER.info(MessageFormat.format(Messages.STARTED_OPERATION_0_BY_USER_1_AND_ORIGIN_OF_2, processInstanceId, authenticatedUser.getId(), authenticatedUser.getToken() diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java index a8aca8efc2..0ec32c40eb 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java @@ -1,7 +1,10 @@ package org.cloudfoundry.multiapps.controller.web.configuration; +import java.lang.management.ManagementFactory; import java.util.Map; +import javax.management.MBeanServer; + import org.cloudfoundry.multiapps.controller.web.monitoring.Metrics; import org.cloudfoundry.multiapps.controller.web.monitoring.UploadDurationMetrics; import org.springframework.context.annotation.Bean; @@ -15,10 +18,17 @@ public class JmxConfiguration { private static final String UPLOAD_METRICS_BEAN = "org.cloudfoundry.multiapps.controller.web.monitoring:type=Metrics,name=UploadMetricsMBean"; + @Bean + public MBeanServer mBeanServer() { + return ManagementFactory.getPlatformMBeanServer(); + } + @Bean public MBeanExporter jmxExporter(Metrics metrics, UploadDurationMetrics uploadDurationMetrics) { MBeanExporter mBeanExporter = new MBeanExporter(); - mBeanExporter.setBeans(Map.of(METRICS_BEAN, metrics, UPLOAD_METRICS_BEAN, uploadDurationMetrics)); + mBeanExporter.setServer(mBeanServer()); + mBeanExporter.setBeans(Map.of(METRICS_BEAN, metrics, + UPLOAD_METRICS_BEAN, uploadDurationMetrics)); return mBeanExporter; } } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsCount.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsCount.java new file mode 100644 index 0000000000..773668d8e3 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsCount.java @@ -0,0 +1,20 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +public class ActiveOperationsCount implements ActiveOperationsCountMBean { + + private volatile long count; + + public ActiveOperationsCount(long count) { + this.count = count; + } + + public void setCount(long count) { + this.count = count; + } + + @Override + public long getCount() { + return count; + } + +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsCountMBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsCountMBean.java new file mode 100644 index 0000000000..b2901e0245 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsCountMBean.java @@ -0,0 +1,7 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +public interface ActiveOperationsCountMBean { + + long getCount(); + +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsJmxReporter.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsJmxReporter.java new file mode 100644 index 0000000000..db89bd909f --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsJmxReporter.java @@ -0,0 +1,117 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.management.MBeanServer; +import javax.management.ObjectName; + +import jakarta.inject.Inject; +import jakarta.inject.Named; +import org.cloudfoundry.multiapps.controller.api.model.Operation; +import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.services.OperationService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; + +@Named +public class ActiveOperationsJmxReporter { + + static final String DOMAIN = "org.cloudfoundry.multiapps.controller.web.monitoring"; + static final String USER_OBJECT_NAME_PATTERN = DOMAIN + ":type=Metrics,name=ActiveOps,user=%s"; + static final String SPACE_OBJECT_NAME_PATTERN = DOMAIN + ":type=Metrics,name=ActiveOps,space=%s"; + + private static final Logger LOGGER = LoggerFactory.getLogger(ActiveOperationsJmxReporter.class); + + private final OperationService operationService; + private final MBeanServer mBeanServer; + private final ApplicationConfiguration applicationConfiguration; + private static final int SELECTED_INSTANCE_FOR_CLEAN_UP = 0; + private final Map userBeans = new ConcurrentHashMap<>(); + private final Map spaceBeans = new ConcurrentHashMap<>(); + + @Inject + public ActiveOperationsJmxReporter(OperationService operationService, MBeanServer mBeanServer, + ApplicationConfiguration applicationConfiguration) { + this.operationService = operationService; + this.mBeanServer = mBeanServer; + this.applicationConfiguration = applicationConfiguration; + } + + @Scheduled(fixedRate = 1, timeUnit = TimeUnit.MINUTES) + public void refresh() { + if (!applicationConfiguration.isOperationRateLimitingEnabled()) { + return; + } + if (applicationConfiguration.getApplicationInstanceIndex() != SELECTED_INSTANCE_FOR_CLEAN_UP) { + return; + } + try { + List activeOperations = operationService.createQuery() + .inNonFinalState() + .list(); + syncMBeans(groupBy(activeOperations, op -> hashUser(op.getUser())), USER_OBJECT_NAME_PATTERN, userBeans); + syncMBeans(groupBy(activeOperations, Operation::getSpaceId), SPACE_OBJECT_NAME_PATTERN, spaceBeans); + } catch (Exception e) { + LOGGER.warn("Failed to refresh active operations JMX metrics", e); + } + } + + private Map groupBy(List operations, Function keyExtractor) { + return operations.stream() + .collect(Collectors.groupingBy(keyExtractor, Collectors.counting())); + } + + private void syncMBeans(Map counts, String pattern, + Map registry) throws Exception { + Set stale = new HashSet<>(registry.keySet()); + for (Map.Entry entry : counts.entrySet()) { + ObjectName name = new ObjectName(pattern.formatted(ObjectName.quote(entry.getKey()))); + stale.remove(name); + upsertMBean(name, entry.getValue(), registry); + } + removeStaleMBeans(stale, registry); + } + + private void upsertMBean(ObjectName name, long count, + Map registry) throws Exception { + ActiveOperationsCount bean = registry.get(name); + if (bean == null) { + bean = new ActiveOperationsCount(count); + mBeanServer.registerMBean(bean, name); + registry.put(name, bean); + } else { + bean.setCount(count); + } + } + + private void removeStaleMBeans(Set stale, + Map registry) throws Exception { + for (ObjectName name : stale) { + mBeanServer.unregisterMBean(name); + registry.remove(name); + } + } + + static String hashUser(String user) { + try { + byte[] digest = MessageDigest.getInstance("SHA-384") + .digest(user.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of() + .formatHex(digest); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimitExceededException.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimitExceededException.java new file mode 100644 index 0000000000..8084c40ccb --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimitExceededException.java @@ -0,0 +1,21 @@ +package org.cloudfoundry.multiapps.controller.web.util; + +/** + * Thrown when an operation cannot be started because a rate limit has been reached. Carries the number of seconds after which the caller may + * retry, which is mappable to an HTTP 429 {@code Retry-After} response header. + */ +public class OperationRateLimitExceededException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final long retryAfterSeconds; + + public OperationRateLimitExceededException(String message, long retryAfterSeconds) { + super(message); + this.retryAfterSeconds = retryAfterSeconds; + } + + public long getRetryAfterSeconds() { + return retryAfterSeconds; + } +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimitKeys.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimitKeys.java new file mode 100644 index 0000000000..7ae9bfd5e5 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimitKeys.java @@ -0,0 +1,39 @@ +package org.cloudfoundry.multiapps.controller.web.util; + +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import com.google.common.primitives.Longs; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Derives stable {@code long} bucket keys for operation rate limiting. + *

+ * Keys are computed from the SHA-256 digest of a namespaced input string and are therefore deterministic across restarts and JVMs. The + * space and user namespaces are disjoint by construction, so a space key can never collide with a user key. Truncating the 256-bit digest + * to its first 64 bits keeps the collision probability negligible for the number of distinct spaces and users a single landscape handles. + */ +public final class OperationRateLimitKeys { + + private static final String SPACE_NAMESPACE_PREFIX = "space:"; + private static final String USER_NAMESPACE_PREFIX = "user:"; + private static final String SEGMENT_SEPARATOR = ":"; + private static final HashFunction HASH_FUNCTION = Hashing.sha384(); + + private OperationRateLimitKeys() { + } + + public static long spaceKey(String spaceGuid) { + return hashToLong(SPACE_NAMESPACE_PREFIX + spaceGuid); + } + + public static long userKey(String spaceGuid, String user) { + return hashToLong(USER_NAMESPACE_PREFIX + spaceGuid + SEGMENT_SEPARATOR + user); + } + + private static long hashToLong(String input) { + byte[] digest = HASH_FUNCTION.hashString(input, UTF_8) + .asBytes(); + return Longs.fromBytes(digest[0], digest[1], digest[2], digest[3], digest[4], digest[5], digest[6], digest[7]); + } +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimiter.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimiter.java new file mode 100644 index 0000000000..8a4b958e55 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimiter.java @@ -0,0 +1,111 @@ +package org.cloudfoundry.multiapps.controller.web.util; + +import java.text.MessageFormat; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import io.github.bucket4j.Bandwidth; +import io.github.bucket4j.Bucket; +import io.github.bucket4j.BucketConfiguration; +import io.github.bucket4j.ConsumptionProbe; +import jakarta.inject.Named; +import org.cloudfoundry.multiapps.controller.api.model.Operation; +import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.services.OperationService; +import org.cloudfoundry.multiapps.controller.process.util.BucketStore; +import org.cloudfoundry.multiapps.controller.web.Messages; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Guards the start of MTA operations against per-space and per-user rate limits. Each start attempt consumes a single token from both the + * space bucket and the user bucket; if either is exhausted an {@link OperationRateLimitExceededException} is raised. + */ +@Named +public class OperationRateLimiter { + + private static final Logger LOGGER = LoggerFactory.getLogger(OperationRateLimiter.class); + private static final Duration REFILL_PERIOD = Duration.ofHours(1); + private static final long TOKENS_PER_OPERATION = 1; + private static final long NO_RETRY_AFTER_SECONDS = 0; + + private final ApplicationConfiguration applicationConfiguration; + private final OperationService operationService; + private final BucketStore bucketStore; + private final BucketConfiguration spaceTokenBucketConfiguration; + private final BucketConfiguration userTokenBucketConfiguration; + + public OperationRateLimiter(ApplicationConfiguration applicationConfiguration, OperationService operationService, + BucketStore bucketStore) { + this.applicationConfiguration = applicationConfiguration; + this.operationService = operationService; + this.bucketStore = bucketStore; + this.spaceTokenBucketConfiguration = buildBucketConfiguration(applicationConfiguration.getOperationRateLimitPerSpaceCapacity(), + applicationConfiguration.getOperationRateLimitPerSpaceRefillPerHour()); + this.userTokenBucketConfiguration = buildBucketConfiguration(applicationConfiguration.getOperationRateLimitPerUserCapacity(), + applicationConfiguration.getOperationRateLimitPerUserRefillPerHour()); + } + + public void checkStartAllowed(String user, String spaceGuid) { + if (!applicationConfiguration.isOperationRateLimitingEnabled()) { + return; + } + checkActiveOperationCaps(user, spaceGuid); + checkTokenBuckets(user, spaceGuid); + } + + private void rejectAndLog(String user, String spaceGuid, String reason, long retryAfterSeconds) { + LOGGER.info(MessageFormat.format(Messages.OPERATION_START_RATE_LIMITED_STRUCTURED, user, spaceGuid, reason)); + throw new OperationRateLimitExceededException(reason, retryAfterSeconds); + } + + private void checkActiveOperationCaps(String user, String spaceGuid) { + List activeOperationsInSpace = operationService.createQuery() + .spaceId(spaceGuid) + .inNonFinalState() + .list(); + if (activeOperationsInSpace.size() >= applicationConfiguration.getMaxActiveOperationsPerSpace()) { + rejectAndLog(user, spaceGuid, Messages.TOO_MANY_ACTIVE_OPERATIONS_IN_SPACE, NO_RETRY_AFTER_SECONDS); + } + long activeOperationsPerUser = activeOperationsInSpace.stream() + .filter(op -> user.equals(op.getUser())) + .count(); + if (activeOperationsPerUser >= applicationConfiguration.getMaxActiveOperationsPerUser()) { + rejectAndLog(user, spaceGuid, Messages.TOO_MANY_ACTIVE_OPERATIONS_FOR_USER, NO_RETRY_AFTER_SECONDS); + } + } + + private void checkTokenBuckets(String user, String spaceGuid) { + consumeUserToken(spaceGuid, user); + consumeSpaceToken(spaceGuid, user); + } + + private void consumeSpaceToken(String spaceGuid, String user) { + Bucket bucket = bucketStore.getBucket(OperationRateLimitKeys.spaceKey(spaceGuid), spaceTokenBucketConfiguration); + consumeToken(bucket, user, spaceGuid); + } + + private void consumeUserToken(String spaceGuid, String user) { + Bucket bucket = bucketStore.getBucket(OperationRateLimitKeys.userKey(spaceGuid, user), userTokenBucketConfiguration); + consumeToken(bucket, user, spaceGuid); + } + + private BucketConfiguration buildBucketConfiguration(int capacity, int refillTokensPerHour) { + Bandwidth bandwidth = Bandwidth.builder() + .capacity(capacity) + .refillGreedy(refillTokensPerHour, REFILL_PERIOD) + .build(); + return BucketConfiguration.builder() + .addLimit(bandwidth) + .build(); + } + + private void consumeToken(Bucket bucket, String user, String spaceGuid) { + ConsumptionProbe probe = bucket.tryConsumeAndReturnRemaining(TOKENS_PER_OPERATION); + if (!probe.isConsumed()) { + long retryAfterSeconds = TimeUnit.NANOSECONDS.toSeconds(probe.getNanosToWaitForRefill()); + rejectAndLog(user, spaceGuid, Messages.OPERATION_RATE_LIMIT_EXCEEDED, retryAfterSeconds); + } + } +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImplTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImplTest.java index ea44fb8882..4af5cfc9b1 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImplTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/api/impl/OperationsApiServiceImplTest.java @@ -39,17 +39,19 @@ import org.cloudfoundry.multiapps.controller.process.util.OperationsHelper; import org.cloudfoundry.multiapps.controller.process.variables.Variables; import org.cloudfoundry.multiapps.controller.web.monitoring.ApiUsageLogger; +import org.cloudfoundry.multiapps.controller.web.util.OperationRateLimitExceededException; +import org.cloudfoundry.multiapps.controller.web.util.OperationRateLimiter; import org.flowable.engine.runtime.ProcessInstance; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Answers; import org.mockito.ArgumentCaptor; -import org.mockito.ArgumentMatchers; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; @@ -60,6 +62,19 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; class OperationsApiServiceImplTest { @@ -91,6 +106,8 @@ class OperationsApiServiceImplTest { private ApiUsageLogger apiUsageLogger; @Mock private HttpServletRequest httpServletRequest; + @Mock + private OperationRateLimiter operationRateLimiter; private OperationsApiServiceImpl operationsApiService; @@ -120,7 +137,7 @@ public void initialize() throws Exception { operationsApiService = new OperationsApiServiceImpl(clientFactory, tokenService, operationService, operationMetadataMapper, logsService, flowableFacade, operationsHelper, progressMessageService, processActionRegistry, operationsApiServiceAuditLog, apiUsageLogger, - httpServletRequest); + httpServletRequest, operationRateLimiter); operations = new LinkedList<>(); operations.add(createOperation(FINISHED_PROCESS, Operation.State.FINISHED, Collections.emptyMap())); operations.add(createOperation(RUNNING_PROCESS, Operation.State.RUNNING, Collections.emptyMap())); @@ -176,8 +193,8 @@ void testGetOperationMissing() { void testExecuteOperationAction() { String processId = RUNNING_PROCESS; operationsApiService.executeOperationAction(SPACE_GUID, processId, Action.ABORT.getActionId()); - Mockito.verify(processAction) - .execute(Mockito.argThat(userInfo -> EXAMPLE_USER.equals(userInfo.getName())), Mockito.eq(processId)); + verify(processAction) + .execute(argThat(userInfo -> EXAMPLE_USER.equals(userInfo.getName())), eq(processId)); } @Test @@ -203,39 +220,77 @@ void testExecuteOperationActionUnauthorized() { void testStartOperation() { Map parameters = Map.of(Variables.MTA_ID.getName(), "test"); Operation operation = createOperation(null, null, parameters); - Mockito.when(operationsHelper.getProcessDefinitionKey(operation)) - .thenReturn("deploy"); - HttpServletRequest httpServletRequestMock = Mockito.mock(HttpServletRequest.class); - Mockito.when(httpServletRequestMock.getRequestURL()) - .thenReturn(new StringBuffer("test/api/path")); + when(operationsHelper.getProcessDefinitionKey(operation)) + .thenReturn("deploy"); + HttpServletRequest httpServletRequestMock = mock(HttpServletRequest.class); + when(httpServletRequestMock.getRequestURL()) + .thenReturn(new StringBuffer("test/api/path")); operationsApiService.startOperation(SPACE_GUID, operation, httpServletRequestMock); - Mockito.verify(flowableFacade) - .startProcess(Mockito.any(), Mockito.anyMap()); + verify(flowableFacade) + .startProcess(any(), anyMap()); + } + + @Test + void testStartOperationWhenRateLimitAllowsStartsProcess() { + Map parameters = Map.of(Variables.MTA_ID.getName(), "test"); + Operation operation = createOperation(null, null, parameters); + when(operationsHelper.getProcessDefinitionKey(operation)) + .thenReturn("deploy"); + HttpServletRequest httpServletRequestMock = mock(HttpServletRequest.class); + when(httpServletRequestMock.getRequestURL()) + .thenReturn(new StringBuffer("test/api/path")); + + ResponseEntity response = operationsApiService.startOperation(SPACE_GUID, operation, httpServletRequestMock); + + assertEquals(HttpStatus.ACCEPTED, response.getStatusCode()); + verify(operationRateLimiter) + .checkStartAllowed(EXAMPLE_USER, SPACE_GUID); + verify(flowableFacade) + .startProcess(any(), anyMap()); + } + + @Test + void testStartOperationWhenRateLimitExceededReturnsTooManyRequests() { + long retryAfterSeconds = 42; + Map parameters = Map.of(Variables.MTA_ID.getName(), "test"); + Operation operation = createOperation(null, null, parameters); + doThrow(new OperationRateLimitExceededException("Operation rate limit exceeded", retryAfterSeconds)) + .when(operationRateLimiter) + .checkStartAllowed(EXAMPLE_USER, SPACE_GUID); + HttpServletRequest httpServletRequestMock = mock(HttpServletRequest.class); + + ResponseEntity response = operationsApiService.startOperation(SPACE_GUID, operation, httpServletRequestMock); + + assertEquals(HttpStatus.TOO_MANY_REQUESTS, response.getStatusCode()); + assertEquals(String.valueOf(retryAfterSeconds), response.getHeaders() + .getFirst(HttpHeaders.RETRY_AFTER)); + verify(flowableFacade, never()) + .startProcess(any(), anyMap()); } @Test void testStartOperationLogsUserGuidAndOriginButNotUsername() { String processInstanceId = "process-instance-id-1"; - ProcessInstance processInstance = Mockito.mock(ProcessInstance.class); - Mockito.when(processInstance.getProcessInstanceId()) - .thenReturn(processInstanceId); - Mockito.when(flowableFacade.startProcess(Mockito.any(), Mockito.anyMap())) - .thenReturn(processInstance); + ProcessInstance processInstance = mock(ProcessInstance.class); + when(processInstance.getProcessInstanceId()) + .thenReturn(processInstanceId); + when(flowableFacade.startProcess(any(), anyMap())) + .thenReturn(processInstance); Map parameters = Map.of(Variables.MTA_ID.getName(), "test"); Operation operation = createOperation(null, null, parameters); - Mockito.when(operationsHelper.getProcessDefinitionKey(operation)) - .thenReturn("deploy"); - HttpServletRequest httpServletRequestMock = Mockito.mock(HttpServletRequest.class); - Mockito.when(httpServletRequestMock.getRequestURL()) - .thenReturn(new StringBuffer("test/api/path")); + when(operationsHelper.getProcessDefinitionKey(operation)) + .thenReturn("deploy"); + HttpServletRequest httpServletRequestMock = mock(HttpServletRequest.class); + when(httpServletRequestMock.getRequestURL()) + .thenReturn(new StringBuffer("test/api/path")); - OperationsApiServiceImpl operationsApiServiceSpy = Mockito.spy(operationsApiService); + OperationsApiServiceImpl operationsApiServiceSpy = spy(operationsApiService); operationsApiServiceSpy.startOperation(SPACE_GUID, operation, httpServletRequestMock); ArgumentCaptor userInfoCaptor = ArgumentCaptor.forClass(UserInfo.class); - Mockito.verify(operationsApiServiceSpy) - .logStartOperation(Mockito.eq(processInstanceId), userInfoCaptor.capture()); + verify(operationsApiServiceSpy) + .logStartOperation(eq(processInstanceId), userInfoCaptor.capture()); UserInfo authenticatedUser = userInfoCaptor.getValue(); assertEquals(USER_GUID, authenticatedUser.getId(), "logStartOperation must receive the user GUID"); assertEquals("test-origin", authenticatedUser.getToken() @@ -252,18 +307,18 @@ void testStartOperationWithInvalidParametersForTheProcess() { Variables.CTS_PROCESS_ID.getName(), "cts_test", Variables.DEPLOY_URI.getName(), "deploy_test"); Operation operation = createOperation(null, null, parameters); - Mockito.when(operationsHelper.getProcessDefinitionKey(operation)) - .thenReturn("deploy"); + when(operationsHelper.getProcessDefinitionKey(operation)) + .thenReturn("deploy"); - HttpServletRequest httpServletRequestMock = Mockito.mock(HttpServletRequest.class); - Mockito.when(httpServletRequestMock.getRequestURL()) - .thenReturn(new StringBuffer("test/api/path")); + HttpServletRequest httpServletRequestMock = mock(HttpServletRequest.class); + when(httpServletRequestMock.getRequestURL()) + .thenReturn(new StringBuffer("test/api/path")); operationsApiService.startOperation(SPACE_GUID, operation, httpServletRequestMock); - Mockito.verify(flowableFacade) - .startProcess(ArgumentMatchers.eq("deploy"), ArgumentMatchers.argThat( - map -> map.containsKey(Variables.MTA_ID.getName()) && map.containsKey(Variables.EXT_DESCRIPTOR_FILE_ID.getName()) - && !map.containsKey(Variables.CTS_PROCESS_ID.getName()) && !map.containsKey(Variables.CTS_PASSWORD.getName()))); + verify(flowableFacade) + .startProcess(eq("deploy"), argThat( + map -> map.containsKey(Variables.MTA_ID.getName()) && map.containsKey(Variables.EXT_DESCRIPTOR_FILE_ID.getName()) + && !map.containsKey(Variables.CTS_PROCESS_ID.getName()) && !map.containsKey(Variables.CTS_PASSWORD.getName()))); } @Test @@ -271,25 +326,25 @@ void testStartOperationWithValidParametersForTheProcess() { Map parameters = Map.of(Variables.MTA_ID.getName(), "test", Variables.EXT_DESCRIPTOR_FILE_ID.getName(), "ext_test", Variables.NO_START.getName(), false, Variables.MTA_NAMESPACE.getName(), "namespace_test"); Operation operation = createOperation(null, null, parameters); - Mockito.when(operationsHelper.getProcessDefinitionKey(operation)) - .thenReturn("deploy"); - HttpServletRequest httpServletRequestMock = Mockito.mock(HttpServletRequest.class); - Mockito.when(httpServletRequestMock.getRequestURL()) - .thenReturn(new StringBuffer("test/api/path")); + when(operationsHelper.getProcessDefinitionKey(operation)) + .thenReturn("deploy"); + HttpServletRequest httpServletRequestMock = mock(HttpServletRequest.class); + when(httpServletRequestMock.getRequestURL()) + .thenReturn(new StringBuffer("test/api/path")); operationsApiService.startOperation(SPACE_GUID, operation, httpServletRequestMock); - Mockito.verify(flowableFacade) - .startProcess(ArgumentMatchers.eq("deploy"), ArgumentMatchers.argThat( - map -> map.containsKey(Variables.MTA_ID.getName()) && map.containsKey(Variables.EXT_DESCRIPTOR_FILE_ID.getName()) - && map.containsKey(Variables.NO_START.getName()) && map.containsKey(Variables.MTA_NAMESPACE.getName()))); + verify(flowableFacade) + .startProcess(eq("deploy"), argThat( + map -> map.containsKey(Variables.MTA_ID.getName()) && map.containsKey(Variables.EXT_DESCRIPTOR_FILE_ID.getName()) + && map.containsKey(Variables.NO_START.getName()) && map.containsKey(Variables.MTA_NAMESPACE.getName()))); } @Test void testGetOperationLogs() throws Exception { String processId = FINISHED_PROCESS; operationsApiService.getOperationLogs(SPACE_GUID, processId); - Mockito.verify(logsService) - .getLogNames(Mockito.eq(SPACE_GUID), Mockito.eq(processId)); + verify(logsService) + .getLogNames(eq(SPACE_GUID), eq(processId)); } @Test @@ -300,8 +355,8 @@ void testGetOperationLogsNotFoundOperation() { @Test void testGetOperationLogsServiceException() throws Exception { String processId = FINISHED_PROCESS; - Mockito.when(logsService.getLogNames(Mockito.eq(SPACE_GUID), Mockito.eq(processId))) - .thenThrow(new FileStorageException("something went wrong")); + when(logsService.getLogNames(eq(SPACE_GUID), eq(processId))) + .thenThrow(new FileStorageException("something went wrong")); Assertions.assertThrows(ContentException.class, () -> operationsApiService.getOperationLogs(SPACE_GUID, processId)); } @@ -310,8 +365,8 @@ void testGetOperationLogContent() throws Exception { String processId = FINISHED_PROCESS; String logName = "OPERATION.log"; String expectedLogContent = "somelogcontentstring\n1234"; - Mockito.when(logsService.getOperationLog(Mockito.eq(SPACE_GUID), Mockito.eq(processId), Mockito.eq(logName))) - .thenReturn(expectedLogContent); + when(logsService.getOperationLog(eq(SPACE_GUID), eq(processId), eq(logName))) + .thenReturn(expectedLogContent); ResponseEntity response = operationsApiService.getOperationLogContent(SPACE_GUID, processId, logName); String logContent = response.getBody(); assertEquals(expectedLogContent, logContent); @@ -321,8 +376,8 @@ void testGetOperationLogContent() throws Exception { void testGetOperationLogContentNotFound() throws Exception { String processId = FINISHED_PROCESS; String logName = "OPERATION.log"; - Mockito.when(logsService.getOperationLog(Mockito.eq(SPACE_GUID), Mockito.eq(processId), Mockito.eq(logName))) - .thenThrow(new NoResultException("log file not found")); + when(logsService.getOperationLog(eq(SPACE_GUID), eq(processId), eq(logName))) + .thenThrow(new NoResultException("log file not found")); Assertions.assertThrows(NoResultException.class, () -> operationsApiService.getOperationLogContent(SPACE_GUID, processId, logName)); } @@ -367,19 +422,17 @@ void testGetOperationActionsNotFound() { } private void mockFlowableFacade() { - Mockito.when(flowableFacade.startProcess(Mockito.any(), Mockito.anyMap())) - .thenReturn(Mockito.mock(ProcessInstance.class)); + when(flowableFacade.startProcess(any(), anyMap())).thenReturn(mock(ProcessInstance.class)); } private void mockClientProvider(boolean shouldReturnAuthorizedClient) { mockClientAuth(shouldReturnAuthorizedClient); CloudSpaceClient mockedClient = mockClient(); - Mockito.when(clientFactory.createSpaceClient(Mockito.any())) - .thenReturn(mockedClient); + when(clientFactory.createSpaceClient(any())).thenReturn(mockedClient); } private void mockClientAuth(boolean shouldReturnAuthorizedClient) { - org.springframework.security.core.context.SecurityContext securityContextMock = Mockito.mock( + org.springframework.security.core.context.SecurityContext securityContextMock = mock( org.springframework.security.core.context.SecurityContext.class); SecurityContextHolder.setContext(securityContextMock); if (shouldReturnAuthorizedClient) { @@ -388,23 +441,23 @@ private void mockClientAuth(boolean shouldReturnAuthorizedClient) { TokenProperties.USER_ID_KEY, USER_GUID, "origin", "test-origin"))); - OAuth2AuthenticationToken auth = Mockito.mock(OAuth2AuthenticationToken.class); + OAuth2AuthenticationToken auth = mock(OAuth2AuthenticationToken.class); Map attributes = Map.of(USER_INFO, userInfo); - OAuth2User principal = Mockito.mock(OAuth2User.class); - Mockito.when(principal.getAttributes()) - .thenReturn(attributes); - Mockito.when(auth.getPrincipal()) - .thenReturn(principal); - Mockito.when(securityContextMock.getAuthentication()) - .thenReturn(auth); + OAuth2User principal = mock(OAuth2User.class); + when(principal.getAttributes()) + .thenReturn(attributes); + when(auth.getPrincipal()) + .thenReturn(principal); + when(securityContextMock.getAuthentication()) + .thenReturn(auth); return; } - Mockito.when(securityContextMock.getAuthentication()) - .thenReturn(null); + when(securityContextMock.getAuthentication()) + .thenReturn(null); } private CloudSpaceClient mockClient() { - CloudSpaceClient client = Mockito.mock(CloudSpaceClient.class); + CloudSpaceClient client = mock(CloudSpaceClient.class); ImmutableCloudOrganization organization = ImmutableCloudOrganization.builder() .metadata(ImmutableCloudMetadata.builder() .guid(UUID.fromString(ORG_GUID)) @@ -418,74 +471,74 @@ private CloudSpaceClient mockClient() { .name(SPACE_NAME) .organization(organization) .build(); - Mockito.when(client.getSpace(Mockito.any())) - .thenReturn(space); + when(client.getSpace(any())) + .thenReturn(space); return client; } private void mockProcessActionRegistry() { - Mockito.when(processActionRegistry.getAction(Mockito.any())) - .thenReturn(processAction); + when(processActionRegistry.getAction(any())) + .thenReturn(processAction); } private HttpServletRequest mockHttpServletRequest(String user) { - HttpServletRequest requestMock = Mockito.mock(HttpServletRequest.class); + HttpServletRequest requestMock = mock(HttpServletRequest.class); if (user != null) { - Principal principalMock = Mockito.mock(Principal.class); - Mockito.when(principalMock.getName()) - .thenReturn(user); - Mockito.when(requestMock.getUserPrincipal()) - .thenReturn(principalMock); + Principal principalMock = mock(Principal.class); + when(principalMock.getName()) + .thenReturn(user); + when(requestMock.getUserPrincipal()) + .thenReturn(principalMock); } return requestMock; } @SuppressWarnings("unchecked") private void setupOperationServiceMock() { - Mockito.when(operationService.createQuery()) - .thenReturn(operationQuery); - - Mockito.doAnswer(invocation -> { - processId = (String) invocation.getArguments()[0]; - return operationQuery; - }) - .when(operationQuery) - .processId(Mockito.anyString()); - Mockito.doAnswer(invocation -> { - Optional foundOperation = operations.stream() - .filter(operation -> operation.getProcessId() - .equals(processId)) - .findFirst(); - if (!foundOperation.isPresent()) { - throw new NoResultException("not found"); - } - return foundOperation.get(); - }) - .when(operationQuery) - .singleResult(); - - Mockito.doAnswer(invocation -> { - operationStatesToFilter = (List) invocation.getArguments()[0]; - return operationQuery; - }) - .when(operationQuery) - .withStateAnyOf(Mockito.anyList()); - Mockito.doAnswer(invocation -> operations.stream() - .filter(operation -> operationStatesToFilter == null || operationStatesToFilter.contains( - operation.getState())) - .collect(Collectors.toList())) - .when(operationQuery) - .list(); + when(operationService.createQuery()) + .thenReturn(operationQuery); + + doAnswer(invocation -> { + processId = (String) invocation.getArguments()[0]; + return operationQuery; + }) + .when(operationQuery) + .processId(anyString()); + doAnswer(invocation -> { + Optional foundOperation = operations.stream() + .filter(operation -> operation.getProcessId() + .equals(processId)) + .findFirst(); + if (!foundOperation.isPresent()) { + throw new NoResultException("not found"); + } + return foundOperation.get(); + }) + .when(operationQuery) + .singleResult(); + + doAnswer(invocation -> { + operationStatesToFilter = (List) invocation.getArguments()[0]; + return operationQuery; + }) + .when(operationQuery) + .withStateAnyOf(anyList()); + doAnswer(invocation -> operations.stream() + .filter(operation -> operationStatesToFilter == null || operationStatesToFilter.contains( + operation.getState())) + .collect(Collectors.toList())) + .when(operationQuery) + .list(); } @SuppressWarnings("unchecked") private void setupOperationsHelperMock() { - Mockito.when(operationsHelper.addErrorType(Mockito.any())) - .thenAnswer(invocation -> invocation.getArgument(0)); - Mockito.when(operationsHelper.releaseLockIfNeeded(Mockito.any())) - .thenAnswer(invocation -> invocation.getArgument(0)); - Mockito.when(operationsHelper.releaseLocksIfNeeded(Mockito.any())) - .thenAnswer(invocation -> invocation.getArgument(0)); + when(operationsHelper.addErrorType(any())) + .thenAnswer(invocation -> invocation.getArgument(0)); + when(operationsHelper.releaseLockIfNeeded(any())) + .thenAnswer(invocation -> invocation.getArgument(0)); + when(operationsHelper.releaseLocksIfNeeded(any())) + .thenAnswer(invocation -> invocation.getArgument(0)); } private Operation createOperation(String processId, Operation.State state, Map parameters) { diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsJmxReporterTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsJmxReporterTest.java new file mode 100644 index 0000000000..801a1bae42 --- /dev/null +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/monitoring/ActiveOperationsJmxReporterTest.java @@ -0,0 +1,129 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.util.List; + +import javax.management.MBeanServer; +import javax.management.MBeanServerFactory; +import javax.management.ObjectName; + +import org.cloudfoundry.multiapps.controller.api.model.Operation; +import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.query.OperationQuery; +import org.cloudfoundry.multiapps.controller.persistence.services.OperationService; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +class ActiveOperationsJmxReporterTest { + + @Mock + private OperationService operationService; + @Mock + private ApplicationConfiguration applicationConfiguration; + + private MBeanServer mBeanServer; + private ActiveOperationsJmxReporter reporter; + private AutoCloseable mocks; + + @BeforeEach + void setUp() { + mocks = MockitoAnnotations.openMocks(this); + mBeanServer = MBeanServerFactory.createMBeanServer(); + reporter = new ActiveOperationsJmxReporter(operationService, mBeanServer, applicationConfiguration); + enableReporting(); + } + + @AfterEach + void tearDown() throws Exception { + MBeanServerFactory.releaseMBeanServer(mBeanServer); + mocks.close(); + } + + @Test + void testRegistersMBeansPerUserAndSpace() throws Exception { + stubOperations(List.of(operation("alice@sap.com", "space-1"), + operation("alice@sap.com", "space-2"), + operation("bob@sap.com", "space-1"))); + + reporter.refresh(); + + assertEquals(2L, getCount(ActiveOperationsJmxReporter.USER_OBJECT_NAME_PATTERN, ActiveOperationsJmxReporter.hashUser("alice@sap.com"))); + assertEquals(1L, getCount(ActiveOperationsJmxReporter.USER_OBJECT_NAME_PATTERN, ActiveOperationsJmxReporter.hashUser("bob@sap.com"))); + assertEquals(2L, getCount(ActiveOperationsJmxReporter.SPACE_OBJECT_NAME_PATTERN, "space-1")); + assertEquals(1L, getCount(ActiveOperationsJmxReporter.SPACE_OBJECT_NAME_PATTERN, "space-2")); + } + + @Test + void testUnregistersMBeanWhenUserHasNoMoreActiveOperations() throws Exception { + stubOperations(List.of(operation("alice@sap.com", "space-1"))); + reporter.refresh(); + + stubOperations(List.of()); + reporter.refresh(); + + ObjectName name = new ObjectName(ActiveOperationsJmxReporter.USER_OBJECT_NAME_PATTERN.formatted(ObjectName.quote(ActiveOperationsJmxReporter.hashUser("alice@sap.com")))); + assertEquals(false, mBeanServer.isRegistered(name)); + } + + @Test + void testUpdatesCountWhenOperationsChange() throws Exception { + stubOperations(List.of(operation("alice@sap.com", "space-1"))); + reporter.refresh(); + assertEquals(1L, getCount(ActiveOperationsJmxReporter.USER_OBJECT_NAME_PATTERN, ActiveOperationsJmxReporter.hashUser("alice@sap.com"))); + + stubOperations(List.of(operation("alice@sap.com", "space-1"), + operation("alice@sap.com", "space-2"))); + reporter.refresh(); + assertEquals(2L, getCount(ActiveOperationsJmxReporter.USER_OBJECT_NAME_PATTERN, ActiveOperationsJmxReporter.hashUser("alice@sap.com"))); + } + + @Test + void testSkipsRefreshWhenRateLimitingDisabled() { + when(applicationConfiguration.isOperationRateLimitingEnabled()).thenReturn(false); + + reporter.refresh(); + + verifyNoInteractions(operationService); + } + + @Test + void testSkipsRefreshWhenNotSelectedInstance() { + when(applicationConfiguration.getApplicationInstanceIndex()).thenReturn(1); + + reporter.refresh(); + + verifyNoInteractions(operationService); + } + + private void enableReporting() { + when(applicationConfiguration.isOperationRateLimitingEnabled()).thenReturn(true); + when(applicationConfiguration.getApplicationInstanceIndex()).thenReturn(0); + } + + private void stubOperations(List operations) { + OperationQuery query = mock(OperationQuery.class); + when(operationService.createQuery()).thenReturn(query); + when(query.inNonFinalState()).thenReturn(query); + when(query.list()).thenReturn(operations); + } + + private long getCount(String pattern, String key) throws Exception { + ObjectName name = new ObjectName(pattern.formatted(ObjectName.quote(key))); + return (long) mBeanServer.getAttribute(name, "Count"); + } + + private Operation operation(String user, String spaceId) { + Operation op = mock(Operation.class); + when(op.getUser()).thenReturn(user); + when(op.getSpaceId()).thenReturn(spaceId); + return op; + } + +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimitKeysTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimitKeysTest.java new file mode 100644 index 0000000000..788f9f419b --- /dev/null +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimitKeysTest.java @@ -0,0 +1,71 @@ +package org.cloudfoundry.multiapps.controller.web.util; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +class OperationRateLimitKeysTest { + + private static final String SPACE_GUID = "3d4d3f9a-1a2b-4c5d-8e9f-0a1b2c3d4e5f"; + private static final String OTHER_SPACE_GUID = "9f8e7d6c-5b4a-3c2d-1e0f-abcdef123456"; + private static final String USER = "john.doe"; + private static final String OTHER_USER = "jane.roe"; + + @Test + void testSpaceKeyIsDeterministic() { + assertEquals(OperationRateLimitKeys.spaceKey(SPACE_GUID), OperationRateLimitKeys.spaceKey(SPACE_GUID)); + } + + @Test + void testUserKeyIsDeterministic() { + assertEquals(OperationRateLimitKeys.userKey(SPACE_GUID, USER), OperationRateLimitKeys.userKey(SPACE_GUID, USER)); + } + + @Test + void testSpaceKeyAndUserKeyAreDisjointForSameSpace() { + assertNotEquals(OperationRateLimitKeys.spaceKey(SPACE_GUID), OperationRateLimitKeys.userKey(SPACE_GUID, USER)); + } + + @Test + void testDifferentSpacesProduceDifferentKeys() { + assertNotEquals(OperationRateLimitKeys.spaceKey(SPACE_GUID), OperationRateLimitKeys.spaceKey(OTHER_SPACE_GUID)); + } + + @Test + void testDifferentUsersInSameSpaceProduceDifferentKeys() { + assertNotEquals(OperationRateLimitKeys.userKey(SPACE_GUID, USER), OperationRateLimitKeys.userKey(SPACE_GUID, OTHER_USER)); + } + + @Test + void testSameUserInDifferentSpacesProduceDifferentKeys() { + assertNotEquals(OperationRateLimitKeys.userKey(SPACE_GUID, USER), OperationRateLimitKeys.userKey(OTHER_SPACE_GUID, USER)); + } + + @Test + void testNoCollisionsAcrossRealisticSamples() { + List spaceGuids = List.of(SPACE_GUID, OTHER_SPACE_GUID, "11111111-2222-3333-4444-555555555555", + "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"); + List users = List.of(USER, OTHER_USER, "admin", "service-account-1"); + Set keys = new HashSet<>(); + for (String spaceGuid : spaceGuids) { + keys.add(OperationRateLimitKeys.spaceKey(spaceGuid)); + for (String user : users) { + keys.add(OperationRateLimitKeys.userKey(spaceGuid, user)); + } + } + int expectedDistinctKeys = spaceGuids.size() + spaceGuids.size() * users.size(); + assertEquals(expectedDistinctKeys, keys.size()); + } + + @Test + void testSpaceKeyIsStableAcrossRuns() { + long firstValue = OperationRateLimitKeys.spaceKey(SPACE_GUID); + long secondValue = OperationRateLimitKeys.spaceKey(SPACE_GUID); + assertEquals(firstValue, secondValue); + } +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimiterTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimiterTest.java new file mode 100644 index 0000000000..a6324e0b8e --- /dev/null +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/util/OperationRateLimiterTest.java @@ -0,0 +1,207 @@ +package org.cloudfoundry.multiapps.controller.web.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import io.github.bucket4j.Bucket; +import io.github.bucket4j.BucketConfiguration; +import io.github.bucket4j.ConsumptionProbe; +import org.cloudfoundry.multiapps.controller.api.model.Operation; +import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.query.OperationQuery; +import org.cloudfoundry.multiapps.controller.persistence.services.OperationService; +import org.cloudfoundry.multiapps.controller.process.util.BucketStore; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +class OperationRateLimiterTest { + + private static final String SPACE_GUID = "3d4d3f9a-1a2b-4c5d-8e9f-0a1b2c3d4e5f"; + private static final String USER = "john.doe"; + private static final int PER_SPACE_CAPACITY = 300; + private static final int PER_SPACE_REFILL_PER_HOUR = 800; + private static final int PER_USER_CAPACITY = 150; + private static final int PER_USER_REFILL_PER_HOUR = 300; + private static final int MAX_ACTIVE_OPERATIONS_PER_SPACE = 500; + private static final int MAX_ACTIVE_OPERATIONS_PER_USER = 200; + private static final long NANOS_TO_WAIT = TimeUnit.SECONDS.toNanos(42); + + @Mock + private ApplicationConfiguration applicationConfiguration; + @Mock + private OperationService operationService; + @Mock + private BucketStore bucketStore; + @Mock + private Bucket spaceBucket; + @Mock + private Bucket userBucket; + + private OperationRateLimiter operationRateLimiter; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + when(applicationConfiguration.getOperationRateLimitPerSpaceCapacity()).thenReturn(PER_SPACE_CAPACITY); + when(applicationConfiguration.getOperationRateLimitPerSpaceRefillPerHour()).thenReturn(PER_SPACE_REFILL_PER_HOUR); + when(applicationConfiguration.getOperationRateLimitPerUserCapacity()).thenReturn(PER_USER_CAPACITY); + when(applicationConfiguration.getOperationRateLimitPerUserRefillPerHour()).thenReturn(PER_USER_REFILL_PER_HOUR); + operationRateLimiter = new OperationRateLimiter(applicationConfiguration, operationService, bucketStore); + } + + private void enableRateLimiting() { + when(applicationConfiguration.isOperationRateLimitingEnabled()).thenReturn(true); + } + + private void stubRateLimitConfiguration() { + when(applicationConfiguration.getMaxActiveOperationsPerSpace()).thenReturn(MAX_ACTIVE_OPERATIONS_PER_SPACE); + when(applicationConfiguration.getMaxActiveOperationsPerUser()).thenReturn(MAX_ACTIVE_OPERATIONS_PER_USER); + when(applicationConfiguration.getOperationRateLimitPerSpaceCapacity()).thenReturn(PER_SPACE_CAPACITY); + when(applicationConfiguration.getOperationRateLimitPerSpaceRefillPerHour()).thenReturn(PER_SPACE_REFILL_PER_HOUR); + when(applicationConfiguration.getOperationRateLimitPerUserCapacity()).thenReturn(PER_USER_CAPACITY); + when(applicationConfiguration.getOperationRateLimitPerUserRefillPerHour()).thenReturn(PER_USER_REFILL_PER_HOUR); + } + + @Test + void testAllowedWhenFeatureFlagOffAndNothingIsTouched() { + when(applicationConfiguration.isOperationRateLimitingEnabled()).thenReturn(false); + + assertDoesNotThrow(() -> operationRateLimiter.checkStartAllowed(USER, SPACE_GUID)); + + verifyNoInteractions(operationService); + verifyNoInteractions(bucketStore); + } + + @Test + void testCheckStartAllowedWhenUnderAllLimits() { + enableRateLimiting(); + stubRateLimitConfiguration(); + stubActiveOperationCounts(0, 0); + stubBucketForKey(OperationRateLimitKeys.spaceKey(SPACE_GUID), spaceBucket); + stubBucketForKey(OperationRateLimitKeys.userKey(SPACE_GUID, USER), userBucket); + stubConsumption(spaceBucket, true); + stubConsumption(userBucket, true); + + assertDoesNotThrow(() -> operationRateLimiter.checkStartAllowed(USER, SPACE_GUID)); + } + + @Test + void testChecksSpaceAndUserBucketsIndependently() { + enableRateLimiting(); + stubRateLimitConfiguration(); + stubActiveOperationCounts(0, 0); + stubBucketForKey(OperationRateLimitKeys.spaceKey(SPACE_GUID), spaceBucket); + stubBucketForKey(OperationRateLimitKeys.userKey(SPACE_GUID, USER), userBucket); + stubConsumption(spaceBucket, true); + stubConsumption(userBucket, true); + + operationRateLimiter.checkStartAllowed(USER, SPACE_GUID); + + verify(bucketStore).getBucket(eq(OperationRateLimitKeys.spaceKey(SPACE_GUID)), any()); + verify(bucketStore).getBucket(eq(OperationRateLimitKeys.userKey(SPACE_GUID, USER)), any()); + } + + @Test + void testThrowExceptionWhenSpaceBucketExhausted() { + enableRateLimiting(); + stubRateLimitConfiguration(); + stubActiveOperationCounts(0, 0); + stubBucketForKey(OperationRateLimitKeys.userKey(SPACE_GUID, USER), userBucket); + stubBucketForKey(OperationRateLimitKeys.spaceKey(SPACE_GUID), spaceBucket); + stubConsumption(userBucket, true); + stubConsumption(spaceBucket, false); + + OperationRateLimitExceededException exception = assertThrows(OperationRateLimitExceededException.class, + () -> operationRateLimiter.checkStartAllowed(USER, SPACE_GUID)); + assertEquals(TimeUnit.NANOSECONDS.toSeconds(NANOS_TO_WAIT), exception.getRetryAfterSeconds()); + } + + @Test + void testThrowExceptionWhenUserBucketExhausted() { + enableRateLimiting(); + stubRateLimitConfiguration(); + stubActiveOperationCounts(0, 0); + stubBucketForKey(OperationRateLimitKeys.spaceKey(SPACE_GUID), spaceBucket); + stubBucketForKey(OperationRateLimitKeys.userKey(SPACE_GUID, USER), userBucket); + stubConsumption(spaceBucket, true); + stubConsumption(userBucket, false); + + OperationRateLimitExceededException exception = assertThrows(OperationRateLimitExceededException.class, + () -> operationRateLimiter.checkStartAllowed(USER, SPACE_GUID)); + assertEquals(TimeUnit.NANOSECONDS.toSeconds(NANOS_TO_WAIT), exception.getRetryAfterSeconds()); + } + + @Test + void testThrowExceptionWhenActiveOperationsPerSpaceReachCap() { + enableRateLimiting(); + when(applicationConfiguration.getMaxActiveOperationsPerSpace()).thenReturn(MAX_ACTIVE_OPERATIONS_PER_SPACE); + stubActiveOperationCounts(MAX_ACTIVE_OPERATIONS_PER_SPACE, 0); + + assertThrows(OperationRateLimitExceededException.class, () -> operationRateLimiter.checkStartAllowed(USER, SPACE_GUID)); + verifyNoInteractions(bucketStore); + } + + @Test + void testThrowExceptionWhenActiveOperationsPerUserReachCap() { + enableRateLimiting(); + when(applicationConfiguration.getMaxActiveOperationsPerSpace()).thenReturn(MAX_ACTIVE_OPERATIONS_PER_SPACE); + when(applicationConfiguration.getMaxActiveOperationsPerUser()).thenReturn(MAX_ACTIVE_OPERATIONS_PER_USER); + stubActiveOperationCounts(MAX_ACTIVE_OPERATIONS_PER_USER, MAX_ACTIVE_OPERATIONS_PER_USER); + + assertThrows(OperationRateLimitExceededException.class, () -> operationRateLimiter.checkStartAllowed(USER, SPACE_GUID)); + verifyNoInteractions(bucketStore); + } + + private void stubActiveOperationCounts(int perSpace, int perUser) { + List ops = activeOperations(perSpace, perUser); + OperationQuery spaceQuery = mockQuery(); + when(spaceQuery.spaceId(SPACE_GUID)).thenReturn(spaceQuery); + when(spaceQuery.inNonFinalState()).thenReturn(spaceQuery); + when(spaceQuery.list()).thenReturn(ops); + + when(operationService.createQuery()).thenReturn(spaceQuery); + } + + private OperationQuery mockQuery() { + return mock(OperationQuery.class); + } + + private List activeOperations(int total, int byUser) { + List ops = new ArrayList<>(); + for (int i = 0; i < byUser; i++) { + Operation op = mock(Operation.class); + when(op.getUser()).thenReturn(USER); + ops.add(op); + } + for (int i = byUser; i < total; i++) { + Operation op = mock(Operation.class); + when(op.getUser()).thenReturn("other.user"); + ops.add(op); + } + return ops; + } + + private void stubBucketForKey(long key, Bucket bucket) { + when(bucketStore.getBucket(eq(key), any(BucketConfiguration.class))).thenReturn(bucket); + } + + private void stubConsumption(Bucket bucket, boolean consumed) { + ConsumptionProbe probe = consumed ? ConsumptionProbe.consumed(1, NANOS_TO_WAIT) + : ConsumptionProbe.rejected(0, NANOS_TO_WAIT, NANOS_TO_WAIT); + when(bucket.tryConsumeAndReturnRemaining(1)).thenReturn(probe); + } +} diff --git a/pom.xml b/pom.xml index 87d54ee76e..7b0854cc65 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,7 @@ 1.0.4 4.0.1 6.3.0 + 8.14.0 multiapps-controller-client @@ -825,6 +826,12 @@ resilience4j-ratelimiter ${resilience4j.version} + + + com.bucket4j + bucket4j_jdk17-postgresql + ${bucket4j.version} +