diff --git a/Makefile b/Makefile index b817a7d9..92c6f62b 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ stop: REGISTRY ?= localhost:5000 WORKERS ?= 2 ENV_FILE ?= .env -SWARM_VARS = REGISTRY CERTBOT_DOMAIN CERTBOT_EMAIL NFS_SERVER_IP NFS_BASE_PATH WORKER_CPU_LIMIT WORKER_MEMORY_LIMIT WORKER_CPU_RESERVATION WORKER_MEMORY_RESERVATION +SWARM_VARS = REGISTRY CERTBOT_DOMAIN CERTBOT_EMAIL NFS_SERVER_IP NFS_BASE_PATH WORKER_CPU_LIMIT WORKER_MEMORY_LIMIT WORKER_CPU_RESERVATION WORKER_MEMORY_RESERVATION WORKER_REPLICAS BACKEND_MEMORY_LIMIT SCHEDULER_MEMORY_LIMIT AV_WORKER_MEMORY_LIMIT SWARM_ENV = ENV_FILE="$(ENV_FILE)" $(foreach v,$(SWARM_VARS),$(v)="$(shell grep '^$(v)=' $(ENV_FILE) | head -1 | cut -d= -f2-)") start-swarm: diff --git a/docker-compose.swarm.nodb.yml b/docker-compose.swarm.nodb.yml index 637275a1..618779c9 100644 --- a/docker-compose.swarm.nodb.yml +++ b/docker-compose.swarm.nodb.yml @@ -23,6 +23,9 @@ services: - validate deploy: replicas: 1 + resources: + limits: + memory: 256M placement: constraints: [node.role == manager] restart_policy: @@ -43,6 +46,9 @@ services: - validate deploy: replicas: 1 + resources: + limits: + memory: ${BACKEND_MEMORY_LIMIT} placement: constraints: [node.role == manager] restart_policy: @@ -69,7 +75,7 @@ services: networks: - validate deploy: - replicas: 2 + replicas: ${WORKER_REPLICAS} # No placement constraint - workers run on any node resources: limits: @@ -97,7 +103,7 @@ services: replicas: 1 resources: limits: - memory: 4G + memory: ${AV_WORKER_MEMORY_LIMIT} # No placement constraint - workers run on any node restart_policy: condition: on-failure @@ -118,6 +124,9 @@ services: - validate deploy: replicas: 1 + resources: + limits: + memory: ${SCHEDULER_MEMORY_LIMIT} placement: constraints: [node.role == manager] restart_policy: @@ -135,6 +144,9 @@ services: - validate deploy: replicas: 1 + resources: + limits: + memory: 512M placement: constraints: [node.role == manager] restart_policy: @@ -163,6 +175,9 @@ services: - validate deploy: replicas: 1 + resources: + limits: + memory: 512M placement: constraints: [node.role == manager] restart_policy: @@ -187,6 +202,9 @@ services: - validate deploy: replicas: 1 + resources: + limits: + memory: 1G placement: constraints: [node.role == manager] restart_policy: @@ -215,6 +233,9 @@ services: - validate deploy: replicas: 1 + resources: + limits: + memory: 512M placement: constraints: [node.role == manager] restart_policy: @@ -231,6 +252,9 @@ services: - validate deploy: mode: global # one per node; Prometheus discovers via tasks.node_exporter + resources: + limits: + memory: 128M restart_policy: condition: on-failure delay: 5s @@ -245,6 +269,9 @@ services: - validate deploy: replicas: 1 + resources: + limits: + memory: 512M restart_policy: condition: on-failure delay: 5s diff --git a/docker/backend/worker-beat-entrypoint.sh b/docker/backend/worker-beat-entrypoint.sh index 8aca4772..f413f81c 100644 --- a/docker/backend/worker-beat-entrypoint.sh +++ b/docker/backend/worker-beat-entrypoint.sh @@ -18,7 +18,8 @@ do done echo "DB is ready." -CELERY_CONCURRENCY=${CELERY_CONCURRENCY:-6} # default 6 worker processes -echo "Celery concurrency: $CELERY_CONCURRENCY" - -celery --app=core worker -Q celery --loglevel=info --concurrency $CELERY_CONCURRENCY --task-events --hostname=worker@%n --beat --scheduler django_celery_beat.schedulers:DatabaseScheduler \ No newline at end of file +# Beat-only: this service schedules periodic tasks and no longer consumes the +# celery queue itself. Task capacity lives in the worker service, where it is +# budgeted (resources.limits x CELERY_CONCURRENCY); the previous embedded worker +# added 4 unbudgeted task slots and peaked at 8 GB RSS without any memory limit. +celery --app=core beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler \ No newline at end of file