From b33eedd403a3907411b9c7ee594acd6d4d944d0b Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 11 Oct 2024 15:21:23 +0200 Subject: [PATCH] Increase size of connection tracking table We have seen occurences of Cinder or Glance activity filling up the conntrack table. This has been seen on controllers and storage nodes (Ceph). We have also seen large amounts of traffic to an Octavia amphora causing the conntrack table of a compute host to fill up. Quadruple the maximum size of the table on these hosts (the default size is 262,144 for hosts with more than 4 GiB of memory [1]). Monitoring hosts are left unchanged since they are believed to be unaffected. [1] https://www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt --- etc/kayobe/compute.yml | 3 ++- etc/kayobe/controllers.yml | 3 ++- etc/kayobe/storage.yml | 3 ++- .../notes/bump-conntrack-table-size-c2d08f2aac04d23f.yaml | 7 +++++++ 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/bump-conntrack-table-size-c2d08f2aac04d23f.yaml diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index 75ff73c5f0..2bebd63b2e 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -106,7 +106,8 @@ # Compute node sysctl configuration. # Dict of sysctl parameters to set. -#compute_sysctl_parameters: +compute_sysctl_parameters: + net.netfilter.nf_conntrack_max: 1048576 ############################################################################### # Compute node tuned configuration. diff --git a/etc/kayobe/controllers.yml b/etc/kayobe/controllers.yml index cdb26a592c..603c3bc279 100644 --- a/etc/kayobe/controllers.yml +++ b/etc/kayobe/controllers.yml @@ -115,7 +115,8 @@ # Controller node sysctl configuration. # Dict of sysctl parameters to set. -#controller_sysctl_parameters: +controller_sysctl_parameters: + net.netfilter.nf_conntrack_max: 1048576 ############################################################################### # Controller node tuned configuration. diff --git a/etc/kayobe/storage.yml b/etc/kayobe/storage.yml index 32a7382aa6..358749d6e3 100644 --- a/etc/kayobe/storage.yml +++ b/etc/kayobe/storage.yml @@ -111,7 +111,8 @@ # Storage node sysctl configuration. # Dict of sysctl parameters to set. -#storage_sysctl_parameters: +storage_sysctl_parameters: + net.netfilter.nf_conntrack_max: 1048576 ############################################################################### # Storage node tuned configuration. diff --git a/releasenotes/notes/bump-conntrack-table-size-c2d08f2aac04d23f.yaml b/releasenotes/notes/bump-conntrack-table-size-c2d08f2aac04d23f.yaml new file mode 100644 index 0000000000..e68cda973c --- /dev/null +++ b/releasenotes/notes/bump-conntrack-table-size-c2d08f2aac04d23f.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Increases maximum size of connection tracking tables to 1,048,576 entries + on controllers, compute and storage hosts. This is to work around loss of + connectivity when the conntrack table becomes full under high Cinder/Glance + activity or high traffic to Octavia amphorae.