Allow exclusion of destination addresses from usage on KVM shared networks

Author: MitchDrage

debian/control

@@ -24,7 +24,7 @@ Description: CloudStack server library
 
 Package: cloudstack-agent
 Architecture: all
-Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17 | openjdk-21-jre-headless | java21-runtime-headless | java21-runtime | zulu-21, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat
+Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17 | openjdk-21-jre-headless | java21-runtime-headless | java21-runtime | zulu-21, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, nftables, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat
 Recommends: init-system-helpers
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent

engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java

@@ -145,9 +145,10 @@ public interface NetworkOrchestrationService {
             "Default data transfer rate in megabits per second allowed in User vm's default network.", true, ConfigKey.Scope.Zone);
 
     ConfigKey<String> NetworkUsageExemptCidrs = new ConfigKey<>(String.class, "network.usage.exempt.cidrs", "Usage", "",
-            "Comma-separated IPv4 or IPv6 addresses or CIDRs. Traffic between guest networks and these addresses doesn't count " +
-            "towards virtual router network usage. Virtual routers only count IPv4 traffic, so they ignore IPv6 entries. " +
-            "Routers pick up a change the next time they start or restart.",
+            "Comma-separated IPv4 or IPv6 addresses or CIDRs. Traffic between guest VMs and these addresses doesn't count " +
+            "towards network usage. Virtual routers only count IPv4 traffic, so they ignore IPv6 entries, and pick up a change " +
+            "the next time they start or restart. On shared networks, KVM hosts exempt both IPv4 and IPv6 and pick up a change " +
+            "straight away.",
             true, ConfigKey.Scope.Zone, null, null, null, null, null, ConfigKey.Kind.CSV, null);
 
     /**

engine/components-api/src/main/java/com/cloud/agent/AgentManager.java

@@ -175,6 +175,12 @@ enum TapAgentsAction {
 
     void propagateChangeToAgents(Map<String, String> params);
 
+    /**
+     * Sends network.usage.exempt.cidrs to the KVM hosts in one zone, or in every zone when zoneId is null,
+     * using the value for each host's zone.
+     */
+    void propagateUsageExemptCidrsToAgents(Long zoneId);
+
     boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance);
 
     int getHostSshPort(HostVO host);

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -23,6 +23,7 @@
 import java.nio.channels.ClosedChannelException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -2288,6 +2289,7 @@ public void processConnect(final Host host, final StartupCommand cmd, final bool
                 params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
                 params.put(NetworkOrchestrationService.TUNGSTEN_ENABLED.key(), String.valueOf(NetworkOrchestrationService.TUNGSTEN_ENABLED.valueIn(host.getDataCenterId())));
                 params.put(ReconcileCommandService.ReconcileCommandsEnabled.key(), String.valueOf(_reconcileCommandsEnabled));
+                params.put(NetworkOrchestrationService.NetworkUsageExemptCidrs.key(), getUsageExemptCidrsParam(host.getDataCenterId()));
 
                     try {
                         SetHostParamsCommand cmds = new SetHostParamsCommand(params);
@@ -2366,6 +2368,31 @@ public void propagateChangeToAgents(Map<String, String> params) {
         }
     }
 
+    @Override
+    public void propagateUsageExemptCidrsToAgents(Long zoneId) {
+        final Map<Long, List<Long>> hostsPerZone = getHostsPerZone();
+        if (hostsPerZone == null) {
+            return;
+        }
+        logger.debug("Sending {} to the agents", NetworkOrchestrationService.NetworkUsageExemptCidrs.key());
+        for (Map.Entry<Long, List<Long>> zone : hostsPerZone.entrySet()) {
+            if (zoneId != null && !zoneId.equals(zone.getKey())) {
+                continue;
+            }
+            final Map<String, String> params = new HashMap<>();
+            params.put(NetworkOrchestrationService.NetworkUsageExemptCidrs.key(), getUsageExemptCidrsParam(zone.getKey()));
+            sendCommandToAgents(Collections.singletonMap(zone.getKey(), zone.getValue()), params);
+        }
+    }
+
+    /**
+     * The zone's exempt list, tidied and joined with commas. An empty string tells the host the list is empty.
+     */
+    protected String getUsageExemptCidrsParam(long zoneId) {
+        return String.join(",", NetworkOrchestrationService.parseUsageExemptCidrs(
+                NetworkOrchestrationService.NetworkUsageExemptCidrs.valueIn(zoneId)));
+    }
+
     @Override
     public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance) {
         return true;

engine/orchestration/src/test/java/com/cloud/agent/manager/AgentManagerImplTest.java

@@ -20,6 +20,7 @@
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.ReadyAnswer;
 import com.cloud.agent.api.ReadyCommand;
+import com.cloud.agent.api.SetHostParamsCommand;
 import com.cloud.agent.api.StartupCommand;
 import com.cloud.agent.api.StartupRoutingCommand;
 import com.cloud.exception.ConnectionException;
@@ -37,7 +38,10 @@
 import org.mockito.Mockito;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public class AgentManagerImplTest {
@@ -172,4 +176,33 @@ public void testGetHostSshPortWithKVMHostCustomPort() {
         int hostSshPort = mgr.getHostSshPort(host);
         Assert.assertEquals(3922, hostSshPort);
     }
+
+    private void mockUsageExemptPush() {
+        Map<Long, List<Long>> hostsPerZone = new HashMap<>();
+        hostsPerZone.put(1L, Arrays.asList(11L, 12L));
+        hostsPerZone.put(2L, Collections.singletonList(21L));
+        Mockito.doReturn(hostsPerZone).when(mgr).getHostsPerZone();
+        Mockito.doReturn("192.0.2.5/32").when(mgr).getUsageExemptCidrsParam(Mockito.anyLong());
+        Answer answer = Mockito.mock(Answer.class);
+        Mockito.when(answer.getResult()).thenReturn(true);
+        Mockito.doReturn(answer).when(mgr).easySend(Mockito.anyLong(), Mockito.any(SetHostParamsCommand.class));
+    }
+
+    @Test
+    public void propagateUsageExemptCidrsToAgentsTestOneZone() {
+        mockUsageExemptPush();
+        mgr.propagateUsageExemptCidrsToAgents(1L);
+        Mockito.verify(mgr).easySend(Mockito.eq(11L), Mockito.any(SetHostParamsCommand.class));
+        Mockito.verify(mgr).easySend(Mockito.eq(12L), Mockito.any(SetHostParamsCommand.class));
+        Mockito.verify(mgr, Mockito.never()).easySend(Mockito.eq(21L), Mockito.any(SetHostParamsCommand.class));
+    }
+
+    @Test
+    public void propagateUsageExemptCidrsToAgentsTestAllZones() {
+        mockUsageExemptPush();
+        mgr.propagateUsageExemptCidrsToAgents(null);
+        Mockito.verify(mgr, Mockito.times(3)).easySend(Mockito.anyLong(), Mockito.any(SetHostParamsCommand.class));
+        Mockito.verify(mgr).getUsageExemptCidrsParam(1L);
+        Mockito.verify(mgr).getUsageExemptCidrsParam(2L);
+    }
 }

packaging/el8/cloud.spec

@@ -114,6 +114,7 @@ Requires: ethtool
 Requires: (net-tools or net-tools-deprecated)
 Requires: iproute
 Requires: ipset
+Requires: nftables
 Requires: perl
 Requires: rsync
 Requires: cifs-utils

packaging/suse15/cloud.spec

@@ -114,6 +114,7 @@ Requires: ethtool
 Requires: (net-tools or net-tools-deprecated)
 Requires: iproute
 Requires: ipset
+Requires: nftables
 Requires: perl
 Requires: rsync
 Requires: cifs-utils

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -623,6 +623,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
     protected MemStat memStat = new MemStat(dom0MinMem, dom0OvercommitMem);
     private final LibvirtUtilitiesHelper libvirtUtilitiesHelper = new LibvirtUtilitiesHelper();
     private LibvirtDomainListener libvirtDomainListener;
+    private final UsageExemptCounters usageExemptCounters = new UsageExemptCounters();
 
     protected Boolean enableManuallySettingCpuTopologyOnKvmVm = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.ENABLE_MANUALLY_SETTING_CPU_TOPOLOGY_ON_KVM_VM);
 
@@ -1790,10 +1791,19 @@ public boolean configureHostParams(final Map<String, String> params) {
             isTungstenEnabled = Boolean.parseBoolean(params.get(NetworkOrchestrationService.TUNGSTEN_ENABLED.key()));
         }
 
-        if (params.get(ReconcileCommandService.ReconcileCommandsEnabled.key()) != null) {
-            isReconcileCommandsEnabled = Boolean.parseBoolean(params.get(ReconcileCommandService.ReconcileCommandsEnabled.key()));
+        final String usageExemptCidrs = params.get(NetworkOrchestrationService.NetworkUsageExemptCidrs.key());
+        if (usageExemptCidrs != null && hypervisorType == HypervisorType.KVM) {
+            usageExemptCounters.setTapOwners(this::getTapOwners);
+            usageExemptCounters.setCidrs(NetworkOrchestrationService.parseUsageExemptCidrs(usageExemptCidrs));
         }
-        if (isReconcileCommandsEnabled) {
+
+        final String reconcileCommandsEnabled = params.get(ReconcileCommandService.ReconcileCommandsEnabled.key());
+        if (reconcileCommandsEnabled != null) {
+            isReconcileCommandsEnabled = Boolean.parseBoolean(reconcileCommandsEnabled);
+        }
+        // Only on connect, which is the only time the server sends this setting. Settings pushed while the agent is
+        // running must leave the logs of commands still in progress alone.
+        if (isReconcileCommandsEnabled && reconcileCommandsEnabled != null) {
             File commandsLogPath = new File(COMMANDS_LOG_PATH);
             if (!commandsLogPath.exists()) {
                 commandsLogPath.mkdirs();
@@ -5197,6 +5207,38 @@ private int executeBashScriptAndRetrieveExitValue(final String script) {
         return command.getExitValue();
     }
 
+    public UsageExemptCounters getUsageExemptCounters() {
+        return usageExemptCounters;
+    }
+
+    /**
+     * Maps each running VM's NIC device name to the VM's name, so the usage exemption counters can skip system VMs.
+     * Returns null if libvirt can't be asked.
+     */
+    protected Map<String, String> getTapOwners() {
+        try {
+            final Connect conn = LibvirtConnection.getConnection();
+            final Map<String, String> owners = new HashMap<>();
+            for (final int id : conn.listDomains()) {
+                final Domain dm = conn.domainLookupByID(id);
+                try {
+                    final String vmName = dm.getName();
+                    for (final InterfaceDef nic : getInterfaces(conn, vmName)) {
+                        if (nic.getDevName() != null) {
+                            owners.put(nic.getDevName(), vmName);
+                        }
+                    }
+                } finally {
+                    dm.free();
+                }
+            }
+            return owners;
+        } catch (final LibvirtException e) {
+            LOGGER.debug("Failed to list the running VMs' NICs for the usage exemption counters", e);
+            return null;
+        }
+    }
+
     public List<VmNetworkStatsEntry> getVmNetworkStat(Connect conn, String vmName) throws LibvirtException {
         Domain dm = null;
         try {
@@ -5209,7 +5251,8 @@ public List<VmNetworkStatsEntry> getVmNetworkStat(Connect conn, String vmName) t
             for (InterfaceDef nic : nics) {
                 DomainInterfaceStats nicStats = dm.interfaceStats(nic.getDevName());
                 String macAddress = nic.getMacAddress();
-                VmNetworkStatsEntry stat = new VmNetworkStatsEntry(vmName, macAddress, nicStats.tx_bytes, nicStats.rx_bytes);
+                long[] usage = usageExemptCounters.subtractExempt(nic.getDevName(), nicStats.tx_bytes, nicStats.rx_bytes);
+                VmNetworkStatsEntry stat = new VmNetworkStatsEntry(vmName, macAddress, usage[0], usage[1]);
                 stats.add(stat);
             }