diff --git a/docs/cloud-controller-manager.md b/docs/cloud-controller-manager.md new file mode 100644 index 00000000..454a37af --- /dev/null +++ b/docs/cloud-controller-manager.md @@ -0,0 +1,23 @@ +# Cloud controller manager + +## Overview + +The cloud controller manager implements the [Kubernetes cloud-controller-manager contract](https://kubernetes.io/docs/concepts/architecture/cloud-controller/#functions-of-the-ccm). + +### Node controller + +The node controller is responsible for updating Node objects when new servers are created in STACKIT infrastructure by obtaining information about the servers. + +For more information check the [Kubernetes documentation](https://kubernetes.io/docs/concepts/architecture/cloud-controller/#node-controller). + +#### Multi Network + +If a server has NICs connected to multiple networks, you can designate the primary network for [Node Addresses](https://kubernetes.io/docs/reference/node/node-status/#addresses) by setting the default network in the config: + +```yaml +instance: + # either network name or id + defaultNetwork: "foo" +``` + +This ensures the IP address for that network's NIC is listed first in the [Node status](https://kubernetes.io/docs/reference/node/node-status/#addresses). diff --git a/docs/migration/configuration.md b/docs/migration/configuration.md index a6cbd6fb..7d340bfa 100644 --- a/docs/migration/configuration.md +++ b/docs/migration/configuration.md @@ -79,6 +79,8 @@ loadBalancer: extraLabels: key1: value1 key2: value2 +instance: {} + # defaultNetwork: # used for multi-network nodes ``` ### CSI Configuration diff --git a/pkg/ccm/instances.go b/pkg/ccm/instances.go index 6e1d9a87..08d3613d 100644 --- a/pkg/ccm/instances.go +++ b/pkg/ccm/instances.go @@ -21,10 +21,12 @@ import ( "errors" "fmt" "regexp" + "slices" "strings" "github.com/stackitcloud/cloud-provider-stackit/pkg/labels" stackitclient "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/client" + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/config" "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" iaas "github.com/stackitcloud/stackit-sdk-go/services/iaas/v2api" corev1 "k8s.io/api/core/v1" @@ -49,13 +51,15 @@ type Instances struct { regionProviderID bool iaasClient stackitclient.IaaSClient region string + defaultNetwork string } -func NewInstance(client stackitclient.IaaSClient, region string) (*Instances, error) { +func NewInstance(client stackitclient.IaaSClient, region string, opts config.InstanceOpts) (*Instances, error) { return &Instances{ iaasClient: client, region: region, regionProviderID: false, + defaultNetwork: opts.DefaultNetwork, }, nil } @@ -104,7 +108,7 @@ func (i *Instances) InstanceMetadata(ctx context.Context, node *corev1.Node) (*c return nil, fmt.Errorf("server has no network interfaces") } - nics := server.GetNics() + nics := sortNics(server.GetNics(), i.defaultNetwork) for i := range nics { nic := &nics[i] if nic.HasIpv4() { @@ -153,6 +157,36 @@ func (i *Instances) makeInstanceID(server *iaas.Server) string { return fmt.Sprintf("%s://%s", ProviderName, server.GetId()) } +// sortNics sorts a slice of server network interfaces alphabetically by their network name +// to ensure a deterministic order. If a non-empty defaultNetwork is provided (matching either +// the NetworkName or NetworkId), that specific network interface is moved to the front (index 0) +// of the returned slice. +func sortNics(nics []iaas.ServerNetwork, defaultNetwork string) []iaas.ServerNetwork { + // nics are returned by IaaS API in a non-deterministic order + // Sort by network name so that every time we use the same order for node addresses + slices.SortFunc(nics, func(a, b iaas.ServerNetwork) int { + return strings.Compare(a.NetworkName, b.NetworkName) + }) + + if defaultNetwork == "" { + return nics + } + + idx := slices.IndexFunc(nics, func(nic iaas.ServerNetwork) bool { + return nic.NetworkName == defaultNetwork || nic.NetworkId == defaultNetwork + }) + // network not found + if idx == -1 { + klog.Infof("no NIC found for default network %s", defaultNetwork) + return nics + } + defaultNic := nics[idx] + nics = slices.Delete(nics, idx, idx+1) + // prepend default nic + nics = slices.Insert(nics, 0, defaultNic) + return nics +} + // addToNodeAddresses appends the NodeAddresses to the passed-by-pointer slice, // only if they do not already exist func addToNodeAddresses(addresses *[]corev1.NodeAddress, addAddresses ...corev1.NodeAddress) { diff --git a/pkg/ccm/instances_test.go b/pkg/ccm/instances_test.go index 304e1709..0f9e558a 100644 --- a/pkg/ccm/instances_test.go +++ b/pkg/ccm/instances_test.go @@ -26,6 +26,7 @@ import ( oapiError "github.com/stackitcloud/stackit-sdk-go/core/oapierror" stackitclientmock "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/client/mock" + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/config" iaas "github.com/stackitcloud/stackit-sdk-go/services/iaas/v2api" "go.uber.org/mock/gomock" corev1 "k8s.io/api/core/v1" @@ -49,7 +50,7 @@ var _ = Describe("Node Controller", func() { nodeMockClient = stackitclientmock.NewMockIaaSClient(ctrl) var err error - instance, err = NewInstance(nodeMockClient, "eu01") + instance, err = NewInstance(nodeMockClient, "eu01", config.InstanceOpts{}) Expect(err).NotTo(HaveOccurred()) }) @@ -281,4 +282,39 @@ var _ = Describe("Node Controller", func() { Expect(metadata).To(BeNil()) }) }) + + Describe("#sortNics", func() { + It("should return the nic of the default network as primary", func() { + nics := []iaas.ServerNetwork{ + { + NetworkName: "abc", + NetworkId: "69", + Ipv4: new("10.0.0.69"), + }, + { + NetworkName: "default", + NetworkId: "69", + Ipv4: new("192.168.0.123"), + }, + { + NetworkName: "foo", + NetworkId: "123", + Ipv4: new("100.80.0.5"), + }, + } + By("with network name") + newNics := sortNics(nics, "default") + Expect(newNics).To(HaveLen(3)) + Expect(newNics[0].NetworkName).To(Equal("default")) + Expect(newNics[1].NetworkName).To(Equal("abc")) + Expect(newNics[2].NetworkName).To(Equal("foo")) + + By("with network id") + newNics = sortNics(nics, "123") + Expect(newNics).To(HaveLen(3)) + Expect(newNics[0].NetworkId).To(Equal("123")) + Expect(newNics[1].NetworkId).To(Equal("69")) + Expect(newNics[2].NetworkId).To(Equal("69")) + }) + }) }) diff --git a/pkg/ccm/stackit.go b/pkg/ccm/stackit.go index 950bd933..10bc5a5f 100644 --- a/pkg/ccm/stackit.go +++ b/pkg/ccm/stackit.go @@ -151,7 +151,7 @@ func NewCloudControllerManager(cfg *stackitconfig.CCMConfig, obs *MetricsRemoteW return nil, fmt.Errorf("failed to create IaaS client: %v", err) } - instances, err := NewInstance(iaasClient, cfg.Global.Region) + instances, err := NewInstance(iaasClient, cfg.Global.Region, cfg.Instance) if err != nil { return nil, err } diff --git a/pkg/stackit/config/config.go b/pkg/stackit/config/config.go index cad4a805..4a88b5d1 100644 --- a/pkg/stackit/config/config.go +++ b/pkg/stackit/config/config.go @@ -19,6 +19,14 @@ type CCMConfig struct { Global GlobalOpts `yaml:"global"` Metadata metadata.Opts `yaml:"metadata"` LoadBalancer LoadBalancerOpts `yaml:"loadBalancer"` + Instance InstanceOpts `yaml:"instance"` +} + +type InstanceOpts struct { + // DefaultNetwork contains the default network to use for a node. + // It can contain either the network name or ID. + // Can be used in mulit-network scenario to indicate which NIC is the primary one. + DefaultNetwork string `yaml:"defaultNetwork"` } type LoadBalancerOpts struct {