diff --git a/cloud/linode/route_controller.go b/cloud/linode/route_controller.go index 3888e265..66de0592 100644 --- a/cloud/linode/route_controller.go +++ b/cloud/linode/route_controller.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "os" + "slices" "strconv" "strings" "sync" @@ -23,7 +24,47 @@ import ( ccmUtils "github.com/linode/linode-cloud-controller-manager/cloud/linode/utils" ) -var ipv6ConfiguredRoutes []*cloudprovider.Route +// ipv6RouteStore keeps the IPv6 routes that CreateRoute accepted without configuring anything, +// so that ListRoutes reports them back to the route controller. The route controller calls +// CreateRoute and DeleteRoute from concurrent goroutines, so access is guarded by a mutex. +type ipv6RouteStore struct { + mu sync.Mutex + routes []*cloudprovider.Route +} + +var ipv6ConfiguredRoutes = &ipv6RouteStore{} + +func sameRoute(a, b *cloudprovider.Route) bool { + return a.TargetNode == b.TargetNode && a.DestinationCIDR == b.DestinationCIDR +} + +// add stores route unless a route for the same node and CIDR is already stored. +func (s *ipv6RouteStore) add(route *cloudprovider.Route) { + s.mu.Lock() + defer s.mu.Unlock() + for _, r := range s.routes { + if sameRoute(r, route) { + return + } + } + s.routes = append(s.routes, route) +} + +// remove drops the stored routes for the same node and CIDR as route. +func (s *ipv6RouteStore) remove(route *cloudprovider.Route) { + s.mu.Lock() + defer s.mu.Unlock() + s.routes = slices.DeleteFunc(s.routes, func(r *cloudprovider.Route) bool { + return sameRoute(r, route) + }) +} + +// list returns a copy of the stored routes. +func (s *ipv6RouteStore) list() []*cloudprovider.Route { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.routes) +} type routeCache struct { Mu sync.RWMutex @@ -140,7 +181,7 @@ func (r *routes) CreateRoute(ctx context.Context, clusterName, nameHint string, } if ipAddr.To4() == nil { // "configure" the route so route controller doesn't keep creating it - ipv6ConfiguredRoutes = append(ipv6ConfiguredRoutes, route) + ipv6ConfiguredRoutes.add(route) return nil } @@ -198,6 +239,16 @@ func (r *routes) CreateRoute(ctx context.Context, clusterName, nameHint string, // DeleteRoute removes route's subnet from ip_ranges of target node's VPC interface func (r *routes) DeleteRoute(ctx context.Context, clusterName string, route *cloudprovider.Route) error { + ipAddr, _, err := net.ParseCIDR(route.DestinationCIDR) + if err != nil { + return err + } + if ipAddr.To4() == nil { + // IPv6 routes are never configured on the interface, only remembered by CreateRoute + ipv6ConfiguredRoutes.remove(route) + return nil + } + instance, err := r.getInstanceFromName(ctx, string(route.TargetNode)) if err != nil { return err @@ -326,7 +377,7 @@ func (r *routes) ListRoutes(ctx context.Context, clusterName string) ([]*cloudpr } // add in "configured" IPv6 routes so cloud-provider is happy - configuredRoutes = append(configuredRoutes, ipv6ConfiguredRoutes...) + configuredRoutes = append(configuredRoutes, ipv6ConfiguredRoutes.list()...) return configuredRoutes, nil } diff --git a/cloud/linode/route_controller_test.go b/cloud/linode/route_controller_test.go index 2d2b9321..ea2f78f3 100644 --- a/cloud/linode/route_controller_test.go +++ b/cloud/linode/route_controller_test.go @@ -1,8 +1,10 @@ package linode import ( + "fmt" "net" "strconv" + "sync" "testing" "github.com/golang/mock/gomock" @@ -663,3 +665,92 @@ func TestDeleteRoute(t *testing.T) { assert.NoError(t, err) }) } + +func TestIPv6RouteBookkeeping(t *testing.T) { + options.Options.VPCNames = []string{"dummy"} + services.VpcIDs["dummy"] = 1 + options.Options.EnableRouteController = true + + existingRoutes := ipv6ConfiguredRoutes + defer func() { + ipv6ConfiguredRoutes = existingRoutes + }() + + existingK8sCache := registeredK8sNodeCache + defer func() { + registeredK8sNodeCache = existingK8sCache + }() + registeredK8sNodeCache = newK8sNodeCache() + + newController := func(t *testing.T) cloudprovider.Routes { + t.Helper() + ctrl := gomock.NewController(t) + client := mocks.NewMockClient(ctrl) + client.EXPECT().ListInstances(gomock.Any(), gomock.Any()).AnyTimes().Return([]linodego.Instance{}, nil) + client.EXPECT().ListVPCIPAddresses(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return([]linodego.VPCIP{}, nil) + client.EXPECT().ListVPCIPv6Addresses(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return([]linodego.VPCIP{}, nil) + routeController, err := newRoutes(client, services.NewInstances(client)) + require.NoError(t, err) + return routeController + } + + t.Run("concurrent CreateRoute calls keep every IPv6 route", func(t *testing.T) { + ipv6ConfiguredRoutes = &ipv6RouteStore{} + routeController := newController(t) + + const nodes = 50 + var wg sync.WaitGroup + for i := range nodes { + wg.Add(1) + go func(i int) { + defer wg.Done() + route := &cloudprovider.Route{ + TargetNode: types.NodeName(fmt.Sprintf("node-%d", i)), + DestinationCIDR: fmt.Sprintf("fd00:0:0:%x::/64", i), + } + assert.NoError(t, routeController.CreateRoute(t.Context(), "dummy", "dummy", route)) + }(i) + } + wg.Wait() + + routes, err := routeController.ListRoutes(t.Context(), "dummy") + require.NoError(t, err) + assert.Len(t, routes, nodes) + }) + + t.Run("CreateRoute does not store the same IPv6 route twice", func(t *testing.T) { + ipv6ConfiguredRoutes = &ipv6RouteStore{} + routeController := newController(t) + route := &cloudprovider.Route{TargetNode: "node-a", DestinationCIDR: "fd00::/64"} + + require.NoError(t, routeController.CreateRoute(t.Context(), "dummy", "dummy", route)) + require.NoError(t, routeController.CreateRoute(t.Context(), "dummy", "dummy", route)) + + routes, err := routeController.ListRoutes(t.Context(), "dummy") + require.NoError(t, err) + assert.Len(t, routes, 1) + }) + + t.Run("DeleteRoute forgets an IPv6 route without looking up the node", func(t *testing.T) { + ipv6ConfiguredRoutes = &ipv6RouteStore{} + routeController := newController(t) + kept := &cloudprovider.Route{TargetNode: "node-a", DestinationCIDR: "fd00:0:0:1::/64"} + deleted := &cloudprovider.Route{TargetNode: "node-b", DestinationCIDR: "fd00:0:0:2::/64"} + + require.NoError(t, routeController.CreateRoute(t.Context(), "dummy", "dummy", kept)) + require.NoError(t, routeController.CreateRoute(t.Context(), "dummy", "dummy", deleted)) + + // node-b is gone from the cluster and from Linode, the delete must still succeed + require.NoError(t, routeController.DeleteRoute(t.Context(), "dummy", deleted)) + + routes, err := routeController.ListRoutes(t.Context(), "dummy") + require.NoError(t, err) + assert.Equal(t, []*cloudprovider.Route{kept}, routes) + }) + + t.Run("DeleteRoute returns an error for an invalid CIDR", func(t *testing.T) { + routeController := newController(t) + err := routeController.DeleteRoute(t.Context(), "dummy", &cloudprovider.Route{TargetNode: "node-a", DestinationCIDR: ":/64"}) + assert.Error(t, err) + }) +}