Add elastic IP annotation to front end NLB (#4330)

* feat(ingress): first draft of adding eip

* chore(annotation): spelling

* test(ingress): add eip tests

* feat(ingress): check that all members of ingress group have the same eip

* fix(ingress): count against chosen subnets

* fix(ingress): assign subnet if eip is not specified

* test(ingress): eip ne subnet

* [feat gw api] Add auth cognito action for secure listeners on ALBs

* feat: add ssl redirect to IngressClassParams

* chore(ingress): remove debug

* fix(ingress): works when no subnets are specified

* test(ingress): current tests pass

* wip(ingress): works but with the current bug where it will pick a private subnet if a public one is not spcified like in #2782

* wip(ingress): sketch out new func

* fix(ingress): first draft of subnet discovery rework

* refactor(ingress): abstract out annotation validation

* test(ingress): exisitng tests pass with subnet discovery

* test(ingress): 1st test passes non-conditional

* test(ingress): all tests except 1 pass without conditionals

* fix(ingress): count wrong way around in error message

* chore(ingress): remove debug

* fix(ingress): refactor validateAndResolveSubnets

* docs(ingress): add docs for new annotation

* Revert "fix(ingress): refactor validateAndResolveSubnets"

This reverts commit fb0db52.

* chore(ingress): remove extra comments

* chore(*): fix bad rebase

---------

Co-authored-by: shraddha bang <shrabang@amazon.com>
Co-authored-by: Michal Szewczyk <vomircom@gmail.com>

Author: 3 people

docs/guide/ingress/annotations.md

@@ -83,6 +83,7 @@ You can add annotations to kubernetes Ingress and Service objects to customize t
 | [alb.ingress.kubernetes.io/frontend-nlb-healthcheck-unhealthy-threshold-count](#frontend-nlb-healthcheck-unhealthy-threshold-count) | integer                     |3| Ingress | N/A           |
 | [alb.ingress.kubernetes.io/frontend-nlb-healthcheck-success-codes](#frontend-nlb-healthcheck-success-codes) | string                                     |200| Ingress | N/A           |
 | [alb.ingress.kubernetes.io/frontend-nlb-tags](#frontend-nlb-tags) | stringMap | N/A | Ingress | Exclusive |
+| [alb.ingress.kubernetes.io/frontend-nlb-eip-allocation](#frontend-nlb-eip-allocation) | stringList                                     |200| Ingress | N/A           |
 
 ## IngressGroup
 IngressGroup feature enables you to group multiple Ingress resources together.
@@ -1199,3 +1200,15 @@ When this option is set to true, the controller will automatically provision a N
         ```
         alb.ingress.kubernetes.io/frontend-nlb-tags: Environment=prod,Team=platform
         ```
+
+- <a name="frontend-nlb-eip-allocation">`alb.ingress.kubernetes.io/frontend-nlb-eip-allocation`</a> specifies a list of [elastic IP address](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html) configuration for an internet-facing NLB.
+
+    !!!note
+        - This configuration is optional, and you can use it to assign static IP addresses to your NLB
+        - If you include the subnets [annotation](#frontend-nlb-subnets) it must have the same number of subnets as this annotation has EIPs.
+        - NLB must be internet-facing
+
+    !!!example
+        ```
+        alb.ingress.kubernetes.io/frontend-nlb-eip-allocation: eipalloc-xyz, eipalloc-zzz
+        ```

pkg/annotations/constants.go

@@ -65,6 +65,7 @@ const (
 	IngressSuffixFrontendNlbSubnets                            = "frontend-nlb-subnets"
 	IngressSuffixFrontendNlbSecurityGroups                     = "frontend-nlb-security-groups"
 	IngressSuffixFrontendNlbListenerPortMapping                = "frontend-nlb-listener-port-mapping"
+	IngressSuffixFrontendNlbEipAlloactions                     = "frontend-nlb-eip-allocations"
 	IngressSuffixFrontendNlbHealthCheckPort                    = "frontend-nlb-healthcheck-port"
 	IngressSuffixFrontendNlbHealthCheckProtocol                = "frontend-nlb-healthcheck-protocol"
 	IngressSuffixFrontendNlbHealthCheckPath                    = "frontend-nlb-healthcheck-path"

pkg/ingress/model_build_frontend_nlb.go

@@ -105,45 +105,99 @@ func (t *defaultModelBuildTask) buildFrontendNlbScheme(ctx context.Context, alb
 	return t.defaultScheme, nil
 }
 
-func (t *defaultModelBuildTask) buildFrontendNlbSubnetMappings(ctx context.Context, scheme elbv2model.LoadBalancerScheme) ([]elbv2model.SubnetMapping, error) {
-	var explicitSubnetNameOrIDsList [][]string
-	for _, member := range t.ingGroup.Members {
-		var rawSubnetNameOrIDs []string
-		if exists := t.annotationParser.ParseStringSliceAnnotation(annotations.IngressSuffixFrontendNlbSubnets, &rawSubnetNameOrIDs, member.Ing.Annotations); !exists {
-			continue
-		}
-		explicitSubnetNameOrIDsList = append(explicitSubnetNameOrIDsList, rawSubnetNameOrIDs)
-	}
-
+func (t *defaultModelBuildTask) validateAndResolveSubnets(ctx context.Context, explicitSubnetNameOrIDsList [][]string, scheme elbv2model.LoadBalancerScheme) ([]ec2types.Subnet, error) {
 	if len(explicitSubnetNameOrIDsList) != 0 {
+		// Check all ingresses have the same subnets
 		chosenSubnetNameOrIDs := explicitSubnetNameOrIDsList[0]
 		for _, subnetNameOrIDs := range explicitSubnetNameOrIDsList[1:] {
 			if !cmp.Equal(chosenSubnetNameOrIDs, subnetNameOrIDs, equality.IgnoreStringSliceOrder()) {
 				return nil, errors.Errorf("conflicting subnets: %v | %v", chosenSubnetNameOrIDs, subnetNameOrIDs)
 			}
 		}
+
 		chosenSubnets, err := t.subnetsResolver.ResolveViaNameOrIDSlice(ctx, chosenSubnetNameOrIDs,
 			networking.WithSubnetsResolveLBType(elbv2model.LoadBalancerTypeNetwork),
 			networking.WithSubnetsResolveLBScheme(scheme),
 		)
 		if err != nil {
 			return nil, err
 		}
+		return chosenSubnets, nil
+	}
 
-		return buildFrontendNlbSubnetMappingsWithSubnets(chosenSubnets), nil
+	// If no explicit subnets, discover public or private subnets based on scheme
+	chosenSubnets, err := t.subnetsResolver.ResolveViaDiscovery(ctx,
+		networking.WithSubnetsResolveLBScheme(scheme),
+	)
+	if err != nil {
+		return nil, err
 	}
+	return chosenSubnets, nil
+}
 
-	return nil, nil
+func (t *defaultModelBuildTask) validateEIPAllocations(eipAllocationsList [][]string, scheme elbv2model.LoadBalancerScheme) ([]string, error) {
+	if len(eipAllocationsList) != 0 {
+		if scheme != elbv2model.LoadBalancerSchemeInternetFacing {
+			return nil, errors.Errorf("EIP allocations can only be set for internet facing load balancers")
+		}
+		chosenEipAllocations := eipAllocationsList[0]
+		for _, eipAllocations := range eipAllocationsList[1:] {
+			if !cmp.Equal(chosenEipAllocations, eipAllocations, equality.IgnoreStringSliceOrder()) {
+				return nil, errors.Errorf("all EIP allocations for the ingress group must be the same: %v | %v", chosenEipAllocations, eipAllocations)
+			}
+		}
+		return chosenEipAllocations, nil
+	}
+	return []string{}, nil
+}
 
+func (t *defaultModelBuildTask) buildFrontendNlbSubnetMappings(ctx context.Context, scheme elbv2model.LoadBalancerScheme) ([]elbv2model.SubnetMapping, error) {
+	var explicitSubnetNameOrIDsList [][]string
+	var eipAllocationsList [][]string
+	// Read annotations
+	for _, member := range t.ingGroup.Members {
+		var rawSubnetNameOrIDs []string
+		if exists := t.annotationParser.ParseStringSliceAnnotation(annotations.IngressSuffixFrontendNlbSubnets, &rawSubnetNameOrIDs, member.Ing.Annotations); exists {
+			explicitSubnetNameOrIDsList = append(explicitSubnetNameOrIDsList, rawSubnetNameOrIDs)
+		}
+		var rawEIP []string
+		if exists := t.annotationParser.ParseStringSliceAnnotation(annotations.IngressSuffixFrontendNlbEipAlloactions, &rawEIP, member.Ing.Annotations); exists {
+			eipAllocationsList = append(eipAllocationsList, rawEIP)
+		}
+	}
+
+	chosenSubnets, err := t.validateAndResolveSubnets(ctx, explicitSubnetNameOrIDsList, scheme)
+	if err != nil {
+		return nil, err
+	}
+
+	chosenEipAllocations, err := t.validateEIPAllocations(eipAllocationsList, scheme)
+	if err != nil {
+		return nil, err
+	}
+
+	// Construct subnet mapping
+	if len(chosenEipAllocations) != 0 {
+		if len(chosenEipAllocations) != len(chosenSubnets) {
+			return nil, errors.Errorf("count of EIP allocations (%d) and subnets (%d) must match", len(chosenEipAllocations), len(chosenSubnets))
+		}
+		return buildFrontendNlbSubnetMappingsWithSubnets(chosenSubnets, chosenEipAllocations), nil
+	}
+
+	return buildFrontendNlbSubnetMappingsWithSubnets(chosenSubnets, []string{}), nil
 }
 
-func buildFrontendNlbSubnetMappingsWithSubnets(subnets []ec2types.Subnet) []elbv2model.SubnetMapping {
+func buildFrontendNlbSubnetMappingsWithSubnets(subnets []ec2types.Subnet, eipAllocation []string) []elbv2model.SubnetMapping {
 	subnetMappings := make([]elbv2model.SubnetMapping, 0, len(subnets))
-	for _, subnet := range subnets {
+	for idx, subnet := range subnets {
 		subnetMappings = append(subnetMappings, elbv2model.SubnetMapping{
 			SubnetID: awssdk.ToString(subnet.SubnetId),
 		})
+		if len(eipAllocation) > 0 {
+			subnetMappings[idx].AllocationID = awssdk.String(eipAllocation[idx])
+		}
 	}
+
 	return subnetMappings
 }
 
@@ -174,11 +228,6 @@ func (t *defaultModelBuildTask) buildFrontendNlbSpec(ctx context.Context, scheme
 		return elbv2model.LoadBalancerSpec{}, err
 	}
 
-	// use alb subnetMappings if it is not explicitly specified
-	if subnetMappings == nil {
-		subnetMappings = alb.Spec.SubnetMappings
-	}
-
 	name, err := t.buildFrontendNlbName(ctx, scheme, alb)
 	if err != nil {
 		return elbv2model.LoadBalancerSpec{}, err