diff --git a/api/core/v1alpha1/evpninstance_types.go b/api/core/v1alpha1/evpninstance_types.go
index 6b58d6296..d38fb01b7 100644
--- a/api/core/v1alpha1/evpninstance_types.go
+++ b/api/core/v1alpha1/evpninstance_types.go
@@ -17,6 +17,8 @@ import (
// [RFC 8365]: https://datatracker.ietf.org/doc/html/rfc8365
//
// +kubebuilder:validation:XValidation:rule="self.type != 'Bridged' || has(self.vlanRef)",message="VLANRef must be specified when Type is Bridged"
+// +kubebuilder:validation:XValidation:rule="self.type != 'Routed' || has(self.vrfRef)",message="VRFRef must be specified when Type is Routed"
+// +kubebuilder:validation:XValidation:rule="self.type != 'Routed' || !has(self.routeDistinguisher)",message="RouteDistinguisher must not be set when Type is Routed"
type EVPNInstanceSpec struct {
// DeviceName is the name of the Device this object belongs to. The Device object must exist in the same namespace.
// Immutable.
@@ -50,7 +52,11 @@ type EVPNInstanceSpec struct {
MulticastGroupAddress string `json:"multicastGroupAddress,omitempty"`
// RouteDistinguisher is the route distinguisher for the EVI.
+ // This field is only applicable when Type is Bridged (MAC-VRF).
+ // For Routed type, the route distinguisher is configured on the referenced VRF instead.
+ // Set to "Auto" for automatic derivation (equivalent to "rd auto").
// Formats supported:
+ // - "Auto" (automatic derivation)
// - Type 0: ASN(0-65535):Number(0-4294967295)
// - Type 1: IPv4:Number(0-65535)
// - Type 2: ASN(65536-4294967295):Number(0-65535)
@@ -71,6 +77,14 @@ type EVPNInstanceSpec struct {
// +optional
// +kubebuilder:validation:XValidation:rule="self.name == oldSelf.name",message="VLANRef is immutable"
VLANRef *LocalObjectReference `json:"vlanRef,omitempty"`
+
+ // VRFRef is a reference to a VRF resource for which this EVPNInstance provides the L3VNI.
+ // This field is only applicable when Type is Routed (L3VNI).
+ // The VRF resource must exist in the same namespace.
+ // Immutable.
+ // +optional
+ // +kubebuilder:validation:XValidation:rule="self.name == oldSelf.name",message="VRFRef is immutable"
+ VRFRef *LocalObjectReference `json:"vrfRef,omitempty"`
}
// EVPNInstanceType defines the type of EVPN instance.
diff --git a/api/core/v1alpha1/vrf_types.go b/api/core/v1alpha1/vrf_types.go
index d4b2452f3..af9d125f3 100644
--- a/api/core/v1alpha1/vrf_types.go
+++ b/api/core/v1alpha1/vrf_types.go
@@ -11,6 +11,9 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
)
+// RouteDistinguisherAuto is the special value for automatic RD derivation (equivalent to "rd auto").
+const RouteDistinguisherAuto = "Auto"
+
// VRFSpec defines the desired state of VRF
type VRFSpec struct {
// DeviceName is the name of the Device this object belongs to. The Device object must exist in the same namespace.
@@ -39,13 +42,17 @@ type VRFSpec struct {
Description string `json:"description,omitempty"`
// VNI is the VXLAN Network Identifier for the VRF (always an L3).
+ //
+ // Deprecated: Use the VNI field on the EVPNInstance resource instead. This field will be removed in a future release.
// +optional
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=16777215
VNI uint32 `json:"vni,omitempty"`
// RouteDistinguisher is the route distinguisher for the VRF.
+ // Set to "Auto" for automatic derivation (equivalent to "rd auto").
// Formats supported:
+ // - "Auto" (automatic derivation)
// - Type 0: ASN(0-65535):Number(0-4294967295)
// - Type 1: IPv4:Number(0-65535)
// - Type 2: ASN(65536-4294967295):Number(0-65535)
diff --git a/api/core/v1alpha1/zz_generated.deepcopy.go b/api/core/v1alpha1/zz_generated.deepcopy.go
index 1fd3266f2..11dfde574 100644
--- a/api/core/v1alpha1/zz_generated.deepcopy.go
+++ b/api/core/v1alpha1/zz_generated.deepcopy.go
@@ -1770,6 +1770,11 @@ func (in *EVPNInstanceSpec) DeepCopyInto(out *EVPNInstanceSpec) {
*out = new(LocalObjectReference)
**out = **in
}
+ if in.VRFRef != nil {
+ in, out := &in.VRFRef, &out.VRFRef
+ *out = new(LocalObjectReference)
+ **out = **in
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EVPNInstanceSpec.
diff --git a/charts/network-operator/templates/crd/evpninstances.networking.metal.ironcore.dev.yaml b/charts/network-operator/templates/crd/evpninstances.networking.metal.ironcore.dev.yaml
index 5d9de7d1d..519d3d2d4 100644
--- a/charts/network-operator/templates/crd/evpninstances.networking.metal.ironcore.dev.yaml
+++ b/charts/network-operator/templates/crd/evpninstances.networking.metal.ironcore.dev.yaml
@@ -133,7 +133,11 @@ spec:
routeDistinguisher:
description: |-
RouteDistinguisher is the route distinguisher for the EVI.
+ This field is only applicable when Type is Bridged (MAC-VRF).
+ For Routed type, the route distinguisher is configured on the referenced VRF instead.
+ Set to "Auto" for automatic derivation (equivalent to "rd auto").
Formats supported:
+ - "Auto" (automatic derivation)
- Type 0: ASN(0-65535):Number(0-4294967295)
- Type 1: IPv4:Number(0-65535)
- Type 2: ASN(65536-4294967295):Number(0-65535)
@@ -207,6 +211,27 @@ spec:
x-kubernetes-validations:
- message: VNI is immutable
rule: self == oldSelf
+ vrfRef:
+ description: |-
+ VRFRef is a reference to a VRF resource for which this EVPNInstance provides the L3VNI.
+ This field is only applicable when Type is Routed (L3VNI).
+ The VRF resource must exist in the same namespace.
+ Immutable.
+ properties:
+ name:
+ description: |-
+ Name of the referent.
+ More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+ maxLength: 63
+ minLength: 1
+ type: string
+ required:
+ - name
+ type: object
+ x-kubernetes-map-type: atomic
+ x-kubernetes-validations:
+ - message: VRFRef is immutable
+ rule: self.name == oldSelf.name
required:
- deviceRef
- type
@@ -215,6 +240,10 @@ spec:
x-kubernetes-validations:
- message: VLANRef must be specified when Type is Bridged
rule: self.type != 'Bridged' || has(self.vlanRef)
+ - message: VRFRef must be specified when Type is Routed
+ rule: self.type != 'Routed' || has(self.vrfRef)
+ - message: RouteDistinguisher must not be set when Type is Routed
+ rule: self.type != 'Routed' || !has(self.routeDistinguisher)
status:
description: |-
Status of the resource. This is set and updated automatically.
diff --git a/charts/network-operator/templates/crd/vrfs.networking.metal.ironcore.dev.yaml b/charts/network-operator/templates/crd/vrfs.networking.metal.ironcore.dev.yaml
index 68b6c6b49..c377a8eb3 100644
--- a/charts/network-operator/templates/crd/vrfs.networking.metal.ironcore.dev.yaml
+++ b/charts/network-operator/templates/crd/vrfs.networking.metal.ironcore.dev.yaml
@@ -137,7 +137,9 @@ spec:
routeDistinguisher:
description: |-
RouteDistinguisher is the route distinguisher for the VRF.
+ Set to "Auto" for automatic derivation (equivalent to "rd auto").
Formats supported:
+ - "Auto" (automatic derivation)
- Type 0: ASN(0-65535):Number(0-4294967295)
- Type 1: IPv4:Number(0-65535)
- Type 2: ASN(65536-4294967295):Number(0-65535)
@@ -186,8 +188,10 @@ spec:
- value
x-kubernetes-list-type: map
vni:
- description: VNI is the VXLAN Network Identifier for the VRF (always
- an L3).
+ description: |-
+ VNI is the VXLAN Network Identifier for the VRF (always an L3).
+
+ Deprecated: Use the VNI field on the EVPNInstance resource instead. This field will be removed in a future release.
format: int32
maximum: 16777215
minimum: 1
diff --git a/config/crd/bases/networking.metal.ironcore.dev_evpninstances.yaml b/config/crd/bases/networking.metal.ironcore.dev_evpninstances.yaml
index 4007ecd93..8ee981b5f 100644
--- a/config/crd/bases/networking.metal.ironcore.dev_evpninstances.yaml
+++ b/config/crd/bases/networking.metal.ironcore.dev_evpninstances.yaml
@@ -130,7 +130,11 @@ spec:
routeDistinguisher:
description: |-
RouteDistinguisher is the route distinguisher for the EVI.
+ This field is only applicable when Type is Bridged (MAC-VRF).
+ For Routed type, the route distinguisher is configured on the referenced VRF instead.
+ Set to "Auto" for automatic derivation (equivalent to "rd auto").
Formats supported:
+ - "Auto" (automatic derivation)
- Type 0: ASN(0-65535):Number(0-4294967295)
- Type 1: IPv4:Number(0-65535)
- Type 2: ASN(65536-4294967295):Number(0-65535)
@@ -204,6 +208,27 @@ spec:
x-kubernetes-validations:
- message: VNI is immutable
rule: self == oldSelf
+ vrfRef:
+ description: |-
+ VRFRef is a reference to a VRF resource for which this EVPNInstance provides the L3VNI.
+ This field is only applicable when Type is Routed (L3VNI).
+ The VRF resource must exist in the same namespace.
+ Immutable.
+ properties:
+ name:
+ description: |-
+ Name of the referent.
+ More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+ maxLength: 63
+ minLength: 1
+ type: string
+ required:
+ - name
+ type: object
+ x-kubernetes-map-type: atomic
+ x-kubernetes-validations:
+ - message: VRFRef is immutable
+ rule: self.name == oldSelf.name
required:
- deviceRef
- type
@@ -212,6 +237,10 @@ spec:
x-kubernetes-validations:
- message: VLANRef must be specified when Type is Bridged
rule: self.type != 'Bridged' || has(self.vlanRef)
+ - message: VRFRef must be specified when Type is Routed
+ rule: self.type != 'Routed' || has(self.vrfRef)
+ - message: RouteDistinguisher must not be set when Type is Routed
+ rule: self.type != 'Routed' || !has(self.routeDistinguisher)
status:
description: |-
Status of the resource. This is set and updated automatically.
diff --git a/config/crd/bases/networking.metal.ironcore.dev_vrfs.yaml b/config/crd/bases/networking.metal.ironcore.dev_vrfs.yaml
index 6e8df1183..535cd7b04 100644
--- a/config/crd/bases/networking.metal.ironcore.dev_vrfs.yaml
+++ b/config/crd/bases/networking.metal.ironcore.dev_vrfs.yaml
@@ -134,7 +134,9 @@ spec:
routeDistinguisher:
description: |-
RouteDistinguisher is the route distinguisher for the VRF.
+ Set to "Auto" for automatic derivation (equivalent to "rd auto").
Formats supported:
+ - "Auto" (automatic derivation)
- Type 0: ASN(0-65535):Number(0-4294967295)
- Type 1: IPv4:Number(0-65535)
- Type 2: ASN(65536-4294967295):Number(0-65535)
@@ -183,8 +185,10 @@ spec:
- value
x-kubernetes-list-type: map
vni:
- description: VNI is the VXLAN Network Identifier for the VRF (always
- an L3).
+ description: |-
+ VNI is the VXLAN Network Identifier for the VRF (always an L3).
+
+ Deprecated: Use the VNI field on the EVPNInstance resource instead. This field will be removed in a future release.
format: int32
maximum: 16777215
minimum: 1
diff --git a/docs/api-reference/index.md b/docs/api-reference/index.md
index c66cf4eba..8aa6954ad 100644
--- a/docs/api-reference/index.md
+++ b/docs/api-reference/index.md
@@ -1412,9 +1412,10 @@ _Appears in:_
| `vni` _integer_ | VNI is the VXLAN Network Identifier.
Immutable. | | Maximum: 1.6777214e+07
Minimum: 1
Required: \{\}
|
| `type` _[EVPNInstanceType](#evpninstancetype)_ | Type specifies the EVPN instance type.
Immutable. | | Enum: [Bridged Routed]
Required: \{\}
|
| `multicastGroupAddress` _string_ | MulticastGroupAddress specifies the IPv4 multicast group address used for BUM (Broadcast, Unknown unicast, Multicast) traffic.
The address must be in the valid multicast range (224.0.0.0 - 239.255.255.255). | | Format: ipv4
Optional: \{\}
|
-| `routeDistinguisher` _string_ | RouteDistinguisher is the route distinguisher for the EVI.
Formats supported:
- Type 0: ASN(0-65535):Number(0-4294967295)
- Type 1: IPv4:Number(0-65535)
- Type 2: ASN(65536-4294967295):Number(0-65535) | | Optional: \{\}
|
+| `routeDistinguisher` _string_ | RouteDistinguisher is the route distinguisher for the EVI.
This field is only applicable when Type is Bridged (MAC-VRF).
For Routed type, the route distinguisher is configured on the referenced VRF instead.
Set to "Auto" for automatic derivation (equivalent to "rd auto").
Formats supported:
- "Auto" (automatic derivation)
- Type 0: ASN(0-65535):Number(0-4294967295)
- Type 1: IPv4:Number(0-65535)
- Type 2: ASN(65536-4294967295):Number(0-65535) | | Optional: \{\}
|
| `routeTargets` _[EVPNRouteTarget](#evpnroutetarget) array_ | RouteTargets is the list of route targets for the EVI. | | MinItems: 1
Optional: \{\}
|
| `vlanRef` _[LocalObjectReference](#localobjectreference)_ | VLANRef is a reference to a VLAN resource for which this EVPNInstance builds the MAC-VRF.
This field is only applicable when Type is Bridged (L2VNI).
The VLAN resource must exist in the same namespace.
Immutable. | | Optional: \{\}
|
+| `vrfRef` _[LocalObjectReference](#localobjectreference)_ | VRFRef is a reference to a VRF resource for which this EVPNInstance provides the L3VNI.
This field is only applicable when Type is Routed (L3VNI).
The VRF resource must exist in the same namespace.
Immutable. | | Optional: \{\}
|
#### EVPNInstanceStatus
@@ -3690,8 +3691,8 @@ _Appears in:_
| `providerConfigRef` _[TypedLocalObjectReference](#typedlocalobjectreference)_ | ProviderConfigRef is a reference to a resource holding the provider-specific configuration of this interface.
This reference is used to link the VRF to its provider-specific configuration. | | Optional: \{\}
|
| `name` _string_ | Name is the name of the VRF.
Immutable. | | MaxLength: 32
MinLength: 1
Required: \{\}
|
| `description` _string_ | Description provides a human-readable description of the VRF. | | MaxLength: 255
MinLength: 1
Optional: \{\}
|
-| `vni` _integer_ | VNI is the VXLAN Network Identifier for the VRF (always an L3). | | Maximum: 1.6777215e+07
Minimum: 1
Optional: \{\}
|
-| `routeDistinguisher` _string_ | RouteDistinguisher is the route distinguisher for the VRF.
Formats supported:
- Type 0: ASN(0-65535):Number(0-4294967295)
- Type 1: IPv4:Number(0-65535)
- Type 2: ASN(65536-4294967295):Number(0-65535)
Validation via admission webhook for the VRF type. | | Optional: \{\}
|
+| `vni` _integer_ | VNI is the VXLAN Network Identifier for the VRF (always an L3).
Deprecated: Use the VNI field on the EVPNInstance resource instead. This field will be removed in a future release. | | Maximum: 1.6777215e+07
Minimum: 1
Optional: \{\}
|
+| `routeDistinguisher` _string_ | RouteDistinguisher is the route distinguisher for the VRF.
Set to "Auto" for automatic derivation (equivalent to "rd auto").
Formats supported:
- "Auto" (automatic derivation)
- Type 0: ASN(0-65535):Number(0-4294967295)
- Type 1: IPv4:Number(0-65535)
- Type 2: ASN(65536-4294967295):Number(0-65535)
Validation via admission webhook for the VRF type. | | Optional: \{\}
|
| `routeTargets` _[RouteTarget](#routetarget) array_ | RouteTargets is the list of route targets for the VRF. | | Optional: \{\}
|
diff --git a/internal/controller/core/evpninstance_controller.go b/internal/controller/core/evpninstance_controller.go
index fa6be4b1a..0d69ef604 100644
--- a/internal/controller/core/evpninstance_controller.go
+++ b/internal/controller/core/evpninstance_controller.go
@@ -204,7 +204,10 @@ func (r *EVPNInstanceReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, nil
}
-var eviVlanRefKey = ".spec.vlanRef.name"
+var (
+ eviVlanRefKey = ".spec.vlanRef.name"
+ eviVrfRefKey = ".spec.vrfRef.name"
+)
// SetupWithManager sets up the controller with the Manager.
func (r *EVPNInstanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
@@ -228,6 +231,16 @@ func (r *EVPNInstanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.
return err
}
+ if err := mgr.GetFieldIndexer().IndexField(ctx, &v1alpha1.EVPNInstance{}, eviVrfRefKey, func(obj client.Object) []string {
+ evi := obj.(*v1alpha1.EVPNInstance)
+ if evi.Spec.VRFRef == nil {
+ return nil
+ }
+ return []string{evi.Spec.VRFRef.Name}
+ }); err != nil {
+ return err
+ }
+
if err := mgr.GetFieldIndexer().IndexField(ctx, &v1alpha1.EVPNInstance{}, v1alpha1.DeviceRefIndexKey, func(obj client.Object) []string {
o := obj.(*v1alpha1.EVPNInstance)
return []string{o.Spec.DeviceRef.Name}
@@ -266,6 +279,20 @@ func (r *EVPNInstanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.
},
}),
).
+ // Watches enqueues EVPNInstances for updates in referenced VRF resources.
+ // Only triggers on create and delete events since VRF names are immutable.
+ Watches(
+ &v1alpha1.VRF{},
+ handler.EnqueueRequestsFromMapFunc(r.vrfToEVPNInstance),
+ builder.WithPredicates(predicate.Funcs{
+ UpdateFunc: func(e event.UpdateEvent) bool {
+ return false
+ },
+ GenericFunc: func(e event.GenericEvent) bool {
+ return false
+ },
+ }),
+ ).
// Watches enqueues EVPNInstances for updates in referenced Device resources.
// Triggers on create, delete, and update events when the device's effective pause state changes.
Watches(
@@ -315,6 +342,15 @@ func (r *EVPNInstanceReconciler) reconcile(ctx context.Context, s *eviScope) (re
}
}
+ var vrf *v1alpha1.VRF
+ if s.EVPNInstance.Spec.Type == v1alpha1.EVPNInstanceTypeRouted && s.EVPNInstance.Spec.VRFRef != nil {
+ var err error
+ vrf, err = r.reconcileVRF(ctx, s)
+ if err != nil {
+ return err
+ }
+ }
+
if err := s.Provider.Connect(ctx, s.Connection); err != nil {
return fmt.Errorf("failed to connect to provider: %w", err)
}
@@ -329,6 +365,7 @@ func (r *EVPNInstanceReconciler) reconcile(ctx context.Context, s *eviScope) (re
EVPNInstance: s.EVPNInstance,
ProviderConfig: s.ProviderConfig,
VLAN: vlan,
+ VRF: vrf,
})
cond := conditions.FromError(err)
@@ -402,11 +439,60 @@ func (r *EVPNInstanceReconciler) reconcileVLAN(ctx context.Context, s *eviScope)
return vlan, nil
}
+// reconcileVRF ensures that the referenced VRF exists and belongs to the same device as the EVPNInstance.
+func (r *EVPNInstanceReconciler) reconcileVRF(ctx context.Context, s *eviScope) (*v1alpha1.VRF, error) {
+ key := client.ObjectKey{
+ Name: s.EVPNInstance.Spec.VRFRef.Name,
+ Namespace: s.EVPNInstance.Namespace,
+ }
+
+ vrf := new(v1alpha1.VRF)
+ if err := r.Get(ctx, key, vrf); err != nil {
+ if apierrors.IsNotFound(err) {
+ conditions.Set(s.EVPNInstance, metav1.Condition{
+ Type: v1alpha1.ReadyCondition,
+ Status: metav1.ConditionFalse,
+ Reason: v1alpha1.VRFNotFoundReason,
+ Message: fmt.Sprintf("referenced VRF %q not found", key),
+ })
+ return nil, reconcile.TerminalError(fmt.Errorf("referenced VRF %q not found", key))
+ }
+ return nil, fmt.Errorf("failed to get referenced VRF %q: %w", key, err)
+ }
+
+ if vrf.Spec.DeviceRef.Name != s.Device.Name {
+ conditions.Set(s.EVPNInstance, metav1.Condition{
+ Type: v1alpha1.ReadyCondition,
+ Status: metav1.ConditionFalse,
+ Reason: v1alpha1.CrossDeviceReferenceReason,
+ Message: fmt.Sprintf("referenced VRF %q does not belong to device %q", vrf.Name, s.Device.Name),
+ })
+ return nil, reconcile.TerminalError(fmt.Errorf("referenced VRF %q does not belong to device %q", vrf.Name, s.Device.Name))
+ }
+
+ return vrf, nil
+}
+
func (r *EVPNInstanceReconciler) finalize(ctx context.Context, s *eviScope) (reterr error) {
if err := r.finalizeVLAN(ctx, s); err != nil {
return err
}
+ var vrf *v1alpha1.VRF
+ if s.EVPNInstance.Spec.VRFRef != nil {
+ vrf = new(v1alpha1.VRF)
+ err := r.Get(ctx, client.ObjectKey{
+ Name: s.EVPNInstance.Spec.VRFRef.Name,
+ Namespace: s.EVPNInstance.Namespace,
+ }, vrf)
+ if client.IgnoreNotFound(err) != nil {
+ return fmt.Errorf("failed to get referenced VRF: %w", err)
+ }
+ if err != nil {
+ vrf = nil
+ }
+ }
+
if err := s.Provider.Connect(ctx, s.Connection); err != nil {
return fmt.Errorf("failed to connect to provider: %w", err)
}
@@ -419,6 +505,7 @@ func (r *EVPNInstanceReconciler) finalize(ctx context.Context, s *eviScope) (ret
return s.Provider.DeleteEVPNInstance(ctx, &provider.EVPNInstanceRequest{
EVPNInstance: s.EVPNInstance,
ProviderConfig: s.ProviderConfig,
+ VRF: vrf,
})
}
@@ -520,6 +607,39 @@ func (r *EVPNInstanceReconciler) vlanToEVPNInstance(ctx context.Context, obj cli
return requests
}
+// vrfToEVPNInstance is a [handler.MapFunc] to be used to enqueue requests for reconciliation
+// for an EVPNInstance when its referenced VRF changes.
+func (r *EVPNInstanceReconciler) vrfToEVPNInstance(ctx context.Context, obj client.Object) []ctrl.Request {
+ vrf, ok := obj.(*v1alpha1.VRF)
+ if !ok {
+ panic(fmt.Sprintf("Expected a VRF but got a %T", obj))
+ }
+
+ log := ctrl.LoggerFrom(ctx, "VRF", klog.KObj(vrf))
+
+ evpnInstances := new(v1alpha1.EVPNInstanceList)
+ if err := r.List(ctx, evpnInstances, client.InNamespace(vrf.Namespace), client.MatchingFields{eviVrfRefKey: vrf.Name}); err != nil {
+ log.Error(err, "Failed to list EVPNInstances")
+ return nil
+ }
+
+ requests := []ctrl.Request{}
+ for _, evi := range evpnInstances.Items {
+ if evi.Spec.VRFRef != nil && evi.Spec.VRFRef.Name == vrf.Name {
+ log.V(2).Info("Enqueuing EVPNInstance for reconciliation", "EVPNInstance", klog.KObj(&evi))
+
+ requests = append(requests, ctrl.Request{
+ NamespacedName: client.ObjectKey{
+ Name: evi.Name,
+ Namespace: evi.Namespace,
+ },
+ })
+ }
+ }
+
+ return requests
+}
+
// evpnInstancesForProviderConfig is a [handler.MapFunc] to be used to enqueue requests for reconciliation
// for a EVPNInstance to update when one of its referenced provider configurations gets updated.
func (r *EVPNInstanceReconciler) evpnInstancesForProviderConfig(ctx context.Context, obj client.Object) []reconcile.Request {
diff --git a/internal/provider/cisco/nxos/evi.go b/internal/provider/cisco/nxos/evi.go
index 5a693183c..95b03b3c7 100644
--- a/internal/provider/cisco/nxos/evi.go
+++ b/internal/provider/cisco/nxos/evi.go
@@ -11,6 +11,7 @@ import (
"strconv"
"strings"
+ "github.com/ironcore-dev/network-operator/api/core/v1alpha1"
"github.com/ironcore-dev/network-operator/internal/transport/gnmiext"
)
@@ -18,8 +19,8 @@ var _ gnmiext.DataElement = (*BDEVI)(nil)
// BDEVI represents a Bridge Domain Ethernet VPN Instance (MAC-VRF).
type BDEVI struct {
- Encap string `json:"encap"`
- Rd string `json:"rd"`
+ Encap string `json:"encap"`
+ Rd Option[string] `json:"rd"`
RttpItems struct {
RttPList gnmiext.List[RttEntryType, *RttEntry] `json:"RttP-list,omitzero"`
} `json:"rttp-items,omitzero"`
@@ -80,7 +81,7 @@ func stdcommunity(s string) (string, error) {
// extcommunity converts a value to an extended community string.
func extcommunity(s string) (string, error) {
- if s == "" {
+ if s == "" || s == v1alpha1.RouteDistinguisherAuto {
return "unknown:0:0", nil
}
parts := strings.SplitN(s, ":", 2)
diff --git a/internal/provider/cisco/nxos/evi_test.go b/internal/provider/cisco/nxos/evi_test.go
index 9b4841964..975e2dd12 100644
--- a/internal/provider/cisco/nxos/evi_test.go
+++ b/internal/provider/cisco/nxos/evi_test.go
@@ -10,8 +10,10 @@ func init() {
rt := &RttEntry{Type: RttEntryTypeExport}
rt.EntItems.RttEntryList.Set(rtt)
+ rd, _ := RouteDistinguisher("10.0.0.10:65000") //nolint:errcheck
+
evi := &BDEVI{Encap: "vxlan-100010"}
- evi.Rd, _ = RouteDistinguisher("10.0.0.10:65000") //nolint:errcheck
+ evi.Rd = NewOption(rd)
evi.RttpItems.RttPList.Set(rt)
Register("evi", evi)
}
diff --git a/internal/provider/cisco/nxos/provider.go b/internal/provider/cisco/nxos/provider.go
index 72ac8e3dd..4032a9358 100644
--- a/internal/provider/cisco/nxos/provider.go
+++ b/internal/provider/cisco/nxos/provider.go
@@ -770,9 +770,12 @@ func (p *Provider) EnsureEVPNInstance(ctx context.Context, req *provider.EVPNIns
case v1alpha1.EVPNInstanceTypeBridged:
evi := new(BDEVI)
evi.Encap = "vxlan-" + strconv.FormatInt(int64(req.EVPNInstance.Spec.VNI), 10)
- evi.Rd, err = RouteDistinguisher(req.EVPNInstance.Spec.RouteDistinguisher)
- if err != nil {
- return fmt.Errorf("evpn instance: invalid route distinguisher: %w", err)
+ if req.EVPNInstance.Spec.RouteDistinguisher != "" {
+ rd, err := RouteDistinguisher(req.EVPNInstance.Spec.RouteDistinguisher)
+ if err != nil {
+ return fmt.Errorf("evpn instance: invalid route distinguisher: %w", err)
+ }
+ evi.Rd = NewOption(rd)
}
imports := &RttEntry{Type: RttEntryTypeImport}
exports := &RttEntry{Type: RttEntryTypeExport}
@@ -810,10 +813,37 @@ func (p *Provider) EnsureEVPNInstance(ctx context.Context, req *provider.EVPNIns
vni.AssociateVrfFlag = true
}
- return p.Update(ctx, updates...)
+ if err := p.Update(ctx, updates...); err != nil {
+ return err
+ }
+
+ // Patch L3VNI/Encap on the VRF separately. This merges into the existing
+ // VRF tree without replacing fields managed by EnsureVRF.
+ if req.EVPNInstance.Spec.Type == v1alpha1.EVPNInstanceTypeRouted && req.VRF != nil {
+ vrf := new(VRFEncap)
+ vrf.Name = req.VRF.Spec.Name
+ vrf.L3Vni = true
+ vrf.Encap = NewOption("vxlan-" + strconv.FormatInt(int64(req.EVPNInstance.Spec.VNI), 10))
+ if err := p.Patch(ctx, vrf); err != nil {
+ return err
+ }
+ }
+
+ return nil
}
func (p *Provider) DeleteEVPNInstance(ctx context.Context, req *provider.EVPNInstanceRequest) error {
+ // Clear L3VNI/Encap on the VRF if this is a Routed EVI and the VRF still exists.
+ // If no VRF is passed in the request, assume it has already been deleted and skip this step.
+ if req.EVPNInstance.Spec.Type == v1alpha1.EVPNInstanceTypeRouted && req.VRF != nil {
+ vrf := new(VRFEncap)
+ vrf.Name = req.VRF.Spec.Name
+ vrf.L3Vni = false
+ if err := p.Patch(ctx, vrf); err != nil {
+ return err
+ }
+ }
+
deletes := make([]gnmiext.DataElement, 0, 3)
evi := new(BDEVI)
@@ -2521,30 +2551,28 @@ func (p *Provider) EnsureVRF(ctx context.Context, req *provider.VRFRequest) erro
if req.VRF.Spec.Description != "" {
v.Descr = NewOption(req.VRF.Spec.Description)
}
- if req.VRF.Spec.VNI > 0 {
- v.L3Vni = true
- v.Encap = NewOption("vxlan-" + strconv.FormatUint(uint64(req.VRF.Spec.VNI), 10))
- }
dom := new(VRFDom)
dom.Name = req.VRF.Spec.Name
- v.DomItems.DomList.Set(dom)
+ domItems := &VRFDomItems{Name: req.VRF.Spec.Name}
+ domItems.DomList.Set(dom)
- // pre: RD format has been already been validated by VRFCustomValidator
+ // RouteDistinguisher is already validated by VRFCustomValidator
if req.VRF.Spec.RouteDistinguisher != "" {
- tokens := strings.Split(req.VRF.Spec.RouteDistinguisher, ":")
- if strings.Contains(tokens[0], ".") {
- dom.Rd = "rd:ipv4-nn2:" + req.VRF.Spec.RouteDistinguisher
- } else {
- asn, err := strconv.ParseUint(tokens[0], 10, 32)
- if err != nil {
- return fmt.Errorf("invalid ASN in route distinguisher: %w", err)
- }
- dom.Rd = "rd:asn2-nn4:" + req.VRF.Spec.RouteDistinguisher
- if asn < math.MaxUint16 {
- dom.Rd = "rd:asn4-nn2:" + req.VRF.Spec.RouteDistinguisher
- }
+ f := new(Feature)
+ f.Name = "bgp"
+ if err := p.client.GetConfig(ctx, f); err != nil && !errors.Is(err, gnmiext.ErrNil) {
+ return err
+ }
+ if f.AdminSt != AdminStEnabled {
+ return apistatus.NewFailedPreconditionError("bgp feature must be enabled to configure route distinguisher")
}
+
+ rd, err := RouteDistinguisher(req.VRF.Spec.RouteDistinguisher)
+ if err != nil {
+ return fmt.Errorf("vrf: invalid route distinguisher: %w", err)
+ }
+ dom.Rd = NewOption(rd)
}
// configure route targets
@@ -2558,32 +2586,14 @@ func (p *Provider) EnsureVRF(ctx context.Context, req *provider.VRFRequest) erro
importEntryIPv6EVPN := &RttEntry{Type: RttEntryTypeImport}
exportEntryIPv6EVPN := &RttEntry{Type: RttEntryTypeExport}
- // route targets are already validated by VRFCustomValidator
+ // RouteTargets are already validated by VRFCustomValidator
for _, rt := range req.VRF.Spec.RouteTargets {
- rttValue := "route-target:"
- tokens := strings.Split(rt.Value, ":")
- if strings.Contains(tokens[0], ".") {
- rttValue += "ipv4-nn2:" + rt.Value
- } else {
- asn, err := strconv.ParseUint(tokens[0], 10, 32)
- if err != nil {
- return fmt.Errorf("invalid ASN in route target: %w", err)
- }
- if asn > math.MaxUint16 {
- rttValue += "as4-nn2:" + rt.Value
- } else {
- nn, err := strconv.ParseUint(tokens[1], 10, 32)
- if err != nil {
- return fmt.Errorf("invalid number in route target: %w", err)
- }
- rttValue += "as2-nn2:" + rt.Value
- if nn > math.MaxUint16 {
- rttValue += "as2-nn4:" + rt.Value
- }
- }
+ rttValue, err := RouteTarget(rt.Value)
+ if err != nil {
+ return fmt.Errorf("invalid route target: %w", err)
}
- rtt := Rtt{Rtt: rttValue}
+ rtt := Rtt{Rtt: rttValue}
for _, af := range rt.AddressFamilies {
switch af {
case v1alpha1.IPv4:
@@ -2621,7 +2631,6 @@ func (p *Provider) EnsureVRF(ctx context.Context, req *provider.VRFRequest) erro
}
if len(req.VRF.Spec.RouteTargets) > 0 {
- // Initialize address families
afIPv4 := &VRFDomAf{Type: AddressFamilyIPv4Unicast}
afIPv6 := &VRFDomAf{Type: AddressFamilyIPv6Unicast}
@@ -2629,7 +2638,6 @@ func (p *Provider) EnsureVRF(ctx context.Context, req *provider.VRFRequest) erro
if importE.EntItems.RttEntryList.Len() == 0 && exportE.EntItems.RttEntryList.Len() == 0 {
return
}
-
ctrl := &VRFDomAfCtrl{Type: afType}
if importE.EntItems.RttEntryList.Len() > 0 {
ctrl.RttpItems.RttPList.Set(importE)
@@ -2650,7 +2658,14 @@ func (p *Provider) EnsureVRF(ctx context.Context, req *provider.VRFRequest) erro
dom.AfItems.DomAfList.Set(afIPv6)
}
- return p.Update(ctx, v)
+ // Patch the VRF fields (name, description), merges into existing tree
+ // to preserve L3Vni/Encap set by EnsureEVPNInstance.
+ if err := p.Patch(ctx, v); err != nil {
+ return err
+ }
+
+ // Replace the VRF domain items (RD, route targets), fully owned by this controller.
+ return p.Update(ctx, domItems)
}
func (p *Provider) DeleteVRF(ctx context.Context, req *provider.VRFRequest) error {
diff --git a/internal/provider/cisco/nxos/testdata/vrf.json b/internal/provider/cisco/nxos/testdata/vrf.json
index 33438bfcd..98413e24b 100644
--- a/internal/provider/cisco/nxos/testdata/vrf.json
+++ b/internal/provider/cisco/nxos/testdata/vrf.json
@@ -2,95 +2,8 @@
"inst-items": {
"Inst-list": [
{
- "encap": "vxlan-101",
- "l3vni": true,
"name": "CC-CLOUD01",
- "descr": "CC-CLOUD01 VRF",
- "dom-items": {
- "Dom-list": [
- {
- "name": "CC-CLOUD01",
- "rd": "rd:as4-nn2:4269539332:101",
- "af-items": {
- "DomAf-list": [
- {
- "type": "ipv4-ucast",
- "ctrl-items": {
- "AfCtrl-list": [
- {
- "type": "l2vpn-evpn",
- "rttp-items": {
- "RttP-list": [
- {
- "type": "export",
- "ent-items": {
- "RttEntry-list": [
- {
- "rtt": "route-target:as2-nn2:65148:4101"
- },
- {
- "rtt": "route-target:as2-nn2:65148:1101"
- }
- ]
- }
- },
- {
- "type": "import",
- "ent-items": {
- "RttEntry-list": [
- {
- "rtt": "route-target:as2-nn2:65148:4101"
- },
- {
- "rtt": "route-target:as2-nn2:65148:1101"
- }
- ]
- }
- }
- ]
- }
- },
- {
- "type": "ipv4-ucast",
- "rttp-items": {
- "RttP-list": [
- {
- "type": "export",
- "ent-items": {
- "RttEntry-list": [
- {
- "rtt": "route-target:as2-nn2:65148:4101"
- },
- {
- "rtt": "route-target:as2-nn2:65148:1101"
- }
- ]
- }
- },
- {
- "type": "import",
- "ent-items": {
- "RttEntry-list": [
- {
- "rtt": "route-target:as2-nn2:65148:4101"
- },
- {
- "rtt": "route-target:as2-nn2:65148:1101"
- }
- ]
- }
- }
- ]
- }
- }
- ]
- }
- }
- ]
- }
- }
- ]
- }
+ "descr": "CC-CLOUD01 VRF"
}
]
}
diff --git a/internal/provider/cisco/nxos/testdata/vrf_dom.json b/internal/provider/cisco/nxos/testdata/vrf_dom.json
new file mode 100644
index 000000000..9e46f0fd4
--- /dev/null
+++ b/internal/provider/cisco/nxos/testdata/vrf_dom.json
@@ -0,0 +1,94 @@
+{
+ "inst-items": {
+ "Inst-list": [
+ {
+ "name": "CC-CLOUD01",
+ "dom-items": {
+ "Dom-list": [
+ {
+ "name": "CC-CLOUD01",
+ "rd": "rd:as4-nn2:4269539332:101",
+ "af-items": {
+ "DomAf-list": [
+ {
+ "type": "ipv4-ucast",
+ "ctrl-items": {
+ "AfCtrl-list": [
+ {
+ "type": "l2vpn-evpn",
+ "rttp-items": {
+ "RttP-list": [
+ {
+ "type": "export",
+ "ent-items": {
+ "RttEntry-list": [
+ {
+ "rtt": "route-target:as2-nn2:65148:4101"
+ },
+ {
+ "rtt": "route-target:as2-nn2:65148:1101"
+ }
+ ]
+ }
+ },
+ {
+ "type": "import",
+ "ent-items": {
+ "RttEntry-list": [
+ {
+ "rtt": "route-target:as2-nn2:65148:4101"
+ },
+ {
+ "rtt": "route-target:as2-nn2:65148:1101"
+ }
+ ]
+ }
+ }
+ ]
+ }
+ },
+ {
+ "type": "ipv4-ucast",
+ "rttp-items": {
+ "RttP-list": [
+ {
+ "type": "export",
+ "ent-items": {
+ "RttEntry-list": [
+ {
+ "rtt": "route-target:as2-nn2:65148:4101"
+ },
+ {
+ "rtt": "route-target:as2-nn2:65148:1101"
+ }
+ ]
+ }
+ },
+ {
+ "type": "import",
+ "ent-items": {
+ "RttEntry-list": [
+ {
+ "rtt": "route-target:as2-nn2:65148:4101"
+ },
+ {
+ "rtt": "route-target:as2-nn2:65148:1101"
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+ ]
+ }
+}
diff --git a/internal/provider/cisco/nxos/testdata/vrf_encap.json b/internal/provider/cisco/nxos/testdata/vrf_encap.json
new file mode 100644
index 000000000..bea632de0
--- /dev/null
+++ b/internal/provider/cisco/nxos/testdata/vrf_encap.json
@@ -0,0 +1,11 @@
+{
+ "inst-items": {
+ "Inst-list": [
+ {
+ "encap": "vxlan-101",
+ "l3vni": true,
+ "name": "CC-CLOUD01"
+ }
+ ]
+ }
+}
diff --git a/internal/provider/cisco/nxos/vrf.go b/internal/provider/cisco/nxos/vrf.go
index f34f5cab2..d51390460 100644
--- a/internal/provider/cisco/nxos/vrf.go
+++ b/internal/provider/cisco/nxos/vrf.go
@@ -11,14 +11,17 @@ const (
ManagementVRFName = "management"
)
-var _ gnmiext.DataElement = (*VRF)(nil)
+var (
+ _ gnmiext.DataElement = (*VRF)(nil)
+ _ gnmiext.DataElement = (*VRFEncap)(nil)
+ _ gnmiext.DataElement = (*VRFDomItems)(nil)
+)
+// VRF represents the VRF YANG container with name and description patched by [Provider.EnsureVRF].
+// It excludes L3Vni/Encap (patched by [Provider.EnsureEVPNInstance]) and DomItems (sent separately via Update).
type VRF struct {
- Encap Option[string] `json:"encap"`
- L3Vni bool `json:"l3vni"`
- Name string `json:"name"`
- Descr Option[string] `json:"descr"`
- DomItems VRFDomItems `json:"dom-items,omitzero"`
+ Name string `json:"name"`
+ Descr Option[string] `json:"descr"`
}
func (*VRF) IsListItem() {}
@@ -27,14 +30,36 @@ func (v *VRF) XPath() string {
return "System/inst-items/Inst-list[name=" + v.Name + "]"
}
+// VRFEncap represents the VRF YANG container with L3VNI and encapsulation fields,
+// used by [Provider.EnsureEVPNInstance] to patch L3VNI configuration on the VRF.
+type VRFEncap struct {
+ Encap Option[string] `json:"encap"`
+ L3Vni bool `json:"l3vni"`
+ Name string `json:"name"`
+}
+
+func (*VRFEncap) IsListItem() {}
+
+func (v *VRFEncap) XPath() string {
+ return "System/inst-items/Inst-list[name=" + v.Name + "]"
+}
+
type VRFDomItems struct {
+ Name string `json:"-"`
+
DomList gnmiext.List[string, *VRFDom] `json:"Dom-list,omitzero"`
}
+func (*VRFDomItems) IsListItem() {}
+
+func (d *VRFDomItems) XPath() string {
+ return "System/inst-items/Inst-list[name=" + d.Name + "]/dom-items"
+}
+
type VRFDom struct {
- Name string `json:"name"`
- Rd string `json:"rd,omitempty"`
- AfItems VRFDomAfItems `json:"af-items,omitzero"`
+ Name string `json:"name"`
+ Rd Option[string] `json:"rd"`
+ AfItems VRFDomAfItems `json:"af-items,omitzero"`
}
func (d *VRFDom) Key() string { return d.Name }
diff --git a/internal/provider/cisco/nxos/vrf_test.go b/internal/provider/cisco/nxos/vrf_test.go
index 7f5a6bf74..ebdeea625 100644
--- a/internal/provider/cisco/nxos/vrf_test.go
+++ b/internal/provider/cisco/nxos/vrf_test.go
@@ -42,14 +42,21 @@ func init() {
dom := new(VRFDom)
dom.Name = "CC-CLOUD01"
- dom.Rd = "rd:as4-nn2:4269539332:101"
+ dom.Rd = NewOption("rd:as4-nn2:4269539332:101")
dom.AfItems.DomAfList.Set(af)
vrf := new(VRF)
vrf.Name = "CC-CLOUD01"
- vrf.L3Vni = true
- vrf.Encap = NewOption("vxlan-101")
vrf.Descr = NewOption("CC-CLOUD01 VRF")
- vrf.DomItems.DomList.Set(dom)
Register("vrf", vrf)
+
+ vrfEncap := new(VRFEncap)
+ vrfEncap.Name = "CC-CLOUD01"
+ vrfEncap.L3Vni = true
+ vrfEncap.Encap = NewOption("vxlan-101")
+ Register("vrf_encap", vrfEncap)
+
+ domItems := &VRFDomItems{Name: "CC-CLOUD01"}
+ domItems.DomList.Set(dom)
+ Register("vrf_dom", domItems)
}
diff --git a/internal/provider/provider.go b/internal/provider/provider.go
index df9a60c74..4ba8ff86f 100644
--- a/internal/provider/provider.go
+++ b/internal/provider/provider.go
@@ -563,6 +563,7 @@ type EVPNInstanceRequest struct {
EVPNInstance *v1alpha1.EVPNInstance
ProviderConfig *ProviderConfig
VLAN *v1alpha1.VLAN
+ VRF *v1alpha1.VRF
}
// PrefixSetProvider is the interface for the realization of the PrefixSet objects over different providers.
diff --git a/internal/webhook/core/v1alpha1/vrf_webhook.go b/internal/webhook/core/v1alpha1/vrf_webhook.go
index d66c9f1aa..d6cd5f2ab 100644
--- a/internal/webhook/core/v1alpha1/vrf_webhook.go
+++ b/internal/webhook/core/v1alpha1/vrf_webhook.go
@@ -41,14 +41,24 @@ var _ admission.Validator[*v1alpha1.VRF] = &VRFCustomValidator{}
func (v *VRFCustomValidator) ValidateCreate(_ context.Context, vrf *v1alpha1.VRF) (admission.Warnings, error) {
vrflog.Info("Validation for VRF upon creation", "name", vrf.GetName())
- return nil, validateVRFSpec(vrf)
+ var warnings admission.Warnings
+ if vrf.Spec.VNI > 0 { //nolint:staticcheck // handling deprecated field for backward compatibility
+ warnings = append(warnings, "spec.vni is deprecated; use the vni field on the EVPNInstance resource instead")
+ }
+
+ return warnings, validateVRFSpec(vrf)
}
// ValidateUpdate implements admission.Validator so a webhook will be registered for the type VRF.
func (v *VRFCustomValidator) ValidateUpdate(_ context.Context, _, vrf *v1alpha1.VRF) (admission.Warnings, error) {
vrflog.Info("Validation for VRF upon update", "name", vrf.GetName())
- return nil, validateVRFSpec(vrf)
+ var warnings admission.Warnings
+ if vrf.Spec.VNI > 0 { //nolint:staticcheck // handling deprecated field for backward compatibility
+ warnings = append(warnings, "spec.vni is deprecated; use the vni field on the EVPNInstance resource instead")
+ }
+
+ return warnings, validateVRFSpec(vrf)
}
// ValidateDelete implements admission.Validator so a webhook will be registered for the type VRF.
@@ -60,7 +70,7 @@ func validateVRFSpec(vrf *v1alpha1.VRF) error {
var errAgg []error
rd := strings.TrimSpace(vrf.Spec.RouteDistinguisher)
- if rd != "" {
+ if rd != "" && rd != v1alpha1.RouteDistinguisherAuto {
if err := validateRouteDistinguisher(rd); err != nil {
errAgg = append(errAgg, fmt.Errorf("invalid route distinguisher value %q: %w", rd, err))
}
diff --git a/internal/webhook/core/v1alpha1/vrf_webhook_test.go b/internal/webhook/core/v1alpha1/vrf_webhook_test.go
index f7ebd1be8..d370be074 100644
--- a/internal/webhook/core/v1alpha1/vrf_webhook_test.go
+++ b/internal/webhook/core/v1alpha1/vrf_webhook_test.go
@@ -42,6 +42,12 @@ var _ = Describe("VRF Webhook", func() {
Expect(err).ToNot(HaveOccurred())
})
+ It("accepts Auto RD", func() {
+ obj.Spec.RouteDistinguisher = "Auto"
+ _, err := validator.ValidateCreate(ctx, obj)
+ Expect(err).ToNot(HaveOccurred())
+ })
+
It("rejects bad format (missing colon)", func() {
obj.Spec.RouteDistinguisher = "badformat"
_, err := validator.ValidateCreate(ctx, obj)
@@ -280,4 +286,27 @@ var _ = Describe("VRF Webhook", func() {
Expect(err).To(HaveOccurred())
})
})
+
+ Context("Deprecated VNI field", func() {
+ It("returns deprecation warning on create when VNI is set", func() {
+ obj.Spec.VNI = 100010 //nolint:staticcheck
+ warnings, err := validator.ValidateCreate(ctx, obj)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(warnings).To(ContainElement(ContainSubstring("spec.vni is deprecated")))
+ })
+
+ It("returns no warning on create when VNI is not set", func() {
+ warnings, err := validator.ValidateCreate(ctx, obj)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(warnings).To(BeEmpty())
+ })
+
+ It("returns deprecation warning on update when VNI is set", func() {
+ newObj := obj.DeepCopy()
+ newObj.Spec.VNI = 100010 //nolint:staticcheck
+ warnings, err := validator.ValidateUpdate(ctx, obj, newObj)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(warnings).To(ContainElement(ContainSubstring("spec.vni is deprecated")))
+ })
+ })
})