feat: support network interface configuration

charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -6,7 +6,7 @@ metadata:
     {{- with .Values.additionalAnnotations }}
       {{- toYaml . | nindent 4 }}
     {{- end }}
-    controller-gen.kubebuilder.io/version: v0.20.0
+    controller-gen.kubebuilder.io/version: v0.20.1
   name: ec2nodeclasses.karpenter.k8s.aws
 spec:
   group: karpenter.k8s.aws
@@ -516,6 +516,36 @@ spec:
                         - optional
                       type: string
                   type: object
+                networkInterfaces:
+                  description: NetworkInterfaces specifies the network interface configurations to be attached to provisioned instances.
+                  items:
+                    description: |-
+                      NetworkInterface specifies the configuration for a network interface to be attached
+                      to provisioned instances.
+                    properties:
+                      deviceIndex:
+                        description: DeviceIndex is the device index for the network interface attachment.
+                        format: int32
+                        minimum: 0
+                        type: integer
+                      interfaceType:
+                        description: InterfaceType is the type of network interface. Valid values are "interface" and "efa-only".
+                        enum:
+                          - interface
+                          - efa-only
+                        type: string
+                      networkCardIndex:
+                        description: NetworkCardIndex is the index of the network card to attach the interface to.
+                        format: int32
+                        minimum: 0
+                        type: integer
+                    required:
+                      - deviceIndex
+                      - interfaceType
+                      - networkCardIndex
+                    type: object
+                  maxItems: 150
+                  type: array
                 role:
                   description: |-
                     Role is the AWS identity that nodes use.

kwok/ec2/ec2.go

@@ -222,6 +222,7 @@ func (c *Client) backupInstances(ctx context.Context) error {
 			}
 		}
 		stored := cm.DeepCopy()
+		//nolint:gosec
 		cm.Data = map[string]string{"instances": string(removeNullFields(lo.Must(json.Marshal(lo.Slice(instances, i*500, (i+1)*500)))))}
 		if !equality.Semantic.DeepEqual(cm, stored) {
 			if err := c.kubeClient.Patch(ctx, cm, client.MergeFrom(stored)); err != nil {
@@ -906,6 +907,7 @@ func (c *Client) toNode(ctx context.Context, instance ec2types.Instance) *corev1
 		nil,
 		nil,
 		nil,
+		nil,
 		// TODO: Eventually support different AMIFamilies from userData
 		"al2023",
 		nil,

pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.20.0
+    controller-gen.kubebuilder.io/version: v0.20.1
   name: ec2nodeclasses.karpenter.k8s.aws
 spec:
   group: karpenter.k8s.aws
@@ -513,6 +513,36 @@ spec:
                         - optional
                       type: string
                   type: object
+                networkInterfaces:
+                  description: NetworkInterfaces specifies the network interface configurations to be attached to provisioned instances.
+                  items:
+                    description: |-
+                      NetworkInterface specifies the configuration for a network interface to be attached
+                      to provisioned instances.
+                    properties:
+                      deviceIndex:
+                        description: DeviceIndex is the device index for the network interface attachment.
+                        format: int32
+                        minimum: 0
+                        type: integer
+                      interfaceType:
+                        description: InterfaceType is the type of network interface. Valid values are "interface" and "efa-only".
+                        enum:
+                          - interface
+                          - efa-only
+                        type: string
+                      networkCardIndex:
+                        description: NetworkCardIndex is the index of the network card to attach the interface to.
+                        format: int32
+                        minimum: 0
+                        type: integer
+                    required:
+                      - deviceIndex
+                      - interfaceType
+                      - networkCardIndex
+                    type: object
+                  maxItems: 150
+                  type: array
                 role:
                   description: |-
                     Role is the AWS identity that nodes use.

pkg/apis/v1/ec2nodeclass.go

@@ -19,6 +19,7 @@ import (
 	"log"
 	"strings"
 
+	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
 	"github.com/google/uuid"
 	"github.com/mitchellh/hashstructure/v2"
 	"github.com/samber/lo"
@@ -119,6 +120,10 @@ type EC2NodeClassSpec struct {
 	// InstanceStorePolicy specifies how to handle instance-store disks.
 	// +optional
 	InstanceStorePolicy *InstanceStorePolicy `json:"instanceStorePolicy,omitempty"`
+	// NetworkInterfaces specifies the network interface configurations to be attached to provisioned instances.
+	// +kubebuilder:validation:MaxItems:=150
+	// +optional
+	NetworkInterfaces []*NetworkInterface `json:"networkInterfaces,omitempty"`
 	// DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched
 	// +optional
 	DetailedMonitoring *bool `json:"detailedMonitoring,omitempty"`
@@ -458,6 +463,30 @@ const (
 	InstanceStorePolicyRAID0 InstanceStorePolicy = "RAID0"
 )
 
+// InterfaceType specifies the network interface type for a network interface.
+type InterfaceType string
+
+const (
+	// InterfaceTypeInterface indicates a standard Elastic Network Adapter (ENA) interface.
+	InterfaceTypeInterface = string(ec2types.NetworkInterfaceTypeInterface)
+	// InterfaceTypeEFAOnly indicates an Elastic Fabric Adapter only (EFA-only) interface for high-performance networking.
+	InterfaceTypeEFAOnly = string(ec2types.NetworkInterfaceTypeEfaOnly)
+)
+
+// NetworkInterface specifies the configuration for a network interface to be attached
+// to provisioned instances.
+type NetworkInterface struct {
+	// NetworkCardIndex is the index of the network card to attach the interface to.
+	// +kubebuilder:validation:Minimum:=0
+	NetworkCardIndex int32 `json:"networkCardIndex"`
+	// DeviceIndex is the device index for the network interface attachment.
+	// +kubebuilder:validation:Minimum:=0
+	DeviceIndex int32 `json:"deviceIndex"`
+	// InterfaceType is the type of network interface. Valid values are "interface" and "efa-only".
+	// +kubebuilder:validation:Enum:={interface,efa-only}
+	InterfaceType InterfaceType `json:"interfaceType"`
+}
+
 // EC2NodeClass is the Schema for the EC2NodeClass API
 // +kubebuilder:object:root=true
 // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description=""
@@ -531,6 +560,10 @@ func (in *EC2NodeClass) InstanceStorePolicy() *InstanceStorePolicy {
 	return in.Spec.InstanceStorePolicy
 }
 
+func (in *EC2NodeClass) NetworkInterfaces() []*NetworkInterface {
+	return in.Spec.NetworkInterfaces
+}
+
 func (in *EC2NodeClass) KubeletConfiguration() *KubeletConfiguration {
 	return in.Spec.Kubelet
 }

pkg/apis/v1/ec2nodeclass_validation_cel_test.go

@@ -1257,6 +1257,62 @@ var _ = Describe("CEL/Validation", func() {
 			Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed()))
 		})
 	})
+	Context("NetworkInterfaces", func() {
+		It("should succeed with valid multiple network interfaces", func() {
+			nc.Spec.NetworkInterfaces = []*v1.NetworkInterface{
+				{
+					NetworkCardIndex: 0,
+					DeviceIndex:      0,
+					InterfaceType:    v1.InterfaceType(v1.InterfaceTypeInterface),
+				},
+				{
+					NetworkCardIndex: 0,
+					DeviceIndex:      1,
+					InterfaceType:    v1.InterfaceType(v1.InterfaceTypeEFAOnly),
+				},
+				{
+					NetworkCardIndex: 1,
+					DeviceIndex:      0,
+					InterfaceType:    v1.InterfaceType(v1.InterfaceTypeInterface),
+				},
+			}
+			Expect(env.Client.Create(ctx, nc)).To(Succeed())
+		})
+		It("should succeed when network interfaces is empty", func() {
+			nc.Spec.NetworkInterfaces = []*v1.NetworkInterface{}
+			Expect(env.Client.Create(ctx, nc)).To(Succeed())
+		})
+		It("should fail with an invalid interface type", func() {
+			nc.Spec.NetworkInterfaces = []*v1.NetworkInterface{
+				{
+					NetworkCardIndex: 0,
+					DeviceIndex:      0,
+					InterfaceType:    "efa",
+				},
+			}
+			Expect(env.Client.Create(ctx, nc)).ToNot(Succeed())
+		})
+		It("should fail with a negative NetworkCardIndex", func() {
+			nc.Spec.NetworkInterfaces = []*v1.NetworkInterface{
+				{
+					NetworkCardIndex: -1,
+					DeviceIndex:      0,
+					InterfaceType:    v1.InterfaceType(v1.InterfaceTypeInterface),
+				},
+			}
+			Expect(env.Client.Create(ctx, nc)).ToNot(Succeed())
+		})
+		It("should fail with a negative DeviceIndex", func() {
+			nc.Spec.NetworkInterfaces = []*v1.NetworkInterface{
+				{
+					NetworkCardIndex: 0,
+					DeviceIndex:      -1,
+					InterfaceType:    v1.InterfaceType(v1.InterfaceTypeInterface),
+				},
+			}
+			Expect(env.Client.Create(ctx, nc)).ToNot(Succeed())
+		})
+	})
 	Context("Role Immutability", func() {
 		It("should fail if role is not defined", func() {
 			nc.Spec.Role = ""

pkg/apis/v1/labels.go

@@ -57,6 +57,7 @@ func init() {
 		LabelTopologyZoneID,
 		LabelInstanceTenancy,
 		corev1.LabelWindowsBuild,
+		LabelEFACount,
 	)
 
 	karpv1.WellKnownValuesForRequirements[LabelInstanceTenancy] = sets.New(string(ec2types.TenancyDedicated), string(ec2types.TenancyDefault))
@@ -161,6 +162,7 @@ var (
 	LabelInstanceAcceleratorCount             = apis.Group + "/instance-accelerator-count"
 	LabelNodeClass                            = apis.Group + "/ec2nodeclass"
 	LabelInstanceTenancy                      = apis.Group + "/instance-tenancy"
+	LabelEFACount                             = apis.Group + "/instance-efa-count"
 
 	LabelTopologyZoneID = "topology.k8s.aws/zone-id"
 

pkg/apis/v1/nodepool_validation_cel_test.go

@@ -57,7 +57,7 @@ var _ = Describe("CEL/Validation", func() {
 	Context("Requirements", func() {
 		It("should allow well known label exceptions", func() {
 			oldNodePool := nodePool.DeepCopy()
-			for label := range karpv1.WellKnownLabels.Difference(sets.New(karpv1.NodePoolLabelKey, karpv1.CapacityTypeLabelKey, v1.LabelInstanceTenancy)) {
+			for label := range karpv1.WellKnownLabels.Difference(sets.New(karpv1.NodePoolLabelKey, karpv1.CapacityTypeLabelKey, v1.LabelInstanceTenancy, v1.LabelEFACount)) {
 				nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{
 					{Key: label, Operator: corev1.NodeSelectorOpIn, Values: []string{"test"}},
 				}
@@ -145,7 +145,7 @@ var _ = Describe("CEL/Validation", func() {
 	Context("Labels", func() {
 		It("should allow well known label exceptions", func() {
 			oldNodePool := nodePool.DeepCopy()
-			for label := range karpv1.WellKnownLabels.Difference(sets.New(karpv1.NodePoolLabelKey)) {
+			for label := range karpv1.WellKnownLabels.Difference(sets.New(karpv1.NodePoolLabelKey, v1.LabelEFACount)) {
 				nodePool.Spec.Template.Labels = map[string]string{
 					label: "test",
 				}

pkg/cloudprovider/cloudprovider.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	stderrors "errors"
 	"fmt"
+	"strconv"
 	"time"
 
 	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
@@ -430,10 +431,10 @@ func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType *
 			if resources.IsZero(v) {
 				return false
 			}
-			// The nodeclaim should only advertise an EFA resource if it was requested. EFA network interfaces are only
-			// added to the launch template if they're requested, otherwise the instance is launched with a normal ENI.
+			// The nodeclaim should only advertise an EFA resource if it was requested by the pod or configured on the NodeClass. EFA network interfaces are
+			// added to the launch template if they're requested or NodeClass network interfaces are configured, otherwise the instance is launched with a normal ENI.
 			if n == v1.ResourceEFA {
-				return i.EFAEnabled
+				return i.EFACount > 0
 			}
 			return true
 		}
@@ -454,6 +455,10 @@ func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType *
 	}
 	labels[karpv1.CapacityTypeLabelKey] = i.CapacityType
 	labels[v1.LabelInstanceTenancy] = i.Tenancy
+	if i.EFACount > 0 {
+		labels[v1.LabelEFACount] = strconv.Itoa(i.EFACount)
+	}
+
 	if i.CapacityType == karpv1.CapacityTypeReserved {
 		labels[cloudprovider.ReservationIDLabel] = *i.CapacityReservationID
 		labels[v1.LabelCapacityReservationType] = string(*i.CapacityReservationType)

pkg/cloudprovider/suite_test.go

@@ -1420,6 +1420,56 @@ var _ = Describe("CloudProvider", func() {
 			Expect(err).To(BeNil())
 			Expect(lo.Keys(cloudProviderNodeClaim.Status.Allocatable)).ToNot(ContainElement(v1.ResourceEFA))
 		})
+		It("should include vpc.amazonaws.com/efa on a nodeclaim if nodeclass configured with EFA-only interfaces", func() {
+			nodeClass.Spec.NetworkInterfaces = []*v1.NetworkInterface{
+				{
+					NetworkCardIndex: 0,
+					DeviceIndex:      0,
+					InterfaceType:    v1.InterfaceType(v1.InterfaceTypeInterface),
+				},
+				{
+					NetworkCardIndex: 0,
+					DeviceIndex:      1,
+					InterfaceType:    v1.InterfaceType(v1.InterfaceTypeEFAOnly),
+				},
+			}
+			nodeClaim.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{
+				{
+					Key:      corev1.LabelInstanceTypeStable,
+					Operator: corev1.NodeSelectorOpIn,
+					Values:   []string{"dl1.24xlarge"},
+				},
+			}
+			ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim)
+			cloudProviderNodeClaim, err := cloudProvider.Create(ctx, nodeClaim)
+			Expect(err).To(BeNil())
+			Expect(lo.Keys(cloudProviderNodeClaim.Status.Allocatable)).To(ContainElement(v1.ResourceEFA))
+		})
+		It("should not include vpc.amazonaws.com/efa on a nodeclaim if nodeclass configured with only interface (ENA) network interfaces", func() {
+			nodeClass.Spec.NetworkInterfaces = []*v1.NetworkInterface{
+				{
+					NetworkCardIndex: 0,
+					DeviceIndex:      0,
+					InterfaceType:    v1.InterfaceType(v1.InterfaceTypeInterface),
+				},
+				{
+					NetworkCardIndex: 1,
+					DeviceIndex:      1,
+					InterfaceType:    v1.InterfaceType(v1.InterfaceTypeInterface),
+				},
+			}
+			nodeClaim.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{
+				{
+					Key:      corev1.LabelInstanceTypeStable,
+					Operator: corev1.NodeSelectorOpIn,
+					Values:   []string{"dl1.24xlarge"},
+				},
+			}
+			ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim)
+			cloudProviderNodeClaim, err := cloudProvider.Create(ctx, nodeClaim)
+			Expect(err).To(BeNil())
+			Expect(lo.Keys(cloudProviderNodeClaim.Status.Allocatable)).ToNot(ContainElement(v1.ResourceEFA))
+		})
 	})
 	Context("Capacity Reservations", func() {
 		const reservationCapacity = 10