Skip to content

Commit 96a9bc8

Browse files
jmdealjmdeal
authored
chore: genericize launch mode evaulation for lts (#8158)
1 parent 014a6cd commit 96a9bc8

File tree

3 files changed

+198
-93
lines changed

3 files changed

+198
-93
lines changed

‎pkg/controllers/nodeclass/validation.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ func (v *Validation) validateCreateLaunchTemplateAuthorization(
205205
if err != nil {
206206
return "", false, fmt.Errorf("generating options, %w", err)
207207
}
208-
createLaunchTemplateInput := launchtemplate.GetCreateLaunchTemplateInput(ctx, opts, corev1.IPv4Protocol, "")
208+
createLaunchTemplateInput := launchtemplate.NewCreateLaunchTemplateInputBuilder(opts, corev1.IPv4Protocol, "").Build(ctx)
209209
createLaunchTemplateInput.DryRun = lo.ToPtr(true)
210210
// Adding NopRetryer to avoid aggressive retry when rate limited
211211
if _, err := v.ec2api.CreateLaunchTemplate(ctx, createLaunchTemplateInput, func(o *ec2.Options) {
@@ -237,7 +237,7 @@ func (v *Validation) validateRunInstancesAuthorization(
237237

238238
// We can directly marshal from CreateLaunchTemplate LaunchTemplate data
239239
runInstancesInput := &ec2.RunInstancesInput{}
240-
raw, err := json.Marshal(launchtemplate.GetCreateLaunchTemplateInput(ctx, opts, corev1.IPv4Protocol, "").LaunchTemplateData)
240+
raw, err := json.Marshal(launchtemplate.NewCreateLaunchTemplateInputBuilder(opts, corev1.IPv4Protocol, "").Build(ctx).LaunchTemplateData)
241241
if err != nil {
242242
return "", false, fmt.Errorf("converting launch template input to run instances input, %w", err)
243243
}

‎pkg/providers/launchtemplate/launchtemplate.go‎

Lines changed: 33 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"sync/atomic"
2525
"time"
2626

27+
"github.com/awslabs/operatorpkg/option"
2728
"github.com/awslabs/operatorpkg/serrors"
2829
"go.uber.org/multierr"
2930
"sigs.k8s.io/controller-runtime/pkg/log"
@@ -40,8 +41,6 @@ import (
4041
"k8s.io/apimachinery/pkg/api/resource"
4142
karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
4243

43-
karpoptions "sigs.k8s.io/karpenter/pkg/operator/options"
44-
4544
v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
4645
awserrors "github.com/aws/karpenter-provider-aws/pkg/errors"
4746
"github.com/aws/karpenter-provider-aws/pkg/operator/options"
@@ -56,23 +55,21 @@ import (
5655
sdk "github.com/aws/karpenter-provider-aws/pkg/aws"
5756
)
5857

59-
type Provider interface {
60-
EnsureAll(context.Context, *v1.EC2NodeClass, *karpv1.NodeClaim,
61-
[]*cloudprovider.InstanceType, string, map[string]string) ([]*LaunchTemplate, error)
62-
DeleteAll(context.Context, *v1.EC2NodeClass) error
63-
InvalidateCache(context.Context, string, string)
64-
ResolveClusterCIDR(context.Context) error
65-
CreateAMIOptions(context.Context, *v1.EC2NodeClass, map[string]string, map[string]string) (*amifamily.Options, error)
58+
type DefaultProviderOpts = option.Function[defaultProviderOpts]
59+
60+
type defaultProviderOpts struct {
61+
launchModeProvider LaunchModeProvider
6662
}
67-
type LaunchTemplate struct {
68-
Name string
69-
InstanceTypes []*cloudprovider.InstanceType
70-
ImageID string
71-
CapacityReservationID string
63+
64+
func WithLaunchModeProvider(provider LaunchModeProvider) DefaultProviderOpts {
65+
return func(opts *defaultProviderOpts) {
66+
opts.launchModeProvider = provider
67+
}
7268
}
7369

7470
type DefaultProvider struct {
7571
sync.Mutex
72+
LaunchModeProvider
7673
ec2api sdk.EC2API
7774
eksapi sdk.EKSAPI
7875
amiFamily amifamily.Resolver
@@ -87,10 +84,26 @@ type DefaultProvider struct {
8784
ClusterIPFamily corev1.IPFamily
8885
}
8986

90-
func NewDefaultProvider(ctx context.Context, cache *cache.Cache, ec2api sdk.EC2API, eksapi sdk.EKSAPI, amiFamily amifamily.Resolver,
91-
securityGroupProvider securitygroup.Provider, subnetProvider subnet.Provider,
92-
caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *DefaultProvider {
87+
func NewDefaultProvider(
88+
ctx context.Context,
89+
cache *cache.Cache,
90+
ec2api sdk.EC2API,
91+
eksapi sdk.EKSAPI,
92+
amiFamily amifamily.Resolver,
93+
securityGroupProvider securitygroup.Provider,
94+
subnetProvider subnet.Provider,
95+
caBundle *string,
96+
startAsync <-chan struct{},
97+
kubeDNSIP net.IP,
98+
clusterEndpoint string,
99+
opts ...DefaultProviderOpts,
100+
) *DefaultProvider {
101+
resolvedOpts := option.Resolve(opts...)
102+
if resolvedOpts.launchModeProvider == nil {
103+
resolvedOpts.launchModeProvider = defaultLaunchModeProvider{}
104+
}
93105
l := &DefaultProvider{
106+
LaunchModeProvider: resolvedOpts.launchModeProvider,
94107
ec2api: ec2api,
95108
eksapi: eksapi,
96109
amiFamily: amiFamily,
@@ -165,9 +178,11 @@ func (p *DefaultProvider) InvalidateCache(ctx context.Context, ltName string, lt
165178
log.FromContext(ctx).V(1).Info("invalidating launch template in the cache because it no longer exists")
166179
p.cache.Delete(ltName)
167180
}
181+
168182
func LaunchTemplateName(options *amifamily.LaunchTemplate) string {
169183
return fmt.Sprintf("%s/%d", v1.LaunchTemplateNamePrefix, lo.Must(hashstructure.Hash(options, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true})))
170184
}
185+
171186
func (p *DefaultProvider) CreateAMIOptions(ctx context.Context, nodeClass *v1.EC2NodeClass, labels, tags map[string]string) (*amifamily.Options, error) {
172187
// Remove any labels passed into userData that are prefixed with "node-restriction.kubernetes.io" or "kops.k8s.io" since the kubelet can't
173188
// register the node with any labels from this domain: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#noderestriction
@@ -242,7 +257,7 @@ func (p *DefaultProvider) createLaunchTemplate(ctx context.Context, options *ami
242257
if err != nil {
243258
return ec2types.LaunchTemplate{}, err
244259
}
245-
createLaunchTemplateInput := GetCreateLaunchTemplateInput(ctx, options, p.ClusterIPFamily, userData)
260+
createLaunchTemplateInput := NewCreateLaunchTemplateInputBuilder(options, p.ClusterIPFamily, userData).WithLaunchModeProvider(p).Build(ctx)
246261
output, err := p.ec2api.CreateLaunchTemplate(ctx, createLaunchTemplateInput)
247262
if err != nil {
248263
return ec2types.LaunchTemplate{}, err
@@ -251,79 +266,6 @@ func (p *DefaultProvider) createLaunchTemplate(ctx context.Context, options *ami
251266
return lo.FromPtr(output.LaunchTemplate), nil
252267
}
253268

254-
// you need UserData, AmiID, tags, blockdevicemappings, instance profile,
255-
func GetCreateLaunchTemplateInput(
256-
ctx context.Context,
257-
options *amifamily.LaunchTemplate,
258-
ClusterIPFamily corev1.IPFamily,
259-
userData string,
260-
) *ec2.CreateLaunchTemplateInput {
261-
launchTemplateDataTags := []ec2types.LaunchTemplateTagSpecificationRequest{
262-
{ResourceType: ec2types.ResourceTypeNetworkInterface, Tags: utils.EC2MergeTags(options.Tags)},
263-
}
264-
if options.CapacityType == karpv1.CapacityTypeSpot {
265-
launchTemplateDataTags = append(launchTemplateDataTags, ec2types.LaunchTemplateTagSpecificationRequest{ResourceType: ec2types.ResourceTypeSpotInstancesRequest, Tags: utils.EC2MergeTags(options.Tags)})
266-
}
267-
networkInterfaces := generateNetworkInterfaces(options, ClusterIPFamily)
268-
lt := &ec2.CreateLaunchTemplateInput{
269-
LaunchTemplateName: aws.String(LaunchTemplateName(options)),
270-
LaunchTemplateData: &ec2types.RequestLaunchTemplateData{
271-
BlockDeviceMappings: blockDeviceMappings(options.BlockDeviceMappings),
272-
IamInstanceProfile: &ec2types.LaunchTemplateIamInstanceProfileSpecificationRequest{
273-
Name: aws.String(options.InstanceProfile),
274-
},
275-
Monitoring: &ec2types.LaunchTemplatesMonitoringRequest{
276-
Enabled: aws.Bool(options.DetailedMonitoring),
277-
},
278-
// If the network interface is defined, the security groups are defined within it
279-
SecurityGroupIds: lo.Ternary(networkInterfaces != nil, nil, lo.Map(options.SecurityGroups, func(s v1.SecurityGroup, _ int) string { return s.ID })),
280-
UserData: aws.String(userData),
281-
ImageId: aws.String(options.AMIID),
282-
MetadataOptions: &ec2types.LaunchTemplateInstanceMetadataOptionsRequest{
283-
HttpEndpoint: ec2types.LaunchTemplateInstanceMetadataEndpointState(lo.FromPtr(options.MetadataOptions.HTTPEndpoint)),
284-
HttpProtocolIpv6: ec2types.LaunchTemplateInstanceMetadataProtocolIpv6(lo.FromPtr(options.MetadataOptions.HTTPProtocolIPv6)),
285-
//Will be removed when we update options.MetadataOptions.HTTPPutResponseHopLimit type to be int32
286-
//nolint: gosec
287-
HttpPutResponseHopLimit: lo.ToPtr(int32(lo.FromPtr(options.MetadataOptions.HTTPPutResponseHopLimit))),
288-
HttpTokens: ec2types.LaunchTemplateHttpTokensState(lo.FromPtr(options.MetadataOptions.HTTPTokens)),
289-
// We statically set the InstanceMetadataTags to "disabled" for all new instances since
290-
// account-wide defaults can override instance defaults on metadata settings
291-
// This can cause instance failure on accounts that default to instance tags since Karpenter
292-
// can't support instance tags with its current tags (e.g. kubernetes.io/cluster/*, karpenter.k8s.aws/ec2nodeclass)
293-
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-options.html#instance-metadata-options-order-of-precedence
294-
InstanceMetadataTags: ec2types.LaunchTemplateInstanceMetadataTagsStateDisabled,
295-
},
296-
NetworkInterfaces: networkInterfaces,
297-
TagSpecifications: launchTemplateDataTags,
298-
},
299-
TagSpecifications: []ec2types.TagSpecification{
300-
{
301-
ResourceType: ec2types.ResourceTypeLaunchTemplate,
302-
Tags: utils.EC2MergeTags(options.Tags),
303-
},
304-
},
305-
}
306-
// Gate this specifically since the update to CapacityReservationPreference will opt od / spot launches out of open
307-
// ODCRs, which is a breaking change from the pre-native ODCR support behavior.
308-
if karpoptions.FromContext(ctx).FeatureGates.ReservedCapacity {
309-
lt.LaunchTemplateData.CapacityReservationSpecification = &ec2types.LaunchTemplateCapacityReservationSpecificationRequest{
310-
CapacityReservationPreference: lo.Ternary(
311-
options.CapacityType == karpv1.CapacityTypeReserved,
312-
ec2types.CapacityReservationPreferenceCapacityReservationsOnly,
313-
ec2types.CapacityReservationPreferenceNone,
314-
),
315-
CapacityReservationTarget: lo.Ternary(
316-
options.CapacityType == karpv1.CapacityTypeReserved,
317-
&ec2types.CapacityReservationTarget{
318-
CapacityReservationId: &options.CapacityReservationID,
319-
},
320-
nil,
321-
),
322-
}
323-
}
324-
return lt
325-
}
326-
327269
// generateNetworkInterfaces generates network interfaces for the launch template.
328270
func generateNetworkInterfaces(options *amifamily.LaunchTemplate, clusterIPFamily corev1.IPFamily) []ec2types.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest {
329271
if options.EFACount != 0 {

‎pkg/providers/launchtemplate/types.go‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
/*
2+
Licensed under the Apache License, Version 2.0 (the "License");
3+
you may not use this file except in compliance with the License.
4+
You may obtain a copy of the License at
5+
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
8+
Unless required by applicable law or agreed to in writing, software
9+
distributed under the License is distributed on an "AS IS" BASIS,
10+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
See the License for the specific language governing permissions and
12+
limitations under the License.
13+
*/
14+
15+
package launchtemplate
16+
17+
import (
18+
"context"
19+
20+
"github.com/aws/aws-sdk-go-v2/service/ec2"
21+
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
22+
"github.com/samber/lo"
23+
corev1 "k8s.io/api/core/v1"
24+
karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
25+
"sigs.k8s.io/karpenter/pkg/cloudprovider"
26+
"sigs.k8s.io/karpenter/pkg/operator/options"
27+
28+
v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
29+
"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily"
30+
"github.com/aws/karpenter-provider-aws/pkg/utils"
31+
)
32+
33+
type Provider interface {
34+
EnsureAll(context.Context, *v1.EC2NodeClass, *karpv1.NodeClaim,
35+
[]*cloudprovider.InstanceType, string, map[string]string) ([]*LaunchTemplate, error)
36+
DeleteAll(context.Context, *v1.EC2NodeClass) error
37+
InvalidateCache(context.Context, string, string)
38+
ResolveClusterCIDR(context.Context) error
39+
CreateAMIOptions(context.Context, *v1.EC2NodeClass, map[string]string, map[string]string) (*amifamily.Options, error)
40+
}
41+
42+
type LaunchTemplate struct {
43+
Name string
44+
InstanceTypes []*cloudprovider.InstanceType
45+
ImageID string
46+
CapacityReservationID string
47+
}
48+
49+
type LaunchMode int
50+
51+
const (
52+
LaunchModeOpen LaunchMode = iota
53+
LaunchModeTargeted
54+
)
55+
56+
type LaunchModeProvider interface {
57+
LaunchMode(context.Context) LaunchMode
58+
}
59+
60+
type defaultLaunchModeProvider struct{}
61+
62+
func (defaultLaunchModeProvider) LaunchMode(ctx context.Context) LaunchMode {
63+
if options.FromContext(ctx).FeatureGates.ReservedCapacity {
64+
return LaunchModeTargeted
65+
}
66+
return LaunchModeOpen
67+
}
68+
69+
type CreateLaunchTemplateInputBuilder struct {
70+
LaunchModeProvider
71+
options *amifamily.LaunchTemplate
72+
clusterIPFamily corev1.IPFamily
73+
userData string
74+
}
75+
76+
func NewCreateLaunchTemplateInputBuilder(
77+
options *amifamily.LaunchTemplate,
78+
clusterIPFamily corev1.IPFamily,
79+
userData string,
80+
) *CreateLaunchTemplateInputBuilder {
81+
return &CreateLaunchTemplateInputBuilder{
82+
LaunchModeProvider: defaultLaunchModeProvider{},
83+
options: options,
84+
clusterIPFamily: clusterIPFamily,
85+
userData: userData,
86+
}
87+
}
88+
89+
func (b *CreateLaunchTemplateInputBuilder) WithLaunchModeProvider(provider LaunchModeProvider) *CreateLaunchTemplateInputBuilder {
90+
b.LaunchModeProvider = provider
91+
return b
92+
}
93+
94+
func (b *CreateLaunchTemplateInputBuilder) Build(ctx context.Context) *ec2.CreateLaunchTemplateInput {
95+
launchTemplateDataTags := []ec2types.LaunchTemplateTagSpecificationRequest{{
96+
ResourceType: ec2types.ResourceTypeNetworkInterface,
97+
Tags: utils.EC2MergeTags(b.options.Tags),
98+
}}
99+
if b.options.CapacityType == karpv1.CapacityTypeSpot {
100+
launchTemplateDataTags = append(launchTemplateDataTags, ec2types.LaunchTemplateTagSpecificationRequest{
101+
ResourceType: ec2types.ResourceTypeSpotInstancesRequest,
102+
Tags: utils.EC2MergeTags(b.options.Tags),
103+
})
104+
}
105+
networkInterfaces := generateNetworkInterfaces(b.options, b.clusterIPFamily)
106+
lt := &ec2.CreateLaunchTemplateInput{
107+
LaunchTemplateName: lo.ToPtr(LaunchTemplateName(b.options)),
108+
LaunchTemplateData: &ec2types.RequestLaunchTemplateData{
109+
BlockDeviceMappings: blockDeviceMappings(b.options.BlockDeviceMappings),
110+
IamInstanceProfile: &ec2types.LaunchTemplateIamInstanceProfileSpecificationRequest{
111+
Name: lo.ToPtr(b.options.InstanceProfile),
112+
},
113+
Monitoring: &ec2types.LaunchTemplatesMonitoringRequest{
114+
Enabled: lo.ToPtr(b.options.DetailedMonitoring),
115+
},
116+
// If the network interface is defined, the security groups are defined within it
117+
SecurityGroupIds: lo.Ternary(networkInterfaces != nil, nil, lo.Map(b.options.SecurityGroups, func(s v1.SecurityGroup, _ int) string { return s.ID })),
118+
UserData: lo.ToPtr(b.userData),
119+
ImageId: lo.ToPtr(b.options.AMIID),
120+
MetadataOptions: &ec2types.LaunchTemplateInstanceMetadataOptionsRequest{
121+
HttpEndpoint: ec2types.LaunchTemplateInstanceMetadataEndpointState(lo.FromPtr(b.options.MetadataOptions.HTTPEndpoint)),
122+
HttpProtocolIpv6: ec2types.LaunchTemplateInstanceMetadataProtocolIpv6(lo.FromPtr(b.options.MetadataOptions.HTTPProtocolIPv6)),
123+
//Will be removed when we update options.MetadataOptions.HTTPPutResponseHopLimit type to be int32
124+
//nolint: gosec
125+
HttpPutResponseHopLimit: lo.ToPtr(int32(lo.FromPtr(b.options.MetadataOptions.HTTPPutResponseHopLimit))),
126+
HttpTokens: ec2types.LaunchTemplateHttpTokensState(lo.FromPtr(b.options.MetadataOptions.HTTPTokens)),
127+
// We statically set the InstanceMetadataTags to "disabled" for all new instances since
128+
// account-wide defaults can override instance defaults on metadata settings
129+
// This can cause instance failure on accounts that default to instance tags since Karpenter
130+
// can't support instance tags with its current tags (e.g. kubernetes.io/cluster/*, karpenter.k8s.aws/ec2nodeclass)
131+
// See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-options.html#instance-metadata-options-order-of-precedence
132+
InstanceMetadataTags: ec2types.LaunchTemplateInstanceMetadataTagsStateDisabled,
133+
},
134+
NetworkInterfaces: networkInterfaces,
135+
TagSpecifications: launchTemplateDataTags,
136+
},
137+
TagSpecifications: []ec2types.TagSpecification{
138+
{
139+
ResourceType: ec2types.ResourceTypeLaunchTemplate,
140+
Tags: utils.EC2MergeTags(b.options.Tags),
141+
},
142+
},
143+
}
144+
// Gate this specifically since the update to CapacityReservationPreference will opt od / spot launches out of open
145+
// ODCRs, which is a breaking change from the pre-native ODCR support behavior.
146+
if b.LaunchMode(ctx) == LaunchModeTargeted {
147+
lt.LaunchTemplateData.CapacityReservationSpecification = &ec2types.LaunchTemplateCapacityReservationSpecificationRequest{
148+
CapacityReservationPreference: lo.Ternary(
149+
b.options.CapacityType == karpv1.CapacityTypeReserved,
150+
ec2types.CapacityReservationPreferenceCapacityReservationsOnly,
151+
ec2types.CapacityReservationPreferenceNone,
152+
),
153+
CapacityReservationTarget: lo.Ternary(
154+
b.options.CapacityType == karpv1.CapacityTypeReserved,
155+
&ec2types.CapacityReservationTarget{
156+
CapacityReservationId: &b.options.CapacityReservationID,
157+
},
158+
nil,
159+
),
160+
}
161+
}
162+
return lt
163+
}

0 commit comments

Comments
 (0)
⚡