Skip to content

Commit dec349e

Browse files
authored
ci: add cleanup for leaked capacity reservations (#9051)
1 parent 943d9d4 commit dec349e

File tree

4 files changed

+185
-22
lines changed

4 files changed

+185
-22
lines changed

test/hack/resource/clean/main.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ func main() {
6767
// prevent deletion
6868
resourceTypes := []resourcetypes.Type{
6969
resourcetypes.NewInstance(ec2Client),
70+
resourcetypes.NewCapacityReservation(ec2Client),
7071
resourcetypes.NewVPCEndpoint(ec2Client),
7172
resourcetypes.NewENI(ec2Client),
7273
resourcetypes.NewSecurityGroup(ec2Client),

test/hack/resource/go.mod

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
module github.com/aws/karpenter-provider-aws/test/hack/resource
22

3-
go 1.23.0
3+
go 1.24
44

55
toolchain go1.24.0
66

77
require (
8-
github.com/aws/aws-sdk-go-v2 v1.26.1
8+
github.com/aws/aws-sdk-go-v2 v1.41.5
99
github.com/aws/aws-sdk-go-v2/config v1.27.11
1010
github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0
11-
github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0
11+
github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.2
1212
github.com/aws/aws-sdk-go-v2/service/iam v1.32.0
1313
github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.25.5
1414
github.com/samber/lo v1.47.0
@@ -20,16 +20,16 @@ require (
2020
require (
2121
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect
2222
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
23-
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
24-
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
23+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
24+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
2525
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
26-
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
26+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
2727
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6 // indirect
28-
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
28+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
2929
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
3030
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
3131
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
32-
github.com/aws/smithy-go v1.20.2 // indirect
32+
github.com/aws/smithy-go v1.24.2 // indirect
3333
github.com/go-logr/logr v1.4.1 // indirect
3434
github.com/gogo/protobuf v1.3.2 // indirect
3535
github.com/google/gofuzz v1.2.0 // indirect

test/hack/resource/go.sum

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,29 @@
1-
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
2-
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
1+
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
2+
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
33
github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
44
github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
55
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
66
github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
77
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
88
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
9-
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
10-
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
11-
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
12-
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
9+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
10+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
11+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
12+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
1313
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
1414
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
1515
github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 h1:Ap5tOJfeAH1hO2UQc3X3uMlwP7uryFeZXMvZCXIlLSE=
1616
github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0/go.mod h1:/v2KYdCW4BaHKayenaWEXOOdxItIwEA3oU0XzuQY3F0=
17-
github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 h1:ooy0OFbrdSwgk32OFGPnvBwry5ySYCKkgTEbQ2hejs8=
18-
github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0/go.mod h1:xejKuuRDjz6z5OqyeLsz01MlOqqW7CqpAB4PabNvpu8=
17+
github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.2 h1:Ytu50ChAxCiDsOlBcBq8jbczXy6+QLb07T65DBJASRs=
18+
github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.2/go.mod h1:R+2BNtUfTfhPY0RH18oL02q116bakeBWjanrbnVBqkM=
1919
github.com/aws/aws-sdk-go-v2/service/iam v1.32.0 h1:ZNlfPdw849gBo/lvLFbEEvpTJMij0LXqiNWZ+lIamlU=
2020
github.com/aws/aws-sdk-go-v2/service/iam v1.32.0/go.mod h1:aXWImQV0uTW35LM0A/T4wEg6R1/ReXUu4SM6/lUHYK0=
21-
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
22-
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
21+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
22+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
2323
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6 h1:6tayEze2Y+hiL3kdnEUxSPsP+pJsUfwLSFspFl1ru9Q=
2424
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6/go.mod h1:qVNb/9IOVsLCZh0x2lnagrBwQ9fxajUpXS7OZfIsKn0=
25-
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
26-
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
25+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
26+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
2727
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
2828
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
2929
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
@@ -32,8 +32,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n
3232
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
3333
github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.25.5 h1:0Ty3j3QkLoqkZ+VagFisIsKYxGAzjv9hIQb84nlt/Jc=
3434
github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.25.5/go.mod h1:9R1IlrgiivwTCZdbKgMPkseFS+moUM+DLh0TEjO6pvE=
35-
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
36-
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
35+
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
36+
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
3737
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
3838
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
3939
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
/*
2+
Licensed under the Apache License, Version 2.0 (the "License");
3+
you may not use this file except in compliance with the License.
4+
You may obtain a copy of the License at
5+
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
8+
Unless required by applicable law or agreed to in writing, software
9+
distributed under the License is distributed on an "AS IS" BASIS,
10+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
See the License for the specific language governing permissions and
12+
limitations under the License.
13+
*/
14+
15+
package resourcetypes
16+
17+
import (
18+
"context"
19+
"time"
20+
21+
"github.com/aws/aws-sdk-go-v2/aws"
22+
"github.com/aws/aws-sdk-go-v2/service/ec2"
23+
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
24+
"github.com/samber/lo"
25+
"go.uber.org/multierr"
26+
)
27+
28+
type CapacityReservation struct {
29+
ec2Client *ec2.Client
30+
}
31+
32+
func NewCapacityReservation(ec2Client *ec2.Client) *CapacityReservation {
33+
return &CapacityReservation{ec2Client: ec2Client}
34+
}
35+
36+
func (cr *CapacityReservation) String() string {
37+
return "CapacityReservations"
38+
}
39+
40+
func (cr *CapacityReservation) Global() bool {
41+
return false
42+
}
43+
44+
func (cr *CapacityReservation) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) {
45+
reservations, err := cr.getAllCapacityReservations(ctx, &ec2.DescribeCapacityReservationsInput{
46+
Filters: []ec2types.Filter{
47+
{
48+
Name: lo.ToPtr("state"),
49+
Values: []string{string(ec2types.CapacityReservationStateActive)},
50+
},
51+
},
52+
})
53+
if err != nil {
54+
return ids, err
55+
}
56+
57+
reservations = lo.Filter(reservations, func(reservation ec2types.CapacityReservation, _ int) bool {
58+
return lo.FromPtr(reservation.CreateDate).Before(expirationTime)
59+
})
60+
ids = lo.Map(reservations, func(reservation ec2types.CapacityReservation, _ int) string {
61+
return lo.FromPtr(reservation.CapacityReservationId)
62+
})
63+
64+
return ids, err
65+
}
66+
67+
func (cr *CapacityReservation) CountAll(ctx context.Context) (count int, err error) {
68+
reservations, err := cr.getAllCapacityReservations(ctx, &ec2.DescribeCapacityReservationsInput{
69+
Filters: []ec2types.Filter{
70+
{
71+
Name: lo.ToPtr("state"),
72+
Values: []string{
73+
string(ec2types.CapacityReservationStateActive),
74+
string(ec2types.CapacityReservationStatePending),
75+
},
76+
},
77+
},
78+
})
79+
if err != nil {
80+
return count, err
81+
}
82+
83+
return len(reservations), err
84+
}
85+
86+
func (cr *CapacityReservation) Get(ctx context.Context, clusterName string) (ids []string, err error) {
87+
return ids, nil
88+
}
89+
90+
// Cleanup cancels capacity reservations. For IODCRs, it sets the allocation to 0.
91+
// For ODCRs the reservations are cancelled.
92+
func (cr *CapacityReservation) Cleanup(ctx context.Context, ids []string) ([]string, error) {
93+
if len(ids) == 0 {
94+
return nil, nil
95+
}
96+
97+
out, err := cr.ec2Client.DescribeCapacityReservations(ctx, &ec2.DescribeCapacityReservationsInput{
98+
CapacityReservationIds: ids,
99+
})
100+
if err != nil {
101+
return nil, err
102+
}
103+
104+
reservationMap := lo.SliceToMap(out.CapacityReservations, func(reservation ec2types.CapacityReservation) (string, ec2types.CapacityReservation) {
105+
return lo.FromPtr(reservation.CapacityReservationId), reservation
106+
})
107+
108+
var iodcrIDs []string
109+
var odcrIDs []string
110+
for _, id := range ids {
111+
if reservation, exists := reservationMap[id]; exists {
112+
if reservation.Interruptible != nil && lo.FromPtr(reservation.Interruptible) {
113+
iodcrIDs = append(iodcrIDs, id)
114+
} else {
115+
odcrIDs = append(odcrIDs, id)
116+
}
117+
}
118+
}
119+
120+
cleaned := make([]string, 0, len(ids))
121+
var errs []error
122+
123+
for _, id := range iodcrIDs {
124+
_, err := cr.ec2Client.UpdateInterruptibleCapacityReservationAllocation(ctx, &ec2.UpdateInterruptibleCapacityReservationAllocationInput{
125+
CapacityReservationId: aws.String(id),
126+
TargetInstanceCount: aws.Int32(0),
127+
})
128+
if err != nil {
129+
errs = append(errs, err)
130+
continue
131+
}
132+
cleaned = append(cleaned, id)
133+
}
134+
135+
for _, id := range odcrIDs {
136+
_, err := cr.ec2Client.CancelCapacityReservation(ctx, &ec2.CancelCapacityReservationInput{
137+
CapacityReservationId: aws.String(id),
138+
})
139+
if err != nil {
140+
errs = append(errs, err)
141+
continue
142+
}
143+
cleaned = append(cleaned, id)
144+
}
145+
146+
return cleaned, multierr.Combine(errs...)
147+
}
148+
149+
func (cr *CapacityReservation) getAllCapacityReservations(ctx context.Context, params *ec2.DescribeCapacityReservationsInput) (reservations []ec2types.CapacityReservation, err error) {
150+
paginator := ec2.NewDescribeCapacityReservationsPaginator(cr.ec2Client, params)
151+
152+
for paginator.HasMorePages() {
153+
page, err := paginator.NextPage(ctx)
154+
if err != nil {
155+
return reservations, err
156+
}
157+
158+
reservations = append(reservations, page.CapacityReservations...)
159+
}
160+
161+
return reservations, nil
162+
}

0 commit comments

Comments
 (0)