Skip to content

Commit d671560

Browse files
drichmdbdrichmdb
committed
testing
This commit changes dbtools to dual write all of its uploads to a new bucket, cdn-origin-db-tools. In the future we will remove puts to the old bucket and only serve artifacts from the new bucket using our CDN.
1 parent 4558399 commit d671560

3 files changed

Lines changed: 143 additions & 18 deletions

File tree

‎common.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,17 +343,31 @@ functions:
343343
content_type: application/octet-stream
344344

345345
"upload release packages to s3":
346+
- command: ec2.assume_role
347+
params:
348+
role_arn: "arn:aws:iam::119629040606:role/s3-access.cdn-origin-db-tools"
346349
- command: shell.exec
347350
params:
348351
working_dir: src/github.com/mongodb/mongo-tools
352+
env:
353+
NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
354+
NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
355+
NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
349356
script: |
350357
${_set_shell_env}
351358
go run release/release.go upload-release
352359
353360
"upload release json feed to s3":
361+
- command: ec2.assume_role
362+
params:
363+
role_arn: "arn:aws:iam::119629040606:role/s3-access.cdn-origin-db-tools"
354364
- command: shell.exec
355365
params:
356366
working_dir: src/github.com/mongodb/mongo-tools
367+
env:
368+
NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
369+
NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
370+
NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
357371
script: |
358372
${_set_shell_env}
359373
go run release/release.go upload-json
@@ -369,9 +383,16 @@ functions:
369383
permissions: public-read
370384

371385
"generate full JSON feed":
386+
- command: ec2.assume_role
387+
params:
388+
role_arn: "arn:aws:iam::119629040606:role/s3-access.cdn-origin-db-tools"
372389
- command: shell.exec
373390
params:
374391
working_dir: src/github.com/mongodb/mongo-tools
392+
env:
393+
NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
394+
NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
395+
NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
375396
script: |
376397
${_set_shell_env}
377398
go run release/release.go generate-full-json

‎release/aws/aws.go‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"regexp"
1515

1616
"github.com/aws/aws-sdk-go/aws"
17+
"github.com/aws/aws-sdk-go/aws/credentials"
1718
"github.com/aws/aws-sdk-go/aws/session"
1819
"github.com/aws/aws-sdk-go/service/s3"
1920
"github.com/aws/aws-sdk-go/service/s3/s3manager"
@@ -24,6 +25,7 @@ var awsClient *AWS
2425

2526
type AWS struct {
2627
session *session.Session
28+
acl string
2729
}
2830

2931
func initializeClient() error {
@@ -40,10 +42,28 @@ func initializeClient() error {
4042

4143
awsClient = &AWS{
4244
session: s,
45+
acl: "public-read",
4346
}
4447
return nil
4548
}
4649

50+
func NewClientFromCredentials(id, secret, token, acl string) (*AWS, error) {
51+
s, err := session.NewSession(&aws.Config{
52+
Region: aws.String("us-east-1"),
53+
Credentials: credentials.NewStaticCredentials(id, secret, token),
54+
})
55+
if err != nil {
56+
return nil, fmt.Errorf("failed to create AWS session: %w", err)
57+
}
58+
59+
c := &AWS{
60+
session: s,
61+
acl: acl,
62+
}
63+
64+
return c, nil
65+
}
66+
4767
// GetClient returns the global AWS client.
4868
// It initializes the AWS client if it hasn't already been initialized.
4969
func GetClient() (*AWS, error) {
@@ -75,7 +95,7 @@ func (a *AWS) UploadBytes(bucket, objPath, filename string, reader io.Reader) er
7595
_, err := uploader.Upload(&s3manager.UploadInput{
7696
Bucket: aws.String(bucket),
7797
Key: aws.String(key),
78-
ACL: aws.String("public-read"),
98+
ACL: aws.String(a.acl),
7999
Body: reader,
80100
})
81101
if err != nil {

‎release/release.go‎

Lines changed: 101 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -953,13 +953,25 @@ func generateFullReleaseJSON(v version.Version) {
953953
return
954954
}
955955

956-
awsClient, err := aws.GetClient()
957-
check(err, "get aws client")
956+
awsClientOld, err := aws.GetClient()
957+
check(err, "get aws client old")
958+
959+
newAccessKeyID := os.Getenv("NEW_AWS_ACCESS_KEY_ID")
960+
newSecretAccessKey := os.Getenv("NEW_AWS_SECRET_ACCESS_KEY")
961+
newSessionToken := os.Getenv("NEW_AWS_SESSION_TOKEN")
962+
963+
awsClientNew, err := aws.NewClientFromCredentials(
964+
newAccessKeyID,
965+
newSecretAccessKey,
966+
newSessionToken,
967+
"private",
968+
)
969+
check(err, "new aws client from credentials")
958970

959-
feed, err := awsClient.GenerateFullReleaseFeedFromObjects()
971+
feed, err := awsClientOld.GenerateFullReleaseFeedFromObjects()
960972
check(err, "generate full release feed from s3 objects")
961973

962-
uploadFeedFile("full.json", feed, awsClient)
974+
uploadFeedFile("full.json", feed, awsClientOld, awsClientNew)
963975
}
964976

965977
func uploadReleaseJSON(v version.Version) {
@@ -996,8 +1008,20 @@ func uploadReleaseJSON(v version.Version) {
9961008
log.Fatalf("found %d sign tasks, but expected %d", len(signTasks), pfCount)
9971009
}
9981010

999-
awsClient, err := aws.GetClient()
1000-
check(err, "get aws client")
1011+
awsClientOld, err := aws.GetClient()
1012+
check(err, "get aws client old")
1013+
1014+
newAccessKeyID := os.Getenv("NEW_AWS_ACCESS_KEY_ID")
1015+
newSecretAccessKey := os.Getenv("NEW_AWS_SECRET_ACCESS_KEY")
1016+
newSessionToken := os.Getenv("NEW_AWS_SESSION_TOKEN")
1017+
1018+
awsClientNew, err := aws.NewClientFromCredentials(
1019+
newAccessKeyID,
1020+
newSecretAccessKey,
1021+
newSessionToken,
1022+
"private",
1023+
)
1024+
check(err, "new aws client from credentials")
10011025

10021026
// Accumulate all downloaded artifacts from sign tasks for JSON feed.
10031027
var dls []*download.ToolsDownload
@@ -1061,8 +1085,18 @@ func uploadReleaseJSON(v version.Version) {
10611085
}
10621086

10631087
// Download the current full.json
1064-
buff, err := awsClient.DownloadFile("downloads.mongodb.org", "tools/db/full.json")
1065-
check(err, "download full.json")
1088+
const addr = "https://downloads.mongodb.org/tools/db/full.json"
1089+
res, err := http.Get(addr)
1090+
check(err, "http get full.json")
1091+
1092+
defer res.Body.Close()
1093+
1094+
buff, err := io.ReadAll(res.Body)
1095+
check(err, "read full.json body")
1096+
1097+
if res.StatusCode != http.StatusOK {
1098+
panic(fmt.Errorf("get full.json status %d: %s", res.StatusCode, string(buff)))
1099+
}
10661100

10671101
var fullFeed download.JSONFeed
10681102

@@ -1074,7 +1108,7 @@ func uploadReleaseJSON(v version.Version) {
10741108
fullFeed.Versions,
10751109
&download.ToolsVersion{Version: v.String(), Downloads: dls},
10761110
)
1077-
uploadFeedFile("full.json", &fullFeed, awsClient)
1111+
uploadFeedFile("full.json", &fullFeed, awsClientOld, awsClientNew)
10781112

10791113
// Upload only the most recent version to release.json
10801114
var feed download.JSONFeed
@@ -1083,23 +1117,42 @@ func uploadReleaseJSON(v version.Version) {
10831117
&download.ToolsVersion{Version: v.String(), Downloads: dls},
10841118
)
10851119

1086-
uploadFeedFile("release.json", &feed, awsClient)
1120+
uploadFeedFile("release.json", &feed, awsClientOld, awsClientNew)
10871121
}
10881122

1089-
func uploadFeedFile(filename string, feed *download.JSONFeed, awsClient *aws.AWS) {
1123+
func uploadFeedFile(filename string, feed *download.JSONFeed, awsClientOld, awsClientNew *aws.AWS) {
10901124
var feedBuffer bytes.Buffer
10911125

10921126
jsonEncoder := json.NewEncoder(&feedBuffer)
10931127
jsonEncoder.SetIndent("", " ")
10941128
err := jsonEncoder.Encode(*feed)
10951129
check(err, "encode json feed")
10961130

1131+
feedBytes := feedBuffer.Bytes()
1132+
10971133
log.Printf(
10981134
"uploading download feed to https://s3.amazonaws.com/downloads.mongodb.org/tools/db/%s\n",
10991135
filename,
11001136
)
1101-
err = awsClient.UploadBytes("downloads.mongodb.org", "/tools/db", filename, &feedBuffer)
1102-
check(err, "upload json feed")
1137+
err = awsClientOld.UploadBytes(
1138+
"downloads.mongodb.org",
1139+
"/tools/db",
1140+
filename,
1141+
bytes.NewReader(feedBytes),
1142+
)
1143+
check(err, "upload json feed old")
1144+
1145+
log.Printf(
1146+
"uploading download feed to s3://cdn-origin-db-tools/tools/db/%s\n",
1147+
filename,
1148+
)
1149+
err = awsClientNew.UploadBytes(
1150+
"cdn-origin-db-tools",
1151+
"/tools/db",
1152+
filename,
1153+
bytes.NewReader(feedBytes),
1154+
)
1155+
check(err, "upload json feed new")
11031156
}
11041157

11051158
func uploadRelease(v version.Version) {
@@ -1129,8 +1182,20 @@ func uploadRelease(v version.Version) {
11291182
log.Fatalf("found %d sign tasks, but expected one", len(signTasks))
11301183
}
11311184

1132-
awsClient, err := aws.GetClient()
1133-
check(err, "get aws client")
1185+
awsClientOld, err := aws.GetClient()
1186+
check(err, "get old aws client")
1187+
1188+
newAccessKeyID := os.Getenv("NEW_AWS_ACCESS_KEY_ID")
1189+
newSecretAccessKey := os.Getenv("NEW_AWS_SECRET_ACCESS_KEY")
1190+
newSessionToken := os.Getenv("NEW_AWS_SESSION_TOKEN")
1191+
1192+
awsClientNew, err := aws.NewClientFromCredentials(
1193+
newAccessKeyID,
1194+
newSecretAccessKey,
1195+
newSessionToken,
1196+
"private",
1197+
)
1198+
check(err, "new aws client from credentials")
11341199

11351200
for _, task := range signTasks {
11361201
log.Printf("\ngetting artifacts for %s\n", task.Variant)
@@ -1182,14 +1247,33 @@ func uploadRelease(v version.Version) {
11821247
" uploading to https://s3.amazonaws.com/downloads.mongodb.org/tools/db/%s\n",
11831248
stableFile,
11841249
)
1185-
err = awsClient.UploadFile("downloads.mongodb.org", "/tools/db", stableFile)
1250+
err = awsClientOld.UploadFile("downloads.mongodb.org", "/tools/db", stableFile)
11861251
check(err, "uploading %q file to S3", stableFile)
11871252
log.Printf(
11881253
" uploading to https://s3.amazonaws.com/downloads.mongodb.org/tools/db/%s\n",
11891254
latestStableFile,
11901255
)
1191-
err = awsClient.UploadFile("downloads.mongodb.org", "/tools/db", latestStableFile)
1256+
1257+
err = awsClientOld.UploadFile(
1258+
"downloads.mongodb.org",
1259+
"/tools/db",
1260+
latestStableFile,
1261+
)
11921262
check(err, "uploading %q file to S3", latestStableFile)
1263+
1264+
log.Printf(
1265+
" uploading to s3://cdn-origin-db-tools/tools/db/%s\n",
1266+
stableFile,
1267+
)
1268+
err = awsClientNew.UploadFile("cdn-origin-db-tools", "/tools/db", stableFile)
1269+
check(err, "uploading %q file to S3 new", stableFile)
1270+
log.Printf(
1271+
" uploading to s3://cdn-origin-db-tools/tools/db/%s\n",
1272+
latestStableFile,
1273+
)
1274+
1275+
err = awsClientNew.UploadFile("cdn-origin-db-tools", "/tools/db", latestStableFile)
1276+
check(err, "uploading %q file to S3 new", latestStableFile)
11931277
}
11941278
}
11951279
}

0 commit comments

Comments
 (0)
⚡