Skip to content

Commit b19382e

Browse files
asifsmohammedasifsmohammed
authored
Updated agent for copyContainer in Data Prepper jenkins (#2168)
* Updated agent for copyContainer Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> * Updated test regression-file for Data Prepper Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> * Updated the agent only for Image copy stage Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> * Created 3 separate stages for Archives, Maven, and Docker Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com> * Restored original agent with alpine3 image Signed-off-by: Asif Sohail Mohammed <nsifmoh@amazon.com>
1 parent ef38d5d commit b19382e

File tree

2 files changed

+158
-113
lines changed

2 files changed

+158
-113
lines changed

‎jenkins/data-prepper/release-data-prepper-all-artifacts.jenkinsfile‎

Lines changed: 121 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -7,102 +7,151 @@ pipeline {
77
agent {
88
docker {
99
label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
10-
image 'opensearchstaging/ci-runner:ci-runner-ubuntu2004-data-prepper-pipeline-v1'
10+
image 'alpine:3'
1111
alwaysPull true
12-
args '-u root'
1312
}
1413
}
1514
parameters {
1615
string(name: 'VERSION', description: 'The version of Data Prepper', trim: true)
1716
string(name: 'DATA_PREPPER_BUILD_NUMBER', description: 'The build number of the Data Prepper build from GitHub to release.', trim: true)
1817
}
1918
stages {
20-
stage('Download Archives') {
21-
steps {
22-
script {
23-
archivePath = "${DATA_PREPPER_ARTIFACT_STAGING_SITE}/${VERSION}/${DATA_PREPPER_BUILD_NUMBER}/archive"
19+
stage('Promote Archives') {
20+
agent {
21+
docker {
22+
label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
23+
image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2'
24+
args '-e JAVA_HOME=/opt/java/openjdk-11'
25+
alwaysPull true
26+
}
27+
}
28+
stages {
29+
stage('Download Archives') {
30+
steps {
31+
script {
32+
archivePath = "${DATA_PREPPER_ARTIFACT_STAGING_SITE}/${VERSION}/${DATA_PREPPER_BUILD_NUMBER}/archive"
2433

25-
dir('archive') {
26-
sh "curl -sSL ${archivePath}/opensearch-data-prepper-${VERSION}-linux-x64.tar.gz -o opensearch-data-prepper-${VERSION}-linux-x64.tar.gz"
27-
sh "curl -sSL ${archivePath}/opensearch-data-prepper-jdk-${VERSION}-linux-x64.tar.gz -o opensearch-data-prepper-jdk-${VERSION}-linux-x64.tar.gz"
34+
dir('archive') {
35+
sh "curl -sSL ${archivePath}/opensearch-data-prepper-${VERSION}-linux-x64.tar.gz -o opensearch-data-prepper-${VERSION}-linux-x64.tar.gz"
36+
sh "curl -sSL ${archivePath}/opensearch-data-prepper-jdk-${VERSION}-linux-x64.tar.gz -o opensearch-data-prepper-jdk-${VERSION}-linux-x64.tar.gz"
37+
}
38+
}
39+
}
40+
}
41+
stage('Sign Archives') {
42+
steps {
43+
script {
44+
signatureType = '.sig'
45+
signArtifacts(
46+
artifactPath: "${WORKSPACE}/archive",
47+
sigtype: signatureType,
48+
platform: 'linux'
49+
)
50+
}
51+
}
52+
}
53+
stage('Release Archives to Production Distribution Bucket') {
54+
steps {
55+
script {
56+
withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
57+
s3Upload(file: 'archive/', bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "data-prepper/${VERSION}/")
58+
}
59+
}
2860
}
2961
}
3062
}
3163
}
32-
stage('Download Maven Artifacts') {
33-
steps {
34-
script {
35-
mavenPath = "${DATA_PREPPER_ARTIFACT_STAGING_SITE}/${VERSION}/${DATA_PREPPER_BUILD_NUMBER}/maven"
36-
group = "org/opensearch/dataprepper"
37-
artifacts = ["data-prepper-api"]
38-
fileTypes = ["-javadoc.jar", ".jar", ".pom", "-sources.jar", ".module"]
39-
checksums = ["", ".md5", ".sha1", ".sha256", ".sha512"]
4064

41-
downloadArtifacts()
65+
stage('Promote Docker') {
66+
agent {
67+
docker {
68+
label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
69+
image 'alpine:3'
70+
alwaysPull true
4271
}
4372
}
44-
}
45-
stage('Sign Archives') {
46-
steps {
47-
script {
48-
signatureType = '.sig'
49-
signArtifacts(
50-
artifactPath: "${WORKSPACE}/archive",
51-
sigtype: signatureType,
52-
platform: 'linux'
53-
)
73+
stages {
74+
stage('Copy Docker Image to DockerHub') {
75+
steps {
76+
script {
77+
dockerCopyHub: {
78+
build job: 'docker-copy',
79+
parameters: [
80+
string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
81+
string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
82+
string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
83+
string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}")
84+
]
85+
}
86+
}
87+
}
88+
}
89+
stage('Copy Docker Image to ECR') {
90+
steps {
91+
script {
92+
dockerCopyECR: {
93+
build job: 'docker-copy',
94+
parameters: [
95+
string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
96+
string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
97+
string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
98+
string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}")
99+
]
100+
}
101+
102+
}
103+
}
54104
}
55105
}
56106
}
57-
stage('Sign Maven Artifacts') {
58-
steps {
59-
script {
60-
signArtifacts(
61-
artifactPath: "${WORKSPACE}/maven",
62-
type: 'maven',
63-
platform: 'linux'
64-
)
107+
108+
stage('Promote Maven') {
109+
agent {
110+
docker {
111+
label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
112+
image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2'
113+
args '-e JAVA_HOME=/opt/java/openjdk-11'
114+
alwaysPull true
65115
}
66116
}
67-
}
68-
stage('Release Archives to Production Distribution Bucket') {
69-
steps {
70-
script {
71-
withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
72-
s3Upload(file: 'archive/', bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "data-prepper/${VERSION}/")
117+
stages {
118+
stage('Download Maven Artifacts') {
119+
steps {
120+
script {
121+
mavenPath = "${DATA_PREPPER_ARTIFACT_STAGING_SITE}/${VERSION}/${DATA_PREPPER_BUILD_NUMBER}/maven"
122+
group = "org/opensearch/dataprepper"
123+
artifacts = ["data-prepper-api"]
124+
fileTypes = ["-javadoc.jar", ".jar", ".pom", "-sources.jar", ".module"]
125+
checksums = ["", ".md5", ".sha1", ".sha256", ".sha512"]
126+
127+
downloadArtifacts()
128+
}
73129
}
74130
}
75-
}
76-
}
77-
stage('Release Docker Image to DockerHub') {
78-
steps {
79-
script {
80-
copyContainer(
81-
sourceImage: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}",
82-
sourceRegistry: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}",
83-
destinationImage: "data-prepper:${VERSION}",
84-
destinationRegistry: 'opensearchproject',
85-
)
86-
copyContainer(
87-
sourceImage: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}",
88-
sourceRegistry: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}",
89-
destinationImage: "data-prepper:latest",
90-
destinationRegistry: 'opensearchproject',
91-
)
131+
stage('Sign Maven Artifacts') {
132+
steps {
133+
script {
134+
signArtifacts(
135+
artifactPath: "${WORKSPACE}/maven",
136+
type: 'maven',
137+
platform: 'linux'
138+
)
139+
}
140+
}
92141
}
93-
}
94-
}
95-
stage('Upload Artifacts to Sonatype') {
96-
environment {
97-
REPO_URL = "https://aws.oss.sonatype.org/"
98-
STAGING_PROFILE_ID = "${SONATYPE_STAGING_PROFILE_ID}"
99-
BUILD_ID = "${BUILD_ID}"
100-
}
101-
steps {
102-
script {
103-
// stage artifacts for release with Sonatype
104-
withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) {
105-
sh('$WORKSPACE/publish/stage-maven-release.sh $WORKSPACE/maven')
142+
stage('Upload Artifacts to Sonatype') {
143+
environment {
144+
REPO_URL = "https://aws.oss.sonatype.org/"
145+
STAGING_PROFILE_ID = "${SONATYPE_STAGING_PROFILE_ID}"
146+
BUILD_ID = "${BUILD_ID}"
147+
}
148+
steps {
149+
script {
150+
// stage artifacts for release with Sonatype
151+
withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) {
152+
sh('$WORKSPACE/publish/stage-maven-release.sh $WORKSPACE/maven')
153+
}
154+
}
106155
}
107156
}
108157
}

0 commit comments

Comments
 (0)
⚡