Skip to content

Commit 950d55c

Browse files
peterzhuamazonpeterzhuamazon
authored
Add RPM Temporary Signing Through Shell Scripts (#2196)
* Add RPM Temporary Signing Through Shell Scripts Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Fix typo Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * More typo Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * More comments Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Add jenkins unittest Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
1 parent ee2d968 commit 950d55c

17 files changed

+445
-145
lines changed

‎jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ pipeline {
44
agent {
55
docker {
66
label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
7-
image 'opensearchstaging/ci-runner:ci-runner-centos7-v1'
7+
image 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2'
88
alwaysPull true
99
}
1010
}
@@ -25,7 +25,7 @@ pipeline {
2525
description: 'What platform is this distribution build for?'
2626
)
2727
choice(
28-
choices: ['.sig'],
28+
choices: ['.sig', '.rpm'],
2929
name: 'SIGNATURE_TYPE',
3030
description: 'What is signature file type?'
3131
)
@@ -52,8 +52,13 @@ pipeline {
5252
println("Note: only supported file types will be signed")
5353

5454
for(filename in downloadedFiles){
55-
filenamesForUrls.add(filename)
56-
filenamesForUrls.add(filename + SIGNATURE_TYPE)
55+
if (SIGNATURE_TYPE.equals('.sig')) {
56+
filenamesForUrls.add(filename)
57+
filenamesForUrls.add(filename + SIGNATURE_TYPE)
58+
}
59+
else {
60+
filenamesForUrls.add(filename)
61+
}
5762
}
5863

5964
finalUploadPath = ([

‎scripts/pkg/sign_templates/rpmmacros‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
%_signature gpg
2+
%_gpg_path ~/.gnupg
3+
%_gpg_name ##key_name##
4+
%_gpg /usr/bin/gpg
5+
%__gpg_sign_cmd %{__gpg} \
6+
gpg --no-verbose --no-armor --batch --yes --pinentry-mode loopback \
7+
--passphrase-file ##passphrase_name## \
8+
%{?_gpg_digest_algo:--digest-algo %{_gpg_digest_algo}} \
9+
--no-secmem-warning \
10+
-u "%{_gpg_name}" -sbo %{__signature_filename} --digest-algo sha512 %{__plaintext_filename}

‎tests/jenkins/TestAssembleManifest.groovy‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ class TestAssembleManifest extends BuildPipelineTest {
1515
void testAssembleManifest_rpm() {
1616
this.registerLibTester(new AssembleManifestLibTester('tests/data/opensearch-build-1.3.0-rpm.yml'))
1717

18+
this.registerLibTester(new SignArtifactsLibTester('.rpm', 'linux', "rpm/dist/opensearch", null, null))
19+
1820
this.registerLibTester(new BuildYumRepoTester(
1921
'tests/data/opensearch-build-1.3.0-rpm.yml',
2022
'https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64'

‎tests/jenkins/TestSignArtifacts.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ class TestSignArtifacts extends BuildPipelineTest {
1717
void setUp() {
1818

1919
this.registerLibTester(new SignArtifactsLibTester('.sig', 'linux', "${this.workspace}/artifacts", null, null))
20+
this.registerLibTester(new SignArtifactsLibTester('.rpm', 'linux', "${this.workspace}/artifacts", 'null', null))
2021
this.registerLibTester(new SignArtifactsLibTester(null, 'linux', "${this.workspace}/file.yml", 'maven', null))
2122
super.setUp()
2223
}

‎tests/jenkins/jenkinsjob-regression-files/data-prepper/release-data-prepper-all-artifacts.jenkinsfile.txt‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,21 +12,22 @@
1212
release-data-prepper-all-artifacts.stage(Sign Archives, groovy.lang.Closure)
1313
release-data-prepper-all-artifacts.script(groovy.lang.Closure)
1414
release-data-prepper-all-artifacts.signArtifacts({artifactPath=/tmp/workspace/archive, sigtype=.sig, platform=linux})
15+
signArtifacts.echo(PGP Signature Signing)
1516
signArtifacts.fileExists(/tmp/workspace/sign.sh)
1617
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
1718
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
1819
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
1920
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
2021
signArtifacts.sh(
21-
#!/bin/bash
22-
set +x
23-
export ROLE=dummy_signer_client_role
24-
export EXTERNAL_ID=signer_client_external_id
25-
export UNSIGNED_BUCKET=signer_client_unsigned_bucket
26-
export SIGNED_BUCKET=signer_client_signed_bucket
22+
#!/bin/bash
23+
set +x
24+
export ROLE=dummy_signer_client_role
25+
export EXTERNAL_ID=signer_client_external_id
26+
export UNSIGNED_BUCKET=signer_client_unsigned_bucket
27+
export SIGNED_BUCKET=signer_client_signed_bucket
2728

28-
/tmp/workspace/sign.sh /tmp/workspace/archive --sigtype=.sig --platform=linux
29-
)
29+
/tmp/workspace/sign.sh /tmp/workspace/archive --sigtype=.sig --platform=linux
30+
)
3031
release-data-prepper-all-artifacts.stage(Release Archives to Production Distribution Bucket, groovy.lang.Closure)
3132
release-data-prepper-all-artifacts.script(groovy.lang.Closure)
3233
release-data-prepper-all-artifacts.withAWS({role=production-role-name, roleAccount=aws-account-artifact, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
@@ -77,21 +78,22 @@
7778
release-data-prepper-all-artifacts.stage(Sign Maven Artifacts, groovy.lang.Closure)
7879
release-data-prepper-all-artifacts.script(groovy.lang.Closure)
7980
release-data-prepper-all-artifacts.signArtifacts({artifactPath=/tmp/workspace/maven, type=maven, platform=linux})
81+
signArtifacts.echo(PGP Signature Signing)
8082
signArtifacts.fileExists(/tmp/workspace/sign.sh)
8183
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
8284
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
8385
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
8486
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
8587
signArtifacts.sh(
86-
#!/bin/bash
87-
set +x
88-
export ROLE=dummy_signer_client_role
89-
export EXTERNAL_ID=signer_client_external_id
90-
export UNSIGNED_BUCKET=signer_client_unsigned_bucket
91-
export SIGNED_BUCKET=signer_client_signed_bucket
88+
#!/bin/bash
89+
set +x
90+
export ROLE=dummy_signer_client_role
91+
export EXTERNAL_ID=signer_client_external_id
92+
export UNSIGNED_BUCKET=signer_client_unsigned_bucket
93+
export SIGNED_BUCKET=signer_client_signed_bucket
9294

93-
/tmp/workspace/sign.sh /tmp/workspace/maven --type=maven --platform=linux
94-
)
95+
/tmp/workspace/sign.sh /tmp/workspace/maven --type=maven --platform=linux
96+
)
9597
release-data-prepper-all-artifacts.stage(Upload Artifacts to Sonatype, groovy.lang.Closure)
9698
release-data-prepper-all-artifacts.script(groovy.lang.Closure)
9799
release-data-prepper-all-artifacts.usernamePassword({credentialsId=Sonatype, usernameVariable=SONATYPE_USERNAME, passwordVariable=SONATYPE_PASSWORD})

‎tests/jenkins/jenkinsjob-regression-files/maven-sign-release/maven-sign-release.jenkinsfile.txt‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,21 +11,22 @@
1111
downloadFromS3.s3Download({file=/tmp/workspace/artifacts, bucket=job-s3-bucket-name, path=distribution-build-opensearch/1.0.0/123/linux/x64/builds/, force=true})
1212
maven-sign-release.echo(Signing Maven artifacts.)
1313
maven-sign-release.signArtifacts({artifactPath=/tmp/workspace/artifacts/distribution-build-opensearch/1.0.0/123/linux/x64/builds/opensearch/manifest.yml, type=maven, platform=linux})
14+
signArtifacts.echo(PGP Signature Signing)
1415
signArtifacts.fileExists(/tmp/workspace/sign.sh)
1516
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
1617
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
1718
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
1819
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
1920
signArtifacts.sh(
20-
#!/bin/bash
21-
set +x
22-
export ROLE=dummy_signer_client_role
23-
export EXTERNAL_ID=signer_client_external_id
24-
export UNSIGNED_BUCKET=signer_client_unsigned_bucket
25-
export SIGNED_BUCKET=signer_client_signed_bucket
21+
#!/bin/bash
22+
set +x
23+
export ROLE=dummy_signer_client_role
24+
export EXTERNAL_ID=signer_client_external_id
25+
export UNSIGNED_BUCKET=signer_client_unsigned_bucket
26+
export SIGNED_BUCKET=signer_client_signed_bucket
2627

27-
/tmp/workspace/sign.sh /tmp/workspace/artifacts/distribution-build-opensearch/1.0.0/123/linux/x64/builds/opensearch/manifest.yml --type=maven --platform=linux
28-
)
28+
/tmp/workspace/sign.sh /tmp/workspace/artifacts/distribution-build-opensearch/1.0.0/123/linux/x64/builds/opensearch/manifest.yml --type=maven --platform=linux
29+
)
2930
maven-sign-release.stage(stage maven artifacts, groovy.lang.Closure)
3031
maven-sign-release.script(groovy.lang.Closure)
3132
maven-sign-release.usernamePassword({credentialsId=Sonatype, usernameVariable=SONATYPE_USERNAME, passwordVariable=SONATYPE_PASSWORD})

‎tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts/sign-standalone-artifacts.jenkinsfile.txt‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,28 +2,29 @@
22
sign-standalone-artifacts.legacySCM(groovy.lang.Closure)
33
sign-standalone-artifacts.library({identifier=jenkins@20211123, retriever=null})
44
sign-standalone-artifacts.pipeline(groovy.lang.Closure)
5-
sign-standalone-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
5+
sign-standalone-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
66
sign-standalone-artifacts.stage(sign, groovy.lang.Closure)
77
sign-standalone-artifacts.script(groovy.lang.Closure)
88
sign-standalone-artifacts.sh(mkdir /tmp/workspace/artifacts)
99
sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_1_artifact.tar.gz -o /tmp/workspace/artifacts/dummy_1_artifact.tar.gz)
1010
sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_2_artifact.tar.gz -o /tmp/workspace/artifacts/dummy_2_artifact.tar.gz)
1111
sign-standalone-artifacts.signArtifacts({artifactPath=/tmp/workspace/artifacts, sigtype=.sig, platform=linux})
12+
signArtifacts.echo(PGP Signature Signing)
1213
signArtifacts.fileExists(/tmp/workspace/sign.sh)
1314
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
1415
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
1516
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
1617
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
1718
signArtifacts.sh(
18-
#!/bin/bash
19-
set +x
20-
export ROLE=dummy_signer_client_role
21-
export EXTERNAL_ID=signer_client_external_id
22-
export UNSIGNED_BUCKET=signer_client_unsigned_bucket
23-
export SIGNED_BUCKET=signer_client_signed_bucket
19+
#!/bin/bash
20+
set +x
21+
export ROLE=dummy_signer_client_role
22+
export EXTERNAL_ID=signer_client_external_id
23+
export UNSIGNED_BUCKET=signer_client_unsigned_bucket
24+
export SIGNED_BUCKET=signer_client_signed_bucket
2425

25-
/tmp/workspace/sign.sh /tmp/workspace/artifacts --sigtype=.sig --platform=linux
26-
)
26+
/tmp/workspace/sign.sh /tmp/workspace/artifacts --sigtype=.sig --platform=linux
27+
)
2728
sign-standalone-artifacts.uploadToS3({sourcePath=/tmp/workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
2829
uploadToS3.withAWS({role=Dummy_Upload_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
2930
uploadToS3.s3Upload({file=/tmp/workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})

‎tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,84 @@
1212
BuildManifest.asBoolean()
1313
BuildManifest.getArtifactRootUrlWithoutDistribution(https://ci.opensearch.org/dbc, vars-build, 123)
1414
assembleManifest.sh(./assemble.sh "tests/data/opensearch-build-1.3.0-rpm.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64)
15+
assembleManifest.signArtifacts({artifactPath=rpm/dist/opensearch, sigtype=.rpm, platform=linux})
16+
signArtifacts.echo(RPM Add Sign)
17+
signArtifacts.withAWS({role=sign_asm_role, roleAccount=sign_asm_account, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure)
18+
signArtifacts.string({credentialsId=jenkins-rpm-signing-asm-pass-id, variable=SIGNING_PASS_ID})
19+
signArtifacts.string({credentialsId=jenkins-rpm-signing-asm-secret-id, variable=SIGNING_SECRET_ID})
20+
signArtifacts.withCredentials([SIGNING_PASS_ID, SIGNING_SECRET_ID], groovy.lang.Closure)
21+
signArtifacts.sh(
22+
set -e
23+
set +x
24+
25+
ARTIFACT_PATH="rpm/dist/opensearch"
26+
27+
echo "------------------------------------------------------------------------"
28+
echo "Check Utility Versions"
29+
gpg_version_limit="2.2.0"
30+
rpm_version_limit="4.13.0" # https://bugzilla.redhat.com/show_bug.cgi?id=227632
31+
32+
gpg_version_check=`gpg --version | head -n 1 | grep -oE '[0-9.]+'`
33+
gpg_version_check_final=`echo $gpg_version_check $gpg_version_limit | tr ' ' '
34+
' | sort -V | head -n 1`
35+
rpm_version_check=`rpm --version | head -n 1 | grep -oE '[0-9.]+'`
36+
rpm_version_check_final=`echo $rpm_version_check $rpm_version_limit | tr ' ' '
37+
' | sort -V | head -n 1`
38+
39+
echo -e "gpg_version_limit gpg_version_check"
40+
echo -e "$gpg_version_limit $gpg_version_check_final"
41+
echo -e "rpm_version_limit rpm_version_check"
42+
echo -e "$rpm_version_limit $rpm_version_check_final"
43+
44+
if [[ $gpg_version_limit = $gpg_version_check_final ]] && [[ $rpm_version_limit = $rpm_version_check_final ]]; then
45+
echo "Utility version is equal or greater than set limit, continue."
46+
else
47+
echo "Utility version is lower than set limit, exit 1"
48+
exit 1
49+
fi
50+
51+
export GPG_TTY=`tty`
52+
53+
echo "------------------------------------------------------------------------"
54+
echo "Setup RPM Macros"
55+
cp -v scripts/pkg/sign_templates/rpmmacros ~/.rpmmacros
56+
sed -i "s/##key_name##/OpenSearch project/g;s/##passphrase_name##/passphrase/g" ~/.rpmmacros
57+
58+
echo "------------------------------------------------------------------------"
59+
echo "Import OpenSearch keys"
60+
aws secretsmanager get-secret-value --region "sign_asm_region" --secret-id "SIGNING_PASS_ID" | jq -r .SecretBinary | base64 --decode > passphrase
61+
aws secretsmanager get-secret-value --region "sign_asm_region" --secret-id "SIGNING_SECRET_ID" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase -
62+
63+
echo "------------------------------------------------------------------------"
64+
echo "Start Signing Rpm"
65+
66+
if file $ARTIFACT_PATH | grep -q directory; then
67+
68+
echo "Sign directory"
69+
for rpm_file in `ls $ARTIFACT_PATH`; do
70+
if file $ARTIFACT_PATH/$rpm_file | grep -q RPM; then
71+
rpm --addsign $ARTIFACT_PATH/$rpm_file
72+
rpm -qip $ARTIFACT_PATH/$rpm_file | grep Signature
73+
fi
74+
done
75+
76+
elif file $ARTIFACT_PATH | grep -q RPM; then
77+
echo "Sign single rpm"
78+
rpm --addsign $ARTIFACT_PATH
79+
rpm -qip $ARTIFACT_PATH | grep Signature
80+
81+
else
82+
echo "This is neither a directory nor a RPM pkg, exit 1"
83+
exit 1
84+
fi
85+
86+
echo "------------------------------------------------------------------------"
87+
echo "Clean up gpg"
88+
gpg --batch --yes --delete-secret-keys sign_asm_keyid
89+
gpg --batch --yes --delete-keys sign_asm_keyid
90+
rm -v passphrase
91+
92+
)
1593
assembleManifest.buildYumRepo({baseUrl=https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64, buildManifest=tests/data/opensearch-build-1.3.0-rpm.yml})
1694
buildYumRepo.legacySCM(groovy.lang.Closure)
1795
buildYumRepo.library({identifier=jenkins@20211123, retriever=null})

0 commit comments

Comments
 (0)
⚡