Skip to content

Commit cc0718c

Browse files
authored
Migrate image and dotnet tool publish to 1es pipelines (#7920)
* Migrate to 1es pipelines * Update build-tools tag
1 parent 8254231 commit cc0718c

8 files changed

Lines changed: 433 additions & 307 deletions

File tree

eng/containers/ci.yml

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,39 +3,29 @@ parameters:
33
type: object
44
default:
55
- name: mock_attestation
6-
pool: azsdk-pool-mms-ubuntu-2204-general
7-
vmImage: ubuntu-22.04
86
dockerRepo: 'keyvault-mock-attestation'
9-
dockerFile: 'tools/keyvault-mock-attestation/Dockerfile'
7+
configPath: 'tools/keyvault-mock-attestation'
108
stableTags:
119
- 'latest'
1210
- name: stress_watcher
13-
pool: azsdk-pool-mms-ubuntu-2204-general
14-
vmImage: ubuntu-22.04
1511
dockerRepo: 'stress/watcher'
16-
dockerFile: 'tools/stress-cluster/services/Stress.Watcher/Dockerfile'
12+
configPath: 'tools/stress-cluster/services/Stress.Watcher'
1713
stableTags:
1814
- 'latest'
1915
- name: stress_deploy_test_resources
20-
pool: azsdk-pool-mms-ubuntu-2204-general
21-
vmImage: ubuntu-22.04
2216
dockerRepo: 'stress/deploy-test-resources'
2317
prepareScript: tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/prepare.ps1
24-
dockerFile: 'tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer/Dockerfile'
18+
configPath: 'tools/stress-cluster/cluster/kubernetes/stress-test-addons/images/test-resource-deployer'
2519
stableTags:
2620
- 'latest'
2721
- name: http_fault_injector
28-
pool: azsdk-pool-mms-ubuntu-2204-general
29-
vmImage: ubuntu-22.04
3022
dockerRepo: 'stress/httpfaultinjector'
31-
dockerFile: 'tools/http-fault-injector/Dockerfile'
23+
configPath: 'tools/http-fault-injector'
3224
stableTags:
3325
- 'latest'
3426
- name: otel_collector
35-
pool: azsdk-pool-mms-ubuntu-2204-general
36-
vmImage: ubuntu-22.04
3727
dockerRepo: 'stress/otelcollector'
38-
dockerFile: 'tools/stress-cluster/services/otelcollector/Dockerfile'
28+
configPath: 'tools/stress-cluster/services/otelcollector'
3929
stableTags:
4030
- '0.94.0'
4131

@@ -70,7 +60,6 @@ variables:
7060
value: 'azsdkengsys'
7161
- name: imageTag
7262
value: $(build.buildid)
73-
- template: ../pipelines/templates/variables/globals.yml
7463

7564
extends:
7665
template: ../pipelines/publish-docker-image.yml

eng/pipelines/mirror-container-images.yml

Lines changed: 36 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -17,32 +17,41 @@ parameters:
1717
- source: ubuntu/squid
1818
mirror: azsdkengsys.azurecr.io/mirror/ubuntu/squid
1919

20-
jobs:
21-
- job: MirrorImages
22-
displayName: Mirror Container Images
20+
extends:
21+
template: /eng/pipelines/templates/stages/1es-redirect.yml
22+
parameters:
23+
stages:
24+
- stage:
25+
displayName: Mirror Images
26+
variables:
27+
- template: /eng/pipelines/templates/variables/image.yml
28+
jobs:
29+
- job: MirrorImages
30+
displayName: Mirror Container Images
2331

24-
pool:
25-
name: azsdk-pool-mms-ubuntu-2004-general
26-
vmImage: MMSUbuntu20.04
32+
pool:
33+
name: $(LINUXPOOL)
34+
image: $(LINUXVMIMAGE)
35+
os: linux
2736

28-
steps:
29-
- ${{ each image in parameters.Images }}:
30-
- task: Docker@2
31-
displayName: Login to ${{ split(image.mirror, '.')[0] }}
32-
inputs:
33-
command: login
34-
containerRegistry: ${{ split(image.mirror, '.')[0] }}
35-
- task: Powershell@2
36-
displayName: Mirror ${{ image.source }} to ${{ image.mirror }}
37-
inputs:
38-
pwsh: true
39-
filePath: $(Build.SourcesDirectory)/eng/scripts/mirror-container-image.ps1
40-
${{ if image.changes }}:
41-
arguments: >
42-
-Image ${{ image.source }}
43-
-Mirror ${{ image.mirror }}
44-
-changes '${{ image.changes }}'
45-
${{ else }}:
46-
arguments: >
47-
-Image ${{ image.source }}
48-
-Mirror ${{ image.mirror }}
37+
steps:
38+
- ${{ each image in parameters.Images }}:
39+
- task: Docker@2
40+
displayName: Login to ${{ split(image.mirror, '.')[0] }}
41+
inputs:
42+
command: login
43+
containerRegistry: ${{ split(image.mirror, '.')[0] }}
44+
- task: Powershell@2
45+
displayName: Mirror ${{ image.source }} to ${{ image.mirror }}
46+
inputs:
47+
pwsh: true
48+
filePath: $(Build.SourcesDirectory)/eng/scripts/mirror-container-image.ps1
49+
${{ if image.changes }}:
50+
arguments: >
51+
-Image ${{ image.source }}
52+
-Mirror ${{ image.mirror }}
53+
-changes '${{ image.changes }}'
54+
${{ else }}:
55+
arguments: >
56+
-Image ${{ image.source }}
57+
-Mirror ${{ image.mirror }}
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
# Additional sample inputs can be found in `eng/containers/ci.yml`, but here is an example.
2+
# - name: test_proxy_linux
3+
# dockerRepo: 'engsys/testproxy-lin'
4+
# prepareScript: tools/test-proxy/docker/prepare.ps1
5+
# excludeFromManifest: true/false
6+
# configPath: 'tools/test-proxy/docker'
7+
# stableTags:
8+
# - 'latest'
9+
10+
# A "ManifestDeployment" is a secondary deployment that can be used to map multiple docker tags to a single platform-sensitive one. It waits until
11+
# all previous tag deployments are created, then triggers a manifest upload that bundles all tags configured in DockerDeployments. To exclude an item from the
12+
# manifest upload, set deployment config property excludeFromManifest to true.
13+
# - name: test_proxy_manifest
14+
# dockerRepo: 'engsys/testproxy'
15+
# stableTags:
16+
# - 'latest'
17+
parameters:
18+
- name: DockerDeployments
19+
type: object
20+
default: []
21+
- name: ManifestDeployment
22+
type: object
23+
default: []
24+
- name: ImageTag
25+
type: string
26+
- name: ContainerRegistry
27+
type: string
28+
default: 'azsdkengsys'
29+
- name: Publish
30+
type: boolean
31+
default: true
32+
33+
stages:
34+
- stage:
35+
displayName: Build/Publish Containers
36+
variables:
37+
- template: /eng/pipelines/templates/variables/globals.yml
38+
- template: /eng/pipelines/templates/variables/image.yml
39+
jobs:
40+
- ${{ each config in parameters.DockerDeployments }}:
41+
- job: container_build_${{ config.name }}
42+
displayName: Build ${{ config.name }} Image
43+
pool:
44+
name: $(LINUXPOOL)
45+
image: $(LINUXVMIMAGE)
46+
os: linux
47+
48+
${{ if parameters.Publish }}:
49+
templateContext:
50+
outputs:
51+
- output: containerImage
52+
displayName: Push ${{ config.name }}:${{ parameters.ImageTag }}
53+
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
54+
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ parameters.ImageTag }}
55+
- ${{ if config.stableTags }}:
56+
- ${{ each stableTag in config.stableTags }}:
57+
- output: containerImage
58+
displayName: Push ${{ config.name }}:${{ stableTag }}
59+
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
60+
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ stableTag }}
61+
62+
steps:
63+
- ${{ if config.prepareScript }}:
64+
- pwsh: |
65+
./${{ config.prepareScript }}
66+
displayName: "Run prep script"
67+
- task: 1ES.BuildContainerImage@1
68+
displayName: Build ${{ config.name }}:${{ parameters.ImageTag }}
69+
inputs:
70+
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ parameters.ImageTag }}
71+
path: ${{ config.configPath }}
72+
enableNetwork: true
73+
useBuildKit: true
74+
- ${{ if config.stableTags }}:
75+
- ${{ each stableTag in config.stableTags }}:
76+
- task: 1ES.BuildContainerImage@1
77+
displayName: Build ${{ config.name }}:${{ stableTag }}
78+
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
79+
inputs:
80+
image: ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ stableTag }}
81+
path: ${{ config.configPath }}
82+
enableNetwork: true
83+
useBuildKit: true
84+
85+
- ${{ if and(parameters.ManifestDeployment, parameters.Publish) }}:
86+
- ${{ each deployment in parameters.ManifestDeployment }}:
87+
- job: container_build_${{ deployment.name }}
88+
displayName: Build ${{ deployment.name }} Manifest
89+
${{ if gt(length(parameters.DockerDeployments), 0) }}:
90+
dependsOn:
91+
- ${{ each config in parameters.DockerDeployments }}:
92+
- container_build_${{ config.name }}
93+
pool:
94+
name: $(LINUXPOOL)
95+
image: $(LINUXVMIMAGE)
96+
os: linux
97+
98+
steps:
99+
- pwsh: |
100+
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
101+
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
102+
-DockerDeploymentJson $configurations `
103+
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
104+
-ImageTag "${{ parameters.ImageTag }}" `
105+
106+
Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
107+
displayName: Generate Manifest Variable
108+
109+
- pwsh: |
110+
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }} $(ManifestVariable)
111+
displayName: Generate Manifest
112+
113+
- pwsh: |
114+
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }}
115+
displayName: Upload Manifest
116+
117+
- ${{ if deployment.stableTags }}:
118+
- ${{ each stableTag in deployment.stableTags }}:
119+
- pwsh: |
120+
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
121+
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
122+
-DockerDeploymentJson $configurations `
123+
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
124+
-ImageTag "${{ stableTag }}" `
125+
126+
Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
127+
displayName: Generate Manifest Variable
128+
129+
- pwsh: |
130+
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }} $(ManifestVariable)
131+
displayName: Generate Manifest
132+
133+
- pwsh: |
134+
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }}
135+
displayName: Upload Manifest
Lines changed: 7 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,9 @@
11
# Additional sample inputs can be found in `eng/containers/ci.yml`, but here is an example.
22
# - name: test_proxy_linux
3-
# pool: azsdk-pool-mms-ubuntu-2204-general
4-
# vmImage: ubuntu-22.04
53
# dockerRepo: 'engsys/testproxy-lin'
6-
# additionalDockerArgs: ''
74
# prepareScript: tools/test-proxy/docker/prepare.ps1
85
# excludeFromManifest: true/false
9-
# dockerFile: 'tools/test-proxy/docker/dockerfile'
6+
# configPath: 'tools/test-proxy/docker'
107
# stableTags:
118
# - 'latest'
129

@@ -33,109 +30,9 @@ parameters:
3330
type: boolean
3431
default: true
3532

36-
jobs:
37-
- ${{ each config in parameters.DockerDeployments }}:
38-
- job: container_build_${{ config.name }}
39-
displayName: Build ${{ config.name }} Image
40-
pool:
41-
name: ${{ config.pool }}
42-
vmImage: ${{ config.vmImage }}
43-
steps:
44-
- ${{ if config.prepareScript }}:
45-
- pwsh: |
46-
./${{ config.prepareScript }}
47-
displayName: "Run prep script"
48-
- task: Docker@2
49-
displayName: Build ${{ config.name }}:${{ parameters.ImageTag }}
50-
inputs:
51-
command: build
52-
Dockerfile: ${{ config.dockerFile }}
53-
tags: ${{ parameters.ImageTag }}
54-
arguments: '-t ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ parameters.ImageTag }} ${{ config.AdditionalDockerArgs }}'
55-
56-
- ${{ if parameters.Publish }}:
57-
- task: Docker@2
58-
displayName: Push ${{ config.name }}:${{ parameters.ImageTag }}
59-
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
60-
inputs:
61-
containerRegistry: ${{ parameters.ContainerRegistry }}
62-
repository: ${{ config.dockerRepo }}
63-
command: push
64-
tags: ${{ parameters.ImageTag }}
65-
66-
- ${{ if config.stableTags }}:
67-
- ${{ each stableTag in config.stableTags }}:
68-
- task: Docker@2
69-
displayName: Build ${{ config.name }}:${{ stableTag }}
70-
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
71-
inputs:
72-
command: build
73-
Dockerfile: ${{ config.dockerFile }}
74-
tags: ${{ stableTag }}
75-
arguments: '-t ${{ parameters.ContainerRegistry }}.azurecr.io/${{ config.dockerRepo }}:${{ stableTag }} ${{ config.exclude }}'
76-
77-
- task: Docker@2
78-
displayName: Push ${{ config.name }}:${{ stableTag }}
79-
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
80-
inputs:
81-
containerRegistry: ${{ parameters.ContainerRegistry }}
82-
repository: ${{ config.dockerRepo }}
83-
command: push
84-
tags: ${{ stableTag }}
85-
86-
- ${{ if and(parameters.ManifestDeployment, parameters.Publish) }}:
87-
- ${{ each deployment in parameters.ManifestDeployment }}:
88-
- job: container_build_${{ deployment.name }}
89-
displayName: Build ${{ deployment.name }} Manifest
90-
${{ if gt(length(parameters.DockerDeployments), 0) }}:
91-
dependsOn:
92-
- ${{ each config in parameters.DockerDeployments }}:
93-
- container_build_${{ config.name }}
94-
pool:
95-
name: azsdk-pool-mms-ubuntu-2204-general
96-
vmImage: ubuntu-22.04
97-
98-
steps:
99-
- task: Docker@2
100-
displayName: Login to ACR
101-
inputs:
102-
command: login
103-
containerRegistry: ${{ parameters.ContainerRegistry }}
104-
105-
- pwsh: |
106-
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
107-
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
108-
-DockerDeploymentJson $configurations `
109-
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
110-
-ImageTag "${{ parameters.ImageTag }}" `
111-
112-
Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
113-
displayName: Generate Manifest Variable
114-
115-
- pwsh: |
116-
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }} $(ManifestVariable)
117-
displayName: Generate Manifest
118-
119-
- pwsh: |
120-
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ parameters.ImageTag }}
121-
displayName: Upload Manifest
122-
123-
- ${{ if deployment.stableTags }}:
124-
- ${{ each stableTag in deployment.stableTags }}:
125-
- pwsh: |
126-
$configurations = '${{ convertToJson(parameters.DockerDeployments) }}' -replace '\\', '/'
127-
$assembledDependentTags = $(Build.SourcesDirectory)/eng/pipelines/templates/scripts/get-docker-manifest-input.ps1 `
128-
-DockerDeploymentJson $configurations `
129-
-ContainerRegistry "${{ parameters.ContainerRegistry }}" `
130-
-ImageTag "${{ stableTag }}" `
131-
132-
Write-Host "##vso[task.setvariable variable=ManifestVariable]$assembledDependentTags"
133-
displayName: Generate Manifest Variable
134-
135-
- pwsh: |
136-
docker manifest create ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }} $(ManifestVariable)
137-
displayName: Generate Manifest
138-
139-
- pwsh: |
140-
docker manifest push ${{ parameters.ContainerRegistry }}.azurecr.io/${{ deployment.dockerRepo }}:${{ stableTag }}
141-
displayName: Upload Manifest
33+
extends:
34+
template: /eng/pipelines/templates/stages/1es-redirect.yml
35+
parameters:
36+
stages:
37+
- template: /eng/pipelines/publish-docker-image-isolated.yml
38+
parameters: ${{ parameters }}

0 commit comments

Comments
 (0)