Skip to content

Commit 39a5f6f

Browse files
zqingqing1zqingqing1
authored
enable cilium with private acr (#1088)
To make cilium image pulling more quicker, we leverage to use private acr feature which can cache the image at private registery. test passed: https://dev.azure.com/akstelescope/telescope/_build/results?buildId=57173&view=results
1 parent 69b6d22 commit 39a5f6f

File tree

12 files changed

+481
-22
lines changed

12 files changed

+481
-22
lines changed

‎modules/terraform/azure/acr/main.tf‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
locals {
2+
acr_config_map = { for i, acr in var.acr_config_list : tostring(i) => acr }
3+
acr_private_dns_zone_name = length(var.acr_config_list) > 0 ? try(var.acr_config_list[0].private_endpoint.private_dns_zone_name, "privatelink.azurecr.io") : "privatelink.azurecr.io"
4+
acr_private_dns_enabled = length([for acr in var.acr_config_list : acr if try(acr.private_endpoint != null, false)]) > 0
5+
6+
# Plan-known switch for whether an aks-cli role needs a kubelet identity / AcrPull grants.
7+
# This is intentionally computed only from input configuration (acr_config_list), not from
8+
# any resource IDs, to keep downstream `count`/`for_each` stable during planning.
9+
acr_pull_enabled_by_aks_cli_role = {
10+
for role in var.aks_cli_roles : role => length([
11+
for acr in var.acr_config_list : 1
12+
if contains(
13+
concat(try(acr.acrpull_aks_cli_roles, []), try(acr.contributor_aks_cli_roles, [])),
14+
role
15+
)
16+
]) > 0
17+
}
18+
19+
acr_private_dns_vnet_roles = distinct([
20+
for acr in var.acr_config_list : var.subnet_to_network_role[acr.private_endpoint.subnet_name]
21+
if try(acr.private_endpoint != null, false)
22+
])
23+
24+
# Terraform doesn't have regexreplace(); replace() supports regex when pattern is wrapped in /.../
25+
acr_default_name_prefix = substr(replace(lower("acr${var.scenario_name}${var.run_id}"), "/[^0-9a-z]/", ""), 0, 45)
26+
27+
acr_name_map = {
28+
for acr_key, acr in local.acr_config_map :
29+
acr_key => (acr.name != null ? acr.name : substr("${local.acr_default_name_prefix}${acr_key}", 0, 50))
30+
}
31+
32+
acr_id_map = {
33+
for acr_key, acr_name in local.acr_name_map :
34+
acr_key => format(
35+
"/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerRegistry/registries/%s",
36+
data.azurerm_client_config.current.subscription_id,
37+
var.resource_group_name,
38+
acr_name
39+
)
40+
}
41+
42+
acr_cache_rule_map = length(var.acr_config_list) > 0 ? merge([
43+
for acr_key, acr in local.acr_config_map : {
44+
for rule in try(acr.cache_rules, []) : "${acr_key}/${rule.name}" => {
45+
acr_key = acr_key
46+
name = rule.name
47+
source_repository = rule.source_repository
48+
target_repository = rule.target_repository
49+
credential_set_resource_id = try(rule.credential_set_resource_id, null)
50+
}
51+
}
52+
]...) : {}
53+
54+
# Plan-stable map form of ACR pull scopes, keyed by the ACR config map key.
55+
# Using stable keys avoids Terraform "Invalid for_each argument" when the list length
56+
# of scopes is unknown at plan time.
57+
acr_pull_scopes_map_by_aks_cli_role = {
58+
for role in var.aks_cli_roles : role => {
59+
for acr_key, acr in local.acr_config_map :
60+
acr_key => local.acr_id_map[acr_key]
61+
if contains(
62+
concat(try(acr.acrpull_aks_cli_roles, []), try(acr.contributor_aks_cli_roles, [])),
63+
role
64+
)
65+
}
66+
}
67+
68+
bootstrap_container_registry_resource_id_by_aks_cli_role = {
69+
for role, scopes_map in local.acr_pull_scopes_map_by_aks_cli_role : role => (
70+
length(scopes_map) > 0 ? scopes_map[sort(keys(scopes_map))[0]] : null
71+
)
72+
}
73+
}
74+
75+
data "azurerm_client_config" "current" {}
76+
77+
resource "azurerm_container_registry" "acr" {
78+
for_each = local.acr_config_map
79+
80+
name = local.acr_name_map[each.key]
81+
resource_group_name = var.resource_group_name
82+
location = var.location
83+
sku = each.value.sku
84+
admin_enabled = each.value.admin_enabled
85+
public_network_access_enabled = each.value.public_network_access_enabled
86+
tags = var.tags
87+
}
88+
89+
resource "azurerm_private_dns_zone" "acr" {
90+
count = local.acr_private_dns_enabled ? 1 : 0
91+
92+
name = local.acr_private_dns_zone_name
93+
resource_group_name = var.resource_group_name
94+
tags = var.tags
95+
}
96+
97+
resource "azurerm_private_dns_zone_virtual_network_link" "acr" {
98+
for_each = local.acr_private_dns_enabled ? { for role in local.acr_private_dns_vnet_roles : role => role } : {}
99+
100+
name = "acr-dns-link-${each.key}"
101+
resource_group_name = var.resource_group_name
102+
private_dns_zone_name = azurerm_private_dns_zone.acr[0].name
103+
virtual_network_id = var.vnet_ids_by_role[each.key]
104+
registration_enabled = false
105+
tags = var.tags
106+
}
107+
108+
resource "azurerm_private_endpoint" "acr" {
109+
for_each = { for k, acr in local.acr_config_map : k => acr if try(acr.private_endpoint != null, false) }
110+
111+
name = "acr-pe-${each.key}"
112+
location = var.location
113+
resource_group_name = var.resource_group_name
114+
subnet_id = var.subnet_ids_by_name[each.value.private_endpoint.subnet_name]
115+
tags = var.tags
116+
117+
private_service_connection {
118+
name = "acr-psc-${each.key}"
119+
private_connection_resource_id = azurerm_container_registry.acr[each.key].id
120+
subresource_names = ["registry"]
121+
is_manual_connection = false
122+
}
123+
124+
private_dns_zone_group {
125+
name = "acr-dns-${each.key}"
126+
private_dns_zone_ids = [azurerm_private_dns_zone.acr[0].id]
127+
}
128+
129+
depends_on = [azurerm_private_dns_zone_virtual_network_link.acr]
130+
}
131+
132+
resource "azapi_resource" "acr_cache_rule" {
133+
for_each = local.acr_cache_rule_map
134+
135+
type = "Microsoft.ContainerRegistry/registries/cacheRules@2023-07-01"
136+
name = each.value.name
137+
parent_id = azurerm_container_registry.acr[each.value.acr_key].id
138+
139+
body = {
140+
properties = merge(
141+
{
142+
sourceRepository = each.value.source_repository
143+
targetRepository = each.value.target_repository
144+
},
145+
each.value.credential_set_resource_id != null ? { credentialSetResourceId = each.value.credential_set_resource_id } : {}
146+
)
147+
}
148+
149+
depends_on = [azurerm_container_registry.acr]
150+
}

‎modules/terraform/azure/acr/output.tf‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
output "acr_pull_enabled_by_aks_cli_role" {
2+
description = "Map of aks-cli role to a plan-known boolean indicating whether to enable kubelet identity + ACR pull grants for that role."
3+
value = local.acr_pull_enabled_by_aks_cli_role
4+
}
5+
6+
output "acr_pull_scopes_map_by_aks_cli_role" {
7+
description = "Map of aks-cli role to map of stable keys -> ACR scope IDs to grant AcrPull on. Use this to drive plan-stable for_each downstream."
8+
value = local.acr_pull_scopes_map_by_aks_cli_role
9+
}
10+
11+
output "bootstrap_container_registry_resource_id_by_aks_cli_role" {
12+
description = "Map of aks-cli role to a single ACR resource ID (first pull scope) used for bootstrap_artifact_source."
13+
value = local.bootstrap_container_registry_resource_id_by_aks_cli_role
14+
}

‎modules/terraform/azure/acr/variables.tf‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
variable "acr_config_list" {
2+
description = "Optional list of Azure Container Registries (ACR) to create. Each entry can also enable a Private Endpoint + Private DNS integration (Private Link)."
3+
type = list(object({
4+
name = optional(string, null)
5+
sku = optional(string, "Premium")
6+
admin_enabled = optional(bool, false)
7+
public_network_access_enabled = optional(bool, true)
8+
9+
private_endpoint = optional(object({
10+
subnet_name = string
11+
private_dns_zone_name = optional(string, "privatelink.azurecr.io")
12+
}), null)
13+
14+
acrpull_aks_cli_roles = optional(list(string), [])
15+
contributor_aks_cli_roles = optional(list(string), [])
16+
17+
cache_rules = optional(list(object({
18+
name = string
19+
source_repository = string
20+
target_repository = string
21+
credential_set_resource_id = optional(string, null)
22+
})), [])
23+
}))
24+
default = []
25+
}
26+
27+
variable "resource_group_name" {
28+
type = string
29+
description = "Resource group name where ACR resources will be created."
30+
}
31+
32+
variable "location" {
33+
type = string
34+
description = "Azure region for ACR resources."
35+
}
36+
37+
variable "tags" {
38+
type = map(string)
39+
description = "Tags to apply to ACR resources."
40+
default = {}
41+
}
42+
43+
variable "scenario_name" {
44+
type = string
45+
description = "Scenario name used for default ACR name generation."
46+
}
47+
48+
variable "run_id" {
49+
type = string
50+
description = "Run ID used for default ACR name generation and resource group name conventions."
51+
}
52+
53+
variable "subnet_ids_by_name" {
54+
type = map(string)
55+
description = "Map of subnet name to subnet ID."
56+
}
57+
58+
variable "vnet_ids_by_role" {
59+
type = map(string)
60+
description = "Map of network role to virtual network ID."
61+
}
62+
63+
variable "subnet_to_network_role" {
64+
type = map(string)
65+
description = "Map of subnet name to network role (used for Private DNS VNet linking)."
66+
}
67+
68+
variable "aks_cli_roles" {
69+
type = list(string)
70+
description = "List of aks-cli roles to compute ACR pull scopes for."
71+
default = []
72+
}

‎modules/terraform/azure/acr/versions.tf‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
terraform {
2+
required_providers {
3+
azurerm = {
4+
source = "hashicorp/azurerm"
5+
}
6+
azapi = {
7+
source = "Azure/azapi"
8+
}
9+
}
10+
}

‎modules/terraform/azure/aks-cli/main.tf‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ locals {
44
format("%s=%s", key, value)
55
]
66

7+
acr_pull_scopes_for_each = (!var.aks_cli_config.dry_run && var.enable_kubelet_identity) ? var.acr_pull_scopes_map : {}
8+
79
extra_pool_map = {
810
for pool in var.aks_cli_config.extra_node_pool :
911
pool.name => pool
@@ -50,6 +52,7 @@ locals {
5052
null :
5153
try(var.subnets_map[var.aks_cli_config.subnet_name], null)
5254
)
55+
5356
api_server_subnet_id = (
5457
var.aks_cli_config.api_server_subnet_name == null ?
5558
null :
@@ -114,6 +117,17 @@ locals {
114117
)
115118
)
116119

120+
kubelet_identity_parameter = (!var.enable_kubelet_identity || var.aks_cli_config.dry_run) ? "" : format(
121+
"%s %s",
122+
"--assign-kubelet-identity",
123+
azurerm_user_assigned_identity.kubelet_identity[0].id,
124+
)
125+
126+
bootstrap_parameters = join(" ", compact([
127+
var.bootstrap_artifact_source != null ? format("--bootstrap-artifact-source %s", var.bootstrap_artifact_source) : null,
128+
var.bootstrap_container_registry_resource_id != null ? format("--bootstrap-container-registry-resource-id %s", var.bootstrap_container_registry_resource_id) : null,
129+
]))
130+
117131

118132
api_server_vnet_integration_parameter = (var.aks_cli_config.enable_apiserver_vnet_integration && local.api_server_subnet_id != null ?
119133
format(
@@ -165,11 +179,13 @@ locals {
165179
local.custom_configurations,
166180
"--no-ssh-key",
167181
local.kubernetes_version,
182+
local.bootstrap_parameters,
168183
local.optional_parameters,
169184
local.kms_parameters,
170185
local.disk_encryption_parameters,
171186
local.subnet_id_parameter,
172187
local.managed_identity_parameter,
188+
local.kubelet_identity_parameter,
173189
local.api_server_vnet_integration_parameter,
174190
local.aad_parameter,
175191
], local.default_node_pool_parameters))
@@ -202,6 +218,14 @@ resource "azurerm_user_assigned_identity" "userassignedidentity" {
202218
tags = var.tags
203219
}
204220

221+
resource "azurerm_user_assigned_identity" "kubelet_identity" {
222+
count = (!var.aks_cli_config.dry_run && var.enable_kubelet_identity) ? 1 : 0
223+
location = var.location
224+
name = "${var.aks_cli_config.aks_name}-kubelet-identity"
225+
resource_group_name = var.resource_group_name
226+
tags = var.tags
227+
}
228+
205229
resource "azurerm_role_assignment" "network_contributor" {
206230
count = var.aks_cli_config.managed_identity_name != null && var.aks_cli_config.subnet_name != null ? 1 : 0
207231
role_definition_name = "Network Contributor"
@@ -217,6 +241,24 @@ resource "azurerm_role_assignment" "network_contributor_api_server_subnet" {
217241
principal_id = azurerm_user_assigned_identity.userassignedidentity[0].principal_id
218242
}
219243

244+
# Grant AcrPull access to ACR for kubelet identity (node identity) BEFORE cluster creation.
245+
resource "azurerm_role_assignment" "acr_pull_kubelet" {
246+
for_each = local.acr_pull_scopes_for_each
247+
248+
scope = each.value
249+
role_definition_name = "AcrPull"
250+
principal_id = azurerm_user_assigned_identity.kubelet_identity[0].principal_id
251+
}
252+
253+
# If the cluster uses a user-assigned identity, it must be able to assign/use the kubelet identity.
254+
resource "azurerm_role_assignment" "managed_identity_operator_kubelet" {
255+
count = (!var.aks_cli_config.dry_run && var.enable_kubelet_identity && var.aks_cli_config.managed_identity_name != null) ? 1 : 0
256+
257+
scope = azurerm_user_assigned_identity.kubelet_identity[0].id
258+
role_definition_name = "Managed Identity Operator"
259+
principal_id = azurerm_user_assigned_identity.userassignedidentity[0].principal_id
260+
}
261+
220262
resource "terraform_data" "enable_aks_cli_preview_extension" {
221263
count = var.aks_cli_config.use_aks_preview_cli_extension == true ? 1 : 0
222264

@@ -247,7 +289,9 @@ resource "terraform_data" "aks_cli" {
247289
terraform_data.enable_aks_cli_preview_extension,
248290
azurerm_role_assignment.network_contributor,
249291
azurerm_role_assignment.network_contributor_api_server_subnet,
250-
azurerm_role_assignment.aks_identity_kms_roles
292+
azurerm_role_assignment.aks_identity_kms_roles,
293+
azurerm_role_assignment.acr_pull_kubelet,
294+
azurerm_role_assignment.managed_identity_operator_kubelet
251295
]
252296

253297
input = {
@@ -351,3 +395,4 @@ resource "azurerm_role_assignment" "des_reader_cluster" {
351395
local.aks_system_assigned_principal_id != null ? local.aks_system_assigned_principal_id : error("Unable to determine AKS system-assigned identity principalId via azapi; cannot grant DES Reader role.")
352396
)
353397
}
398+

‎modules/terraform/azure/aks-cli/variables.tf‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,30 @@ variable "aks_aad_enabled" {
4747
default = false
4848
}
4949

50+
variable "acr_pull_scopes_map" {
51+
description = "Map of stable keys to Azure resource IDs (scopes) to grant the AKS kubelet identity AcrPull access to. Prefer this over acr_pull_scopes to keep for_each keys plan-stable."
52+
type = map(string)
53+
default = {}
54+
}
55+
56+
variable "enable_kubelet_identity" {
57+
description = "Whether to create and assign a user-assigned kubelet identity (and grant it ACR pull access when acr_pull_scopes are provided). Set false to disable kubelet identity entirely."
58+
type = bool
59+
default = false
60+
}
61+
62+
variable "bootstrap_artifact_source" {
63+
description = "Optional value for --bootstrap-artifact-source (for example, Cache or Direct). If null, the flag is omitted."
64+
type = string
65+
default = null
66+
}
67+
68+
variable "bootstrap_container_registry_resource_id" {
69+
description = "Optional ACR resource ID to pass to --bootstrap-container-registry-resource-id. If null, the flag is omitted."
70+
type = string
71+
default = null
72+
}
73+
5074
variable "aks_cli_config" {
5175
type = object({
5276
role = string

0 commit comments

Comments
 (0)
⚡