From a018a5551fa04b122cfa463d94a768c08a2f0470 Mon Sep 17 00:00:00 2001 From: Jaime Hidalgo Date: Wed, 18 Mar 2020 12:06:51 -0500 Subject: [PATCH 1/6] Add additional volumes feature to launch templates Now the user can specify additional volumes for its instances. This is specially useful when the cluster is going to be used for specific applications such as storage. Example 1 (Just required fields) additional_volumes = [{ block_device_name = "/dev/sdb", volume_size = 200, delete_on_termination = false }] Example 2 (Multiple additional volumes & All features) additional_volumes = [{ block_device_name = "/dev/sdb", volume_size = 200, volume_type = "io1" iops = 100 encrypted = true kms_key_id = "arn:aws:kms:us-east-1:89349:key/a4-ds-34" delete_on_termination = false },{ block_device_name = "/dev/sdc", volume_size = 500, delete_on_termination = false }] --- local.tf | 1 + workers_launch_template.tf | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/local.tf b/local.tf index 704b5cdcb2..371184582c 100644 --- a/local.tf +++ b/local.tf @@ -73,6 +73,7 @@ locals { root_encrypted = false # Whether the volume should be encrypted or not eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs + additional_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id, delete_on_termination. Optional values are grabbed from root volume or from defaults market_type = null # Settings for launch templates with mixed instances policy override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] # A list of override instance types for mixed instances policy diff --git a/workers_launch_template.tf b/workers_launch_template.tf index 52ec738588..aa98297417 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -351,6 +351,43 @@ resource "aws_launch_template" "workers_launch_template" { } } + dynamic "block_device_mappings" { + for_each = lookup(var.worker_groups_launch_template[count.index], "additional_volumes", lookup(local.workers_group_defaults, "additional_volumes", [])) + content { + device_name = block_device_mappings.value.block_device_name + + ebs { + volume_size = lookup( + block_device_mappings.value, + "volume_size", + local.workers_group_defaults["root_volume_size"], + ) + volume_type = lookup( + block_device_mappings.value, + "volume_type", + local.workers_group_defaults["root_volume_type"], + ) + iops = lookup( + block_device_mappings.value, + "iops", + local.workers_group_defaults["root_iops"], + ) + encrypted = lookup( + block_device_mappings.value, + "encrypted", + local.workers_group_defaults["root_encrypted"], + ) + kms_key_id = lookup( + block_device_mappings.value, + "kms_key_id", + local.workers_group_defaults["root_kms_key_id"], + ) + delete_on_termination = lookup(block_device_mappings.value, "delete_on_termination", true) + } + } + + } + tag_specifications { resource_type = "volume" From 4701e81c9297611f42e690760eac504461a3ade6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Hidalgo=20Garc=C3=ADa?= Date: Sun, 22 Mar 2020 23:33:06 -0500 Subject: [PATCH 2/6] Simplify expression to assign empty list as default for additional volumes As @barryib pointed out, locals already defaults an empty list, so it is no needed to specify it in the expression, making it more readable Co-Authored-By: Thierno IB. BARRY --- workers_launch_template.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers_launch_template.tf b/workers_launch_template.tf index aa98297417..ca8e2018a9 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -352,7 +352,7 @@ resource "aws_launch_template" "workers_launch_template" { } dynamic "block_device_mappings" { - for_each = lookup(var.worker_groups_launch_template[count.index], "additional_volumes", lookup(local.workers_group_defaults, "additional_volumes", [])) + for_each = lookup(var.worker_groups_launch_template[count.index], "additional_volumes", local.workers_group_defaults["additional_volumes"]) content { device_name = block_device_mappings.value.block_device_name From 4832c31d6768207c777d7211714a335c55c8a017 Mon Sep 17 00:00:00 2001 From: Jaime Hidalgo Date: Mon, 23 Mar 2020 12:42:34 -0500 Subject: [PATCH 3/6] feat: additional ebs volumes on worker groups launch configuration --- workers.tf | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/workers.tf b/workers.tf index ce2b27c601..baad9f0388 100644 --- a/workers.tf +++ b/workers.tf @@ -223,6 +223,37 @@ resource "aws_launch_configuration" "workers" { delete_on_termination = true } + dynamic "ebs_block_device" { + for_each = lookup(var.worker_groups[count.index], "additional_volumes", local.workers_group_defaults["additional_volumes"]) + device_name = ebs_block_device.value.block_device_name + volume_size = lookup( + ebs_block_device.value, + "volume_size", + local.workers_group_defaults["root_volume_size"], + ) + volume_type = lookup( + ebs_block_device.value, + "volume_type", + local.workers_group_defaults["root_volume_type"], + ) + iops = lookup( + ebs_block_device.value, + "iops", + local.workers_group_defaults["root_iops"], + ) + encrypted = lookup( + ebs_block_device.value, + "encrypted", + local.workers_group_defaults["root_encrypted"], + ) + kms_key_id = lookup( + ebs_block_device.value, + "kms_key_id", + local.workers_group_defaults["root_kms_key_id"], + ) + delete_on_termination = lookup(ebs_block_device.value, "delete_on_termination", true) + } + lifecycle { create_before_destroy = true } From 1dc40b472ca4ed35638d3fc234ded2df00320781 Mon Sep 17 00:00:00 2001 From: Jaime Hidalgo Date: Mon, 23 Mar 2020 12:48:20 -0500 Subject: [PATCH 4/6] additional_volumes is now in the generic section, not inside launch templates-only configuration --- local.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local.tf b/local.tf index 371184582c..ca1b374f06 100644 --- a/local.tf +++ b/local.tf @@ -64,6 +64,7 @@ locals { service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. platform = "linux" # Platform of workers. either "linux" or "windows" + additional_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id, delete_on_termination. Optional values are grabbed from root volume or from defaults # Settings for launch templates root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device @@ -73,7 +74,6 @@ locals { root_encrypted = false # Whether the volume should be encrypted or not eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs - additional_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id, delete_on_termination. Optional values are grabbed from root volume or from defaults market_type = null # Settings for launch templates with mixed instances policy override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] # A list of override instance types for mixed instances policy From ffa909992e22dd4d9816bb8a4cf777874b231459 Mon Sep 17 00:00:00 2001 From: Jaime Hidalgo Date: Mon, 23 Mar 2020 13:22:20 -0500 Subject: [PATCH 5/6] Fix additional volumes on launch configuration + LC don't support setting kms key --- local.tf | 2 +- workers.tf | 55 +++++++++++++++++++++++++++--------------------------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/local.tf b/local.tf index ca1b374f06..66446da71a 100644 --- a/local.tf +++ b/local.tf @@ -64,7 +64,7 @@ locals { service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. platform = "linux" # Platform of workers. either "linux" or "windows" - additional_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id, delete_on_termination. Optional values are grabbed from root volume or from defaults + additional_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults # Settings for launch templates root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device diff --git a/workers.tf b/workers.tf index baad9f0388..cabf4efbe8 100644 --- a/workers.tf +++ b/workers.tf @@ -224,34 +224,33 @@ resource "aws_launch_configuration" "workers" { } dynamic "ebs_block_device" { - for_each = lookup(var.worker_groups[count.index], "additional_volumes", local.workers_group_defaults["additional_volumes"]) - device_name = ebs_block_device.value.block_device_name - volume_size = lookup( - ebs_block_device.value, - "volume_size", - local.workers_group_defaults["root_volume_size"], - ) - volume_type = lookup( - ebs_block_device.value, - "volume_type", - local.workers_group_defaults["root_volume_type"], - ) - iops = lookup( - ebs_block_device.value, - "iops", - local.workers_group_defaults["root_iops"], - ) - encrypted = lookup( - ebs_block_device.value, - "encrypted", - local.workers_group_defaults["root_encrypted"], - ) - kms_key_id = lookup( - ebs_block_device.value, - "kms_key_id", - local.workers_group_defaults["root_kms_key_id"], - ) - delete_on_termination = lookup(ebs_block_device.value, "delete_on_termination", true) + for_each = lookup(var.worker_groups[count.index], "additional_volumes", local.workers_group_defaults["additional_volumes"]) + + content { + device_name = ebs_block_device.value.block_device_name + volume_size = lookup( + ebs_block_device.value, + "volume_size", + local.workers_group_defaults["root_volume_size"], + ) + volume_type = lookup( + ebs_block_device.value, + "volume_type", + local.workers_group_defaults["root_volume_type"], + ) + iops = lookup( + ebs_block_device.value, + "iops", + local.workers_group_defaults["root_iops"], + ) + encrypted = lookup( + ebs_block_device.value, + "encrypted", + local.workers_group_defaults["root_encrypted"], + ) + delete_on_termination = lookup(ebs_block_device.value, "delete_on_termination", true) + } + } lifecycle { From b09dca1819232382b95f5eb0aa40296c1548d9a2 Mon Sep 17 00:00:00 2001 From: Jaime Hidalgo Date: Mon, 23 Mar 2020 13:25:26 -0500 Subject: [PATCH 6/6] rename: additional_volumes to additional_ebs_volumes to be more clear and precise --- local.tf | 2 +- workers.tf | 2 +- workers_launch_template.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/local.tf b/local.tf index 66446da71a..4ea1b72ffb 100644 --- a/local.tf +++ b/local.tf @@ -64,7 +64,7 @@ locals { service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. platform = "linux" # Platform of workers. either "linux" or "windows" - additional_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults + additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults # Settings for launch templates root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. root_kms_key_id = "" # The KMS key to use when encrypting the root storage device diff --git a/workers.tf b/workers.tf index cabf4efbe8..2eed088c4f 100644 --- a/workers.tf +++ b/workers.tf @@ -224,7 +224,7 @@ resource "aws_launch_configuration" "workers" { } dynamic "ebs_block_device" { - for_each = lookup(var.worker_groups[count.index], "additional_volumes", local.workers_group_defaults["additional_volumes"]) + for_each = lookup(var.worker_groups[count.index], "additional_ebs_volumes", local.workers_group_defaults["additional_ebs_volumes"]) content { device_name = ebs_block_device.value.block_device_name diff --git a/workers_launch_template.tf b/workers_launch_template.tf index ca8e2018a9..4eea118167 100644 --- a/workers_launch_template.tf +++ b/workers_launch_template.tf @@ -352,7 +352,7 @@ resource "aws_launch_template" "workers_launch_template" { } dynamic "block_device_mappings" { - for_each = lookup(var.worker_groups_launch_template[count.index], "additional_volumes", local.workers_group_defaults["additional_volumes"]) + for_each = lookup(var.worker_groups_launch_template[count.index], "additional_ebs_volumes", local.workers_group_defaults["additional_ebs_volumes"]) content { device_name = block_device_mappings.value.block_device_name