Skip to content

Commit 4d3799e

Browse files
rajhollarajholla
authored
resource/aws_kinesis_firehose_delivery_stream: Add elasticsearch_configuration vpc_config configuration block (support Elasticsearch VPC configuration) (#13269)
Output from acceptance testing: ``` --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_s3KinesisStreamSource (102.19s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_s3basicWithTags (106.75s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3_DataFormatConversionConfiguration_ParquetSerDe_Empty (118.90s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3_DataFormatConversionConfiguration_OpenXJsonSerDe_Empty (123.17s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_s3basic (133.80s) --- FAIL: TestAccAWSKinesisFirehoseDeliveryStream_basic (146.83s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3_DataFormatConversionConfiguration_OrcSerDe_Empty (148.34s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_disappears (149.37s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3_DataFormatConversionConfiguration_HiveJsonSerDe_Empty (149.70s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3_DataFormatConversionConfiguration_Serializer_Update (152.19s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3_ExternalUpdate (163.95s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_s3WithCloudwatchLogging (167.63s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3basic (168.05s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3KmsKeyArn (173.94s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_s3ConfigUpdates (175.01s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3_ProcessingConfiguration_Empty (176.11s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3_ErrorOutputPrefix (180.04s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3_DataFormatConversionConfiguration_Deserializer_Update (187.17s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3_DataFormatConversionConfiguration_Enabled (197.56s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3_KinesisStreamSource (98.68s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_s3basicWithSSE (253.21s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_missingProcessingConfiguration (107.01s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_SplunkConfigUpdates (139.39s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ExtendedS3Updates (163.46s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_RedshiftConfigUpdates (340.62s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ElasticsearchConfigUpdates (991.76s) --- PASS: TestAccAWSKinesisFirehoseDeliveryStream_ElasticsearchWithVpcConfigUpdates (1574.83s) ```
1 parent e208508 commit 4d3799e

3 files changed

Lines changed: 392 additions & 1 deletion

‎aws/resource_aws_kinesis_firehose_delivery_stream.go‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,7 @@ func flattenFirehoseElasticsearchConfiguration(description *firehose.Elasticsear
183183
"index_name": aws.StringValue(description.IndexName),
184184
"s3_backup_mode": aws.StringValue(description.S3BackupMode),
185185
"index_rotation_period": aws.StringValue(description.IndexRotationPeriod),
186+
"vpc_config": flattenVpcConfiguration(description.VpcConfigurationDescription),
186187
"processing_configuration": flattenProcessingConfiguration(description.ProcessingConfiguration, aws.StringValue(description.RoleARN)),
187188
}
188189

@@ -198,6 +199,21 @@ func flattenFirehoseElasticsearchConfiguration(description *firehose.Elasticsear
198199
return []map[string]interface{}{m}
199200
}
200201

202+
func flattenVpcConfiguration(description *firehose.VpcConfigurationDescription) []map[string]interface{} {
203+
if description == nil {
204+
return []map[string]interface{}{}
205+
}
206+
207+
m := map[string]interface{}{
208+
"vpc_id": aws.StringValue(description.VpcId),
209+
"subnet_ids": flattenStringSet(description.SubnetIds),
210+
"security_group_ids": flattenStringSet(description.SecurityGroupIds),
211+
"role_arn": aws.StringValue(description.RoleARN),
212+
}
213+
214+
return []map[string]interface{}{m}
215+
}
216+
201217
func flattenFirehoseExtendedS3Configuration(description *firehose.ExtendedS3DestinationDescription) []map[string]interface{} {
202218
if description == nil {
203219
return []map[string]interface{}{}
@@ -1249,6 +1265,39 @@ func resourceAwsKinesisFirehoseDeliveryStream() *schema.Resource {
12491265
},
12501266
},
12511267

1268+
"vpc_config": {
1269+
Type: schema.TypeList,
1270+
Optional: true,
1271+
ForceNew: true,
1272+
MaxItems: 1,
1273+
Elem: &schema.Resource{
1274+
Schema: map[string]*schema.Schema{
1275+
"vpc_id": {
1276+
Type: schema.TypeString,
1277+
Computed: true,
1278+
},
1279+
"subnet_ids": {
1280+
Type: schema.TypeSet,
1281+
Required: true,
1282+
ForceNew: true,
1283+
Elem: &schema.Schema{Type: schema.TypeString},
1284+
},
1285+
"security_group_ids": {
1286+
Type: schema.TypeSet,
1287+
Required: true,
1288+
ForceNew: true,
1289+
Elem: &schema.Schema{Type: schema.TypeString},
1290+
},
1291+
"role_arn": {
1292+
Type: schema.TypeString,
1293+
Required: true,
1294+
ForceNew: true,
1295+
ValidateFunc: validateArn,
1296+
},
1297+
},
1298+
},
1299+
},
1300+
12521301
"cloudwatch_logging_options": cloudWatchLoggingOptionsSchema(),
12531302

12541303
"processing_configuration": processingConfigurationSchema(),
@@ -1792,6 +1841,21 @@ func extractCloudWatchLoggingConfiguration(s3 map[string]interface{}) *firehose.
17921841

17931842
}
17941843

1844+
func extractVpcConfiguration(es map[string]interface{}) *firehose.VpcConfiguration {
1845+
config := es["vpc_config"].([]interface{})
1846+
if len(config) == 0 {
1847+
return nil
1848+
}
1849+
1850+
vpcConfig := config[0].(map[string]interface{})
1851+
1852+
return &firehose.VpcConfiguration{
1853+
RoleARN: aws.String(vpcConfig["role_arn"].(string)),
1854+
SubnetIds: expandStringSet(vpcConfig["subnet_ids"].(*schema.Set)),
1855+
SecurityGroupIds: expandStringSet(vpcConfig["security_group_ids"].(*schema.Set)),
1856+
}
1857+
}
1858+
17951859
func extractPrefixConfiguration(s3 map[string]interface{}) *string {
17961860
if v, ok := s3["prefix"]; ok {
17971861
return aws.String(v.(string))
@@ -1900,6 +1964,10 @@ func createElasticsearchConfig(d *schema.ResourceData, s3Config *firehose.S3Dest
19001964
config.S3BackupMode = aws.String(s3BackupMode.(string))
19011965
}
19021966

1967+
if _, ok := es["vpc_config"]; ok {
1968+
config.VpcConfiguration = extractVpcConfiguration(es)
1969+
}
1970+
19031971
return config, nil
19041972
}
19051973

‎aws/resource_aws_kinesis_firehose_delivery_stream_test.go‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1103,6 +1103,78 @@ func TestAccAWSKinesisFirehoseDeliveryStream_ElasticsearchConfigUpdates(t *testi
11031103
})
11041104
}
11051105

1106+
func TestAccAWSKinesisFirehoseDeliveryStream_ElasticsearchWithVpcConfigUpdates(t *testing.T) {
1107+
var stream firehose.DeliveryStreamDescription
1108+
1109+
resourceName := "aws_kinesis_firehose_delivery_stream.test"
1110+
ri := acctest.RandInt()
1111+
rString := acctest.RandString(8)
1112+
funcName := fmt.Sprintf("aws_kinesis_firehose_delivery_stream_test_%s", rString)
1113+
policyName := fmt.Sprintf("tf_acc_policy_%s", rString)
1114+
roleName := fmt.Sprintf("tf_acc_role_%s", rString)
1115+
preConfigWithVpc := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_ElasticsearchVpcBasic,
1116+
ri, ri, ri, ri, ri)
1117+
1118+
postConfigWithVpc := testAccFirehoseAWSLambdaConfigBasic(funcName, policyName, roleName) +
1119+
fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_ElasticsearchVpcUpdate,
1120+
ri, ri, ri, ri, ri)
1121+
1122+
updatedElasticSearchConfig := &firehose.ElasticsearchDestinationDescription{
1123+
BufferingHints: &firehose.ElasticsearchBufferingHints{
1124+
IntervalInSeconds: aws.Int64(500),
1125+
},
1126+
ProcessingConfiguration: &firehose.ProcessingConfiguration{
1127+
Enabled: aws.Bool(true),
1128+
Processors: []*firehose.Processor{
1129+
{
1130+
Type: aws.String("Lambda"),
1131+
Parameters: []*firehose.ProcessorParameter{
1132+
{
1133+
ParameterName: aws.String("LambdaArn"),
1134+
ParameterValue: aws.String("valueNotTested"),
1135+
},
1136+
},
1137+
},
1138+
},
1139+
},
1140+
}
1141+
1142+
resource.ParallelTest(t, resource.TestCase{
1143+
PreCheck: func() { testAccPreCheck(t); testAccPreCheckIamServiceLinkedRoleEs(t) },
1144+
Providers: testAccProviders,
1145+
CheckDestroy: testAccCheckKinesisFirehoseDeliveryStreamDestroy,
1146+
Steps: []resource.TestStep{
1147+
{
1148+
Config: preConfigWithVpc,
1149+
Check: resource.ComposeTestCheckFunc(
1150+
testAccCheckKinesisFirehoseDeliveryStreamExists(resourceName, &stream),
1151+
testAccCheckAWSKinesisFirehoseDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil),
1152+
resource.TestCheckResourceAttrPair(resourceName, "elasticsearch_configuration.0.vpc_config.0.vpc_id", "aws_vpc.elasticsearch_in_vpc", "id"),
1153+
resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.vpc_config.0.subnet_ids.#", "2"),
1154+
resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.vpc_config.0.security_group_ids.#", "2"),
1155+
resource.TestCheckResourceAttrPair(resourceName, "elasticsearch_configuration.0.vpc_config.0.role_arn", "aws_iam_role.firehose", "arn"),
1156+
),
1157+
},
1158+
{
1159+
ResourceName: resourceName,
1160+
ImportState: true,
1161+
ImportStateVerify: true,
1162+
},
1163+
{
1164+
Config: postConfigWithVpc,
1165+
Check: resource.ComposeTestCheckFunc(
1166+
testAccCheckKinesisFirehoseDeliveryStreamExists(resourceName, &stream),
1167+
testAccCheckAWSKinesisFirehoseDeliveryStreamAttributes(&stream, nil, nil, nil, updatedElasticSearchConfig, nil),
1168+
resource.TestCheckResourceAttrPair(resourceName, "elasticsearch_configuration.0.vpc_config.0.vpc_id", "aws_vpc.elasticsearch_in_vpc", "id"),
1169+
resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.vpc_config.0.subnet_ids.#", "2"),
1170+
resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.vpc_config.0.security_group_ids.#", "2"),
1171+
resource.TestCheckResourceAttrPair(resourceName, "elasticsearch_configuration.0.vpc_config.0.role_arn", "aws_iam_role.firehose", "arn"),
1172+
),
1173+
},
1174+
},
1175+
})
1176+
}
1177+
11061178
// Regression test for https://github.com/terraform-providers/terraform-provider-aws/issues/1657
11071179
func TestAccAWSKinesisFirehoseDeliveryStream_missingProcessingConfiguration(t *testing.T) {
11081180
var stream firehose.DeliveryStreamDescription
@@ -2452,6 +2524,108 @@ EOF
24522524
}
24532525
`
24542526

2527+
// ElasticSearch associated with VPC
2528+
var testAccKinesisFirehoseDeliveryStreamBaseElasticsearchVpcConfig = testAccKinesisFirehoseDeliveryStreamBaseConfig + `
2529+
data "aws_availability_zones" "available" {
2530+
state = "available"
2531+
2532+
filter {
2533+
name = "opt-in-status"
2534+
values = ["opt-in-not-required"]
2535+
}
2536+
}
2537+
2538+
resource "aws_vpc" "elasticsearch_in_vpc" {
2539+
cidr_block = "192.168.0.0/22"
2540+
2541+
tags = {
2542+
Name = "terraform-testacc-elasticsearch-domain-in-vpc"
2543+
}
2544+
}
2545+
2546+
resource "aws_subnet" "first" {
2547+
vpc_id = aws_vpc.elasticsearch_in_vpc.id
2548+
availability_zone = data.aws_availability_zones.available.names[0]
2549+
cidr_block = "192.168.0.0/24"
2550+
2551+
tags = {
2552+
Name = "tf-acc-elasticsearch-domain-in-vpc-first"
2553+
}
2554+
}
2555+
2556+
resource "aws_subnet" "second" {
2557+
vpc_id = aws_vpc.elasticsearch_in_vpc.id
2558+
availability_zone = data.aws_availability_zones.available.names[1]
2559+
cidr_block = "192.168.1.0/24"
2560+
2561+
tags = {
2562+
Name = "tf-acc-elasticsearch-domain-in-vpc-second"
2563+
}
2564+
}
2565+
2566+
resource "aws_security_group" "first" {
2567+
vpc_id = aws_vpc.elasticsearch_in_vpc.id
2568+
}
2569+
2570+
resource "aws_security_group" "second" {
2571+
vpc_id = aws_vpc.elasticsearch_in_vpc.id
2572+
}
2573+
2574+
resource "aws_elasticsearch_domain" "test_cluster" {
2575+
domain_name = "es-test-%d"
2576+
2577+
cluster_config {
2578+
instance_count = 2
2579+
zone_awareness_enabled = true
2580+
instance_type = "t2.small.elasticsearch"
2581+
}
2582+
2583+
ebs_options {
2584+
ebs_enabled = true
2585+
volume_size = 10
2586+
}
2587+
2588+
vpc_options {
2589+
security_group_ids = [aws_security_group.first.id, aws_security_group.second.id]
2590+
subnet_ids = [aws_subnet.first.id, aws_subnet.second.id]
2591+
}
2592+
}
2593+
2594+
resource "aws_iam_role_policy" "firehose-elasticsearch" {
2595+
name = "elasticsearch"
2596+
role = aws_iam_role.firehose.id
2597+
policy = <<EOF
2598+
{
2599+
"Version":"2012-10-17",
2600+
"Statement":[
2601+
{
2602+
"Effect":"Allow",
2603+
"Action":[
2604+
"es:*"
2605+
],
2606+
"Resource":[
2607+
"${aws_elasticsearch_domain.test_cluster.arn}",
2608+
"${aws_elasticsearch_domain.test_cluster.arn}/*"
2609+
]
2610+
},
2611+
{
2612+
"Effect":"Allow",
2613+
"Action":[
2614+
"ec2:Describe*",
2615+
"ec2:CreateNetworkInterface",
2616+
"ec2:CreateNetworkInterfacePermission",
2617+
"ec2:DeleteNetworkInterface"
2618+
],
2619+
"Resource":[
2620+
"*"
2621+
]
2622+
}
2623+
]
2624+
}
2625+
EOF
2626+
}
2627+
`
2628+
24552629
var testAccKinesisFirehoseDeliveryStreamConfig_ElasticsearchBasic = testAccKinesisFirehoseDeliveryStreamBaseElasticsearchConfig + `
24562630
resource "aws_kinesis_firehose_delivery_stream" "test" {
24572631
depends_on = [aws_iam_role_policy.firehose-elasticsearch]
@@ -2473,6 +2647,30 @@ resource "aws_kinesis_firehose_delivery_stream" "test" {
24732647
}
24742648
`
24752649

2650+
var testAccKinesisFirehoseDeliveryStreamConfig_ElasticsearchVpcBasic = testAccKinesisFirehoseDeliveryStreamBaseElasticsearchVpcConfig + `
2651+
resource "aws_kinesis_firehose_delivery_stream" "test" {
2652+
depends_on = [aws_iam_role_policy.firehose-elasticsearch]
2653+
2654+
name = "terraform-kinesis-firehose-es-%d"
2655+
destination = "elasticsearch"
2656+
s3_configuration {
2657+
role_arn = aws_iam_role.firehose.arn
2658+
bucket_arn = aws_s3_bucket.bucket.arn
2659+
}
2660+
elasticsearch_configuration {
2661+
domain_arn = aws_elasticsearch_domain.test_cluster.arn
2662+
role_arn = aws_iam_role.firehose.arn
2663+
index_name = "test"
2664+
type_name = "test"
2665+
2666+
vpc_config {
2667+
subnet_ids = [aws_subnet.first.id, aws_subnet.second.id]
2668+
security_group_ids = [aws_security_group.first.id, aws_security_group.second.id]
2669+
role_arn = aws_iam_role.firehose.arn
2670+
}
2671+
}
2672+
}
2673+
`
24762674
var testAccKinesisFirehoseDeliveryStreamConfig_ElasticsearchUpdate = testAccKinesisFirehoseDeliveryStreamBaseElasticsearchConfig + `
24772675
resource "aws_kinesis_firehose_delivery_stream" "test" {
24782676
depends_on = [aws_iam_role_policy.firehose-elasticsearch]
@@ -2508,6 +2706,40 @@ resource "aws_kinesis_firehose_delivery_stream" "test" {
25082706
}
25092707
`
25102708

2709+
var testAccKinesisFirehoseDeliveryStreamConfig_ElasticsearchVpcUpdate = testAccKinesisFirehoseDeliveryStreamBaseElasticsearchVpcConfig + `
2710+
resource "aws_kinesis_firehose_delivery_stream" "test" {
2711+
depends_on = [aws_iam_role_policy.firehose-elasticsearch]
2712+
2713+
name = "terraform-kinesis-firehose-es-%d"
2714+
destination = "elasticsearch"
2715+
s3_configuration {
2716+
role_arn = aws_iam_role.firehose.arn
2717+
bucket_arn = aws_s3_bucket.bucket.arn
2718+
}
2719+
elasticsearch_configuration {
2720+
domain_arn = aws_elasticsearch_domain.test_cluster.arn
2721+
role_arn = aws_iam_role.firehose.arn
2722+
index_name = "test"
2723+
type_name = "test"
2724+
buffering_interval = 500
2725+
vpc_config {
2726+
subnet_ids = [aws_subnet.first.id, aws_subnet.second.id]
2727+
security_group_ids = [aws_security_group.first.id, aws_security_group.second.id]
2728+
role_arn = aws_iam_role.firehose.arn
2729+
}
2730+
processing_configuration {
2731+
enabled = false
2732+
processors {
2733+
type = "Lambda"
2734+
parameters {
2735+
parameter_name = "LambdaArn"
2736+
parameter_value = "${aws_lambda_function.lambda_function_test.arn}:$LATEST"
2737+
}
2738+
}
2739+
}
2740+
}
2741+
}`
2742+
25112743
func testAccKinesisFirehoseDeliveryStreamConfig_missingProcessingConfiguration(rInt int) string {
25122744
return fmt.Sprintf(`
25132745
data "aws_partition" "current" {}

0 commit comments

Comments
 (0)
⚡