resource/aws_subnet: an unexpected new value for was present, but now absent.

[ialidzhikov]

Community Note

• Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request
• Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
• If you are interested in working on this issue or have submitted a pull request, please leave a comment

Terraform Version

terraform version - 0.12.9
provider-aws version - 2.26.0

Affected Resource(s)

• aws_subnet
• aws_vpc
• aws_route

Terraform Configuration Files

    provider "aws" {
      access_key = "${var.ACCESS_KEY_ID}"
      secret_key = "${var.SECRET_ACCESS_KEY}"
      region     = "eu-west-1"
    }

    resource "aws_vpc_dhcp_options" "vpc_dhcp_options" {
      domain_name         = "eu-west-1.compute.internal"
      domain_name_servers = ["AmazonProvidedDNS"]
    }

    resource "aws_vpc" "vpc" {
      cidr_block           = "10.250.0.0/16"
      enable_dns_support   = true
      enable_dns_hostnames = true
    }

    resource "aws_vpc_dhcp_options_association" "vpc_dhcp_options_association" {
      vpc_id          = "${aws_vpc.vpc.id}"
      dhcp_options_id = "${aws_vpc_dhcp_options.vpc_dhcp_options.id}"
    }

    resource "aws_default_security_group" "default" {
      vpc_id = "${aws_vpc.vpc.id}"
    }

    resource "aws_internet_gateway" "igw" {
      vpc_id = "${aws_vpc.vpc.id}"
    }

    resource "aws_route_table" "routetable_main" {
      vpc_id = "${aws_vpc.vpc.id}"
    }

    resource "aws_route" "public" {
      route_table_id         = "${aws_route_table.routetable_main.id}"
      destination_cidr_block = "0.0.0.0/0"
      gateway_id             = "${aws_internet_gateway.igw.id}"
    }

    resource "aws_security_group" "nodes" {
      name        = "foo-nodes"
      description = "Security group for nodes"
      vpc_id      = "${aws_vpc.vpc.id}"
    }

    resource "aws_security_group_rule" "nodes_self" {
      type              = "ingress"
      from_port         = 0
      to_port           = 0
      protocol          = "-1"
      self              = true
      security_group_id = "${aws_security_group.nodes.id}"
    }

    resource "aws_security_group_rule" "nodes_tcp_all" {
      type              = "ingress"
      from_port         = 30000
      to_port           = 32767
      protocol          = "tcp"
      cidr_blocks       = ["0.0.0.0/0"]
      security_group_id = "${aws_security_group.nodes.id}"
    }

    resource "aws_security_group_rule" "nodes_udp_all" {
      type              = "ingress"
      from_port         = 30000
      to_port           = 32767
      protocol          = "udp"
      cidr_blocks       = ["0.0.0.0/0"]
      security_group_id = "${aws_security_group.nodes.id}"
    }

    resource "aws_security_group_rule" "nodes_egress_all" {
      type              = "egress"
      from_port         = 0
      to_port           = 0
      protocol          = "-1"
      cidr_blocks       = ["0.0.0.0/0"]
      security_group_id = "${aws_security_group.nodes.id}"
    }


    resource "aws_subnet" "nodes_z0" {
      vpc_id            = "${aws_vpc.vpc.id}"
      cidr_block        = "10.250.0.0/19"
      availability_zone = "eu-west-1c"
    }

    output "subnet_nodes_z0" {
      value = "${aws_subnet.nodes_z0.id}"
    }

    resource "aws_subnet" "private_utility_z0" {
      vpc_id            = "${aws_vpc.vpc.id}"
      cidr_block        = "10.250.112.0/22"
      availability_zone = "eu-west-1c"
    }

    resource "aws_security_group_rule" "nodes_tcp_internal_z0" {
      type              = "ingress"
      from_port         = 30000
      to_port           = 32767
      protocol          = "tcp"
      cidr_blocks       = ["10.250.112.0/22"]
      security_group_id = "${aws_security_group.nodes.id}"
    }

    resource "aws_security_group_rule" "nodes_udp_internal_z0" {
      type              = "ingress"
      from_port         = 30000
      to_port           = 32767
      protocol          = "udp"
      cidr_blocks       = ["10.250.112.0/22"]
      security_group_id = "${aws_security_group.nodes.id}"
    }

    resource "aws_subnet" "public_utility_z0" {
      vpc_id            = "${aws_vpc.vpc.id}"
      cidr_block        = "10.250.96.0/22"
      availability_zone = "eu-west-1c"
    }

    output "subnet_public_utility_z0" {
      value = "${aws_subnet.public_utility_z0.id}"
    }

    resource "aws_security_group_rule" "nodes_tcp_public_z0" {
      type              = "ingress"
      from_port         = 30000
      to_port           = 32767
      protocol          = "tcp"
      cidr_blocks       = ["10.250.96.0/22"]
      security_group_id = "${aws_security_group.nodes.id}"
    }

    resource "aws_security_group_rule" "nodes_udp_public_z0" {
      type              = "ingress"
      from_port         = 30000
      to_port           = 32767
      protocol          = "udp"
      cidr_blocks       = ["10.250.96.0/22"]
      security_group_id = "${aws_security_group.nodes.id}"
    }

    resource "aws_eip" "eip_natgw_z0" {
      vpc = true
    }

    resource "aws_nat_gateway" "natgw_z0" {
      allocation_id = "${aws_eip.eip_natgw_z0.id}"
      subnet_id     = "${aws_subnet.public_utility_z0.id}"
    }

    resource "aws_route_table" "routetable_private_utility_z0" {
      vpc_id = "${aws_vpc.vpc.id}"
    }

    resource "aws_route" "private_utility_z0_nat" {
      route_table_id         = "${aws_route_table.routetable_private_utility_z0.id}"
      destination_cidr_block = "0.0.0.0/0"
      nat_gateway_id         = "${aws_nat_gateway.natgw_z0.id}"

      timeouts {
        create = "5m"
      }
    }

    resource "aws_route_table_association" "routetable_private_utility_z0_association_private_utility_z0" {
      subnet_id      = "${aws_subnet.private_utility_z0.id}"
      route_table_id = "${aws_route_table.routetable_private_utility_z0.id}"
    }

    resource "aws_route_table_association" "routetable_main_association_public_utility_z0" {
      subnet_id      = "${aws_subnet.public_utility_z0.id}"
      route_table_id = "${aws_route_table.routetable_main.id}"
    }

    resource "aws_route_table_association" "routetable_private_utility_z0_association_nodes_z0" {
      subnet_id      = "${aws_subnet.nodes_z0.id}"
      route_table_id = "${aws_route_table.routetable_private_utility_z0.id}"
    }


    //=====================================================================
    //= IAM instance profiles
    //=====================================================================

    resource "aws_iam_role" "bastions" {
      name = "foo-bastions"
      path = "/"

      assume_role_policy = <<EOF
    {
      "Version": "2012-10-17",
      "Statement": [
        {
          "Effect": "Allow",
          "Principal": {
            "Service": "ec2.amazonaws.com"
          },
          "Action": "sts:AssumeRole"
        }
      ]
    }
    EOF
    }

    resource "aws_iam_instance_profile" "bastions" {
      name = "foo-bastions"
      role = "${aws_iam_role.bastions.name}"
    }

    resource "aws_iam_role_policy" "bastions" {
      name = "foo-bastions"
      role = "${aws_iam_role.bastions.id}"

      policy = <<EOF
    {
      "Version": "2012-10-17",
      "Statement": [
        {
          "Effect": "Allow",
          "Action": [
            "ec2:DescribeRegions"
          ],
          "Resource": [
            "*"
          ]
        }
      ]
    }
    EOF
    }

    resource "aws_iam_role" "nodes" {
      name = "foo-nodes"
      path = "/"

      assume_role_policy = <<EOF
    {
      "Version": "2012-10-17",
      "Statement": [
        {
          "Effect": "Allow",
          "Principal": {
            "Service": "ec2.amazonaws.com"
          },
          "Action": "sts:AssumeRole"
        }
      ]
    }
    EOF
    }

    resource "aws_iam_instance_profile" "nodes" {
      name = "foo-nodes"
      role = "${aws_iam_role.nodes.name}"
    }

    resource "aws_iam_role_policy" "nodes" {
      name = "foo-nodes"
      role = "${aws_iam_role.nodes.id}"

      policy = <<EOF
    {
      "Version": "2012-10-17",
      "Statement": [
        {
          "Effect": "Allow",
          "Action": [
            "ec2:DescribeInstances"
          ],
          "Resource": [
            "*"
          ]
        },
        {
          "Effect": "Allow",
          "Action": [
            "ecr:GetAuthorizationToken",
            "ecr:BatchCheckLayerAvailability",
            "ecr:GetDownloadUrlForLayer",
            "ecr:GetRepositoryPolicy",
            "ecr:DescribeRepositories",
            "ecr:ListImages",
            "ecr:BatchGetImage"
          ],
          "Resource": [
            "*"
          ]
        }
      ]
    }
    EOF
    }

    //=====================================================================
    //= EC2 Key Pair
    //=====================================================================

    resource "aws_key_pair" "kubernetes" {
      key_name   = "foo-ssh-publickey"
      public_key = "ssh-rsa bar"
    }

    //=====================================================================
    //= Output variables
    //=====================================================================

    output "vpc_id" {
      value = "${aws_vpc.vpc.id}"
    }

    output "iamInstanceProfileNodes" {
      value = "${aws_iam_instance_profile.nodes.name}"
    }

    output "keyName" {
      value = "${aws_key_pair.kubernetes.key_name}"
    }

    output "security_group_nodes" {
      value = "${aws_security_group.nodes.id}"
    }

    output "nodes_role_arn" {
      value = "${aws_iam_role.nodes.arn}"
    }

Debug Output

Panic Output

Expected Behavior

Actual Behavior

Initializing the backend...

Initializing provider plugins...

The following providers do not have any version constraints in configuration,
so the latest version was installed.

To prevent automatic upgrades to new major versions that may contain breaking
changes, it is recommended to add version = "..." constraints to the
corresponding provider blocks in configuration, with the constraint strings
suggested below.

* provider.aws: version = "~> 2.26"


Terraform has been successfully initialized!

You may now begin working with Terraform. Try running "terraform plan" to see
any changes that are required for your infrastructure. All Terraform commands
should now work.

If you ever set or change modules or backend configuration for Terraform,
rerun this command to reinitialize your working directory. If you forget, other
commands will detect it and remind you to do so if necessary.

aws_key_pair.kubernetes: Creating...
aws_iam_role.bastions: Creating...
aws_iam_role.nodes: Creating...
aws_vpc.vpc: Creating...
aws_key_pair.kubernetes: Creation complete after 0s [id=foo-ssh-publickey]
aws_vpc_dhcp_options.vpc_dhcp_options: Creating...
aws_vpc_dhcp_options.vpc_dhcp_options: Creation complete after 1s [id=dopt-05e353c455684bd20]
aws_eip.eip_natgw_z0: Creating...
aws_iam_role.nodes: Creation complete after 1s [id=foo-nodes]
aws_iam_instance_profile.nodes: Creating...
aws_iam_role.bastions: Creation complete after 1s [id=foo-bastions]
aws_iam_role_policy.nodes: Creating...
aws_eip.eip_natgw_z0: Creation complete after 0s [id=eipalloc-034584994a6456a50]
aws_iam_role_policy.bastions: Creating...
aws_vpc.vpc: Creation complete after 1s [id=vpc-06797e045ce6cd937]
aws_iam_instance_profile.bastions: Creating...
aws_iam_role_policy.nodes: Creation complete after 1s [id=foo-nodes:foo-nodes]
aws_subnet.public_utility_z0: Creating...
aws_iam_role_policy.bastions: Creation complete after 1s [id=foo-bastions:foo-bastions]
aws_subnet.private_utility_z0: Creating...
aws_iam_instance_profile.nodes: Creation complete after 1s [id=foo-nodes]
aws_internet_gateway.igw: Creating...
aws_route_table.routetable_main: Creating...
aws_internet_gateway.igw: Creation complete after 0s [id=igw-08e178cb0ec8205a1]
aws_route_table.routetable_main: Creation complete after 0s [id=rtb-0076a5ae49f37d94f]
aws_iam_instance_profile.bastions: Creation complete after 2s [id=foo-bastions]
aws_subnet.private_utility_z0: Creation complete after 1s [id=subnet-08bc13f9841ee7d8b]
aws_default_security_group.default: Creating...
aws_vpc_dhcp_options_association.vpc_dhcp_options_association: Creating...
aws_route_table.routetable_private_utility_z0: Creating...
aws_subnet.nodes_z0: Creating...
aws_vpc_dhcp_options_association.vpc_dhcp_options_association: Creation complete after 0s [id=dopt-05e353c455684bd20-vpc-06797e045ce6cd937]
aws_security_group.nodes: Creating...
aws_route_table.routetable_private_utility_z0: Creation complete after 0s [id=rtb-0667ef879c08fcc62]
aws_route.public: Creating...
aws_route.public: Creation complete after 0s [id=r-rtb-0076a5ae49f37d94f1080289494]
aws_route_table_association.routetable_private_utility_z0_association_private_utility_z0: Creating...
aws_route_table_association.routetable_private_utility_z0_association_private_utility_z0: Creation complete after 0s [id=rtbassoc-030ca4ee1c5844779]
aws_default_security_group.default: Creation complete after 0s [id=sg-0a607654130892970]
aws_security_group.nodes: Creation complete after 0s [id=sg-070307b96a244e080]
aws_security_group_rule.nodes_tcp_internal_z0: Creating...
aws_security_group_rule.nodes_udp_public_z0: Creating...
aws_security_group_rule.nodes_udp_all: Creating...
aws_subnet.nodes_z0: Creation complete after 1s [id=subnet-01ab14436e58bbca9]
aws_security_group_rule.nodes_udp_internal_z0: Creating...
aws_security_group_rule.nodes_tcp_internal_z0: Creation complete after 1s [id=sgrule-183128067]
aws_security_group_rule.nodes_egress_all: Creating...
aws_security_group_rule.nodes_udp_all: Creation complete after 1s [id=sgrule-86703399]
aws_security_group_rule.nodes_tcp_all: Creating...
aws_security_group_rule.nodes_udp_public_z0: Creation complete after 1s [id=sgrule-2144436421]
aws_security_group_rule.nodes_self: Creating...
aws_security_group_rule.nodes_udp_internal_z0: Creation complete after 1s [id=sgrule-3649760152]
aws_security_group_rule.nodes_tcp_public_z0: Creating...
aws_security_group_rule.nodes_egress_all: Creation complete after 1s [id=sgrule-2510112059]
aws_route_table_association.routetable_private_utility_z0_association_nodes_z0: Creating...
aws_route_table_association.routetable_private_utility_z0_association_nodes_z0: Creation complete after 0s [id=rtbassoc-024a4bd3bd8f5545d]
aws_security_group_rule.nodes_tcp_all: Creation complete after 1s [id=sgrule-2078613439]
aws_security_group_rule.nodes_self: Creation complete after 1s [id=sgrule-3504389354]
aws_security_group_rule.nodes_tcp_public_z0: Creation complete after 1s [id=sgrule-3382632610]

Error: Provider produced inconsistent result after apply
When applying changes to aws_subnet.public_utility_z0, provider \"aws\" produced
an unexpected new value for was present, but now absent.


This is a bug in the provider, which should be reported in the provider's own
issue tracker.

Steps to Reproduce

1. terraform apply the configuration from above

2. Ensure that some times the apply fails with

Error: Provider produced inconsistent result after apply
When applying changes to aws_subnet.public_utility_z0, provider \"aws\" produced
an unexpected new value for was present, but now absent.


This is a bug in the provider, which should be reported in the provider's own
issue tracker.

Important Factoids

References

• #0000

[ialidzhikov]

@bflad, @ewbankkit, @ryndaniels, do you have idea why this happens?

[ialidzhikov]

We see the same error during terraform apply also for aws_vpc and aws_route. For example:

Provider produced inconsistent result after apply
When applying changes to aws_vpc.vpc, provider \"aws\" produced an unexpected
new value for was present, but now absent.
This is a bug in the provider, which should be reported in the provider's own
issue tracker.

[ewbankkit]

Similar:

• Provider produced inconsistent result after apply - aws_vpc_endpoint_route_table_association.s3_vpc_endpoint #12449

This is most likely an eventual consistency issue - The relevant resource (subnet, VPC, ...) is created successfully but when Terraform then attempts to read the resource back, the AWS APIs report that the resource does not exist.