Add sleep for network rule application

Author: benbp

eng/common/TestResources/New-TestResources.ps1

@@ -117,6 +117,7 @@ param (
     $NewTestResourcesRemainingArguments
 )
 
+. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1)
 . $PSScriptRoot/TestResources-Helpers.ps1
 . $PSScriptRoot/SubConfig-Helpers.ps1
 

eng/common/TestResources/Remove-TestResources.ps1

@@ -257,8 +257,8 @@ $verifyDeleteScript = {
 # Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty.
 $purgeableResources = Get-PurgeableGroupResources $ResourceGroupName
 
-SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -Override -CI:$CI
-Remove-WormStorageAccounts -GroupPrefix $ResourceGroupName
+SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -Override -CI:$CI
+Remove-WormStorageAccounts -GroupPrefix $ResourceGroupName -CI:$CI
 
 Log "Deleting resource group '$ResourceGroupName'"
 if ($Force -and !$purgeableResources) {

eng/common/scripts/Helpers/Resource-Helpers.ps1

@@ -213,7 +213,8 @@ function Wait-PurgeableResourceJob {
 function Remove-WormStorageAccounts() {
   [CmdletBinding(SupportsShouldProcess = $True)]
   param(
-    [string]$GroupPrefix
+    [string]$GroupPrefix,
+    [switch]$CI
   )
 
   $ErrorActionPreference = 'Stop'
@@ -222,8 +223,8 @@ function Remove-WormStorageAccounts() {
   # DO NOT REMOVE THIS
   # We call this script from live test pipelines as well, and a string mismatch/error could blow away
   # some static storage accounts we rely on
-  if (!$groupPrefix -or !$GroupPrefix.StartsWith('rg-')) {
-    throw "The -GroupPrefix parameter must start with 'rg-'"
+  if (!$groupPrefix -or ($CI -and !$GroupPrefix.StartsWith('rg-'))) {
+    throw "The -GroupPrefix parameter must not be empty, or must start with 'rg-' in CI contexts"
   }
 
   $groups = Get-AzResourceGroup | Where-Object { $_.ResourceGroupName.StartsWith($GroupPrefix) } | Where-Object { $_.ProvisioningState -ne 'Deleting' }
@@ -274,12 +275,14 @@ function Remove-WormStorageAccounts() {
           try {
             Write-Host "Removing immutability policies - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)"
             $null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy
-          } catch {}
+          }
+          catch {}
 
           try {
             $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force
             $succeeded = $true
-          } catch {
+          }
+          catch {
             Write-Warning "Failed to remove blobs - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)"
             Write-Warning $_
           }
@@ -314,6 +317,7 @@ function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowI
   $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" }
   # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access
   if ($storageAccounts) {
+    $appliedRule = $false
     foreach ($account in $storageAccounts) {
       $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name
       if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) {
@@ -322,13 +326,15 @@ function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowI
         if ($CI -and $env:PoolSubnet) {
           Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)"
           Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet }
+          $appliedRule = $true
         }
         elseif ($AllowIpRanges) {
           Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges"
           $ipRanges = $AllowIpRanges | ForEach-Object {
             @{ Action = 'allow'; IPAddressOrRange = $_ }
           }
           Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null }
+          $appliedRule = $true
         }
         elseif (!$CI) {
           Write-Host "Enabling access to '$($account.Name)' from client IP"
@@ -343,9 +349,14 @@ function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowI
             }
           }
           Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null }
+          $appliedRule = $true
         }
       }
     }
+    if ($appliedRule) {
+      Write-Host "Sleeping for 15 seconds to allow network rules to take effect"
+      Start-Sleep 15
+    }
   }
 }
 

eng/scripts/live-test-resource-cleanup.ps1

@@ -436,7 +436,7 @@ function DeleteAndPurgeGroups([array]$toDelete) {
         # can be left around which prevent deletion.
         if ($rg.Tags?.ContainsKey('ServiceDirectory') -and $rg.Tags.ServiceDirectory -like '*storage*') {
           SetStorageNetworkAccessRules -ResourceGroupName $rg.ResourceGroupName -Override -CI:($null -ne $env:SYSTEM_TEAMPROJECTID)
-          Remove-WormStorageAccounts -GroupPrefix $rg.ResourceGroupName
+          Remove-WormStorageAccounts -GroupPrefix $rg.ResourceGroupName -CI:($null -ne $env:SYSTEM_TEAMPROJECTID)
         } else {
           Write-Host ($rg | Remove-AzResourceGroup -Force -AsJob).Name
         }