diff --git a/.github/workflows/event-processor.yml b/.github/workflows/event-processor.yml index 8bb1d3a1bd..06092e395a 100644 --- a/.github/workflows/event-processor.yml +++ b/.github/workflows/event-processor.yml @@ -58,7 +58,7 @@ jobs: run: > dotnet tool install Azure.Sdk.Tools.GitHubEventProcessor - --version 1.0.0-dev.20240610.2 + --version 1.0.0-dev.20240708.1 --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json --global shell: bash @@ -114,7 +114,7 @@ jobs: run: > dotnet tool install Azure.Sdk.Tools.GitHubEventProcessor - --version 1.0.0-dev.20240610.2 + --version 1.0.0-dev.20240708.1 --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json --global shell: bash diff --git a/.github/workflows/scheduled-event-processor.yml b/.github/workflows/scheduled-event-processor.yml index db36be4e56..a20efcfcb9 100644 --- a/.github/workflows/scheduled-event-processor.yml +++ b/.github/workflows/scheduled-event-processor.yml @@ -39,7 +39,7 @@ jobs: run: > dotnet tool install Azure.Sdk.Tools.GitHubEventProcessor - --version 1.0.0-dev.20240610.2 + --version 1.0.0-dev.20240708.1 --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json --global shell: bash diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 633cad9cbf..21c218a116 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -216,11 +216,13 @@ "rehydrated", "Reitz", "retriable", + "rschu", "rtti", "rwxrw", "sasia", "sbom", "Schonberger", + "Schulze", "scus", "SDDL", "sdpath", diff --git a/README.md b/README.md index 8fbe1a5736..cefb367405 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ vcpkg add port azure-identity-cpp azure-storage-blobs-cpp } ``` -### Configure project files +### Configure CMake project files - Replace the contents of `CMakeLists.txt` with the following: @@ -57,6 +57,12 @@ add_executable(HelloWorld helloworld.cpp) target_link_libraries(HelloWorld PRIVATE Azure::azure-identity Azure::azure-storage-blobs) ``` +### Installing Azure SDK packages in a Visual Studio MSBuild (.vcxproj) project + +Here's a walkthrough video on how to install the Azure SDK packages, using vcpkg, into an MSBuild project in VS: https://aka.ms/azsdk/cpp/gettingstarted-vcpkg-msbuild-video + +See the [vcpkg documentation](https://learn.microsoft.com/en-us/vcpkg/get_started/get-started-msbuild?pivots=shell-cmd) for more details. + ### Additional methods for installing and configuring diff --git a/cmake-modules/AzureVcpkg.cmake b/cmake-modules/AzureVcpkg.cmake index 0b936d7767..6c9e1dedf1 100644 --- a/cmake-modules/AzureVcpkg.cmake +++ b/cmake-modules/AzureVcpkg.cmake @@ -18,7 +18,7 @@ macro(az_vcpkg_integrate) message("AZURE_SDK_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.") # GET VCPKG FROM SOURCE # User can set env var AZURE_SDK_VCPKG_COMMIT to pick the VCPKG commit to fetch - set(VCPKG_COMMIT_STRING 8150939b69720adc475461978e07c2d2bf5fb76e) # default SDK tested commit + set(VCPKG_COMMIT_STRING 5312e9f976e89b256954f571433e34f783dd2d12) # default SDK tested commit if(DEFINED ENV{AZURE_SDK_VCPKG_COMMIT}) message("AZURE_SDK_VCPKG_COMMIT is defined. Using that instead of the default.") set(VCPKG_COMMIT_STRING "$ENV{AZURE_SDK_VCPKG_COMMIT}") # default SDK tested commit diff --git a/eng/common/InterdependencyGraph.html b/eng/common/InterdependencyGraph.html deleted file mode 100644 index 21f78563ed..0000000000 --- a/eng/common/InterdependencyGraph.html +++ /dev/null @@ -1,356 +0,0 @@ - - - -Interdependency Graph - - - - - - - - -
-

Dependency Graph

- - -
-
-
- - - diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 6ee09ff3b2..6ccf55a781 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -117,6 +117,8 @@ param ( $NewTestResourcesRemainingArguments ) +. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1) +. $PSScriptRoot/TestResources-Helpers.ps1 . $PSScriptRoot/SubConfig-Helpers.ps1 if (!$ServicePrincipalAuth) { @@ -131,272 +133,6 @@ if (!$PSBoundParameters.ContainsKey('ErrorAction')) { $ErrorActionPreference = 'Stop' } -function Log($Message) -{ - Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) -} - -# vso commands are specially formatted log lines that are parsed by Azure Pipelines -# to perform additional actions, most commonly marking values as secrets. -# https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands -function LogVsoCommand([string]$message) -{ - if (!$CI -or $SuppressVsoCommands) { - return - } - Write-Host $message -} - -function Retry([scriptblock] $Action, [int] $Attempts = 5) -{ - $attempt = 0 - $sleep = 5 - - while ($attempt -lt $Attempts) { - try { - $attempt++ - return $Action.Invoke() - } catch { - if ($attempt -lt $Attempts) { - $sleep *= 2 - - Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..." - Start-Sleep -Seconds $sleep - } else { - throw - } - } - } -} - -# NewServicePrincipalWrapper creates an object from an AAD graph or Microsoft Graph service principal object type. -# This is necessary to work around breaking changes introduced in Az version 7.0.0: -# https://azure.microsoft.com/en-us/updates/update-your-apps-to-use-microsoft-graph-before-30-june-2022/ -function NewServicePrincipalWrapper([string]$subscription, [string]$resourceGroup, [string]$displayName) -{ - if ((Get-Module Az.Resources).Version -eq "5.3.0") { - # https://github.com/Azure/azure-powershell/issues/17040 - # New-AzAdServicePrincipal calls will fail with: - # "You cannot call a method on a null-valued expression." - Write-Warning "Az.Resources version 5.3.0 is not supported. Please update to >= 5.3.1" - Write-Warning "Update-Module Az.Resources -RequiredVersion 5.3.1" - exit 1 - } - - try { - $servicePrincipal = Retry { - New-AzADServicePrincipal -Role "Owner" -Scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName" -DisplayName $displayName - } - } catch { - # The underlying error "The directory object quota limit for the Principal has been exceeded" gets overwritten by the module trying - # to call New-AzADApplication with a null object instead of stopping execution, which makes this case hard to diagnose because it prints the following: - # "Cannot bind argument to parameter 'ObjectId' because it is an empty string." - # Provide a more helpful diagnostic prompt to the user if appropriate: - $totalApps = (Get-AzADApplication -OwnedApplication).Length - $msg = "App Registrations owned by you total $totalApps and may exceed the max quota (likely around 135)." + ` - "`nTry removing some at https://ms.portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/RegisteredApps" + ` - " or by running the following command to remove apps created by this script:" + ` - "`n Get-AzADApplication -DisplayNameStartsWith '$baseName' | Remove-AzADApplication" + ` - "`nNOTE: You may need to wait for the quota number to be updated after removing unused applications." - Write-Warning $msg - throw - } - - $spPassword = "" - $appId = "" - if (Get-Member -Name "Secret" -InputObject $servicePrincipal -MemberType property) { - Write-Verbose "Using legacy PSADServicePrincipal object type from AAD graph API" - # Secret property exists on PSADServicePrincipal type from AAD graph in Az # module versions < 7.0.0 - $spPassword = $servicePrincipal.Secret - $appId = $servicePrincipal.ApplicationId - } else { - if ((Get-Module Az.Resources).Version -eq "5.1.0") { - Write-Verbose "Creating password and credential for service principal via MS Graph API" - Write-Warning "Please update Az.Resources to >= 5.2.0 by running 'Update-Module Az'" - # Microsoft graph objects (Az.Resources version == 5.1.0) do not provision a secret on creation so it must be added separately. - # Submitting a password credential object without specifying a password will result in one being generated on the server side. - $password = New-Object -TypeName "Microsoft.Azure.PowerShell.Cmdlets.Resources.MSGraph.Models.ApiV10.MicrosoftGraphPasswordCredential" - $password.DisplayName = "Password for $displayName" - $credential = Retry { New-AzADSpCredential -PasswordCredentials $password -ServicePrincipalObject $servicePrincipal -ErrorAction 'Stop' } - $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force - $appId = $servicePrincipal.AppId - } else { - Write-Verbose "Creating service principal credential via MS Graph API" - # In 5.2.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the - # parameter set was changed making the above call fail due to a missing ServicePrincipalId parameter. - $credential = Retry { $servicePrincipal | New-AzADSpCredential -ErrorAction 'Stop' } - $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force - $appId = $servicePrincipal.AppId - } - } - - return @{ - AppId = $appId - ApplicationId = $appId - # This is the ObjectId/OID but most return objects use .Id so keep it consistent to prevent confusion - Id = $servicePrincipal.Id - DisplayName = $servicePrincipal.DisplayName - Secret = $spPassword - } -} - -function LoadCloudConfig([string] $env) -{ - $configPath = "$PSScriptRoot/clouds/$env.json" - if (!(Test-Path $configPath)) { - Write-Warning "Could not find cloud configuration for environment '$env'" - return @{} - } - - $config = Get-Content $configPath | ConvertFrom-Json -AsHashtable - return $config -} - -function MergeHashes([hashtable] $source, [psvariable] $dest) -{ - foreach ($key in $source.Keys) { - if ($dest.Value.Contains($key) -and $dest.Value[$key] -ne $source[$key]) { - Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " + - "to new value '$($source[$key])'") - } - $dest.Value[$key] = $source[$key] - } -} - -function BuildBicepFile([System.IO.FileSystemInfo] $file) -{ - if (!(Get-Command bicep -ErrorAction Ignore)) { - Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See aka.ms/bicep-install" - throw - } - - $tmp = $env:TEMP ? $env:TEMP : [System.IO.Path]::GetTempPath() - $templateFilePath = Join-Path $tmp "$ResourceType-resources.$(New-Guid).compiled.json" - - # Az can deploy bicep files natively, but by compiling here it becomes easier to parse the - # outputted json for mismatched parameter declarations. - bicep build $file.FullName --outfile $templateFilePath - if ($LASTEXITCODE) { - Write-Error "Failure building bicep file '$($file.FullName)'" - throw - } - - return $templateFilePath -} - -function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) { - $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName - # Add default values - $deploymentOutputs = [Ordered]@{ - "${serviceDirectoryPrefix}SUBSCRIPTION_ID" = $azContext.Subscription.Id; - "${serviceDirectoryPrefix}RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; - "${serviceDirectoryPrefix}LOCATION" = $resourceGroup.Location; - "${serviceDirectoryPrefix}ENVIRONMENT" = $azContext.Environment.Name; - "${serviceDirectoryPrefix}AZURE_AUTHORITY_HOST" = $azContext.Environment.ActiveDirectoryAuthority; - "${serviceDirectoryPrefix}RESOURCE_MANAGER_URL" = $azContext.Environment.ResourceManagerUrl; - "${serviceDirectoryPrefix}SERVICE_MANAGEMENT_URL" = $azContext.Environment.ServiceManagementUrl; - "AZURE_SERVICE_DIRECTORY" = $serviceName.ToUpperInvariant(); - } - - if ($ServicePrincipalAuth) { - $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_ID"] = $TestApplicationId; - $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_SECRET"] = $TestApplicationSecret; - $deploymentOutputs["${serviceDirectoryPrefix}TENANT_ID"] = $azContext.Tenant.Id; - } - - MergeHashes $environmentVariables $(Get-Variable deploymentOutputs) - - foreach ($key in $deployment.Outputs.Keys) { - $variable = $deployment.Outputs[$key] - - # Work around bug that makes the first few characters of environment variables be lowercase. - $key = $key.ToUpperInvariant() - - if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') { - $deploymentOutputs[$key] = $variable.Value - } - } - - # Force capitalization of all keys to avoid Azure Pipelines confusion with - # variable auto-capitalization and OS env var capitalization differences - $capitalized = @{} - foreach ($item in $deploymentOutputs.GetEnumerator()) { - $capitalized[$item.Name.ToUpperInvariant()] = $item.Value - } - - return $capitalized -} - -function SetDeploymentOutputs( - [string]$serviceName, - [object]$azContext, - [object]$deployment, - [object]$templateFile, - [hashtable]$environmentVariables = @{} -) { - $deploymentEnvironmentVariables = $environmentVariables.Clone() - $deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables - - if ($OutFile) { - if (!$IsWindows) { - Write-Host 'File option is supported only on Windows' - } - - $outputFile = "$($templateFile.originalFilePath).env" - - $environmentText = $deploymentOutputs | ConvertTo-Json; - $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) - $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) - - Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force - - Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" - } else { - if (!$CI) { - # Write an extra new line to isolate the environment variables for easy reading. - Log "Persist the following environment variables based on your detected shell ($shell):`n" - } - - # Write overwrite warnings first, since local execution prints a runnable command to export variables - foreach ($key in $deploymentOutputs.Keys) { - if ([Environment]::GetEnvironmentVariable($key)) { - Write-Warning "Deployment outputs will overwrite pre-existing environment variable '$key'" - } - } - - # Marking values as secret by allowed keys below is not sufficient, as there may be outputs set in the ARM/bicep - # file that re-mark those values as secret (since all user-provided deployment outputs are treated as secret by default). - # This variable supports a second check on not marking previously allowed keys/values as secret. - $notSecretValues = @() - foreach ($key in $deploymentOutputs.Keys) { - $value = $deploymentOutputs[$key] - $deploymentEnvironmentVariables[$key] = $value - - if ($CI) { - if (ShouldMarkValueAsSecret $serviceName $key $value $notSecretValues) { - # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. - # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. - LogVsoCommand "##vso[task.setvariable variable=_$key;issecret=true;]$value" - Write-Host "Setting variable as secret '$key'" - } else { - Write-Host "Setting variable '$key': $value" - $notSecretValues += $value - } - LogVsoCommand "##vso[task.setvariable variable=$key;]$value" - } else { - Write-Host ($shellExportFormat -f $key, $value) - } - } - - if ($key) { - # Isolate the environment variables for easy reading. - Write-Host "`n" - $key = $null - } - } - - return $deploymentEnvironmentVariables, $deploymentOutputs -} # Support actions to invoke on exit. $exitActions = @({ @@ -843,31 +579,7 @@ try { -templateFile $templateFile ` -environmentVariables $EnvironmentVariables - $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } - # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access - if ($storageAccounts) { - foreach ($account in $storageAccounts) { - $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name - if ($rules -and $rules.DefaultAction -eq "Allow") { - Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default" - Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny } - if ($CI -and $env:PoolSubnet) { - Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" - Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } - } elseif ($AllowIpRanges) { - Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" - $ipRanges = $AllowIpRanges | ForEach-Object { - @{ Action = 'allow'; IPAddressOrRange = $_ } - } - Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } - } elseif (!$CI) { - Write-Host "Enabling access to '$($account.Name)' from client IP" - $clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site - Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } - } - } - } - } + SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI $postDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath "$ResourceType-resources-post.ps1" if (Test-Path $postDeploymentScript) { diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index 490b41b8eb..12411c4ee2 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -61,6 +61,19 @@ param ( [Parameter()] [switch] $ServicePrincipalAuth, + # List of CIDR ranges to add to specific resource firewalls, e.g. @(10.100.0.0/16, 10.200.0.0/16) + [Parameter()] + [ValidateCount(0,399)] + [Validatescript({ + foreach ($range in $PSItem) { + if ($range -like '*/31' -or $range -like '*/32') { + throw "Firewall IP Ranges cannot contain a /31 or /32 CIDR" + } + } + return $true + })] + [array] $AllowIpRanges = @(), + [Parameter()] [switch] $Force, @@ -69,6 +82,9 @@ param ( $RemoveTestResourcesRemainingArguments ) +. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1) +. (Join-Path $PSScriptRoot TestResources-Helpers.ps1) + # By default stop for any error. if (!$PSBoundParameters.ContainsKey('ErrorAction')) { $ErrorActionPreference = 'Stop' @@ -241,6 +257,9 @@ $verifyDeleteScript = { # Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty. $purgeableResources = Get-PurgeableGroupResources $ResourceGroupName +SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -SetFirewall -CI:$CI +Remove-WormStorageAccounts -GroupPrefix $ResourceGroupName -CI:$CI + Log "Deleting resource group '$ResourceGroupName'" if ($Force -and !$purgeableResources) { Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force -AsJob diff --git a/eng/common/TestResources/TestResources-Helpers.ps1 b/eng/common/TestResources/TestResources-Helpers.ps1 new file mode 100644 index 0000000000..6dee017aec --- /dev/null +++ b/eng/common/TestResources/TestResources-Helpers.ps1 @@ -0,0 +1,267 @@ +function Log($Message) { + Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) +} + +# vso commands are specially formatted log lines that are parsed by Azure Pipelines +# to perform additional actions, most commonly marking values as secrets. +# https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands +function LogVsoCommand([string]$message) { + if (!$CI -or $SuppressVsoCommands) { + return + } + Write-Host $message +} + +function Retry([scriptblock] $Action, [int] $Attempts = 5) { + $attempt = 0 + $sleep = 5 + + while ($attempt -lt $Attempts) { + try { + $attempt++ + return $Action.Invoke() + } + catch { + if ($attempt -lt $Attempts) { + $sleep *= 2 + + Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..." + Start-Sleep -Seconds $sleep + } + else { + throw + } + } + } +} + +# NewServicePrincipalWrapper creates an object from an AAD graph or Microsoft Graph service principal object type. +# This is necessary to work around breaking changes introduced in Az version 7.0.0: +# https://azure.microsoft.com/en-us/updates/update-your-apps-to-use-microsoft-graph-before-30-june-2022/ +function NewServicePrincipalWrapper([string]$subscription, [string]$resourceGroup, [string]$displayName) { + if ((Get-Module Az.Resources).Version -eq "5.3.0") { + # https://github.com/Azure/azure-powershell/issues/17040 + # New-AzAdServicePrincipal calls will fail with: + # "You cannot call a method on a null-valued expression." + Write-Warning "Az.Resources version 5.3.0 is not supported. Please update to >= 5.3.1" + Write-Warning "Update-Module Az.Resources -RequiredVersion 5.3.1" + exit 1 + } + + try { + $servicePrincipal = Retry { + New-AzADServicePrincipal -Role "Owner" -Scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName" -DisplayName $displayName + } + } + catch { + # The underlying error "The directory object quota limit for the Principal has been exceeded" gets overwritten by the module trying + # to call New-AzADApplication with a null object instead of stopping execution, which makes this case hard to diagnose because it prints the following: + # "Cannot bind argument to parameter 'ObjectId' because it is an empty string." + # Provide a more helpful diagnostic prompt to the user if appropriate: + $totalApps = (Get-AzADApplication -OwnedApplication).Length + $msg = "App Registrations owned by you total $totalApps and may exceed the max quota (likely around 135)." + ` + "`nTry removing some at https://ms.portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/RegisteredApps" + ` + " or by running the following command to remove apps created by this script:" + ` + "`n Get-AzADApplication -DisplayNameStartsWith '$baseName' | Remove-AzADApplication" + ` + "`nNOTE: You may need to wait for the quota number to be updated after removing unused applications." + Write-Warning $msg + throw + } + + $spPassword = "" + $appId = "" + if (Get-Member -Name "Secret" -InputObject $servicePrincipal -MemberType property) { + Write-Verbose "Using legacy PSADServicePrincipal object type from AAD graph API" + # Secret property exists on PSADServicePrincipal type from AAD graph in Az # module versions < 7.0.0 + $spPassword = $servicePrincipal.Secret + $appId = $servicePrincipal.ApplicationId + } + else { + if ((Get-Module Az.Resources).Version -eq "5.1.0") { + Write-Verbose "Creating password and credential for service principal via MS Graph API" + Write-Warning "Please update Az.Resources to >= 5.2.0 by running 'Update-Module Az'" + # Microsoft graph objects (Az.Resources version == 5.1.0) do not provision a secret on creation so it must be added separately. + # Submitting a password credential object without specifying a password will result in one being generated on the server side. + $password = New-Object -TypeName "Microsoft.Azure.PowerShell.Cmdlets.Resources.MSGraph.Models.ApiV10.MicrosoftGraphPasswordCredential" + $password.DisplayName = "Password for $displayName" + $credential = Retry { New-AzADSpCredential -PasswordCredentials $password -ServicePrincipalObject $servicePrincipal -ErrorAction 'Stop' } + $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force + $appId = $servicePrincipal.AppId + } + else { + Write-Verbose "Creating service principal credential via MS Graph API" + # In 5.2.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the + # parameter set was changed making the above call fail due to a missing ServicePrincipalId parameter. + $credential = Retry { $servicePrincipal | New-AzADSpCredential -ErrorAction 'Stop' } + $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force + $appId = $servicePrincipal.AppId + } + } + + return @{ + AppId = $appId + ApplicationId = $appId + # This is the ObjectId/OID but most return objects use .Id so keep it consistent to prevent confusion + Id = $servicePrincipal.Id + DisplayName = $servicePrincipal.DisplayName + Secret = $spPassword + } +} + +function LoadCloudConfig([string] $env) { + $configPath = "$PSScriptRoot/clouds/$env.json" + if (!(Test-Path $configPath)) { + Write-Warning "Could not find cloud configuration for environment '$env'" + return @{} + } + + $config = Get-Content $configPath | ConvertFrom-Json -AsHashtable + return $config +} + +function MergeHashes([hashtable] $source, [psvariable] $dest) { + foreach ($key in $source.Keys) { + if ($dest.Value.Contains($key) -and $dest.Value[$key] -ne $source[$key]) { + Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " + + "to new value '$($source[$key])'") + } + $dest.Value[$key] = $source[$key] + } +} + +function BuildBicepFile([System.IO.FileSystemInfo] $file) { + if (!(Get-Command bicep -ErrorAction Ignore)) { + Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See aka.ms/bicep-install" + throw + } + + $tmp = $env:TEMP ? $env:TEMP : [System.IO.Path]::GetTempPath() + $templateFilePath = Join-Path $tmp "$ResourceType-resources.$(New-Guid).compiled.json" + + # Az can deploy bicep files natively, but by compiling here it becomes easier to parse the + # outputted json for mismatched parameter declarations. + bicep build $file.FullName --outfile $templateFilePath + if ($LASTEXITCODE) { + Write-Error "Failure building bicep file '$($file.FullName)'" + throw + } + + return $templateFilePath +} + +function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) { + $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName + # Add default values + $deploymentOutputs = [Ordered]@{ + "${serviceDirectoryPrefix}SUBSCRIPTION_ID" = $azContext.Subscription.Id; + "${serviceDirectoryPrefix}RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; + "${serviceDirectoryPrefix}LOCATION" = $resourceGroup.Location; + "${serviceDirectoryPrefix}ENVIRONMENT" = $azContext.Environment.Name; + "${serviceDirectoryPrefix}AZURE_AUTHORITY_HOST" = $azContext.Environment.ActiveDirectoryAuthority; + "${serviceDirectoryPrefix}RESOURCE_MANAGER_URL" = $azContext.Environment.ResourceManagerUrl; + "${serviceDirectoryPrefix}SERVICE_MANAGEMENT_URL" = $azContext.Environment.ServiceManagementUrl; + "AZURE_SERVICE_DIRECTORY" = $serviceName.ToUpperInvariant(); + } + + if ($ServicePrincipalAuth) { + $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_ID"] = $TestApplicationId; + $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_SECRET"] = $TestApplicationSecret; + $deploymentOutputs["${serviceDirectoryPrefix}TENANT_ID"] = $azContext.Tenant.Id; + } + + MergeHashes $environmentVariables $(Get-Variable deploymentOutputs) + + foreach ($key in $deployment.Outputs.Keys) { + $variable = $deployment.Outputs[$key] + + # Work around bug that makes the first few characters of environment variables be lowercase. + $key = $key.ToUpperInvariant() + + if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') { + $deploymentOutputs[$key] = $variable.Value + } + } + + # Force capitalization of all keys to avoid Azure Pipelines confusion with + # variable auto-capitalization and OS env var capitalization differences + $capitalized = @{} + foreach ($item in $deploymentOutputs.GetEnumerator()) { + $capitalized[$item.Name.ToUpperInvariant()] = $item.Value + } + + return $capitalized +} + +function SetDeploymentOutputs( + [string]$serviceName, + [object]$azContext, + [object]$deployment, + [object]$templateFile, + [hashtable]$environmentVariables = @{} +) { + $deploymentEnvironmentVariables = $environmentVariables.Clone() + $deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables + + if ($OutFile) { + if (!$IsWindows) { + Write-Host 'File option is supported only on Windows' + } + + $outputFile = "$($templateFile.originalFilePath).env" + + $environmentText = $deploymentOutputs | ConvertTo-Json; + $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) + $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + + Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force + + Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" + } + else { + if (!$CI) { + # Write an extra new line to isolate the environment variables for easy reading. + Log "Persist the following environment variables based on your detected shell ($shell):`n" + } + + # Write overwrite warnings first, since local execution prints a runnable command to export variables + foreach ($key in $deploymentOutputs.Keys) { + if ([Environment]::GetEnvironmentVariable($key)) { + Write-Warning "Deployment outputs will overwrite pre-existing environment variable '$key'" + } + } + + # Marking values as secret by allowed keys below is not sufficient, as there may be outputs set in the ARM/bicep + # file that re-mark those values as secret (since all user-provided deployment outputs are treated as secret by default). + # This variable supports a second check on not marking previously allowed keys/values as secret. + $notSecretValues = @() + foreach ($key in $deploymentOutputs.Keys) { + $value = $deploymentOutputs[$key] + $deploymentEnvironmentVariables[$key] = $value + + if ($CI) { + if (ShouldMarkValueAsSecret $serviceName $key $value $notSecretValues) { + # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. + # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. + LogVsoCommand "##vso[task.setvariable variable=_$key;issecret=true;]$value" + Write-Host "Setting variable as secret '$key'" + } + else { + Write-Host "Setting variable '$key': $value" + $notSecretValues += $value + } + LogVsoCommand "##vso[task.setvariable variable=$key;]$value" + } + else { + Write-Host ($shellExportFormat -f $key, $value) + } + } + + if ($key) { + # Isolate the environment variables for easy reading. + Write-Host "`n" + $key = $null + } + } + + return $deploymentEnvironmentVariables, $deploymentOutputs +} diff --git a/eng/common/TestResources/build-test-resource-config.yml b/eng/common/TestResources/build-test-resource-config.yml index c64664f852..e3dd306449 100644 --- a/eng/common/TestResources/build-test-resource-config.yml +++ b/eng/common/TestResources/build-test-resource-config.yml @@ -8,7 +8,7 @@ parameters: # EnvVars is used to help diagnose variable conflict issues early - name: EnvVars type: object - default: null + default: {} - name: SubscriptionConfigurationFilePaths type: object default: null @@ -16,6 +16,7 @@ parameters: steps: - task: AzurePowerShell@5 displayName: Set Pipeline Subnet Info + condition: and(succeeded(), ne(variables['Pool'], '')) env: ${{ parameters.EnvVars }} inputs: azureSubscription: azure-sdk-tests @@ -67,6 +68,12 @@ steps: if ($subConfigFilesRaw) { $subConfigFiles = $subConfigFilesRaw | ConvertFrom-Json -AsHashtable foreach ($file in $subConfigFiles) { + # In some cases, $file could be an empty string. Get-Content will fail + # if $file is an empty string, so skip those cases. + if (!$file) { + continue + } + Write-Host "Merging sub config from file: $file" $subConfig = Get-Content $file | ConvertFrom-Json -AsHashtable $finalConfig = UpdateSubscriptionConfiguration $finalConfig $subConfig diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml index b877d72139..025e90dd4c 100644 --- a/eng/common/TestResources/remove-test-resources.yml +++ b/eng/common/TestResources/remove-test-resources.yml @@ -29,7 +29,9 @@ steps: displayName: Remove test resources condition: and(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), ne(variables['Skip.RemoveTestResources'], 'true')) continueOnError: true - env: ${{ parameters.EnvVars }} + env: + PoolSubnet: $(PoolSubnet) + ${{ insert }}: ${{ parameters.EnvVars }} inputs: azureSubscription: ${{ parameters.ServiceConnection }} azurePowerShellVersion: LatestVersion @@ -46,6 +48,7 @@ steps: @subscriptionConfiguration ` -ResourceType '${{ parameters.ResourceType }}' ` -ServiceDirectory "${{ parameters.ServiceDirectory }}" ` + -AllowIpRanges ('$(azsdk-corp-net-ip-ranges)' -split ',') ` -CI ` -Force ` -Verbose @@ -63,10 +66,13 @@ steps: -ResourceType '${{ parameters.ResourceType }}' ` -ServiceDirectory "${{ parameters.ServiceDirectory }}" ` -ServicePrincipalAuth ` + -AllowIpRanges ('$(azsdk-corp-net-ip-ranges)' -split ',') ` -CI ` -Force ` -Verbose displayName: Remove test resources condition: and(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), ne(variables['Skip.RemoveTestResources'], 'true')) continueOnError: true - env: ${{ parameters.EnvVars }} + env: + PoolSubnet: $(PoolSubnet) + ${{ insert }}: ${{ parameters.EnvVars }} diff --git a/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml b/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml index 5d53a5265c..37efd0bd03 100644 --- a/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml +++ b/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml @@ -2,6 +2,7 @@ # is used when this pipeline is going to be generating and publishing daily dev builds. parameters: ServiceDirectory: '' + Condition: succeeded() steps: - ${{if ne(parameters.ServiceDirectory, '')}}: - task: Powershell@2 @@ -13,7 +14,7 @@ steps: pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Dump Package properties - condition: succeeded() + condition: ${{ parameters.Condition }} - pwsh: | $setDailyDevBuild = "false" if (('$(Build.Reason)' -eq 'Schedule') -and ('$(System.TeamProject)' -eq 'internal')) { @@ -21,4 +22,4 @@ steps: } echo "##vso[task.setvariable variable=SetDevVersion]$setDailyDevBuild" displayName: "Setup Versioning Properties" - condition: and(succeeded(), eq(variables['SetDevVersion'], '')) + condition: and(${{ parameters.Condition }}, eq(variables['SetDevVersion'], '')) diff --git a/eng/common/pipelines/templates/steps/verify-changelog.yml b/eng/common/pipelines/templates/steps/verify-changelog.yml index 887ad1a97d..113e0a5e69 100644 --- a/eng/common/pipelines/templates/steps/verify-changelog.yml +++ b/eng/common/pipelines/templates/steps/verify-changelog.yml @@ -11,6 +11,9 @@ parameters: - name: ForRelease type: boolean default: false +- name: Condition + type: string + default: succeeded() steps: - task: Powershell@2 @@ -23,4 +26,5 @@ steps: pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Verify ChangeLogEntry for ${{ parameters.PackageName }} + condition: ${{ parameters.Condition }} continueOnError: false diff --git a/eng/common/pipelines/templates/steps/verify-path-length.yml b/eng/common/pipelines/templates/steps/verify-path-length.yml index e797292169..259c663b9d 100644 --- a/eng/common/pipelines/templates/steps/verify-path-length.yml +++ b/eng/common/pipelines/templates/steps/verify-path-length.yml @@ -1,11 +1,13 @@ # Template for all Python Scripts in this repository -parameters: +parameters: SourceDirectory: '' BasePathLength: 49 + Condition: succeeded() steps: - task: PythonScript@0 displayName: Analyze Path Lengths + condition: ${{ parameters.Condition }} inputs: scriptSource: inline script: | diff --git a/eng/common/pipelines/templates/steps/verify-readme.yml b/eng/common/pipelines/templates/steps/verify-readme.yml index 7b9217ade3..6eeb174b32 100644 --- a/eng/common/pipelines/templates/steps/verify-readme.yml +++ b/eng/common/pipelines/templates/steps/verify-readme.yml @@ -15,10 +15,14 @@ parameters: - name: DocWardenVersion type: string default: '' +- name: Condition + type: string + default: succeeded() steps: - task: PowerShell@2 displayName: "Verify Readmes" + condition: ${{ parameters.Condition }} inputs: filePath: "eng/common/scripts/Verify-Readme.ps1" arguments: > diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 6c02e9150e..fdbb44186e 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -4,7 +4,7 @@ function Get-PurgeableGroupResources { param ( - [Parameter(Mandatory=$true, Position=0)] + [Parameter(Mandatory = $true, Position = 0)] [string] $ResourceGroupName ) @@ -27,8 +27,8 @@ function Get-PurgeableGroupResources { # Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled. $deletedKeyVaults = @(Get-AzKeyVault -ResourceGroupName $ResourceGroupName -ErrorAction Ignore | ForEach-Object { - # Enumerating vaults from a resource group does not return all properties we required. - Get-AzKeyVault -VaultName $_.VaultName -ErrorAction Ignore | Where-Object { $_.EnableSoftDelete } ` + # Enumerating vaults from a resource group does not return all properties we required. + Get-AzKeyVault -VaultName $_.VaultName -ErrorAction Ignore | Where-Object { $_.EnableSoftDelete } ` | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru ` | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru }) @@ -56,13 +56,13 @@ function Get-PurgeableResources { $deletedHsms = @() foreach ($r in $content.value) { $deletedHsms += [pscustomobject] @{ - AzsdkResourceType = 'Managed HSM' - AzsdkName = $r.name - Id = $r.id - Name = $r.name - Location = $r.properties.location - DeletionDate = $r.properties.deletionDate -as [DateTime] - ScheduledPurgeDate = $r.properties.scheduledPurgeDate -as [DateTime] + AzsdkResourceType = 'Managed HSM' + AzsdkName = $r.name + Id = $r.id + Name = $r.name + Location = $r.properties.location + DeletionDate = $r.properties.deletionDate -as [DateTime] + ScheduledPurgeDate = $r.properties.scheduledPurgeDate -as [DateTime] EnablePurgeProtection = $r.properties.purgeProtectionEnabled } } @@ -91,7 +91,8 @@ function Get-PurgeableResources { Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge." $purgeableResources += $deletedKeyVaults } - } catch { } + } + catch { } return $purgeableResources } @@ -100,7 +101,7 @@ function Get-PurgeableResources { # This allows you to pipe a collection and process each item in the collection. filter Remove-PurgeableResources { param ( - [Parameter(Position=0, ValueFromPipeline=$true)] + [Parameter(Position = 0, ValueFromPipeline = $true)] [object[]] $Resource, [Parameter()] @@ -128,7 +129,7 @@ filter Remove-PurgeableResources { # Use `-AsJob` to start a lightweight, cancellable job and pass to `Wait-PurgeableResoruceJob` for consistent behavior. Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue -AsJob ` - | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru + | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru } 'Managed HSM' { @@ -139,18 +140,19 @@ filter Remove-PurgeableResources { # Use `GetNewClosure()` on the `-Action` ScriptBlock to make sure variables are captured. Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2023-02-01" -ErrorAction Ignore -AsJob ` - | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru -Action { - param ( $response ) - if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { - Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged." - } elseif ($response.Content) { - $content = $response.Content | ConvertFrom-Json - if ($content.error) { - $err = $content.error - Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)" - } - } - }.GetNewClosure() + | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru -Action { + param ( $response ) + if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { + Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged." + } + elseif ($response.Content) { + $content = $response.Content | ConvertFrom-Json + if ($content.error) { + $err = $content.error + Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)" + } + } + }.GetNewClosure() } default { @@ -167,12 +169,12 @@ function Log($Message) { function Wait-PurgeableResourceJob { param ( - [Parameter(Mandatory=$true, ValueFromPipeline=$true)] + [Parameter(Mandatory = $true, ValueFromPipeline = $true)] $Job, # The resource is used for logging and to return if `-PassThru` is specified # so we can easily see all resources that may be in a bad state when the script has completed. - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] $Resource, # Optional ScriptBlock should define params corresponding to the associated job's `Output` property. @@ -195,7 +197,8 @@ function Wait-PurgeableResourceJob { if ($Action) { $null = $Action.Invoke($result) } - } else { + } + else { Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)'. Cancelling job." $Job.Cancel() @@ -204,3 +207,191 @@ function Wait-PurgeableResourceJob { } } } + +# Helper function for removing storage accounts with WORM that sometimes get leaked from live tests not set up to clean +# up their resource policies +function Remove-WormStorageAccounts() { + [CmdletBinding(SupportsShouldProcess = $True)] + param( + [string]$GroupPrefix, + [switch]$CI + ) + + $ErrorActionPreference = 'Stop' + + # Be a little defensive so we don't delete non-live test groups via naming convention + # DO NOT REMOVE THIS + # We call this script from live test pipelines as well, and a string mismatch/error could blow away + # some static storage accounts we rely on + if (!$groupPrefix -or ($CI -and !$GroupPrefix.StartsWith('rg-'))) { + throw "The -GroupPrefix parameter must not be empty, or must start with 'rg-' in CI contexts" + } + + $groups = Get-AzResourceGroup | Where-Object { $_.ResourceGroupName.StartsWith($GroupPrefix) } | Where-Object { $_.ProvisioningState -ne 'Deleting' } + + foreach ($group in $groups) { + Write-Host "=========================================" + $accounts = Get-AzStorageAccount -ResourceGroupName $group.ResourceGroupName + if ($accounts) { + foreach ($account in $accounts) { + if ($WhatIfPreference) { + Write-Host "What if: Removing $($account.StorageAccountName) in $($account.ResourceGroupName)" + } + else { + Write-Host "Removing $($account.StorageAccountName) in $($account.ResourceGroupName)" + } + + $hasContainers = ($account.Kind -ne "FileStorage") + + # If it doesn't have containers then we can skip the explicit clean-up of this storage account + if (!$hasContainers) { continue } + + $ctx = New-AzStorageContext -StorageAccountName $account.StorageAccountName + + $immutableBlobs = $ctx ` + | Get-AzStorageContainer ` + | Where-Object { $_.BlobContainerProperties.HasImmutableStorageWithVersioning } ` + | Get-AzStorageBlob + try { + foreach ($blob in $immutableBlobs) { + Write-Host "Removing legal hold - blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)" + $blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null + } + } + catch { + Write-Warning "User must have 'Storage Blob Data Owner' RBAC permission on subscription or resource group" + Write-Error $_ + throw + } + # Sometimes we get a 404 blob not found but can still delete containers, + # and sometimes we must delete the blob if there's a legal hold. + # Try to remove the blob, but keep running regardless. + $succeeded = $false + for ($attempt = 0; $attempt -lt 2; $attempt++) { + if ($succeeded) { + break + } + + try { + Write-Host "Removing immutability policies - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" + $null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy + } + catch {} + + try { + $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force + $succeeded = $true + } + catch { + Write-Warning "Failed to remove blobs - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" + Write-Warning $_ + } + } + + try { + # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs + $ctx | Get-AzStorageContainer | ForEach-Object { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force } + } + catch { + Write-Warning "Container removal failed. Ignoring the error and trying to delete the storage account." + Write-Warning $_ + } + Remove-AzStorageAccount -StorageAccountName $account.StorageAccountName -ResourceGroupName $account.ResourceGroupName -Force + } + } + if ($WhatIfPreference) { + Write-Host "What if: Removing resource group $($group.ResourceGroupName)" + } + else { + Remove-AzResourceGroup -ResourceGroupName $group.ResourceGroupName -Force -AsJob + } + } +} + +function SetResourceNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI, [switch]$SetFirewall) { + SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI -SetFirewall:$SetFirewall +} + +function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI, [switch]$SetFirewall) { + $clientIp = $null + $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } + # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access + if ($storageAccounts) { + $appliedRule = $false + foreach ($account in $storageAccounts) { + $properties = Get-AzStorageAccount -ResourceGroupName $ResourceGroupName -AccountName $account.Name + $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name + + if ($properties.AllowBlobPublicAccess) { + Write-Host "Restricting public blob access in storage account '$($account.Name)'" + Set-AzStorageAccount -ResourceGroupName $ResourceGroupName -StorageAccountName $account.Name -AllowBlobPublicAccess $false + } + + # In override mode, we only want to capture storage accounts that have had incomplete network rules applied, + # otherwise it's not worth updating due to timing and throttling issues. + # If the network rules are deny only without any vnet/ip allowances, then we can't ever purge the storage account + # when immutable blobs need to be removed. + if (!$rules -or !$SetFirewall -or $rules.DefaultAction -eq "Allow") { + return + } + + # Add firewall rules in cases where existing rules added were incomplete to enable blob removal + Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default" + Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny } + if ($CI -and $env:PoolSubnet) { + Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" + Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } + $appliedRule = $true + } + elseif ($AllowIpRanges) { + Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" + $ipRanges = $AllowIpRanges | ForEach-Object { + @{ Action = 'allow'; IPAddressOrRange = $_ } + } + Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } + $appliedRule = $true + } + elseif (!$CI) { + Write-Host "Enabling access to '$($account.Name)' from client IP" + $clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site + $clientIp = $clientIp.Trim() + $ipRanges = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name + if ($ipRanges) { + foreach ($range in $ipRanges.IpRules) { + if (DoesSubnetOverlap $range.IPAddressOrRange $clientIp) { + return + } + } + } + Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } + $appliedRule = $true + } + } + + if ($appliedRule) { + Write-Host "Sleeping for 15 seconds to allow network rules to take effect" + Start-Sleep 15 + } + } +} + +function DoesSubnetOverlap([string]$ipOrCidr, [string]$overlapIp) { + [System.Net.IPAddress]$overlapIpAddress = $overlapIp + $parsed = $ipOrCidr -split '/' + [System.Net.IPAddress]$baseIp = $parsed[0] + if ($parsed.Length -eq 1) { + return $baseIp -eq $overlapIpAddress + } + + $subnet = $parsed[1] + $subnetNum = [int]$subnet + + $baseMask = [math]::pow(2, 31) + $mask = 0 + for ($i = 0; $i -lt $subnetNum; $i++) { + $mask = $mask + $baseMask; + $baseMask = $baseMask / 2 + } + + return $baseIp.Address -eq ($overlapIpAddress.Address -band ([System.Net.IPAddress]$mask).Address) +} diff --git a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 index a79db98e7c..bec510cd95 100644 --- a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 +++ b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 @@ -75,7 +75,7 @@ function ParseChart([string]$chartFile) { function MatchesAnnotations([hashtable]$chart, [hashtable]$filters) { foreach ($filter in $filters.GetEnumerator()) { - if (!$chart["annotations"] -or $chart["annotations"][$filter.Key] -ne $filter.Value) { + if (!$chart["annotations"] -or $chart["annotations"][$filter.Key] -notmatch $filter.Value) { return $false } } diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index aa426d7b46..c67540f3b3 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -122,6 +122,12 @@ function DeployStressTests( } $clusterGroup = 'rg-stress-cluster-prod' $subscription = 'Azure SDK Test Resources' + } elseif ($environment -eq 'storage') { + if ($clusterGroup -or $subscription) { + Write-Warning "Overriding cluster group and subscription with defaults for 'storage' environment." + } + $clusterGroup = 'rg-stress-cluster-storage' + $subscription = 'XClient' } elseif (!$clusterGroup -or !$subscription) { throw "clusterGroup and subscription parameters must be specified when deploying to an environment that is not pg or prod." } diff --git a/eng/common/testproxy/install-test-proxy.ps1 b/eng/common/testproxy/install-test-proxy.ps1 new file mode 100644 index 0000000000..402e5ddc8c --- /dev/null +++ b/eng/common/testproxy/install-test-proxy.ps1 @@ -0,0 +1,33 @@ +<# +.SYNOPSIS +Installs a standalone version of the test-proxy for use in tests. This function is intended to be used in CI/CD pipelines, and leaves behind +the pipeline variable PROXY_EXE which contains the path to the test-proxy executable. + +.PARAMETER Version +The version of the proxy to install. Requires a full version to be provided. EG "1.0.0-dev.20240617.1" + +.PARAMETER Directory +The directory within which the test-proxy exe will exist after this function invokes. Defaults to CWD. +#> +param( + [Parameter(Mandatory = $true)] + $Version, + [Parameter(Mandatory = $true)] + $InstallDirectory +) + +. (Join-Path $PSScriptRoot test-proxy.ps1) + +Write-Host "Attempting to download and install version `"$Version`" into `"$InstallDirectory`"" + +Install-Standalone-TestProxy -Version $Version -Directory $InstallDirectory + +$PROXY_EXE = "" + +if ($IsWindows) { + $PROXY_EXE = Join-Path $InstallDirectory "Azure.Sdk.Tools.TestProxy.exe" +} else { + $PROXY_EXE = Join-Path $InstallDirectory "Azure.Sdk.Tools.TestProxy" +} +Write-Host "Downloaded test-proxy available at $PROXY_EXE." +Write-Host "##vso[task.setvariable variable=PROXY_EXE]$PROXY_EXE" diff --git a/eng/common/testproxy/test-proxy-standalone-tool.yml b/eng/common/testproxy/test-proxy-standalone-tool.yml new file mode 100644 index 0000000000..a836427ad7 --- /dev/null +++ b/eng/common/testproxy/test-proxy-standalone-tool.yml @@ -0,0 +1,83 @@ +# This template sets variable PROXY_PID to be used for shutdown later. +parameters: + rootFolder: '$(Build.SourcesDirectory)' + runProxy: true + targetVersion: '' + templateRoot: '$(Build.SourcesDirectory)' + condition: true + +steps: + - pwsh: | + ${{ parameters.templateRoot }}/eng/common/scripts/trust-proxy-certificate.ps1 + displayName: 'Language Specific Certificate Trust' + condition: and(succeeded(), ${{ parameters.condition }}) + + - task: PowerShell@2 + displayName: 'Override proxy version if necessary' + condition: and(succeeded(), ${{ parameters.condition }}, ne('${{ parameters.targetVersion }}', '')) + inputs: + targetType: filePath + filePath: '${{ parameters.templateRoot }}/eng/common/testproxy/scripts/override-proxy-version.ps1' + arguments: '-TargetVersion "${{ parameters.targetVersion }}"' + pwsh: true + + - pwsh: | + $standardVersion = "${{ parameters.templateRoot }}/eng/common/testproxy/target_version.txt" + $overrideVersion = "${{ parameters.templateRoot }}/eng/target_proxy_version.txt" + + $version = $(Get-Content $standardVersion -Raw).Trim() + + if (Test-Path $overrideVersion) { + $version = $(Get-Content $overrideVersion -Raw).Trim() + } + + Write-Host "Installing test-proxy version $version" + ${{ parameters.templateRoot }}/eng/common/testproxy/install-test-proxy.ps1 -Version $version -InstallDirectory $(Build.BinariesDirectory)/test-proxy + displayName: "Install test-proxy" + condition: and(succeeded(), ${{ parameters.condition }}) + + - pwsh: | + Write-Host "##vso[task.prependpath]$(Build.BinariesDirectory)/test-proxy" + displayName: "Prepend path with test-proxy tool install location" + + - ${{ if eq(parameters.runProxy, 'true') }}: + - pwsh: | + Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Path]${{ parameters.templateRoot }}/eng/common/testproxy/dotnet-devcert.pfx" + Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password" + Write-Host "##vso[task.setvariable variable=PROXY_MANUAL_START]true" + displayName: 'Configure Kestrel and PROXY_MANUAL_START Variables' + condition: and(succeeded(), ${{ parameters.condition }}) + + - pwsh: | + $Process = Start-Process $(PROXY_EXE) ` + -ArgumentList "start -u --storage-location ${{ parameters.rootFolder }}" ` + -NoNewWindow -PassThru -RedirectStandardOutput ${{ parameters.rootFolder }}/test-proxy.log + + Write-Host "##vso[task.setvariable variable=PROXY_PID]$($Process.Id)" + displayName: 'Run the testproxy - windows' + condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT'), ${{ parameters.condition }}) + + # nohup does NOT continue beyond the current session if you use it within powershell + - bash: | + nohup $(PROXY_EXE) &>${{ parameters.rootFolder }}/test-proxy.log & + + echo $! > $(Build.SourcesDirectory)/test-proxy.pid + echo "##vso[task.setvariable variable=PROXY_PID]$(cat $(Build.SourcesDirectory)/test-proxy.pid)" + displayName: "Run the testproxy - linux/mac" + condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT'), ${{ parameters.condition }}) + workingDirectory: "${{ parameters.rootFolder }}" + + - pwsh: | + for ($i = 0; $i -lt 10; $i++) { + try { + Invoke-WebRequest -Uri "http://localhost:5000/Admin/IsAlive" | Out-Null + exit 0 + } catch { + Write-Warning "Failed to successfully connect to test proxy. Retrying..." + Start-Sleep 6 + } + } + Write-Error "Could not connect to test proxy." + exit 1 + displayName: Test Proxy IsAlive + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/common/testproxy/test-proxy.ps1 b/eng/common/testproxy/test-proxy.ps1 new file mode 100644 index 0000000000..f1bf1eca8f --- /dev/null +++ b/eng/common/testproxy/test-proxy.ps1 @@ -0,0 +1,162 @@ +Set-StrictMode -Version 4 +$AVAILABLE_TEST_PROXY_BINARIES = @{ + "Windows" = @{ + "AMD64" = @{ + "system" = "Windows" + "machine" = "AMD64" + "file_name" = "test-proxy-standalone-win-x64.zip" + "executable" = "Azure.Sdk.Tools.TestProxy.exe" + } + } + "Linux" = @{ + "X86_64" = @{ + "system" = "Linux" + "machine" = "X86_64" + "file_name" = "test-proxy-standalone-linux-x64.tar.gz" + "executable" = "Azure.Sdk.Tools.TestProxy" + } + "ARM64" = @{ + "system" = "Linux" + "machine" = "ARM64" + "file_name" = "test-proxy-standalone-linux-arm64.tar.gz" + "executable" = "Azure.Sdk.Tools.TestProxy" + } + } + "Darwin" = @{ + "X86_64" = @{ + "system" = "Darwin" + "machine" = "X86_64" + "file_name" = "test-proxy-standalone-osx-x64.zip" + "executable" = "Azure.Sdk.Tools.TestProxy" + } + "ARM64" = @{ + "system" = "Darwin" + "machine" = "ARM64" + "file_name" = "test-proxy-standalone-osx-arm64.zip" + "executable" = "Azure.Sdk.Tools.TestProxy" + } + } +} + +function Get-SystemArchitecture { + $unameOutput = uname -m + switch ($unameOutput) { + "x86_64" { return "X86_64" } + "aarch64" { return "ARM64" } + "arm64" { return "ARM64" } + default { throw "Unable to determine system architecture. uname -m returned $unameOutput." } + } +} + +function Get-Proxy-Meta () { + $ErrorActionPreferenceDefault = $ErrorActionPreference + $ErrorActionPreference = "Stop" + + $os = "unknown" + $machine = Get-SystemArchitecture + + if ($IsWindows) { + $os = "Windows" + # we only support x64 on windows, if that doesn't work the platform is unsupported + $machine = "AMD64" + } elseif ($IsLinux) { + $os = "Linux" + } elseif ($IsMacOS) { + $os = "Darwin" + } + + $ErrorActionPreference = $ErrorActionPreferenceDefault + + return $AVAILABLE_TEST_PROXY_BINARIES[$os][$machine] +} + +function Get-Proxy-Url ( + [Parameter(mandatory=$true)]$Version +) { + $systemDetails = Get-Proxy-Meta + + $file = $systemDetails.file_name + $url = "https://github.com/Azure/azure-sdk-tools/releases/download/Azure.Sdk.Tools.TestProxy_$Version/$file" + + return $url +} + +function Cleanup-Directory ($path) { + if (Test-Path -Path $path) { + Remove-Item -Path $path -Recurse -Force + } + New-Item -ItemType Directory -Path $path -Force +} + +function Is-Work-Necessary ( + [Parameter(mandatory=$true)] + $Version, + [Parameter(mandatory=$true)] + $Directory +) { + $savedVersionTxt = Join-Path $Directory "downloaded_version.txt" + if (Test-Path $savedVersionTxt) { + $result = (Get-Content -Raw $savedVersionTxt).Trim() + + if ($result -eq $Version) { + return $false + } + } + + return $true +} + +<# +.SYNOPSIS +Installs a standalone version of the test-proxy. +.PARAMETER Version +The version of the proxy to install. Requires a full version to be provided. EG "1.0.0-dev.20240617.1" +.PARAMETER Directory +The directory within which the test-proxy exe will exist after this function invokes. Defaults to "." +#> +function Install-Standalone-TestProxy ( + [Parameter(mandatory=$true)] + $Version, + $Directory="." +) { + $ErrorActionPreference = "Stop" + $systemDetails = Get-Proxy-Meta + + if (!(Test-Path $Directory) -and $Directory -ne ".") { + New-Item -ItemType Directory -Path $Directory -Force + } + + $downloadFolder = Resolve-Path $Directory + $downloadUrl = Get-Proxy-Url $Version + $downloadFile = $downloadUrl.Split('/')[-1] + $downloadLocation = Join-Path $downloadFolder $downloadFile + $savedVersionTxt = Join-Path $downloadFolder "downloaded_version.txt" + + if (Is-Work-Necessary $version $downloadFolder) { + Write-Host "Commencing installation of `"$Version`" to `"$downloadFolder`" from $downloadUrl." + Invoke-WebRequest -Uri $downloadUrl -OutFile $downloadLocation + + if ($downloadFile -like "*.zip") { + Expand-Archive -Path $downloadLocation -DestinationPath $downloadFolder -Force + } elseif ($downloadFile -like "*.tar.gz") { + tar -xzf $downloadLocation -C $downloadFolder + } else { + throw "Unsupported file format" + } + + # Remove the downloaded file after extraction + Remove-Item -Path $downloadLocation -Force + + # Record downloaded version + Set-Content -Path $savedVersionTxt -Value $Version + + # Set executable permissions if on macOS (Darwin) + $executable_path = Join-Path $downloadFolder $systemDetails.executable + if ($IsMacOS) { + chmod 755 $executable_path + } + } + else { + Write-Host "Target version `"$Version`" already present in target directory `"$downloadFolder.`"" + } +} diff --git a/eng/pipelines/templates/jobs/live.tests.yml b/eng/pipelines/templates/jobs/live.tests.yml index bda375013b..dd90006382 100644 --- a/eng/pipelines/templates/jobs/live.tests.yml +++ b/eng/pipelines/templates/jobs/live.tests.yml @@ -244,32 +244,6 @@ jobs: # Will run samples described on a file name [service]-samples.txt within the build directory. # For example keyvault-samples.txt. # The file is written by CMake during configuration when building samples. - - bash: | - IFS=$'\n' - if [[ -f "./${{ parameters.ServiceDirectory }}-samples.txt" ]]; then - for sample in `cat ./${{ parameters.ServiceDirectory }}-samples.txt` - do - export AZURE_CLIENT_ID=$(${{parameters.ServiceDirectory}}_CLIENT_ID) - export AZURE_TENANT_ID=$(${{parameters.ServiceDirectory}}_TENANT_ID) - export AZURE_CLIENT_SECRET=$(${{parameters.ServiceDirectory}}_CLIENT_SECRET) - echo "**********Running sample: ${sample}" - bash -c "$sample" - status=$? - if [[ $status -eq 0 ]]; then - echo "*********Sample completed*********" - else - echo "*Sample returned a failed code: $status" - exit 1 - fi - done - fi - workingDirectory: build - displayName: "Run Samples for : ${{ parameters.ServiceDirectory }}" - condition: and(succeeded(), eq(variables['RunSamples'], '1')) - env: - ${{ insert }}: ${{ parameters.EnvVars }} - - - ${{ else }}: - task: AzurePowerShell@5 displayName: "Run Samples for : ${{ parameters.ServiceDirectory }}" condition: and(succeeded(), eq(variables['RunSamples'], '1')) @@ -299,6 +273,32 @@ jobs: SYSTEM_ACCESSTOKEN: $(System.AccessToken) ${{ insert }}: ${{ parameters.EnvVars }} + - ${{ else }}: + - bash: | + IFS=$'\n' + if [[ -f "./${{ parameters.ServiceDirectory }}-samples.txt" ]]; then + for sample in `cat ./${{ parameters.ServiceDirectory }}-samples.txt` + do + export AZURE_CLIENT_ID=$(${{parameters.ServiceDirectory}}_CLIENT_ID) + export AZURE_TENANT_ID=$(${{parameters.ServiceDirectory}}_TENANT_ID) + export AZURE_CLIENT_SECRET=$(${{parameters.ServiceDirectory}}_CLIENT_SECRET) + echo "**********Running sample: ${sample}" + bash -c "$sample" + status=$? + if [[ $status -eq 0 ]]; then + echo "*********Sample completed*********" + else + echo "*Sample returned a failed code: $status" + exit 1 + fi + done + fi + workingDirectory: build + displayName: "Run Samples for : ${{ parameters.ServiceDirectory }}" + condition: and(succeeded(), eq(variables['RunSamples'], '1')) + env: + ${{ insert }}: ${{ parameters.EnvVars }} + # Make coverage targets (specified in coverage_targets.txt) and assemble # coverage report - bash: | diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 488e91c870..b33dd34d01 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -62,6 +62,8 @@ parameters: Preview: SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview) ServiceConnection: azure-sdk-tests + # Temporary fix until an eng/common config for Preview can be merged + SubscriptionConfigurationFilePaths: [] Canary: SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) ServiceConnection: azure-sdk-tests @@ -117,6 +119,10 @@ extends: compiled: true break: true policy: M365 + codeql: + compiled: + enabled: false + justificationForDisabling: CodeQL times our pipelines out by running for 2+ hours before being force canceled. credscan: suppressionsFile: $(Build.SourcesDirectory)/eng/CredScanSuppression.json toolVersion: 2.3.12.23 diff --git a/eng/scripts/Show-FailureLogs.ps1 b/eng/scripts/Show-FailureLogs.ps1 index 5d88263315..1bf7e963fd 100644 --- a/eng/scripts/Show-FailureLogs.ps1 +++ b/eng/scripts/Show-FailureLogs.ps1 @@ -6,7 +6,8 @@ # sensitive information. $logFiles = Get-ChildItem -Recurse -Filter *.log -$filteredLogs = $logFiles.Where({ $_.Name -in ('vcpkg-bootstrap.log', 'vcpkg-manifest-install.log') }) +$vcpkgLogFileNames = @('vcpkg-bootstrap.log', 'vcpkg-manifest-install.log') +$filteredLogs = $logFiles.Where({ $_.Name -in $vcpkgLogFileNames }) $filteredLogs.FullName | Write-Host @@ -15,14 +16,33 @@ if (!$filteredLogs) { exit 0 } -foreach ($logFile in $filteredLogs) +$filteredLogs = @($filteredLogs.FullName) + +for ($i = 0; $i -lt $filteredLogs.Length; $i += 1) { + $logFile = $filteredLogs[$i] + Write-Host "//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////" Write-Host "==============================================================================================================================" Write-Host "Log file: $logFile" Write-Host "==============================================================================================================================" try { Get-Content $logFile | Write-Host + + # vcpkg logs do reference build log paths after "See logs for more information:". Sometimes there are multiple files to see. + # And should there be a C++ build error, for example - that is where the error message would be. + # So, we parse known logs (contained in vcpkgLogFileNames), and see if more logs are mentioned there. If there are extra logs, + # we add them to the end of the list that we're iterating over, so that the format is the same, and this code gets reused + # (i.e. formatting, Log file name header, try-catch, and so on). + if ($i -lt $vcpkgLogFileNames.Length) + { + $rawContents = Get-Content $logFile -Raw + $regexMatches = Select-String "See logs for more information\:\s*(\r|\n|\r\n|\n\r)(\s+(?\S*)\s*(\r|\n|\r\n|\n\r))+" -input $rawContents -AllMatches + foreach ($additionalLogFile in $regexMatches.matches.groups.Where({ $_.Name -eq "logFilePath" })) + { + $filteredLogs += $additionalLogFile.Value + } + } } catch { Write-Host "Could not locate file found using Get-ChildItem $logFile" } diff --git a/sdk/attestation/ci.yml b/sdk/attestation/ci.yml index 9be5362525..2d7e75aaa7 100644 --- a/sdk/attestation/ci.yml +++ b/sdk/attestation/ci.yml @@ -32,6 +32,7 @@ extends: LiveTestCtestRegex: azure-security-attestation.* LineCoverageTarget: 70 BranchCoverageTarget: 34 + UseFederatedAuth: true Artifacts: - Name: azure-security-attestation Path: azure-security-attestation diff --git a/sdk/core/azure-core-test/inc/azure/core/test/test_base.hpp b/sdk/core/azure-core-test/inc/azure/core/test/test_base.hpp index de9c152fd0..5305aba572 100644 --- a/sdk/core/azure-core-test/inc/azure/core/test/test_base.hpp +++ b/sdk/core/azure-core-test/inc/azure/core/test/test_base.hpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include @@ -246,7 +248,17 @@ namespace Azure { namespace Core { namespace Test { } if (clientSecret.empty()) { - m_testCredential = std::make_shared(); + m_testCredential = std::make_shared( + Azure::Identity::ChainedTokenCredential::Sources{ + std ::make_shared( + Azure::Core::_internal::Environment::GetVariable( + "AZURESUBSCRIPTION_TENANT_ID"), + Azure::Core::_internal::Environment::GetVariable( + "AZURESUBSCRIPTION_CLIENT_ID"), + Azure::Core::_internal::Environment::GetVariable( + "AZURESUBSCRIPTION_SERVICE_CONNECTION_ID"), + Azure::Core::_internal::Environment::GetVariable("SYSTEM_ACCESSTOKEN")), + std::make_shared()}); } else { @@ -302,7 +314,7 @@ namespace Azure { namespace Core { namespace Test { * * @return The value of the environment variable retrieved. * - * @note If AZURE_TENANT_ID, AZURE_CLIENT_ID, or AZURE_CLIENT_SECRET are not available in the + * @note If AZURE_TENANT_ID or AZURE_CLIENT_ID are not available in the * environment, the AZURE_SERVICE_DIRECTORY environment variable is used to set those values * with the values emitted by the New-TestResources.ps1 script. * diff --git a/sdk/core/azure-core-tracing-opentelemetry/src/opentelemetry.cpp b/sdk/core/azure-core-tracing-opentelemetry/src/opentelemetry.cpp index bf88940bc8..7a9854bfa6 100644 --- a/sdk/core/azure-core-tracing-opentelemetry/src/opentelemetry.cpp +++ b/sdk/core/azure-core-tracing-opentelemetry/src/opentelemetry.cpp @@ -232,7 +232,7 @@ namespace Azure { namespace Core { namespace Tracing { namespace OpenTelemetry { { return header.Value(); } - return std::string(); + return {}; } /** @brief Sets the value of an HTTP header in the request. diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 456ca77c7e..0cf907c6a5 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.13.0-beta.2 (Unreleased) +## 1.14.0-beta.1 (Unreleased) ### Features Added @@ -10,17 +10,35 @@ ### Other Changes +## 1.13.0 (2024-07-12) + +### Bugs Fixed + +- [[#5589]](https://github.com/Azure/azure-sdk-for-cpp/pull/5589) Fix possible endless loop while polling curl socket. (A community contribution, courtesy of _[CurtizJ](https://github.com/CurtizJ)_) + +### Other Changes + - Updated JSON library to 3.11.3. +- [[#5622]](https://github.com/Azure/azure-sdk-for-cpp/pull/5622) Documentation fix for building the SDK with specific OpenSSL version. (A community contribution, courtesy of _[ByteYue](https://github.com/ByteYue)_) +- [[#5766]](https://github.com/Azure/azure-sdk-for-cpp/pull/5766) Add missing include. (A community contribution, courtesy of _[rschu1ze](https://github.com/rschu1ze)_) + +### Acknowledgments + +Thank you to our developer community members who helped to make Azure Core better with their contributions to this release: + +- Anton Popov _([GitHub](https://github.com/CurtizJ))_ +- AlexYue _([GitHub](https://github.com/ByteYue))_ +- Robert Schulze _([GitHub](https://github.com/rschu1ze))_ ## 1.13.0-beta.1 (2024-06-06) ### Bugs Fixed -- [[#5589]](https://github.com/Azure/azure-sdk-for-cpp/pull/5589) Fix possible endless loop while polling curl socket. (A community contribution, courtesy of _[CurtizJ](https://github.com/CurtizJ)_) +- [[#5589]](https://github.com/Azure/azure-sdk-for-cpp/pull/5589) Fix possible endless loop while polling curl socket. (A community contribution, courtesy of _[CurtizJ](https://github.com/CurtizJ)_) ### Other Changes -- [[#5622]](https://github.com/Azure/azure-sdk-for-cpp/pull/5622) Documentation fix for building the SDK with specific OpenSSL version. (A community contribution, courtesy of _[ByteYue](https://github.com/ByteYue)_) +- [[#5622]](https://github.com/Azure/azure-sdk-for-cpp/pull/5622) Documentation fix for building the SDK with specific OpenSSL version. (A community contribution, courtesy of _[ByteYue](https://github.com/ByteYue)_) - [[#5515]](https://github.com/Azure/azure-sdk-for-cpp/issues/5515) Add a `ShouldRetry` virtual method to the retry policy to enable customization of service-specific retry logic. ### Acknowledgments diff --git a/sdk/core/azure-core/inc/azure/core/http/policies/policy.hpp b/sdk/core/azure-core/inc/azure/core/http/policies/policy.hpp index 7eae35118a..511a1c3a7a 100644 --- a/sdk/core/azure-core/inc/azure/core/http/policies/policy.hpp +++ b/sdk/core/azure-core/inc/azure/core/http/policies/policy.hpp @@ -363,7 +363,11 @@ namespace Azure { namespace Core { namespace Http { namespace Policies { /** * @brief HTTP retry policy. */ - class RetryPolicy : public HttpPolicy { + class RetryPolicy +#if !defined(_azure_TESTING_BUILD) + final +#endif + : public HttpPolicy { private: RetryOptions m_retryOptions; @@ -411,26 +415,6 @@ namespace Azure { namespace Core { namespace Http { namespace Policies { int32_t attempt, std::chrono::milliseconds& retryAfter, double jitterFactor = -1) const; - - /** - * @brief Overriding this method customizes the logic of when the RetryPolicy will re-attempt - * a request, based on the returned HTTP response. - * - * @remark A null response pointer means there was no response received from the corresponding - * request. Custom implementations of this method that override the retry behavior, should - * handle that error case, if that needs to be customized. - * - * @remark Unless overriden, the default implementation is to always return `false`. The - * non-retriable errors, including those specified in the RetryOptions, remain evaluated - * before calling ShouldRetry. - * - * @param response An HTTP response returned corresponding to the request sent by the policy. - * @param retryOptions The set of options provided to the RetryPolicy. - * @return Whether or not the HTTP request should be sent again through the pipeline. - */ - virtual bool ShouldRetry( - std::unique_ptr const& response, - RetryOptions const& retryOptions) const; }; /** diff --git a/sdk/core/azure-core/src/http/http_sanitizer.cpp b/sdk/core/azure-core/src/http/http_sanitizer.cpp index 7d739581d9..6fc637259f 100644 --- a/sdk/core/azure-core/src/http/http_sanitizer.cpp +++ b/sdk/core/azure-core/src/http/http_sanitizer.cpp @@ -5,6 +5,7 @@ #include "azure/core/url.hpp" +#include #include #include diff --git a/sdk/core/azure-core/src/http/retry_policy.cpp b/sdk/core/azure-core/src/http/retry_policy.cpp index 32067e5726..217e0bd898 100644 --- a/sdk/core/azure-core/src/http/retry_policy.cpp +++ b/sdk/core/azure-core/src/http/retry_policy.cpp @@ -118,11 +118,6 @@ int32_t RetryPolicy::GetRetryCount(Context const& context) return *ptr; } -bool RetryPolicy::ShouldRetry(std::unique_ptr const&, RetryOptions const&) const -{ - return false; -} - std::unique_ptr RetryPolicy::Send( Request& request, NextHttpPolicy nextPolicy, @@ -145,24 +140,9 @@ std::unique_ptr RetryPolicy::Send( { auto response = nextPolicy.Send(request, retryContext); - // Are we out of retry attempts? - // Checking this first, before checking the response so that the extension point of - // ShouldRetry doesn't have the responsibility of checking the number of retries (again). - if (WasLastAttempt(m_retryOptions, attempt)) - { - return response; - } - - // If a response is non-retriable (or simply 200 OK, i.e doesn't need to be retried), then - // ShouldRetryOnResponse returns false. Service SDKs can inject custom logic to define whether - // the request should be retried, based on the response. The default of `ShouldRetry` is - // false. - // Because of boolean short-circuit evaluation, if ShouldRetryOnResponse returns true, the - // overriden ShouldRetry is not called. This is expected, since overriding ShouldRetry enables - // loosening the retry conditions (retrying where otherwise the request wouldn't be), but not - // strengthening it. - if (!ShouldRetryOnResponse(*response.get(), m_retryOptions, attempt, retryAfter) - && !ShouldRetry(response, m_retryOptions)) + // If we are out of retry attempts, if a response is non-retriable (or simply 200 OK, i.e + // doesn't need to be retried), then ShouldRetry returns false. + if (!ShouldRetryOnResponse(*response.get(), m_retryOptions, attempt, retryAfter)) { // If this is the second attempt and StartTry was called, we need to stop it. Otherwise // trying to perform same request would use last retry query/headers @@ -235,6 +215,12 @@ bool RetryPolicy::ShouldRetryOnResponse( using Azure::Core::Diagnostics::Logger; using Azure::Core::Diagnostics::_internal::Log; + // Are we out of retry attempts? + if (WasLastAttempt(retryOptions, attempt)) + { + return false; + } + // Should we retry on the given response retry code? { auto const& statusCodes = retryOptions.StatusCodes; diff --git a/sdk/core/azure-core/src/private/package_version.hpp b/sdk/core/azure-core/src/private/package_version.hpp index 8c7bd0ac9c..b9a096e5e2 100644 --- a/sdk/core/azure-core/src/private/package_version.hpp +++ b/sdk/core/azure-core/src/private/package_version.hpp @@ -11,9 +11,9 @@ #include #define AZURE_CORE_VERSION_MAJOR 1 -#define AZURE_CORE_VERSION_MINOR 13 +#define AZURE_CORE_VERSION_MINOR 14 #define AZURE_CORE_VERSION_PATCH 0 -#define AZURE_CORE_VERSION_PRERELEASE "beta.2" +#define AZURE_CORE_VERSION_PRERELEASE "beta.1" #define AZURE_CORE_VERSION_ITOA_HELPER(i) #i #define AZURE_CORE_VERSION_ITOA(i) AZURE_CORE_VERSION_ITOA_HELPER(i) diff --git a/sdk/core/azure-core/test/ut/retry_policy_test.cpp b/sdk/core/azure-core/test/ut/retry_policy_test.cpp index 5c5b069ac0..b482418ce5 100644 --- a/sdk/core/azure-core/test/ut/retry_policy_test.cpp +++ b/sdk/core/azure-core/test/ut/retry_policy_test.cpp @@ -124,239 +124,8 @@ class RetryPolicyTest final : public RetryPolicy { return m_shouldRetryOnResponse(response, retryOptions, attempt, retryAfter, jitterFactor); } }; - -class RetryPolicyTest_CustomRetry_True final : public RetryPolicy { -public: - RetryPolicyTest_CustomRetry_True(RetryOptions const& retryOptions) : RetryPolicy(retryOptions) {} - - std::unique_ptr Clone() const override - { - return std::make_unique(*this); - } - -protected: - bool ShouldRetry(std::unique_ptr const&, RetryOptions const&) const override - { - return true; - } -}; - -class RetryPolicyTest_CustomRetry_Throw final : public RetryPolicy { -public: - RetryPolicyTest_CustomRetry_Throw(RetryOptions const& retryOptions) : RetryPolicy(retryOptions) {} - - std::unique_ptr Clone() const override - { - return std::make_unique(*this); - } - -protected: - bool ShouldRetry(std::unique_ptr const&, RetryOptions const&) const override - { - throw TransportException("Short-circuit evaluation means this should never be called."); - } -}; - -class RetryPolicyTest_CustomRetry_CopySource final : public RetryPolicy { -public: - RetryPolicyTest_CustomRetry_CopySource(RetryOptions const& retryOptions) - : RetryPolicy(retryOptions) - { - } - - std::unique_ptr Clone() const override - { - return std::make_unique(*this); - } - -protected: - bool ShouldRetry(std::unique_ptr const& response, RetryOptions const&) const override - { - if (response == nullptr) - { - return true; - } - - if (static_cast>( - response->GetStatusCode()) - >= 400) - { - const auto& headers = response->GetHeaders(); - auto ite = headers.find("x-ms-error-code"); - if (ite != headers.end()) - { - const std::string error = ite->second; - - if (error == "InternalError" || error == "OperationTimedOut" || error == "ServerBusy") - { - return true; - } - } - } - - if (static_cast>( - response->GetStatusCode()) - >= 400) - { - const auto& headers = response->GetHeaders(); - auto ite = headers.find("x-ms-copy-source-error-code"); - if (ite != headers.end()) - { - const std::string error = ite->second; - - if (error == "InternalError" || error == "OperationTimedOut" || error == "ServerBusy") - { - return true; - } - } - } - return false; - } -}; } // namespace -TEST(RetryPolicy, ShouldRetry) -{ - using namespace std::chrono_literals; - RetryOptions const retryOptions{3, 10ms, 60s, {HttpStatusCode::Ok}}; - - // The default ShouldRetry implementation is to always return false, - // which means we will retry up to the retry count in the options, unless the status code isn't - // one that is retriable. - { - Azure::Core::Context context = Azure::Core::Context(); - auto policy = RetryPolicy(retryOptions); - - RawResponse const* responsePtrSent = nullptr; - int32_t retryCounter = -1; - - std::vector> policies; - policies.emplace_back(std::make_unique(policy)); - policies.emplace_back(std::make_unique([&]() { - auto response = std::make_unique(1, 1, HttpStatusCode::Ok, "Test"); - responsePtrSent = response.get(); - retryCounter++; - return response; - })); - - Azure::Core::Http::_internal::HttpPipeline pipeline(policies); - - Request request(HttpMethod::Get, Azure::Core::Url("https://www.microsoft.com")); - pipeline.Send(request, context); - - EXPECT_NE(responsePtrSent, nullptr); - EXPECT_EQ(retryCounter, 3); - } - - // If the status code is retriable based on the options, ShouldRetry doesn't get called. - // Testing short-circuit evaluation - { - Azure::Core::Context context = Azure::Core::Context(); - auto policy = RetryPolicyTest_CustomRetry_Throw(retryOptions); - - RawResponse const* responsePtrSent = nullptr; - int32_t retryCounter = -1; - - std::vector> policies; - policies.emplace_back(std::make_unique(policy)); - policies.emplace_back(std::make_unique([&]() { - auto response = std::make_unique(1, 1, HttpStatusCode::Ok, "Test"); - responsePtrSent = response.get(); - retryCounter++; - return response; - })); - - Azure::Core::Http::_internal::HttpPipeline pipeline(policies); - - Request request(HttpMethod::Get, Azure::Core::Url("https://www.microsoft.com")); - pipeline.Send(request, context); - - EXPECT_NE(responsePtrSent, nullptr); - EXPECT_EQ(retryCounter, 3); - } - - // If the status code isn't retriable based on the options, only then does ShouldRetry get called. - // Since the default of ShouldRetry is false, we don't expect any retries. - { - Azure::Core::Context context = Azure::Core::Context(); - auto policy = RetryPolicy(retryOptions); - - RawResponse const* responsePtrSent = nullptr; - int32_t retryCounter = -1; - - std::vector> policies; - policies.emplace_back(std::make_unique(policy)); - policies.emplace_back(std::make_unique([&]() { - auto response = std::make_unique(1, 1, HttpStatusCode::Accepted, "Test"); - responsePtrSent = response.get(); - retryCounter++; - return response; - })); - - Azure::Core::Http::_internal::HttpPipeline pipeline(policies); - - Request request(HttpMethod::Get, Azure::Core::Url("https://www.microsoft.com")); - pipeline.Send(request, context); - - EXPECT_NE(responsePtrSent, nullptr); - EXPECT_EQ(retryCounter, 0); - } - - // Overriding ShouldRetry to return true will mean we will retry, when the status code isn't - // retriable based on the options. - { - Azure::Core::Context context = Azure::Core::Context(); - auto policy = RetryPolicyTest_CustomRetry_True(retryOptions); - - RawResponse const* responsePtrSent = nullptr; - int32_t retryCounter = -1; - - std::vector> policies; - policies.emplace_back(std::make_unique(policy)); - policies.emplace_back(std::make_unique([&]() { - auto response = std::make_unique(1, 1, HttpStatusCode::Accepted, "Test"); - responsePtrSent = response.get(); - retryCounter++; - return response; - })); - - Azure::Core::Http::_internal::HttpPipeline pipeline(policies); - - Request request(HttpMethod::Get, Azure::Core::Url("https://www.microsoft.com")); - pipeline.Send(request, context); - - EXPECT_NE(responsePtrSent, nullptr); - EXPECT_EQ(retryCounter, 3); - } - - // Copy Status Code scenario (non-retriable HTTP status code) - { - Azure::Core::Context context = Azure::Core::Context(); - auto policy = RetryPolicyTest_CustomRetry_CopySource(RetryOptions()); - - RawResponse const* responsePtrSent = nullptr; - int32_t retryCounter = -1; - - std::vector> policies; - policies.emplace_back(std::make_unique(policy)); - policies.emplace_back(std::make_unique([&]() { - auto response = std::make_unique(1, 1, HttpStatusCode::Conflict, "Test"); - response->SetHeader("x-ms-copy-source-error-code", "OperationTimedOut"); - responsePtrSent = response.get(); - retryCounter++; - return response; - })); - - Azure::Core::Http::_internal::HttpPipeline pipeline(policies); - - Request request(HttpMethod::Get, Azure::Core::Url("https://www.microsoft.com")); - pipeline.Send(request, context); - - EXPECT_NE(responsePtrSent, nullptr); - EXPECT_EQ(retryCounter, 3); - } -} - TEST(RetryPolicy, ShouldRetryOnResponse) { using namespace std::chrono_literals; diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 618999e744..8606dd261d 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -53,6 +53,7 @@ extends: LiveTestTimeoutInMinutes: 90 # default is 60 min. We need a little longer on worst case for Win+jsonTests LineCoverageTarget: 88 BranchCoverageTarget: 50 + UseFederatedAuth: true # PreTestSteps: # - pwsh: | # docker build -t squid-local $(Build.SourcesDirectory)/sdk/core/azure-core/test/ut/proxy_tests/localproxy diff --git a/sdk/core/perf/inc/azure/perf/base_test.hpp b/sdk/core/perf/inc/azure/perf/base_test.hpp index cc0c7f5f05..6b2d0ece71 100644 --- a/sdk/core/perf/inc/azure/perf/base_test.hpp +++ b/sdk/core/perf/inc/azure/perf/base_test.hpp @@ -100,7 +100,7 @@ namespace Azure { namespace Perf { * * @return The value of the environment variable retrieved. * - * @note If AZURE_TENANT_ID, AZURE_CLIENT_ID, or AZURE_CLIENT_SECRET are not available in the + * @note If AZURE_TENANT_ID or AZURE_CLIENT_ID are not available in the * environment, the AZURE_SERVICE_DIRECTORY environment variable is used to set those values * with the values emitted by the New-TestResources.ps1 script. * diff --git a/sdk/core/perf/src/base_test.cpp b/sdk/core/perf/src/base_test.cpp index c5ab583014..037071d13f 100644 --- a/sdk/core/perf/src/base_test.cpp +++ b/sdk/core/perf/src/base_test.cpp @@ -11,6 +11,8 @@ #endif #include #include +#include +#include #include #include @@ -285,7 +287,15 @@ namespace Azure { namespace Perf { } if (clientSecret.empty()) { - m_testCredential = std::make_shared(); + m_testCredential = std::make_shared( + Azure::Identity::ChainedTokenCredential::Sources{ + std ::make_shared( + Azure::Core::_internal::Environment::GetVariable("AZURESUBSCRIPTION_TENANT_ID"), + Azure::Core::_internal::Environment::GetVariable("AZURESUBSCRIPTION_CLIENT_ID"), + Azure::Core::_internal::Environment::GetVariable( + "AZURESUBSCRIPTION_SERVICE_CONNECTION_ID"), + Azure::Core::_internal::Environment::GetVariable("SYSTEM_ACCESSTOKEN")), + std::make_shared()}); } else { diff --git a/sdk/eventhubs/azure-messaging-eventhubs/test/eventhubs-stress-test/src/eventhubs_stress_test.cpp b/sdk/eventhubs/azure-messaging-eventhubs/test/eventhubs-stress-test/src/eventhubs_stress_test.cpp index d43fd9af20..f2845599e7 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/test/eventhubs-stress-test/src/eventhubs_stress_test.cpp +++ b/sdk/eventhubs/azure-messaging-eventhubs/test/eventhubs-stress-test/src/eventhubs_stress_test.cpp @@ -90,8 +90,28 @@ void InitTracer(const std::string& stressScenarioName) auto processor = trace_sdk::SimpleSpanProcessorFactory::Create(std::move(exporter)); auto resource{GetTraceResource(stressScenarioName)}; + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4996) +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + // https://github.com/Azure/azure-sdk-for-cpp/issues/5784 + std::shared_ptr provider = trace_sdk::TracerProviderFactory::Create(std::move(processor), std::move(resource)); +#ifdef _MSC_VER +#pragma warning(pop) +#elif defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif // Set the global trace provider trace::Provider::SetTracerProvider(provider); @@ -181,8 +201,27 @@ void InitLogger(const std::string& stressScenarioName) auto resource{GetTraceResource(stressScenarioName)}; +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4996) +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + // https://github.com/Azure/azure-sdk-for-cpp/issues/5784 + std::shared_ptr provider = logs_sdk::LoggerProviderFactory::Create(std::move(processor), std::move(resource)); +#ifdef _MSC_VER +#pragma warning(pop) +#elif defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif // Set the global log provider. logs::Provider::SetLoggerProvider(provider); diff --git a/sdk/eventhubs/ci.yml b/sdk/eventhubs/ci.yml index a87a348b36..b6bce45415 100644 --- a/sdk/eventhubs/ci.yml +++ b/sdk/eventhubs/ci.yml @@ -32,6 +32,7 @@ extends: LiveTestTimeoutInMinutes: 120 LineCoverageTarget: 27 BranchCoverageTarget: 13 + UseFederatedAuth: true Artifacts: - Name: azure-messaging-eventhubs Path: azure-messaging-eventhubs diff --git a/sdk/identity/azure-identity/samples/CMakeLists.txt b/sdk/identity/azure-identity/samples/CMakeLists.txt index c314f8ee70..9eec6e9a6e 100644 --- a/sdk/identity/azure-identity/samples/CMakeLists.txt +++ b/sdk/identity/azure-identity/samples/CMakeLists.txt @@ -31,22 +31,22 @@ target_link_libraries(workload_identity_credential_sample PRIVATE azure-identity target_include_directories(workload_identity_credential_sample PRIVATE .) create_per_service_target_build_for_sample(identity workload_identity_credential_sample) -add_executable(client_secret_credential_sample client_secret_credential.cpp) -target_link_libraries(client_secret_credential_sample PRIVATE azure-identity service get-env-helper) -target_include_directories(client_secret_credential_sample PRIVATE .) -create_per_service_target_build_for_sample(identity client_secret_credential_sample) - -add_executable(default_azure_credential_sample default_azure_credential.cpp) -target_link_libraries(default_azure_credential_sample PRIVATE azure-identity service) -target_include_directories(default_azure_credential_sample PRIVATE .) -create_per_service_target_build_for_sample(identity default_azure_credential_sample) - -add_executable(environment_credential_sample environment_credential.cpp) -target_link_libraries(environment_credential_sample PRIVATE azure-identity service) -target_include_directories(environment_credential_sample PRIVATE .) -create_per_service_target_build_for_sample(identity environment_credential_sample) - -add_executable(managed_identity_credential_sample managed_identity_credential.cpp) -target_link_libraries(managed_identity_credential_sample PRIVATE azure-identity service) -target_include_directories(managed_identity_credential_sample PRIVATE .) -create_per_service_target_build_for_sample(identity managed_identity_credential_sample) +#add_executable(client_secret_credential_sample client_secret_credential.cpp) +#target_link_libraries(client_secret_credential_sample PRIVATE azure-identity service get-env-helper) +#target_include_directories(client_secret_credential_sample PRIVATE .) +#create_per_service_target_build_for_sample(identity client_secret_credential_sample) + +#add_executable(default_azure_credential_sample default_azure_credential.cpp) +#target_link_libraries(default_azure_credential_sample PRIVATE azure-identity service) +#target_include_directories(default_azure_credential_sample PRIVATE .) +#create_per_service_target_build_for_sample(identity default_azure_credential_sample) + +#add_executable(environment_credential_sample environment_credential.cpp) +#target_link_libraries(environment_credential_sample PRIVATE azure-identity service) +#target_include_directories(environment_credential_sample PRIVATE .) +#create_per_service_target_build_for_sample(identity environment_credential_sample) + +#add_executable(managed_identity_credential_sample managed_identity_credential.cpp) +#target_link_libraries(managed_identity_credential_sample PRIVATE azure-identity service) +#target_include_directories(managed_identity_credential_sample PRIVATE .) +#create_per_service_target_build_for_sample(identity managed_identity_credential_sample) diff --git a/sdk/identity/azure-identity/test/ut/azure_pipelines_credential_test.cpp b/sdk/identity/azure-identity/test/ut/azure_pipelines_credential_test.cpp index 02f110e33b..8d5280be6a 100644 --- a/sdk/identity/azure-identity/test/ut/azure_pipelines_credential_test.cpp +++ b/sdk/identity/azure-identity/test/ut/azure_pipelines_credential_test.cpp @@ -644,7 +644,7 @@ TEST(AzurePipelinesCredential, InvalidServiceConnectionId_LIVEONLY_) } } -TEST(AzurePipelinesCredential, InvalidSystemAccessToken_LIVEONLY_) +TEST(AzurePipelinesCredential, DISABLED_InvalidSystemAccessToken_LIVEONLY_) { std::string tenantId = Environment::GetVariable("AZURESUBSCRIPTION_TENANT_ID"); std::string clientId = Environment::GetVariable("AZURESUBSCRIPTION_CLIENT_ID"); diff --git a/sdk/identity/azure-identity/test/ut/token_credential_test.cpp b/sdk/identity/azure-identity/test/ut/token_credential_test.cpp index 319486c253..e9468b5533 100644 --- a/sdk/identity/azure-identity/test/ut/token_credential_test.cpp +++ b/sdk/identity/azure-identity/test/ut/token_credential_test.cpp @@ -55,6 +55,12 @@ using namespace Azure::Identity; TEST_F(TokenCredentialTest, ClientSecret) { + if (m_testContext.IsLiveMode()) + { + GTEST_SKIP_( + "Skipping ClientSecret test since it requires env vars that aren't set in live mode."); + } + std::string const testName(GetTestName()); auto const clientSecretCredential = GetClientSecretCredential(testName); @@ -71,6 +77,12 @@ TEST_F(TokenCredentialTest, ClientSecret) TEST_F(TokenCredentialTest, EnvironmentCredential) { + if (m_testContext.IsLiveMode()) + { + GTEST_SKIP_("Skipping EnvironmentCredential test since it requires env vars that aren't set in " + "live mode."); + } + std::string const testName(GetTestName()); auto const clientSecretCredential = GetEnvironmentCredential(testName); diff --git a/sdk/identity/ci.yml b/sdk/identity/ci.yml index 2fcb520b3b..053746e82d 100644 --- a/sdk/identity/ci.yml +++ b/sdk/identity/ci.yml @@ -30,6 +30,7 @@ extends: LiveTestCtestRegex: azure-identity. LineCoverageTarget: 95 BranchCoverageTarget: 56 + UseFederatedAuth: true Artifacts: - Name: azure-identity Path: azure-identity diff --git a/sdk/identity/test-resources.json b/sdk/identity/test-resources.json index b7d84ff613..b71f8c95d4 100644 --- a/sdk/identity/test-resources.json +++ b/sdk/identity/test-resources.json @@ -14,12 +14,6 @@ "metadata": { "description": "The application client ID used to run tests." } - }, - "testApplicationSecret": { - "type": "string", - "metadata": { - "description": "The application client secret used to run tests." - } } }, "resources": [], @@ -31,10 +25,6 @@ "AZURE_CLIENT_ID": { "type": "string", "value": "[parameters('testApplicationId')]" - }, - "AZURE_CLIENT_SECRET": { - "type": "string", - "value": "[parameters('testApplicationSecret')]" } } } diff --git a/sdk/keyvault/azure-security-keyvault-certificates/CMakeLists.txt b/sdk/keyvault/azure-security-keyvault-certificates/CMakeLists.txt index b6883613d5..9099e28528 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/CMakeLists.txt +++ b/sdk/keyvault/azure-security-keyvault-certificates/CMakeLists.txt @@ -117,7 +117,7 @@ if (BUILD_PERFORMANCE_TESTS) add_subdirectory(test/perf) endif() -if(BUILD_SAMPLES) +if(BUILD_SAMPLES_DISABLED) add_subdirectory(samples) endif() diff --git a/sdk/keyvault/azure-security-keyvault-keys/CMakeLists.txt b/sdk/keyvault/azure-security-keyvault-keys/CMakeLists.txt index 9a18bee072..e15956e358 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/CMakeLists.txt +++ b/sdk/keyvault/azure-security-keyvault-keys/CMakeLists.txt @@ -154,7 +154,7 @@ if (BUILD_PERFORMANCE_TESTS) add_subdirectory(test/perf) endif() -if(BUILD_SAMPLES) +if(BUILD_SAMPLES_DISABLED) add_subdirectory(samples) endif() diff --git a/sdk/keyvault/azure-security-keyvault-keys/README.md b/sdk/keyvault/azure-security-keyvault-keys/README.md index 9a4344fc0f..98c2dd7241 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/README.md +++ b/sdk/keyvault/azure-security-keyvault-keys/README.md @@ -79,11 +79,10 @@ Use the [Azure CLI][azure_cli] snippet below to create/get client secret credent ``` "" ``` -- Use the returned credentials above to set **AZURE_CLIENT_ID** (appId), **AZURE_CLIENT_SECRET** (password), and **AZURE_TENANT_ID** (tenant) environment variables. The following example shows a way to do this in Powershell: +- Use the returned credentials above to set **AZURE_CLIENT_ID** (appId) and **AZURE_TENANT_ID** (tenant) environment variables. The following example shows a way to do this in Powershell: ```PowerShell $Env:AZURE_CLIENT_ID="generated-app-ID" - $Env:AZURE_CLIENT_SECRET="random-password" $Env:AZURE_TENANT_ID="tenant-ID" ``` diff --git a/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt b/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt index 56b5e0a890..f6521b342b 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt +++ b/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt @@ -117,7 +117,7 @@ if (BUILD_PERFORMANCE_TESTS) add_subdirectory(test/perf) endif() -if(BUILD_SAMPLES) +if(BUILD_SAMPLES_DISABLED) add_subdirectory(samples) endif() diff --git a/sdk/keyvault/ci.yml b/sdk/keyvault/ci.yml index 9b3a6358db..c491e25ee7 100644 --- a/sdk/keyvault/ci.yml +++ b/sdk/keyvault/ci.yml @@ -32,6 +32,7 @@ extends: LiveTestTimeoutInMinutes: 120 LineCoverageTarget: 81 BranchCoverageTarget: 42 + UseFederatedAuth: true Artifacts: - Name: azure-security-keyvault-keys Path: azure-security-keyvault-keys diff --git a/sdk/storage/README.md b/sdk/storage/README.md index f15f3b51c5..e98d4ae856 100644 --- a/sdk/storage/README.md +++ b/sdk/storage/README.md @@ -36,4 +36,3 @@ additional questions or comments. [coc]: https://opensource.microsoft.com/codeofconduct/ [coc_faq]: https://opensource.microsoft.com/codeofconduct/faq/ [coc_contact]: mailto:opencode@microsoft.com - \ No newline at end of file diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index ddf0129e76..ca8ba906d7 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -33,6 +33,7 @@ extends: LiveTestCtestRegex: azure-storage Clouds: Preview SupportedClouds: Preview + UseFederatedAuth: false Artifacts: - Name: azure-storage-common Path: azure-storage-common diff --git a/sdk/tables/ci.yml b/sdk/tables/ci.yml index 350b30e121..eb9e9e09b0 100644 --- a/sdk/tables/ci.yml +++ b/sdk/tables/ci.yml @@ -30,6 +30,7 @@ extends: CtestRegex: azure-data LineCoverageTarget: 77 BranchCoverageTarget: 42 + UseFederatedAuth: true LiveTestCtestRegex: azure-data Clouds: Preview SupportedClouds: Preview diff --git a/vcpkg.json b/vcpkg.json index dee6e60122..a6b6cd1350 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,7 +1,7 @@ { "name": "azure-sdk-for-cpp", "version": "1.5.0", - "builtin-baseline": "8150939b69720adc475461978e07c2d2bf5fb76e", + "builtin-baseline": "5312e9f976e89b256954f571433e34f783dd2d12", "dependencies": [ { "name": "curl"