Skip to content

Commit c59ab4e

Browse files
authored
Use Azure cli or PowerShell credentials for devops and oss service (#8298)
* Use Azure cli or PowerShell credentials for devops and oss service * Move alias caching into GitHubToAADConverter and update pipelines * Stop retrying on 404s * Use new app ID for OSS portal * Skip tests for notification tool
1 parent b6035ed commit c59ab4e

18 files changed

Lines changed: 140 additions & 286 deletions

File tree

eng/pipelines/notifications.yml

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -54,23 +54,20 @@ stages:
5454
arguments: 'install --global --add-source "$(DotNetDevOpsFeed)" --version "$(NotificationsCreatorVersion)" "Azure.Sdk.Tools.NotificationConfiguration"'
5555
workingDirectory: '$(Agent.BuildDirectory)'
5656

57-
- pwsh: |
58-
notification-creator `
59-
--organization $(Organization) `
60-
--project $(Project) `
61-
--path-prefix "\$(PathPrefix)" `
62-
--token-variable-name DEVOPS_TOKEN `
63-
--aad-app-id-var OPENSOURCE_AAD_APP_ID `
64-
--aad-app-secret-var OPENSOURCE_AAD_APP_SECRET `
65-
--aad-tenant-var OPENSOURCE_AAD_TENANT_ID `
66-
--selection-strategy Scheduled `
67-
$(AdditionalParameters)
57+
- task: AzureCLI@2
6858
displayName: 'Run Team/Notification Creator'
59+
inputs:
60+
azureSubscription: 'opensource-api-connection'
61+
scriptType: pscore
62+
scriptLocation: inlineScript
63+
inlineScript:
64+
notification-creator `
65+
--organization $(Organization) `
66+
--project $(Project) `
67+
--path-prefix "\$(PathPrefix)" `
68+
--selection-strategy Scheduled `
69+
$(AdditionalParameters)
6970
env:
7071
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
7172
DOTNET_CLI_TELEMETRY_OPTOUT: 1
7273
DOTNET_MULTILEVEL_LOOKUP: 0
73-
DEVOPS_TOKEN: $(azure-sdk-notification-tools-pat)
74-
OPENSOURCE_AAD_APP_ID: $(opensource-aad-app-id)
75-
OPENSOURCE_AAD_APP_SECRET: $(opensource-aad-secret)
76-
OPENSOURCE_AAD_TENANT_ID: $(opensource-aad-tenant-id)

eng/pipelines/pipeline-owners-extraction.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ stages:
4949
- task: AzureCLI@2
5050
displayName: Run Pipeline Owners Extractor
5151
inputs:
52-
azureSubscription: 'Azure SDK Engineering System'
52+
azureSubscription: 'opensource-api-connection'
5353
scriptType: pscore
5454
scriptLocation: inlineScript
5555
inlineScript: pipeline-owners-extractor --output "$(OutputPath)"

tools/codeowners-utils/Azure.Sdk.Tools.CodeownersUtils/Utils/FileHelpers.cs

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,10 @@ private static string GetUrlContents(string url)
5757
using HttpClient client = new HttpClient();
5858
while (attempts <= maxRetries)
5959
{
60+
HttpResponseMessage response = null;
6061
try
6162
{
62-
HttpResponseMessage response = client.GetAsync(url).ConfigureAwait(false).GetAwaiter().GetResult();
63+
response = client.GetAsync(url).ConfigureAwait(false).GetAwaiter().GetResult();
6364
if (response.StatusCode == HttpStatusCode.OK)
6465
{
6566
// This writeline is probably unnecessary but good to have if there are previous attempts that failed
@@ -76,17 +77,22 @@ private static string GetUrlContents(string url)
7677
// HttpRequestException means the request failed due to an underlying issue such as network connectivity,
7778
// DNS failure, server certificate validation or timeout.
7879
Console.WriteLine($"GetUrlContents attempt number {attempts}. HttpRequestException trying to fetch {url}. Exception message = {httpReqEx.Message}");
79-
if (attempts == maxRetries)
80-
{
81-
// At this point the retries have been exhausted, let this rethrow
82-
throw;
83-
}
8480
}
85-
System.Threading.Thread.Sleep(delayTimeInMs);
81+
82+
// Skip retries on a NotFound response
83+
if (response?.StatusCode == HttpStatusCode.NotFound)
84+
{
85+
break;
86+
}
87+
88+
if (attempts < maxRetries)
89+
{
90+
System.Threading.Thread.Sleep(delayTimeInMs);
91+
}
8692
attempts++;
8793
}
8894
// This will only get hit if the final retry is non-OK status code
89-
throw new FileLoadException($"Unable to fetch {url} after {maxRetries}. See above for status codes for each attempt.");
95+
throw new FileLoadException($"Unable to fetch {url} after {attempts} attempts. See above for status codes for each attempt.");
9096
}
9197
}
9298
}
Lines changed: 61 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,96 +1,107 @@
11
using System;
2-
using System.Net;
2+
using System.Collections.Generic;
3+
using System.Linq;
34
using System.Net.Http;
5+
using System.Threading;
46
using System.Threading.Tasks;
57
using Azure.Core;
6-
using Azure.Identity;
78
using Microsoft.Extensions.Logging;
89
using Models.OpenSourcePortal;
910
using Newtonsoft.Json;
1011

1112
namespace Azure.Sdk.Tools.NotificationConfiguration.Helpers
1213
{
14+
/// <summary>
15+
/// Utility class for converting GitHub usernames to AAD user principal names.
16+
/// </summary>
17+
/// <remarks>
18+
/// A map of GitHub usernames to AAD user principal names is cached in memory to avoid making multiple calls to the
19+
/// OpenSource portal API. The cache is initialized with the full alias list on the first call to
20+
/// GetUserPrincipalNameFromGithubAsync.
21+
/// </remarks>
1322
public class GitHubToAADConverter
1423
{
24+
private readonly TokenCredential credential;
25+
private readonly ILogger<GitHubToAADConverter> logger;
26+
private readonly SemaphoreSlim cacheLock = new(1);
27+
private Dictionary<string, string> lookupCache;
28+
1529
/// <summary>
1630
/// GitHubToAadConverter constructor for generating new token, and initialize http client.
1731
/// </summary>
1832
/// <param name="credential">The aad token auth class.</param>
1933
/// <param name="logger">Logger</param>
20-
public GitHubToAADConverter(
21-
ClientSecretCredential credential,
22-
ILogger<GitHubToAADConverter> logger)
34+
public GitHubToAADConverter(TokenCredential credential, ILogger<GitHubToAADConverter> logger)
2335
{
36+
this.credential = credential;
2437
this.logger = logger;
25-
var opsAuthToken = "";
38+
39+
}
40+
41+
public async Task<string> GetUserPrincipalNameFromGithubAsync(string gitHubUserName)
42+
{
43+
await EnsureCacheExistsAsync();
44+
45+
if (this.lookupCache.TryGetValue(gitHubUserName, out string aadUserPrincipalName))
46+
{
47+
return aadUserPrincipalName;
48+
}
49+
50+
return null;
51+
}
52+
53+
public async Task EnsureCacheExistsAsync()
54+
{
55+
await this.cacheLock.WaitAsync();
2656
try
2757
{
28-
// This is aad scope of opensource rest API.
29-
string[] scopes = new string[]
58+
if (this.lookupCache == null)
3059
{
31-
"api://2789159d-8d8b-4d13-b90b-ca29c1707afd/.default"
32-
};
33-
opsAuthToken = credential.GetToken(new TokenRequestContext(scopes)).Token;
60+
var peopleLinks = await GetPeopleLinksAsync();
61+
this.lookupCache = peopleLinks.ToDictionary(
62+
x => x.GitHub.Login,
63+
x => x.Aad.UserPrincipalName,
64+
StringComparer.OrdinalIgnoreCase);
65+
}
3466
}
35-
catch (Exception ex)
67+
finally
3668
{
37-
logger.LogError("Failed to generate aad token. " + ex.Message);
69+
this.cacheLock.Release();
3870
}
39-
client = new HttpClient();
40-
client.DefaultRequestHeaders.Add("content_type", "application/json");
41-
client.DefaultRequestHeaders.Add("api-version", "2019-10-01");
42-
client.DefaultRequestHeaders.Add("Authorization", $"Bearer {opsAuthToken}");
4371
}
4472

45-
private readonly HttpClient client;
46-
private readonly ILogger<GitHubToAADConverter> logger;
47-
48-
/// <summary>
49-
/// Get the user principal name from github. User principal name is in format of ms email.
50-
/// </summary>
51-
/// <param name="githubUserName">github user name</param>
52-
/// <returns>Aad user principal name</returns>
53-
public string GetUserPrincipalNameFromGithub(string githubUserName)
73+
private async Task<UserLink[]> GetPeopleLinksAsync()
5474
{
55-
return GetUserPrincipalNameFromGithubAsync(githubUserName).Result;
56-
}
75+
AccessToken opsAuthToken;
5776

58-
public async Task<string> GetUserPrincipalNameFromGithubAsync(string githubUserName)
59-
{
6077
try
6178
{
62-
var responseJsonString = await client.GetStringAsync($"https://repos.opensource.microsoft.com/api/people/links/github/{githubUserName}");
63-
dynamic contentJson = JsonConvert.DeserializeObject(responseJsonString);
64-
return contentJson.aad.userPrincipalName;
65-
}
66-
catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
67-
{
68-
logger.LogWarning("Github username {Username} not found", githubUserName);
79+
// This is aad scope of opensource rest API.
80+
string[] scopes = new [] { "api://66b6ea26-954d-4b68-8f48-71e3faec7ad1/.default" };
81+
opsAuthToken = await credential.GetTokenAsync(new TokenRequestContext(scopes), CancellationToken.None);
6982
}
7083
catch (Exception ex)
7184
{
72-
logger.LogError(ex.Message);
85+
this.logger.LogError("Failed to generate aad token. {ExceptionMessage}", ex.Message);
86+
throw;
7387
}
7488

75-
return null;
76-
}
77-
78-
public async Task<UserLink[]> GetPeopleLinksAsync()
79-
{
8089
try
8190
{
82-
logger.LogInformation("Calling GET https://repos.opensource.microsoft.com/api/people/links");
83-
var responseJsonString = await client.GetStringAsync($"https://repos.opensource.microsoft.com/api/people/links");
84-
var allLinks = JsonConvert.DeserializeObject<UserLink[]>(responseJsonString);
91+
using HttpClient client = new ();
92+
client.DefaultRequestHeaders.Add("content_type", "application/json");
93+
client.DefaultRequestHeaders.Add("api-version", "2019-10-01");
94+
client.DefaultRequestHeaders.Add("Authorization", $"Bearer {opsAuthToken.Token}");
8595

86-
return allLinks;
96+
this.logger.LogInformation("Calling GET https://repos.opensource.microsoft.com/api/people/links");
97+
string responseJsonString = await client.GetStringAsync($"https://repos.opensource.microsoft.com/api/people/links");
98+
return JsonConvert.DeserializeObject<UserLink[]>(responseJsonString);
8799
}
88100
catch (Exception ex)
89101
{
90-
logger.LogError(ex.Message);
102+
this.logger.LogError(ex, "Error getting people links from opensource.microsoft.com: {ExceptionMessage}", ex.Message);
103+
throw;
91104
}
92-
93-
return null;
94105
}
95106
}
96107
}

tools/identity-resolution/Services/AzureDevOpsService.cs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,12 @@
99
using Microsoft.VisualStudio.Services.Notifications.WebApi;
1010
using System;
1111
using Microsoft.Extensions.Logging;
12-
using Microsoft.VisualStudio.Services.Common;
12+
using Microsoft.VisualStudio.Services.Client;
1313
using Microsoft.VisualStudio.Services.Identity.Client;
1414
using System.Threading;
1515
using System.Linq;
1616
using MicrosoftIdentityAlias = Microsoft.VisualStudio.Services.Identity;
17+
using Azure.Core;
1718

1819
namespace Azure.Sdk.Tools.NotificationConfiguration.Services
1920
{
@@ -27,9 +28,9 @@ public class AzureDevOpsService
2728
private Dictionary<Type, VssHttpClientBase> clientCache = new Dictionary<Type, VssHttpClientBase>();
2829
private SemaphoreSlim clientCacheSemaphore = new SemaphoreSlim(1);
2930

30-
public static AzureDevOpsService CreateAzureDevOpsService(string token, string url, ILogger<AzureDevOpsService> logger)
31+
public static AzureDevOpsService CreateAzureDevOpsService(TokenCredential tokenCredential, string url, ILogger<AzureDevOpsService> logger)
3132
{
32-
var devOpsCreds = new VssBasicCredential("nobody", token);
33+
var devOpsCreds = new VssAzureIdentityCredential(tokenCredential);
3334
var devOpsConnection = new VssConnection(new Uri(url), devOpsCreds);
3435
var result = new AzureDevOpsService(devOpsConnection, logger);
3536

tools/identity-resolution/Services/GitHubService.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
using System.Collections.Concurrent;
44
using System.Collections.Generic;
55
using System.Linq;
6-
using System.Net.Http;
7-
using System.Threading.Tasks;
86
using Azure.Sdk.Tools.CodeownersUtils.Parsing;
97
using System.IO;
108

tools/identity-resolution/identity-resolution.csproj

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77

88
<ItemGroup>
99
<PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
10-
<PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.170.0" />
11-
<PackageReference Include="Microsoft.VisualStudio.Services.Notifications.WebApi" Version="16.170.0" />
10+
<PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="19.239.0-preview" />
11+
<PackageReference Include="Microsoft.VisualStudio.Services.Notifications.WebApi" Version="19.239.0-preview" />
12+
<PackageReference Include="Microsoft.VisualStudio.Services.InteractiveClient" Version="19.239.0-preview" />
1213
<PackageReference Include="YamlDotNet" Version="6.1.1" />
1314
<PackageReference Include="Azure.Identity" Version="1.10.2" />
1415
</ItemGroup>

tools/notification-configuration/ci.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,4 @@ extends:
2929
template: /eng/pipelines/templates/stages/archetype-sdk-tool-dotnet.yml
3030
parameters:
3131
ToolDirectory: tools/notification-configuration
32+
TestMatrix: {}

tools/notification-configuration/notification-creator.Tests/ProgramTests.cs

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,13 @@ public class ProgramTests
99
[Test]
1010
public void ThrowsVssUnauthorizedException()
1111
{
12-
Environment.SetEnvironmentVariable("aadAppIdVar", "aadAppIdVarValue");
13-
Environment.SetEnvironmentVariable("aadAppSecretVar", "aadAppSecretVarValue");
14-
Environment.SetEnvironmentVariable("aadTenantVar", "aadTenantVarValue");
1512
Assert.ThrowsAsync<Microsoft.VisualStudio.Services.Common.VssUnauthorizedException>(
1613
async () =>
1714
// Act
1815
await Program.Main(
1916
organization: "fooOrg",
2017
project: "barProj",
2118
pathPrefix: "qux",
22-
tokenVariableName: "token",
23-
aadAppIdVar: "aadAppIdVar",
24-
aadAppSecretVar: "aadAppSecretVar",
25-
aadTenantVar: "aadTenantVar",
2619
selectionStrategy: PipelineSelectionStrategy.Scheduled,
2720
dryRun: true)
2821
);

tools/notification-configuration/notification-creator/NotificationConfigurator.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ private async Task SyncTeamWithCodeownersFile(
196196
if (!contactsCache.ContainsKey(contact))
197197
{
198198
// TODO: Better to have retry if no success on this call.
199-
var userPrincipal = gitHubToAADConverter.GetUserPrincipalNameFromGithub(contact);
199+
var userPrincipal = await gitHubToAADConverter.GetUserPrincipalNameFromGithubAsync(contact);
200200
if (!string.IsNullOrEmpty(userPrincipal))
201201
{
202202
contactsCache[contact] = await service.GetDescriptorForPrincipal(userPrincipal);

0 commit comments

Comments
 (0)