Skip to content

Commit 51d0611

Browse files
JimSuplizioJimSuplizio
authored
Add RestAPISpecsDocsRepos to branch cleanup and update the delete scripts to output limits and telemetry data (#8830)
* Add RestAPISpecsDocsRepos to branch cleanup. * Remove commented out yml, add pre-message to limit output and output core limit at the start of processing * updates for feedback
1 parent a62328f commit 51d0611

3 files changed

Lines changed: 268 additions & 50 deletions

File tree

‎eng/common/scripts/Delete-RemoteBranches.ps1‎

Lines changed: 104 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ param(
1515
$AuthToken
1616
)
1717
Set-StrictMode -version 3
18-
1918
. (Join-Path $PSScriptRoot common.ps1)
2019

2120
function Get-AllBranchesAndPullRequestInfo($owner, $repo) {
@@ -66,67 +65,124 @@ if ($AuthToken) {
6665
}
6766

6867
$owner, $repo = $RepoId -split "/"
69-
$branches = Get-AllBranchesAndPullRequestInfo $owner $repo
7068

71-
foreach ($branch in $branches)
72-
{
73-
$branchName = $branch.Name
74-
if ($branchName -notmatch $BranchRegex) {
75-
continue
69+
# These will always be output at the end of the script. Their only purpose is for information gathering
70+
# Total number returned from query
71+
$totalBranchesFromQuery = 0
72+
# reasons why a branch was skipped
73+
$skippedBranchNotMatchRegex = 0
74+
$skippedForCommitDate = 0
75+
$skippedForOpenPRs = 0
76+
$skippedForPRNotInBranch = 0
77+
$skippedForPRNotInRepo = 0
78+
# gh call counters
79+
$ghPRViewCalls = 0
80+
$ghBranchDeleteCalls = 0
81+
82+
try {
83+
# Output the core rate limit at the start of processing. There's no real need
84+
# to output this at the end because the GH call counts are being output
85+
$coreRateLimit = Get-RateLimit core
86+
Write-RateLimit $coreRateLimit
87+
# Output the GraphQL rate limit before and after the call
88+
$graphqlRateLimit = Get-RateLimit graphql
89+
Write-RateLimit $graphqlRateLimit "Before GraphQL Call"
90+
$branches = Get-AllBranchesAndPullRequestInfo $owner $repo
91+
$graphqlRateLimit = Get-RateLimit graphql
92+
Write-RateLimit $graphqlRateLimit "After GraphQL Call"
93+
94+
if ($branches) {
95+
$totalBranchesFromQuery = $branches.Count
7696
}
77-
$openPullRequests = @($branch.pullRequests | Where-Object { !$_.Closed })
7897

79-
# If we have a central PR that created this branch still open don't delete the branch
80-
if ($CentralRepoId)
98+
foreach ($branch in $branches)
8199
{
82-
$pullRequestNumber = $matches["PrNumber"]
83-
# If central PR number is not found, then skip
84-
if (!$pullRequestNumber) {
85-
LogError "No PR number found in the branch name. Please check the branch name '$branchName'. Skipping..."
100+
$branchName = $branch.Name
101+
if ($branchName -notmatch $BranchRegex) {
102+
$skippedBranchNotMatchRegex++
86103
continue
87104
}
105+
$openPullRequests = @($branch.pullRequests | Where-Object { !$_.Closed })
88106

89-
$centralPR = gh pr view --json 'url,closed' --repo $CentralRepoId $pullRequestNumber | ConvertFrom-Json
90-
if ($LASTEXITCODE) {
91-
LogError "PR '$pullRequestNumber' not found in repo '$CentralRepoId'. Skipping..."
92-
continue;
93-
} else {
94-
LogDebug "Found central PR $($centralPR.url) and Closed=$($centralPR.closed)"
95-
if (!$centralPR.Closed) {
96-
# Skipping if there is an open central PR open for the branch.
97-
LogDebug "Central PR is still open so skipping the deletion of branch '$branchName'. Skipping..."
98-
continue;
107+
# If we have a central PR that created this branch still open don't delete the branch
108+
if ($CentralRepoId)
109+
{
110+
$pullRequestNumber = $matches["PrNumber"]
111+
# If central PR number is not found, then skip
112+
if (!$pullRequestNumber) {
113+
LogError "No PR number found in the branch name. Please check the branch name '$branchName'. Skipping..."
114+
$skippedForPRNotInBranch++
115+
continue
116+
}
117+
118+
$ghPRViewCalls++
119+
$centralPR = gh pr view --json 'url,closed' --repo $CentralRepoId $pullRequestNumber | ConvertFrom-Json
120+
if ($LASTEXITCODE) {
121+
LogError "PR '$pullRequestNumber' not found in repo '$CentralRepoId'. Skipping..."
122+
$skippedForPRNotInRepo++
123+
continue
124+
} else {
125+
LogDebug "Found central PR $($centralPR.url) and Closed=$($centralPR.closed)"
126+
if (!$centralPR.Closed) {
127+
$skippedForOpenPRs++
128+
# Skipping if there is an open central PR open for the branch.
129+
LogDebug "Central PR is still open so skipping the deletion of branch '$branchName'. Skipping..."
130+
continue
131+
}
99132
}
100133
}
101-
}
102-
else {
103-
# Not CentralRepoId - not associated with a central repo PR
104-
if ($openPullRequests.Count -gt 0 -and !$DeleteBranchesEvenIfThereIsOpenPR) {
105-
LogDebug "Found open PRs associate with branch '$branchName'. Skipping..."
106-
continue
134+
else {
135+
# Not CentralRepoId - not associated with a central repo PR
136+
if ($openPullRequests.Count -gt 0 -and !$DeleteBranchesEvenIfThereIsOpenPR) {
137+
$skippedForOpenPRs++
138+
LogDebug "Found open PRs associate with branch '$branchName'. Skipping..."
139+
continue
140+
}
107141
}
108-
}
109142

110-
# If there is date filter, then check if branch last commit is older than the date.
111-
if ($LastCommitOlderThan)
112-
{
113-
$commitDate = $branch.committedDate
114-
if ($commitDate -gt $LastCommitOlderThan) {
115-
LogDebug "The branch $branch last commit date '$commitDate' is newer than the date '$LastCommitOlderThan'. Skipping..."
116-
continue
143+
# If there is date filter, then check if branch last commit is older than the date.
144+
if ($LastCommitOlderThan)
145+
{
146+
$commitDate = $branch.committedDate
147+
if ($commitDate -gt $LastCommitOlderThan) {
148+
$skippedForCommitDate++
149+
LogDebug "The branch $branch last commit date '$commitDate' is newer than the date '$LastCommitOlderThan'. Skipping..."
150+
continue
151+
}
117152
}
118-
}
119153

120-
foreach ($openPullRequest in $openPullRequests) {
121-
Write-Host "Note: Open pull Request '$($openPullRequest.url)' will be closed after branch deletion, given the central PR is closed."
122-
}
154+
foreach ($openPullRequest in $openPullRequests) {
155+
LogDebug "Note: Open pull Request '$($openPullRequest.url)' will be closed after branch deletion, given the central PR is closed."
156+
}
123157

124-
$commitUrl = $branch.commitUrl
125-
if ($PSCmdlet.ShouldProcess("'$branchName' in '$RepoId'", "Deleting branch on cleanup script")) {
126-
gh api "repos/${RepoId}/git/refs/heads/${branchName}" -X DELETE
127-
if ($LASTEXITCODE) {
128-
LogError "Deletion of branch '$branchName` failed"
158+
$commitUrl = $branch.commitUrl
159+
if ($PSCmdlet.ShouldProcess("'$branchName' in '$RepoId'", "Deleting branch on cleanup script")) {
160+
$ghBranchDeleteCalls++
161+
gh api "repos/${RepoId}/git/refs/heads/${branchName}" -X DELETE
162+
if ($LASTEXITCODE) {
163+
LogError "Deletion of branch '$branchName` failed, see command output above"
164+
exit $LASTEXITCODE
165+
}
166+
LogDebug "The branch '$branchName' at commit '$commitUrl' in '$RepoId' has been deleted."
129167
}
130-
Write-Host "The branch '$branchName' at commit '$commitUrl' in '$RepoId' has been deleted."
168+
}
169+
}
170+
finally {
171+
172+
173+
Write-Host "Number of branches returned from graphql query: $totalBranchesFromQuery"
174+
# The $BranchRegex seems to be always set
175+
if ($BranchRegex) {
176+
Write-Host "Number of branches that didn't match the BranchRegex: $skippedBranchNotMatchRegex"
177+
}
178+
Write-Host "Number of branches skipped for newer last commit date: $skippedForCommitDate"
179+
Write-Host "Number of branches skipped for open PRs: $skippedForOpenPRs"
180+
Write-Host "Number of gh api calls to delete branches: $ghBranchDeleteCalls"
181+
# The following are only applicable when $CentralRepoId is passed in
182+
if ($CentralRepoId) {
183+
Write-Host "The following are applicable because CentralRepoId was passed in:"
184+
Write-Host " Number of gh pr view calls: $ghPRViewCalls"
185+
Write-Host " Number of branches skipped due to PR not in the repository: $skippedForPRNotInRepo "
186+
Write-Host " Number of branches skipped due to PR not in the branch name: $skippedForPRNotInBranch"
131187
}
132188
}

‎eng/common/scripts/Helpers/git-helpers.ps1‎

Lines changed: 140 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ function Get-ChangedFiles {
1515
return ""
1616
}
1717

18-
# Add config to disable the quote and encoding on file name.
18+
# Add config to disable the quote and encoding on file name.
1919
# Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-quoted-file-names
2020
# Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-commit-message-transcoding
2121
# Git PR diff: https://docs.github.com/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-comparing-branches-in-pull-requests#three-dot-and-two-dot-git-diff-comparisons
@@ -91,7 +91,7 @@ class ConflictedFile {
9191
$lines = $IncomingContent -split "`r?`n"
9292
$l = @()
9393
$r = @()
94-
94+
9595
foreach($line in $lines) {
9696
if ($line -match "^<<<<<<<\s*(.+)") {
9797
$this.IsConflicted = $true
@@ -110,3 +110,141 @@ class ConflictedFile {
110110
}
111111
}
112112
}
113+
114+
# The rate limit comes back in the following format:
115+
# The top level "rate" object is deprecated and the resources->core object should be used
116+
# in its place.
117+
# {
118+
# "resources": {
119+
# "core": {
120+
# "limit": 5000,
121+
# "used": 1087,
122+
# "remaining": 3913,
123+
# "reset": 1722876411
124+
# },
125+
# "search": {
126+
# "limit": 30,
127+
# "used": 0,
128+
# "remaining": 30,
129+
# "reset": 1722875519
130+
# },
131+
# "graphql": {
132+
# "limit": 5000,
133+
# "used": 0,
134+
# "remaining": 5000,
135+
# "reset": 1722879059
136+
# },
137+
# "integration_manifest": {
138+
# "limit": 5000,
139+
# "used": 0,
140+
# "remaining": 5000,
141+
# "reset": 1722879059
142+
# },
143+
# "source_import": {
144+
# "limit": 100,
145+
# "used": 0,
146+
# "remaining": 100,
147+
# "reset": 1722875519
148+
# },
149+
# "code_scanning_upload": {
150+
# "limit": 1000,
151+
# "used": 0,
152+
# "remaining": 1000,
153+
# "reset": 1722879059
154+
# },
155+
# "actions_runner_registration": {
156+
# "limit": 10000,
157+
# "used": 0,
158+
# "remaining": 10000,
159+
# "reset": 1722879059
160+
# },
161+
# "scim": {
162+
# "limit": 15000,
163+
# "used": 0,
164+
# "remaining": 15000,
165+
# "reset": 1722879059
166+
# },
167+
# "dependency_snapshots": {
168+
# "limit": 100,
169+
# "used": 0,
170+
# "remaining": 100,
171+
# "reset": 1722875519
172+
# },
173+
# "audit_log": {
174+
# "limit": 1750,
175+
# "used": 0,
176+
# "remaining": 1750,
177+
# "reset": 1722879059
178+
# },
179+
# "audit_log_streaming": {
180+
# "limit": 15,
181+
# "used": 0,
182+
# "remaining": 15,
183+
# "reset": 1722879059
184+
# },
185+
# "code_search": {
186+
# "limit": 10,
187+
# "used": 0,
188+
# "remaining": 10,
189+
# "reset": 1722875519
190+
# }
191+
# },
192+
# "rate": {
193+
# "limit": 5000,
194+
# "used": 1087,
195+
# "remaining": 3913,
196+
# "reset": 1722876411
197+
# }
198+
# }
199+
200+
# These are the rate limit types we care about. If others needed in the future they
201+
# can be defined here. The reason these need to be defined is because Get-RateLimit
202+
# call needs to select the particular property to return the right limit. This ensures
203+
# that rate limit type being passed to the function will exist.
204+
enum RateLimitTypes {
205+
core
206+
search
207+
graphql
208+
}
209+
210+
# Fetch the rate limit for the given RateLimitType
211+
function Get-RateLimit([RateLimitTypes]$RateLimitType) {
212+
$returnValue = gh api rate_limit
213+
if ($LASTEXITCODE) {
214+
LogError "Get-RateLimit::unable to get rate limit"
215+
exit $LASTEXITCODE
216+
}
217+
# All rate limits have the following fields: limit, used, remaning, reset.
218+
# Returning -AsHashtable allows easier access, eg. $rate_limit.remaining
219+
$rate_limit = $returnValue | ConvertFrom-Json -AsHashtable | Select-Object -ExpandProperty resources | Select-Object -ExpandProperty $RateLimitType
220+
# Add the limit type for convenance
221+
$rate_limit["type"] = $RateLimitType
222+
return $rate_limit
223+
}
224+
225+
# Get the number of minutes until RateLimit reset rounded up to the nearest minute
226+
# for the passed in RateLimit. This is more applicable to the core and graphql rate
227+
# limits than search because the search rate limit resets every minute
228+
function Get-MinutesUntilRateLimitReset($RateLimit) {
229+
$TimeSpan = [System.DateTimeOffset]::FromUnixTimeSeconds($rate.reset).UtcDateTime.Subtract([System.DateTime]::UtcNow)
230+
$MinutesRoundedUp = [Math]::Ceiling($TimeSpan.TotalMinutes)
231+
return $MinutesRoundedUp
232+
}
233+
234+
# Output the rate limit
235+
function Write-RateLimit {
236+
param (
237+
$RateLimit,
238+
[string]$PreMsg = $null
239+
)
240+
241+
if ($PreMsg) {
242+
Write-Host $PreMsg
243+
}
244+
Write-Host "Limit Type=$($RateLimit.type)"
245+
Write-Host " limit=$($RateLimit.limit)"
246+
Write-Host " used=$($RateLimit.used)"
247+
Write-Host " remaining=$($RateLimit.remaining)"
248+
Write-Host " reset=$($RateLimit.reset)"
249+
Write-Host ""
250+
}

‎eng/pipelines/branch-cleanup.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@ parameters:
5050
- MicrosoftDocs/azure-docs-sdk-node
5151
- MicrosoftDocs/azure-docs-sdk-python
5252

53+
- name: RestAPISpecsDocsRepos
54+
type: object
55+
default:
56+
- MicrosoftDocs/AzureRestPreview
57+
5358
jobs:
5459
- job:
5560
displayName: Branch Clean-up
@@ -159,3 +164,22 @@ jobs:
159164
-LastCommitOlderThan ((Get-Date).AddDays(-7))
160165
-AuthToken $(azuresdk-github-pat)
161166
-WhatIf:$${{parameters.WhatIfPreference}}
167+
168+
- ${{ each repo in parameters.RestAPISpecsDocsRepos }}:
169+
- task: PowerShell@2
170+
displayName: ${{ repo }} azure-rest-api-specs docs branch clean-up
171+
condition: succeededOrFailed()
172+
continueOnError: true
173+
inputs:
174+
pwsh: true
175+
workingDirectory: $(System.DefaultWorkingDirectory)
176+
filePath: $(System.DefaultWorkingDirectory)/eng/common/scripts/Delete-RemoteBranches.ps1
177+
# TODO: 00a0dc86-3419-4dd5-b119-e83edaf17e7e needs to be skipped because it's the default
178+
# branch for the repository. This will be removed once the default branch has been changed
179+
# to main
180+
arguments: >
181+
-RepoId "${{ repo }}"
182+
-BranchRegex "^(result_)?(?!00a0dc86-3419-4dd5-b119-e83edaf17e7e)([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-)|(openapiHub_production_)|(openapiHub_preproduction_)[0-9a-z]{12}$"
183+
-LastCommitOlderThan ((Get-Date).AddDays(-14))
184+
-AuthToken $(azuresdk-github-pat)
185+
-WhatIf:$${{parameters.WhatIfPreference}}

0 commit comments

Comments
 (0)
⚡