Add RestAPISpecsDocsRepos to branch cleanup and update the delete scripts to output limits and telemetry data

eng/common/scripts/Delete-RemoteBranches.ps1

@@ -15,7 +15,6 @@ param(
   $AuthToken
 )
 Set-StrictMode -version 3
-
 . (Join-Path $PSScriptRoot common.ps1)
 
 function Get-AllBranchesAndPullRequestInfo($owner, $repo) {
@@ -66,67 +65,124 @@ if ($AuthToken) {
 }
 
 $owner, $repo = $RepoId -split "/"
-$branches = Get-AllBranchesAndPullRequestInfo $owner $repo
 
-foreach ($branch in $branches)
-{
-  $branchName = $branch.Name
-  if ($branchName -notmatch $BranchRegex) {
-    continue
+# These will always be output at the end of the script. Their only purpose is for information gathering
+# Total number returned from query
+$totalBranchesFromQuery = 0
+# reasons why a branch was skipped
+$skippedBranchNotMatchRegex = 0
+$skippedForCommitDate = 0
+$skippedForOpenPRs = 0
+$skippedForPRNotInBranch = 0
+$skippedForPRNotInRepo = 0
+# gh call counters
+$ghPRViewCalls = 0
+$ghBranchDeleteCalls = 0
+
+try {
+  # Output the core rate limit at the start of processing. There's no real need
+  # to output this at the end because the GH call counts are being output
+  $coreRateLimit = Get-RateLimit ([RateLimitTypes]::core)
+  Write-RateLimit $coreRateLimit
+  # Output the GraphQL rate limit before and after the call
+  $graphqlRateLimit = Get-RateLimit ([RateLimitTypes]::graphql)
+  Write-RateLimit $graphqlRateLimit "Before GraphQL Call"
+  $branches = Get-AllBranchesAndPullRequestInfo $owner $repo
+  $graphqlRateLimit = Get-RateLimit ([RateLimitTypes]::graphql)
+  Write-RateLimit $graphqlRateLimit "After GraphQL Call"
+
+  if ($branches) {
+    $totalBranchesFromQuery = $branches.Count
   }
-  $openPullRequests = @($branch.pullRequests | Where-Object { !$_.Closed })
 
-  # If we have a central PR that created this branch still open don't delete the branch
-  if ($CentralRepoId)
+  foreach ($branch in $branches)
   {
-    $pullRequestNumber = $matches["PrNumber"]
-    # If central PR number is not found, then skip
-    if (!$pullRequestNumber) {
-      LogError "No PR number found in the branch name. Please check the branch name '$branchName'. Skipping..."
+    $branchName = $branch.Name
+    if ($branchName -notmatch $BranchRegex) {
+      $skippedBranchNotMatchRegex++
       continue
     }
+    $openPullRequests = @($branch.pullRequests | Where-Object { !$_.Closed })
 
-    $centralPR = gh pr view --json 'url,closed' --repo $CentralRepoId $pullRequestNumber | ConvertFrom-Json
-    if ($LASTEXITCODE) {
-      LogError "PR '$pullRequestNumber' not found in repo '$CentralRepoId'. Skipping..."
-      continue;
-    } else {
-      LogDebug "Found central PR $($centralPR.url) and Closed=$($centralPR.closed)"
-      if (!$centralPR.Closed) {
-        # Skipping if there is an open central PR open for the branch.
-        LogDebug "Central PR is still open so skipping the deletion of branch '$branchName'. Skipping..."
-        continue;
+    # If we have a central PR that created this branch still open don't delete the branch
+    if ($CentralRepoId)
+    {
+      $pullRequestNumber = $matches["PrNumber"]
+      # If central PR number is not found, then skip
+      if (!$pullRequestNumber) {
+        LogError "No PR number found in the branch name. Please check the branch name '$branchName'. Skipping..."
+        $skippedForPRNotInBranch++
+        continue
+      }
+
+      $ghPRViewCalls++
+      $centralPR = gh pr view --json 'url,closed' --repo $CentralRepoId $pullRequestNumber | ConvertFrom-Json
+      if ($LASTEXITCODE) {
+        LogError "PR '$pullRequestNumber' not found in repo '$CentralRepoId'. Skipping..."
+        $skippedForPRNotInRepo++
+        continue
+      } else {
+        LogDebug "Found central PR $($centralPR.url) and Closed=$($centralPR.closed)"
+        if (!$centralPR.Closed) {
+          $skippedForOpenPRs++
+          # Skipping if there is an open central PR open for the branch.
+          LogDebug "Central PR is still open so skipping the deletion of branch '$branchName'. Skipping..."
+          continue
+        }
       }
     }
-  }
-  else {
-    # Not CentralRepoId - not associated with a central repo PR
-    if ($openPullRequests.Count -gt 0 -and !$DeleteBranchesEvenIfThereIsOpenPR) {
-      LogDebug "Found open PRs associate with branch '$branchName'. Skipping..."
-      continue
+    else {
+      # Not CentralRepoId - not associated with a central repo PR
+      if ($openPullRequests.Count -gt 0 -and !$DeleteBranchesEvenIfThereIsOpenPR) {
+        $skippedForOpenPRs++
+        LogDebug "Found open PRs associate with branch '$branchName'. Skipping..."
+        continue
+      }
     }
-  }
 
-  # If there is date filter, then check if branch last commit is older than the date.
-  if ($LastCommitOlderThan)
-  {
-    $commitDate = $branch.committedDate
-    if ($commitDate -gt $LastCommitOlderThan) {
-      LogDebug "The branch $branch last commit date '$commitDate' is newer than the date '$LastCommitOlderThan'. Skipping..."
-      continue
+    # If there is date filter, then check if branch last commit is older than the date.
+    if ($LastCommitOlderThan)
+    {
+      $commitDate = $branch.committedDate
+      if ($commitDate -gt $LastCommitOlderThan) {
+        $skippedForCommitDate++
+        LogDebug "The branch $branch last commit date '$commitDate' is newer than the date '$LastCommitOlderThan'. Skipping..."
+        continue
+      }
     }
-  }
 
-  foreach ($openPullRequest in $openPullRequests) {
-    Write-Host "Note: Open pull Request '$($openPullRequest.url)' will be closed after branch deletion, given the central PR is closed."
-  }
+    foreach ($openPullRequest in $openPullRequests) {
+      LogDebug "Note: Open pull Request '$($openPullRequest.url)' will be closed after branch deletion, given the central PR is closed."
+    }
 
-  $commitUrl = $branch.commitUrl
-  if ($PSCmdlet.ShouldProcess("'$branchName' in '$RepoId'", "Deleting branch on cleanup script")) {
-    gh api "repos/${RepoId}/git/refs/heads/${branchName}" -X DELETE
-    if ($LASTEXITCODE) {
-      LogError "Deletion of branch '$branchName` failed"
+    $commitUrl = $branch.commitUrl
+    if ($PSCmdlet.ShouldProcess("'$branchName' in '$RepoId'", "Deleting branch on cleanup script")) {
+      $ghBranchDeleteCalls++
+      gh api "repos/${RepoId}/git/refs/heads/${branchName}" -X DELETE
+      if ($LASTEXITCODE) {
+        LogError "Deletion of branch '$branchName` failed, see command output above"
+        exit $LASTEXITCODE
+      }
+      LogDebug "The branch '$branchName' at commit '$commitUrl' in '$RepoId' has been deleted."
     }
-    Write-Host "The branch '$branchName' at commit '$commitUrl' in '$RepoId' has been deleted."
+  }
+}
+finally {
+
+
+  Write-Host "Number of branches returned from graphql query: $totalBranchesFromQuery"
+  # The $BranchRegex seems to be always set
+  if ($BranchRegex) {
+    Write-Host "Number of branches that didn't match the BranchRegex: $skippedBranchNotMatchRegex"
+  }
+  Write-Host "Number of branches skipped for newer last commit date: $skippedForCommitDate"
+  Write-Host "Number of branches skipped for open PRs: $skippedForOpenPRs"
+  Write-Host "Number of gh api calls to delete branches: $ghBranchDeleteCalls"
+  # The following are only applicable when $CentralRepoId is passed in
+  if ($CentralRepoId) {
+    Write-Host "The following are applicable because CentralRepoId was passed in:"
+    Write-Host "  Number of gh pr view calls: $ghPRViewCalls"
+    Write-Host "  Number of branches skipped due to PR not in the repository: $skippedForPRNotInRepo "
+    Write-Host "  Number of branches skipped due to PR not in the branch name: $skippedForPRNotInBranch"
   }
 }

eng/common/scripts/Helpers/git-helpers.ps1

@@ -15,7 +15,7 @@ function Get-ChangedFiles {
     return ""
   }
 
-  # Add config to disable the quote and encoding on file name. 
+  # Add config to disable the quote and encoding on file name.
   # Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-quoted-file-names
   # Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-commit-message-transcoding
   # Git PR diff: https://docs.github.com/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-comparing-branches-in-pull-requests#three-dot-and-two-dot-git-diff-comparisons
@@ -91,7 +91,7 @@ class ConflictedFile {
     $lines = $IncomingContent -split "`r?`n"
     $l = @()
     $r = @()
-    
+
     foreach($line in $lines) {
       if ($line -match "^<<<<<<<\s*(.+)") {
         $this.IsConflicted = $true
@@ -110,3 +110,143 @@ class ConflictedFile {
     }
   }
 }
+
+# The rate limit comes back in the following format:
+# The top level "rate" object is deprecated and the resources->core object should be used
+# in its place.
+# {
+#   "resources": {
+#     "core": {
+#       "limit": 5000,
+#       "used": 1087,
+#       "remaining": 3913,
+#       "reset": 1722876411
+#     },
+#     "search": {
+#       "limit": 30,
+#       "used": 0,
+#       "remaining": 30,
+#       "reset": 1722875519
+#     },
+#     "graphql": {
+#       "limit": 5000,
+#       "used": 0,
+#       "remaining": 5000,
+#       "reset": 1722879059
+#     },
+#     "integration_manifest": {
+#       "limit": 5000,
+#       "used": 0,
+#       "remaining": 5000,
+#       "reset": 1722879059
+#     },
+#     "source_import": {
+#       "limit": 100,
+#       "used": 0,
+#       "remaining": 100,
+#       "reset": 1722875519
+#     },
+#     "code_scanning_upload": {
+#       "limit": 1000,
+#       "used": 0,
+#       "remaining": 1000,
+#       "reset": 1722879059
+#     },
+#     "actions_runner_registration": {
+#       "limit": 10000,
+#       "used": 0,
+#       "remaining": 10000,
+#       "reset": 1722879059
+#     },
+#     "scim": {
+#       "limit": 15000,
+#       "used": 0,
+#       "remaining": 15000,
+#       "reset": 1722879059
+#     },
+#     "dependency_snapshots": {
+#       "limit": 100,
+#       "used": 0,
+#       "remaining": 100,
+#       "reset": 1722875519
+#     },
+#     "audit_log": {
+#       "limit": 1750,
+#       "used": 0,
+#       "remaining": 1750,
+#       "reset": 1722879059
+#     },
+#     "audit_log_streaming": {
+#       "limit": 15,
+#       "used": 0,
+#       "remaining": 15,
+#       "reset": 1722879059
+#     },
+#     "code_search": {
+#       "limit": 10,
+#       "used": 0,
+#       "remaining": 10,
+#       "reset": 1722875519
+#     }
+#   },
+#   "rate": {
+#     "limit": 5000,
+#     "used": 1087,
+#     "remaining": 3913,
+#     "reset": 1722876411
+#   }
+# }
+
+# These are the rate limit types we care about. If others needed in the future they
+# can be defined here. The reason these need to be defined is because Get-RateLimit
+# call needs to select the particular property to return the right limit. This ensures
+# that rate limit type being passed to the function will exist.
+enum RateLimitTypes {
+  core
+  search
+  graphql
+}
+
+# Fetch the rate limit for the given RateLimitType
+# TODO: Possibly change the parameter to take in an array of types and return an
+# array of limits?
+function Get-RateLimit([RateLimitTypes]$RateLimitType) {
+  $returnValue = gh api rate_limit
+  if ($LASTEXITCODE) {
+    LogError "Get-RateLimit::unable to get rate limit"
+    exit $LASTEXITCODE
+  }
+  # All rate limits have the following fields: limit, used, remaning, reset.
+  # Returning -AsHashtable allows easier access, eg. $rate_limit.remaining
+  $rate_limit = $returnValue | ConvertFrom-Json -AsHashtable | Select-Object -ExpandProperty resources | Select-Object -ExpandProperty $RateLimitType
+  # Add the limit type for convenance
+  $rate_limit["type"] = $RateLimitType
+  return $rate_limit
+}
+
+# Get the number of minutes until RateLimit reset rounded up to the nearest minute
+# for the passed in RateLimit. This is more applicable to the core and graphql rate
+# limits than search because the search rate limit resets every minute
+function Get-MinutesUntilRateLimitReset($RateLimit) {
+  $TimeSpan = [System.DateTimeOffset]::FromUnixTimeSeconds($rate.reset).UtcDateTime.Subtract([System.DateTime]::UtcNow)
+  $MinutesRoundedUp = [Math]::Ceiling($TimeSpan.TotalMinutes)
+  return $MinutesRoundedUp
+}
+
+# Output the rate limit
+function Write-RateLimit {
+  param (
+    $RateLimit,
+    [string]$PreMsg = $null
+  )
+
+  if ($PreMsg) {
+    Write-Host $PreMsg
+  }
+  Write-Host "Limit Type=$($RateLimit.type)"
+  Write-Host "    limit=$($RateLimit.limit)"
+  Write-Host "    used=$($RateLimit.used)"
+  Write-Host "    remaining=$($RateLimit.remaining)"
+  Write-Host "    reset=$($RateLimit.reset)"
+  Write-Host ""
+}

eng/pipelines/branch-cleanup.yml

@@ -50,6 +50,11 @@ parameters:
     - MicrosoftDocs/azure-docs-sdk-node
     - MicrosoftDocs/azure-docs-sdk-python
 
+- name: RestAPISpecsDocsRepos
+  type: object
+  default:
+    - MicrosoftDocs/AzureRestPreview
+
 jobs:
   - job:
     displayName: Branch Clean-up
@@ -159,3 +164,22 @@ jobs:
               -LastCommitOlderThan ((Get-Date).AddDays(-7))
               -AuthToken $(azuresdk-github-pat)
               -WhatIf:$${{parameters.WhatIfPreference}}
+
+      - ${{ each repo in parameters.RestAPISpecsDocsRepos }}:
+        - task: PowerShell@2
+          displayName: ${{ repo }} azure-rest-api-specs docs branch clean-up
+          condition: succeededOrFailed()
+          continueOnError: true
+          inputs:
+            pwsh: true
+            workingDirectory: $(System.DefaultWorkingDirectory)
+            filePath: $(System.DefaultWorkingDirectory)/eng/common/scripts/Delete-RemoteBranches.ps1
+            # TODO: 00a0dc86-3419-4dd5-b119-e83edaf17e7e needs to be skipped because it's the default
+            # branch for the repository. This will be removed once the default branch has been changed
+            # to main
+            arguments: >
+              -RepoId "${{ repo }}"
+              -BranchRegex "^(result_)?(?!00a0dc86-3419-4dd5-b119-e83edaf17e7e)([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-)|(openapiHub_production_)|(openapiHub_preproduction_)[0-9a-z]{12}$"
+              -LastCommitOlderThan ((Get-Date).AddDays(-14))
+              -AuthToken $(azuresdk-github-pat)
+              -WhatIf:$${{parameters.WhatIfPreference}}