Discover Affected Recordings (#8391)

scbedd · benbp · web-flow · commit 99c86b9389f2 · 2024-06-10T14:03:51.000-07:00
* adjustments to Assets.Maintenance.Tool to allow a bit more flexibility as to which commits are pulled down
* add a script which can retrieve multiple assets tags at a time, then scan them for contents


Co-authored-by: Ben Broderick Phillips &lt;ben@benbp.net&gt;
diff --git a/.gitignore b/.gitignore
@@ -485,8 +485,10 @@ local.settings.json
 # Java tooling
 *.iml
 
+# generated by test-proxy related scripts and packages when testing
 .assets
 .testruns
+.results
 
 # oav converter
 tools/oav-traffic-converter/input-example/
diff --git a/tools/assets-automation/assets-maintenance-tool/Azure.Sdk.Tools.Assets.MaintenanceTool/Model/RepoConfiguration.cs b/tools/assets-automation/assets-maintenance-tool/Azure.Sdk.Tools.Assets.MaintenanceTool/Model/RepoConfiguration.cs
@@ -24,11 +24,20 @@ public RepoConfiguration() {
     /// almost arbitrarily. Official test-proxy began supported external assets in late November of 2022, so we don't
     /// need to go further back then that when examining the SHAs in the language repos. There is no possibility of an
     /// assets.json past this date!
+    /// 
+    /// If provided with "latest" argument, only the most recent commit on each considered branch will be included.
     /// </summary>
-    public DateTime ScanStartDate { get; set; } = DateTime.Parse("2022-12-01");
+    public string ScanStartDate { get; set; } = "2022-12-01";
 
     /// <summary>
     /// The set of branches that we will examine. Defaults to just 'main'.
     /// </summary>
     public List<string> Branches { get; set; } = new List<string> { "main" };
+
+    /// <summary>
+    /// The folder patterns that are used to filter the repo. Functionally, these strings
+    /// will be combined with **/assets.json while searching for assets. Non-presence indicates
+    /// the intent to scan the entire repository.
+    /// </summary>
+    public List<string> ScanFolders { get; set; } = new List<string>{};
 }
diff --git a/tools/assets-automation/assets-maintenance-tool/Azure.Sdk.Tools.Assets.MaintenanceTool/Scan/AssetsScanner.cs b/tools/assets-automation/assets-maintenance-tool/Azure.Sdk.Tools.Assets.MaintenanceTool/Scan/AssetsScanner.cs
@@ -85,36 +85,29 @@ private List<AssetsResult> ScanRepo(RepoConfiguration config, AssetsResultSet? p
         }
 
         var targetRepoUri = $"https://{authString}github.com/{config.LanguageRepo}.git";
-        var workingDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+        var workingDirectory = Path.Combine(WorkingDirectory, config.LanguageRepo.Replace("/", "_"));
         var results = new List<AssetsResult>();
 
-        try
+        if (!Directory.Exists(workingDirectory))
         {
-            if (!Directory.Exists(workingDirectory))
-            {
-                Directory.CreateDirectory(workingDirectory);
-            }
+            Directory.CreateDirectory(workingDirectory);
+        }
 
-            foreach (var branch in config.Branches)
-            {
-                var commitsOnBranch = GetBranchCommits(targetRepoUri, branch, config.ScanStartDate, workingDirectory);
-                var unretrievedCommits = ResolveUnhandledCommits(commitsOnBranch, previousOutput);
+        foreach (var branch in config.Branches)
+        {
+            var commitsOnBranch = GetBranchCommits(targetRepoUri, branch, config.ScanStartDate, workingDirectory);
+            var unretrievedCommits = ResolveUnhandledCommits(commitsOnBranch, previousOutput);
 
-                results.AddRange(GetAssetsResults(config.LanguageRepo, unretrievedCommits, workingDirectory));
+            results.AddRange(GetAssetsResults(config.LanguageRepo, unretrievedCommits, workingDirectory, config.ScanFolders));
 
-                if (previousOutput != null)
+            if (previousOutput != null)
+            {
+                foreach (var commit in commitsOnBranch.Where(commit => !unretrievedCommits.Contains(commit)))
                 {
-                    foreach (var commit in commitsOnBranch.Where(commit => !unretrievedCommits.Contains(commit)))
-                    {
-                        results.AddRange(previousOutput.ByOriginSHA[commit]);
-                    }
+                    results.AddRange(previousOutput.ByOriginSHA[commit]);
                 }
             }
         }
-        finally
-        {
-            CleanupWorkingDirectory(workingDirectory);
-        }
 
         return results;
     }
@@ -123,9 +116,10 @@ private List<AssetsResult> ScanRepo(RepoConfiguration config, AssetsResultSet? p
     /// Clones a specific branch, then returns all commit shas newer than our targeted date.
     /// </summary>
     /// <returns>A list of commits (limited to after a startdate) from the targeted branch.</returns>
-    private List<string> GetBranchCommits(string uri, string branch, DateTime since, string workingDirectory)
+    private List<string> GetBranchCommits(string uri, string branch, string since, string workingDirectory)
     {
         var commitSHAs = new List<string>();
+
         try
         {
             // if git is already initialized, we just need to checkout a specific branch
@@ -141,7 +135,15 @@ private List<string> GetBranchCommits(string uri, string branch, DateTime since,
                 Cleanup(workingDirectory);
             }
 
-            var tagResult = handler.Run($"log --since={since.ToString("yyyy-MM-dd")} --format=format:%H", workingDirectory);
+            CommandResult tagResult;
+            if (since == "latest")
+            {
+                tagResult = handler.Run($"log -n 1 --format=format:%H", workingDirectory);
+            }
+            else
+            {
+                tagResult = handler.Run($"log --since={since} --format=format:%H", workingDirectory);
+            }
             commitSHAs.AddRange(tagResult.StdOut.Split(Environment.NewLine).Select(x => x.Trim()).Where(x => !string.IsNullOrWhiteSpace(x)));
         }
         catch (GitProcessException gitException)
@@ -207,11 +209,23 @@ public Assets()
     /// Find all assets.jsons beneath a targeted folder.
     /// </summary>
     /// <returns>AssetsResults for each discovered assets.json, populating other metadata as necessary.</returns>
-    private List<AssetsResult> ScanDirectory(string repo, string commit, string workingDirectory)
+    private List<AssetsResult> ScanDirectory(string repo, string commit, string workingDirectory, List<string> scanFolders)
     {
         Matcher matcher = new();
         List<AssetsResult> locatedAssets = new List<AssetsResult>();
-        matcher.AddIncludePatterns(new[] { "**/assets.json" });
+
+        if (scanFolders.Count > 0)
+        {
+            foreach (string folder in scanFolders)
+            {
+                matcher.AddIncludePatterns(new[] { Path.Combine(folder, "**/assets.json") });
+            }
+        }
+        else
+        {
+            matcher.AddIncludePatterns(new[] { "**/assets.json" });
+        }
+        
         IEnumerable<string> assetsJsons = matcher.GetResultsInFullPath(workingDirectory);
 
         foreach (var assetsJson in assetsJsons)
@@ -233,14 +247,14 @@ private List<AssetsResult> ScanDirectory(string repo, string commit, string work
     /// Walks a set of targeted commits, extracting all available assets.jsons from each.
     /// </summary>
     /// <returns>A list of AssetsResults reflecting all discovered assets.jsons from each targeted commit.</returns>
-    private List<AssetsResult> GetAssetsResults(string repo, List<string> commits, string workingDirectory)
+    private List<AssetsResult> GetAssetsResults(string repo, List<string> commits, string workingDirectory, List<string> folderGlobs)
     {
         var allResults = new List<AssetsResult>();
         foreach (var commit in commits)
         {
             handler.Run($"checkout {commit}", workingDirectory);
             Cleanup(workingDirectory);
-            allResults.AddRange(ScanDirectory(repo, commit, workingDirectory));
+            allResults.AddRange(ScanDirectory(repo, commit, workingDirectory, folderGlobs));
         }
 
         return allResults;
@@ -275,7 +289,7 @@ private void Cleanup(string workingDirectory)
     /// This is necessary because certain `.pack` files created by git cannot be deleted without
     /// adjusting these permissions.
     /// </summary>
-    private void SetPermissionsAndDelete(string gitfolder)
+    public static void SetPermissionsAndDelete(string gitfolder)
     {
         File.SetAttributes(gitfolder, FileAttributes.Normal);
 
@@ -300,7 +314,7 @@ private void SetPermissionsAndDelete(string gitfolder)
     /// The .git folder's .pack files can be super finicky to delete from code.
     /// This function abstracts the necessary permissions update and cleans that folder for us.
     /// </summary>
-    private void CleanupWorkingDirectory(string workingDirectory)
+    public static void CleanupGitDirectory(string workingDirectory)
     {
         var gitDir = Path.Combine(workingDirectory, ".git");
 
@@ -319,7 +333,11 @@ public void Save(AssetsResultSet newResults)
     {
         using (var stream = System.IO.File.OpenWrite(ResultsFile))
         {
-            stream.Write(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(newResults.Results)));
+            var options = new JsonSerializerOptions
+            {
+                WriteIndented = true
+            };
+            stream.Write(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(newResults.Results, options: options)));
         }
     }
 }
diff --git a/tools/assets-automation/assets-task-scripts/find-assets-files/locate-involved-assets.ps1 b/tools/assets-automation/assets-task-scripts/find-assets-files/locate-involved-assets.ps1
@@ -0,0 +1,67 @@
+<#
+.SYNOPSIS
+Used to retrieve a list of references to test recordings that that "contain" a string in their contents.
+
+.DESCRIPTION
+Uses the Azure.Sdk.Tools.Assets.MaintenanceTool to retrieve a copy of all the assets files, then grep each tag's contents
+for the specified string. Intermediary results are stored in the .results folder, and the final ressults will be output to the
+console on successful completion of the script.
+
+This will enable easy access to the test recordings that are affected by a change in the SDK, and will help to identify
+which tests need to be updated.
+
+PreReqs:
+    - Azure.Sdk.Tools.Assets.MaintenanceTool must be available on the PATH
+    - git is available on the PATH
+    - Running powershell core
+    - ripgrip installed on the machine. 'choco install ripgrep' on elevated terminal
+
+.PARAMETER ConfigFilePath
+The query configuration file, which contains targeted repos, branches, and paths.
+
+.PARAMETER SearchString
+The regex string to search for in the contents of the test recordings.
+#>
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$ConfigFilePath,
+    [Parameter(Mandatory = $true)]
+    [string]$SearchString
+)
+
+Set-StrictMode -Version 4
+$ErrorActionPreference = "Stop"
+. (Join-Path $PSScriptRoot ".." "utilities.ps1")
+
+if (!(Test-Path $ConfigFilePath -PathType Leaf)) {
+    Write-Error "Config file not found: $ConfigFilePath"
+    exit 1
+}
+
+$ResultsFolder = Create-If-Not-Exists (Join-Path $PSScriptRoot ".results")
+$TagsFolder = Create-If-Not-Exists (Join-Path $ResultsFolder "tags")
+$ScanOutputJson = Join-Path $ResultsFolder "output.json"
+
+try {
+    Push-Location $ResultsFolder
+
+    if (!(Test-Path $ScanOutputJson)) {
+        Azure.Sdk.Tools.Assets.MaintenanceTool scan --config $ConfigFilePath
+        if ($LASTEXITCODE) { exit $LASTEXITCODE }
+    }
+    else {
+        Write-Host "Skipping scan, using cached results"
+    }
+}
+finally {
+    Pop-Location
+}
+
+$DiscoveredAssets = Get-Content $ScanOutputJson | ConvertFrom-Json 
+
+foreach ($asset in $DiscoveredAssets) {
+    $TagFolder = Get-AssetsRepoSlice -Tag $asset.Tag -WorkDirectory $ResultsFolder
+}
+
+Write-Host "rg $SearchString -g `"*.json`" $TagsFolder"
+rg $SearchString -g "*.json" $TagsFolder
diff --git a/tools/assets-automation/assets-task-scripts/find-assets-files/sample-query-config.json b/tools/assets-automation/assets-task-scripts/find-assets-files/sample-query-config.json
@@ -0,0 +1,39 @@
+{
+  "LanguageRepos": [
+    {
+      "LanguageRepo": "Azure/azure-sdk-for-java",
+      "ScanStartDate": "latest",
+      "ScanFolders": [
+        "sdk/containerregistry"
+      ]
+    },
+    {
+      "LanguageRepo": "Azure/azure-sdk-for-python",
+      "ScanStartDate": "latest",
+      "ScanFolders": [
+        "sdk/containerregistry"
+      ]
+    },
+    {
+      "LanguageRepo": "Azure/azure-sdk-for-go",
+      "ScanStartDate": "latest",
+      "ScanFolders": [
+        "sdk/containers"
+      ]
+    },
+    {
+      "LanguageRepo": "Azure/azure-sdk-for-net",
+      "ScanStartDate": "latest",
+      "ScanFolders": [
+        "sdk/containerregistry"
+      ]
+    },
+    {
+      "LanguageRepo": "Azure/azure-sdk-for-js",
+      "ScanStartDate": "latest",
+      "ScanFolders": [
+        "sdk/containerregistry"
+      ]
+    }
+  ]
+}
diff --git a/tools/assets-automation/assets-task-scripts/utilities.ps1 b/tools/assets-automation/assets-task-scripts/utilities.ps1
@@ -0,0 +1,76 @@
+Set-StrictMode -Version 4
+
+function Create-If-Not-Exists {
+    param(
+        [string]$Path
+    )
+
+    if (!(Test-Path $Path)) {
+        New-Item -ItemType Directory -Path $Path -Force | Out-Null
+    }
+
+    return $Path
+}
+
+<#
+.SYNOPSIS
+Retrieve a specific tag from an assets repo and store it on disk within the work directory.
+
+.DESCRIPTION
+Clones a specific tag from an assets repo (defaults to azure-sdk-assets) and stores it on disk
+within the work directory under a folder with pattern:
+
+.results <-- this should be WorkDirectory arg
+    tags/
+        <tagname>/
+            <tagged repo contents>
+        <tagname>/
+            <tagged repo contents>
+        <tagname>/
+            <tagged repo contents>
+        ...
+  
+Returns the location of the folder after the work is complete.
+
+.PARAMETER Tag
+The tag to retrieve from the assets repo.
+
+.PARAMETER WorkDirectory
+The path to the .results directory within which this script will operate.
+
+.PARAMETER TargetRepo
+Defaults to "Azure/azure-sdk-assets". This is the repo that will be cloned from.
+#>
+function Get-AssetsRepoSlice {
+    param(
+        [Parameter(Mandatory=$true)]
+        [string]$Tag,
+        [Parameter(Mandatory=$true)]
+        [string]$WorkDirectory,
+        [string]$TargetRepo = "Azure/azure-sdk-assets"
+    )
+    $CloneUri = "https://github.com/$TargetRepo.git"
+    $TagFolderName = Join-Path $WorkDirectory "tags" $Tag.Replace("/", "-")
+
+    $TagFolder = Create-If-Not-Exists -Path $TagFolderName
+
+    Write-Host "TagFolder is $TagFolder"
+
+    if (Test-Path $TagFolder/.git) {
+        Write-Host "TagFolder already exists, skipping clone, returning $TagFolder"
+        return $TagFolder
+    }
+    else {
+        try {
+            Push-Location $TagFolder
+            git clone -c core.longpaths=true --no-checkout --filter=tree:0 $CloneUri .
+            git fetch origin "refs/tags/$($Tag):refs/tags/$Tag"
+            git checkout $Tag
+        }
+        finally {
+            Pop-Location
+        }
+
+        return $TagFolder
+    }
+}
