add integration test exercising a push with only a delete. updating logic to include deleted files in the detect pending changes. something that was erroneously excluded when adding security scanning to the project (#8681)

Author: scbedd

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/IntegrationTests/GitStoreIntegrationPushTests.cs

@@ -506,6 +506,85 @@ public async Task Scenario5(string inputJson)
             }
         }
 
+
+        /// <summary>
+        /// 1. Restore from the initial tag in pull/scenarios
+        /// 2. Only delete a file
+        /// 3. Push to new branch
+        /// 4. Verify local files are what is expected
+        /// 5. Verify assets.json was updated with the new commit Tag
+        /// </summary>
+        /// <param name="inputJson"></param>
+        /// <returns></returns>
+        [EnvironmentConditionalSkipTheory]
+        [InlineData(
+        @"{
+              ""AssetsRepo"": ""Azure/azure-sdk-assets-integration"",
+              ""AssetsRepoPrefixPath"": ""pull/scenarios"",
+              ""AssetsRepoId"": """",
+              ""TagPrefix"": ""language/tables"",
+              ""Tag"": ""python/tables_fc54d0""
+        }")]
+        [Trait("Category", "Integration")]
+        public async Task EnsureOnlyDeletesDetected(string inputJson)
+        {
+            var folderStructure = new string[]
+            {
+                GitStoretests.AssetsJson
+            };
+
+            Assets assets = JsonSerializer.Deserialize<Assets>(inputJson);
+            Assets updatedAssets = null;
+            string originalTagPrefix = assets.TagPrefix;
+            string originalTag = assets.Tag;
+            var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure, isPushTest: true);
+            try
+            {
+                // Ensure that the Tag was updated
+                Assert.NotEqual(originalTag, assets.TagPrefix);
+
+                var jsonFileLocation = Path.Join(testFolder, GitStoretests.AssetsJson);
+
+                var parsedConfiguration = await _defaultStore.ParseConfigurationFile(jsonFileLocation);
+                await _defaultStore.Restore(jsonFileLocation);
+
+                // Calling Path.GetFullPath of the Path.Combine will ensure any directory separators are normalized for
+                // the OS the test is running on. The reason being is that AssetsRepoPrefixPath, if there's a separator,
+                // will be a forward one as expected by git but on Windows this won't result in a usable path.
+                string localFilePath = Path.GetFullPath(Path.Combine(parsedConfiguration.AssetsRepoLocation.ToString(), parsedConfiguration.AssetsRepoPrefixPath.ToString()));
+
+                // These are the files pulled down with the original Tag
+                Assert.Equal(3, System.IO.Directory.EnumerateFiles(localFilePath).Count());
+                Assert.True(TestHelpers.VerifyFileVersion(localFilePath, "file1.txt", 1));
+                Assert.True(TestHelpers.VerifyFileVersion(localFilePath, "file2.txt", 1));
+                Assert.True(TestHelpers.VerifyFileVersion(localFilePath, "file3.txt", 1));
+
+                File.Delete(Path.Combine(localFilePath, "file2.txt"));
+
+                // Push the update, it should be a clean push
+                await _defaultStore.Push(jsonFileLocation);
+
+                // Verify that after the push the directory still contains our updated files
+                Assert.Equal(2, System.IO.Directory.EnumerateFiles(localFilePath).Count());
+                Assert.True(TestHelpers.VerifyFileVersion(localFilePath, "file1.txt", 1));
+                Assert.True(TestHelpers.VerifyFileVersion(localFilePath, "file3.txt", 1));
+
+                // Ensure that the config was updated with the new Tag as part of the push
+                updatedAssets = TestHelpers.LoadAssetsFromFile(jsonFileLocation);
+                Assert.NotEqual(originalTag, updatedAssets.Tag);
+
+                // Ensure that the targeted tag is present on the repo
+                TestHelpers.CheckExistenceOfTag(updatedAssets, localFilePath);
+                await TestHelpers.CheckBreadcrumbAgainstAssetsJsons(new string[] { jsonFileLocation });
+            }
+            finally
+            {
+                DirectoryHelper.DeleteGitDirectory(testFolder);
+                TestHelpers.CleanupIntegrationTestTag(assets);
+                TestHelpers.CleanupIntegrationTestTag(updatedAssets);
+            }
+        }
+
         /// <summary>
         /// 1. Restore from a tag that has minimal existing files and a long destination path
         /// 2. Add a _bunch_ of files

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SecretScanner.cs

@@ -29,37 +29,41 @@ public List<Tuple<string, Detection>> DiscoverSecrets(string assetRepoRoot, IEnu
             var detectedSecrets = new ConcurrentBag<Tuple<string, Detection>>();
             var total = relativePaths.Count();
             var seen = 0;
-            Console.WriteLine(string.Empty);
 
-            var options = new ParallelOptions
+            if (relativePaths.Count() > 0)
             {
-                MaxDegreeOfParallelism = Environment.ProcessorCount
-            };
+                Console.WriteLine(string.Empty);
 
-            Parallel.ForEach(relativePaths, options, (filePath) =>
-            {
-                var path = Path.Combine(assetRepoRoot, filePath);
+                var options = new ParallelOptions
+                {
+                    MaxDegreeOfParallelism = Environment.ProcessorCount
+                };
 
-                if (File.Exists(path))
+                Parallel.ForEach(relativePaths, options, (filePath) =>
                 {
-                    var content = File.ReadAllText(path);
-                    var fileDetections = DetectSecrets(content);
+                    var path = Path.Combine(assetRepoRoot, filePath);
 
-                    if (fileDetections != null && fileDetections.Count > 0)
+                    if (File.Exists(path))
                     {
-                        foreach (Detection detection in fileDetections)
+                        var content = File.ReadAllText(path);
+                        var fileDetections = DetectSecrets(content);
+
+                        if (fileDetections != null && fileDetections.Count > 0)
                         {
-                            detectedSecrets.Add(Tuple.Create(filePath, detection));
+                            foreach (Detection detection in fileDetections)
+                            {
+                                detectedSecrets.Add(Tuple.Create(filePath, detection));
+                            }
                         }
-                    }
 
-                    Interlocked.Increment(ref seen);
+                        Interlocked.Increment(ref seen);
 
-                    Console.Write($"\r\u001b[2KScanned {seen}/{total}.");
-                }
-            });
+                        Console.Write($"\r\u001b[2KScanned {seen}/{total}.");
+                    }
+                });
 
-            Console.WriteLine(string.Empty);
+                Console.WriteLine(string.Empty);
+            }
 
             return detectedSecrets.ToList();
         }

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs

@@ -378,19 +378,11 @@ public string[] DetectPendingChanges(GitAssetsConfiguration config)
 
             if (!string.IsNullOrWhiteSpace(diffResult.StdOut))
             {
-                /* 
-                 * Normally, we'd use Environment.NewLine here but this doesn't work on Windows since its NewLine is \r\n and Git's NewLine is just \n
-                 * 
-                 * The output from git status porcelain will include two possible additional values
-                 * " ?? path/to/file" -> File that is new
-                 * " M path/to/file" -> File that is modified
-                 * " D path/to/file" -> File that is deleted
-                */
+                // Normally, we'd use Environment.NewLine here but this doesn't work on Windows since its NewLine is \r\n and
+                // Git's NewLine is just \n
                 var individualResults = diffResult.StdOut.Split("\n")
-                    // strim the leading space, the characters for ADDED or MODIFIED, and the space after them
-                    .Select(x => x.Trim().TrimStart('?', 'M').Trim())
-                    // exclude empty paths or paths that have been DELETED
-                    .Where(x => !string.IsNullOrWhiteSpace(x) && !x.StartsWith("D")).ToArray();
+                    .Select(x => x.Trim())
+                    .Where(x => !string.IsNullOrWhiteSpace(x)).ToArray();
                 return individualResults;
             }