Support resolving environment variable references in matrix config (#7682)

benbp · web-flow · commit 3ecd831c04cc · 2024-02-29T21:42:10.000Z
* Support resolving environment variable references in matrix config

* Improve type and null handling

* Fix reference bug

* Change behavior on missing env vars to throw
diff --git a/doc/common/matrix_generator.md b/doc/common/matrix_generator.md
@@ -17,6 +17,7 @@
   * [all](#all)
   * [sparse](#sparse)
   * [include/exclude](#includeexclude)
+  * [Environment Variable References](#environment-variable-references)
   * [Generated display name](#generated-display-name)
   * [Filters](#filters)
   * [Replace/Modify/Append](#replacemodifyappend-values)
@@ -512,6 +513,29 @@ will be generated, but the full matrix of both include and exclude will be proce
 Excludes are processed first, so includes can be used to forcefully add specific combinations to the matrix,
 regardless of exclusions.
 
+### Environment Variable References
+
+The matrix config supports values that reference environment variables and resolves them at
+matrix generation time. This is useful especially in pipeline scenarios where we want to reference
+common values that are defined elsewhere in the environment and/or pipeline config.
+
+Prefix a matrix value with `env:` to trigger an environment variable
+lookup. If the variable does not exist, then the value will resolve to empty string.
+
+For example:
+
+``` yaml
+{
+  "net461_macOS1015": {
+    "framework": "net461",
+    "operatingSystem": "env:OperatingSystem"
+  }
+}
+```
+
+Matrix filters and replace parameters evaluate before environment variables are resolved. Matrix
+display name renames and display name filters evaluate after variables are resolved.
+
 ### Generated display name
 
 In the matrix job output that azure pipelines consumes, the format is a map of maps. For example:
diff --git a/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1 b/eng/common-tests/matrix-generator/tests/job-matrix-functions.modification.tests.ps1
@@ -2,7 +2,7 @@ Import-Module Pester
 
 
 BeforeAll {
-  . $PSScriptRoot/../../../common/scripts/job-matrix/job-matrix-functions.ps1
+    . $PSScriptRoot/../../../common/scripts/job-matrix/job-matrix-functions.ps1
 
     function CompareMatrices([Array]$matrix, [Array]$expected) {
         $matrix.Length | Should -Be $expected.Length
@@ -33,25 +33,25 @@ Describe "Platform Matrix nonSparse" -Tag "UnitTest", "nonsparse" {
     }
 
     It "Should process nonSparse parameters" {
-        $parameters, $nonSparse = ProcessNonSparseParameters $config.matrixParameters "testField1","testField3"
+        $parameters, $nonSparse = ProcessNonSparseParameters $config.matrixParameters "testField1", "testField3"
 
         $parameters.Count | Should -Be 1
         $parameters[0].Name | Should -Be "testField2"
-        $parameters[0].Value | Should -Be 1,2,3
+        $parameters[0].Value | Should -Be 1, 2, 3
 
         $nonSparse.Count | Should -Be 2
         $nonSparse[0].Name | Should -Be "testField1"
-        $nonSparse[0].Value | Should -Be 1,2
+        $nonSparse[0].Value | Should -Be 1, 2
         $nonSparse[1].Name | Should -Be "testField3"
-        $nonSparse[1].Value | Should -Be 1,2,3,4
+        $nonSparse[1].Value | Should -Be 1, 2, 3, 4
 
         $parameters, $nonSparse = ProcessNonSparseParameters $config.matrixParameters "testField3"
         $parameters.Count | Should -Be 2
         ($parameters).Name -match "testField3" | Should -Be $null
 
         $nonSparse.Count | Should -Be 1
         $nonSparse[0].Name | Should -Be "testField3"
-        $nonSparse[0].Value | Should -Be 1,2,3,4
+        $nonSparse[0].Value | Should -Be 1, 2, 3, 4
     }
 
     It "Should ignore nonSparse with all selection" {
@@ -77,10 +77,10 @@ Describe "Platform Matrix nonSparse" -Tag "UnitTest", "nonsparse" {
 '@
         $config = GetMatrixConfigFromJson $matrixJson
 
-        $matrix = GenerateMatrix $config "all" -nonSparseParameters "testField3","testField4"
+        $matrix = GenerateMatrix $config "all" -nonSparseParameters "testField3", "testField4"
         $matrix.Length | Should -Be 16
 
-        $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3","testField4"
+        $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3", "testField4"
         $matrix.Length | Should -Be 8
     }
 
@@ -125,13 +125,13 @@ Describe "Platform Matrix nonSparse" -Tag "UnitTest", "nonsparse" {
     }
 }
 
-# This test is currently disabled (it doesn't have "UnitTest" tag) as it fails 
+# This test is currently disabled (it doesn't have "UnitTest" tag) as it fails
 # in test "Should generate a sparse matrix where the entire base matrix is imported" on line:
 #
 #   $matrix = GenerateMatrix $importConfig "sparse"
 #
 # with message:
-# 
+#
 #   ParameterBindingArgumentTransformationException: Cannot process argument transformation on parameter 'parameters'. Cannot convert the "System.Collections.Hashtable" value of type "System.Collections.Hashtable" to type "MatrixParameter".
 #
 # See full build failure:
@@ -392,11 +392,11 @@ Describe "Platform Matrix Import" -Tag "import" {
 
 Describe "Platform Matrix Replace" -Tag "UnitTest", "replace" {
     It "Should parse replacement syntax" -TestCases @(
-         @{ query = 'foo=bar/baz'; key = '^foo$'; value = '^bar$'; replace = 'baz' },
-         @{ query = 'foo=\/p:bar/\/p:baz'; key = '^foo$'; value = '^\/p:bar$'; replace = '/p:baz' },
-         @{ query = 'f\=o\/o=\/p:b\=ar/\/p:b\=az'; key = '^f\=o\/o$'; value = '^\/p:b\=ar$'; replace = '/p:b=az' },
-         @{ query = 'foo=bar/'; key = '^foo$'; value = '^bar$'; replace = '' },
-         @{ query = 'foo=/baz'; key = '^foo$'; value = '^$'; replace = 'baz' }
+        @{ query = 'foo=bar/baz'; key = '^foo$'; value = '^bar$'; replace = 'baz' },
+        @{ query = 'foo=\/p:bar/\/p:baz'; key = '^foo$'; value = '^\/p:bar$'; replace = '/p:baz' },
+        @{ query = 'f\=o\/o=\/p:b\=ar/\/p:b\=az'; key = '^f\=o\/o$'; value = '^\/p:b\=ar$'; replace = '/p:b=az' },
+        @{ query = 'foo=bar/'; key = '^foo$'; value = '^bar$'; replace = '' },
+        @{ query = 'foo=/baz'; key = '^foo$'; value = '^$'; replace = 'baz' }
     ) {
         $parsed = ParseReplacement $query
         $parsed.key | Should -Be $key
@@ -582,5 +582,100 @@ Describe "Platform Matrix Replace" -Tag "UnitTest", "replace" {
         $matrix[2].parameters.Baz | Should -Be "importedBaz"
         $matrix[2].parameters.replaceme | Should -Be "replaceme"
     }
+}
 
+Describe "Platform Matrix Environment Variables" -Tag "UnitTest", "envvar" {
+    It "Should parse environment variable reference syntax" {
+        $matrixJson = @'
+{
+  "matrix": {
+    "foo": "bar",
+    "envReference": ["env:TestMatrixEnvReference", "env:TestMatrixEnvReference2", "noref"]
+  },
+  "include": [
+    {
+      "foo": "bar",
+      "envReference": "env:TestMatrixEnvReference"
+    }
+  ]
+}
+'@
+
+        [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "")
+        { GenerateMatrix (GetMatrixConfigFromJson $matrixJson) "sparse" } | Should -Throw
+
+        [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "replaced")
+        [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference2", "replaced2")
+        [array]$replacedMatrix = GenerateMatrix (GetMatrixConfigFromJson $matrixJson) "sparse"
+        $replacedMatrix.Length | Should -Be 4
+        $replacedMatrix[0].name | Should -Be "bar_replaced"
+        $replacedMatrix[0].parameters.envReference | Should -Be "replaced"
+        $replacedMatrix[1].name | Should -Be "bar_replaced2"
+        $replacedMatrix[1].parameters.envReference | Should -Be "replaced2"
+        $replacedMatrix[2].name | Should -Be "bar_noref"
+        $replacedMatrix[2].parameters.envReference | Should -Be "noref"
+        $replacedMatrix[3].name | Should -Be "bar_replaced"
+        $replacedMatrix[3].parameters.envReference | Should -Be "replaced"
+    }
+
+    It "Should support filter/replace with variable reference syntax" {
+        $matrixJson = @'
+{
+  "displayNames": {
+    "env:replaceme": "env:TestMatrixEnvReference"
+  },
+  "matrix": {
+    "foo": "bar",
+    "envReference": "env:replaceme"
+  }
+}
+'@
+
+        [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "replaced")
+
+        [array]$replacedMatrix = GenerateMatrix `
+            -config (GetMatrixConfigFromJson $matrixJson) `
+            -selectFromMatrixType "sparse" `
+            -replace @("envReference=env:replaceme/env:TestMatrixEnvReference")
+        $replacedMatrix.Length | Should -Be 1
+        $replacedMatrix[0].name | Should -Be "bar_replaced"
+        $replacedMatrix[0].parameters.envReference | Should -Be "replaced"
+
+        # Don't filter out by replaced values, but by original references
+        [System.Environment]::SetEnvironmentVariable("replaceme", "filter_replaced")
+        [array]$replacedMatrix = GenerateMatrix `
+            -config (GetMatrixConfigFromJson $matrixJson) `
+            -selectFromMatrixType "sparse" `
+            -filter @("envReference=env:replaceme")
+        $replacedMatrix.Length | Should -Be 1
+        $replacedMatrix[0].name | Should -Be "bar_filter_replaced"
+        $replacedMatrix[0].parameters.envReference | Should -Be "filter_replaced"
+    }
+
+    It "Should support display name and display name filter with variable reference syntax" {
+        $matrixJson = @'
+{
+  "displayNames": {
+    "replaced": "display"
+  },
+  "matrix": {
+    "foo": "bar",
+    "envReference": "env:TestMatrixEnvReference"
+  }
+}
+'@
+
+        [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "replaced")
+        [array]$replacedMatrix = GenerateMatrix (GetMatrixConfigFromJson $matrixJson) "sparse"
+        $replacedMatrix.Length | Should -Be 1
+        $replacedMatrix[0].name | Should -Be "bar_display"
+        $replacedMatrix[0].parameters.envReference | Should -Be "replaced"
+
+        [System.Environment]::SetEnvironmentVariable("TestMatrixEnvReference", "replaced")
+        [array]$replacedMatrix = GenerateMatrix `
+            -config (GetMatrixConfigFromJson $matrixJson) `
+            -selectFromMatrixType "sparse" `
+            -displayNameFilter "doesnotexist"
+        $replacedMatrix | Should -BeNullOrEmpty
+    }
 }
diff --git a/eng/common-tests/matrix-generator/tests/job-matrix-functions.tests.ps1 b/eng/common-tests/matrix-generator/tests/job-matrix-functions.tests.ps1
@@ -228,17 +228,17 @@ Describe "Matrix-Reverse-Lookup" -Tag "UnitTest", "lookup" {
          @{ index = 1; expected = @(0,0,0,1) }
          @{ index = 2; expected = @(0,0,0,2) }
          @{ index = 3; expected = @(0,0,0,3) }
-                      
+
          @{ index = 4; expected = @(0,0,1,0) }
          @{ index = 5; expected = @(0,0,1,1) }
          @{ index = 6; expected = @(0,0,1,2) }
          @{ index = 7; expected = @(0,0,1,3) }
-                      
+
          @{ index = 8; expected = @(0,1,0,0) }
          @{ index = 9; expected = @(0,1,0,1) }
          @{ index = 10; expected = @(0,1,0,2) }
          @{ index = 11; expected = @(0,1,0,3) }
-                      
+
          @{ index = 12; expected = @(0,1,1,0) }
          @{ index = 13; expected = @(0,1,1,1) }
          @{ index = 14; expected = @(0,1,1,2) }
diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1
