js - pullrequest (#31420)

This pullrequest adds a definition for an expanding and contracting
build that will be used to replace all `js - <service> - ci` build
definitions in the `public` devops project. This is an overall package
selection change, where we find the targeted packages using
`Save-Package-Properties.ps1` primarily. Analyze checks have also been
updated to honor this new selection methodology, rather than the hardcoded
artifactslist.

Only `js - pullrequest` will actually honor the expand and contract for
the packages that were changed in the PR. Meanwhile the `internal`
versions of the `service`-specific definitions will still exist, still
select packages for the service directory, and still be used to release
as normal.

`js - pullrequest` will run _alongside_ the existing builds for a short
period while we identify any outstanding bugs or collection adjustments.
After we're happy, we'll disable all public js builds in favor of the
single `js - pullrequest` build.

---------

Co-authored-by: Ben Broderick Phillips <[email protected]>

Author: scbedd

.vscode/cspell.json

@@ -168,7 +168,11 @@
   "overrides": [
     {
       "filename": "eng/pipelines",
-      "words": ["azuresdkartifacts", "gdnbaselines", "policheck"]
+      "words": ["azuresdkartifacts", "gdnbaselines", "policheck", "issecret", "dlrw"]
+    },
+    {
+      "filename": "eng/tools/rush-runner.js",
+      "words": ["Jsons"]
     },
     {
       "filename": "sdk/apimanagement/api-management-custom-widgets-scaffolder/review/api-management-custom-widgets-scaffolder.api.md",

eng/pipelines/templates/jobs/ci.yml

@@ -74,9 +74,16 @@ jobs:
           MatrixConfigs: ${{ parameters.MatrixConfigs }}
           MatrixFilters: ${{ parameters.MatrixFilters }}
           MatrixReplace: ${{ parameters.MatrixReplace }}
+          SparseCheckoutPaths: [ "sdk/", ".vscode"]
           CloudConfig:
             Cloud: Public
           AdditionalParameters:
             ServiceDirectory: ${{ parameters.ServiceDirectory }}
             Artifacts: ${{ parameters.Artifacts }}
             TestProxy: ${{ parameters.TestProxy }}
+          ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
+            EnablePRGeneration: true
+            PreGenerationSteps:
+              - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+                parameters:
+                  ServiceDirectory: ${{parameters.ServiceDirectory}}

eng/pipelines/templates/steps/analyze.yml

@@ -4,13 +4,22 @@ parameters:
   TestPipeline: false
 
 steps:
+  - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+    parameters:
+      ServiceDirectory: ${{parameters.ServiceDirectory}}
+
+  - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
+    parameters:
+      PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
+      Artifacts: ${{ parameters.Artifacts }}
+
   - template: /eng/common/pipelines/templates/steps/check-spelling.yml
 
   - task: PowerShell@2
     inputs:
       targetType: 'filePath'
-      filePath: eng/scripts/spell-check-public-api.ps1
-      arguments: -ServiceDirectory ${{ parameters.ServiceDirectory }}
+      filePath: eng/scripts/spell-check-public-apis.ps1
+      arguments: -ChangedServices "$(ChangedServices)"
       pwsh: true
     displayName: Spell check public API
 
@@ -26,10 +35,9 @@ steps:
       ArtifactName: 'package-diffs'
       SbomEnabled: false
 
-
-  - template: /eng/common/pipelines/templates/steps/verify-readme.yml
+  - template: /eng/common/pipelines/templates/steps/verify-readmes.yml
     parameters:
-      ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
+      PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
 
   - template: /eng/common/pipelines/templates/steps/verify-path-length.yml
     parameters:
@@ -46,7 +54,7 @@ steps:
 
   - template: /eng/common/pipelines/templates/steps/verify-samples.yml
     parameters:
-      ServiceDirectory: ${{ parameters.ServiceDirectory }}
+      ServiceDirectories: $(ChangedServicesCsv)
 
   - script: |
       npm ci
@@ -62,28 +70,21 @@ steps:
       node common/scripts/install-run-rush.js install
     displayName: "Install dependencies"
 
-  - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
-    parameters:
-      Artifacts: ${{ parameters.Artifacts }}
-
   - script: |
-      node eng/tools/rush-runner.js build "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
+      node eng/tools/rush-runner.js build $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Build libraries"
 
   - template: /eng/pipelines/templates/steps/run-eslint.yml
     parameters:
-      ServiceDirectory: ${{ parameters.ServiceDirectory }}
+      ServiceDirectories: $(ChangedServices)
 
   - pwsh: |
-      node eng/tools/rush-runner.js check-format "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose
+      node eng/tools/rush-runner.js check-format $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose
     displayName: "Check Format in Libraries"
 
-  - ${{ each artifact in parameters.Artifacts }}:
-      - template: /eng/common/pipelines/templates/steps/verify-changelog.yml
-        parameters:
-          PackageName: ${{artifact.name}}
-          ServiceName: ${{parameters.ServiceDirectory}}
-          ForRelease: false
+  - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml
+    parameters:
+      PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
 
   - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
     # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR

eng/pipelines/templates/steps/build.yml

@@ -13,32 +13,23 @@ steps:
 
   - pwsh: |
       $folder = "${{parameters.ServiceDirectory}}"
-      if ($folder -eq "*") { $folder = "" }
+      if ($folder -eq "*" -or $folder -eq "auto") { $folder = "" }
       echo "##vso[task.setvariable variable=folder]$folder"
     displayName: "Set folder variable for readme links"
 
+  # we are not passing service directory, so we only ever set dev build to true
   - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml
-    parameters:
-      ServiceDirectory: ${{ parameters.ServiceDirectory }}
 
   - script: |
       npm install ./eng/tools/versioning
-      node eng/tools/versioning/set-dev.js --build-id "$(Build.BuildNumber)" --repo-root "$(Build.SourcesDirectory)" --service "$(folder)"
+      node eng/tools/versioning/set-dev.js --build-id "$(Build.BuildNumber)" --repo-root "$(Build.SourcesDirectory)"
       node common/scripts/install-run-rush.js update
     condition: and(succeeded(),eq(variables['SetDevVersion'],'true'))
     displayName: "Update package versions for dev build"
 
-  - task: Powershell@2
-    inputs:
-      filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
-      arguments: >
-        -ServiceDirectory ${{parameters.ServiceDirectory}}
-        -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
-        -AddDevVersion
-      pwsh: true
-      workingDirectory: $(Pipeline.Workspace)
-    displayName: Update package properties with dev version
-    condition: and(succeeded(),eq(variables['SetDevVersion'],'true'))
+  - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+    parameters:
+      ServiceDirectory: ${{parameters.ServiceDirectory}}
 
   - script: |
       node common/scripts/install-run-rush.js install
@@ -47,7 +38,9 @@ steps:
 
   - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
     parameters:
+      PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
       Artifacts: ${{ parameters.Artifacts }}
+
   - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
     - task: AzureCLI@2
       inputs:
@@ -60,26 +53,24 @@ steps:
           sasToken=$(az storage container generate-sas --account-name azuresdkartifacts --name azure-sdk-for-js-rush-cache --permissions dlrw --auth-mode login --as-user --expiry $expiry --https-only -o tsv)
           echo "##vso[task.setvariable variable=rushBuildCacheCred;issecret=true;]$sasToken"
 
-  # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
-  # The default on Windows is "cores - 1" (microsoft/rushstack#436).
-  - script: |
-      node eng/tools/rush-runner.js build "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
+  - pwsh: |
+      node eng/tools/rush-runner.js build $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Build libraries"
     env:
       ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
         RUSH_BUILD_CACHE_CREDENTIAL: $(rushBuildCacheCred)
         RUSH_BUILD_CACHE_WRITE_ALLOWED: 1
 
   - script: |
-      node eng/tools/rush-runner.js build:samples "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose
-    displayName: "Build samples"
+      node eng/tools/rush-runner.js build:samples "$(ChangedServices)" -packages "$(ArtifactPackageNames)" --verbose
+    displayName: "Build samples for PR"
 
   - pwsh: |
       eng/tools/check-api-warning.ps1
     displayName: "Check api extractor output changes"
 
   - script: |
-      node eng/tools/rush-runner.js pack "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose
+      node eng/tools/rush-runner.js pack "$(ChangedServices)" -packages "$(ArtifactPackageNames)" --verbose
     displayName: "Pack libraries"
 
   # Unlink node_modules folders to significantly improve performance of subsequent tasks
@@ -93,18 +84,22 @@ steps:
       ServiceDirectory: ${{parameters.ServiceDirectory}}
 
   - pwsh: |
-      $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
+      $artifacts = Get-ChildItem -Recurse -Force "$(Build.ArtifactStagingDirectory)/PackageInfo/*.json" `
+          | ForEach-Object { $_.Name.Replace(".json", "") }
+
       foreach ($artifact in $artifacts)
       {
-        $artifactName = $artifact.name
-        Write-Host "Copying $artifactName artifacts to $(Build.ArtifactStagingDirectory)/$artifactName"
-        New-Item -Type Directory -Name $artifactName -Path $(Build.ArtifactStagingDirectory) > $null
-        Copy-Item sdk/${{parameters.ServiceDirectory}}/**/$artifactName-[0-9]*.[0-9]*.[0-9]*.tgz $(Build.ArtifactStagingDirectory)/$artifactName
-        Copy-Item sdk/${{parameters.ServiceDirectory}}/**/browser/$artifactName-[0-9]*.[0-9]*.[0-9]*.zip $(Build.ArtifactStagingDirectory)/$artifactName
-        if ($${{ parameters.IncludeRelease }} -eq $true -and $artifact.skipPublishDocMs -ne $true)
+        $artifactDetails = Get-Content -Raw $(Build.ArtifactStagingDirectory)/PackageInfo/$artifact.json | ConvertFrom-Json
+
+        Write-Host "Copying $artifact artifacts to $(Build.ArtifactStagingDirectory)/$artifact"
+        New-Item -Type Directory -Force -Name $artifact -Path $(Build.ArtifactStagingDirectory) > $null
+        Copy-Item sdk/$($artifactDetails.Service)/**/$artifact-[0-9]*.[0-9]*.[0-9]*.tgz $(Build.ArtifactStagingDirectory)/$artifact
+        Copy-Item sdk/$($artifactDetails.Service)/**/browser/$artifact-[0-9]*.[0-9]*.[0-9]*.zip $(Build.ArtifactStagingDirectory)/$artifact
+
+        if ($${{ parameters.IncludeRelease }} -eq $true -and $artifactDetails.ArtifactDetails.skipPublishDocMs -ne $true)
         {
-          New-Item -Type Directory -Name documentation -Path $(Build.ArtifactStagingDirectory)/$artifactName > $null
-          Copy-Item $(Build.SourcesDirectory)/docGen/$artifactName.zip $(Build.ArtifactStagingDirectory)/$artifactName/documentation
+          New-Item -Type Directory -Force -Name documentation -Path $(Build.ArtifactStagingDirectory)/$artifact > $null
+          Copy-Item $(Build.SourcesDirectory)/docGen/$artifact.zip $(Build.ArtifactStagingDirectory)/$artifact/documentation
         }
       }
     displayName: 'Copy Packages'

eng/pipelines/templates/steps/generate-doc.yml

@@ -15,7 +15,17 @@ steps:
   - pwsh: |
       node $(Build.SourcesDirectory)/eng/tools/generate-doc/dist/index.js --serviceDir "${{parameters.ServiceDirectory}}"
     displayName: "Run Typedoc Docs"
+    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+  - pwsh: |
+      if ('$(ChangedServices)' -and '$(ChangedServices)' -notlike '*ChangedServices*') {
+        foreach($service in '$(ChangedServices)'.Split(" ")) {
+          node $(Build.SourcesDirectory)/eng/tools/generate-doc/dist/index.js --serviceDir "$service"
+        }
+      }
+    displayName: "Run Typedocs for PR"
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
 
   - pwsh: |
       $(Build.SourcesDirectory)/eng/tools/compress-subfolders.ps1 "$(Build.SourcesDirectory)/docGen" "$(Build.SourcesDirectory)/docGen"
-    displayName: "Generate Typedoc Docs"
+    displayName: "Compress generated docs"

eng/pipelines/templates/steps/run-eslint.yml

@@ -1,5 +1,5 @@
 parameters:
-  ServiceDirectory: ''
+  ServiceDirectories: ''
 
 steps:
   - script: |
@@ -8,5 +8,5 @@ steps:
 
   - pwsh: |
       node common/scripts/install-run-rush.js build -t @azure/eslint-plugin-azure-sdk -t @azure/monitor-opentelemetry-exporter
-      node eng/tools/rush-runner.js lint "${{parameters.ServiceDirectory}}" -p max
+      node eng/tools/rush-runner.js lint "${{parameters.ServiceDirectories}}" -p max
     displayName: "Build ESLint Plugin and Lint Libraries"

eng/pipelines/templates/steps/set-artifact-packages.yml

@@ -1,12 +1,51 @@
 parameters:
+  PackageInfo: ''
   Artifacts: []
 steps:
+  # Package-Properties folder contains the package properties for all discovered packages that were either A) affected by the PR or
+  # B) explicitly present in the service directory. This repo splits the builds into two categories: "mgmt" and "dataplane". While
+  # a given directory may contain both, there will be a separate build definition to release the management packages. Due to this
+  # we need to artificially filter the packages to ensure that only the packages targeted for THAT ci.yml are built.
+  # When we merge the PR adding js - pullrequest, this code will also merge, meaning the public - <service> - ci builds will still operate
+  # EXACTLY the same as they did before the pipelinev3 change. This is important to ensure that we don't accidentally build the wrong packages
+  # while in the integration period. After we disable all the public `js - <service> - ci` builds, only the `js - pullrequest` build WONT provide
+  # an artifact list, which will allow the expand/contract. Meanwhile the internal builds will continue to provide the artifact list from their
+  # individual ci.yml and ci.mgmt.yml files.
   - pwsh: |
       $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
-      $packages = ""
-      foreach ($artifact in $artifacts)
-      {
-        $packages += "$($artifact.name),"
+      $knownArtifacts = @()
+      if ($artifacts) {
+        $knownArtifacts = $artifacts | ForEach-Object { $_.name }
       }
-      echo "##vso[task.setvariable variable=ArtifactPackageNames]$packages"
-    displayName: "Find Packages to build"
+
+      $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json `
+        | Where-Object { if($knownArtifacts) { $knownArtifacts -contains $_.Name.Replace(".json", "") } else { $true } }
+
+      $pkgNames = $packageProperties | ForEach-Object { $_.Name.Replace(".json", "") }
+
+      $setting = $pkgNames -join ","
+      Write-Host "Setting ArtifactPackageNames to: `n$setting"
+      Write-Host "##vso[task.setvariable variable=ArtifactPackageNames;]$setting"
+    displayName: Resolve Targeted Packages
+    condition: eq(variables['ArtifactPackageNames'], '')
+
+  - pwsh: |
+      # set changed services given the set of changed packages, this will mean that
+      # ChangedServices will be appropriate for the batched set of packages if that is indeed how
+      # we set the targeted artifacts
+      $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackageInfo }}" *.json `
+        | Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json }
+
+      $packageSet = "$(ArtifactPackageNames)" -split ","
+
+      $changedServicesArray = $packageProperties | Where-Object { $packageSet -contains $_.ArtifactName }
+        | ForEach-Object { $_.ServiceDirectory } | Get-Unique
+
+      $changedServices = $changedServicesArray -join " "
+      $commaChangedServices = $changedServicesArray -join ","
+      Write-Host "##vso[task.setvariable variable=ChangedServices;]$changedServices"
+      Write-Host "##vso[task.setvariable variable=ChangedServicesCsv;]$commaChangedServices"
+
+      Write-Host "This run is targeting: `n$(ArtifactPackageNames) in [$changedServices]"
+    displayName: Resolve Targeted Packages
+    condition: ne(variables['ArtifactPackageNames'], '')

eng/pipelines/templates/steps/test.yml

@@ -9,42 +9,47 @@ steps:
     parameters:
       AgentImage: ${{ parameters.OSName}}
 
+  - template: /eng/common/pipelines/templates/steps/save-package-properties.yml
+    parameters:
+      ServiceDirectory: ${{parameters.ServiceDirectory}}
+
   - script: |
       node common/scripts/install-run-rush.js install
     displayName: "Install dependencies"
 
   - template: /eng/pipelines/templates/steps/set-artifact-packages.yml
     parameters:
+      PackageInfo: $(Build.ArtifactStagingDirectory)/PackageInfo
       Artifacts: ${{ parameters.Artifacts }}
 
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).
   - script: |
-      node eng/tools/rush-runner.js build "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
+      node eng/tools/rush-runner.js build $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Build libraries"
 
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).
   - script: |
-      node eng/tools/rush-runner.js build:test "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
+      node eng/tools/rush-runner.js build:test $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Build test assets"
 
   - template: ../steps/use-node-test-version.yml
 
   - ${{ if eq(parameters.TestProxy, true) }}:
     - template: /eng/common/testproxy/test-proxy-standalone-tool.yml
-  
+
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).
   - script: |
-      node eng/tools/rush-runner.js unit-test:node "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
+      node eng/tools/rush-runner.js unit-test:node $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Test libraries"
     condition: and(succeeded(),eq(variables['TestType'], 'node'))
 
   # Option "-p max" ensures parallelism is set to the number of cores on all platforms, which improves build times.
   # The default on Windows is "cores - 1" (microsoft/rushstack#436).
   - script: |
-      node eng/tools/rush-runner.js unit-test:browser "${{parameters.ServiceDirectory}}" -packages "$(ArtifactPackageNames)" --verbose -p max
+      node eng/tools/rush-runner.js unit-test:browser $(ChangedServices) -packages "$(ArtifactPackageNames)" --verbose -p max
     displayName: "Test libraries"
     condition: and(succeeded(),eq(variables['TestType'], 'browser'))
 
@@ -53,7 +58,7 @@ steps:
         copy $(Build.SourcesDirectory)/test-proxy.log $(Build.ArtifactStagingDirectory)
       displayName: 'Dump Test Proxy logs'
       condition: succeededOrFailed()
-      
+
   # Unlink node_modules folders to significantly improve performance of subsequent tasks
   # which need to walk the directory tree (and are hardcoded to follow symlinks).
   # Retry for 30 seconds, since this command may fail with error "Another rush command is already