diff --git a/.editorconfig b/.editorconfig index c3bceb7b72f..823d0dac3fc 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,6 +10,9 @@ trim_trailing_whitespace = true [*.json] indent_size = 2 +[*.ps1] +indent_size = 2 + [*.rs] end_of_line = lf diff --git a/.vscode/settings.json b/.vscode/settings.json index 4fd4711c61c..9bf3923fb8a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,9 @@ { "cSpell.enabled": true, "editor.formatOnSave": true, - "rust-analyzer.cargo.features": "all" + "rust-analyzer.cargo.features": "all", + "[powershell]": { + "editor.defaultFormatter": "ms-vscode.powershell", + }, + "yaml.format.printWidth": 240 } diff --git a/eng/CredScanSuppression.json b/eng/CredScanSuppression.json new file mode 100644 index 00000000000..5cfa9c47fe2 --- /dev/null +++ b/eng/CredScanSuppression.json @@ -0,0 +1,17 @@ +{ + "tool": "Credential Scanner", + "suppressions": [ + { + "placeholder": [ + "placeholder" + ], + "_justification": "Secret used by test code, it is fake." + }, + { + "file": [ + "eng/common/testproxy/dotnet-devcert.pfx" + ], + "_justification": "File contains private key used by test code." + } + ] +} \ No newline at end of file diff --git a/eng/pipelines/pr.yml b/eng/pipelines/pr.yml new file mode 100644 index 00000000000..4202446f655 --- /dev/null +++ b/eng/pipelines/pr.yml @@ -0,0 +1,12 @@ +trigger: none + +pr: + branches: + include: + - main + - feature/* + - hotfix/* + - release/* + +extends: + template: /eng/pipelines/templates/stages/archetype-sdk-client.yml diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml new file mode 100644 index 00000000000..39b4d42633c --- /dev/null +++ b/eng/pipelines/templates/jobs/ci.yml @@ -0,0 +1,114 @@ +parameters: + - name: ServiceDirectory + type: string + default: auto + - name: BeforeTestSteps + type: object + default: [] + - name: AfterTestSteps + type: object + default: [] + - name: TestTimeoutInMinutes + type: number + default: 60 + - name: BuildDocs + type: boolean + default: true + - name: TestProxy + type: boolean + default: false + - name: TestPipeline + type: boolean + default: false + - name: GenerateApiReviewForManualOnly + type: boolean + default: false + - name: BuildMatrix + type: object + default: + - pool: + os: linux + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + Toolchains: + - name: stable + publish: true + - name: msrv + - name: nightly + - pool: + os: windows + name: $(WINDOWSPOOL) + image: $(WINDOWSVMIMAGE) + Toolchains: + - name: stable + - name: msrv + - name: nightly + - pool: + os: macOS + name: $(MACPOOL) + vmImage: $(MACVMIMAGE) + Toolchains: + - name: stable + - name: msrv + - name: nightly + - name: AnalyzeToolchains + type: object + default: [stable] + +jobs: + - ${{ each matrix in parameters.BuildMatrix }}: + - ${{ each toolchain in matrix.Toolchains }}: + - job: Build_${{ matrix.pool.os }}_${{ toolchain.name }} + timeoutInMinutes: 90 + + pool: ${{ matrix.pool }} + + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: [/*] + + - template: /eng/pipelines/templates/steps/test-packages.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Toolchain: ${{ toolchain.name }} + ArtifactSuffix: ${{ matrix.pool.os }}_${{ toolchain.name }} + PublishArtifacts: ${{ eq(toolchain.publish, 'true') }} + UnitTests: true + FunctionalTests: ${{ ne(variables['Build.Reason'], 'PullRequest') }} + + - ${{ each toolchain in parameters.AnalyzeToolchains }}: + - job: "Analyze_${{ toolchain }}" + condition: and(succeededOrFailed(), ne(variables['Skip.Analyze'], 'true')) + timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} + + pool: + os: linux + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: [/*] + + - template: /eng/pipelines/templates/steps/analyze.yml + parameters: + Toolchain: ${{ toolchain }} + + - template: /eng/common/pipelines/templates/steps/check-spelling.yml + parameters: + ContinueOnError: false + + # Disabled until we fix crates.io link checking + # - template: /eng/common/pipelines/templates/steps/verify-links.yml + # parameters: + # ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + # Directory: "" + # Urls: (eng/common/scripts/get-markdown-files-from-changed-files.ps1) + # ${{ elseif eq(parameters.ServiceDirectory, 'auto') }}: + # Directory: "" + # ${{ else }}: + # Directory: sdk/${{ parameters.ServiceDirectory }} + # CheckLinkGuidance: $true + # Condition: succeededOrFailed() diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml new file mode 100644 index 00000000000..d2ac6eab6f3 --- /dev/null +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -0,0 +1,61 @@ +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: 1ESPipelineTemplatesCanary + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/canary + +parameters: + - name: stages + type: stageList + default: [] + - name: Use1ESOfficial + type: boolean + default: true + - name: oneESTemplateTag + type: string + default: release + +extends: + ${{ if and(parameters.Use1ESOfficial, eq(parameters.oneESTemplateTag, 'canary')) }}: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplatesCanary + ${{ elseif eq(parameters.oneESTemplateTag, 'canary') }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplatesCanary + ${{ elseif and(parameters.Use1ESOfficial, eq(variables['System.TeamProject'], 'internal')) }}: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + ${{ if eq(parameters.oneESTemplateTag, 'canary') }}: + # Enable 1es template team to verify validation has been run on canary + customBuildTags: + - 1ES.PT.Tag-refs/tags/canary + settings: + skipBuildTagsForGitHubPullRequests: true + sdl: + sourceAnalysisPool: + name: azsdk-pool-mms-win-2022-general + image: azsdk-pool-mms-win-2022-1espt + os: windows + eslint: + enabled: false + justificationForDisabling: "ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=3556850" + codeql: + compiled: + enabled: false + justificationForDisabling: "CodeQL times our pipelines out by running for 2+ hours before being force canceled." + credscan: + suppressionsFile: $(Build.SourcesDirectory)/eng/CredScanSuppression.json + toolVersion: "2.3.12.23" + baselineFiles: $(Build.SourcesDirectory)/eng/python.gdnbaselines + psscriptanalyzer: + compiled: true + break: true + policy: M365 + # Turn off the build warnings caused by disabling some sdl checks + createAdoIssuesForJustificationsForDisablement: false + stages: ${{ parameters.stages }} diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml new file mode 100644 index 00000000000..a80814864f4 --- /dev/null +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -0,0 +1,38 @@ +parameters: + - name: ServiceDirectory + type: string + default: auto + - name: TestTimeoutInMinutes + type: number + default: 60 + - name: BuildDocs + type: boolean + default: true + - name: TestProxy + type: boolean + default: true + - name: GenerateApiReviewForManualOnly + type: boolean + default: false + - name: oneESTemplateTag + type: string + default: release + +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + oneESTemplateTag: ${{ parameters.oneESTemplateTag }} + stages: + - stage: Build + variables: + - template: /eng/pipelines/templates/variables/image.yml + jobs: + - template: /eng/pipelines/templates/jobs/ci.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + ${{ if eq(parameters.ServiceDirectory, 'template') }}: + TestPipeline: true + TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} + BuildDocs: ${{ parameters.BuildDocs }} + TestProxy: ${{ parameters.TestProxy }} + GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }} diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml new file mode 100644 index 00000000000..ed0b41961cf --- /dev/null +++ b/eng/pipelines/templates/steps/analyze.yml @@ -0,0 +1,21 @@ +parameters: + - name: Toolchain + type: string + default: stable + +steps: + - template: /eng/pipelines/templates/steps/use-rust.yml@self + parameters: + Toolchain: ${{ parameters.Toolchain }} + + - pwsh: | + . ./eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 + + $env:RUSTDOCFLAGS = "-D warnings" + $env:RUSTFLAGS = "-Dwarnings" + + Invoke-LoggedCommand "cargo +$(Toolchain) check -p azure_core --no-default-features" + Invoke-LoggedCommand "cargo +$(Toolchain) fmt --all -- --check" + Invoke-LoggedCommand "cargo +$(Toolchain) clippy --all" + Invoke-LoggedCommand "cargo +$(Toolchain) doc --all --no-deps" + displayName: "Run source analysis" diff --git a/eng/pipelines/templates/steps/test-packages.yml b/eng/pipelines/templates/steps/test-packages.yml new file mode 100644 index 00000000000..f4c3950312d --- /dev/null +++ b/eng/pipelines/templates/steps/test-packages.yml @@ -0,0 +1,105 @@ +parameters: + - name: ServiceDirectory + type: string + default: auto + - name: Toolchain + type: string + default: stable + - name: ArtifactSuffix + type: string + default: "linux" + - name: PublishArtifacts + type: boolean + default: false + - name: UnitTests + type: boolean + default: false + - name: FunctionalTests + type: boolean + default: false + - name: TestTimeoutInMinutes + type: number + default: 60 + +steps: + - template: /eng/pipelines/templates/steps/use-rust.yml@self + parameters: + Toolchain: ${{ parameters.Toolchain }} + + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml@self + + - script: | + echo "##vso[build.addbuildtag]Scheduled" + displayName: "Tag scheduled builds" + condition: and(eq(variables['Build.SourceBranchName'], variables['DefaultBranch']), eq(variables['Build.Reason'],'Schedule')) + + # now we need to call Save-Package-Properties so that we can filter on it + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - pwsh: | + mkdir -p $(Build.ArtifactStagingDirectory)/diff + displayName: Create PR Diff Folder + + - pwsh: | + $location = Join-Path "$(Build.ArtifactStagingDirectory)" "diff" + + Write-Host "./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath `"$(Build.SourcesDirectory)`" -ArtifactPath `"$location`"" + ./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath "$(Build.SourcesDirectory)" -ArtifactPath "$location" + displayName: Generate PR Diff + + - pwsh: | + Write-Host "Freshly generated the PR diff:" + Get-ChildItem -R -Force $(Build.ArtifactStagingDirectory)/diff | % { $_.FullName } + cat $(Build.ArtifactStagingDirectory)/diff/diff.json + displayName: Dump PR Diff + + - task: Powershell@2 + displayName: Save package properties filtered for PR + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -PrDiff $(Build.ArtifactStagingDirectory)/diff/diff.json + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + pwsh: true + + - ${{ else }}: + - task: Powershell@2 + displayName: Save package properties with dev version + condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -AddDevVersion + pwsh: true + - task: Powershell@2 + displayName: Save package properties for service + condition: and(succeeded(), ne(variables['SetDevVersion'],'true')) + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + pwsh: true + + - task: Powershell@2 + displayName: "Test Packages" + condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) + timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} + env: + CIBW_BUILD_VERBOSITY: 3 + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/Test-Packages.ps1 + arguments: > + -PackageInfoPath '(Build.ArtifactStagingDirectory)/PackageInfo' + -Toolchain '$(Toolchain)' + -UnitTests $${{ parameters.UnitTests }} + -FunctionalTests $${{ parameters.FunctionalTests }} + + - ${{ if parameters.PublishArtifacts}}: + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: "$(Build.ArtifactStagingDirectory)" + ArtifactName: "packages_${{ parameters.ArtifactSuffix }}" + CustomCondition: and(succeeded(), ne(variables['NoPackagesChanged'],'true')) diff --git a/eng/pipelines/templates/steps/use-rust.yml b/eng/pipelines/templates/steps/use-rust.yml new file mode 100644 index 00000000000..e8b83170d05 --- /dev/null +++ b/eng/pipelines/templates/steps/use-rust.yml @@ -0,0 +1,19 @@ +parameters: + - name: Toolchain + type: string + default: stable + +steps: + - pwsh: | + $toolchain = '${{ parameters.Toolchain }}' + if ($toolchain -eq 'msrv') { + Write-Host "Reading MSRV from azure_core" + $toolchain = cargo metadata --manifest-path ./sdk/core/azure_core/Cargo.toml --format-version 1 | convertfrom-json | select -expand packages | where { $_.name -eq 'azure_core' } | select -expand rust_version + } + + Write-Host "Setting Toolchain variable to $toolchain" + Write-Host "##vso[task.setvariable variable=Toolchain]$toolchain" + + Write-Host "> rustup update --no-self-update $toolchain" + rustup update --no-self-update $toolchain + displayName: "Use Rust ${{ parameters.Toolchain }}" diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml new file mode 100644 index 00000000000..0b0cbb2935c --- /dev/null +++ b/eng/pipelines/templates/variables/globals.yml @@ -0,0 +1 @@ +variables: diff --git a/eng/pipelines/templates/variables/image.yml b/eng/pipelines/templates/variables/image.yml new file mode 100644 index 00000000000..681e4709f4b --- /dev/null +++ b/eng/pipelines/templates/variables/image.yml @@ -0,0 +1,26 @@ +# Default pool image selection. Set as variable so we can override at pipeline level + +variables: + - name: LINUXPOOL + value: azsdk-pool-mms-ubuntu-2004-general + - name: WINDOWSPOOL + value: azsdk-pool-mms-win-2022-general + - name: MACPOOL + value: Azure Pipelines + + - name: LINUXVMIMAGE + value: azsdk-pool-mms-ubuntu-2004-1espt + - name: LINUXNEXTVMIMAGE + value: ubuntu-22.04 + - name: WINDOWSVMIMAGE + value: azsdk-pool-mms-win-2022-1espt + - name: MACVMIMAGE + value: macos-latest + + # Values required for pool.os field in 1es pipeline templates + - name: LINUXOS + value: linux + - name: WINDOWSOS + value: windows + - name: MACOS + value: macOS diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 new file mode 100644 index 00000000000..2780cd3c516 --- /dev/null +++ b/eng/scripts/Language-Settings.ps1 @@ -0,0 +1,93 @@ +$Language = "rust" +$LanguageDisplayName = "Rust" +$PackageRepository = "crates.io" +$packagePattern = "Cargo.toml" +#$MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/rust-packages.csv" +$GithubUri = "https://github.com/Azure/azure-sdk-for-rust" +$PackageRepositoryUri = "https://crates.io/crates" + +function Get-AllPackageInfoFromRepo ([string] $ServiceDirectory) { + $allPackageProps = @() + Push-Location $RepoRoot + try { + $searchPath = Join-Path $RepoRoot 'sdk' -Resolve + + if ($ServiceDirectory -and $ServiceDirectory -ne 'auto') { + $searchPath = Join-Path 'sdk' $ServiceDirectory + } + + $packages = cargo metadata --format-version 1 + | ConvertFrom-Json -AsHashtable + | Select-Object -ExpandProperty packages + | Where-Object { $_.manifest_path.StartsWith($searchPath) } + + $packageManifests = @{} + foreach ($package in $packages) { + if ($package.manifest_path -replace '\\', '/' -match '/sdk/([^/]+)/') { + $package.ServiceDirectoryName = $Matches[1] + } + else { + # ignore manifests that are not in a service directory + continue + } + + $package.RelativePath = (Split-Path $package.manifest_path -Parent).Replace($RepoRoot, "").SubString(1) + $package.DependentPackages = @() + $packageManifests[$package.name] = $package + } + } + catch { + # This is soft error and failure is expected for python metapackages + LogError "Failed to get all package properties" + } + finally { + Pop-Location + } + + # Invert the manifest dependency graph + foreach ($package in $packageManifests.Values) { + foreach ($dependency in $package.dependencies) { + $dependencyManifest = $packageManifests[$dependency.name] + if ($dependencyManifest) { + $dependencyManifest.DependentPackages += $package + } + } + } + + # Flatten the dependency graph recursively + function GetDependentPackages($package, $dependantPackages) { + if (!$dependantPackages) { + $dependantPackages = @() + } + + foreach ($dependency in $package.DependentPackages) { + if ($dependantPackages.Contains($dependency)) { + continue + } + $dependantPackages += $dependency + [array]$dependantPackages = GetDependentPackages $dependency $dependantPackages + } + + return $dependantPackages; + } + + foreach ($package in $packageManifests.Values) { + $absolutePath = Split-Path $package.manifest_path -Parent -Resolve + $pkgProp = [PackageProps]::new($package.name, $package.version, $absolutePath, $package.ServiceDirectoryName) + $pkgProp.IsNewSdk = $true + $pkgProp.ArtifactName = $package.name + + if ($package.name -match "mgmt") { + $pkgProp.SdkType = "mgmt" + } + else { + $pkgProp.SdkType = "client" + } + + $pkgProp.DependentPackages = GetDependentPackages $package | Select-Object -ExpandProperty RelativePath + + $allPackageProps += $pkgProp + } + + return $allPackageProps +} diff --git a/eng/scripts/Test-Packages.ps1 b/eng/scripts/Test-Packages.ps1 new file mode 100644 index 00000000000..cdbccdbfc59 --- /dev/null +++ b/eng/scripts/Test-Packages.ps1 @@ -0,0 +1,64 @@ +#Requires -Version 7.0 + +param( + [string]$Toolchain = 'stable', + [bool]$UnitTests = $true, + [bool]$FunctionalTests = $true, + [string]$PackageInfoPath +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +. "$PSScriptRoot\..\common\scripts\common.ps1" +. (Join-Path $EngCommonScriptsDir "Helpers" CommandInvocation-Helpers.ps1) + +Write-Host "Testing packages with + Toolchain: '$Toolchain' + UnitTests: '$UnitTests' + FunctionalTests: '$FunctionalTests' + PackageInfoPath: '$PackageInfoPath'" + +if ($PackageInfoPath) { + if (!(Test-Path $PackageInfoPath)) { + Write-Error "Package info path '$PackageInfoPath' does not exist." + exit 1 + } + + $pacakgesToTest = Get-ChildItem $PackageInfoPath -Filter "*.json" -Recurse + | ConvertFrom-Json +} +else { + $pacakgesToTest = Get-AllPackagesInRepo +} + +Write-Host "Testing packages:" +foreach ($package in $pacakgesToTest) { + Write-Host " '$($package.Name)'" +} + +Write-Host "Setting RUSTFLAGS to '-Dwarnings'" +$env:RUSTFLAGS = "-Dwarnings" + +$verifyDependenciesScript = Join-Path $RepoRoot '/eng/scripts/verify-dependencies.rs' -Resolve + +foreach ($package in $pacakgesToTest) { + Push-Location $package.DirectoryPath + try { + Write-Host "`n`nTesting package: '$($package.Name)' in directory: '$($package.DirectoryPath)'`n" + + Invoke-LoggedCommand "cargo +$Toolchain build --keep-going" + Write-Host "`n`n" + Invoke-LoggedCommand "cargo +$Toolchain test --lib --no-fail-fast" + Write-Host "`n`n" + Invoke-LoggedCommand "cargo +$Toolchain test --doc --no-fail-fast" + + if ($Toolchain -eq 'nightly') { + Write-Host "`n`n" + Invoke-LoggedCommand "cargo +nightly -Zscript $verifyDependenciesScript" + } + } + finally { + Pop-Location + } +}