[main] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24352.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="10.0.0-beta.24620.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4a7d983f833d6b86365ea1b2b4d6ee72fbdbf944</Sha>
+      <Sha>ddb0b75f326734f82e94a85d7dee9b299ecdf7fc</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/SetupNugetSources.ps1

@@ -157,7 +157,7 @@ if ($dotnet31Source -ne $null) {
     AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
 }
 
-$dotnetVersions = @('5','6','7','8')
+$dotnetVersions = @('5','6','7','8','9')
 
 foreach ($dotnetVersion in $dotnetVersions) {
     $feedPrefix = "dotnet" + $dotnetVersion;

eng/common/SetupNugetSources.sh

@@ -99,7 +99,7 @@ if [ "$?" == "0" ]; then
     PackageSources+=('dotnet3.1-internal-transport')
 fi
 
-DotNetVersions=('5' '6' '7' '8')
+DotNetVersions=('5' '6' '7' '8' '9')
 
 for DotNetVersion in ${DotNetVersions[@]} ; do
     FeedPrefix="dotnet${DotNetVersion}";

eng/common/build.sh

@@ -231,7 +231,6 @@ function Build {
     /p:Restore=$restore \
     /p:Build=$build \
     /p:DotNetBuildRepo=$product_build \
-    /p:ArcadeBuildFromSource=$source_build \
     /p:DotNetBuildSourceOnly=$source_build \
     /p:Rebuild=$rebuild \
     /p:Test=$test \

eng/common/core-templates/job/job.yml

@@ -19,6 +19,7 @@ parameters:
   # publishing defaults
   artifacts: ''
   enableMicrobuild: false
+  enableMicrobuildForMacAndLinux: false
   enablePublishBuildArtifacts: false
   enablePublishBuildAssets: false
   enablePublishTestResults: false
@@ -33,11 +34,6 @@ parameters:
   artifactPublishSteps: []
   runAsPublic: false
 
-# Sbom related params
-  enableSbom: true
-  PackageVersion: 9.0.0
-  BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
-
 # 1es specific parameters
   is1ESPipeline: ''
 
@@ -132,18 +128,11 @@ jobs:
       - ${{ preStep }}
 
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
-      - task: MicroBuildSigningPlugin@4
-        displayName: Install MicroBuild plugin
-        inputs:
-          signType: $(_SignType)
-          zipSources: false
-          feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
-        env:
-          TeamName: $(_TeamName)
-          MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
+    - template: /eng/common/core-templates/steps/install-microbuild.yml
+      parameters:
+        enableMicrobuild: ${{ parameters.enableMicrobuild }}
+        enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
         continueOnError: ${{ parameters.continueOnError }}
-        condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
 
   - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
     - task: NuGetAuthenticate@1
@@ -172,14 +161,12 @@ jobs:
   - ${{ each step in parameters.componentGovernanceSteps }}:
     - ${{ step }}
 
-  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
-    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-      - task: MicroBuildCleanup@1
-        displayName: Execute Microbuild cleanup tasks
-        condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - template: /eng/common/core-templates/steps/cleanup-microbuild.yml
+      parameters:
+        enableMicrobuild: ${{ parameters.enableMicrobuild }}
+        enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
         continueOnError: ${{ parameters.continueOnError }}
-        env:
-          TeamName: $(_TeamName)
 
   # Publish test results
   - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:

eng/common/core-templates/job/publish-build-assets.yml

@@ -113,38 +113,19 @@ jobs:
           Add-Content -Path $filePath -Value "$(DefaultChannels)"
           Add-Content -Path $filePath -Value $(IsStableBuild)
 
-    - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
-      parameters:
-        is1ESPipeline: ${{ parameters.is1ESPipeline }}
-        args:
-          displayName: Publish ReleaseConfigs Artifact
-          pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs'
-          publishLocation: Container
-          artifactName: ReleaseConfigs
-
-    - task: powershell@2
-      displayName: Check if SymbolPublishingExclusionsFile.txt exists
-      inputs:
-        targetType: inline
-        script: |
           $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt"
-          if(Test-Path -Path $symbolExclusionfile)
+          if (Test-Path -Path $symbolExclusionfile)
           {
             Write-Host "SymbolExclusionFile exists"
-            Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true"
-          }
-          else{
-           Write-Host "Symbols Exclusion file does not exist"
-           Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false"
+            Copy-Item -Path $symbolExclusionfile -Destination "$(Build.StagingDirectory)/ReleaseConfigs"
           }
 
     - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
       parameters:
         is1ESPipeline: ${{ parameters.is1ESPipeline }}
         args:
-          displayName: Publish SymbolPublishingExclusionsFile Artifact
-          condition: eq(variables['SymbolExclusionFile'], 'true') 
-          pathToPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+          displayName: Publish ReleaseConfigs Artifact
+          pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs'
           publishLocation: Container
           artifactName: ReleaseConfigs
 
@@ -162,9 +143,10 @@ jobs:
           scriptType: ps
           scriptLocation: scriptPath
           scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
-          arguments: -BuildId $(BARBuildId) 
+          arguments: >
+            -BuildId $(BARBuildId)
             -PublishingInfraVersion 3
-            -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
+            -AzdoToken '$(System.AccessToken)'
             -WaitPublishingFinish true
             -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
             -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/common/core-templates/job/source-build.yml

@@ -12,9 +12,10 @@ parameters:
   #   The name of the job. This is included in the job ID.
   # targetRID: ''
   #   The name of the target RID to use, instead of the one auto-detected by Arcade.
-  # nonPortable: false
+  # portableBuild: false
   #   Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
-  #   linux-x64), and compiling against distro-provided packages rather than portable ones.
+  #   linux-x64), and compiling against distro-provided packages rather than portable ones. The
+  #   default is portable mode.
   # skipPublishValidation: false
   #   Disables publishing validation.  By default, a check is performed to ensure no packages are
   #   published by source-build.

eng/common/core-templates/job/source-index-stage1.yml

@@ -1,8 +1,5 @@
 parameters:
   runAsPublic: false
-  sourceIndexUploadPackageVersion: 2.0.0-20240522.1
-  sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1
-  sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
   sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
   preSteps: []
   binlogPath: artifacts/log/Debug/Build.binlog
@@ -16,12 +13,6 @@ jobs:
   dependsOn: ${{ parameters.dependsOn }}
   condition: ${{ parameters.condition }}
   variables:
-  - name: SourceIndexUploadPackageVersion
-    value: ${{ parameters.sourceIndexUploadPackageVersion }}
-  - name: SourceIndexProcessBinlogPackageVersion
-    value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
-  - name: SourceIndexPackageSource
-    value: ${{ parameters.sourceIndexPackageSource }}
   - name: BinlogPath
     value: ${{ parameters.binlogPath }}
   - template: /eng/common/core-templates/variables/pool-providers.yml
@@ -45,35 +36,9 @@ jobs:
 
   - ${{ each preStep in parameters.preSteps }}:
     - ${{ preStep }}
-
-  - task: UseDotNet@2
-    displayName: Use .NET 8 SDK
-    inputs:
-      packageType: sdk
-      version: 8.0.x
-      installationPath: $(Agent.TempDirectory)/dotnet
-      workingDirectory: $(Agent.TempDirectory)
-
-  - script: |
-      $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
-      $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
-    displayName: Download Tools
-    # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
-    workingDirectory: $(Agent.TempDirectory)
-
   - script: ${{ parameters.sourceIndexBuildCommand }}
     displayName: Build Repository
 
-  - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
-    displayName: Process Binlog into indexable sln
-
-  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - task: AzureCLI@2
-      displayName: Log in to Azure and upload stage1 artifacts to source index
-      inputs:
-        azureSubscription: 'SourceDotNet Stage1 Publish'
-        addSpnToEnvironment: true
-        scriptType: 'ps'
-        scriptLocation: 'inlineScript'
-        inlineScript: |
-          $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
+  - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml
+    parameters:
+      binLogPath: ${{ parameters.binLogPath }}

eng/common/core-templates/post-build/post-build.yml

@@ -307,9 +307,10 @@ stages:
             scriptType: ps
             scriptLocation: scriptPath
             scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
-            arguments: -BuildId $(BARBuildId) 
+            arguments: >
+              -BuildId $(BARBuildId)
               -PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
-              -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
+              -AzdoToken '$(System.AccessToken)'
               -WaitPublishingFinish true
               -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
               -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/common/core-templates/steps/cleanup-microbuild.yml

@@ -0,0 +1,28 @@
+parameters:
+  # Enable cleanup tasks for MicroBuild
+  enableMicrobuild: false
+  # Enable cleanup tasks for MicroBuild on Mac and Linux
+  # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
+  enableMicrobuildForMacAndLinux: false
+  continueOnError: false
+
+steps:
+  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+    - task: MicroBuildCleanup@1
+      displayName: Execute Microbuild cleanup tasks
+      condition: and(
+        always(),
+        or(
+          and(
+            eq(variables['Agent.Os'], 'Windows_NT'),
+            in(variables['_SignType'], 'real', 'test')
+          ),
+          and(
+            ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
+            ne(variables['Agent.Os'], 'Windows_NT'),
+            eq(variables['_SignType'], 'real')
+          )
+        ))
+      continueOnError: ${{ parameters.continueOnError }}
+      env:
+        TeamName: $(_TeamName)

eng/common/core-templates/steps/get-federated-access-token.yml

@@ -3,17 +3,31 @@ parameters:
   type: string
 - name: outputVariableName
   type: string
+- name: is1ESPipeline
+  type: boolean
+- name: stepName
+  type: string
+  default: 'getFederatedAccessToken'
+- name: condition
+  type: string
+  default: ''
 # Resource to get a token for. Common values include:
 # - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps
 # - 'https://storage.azure.com/' for storage
 # Defaults to Azure DevOps
 - name: resource
   type: string
   default: '499b84ac-1321-427f-aa17-267ca6975798'
+- name: isStepOutputVariable
+  type: boolean
+  default: false
 
 steps:
 - task: AzureCLI@2
   displayName: 'Getting federated access token for feeds'
+  name: ${{ parameters.stepName }}
+  ${{ if ne(parameters.condition, '') }}:
+    condition: ${{ parameters.condition }}
   inputs:
     azureSubscription: ${{ parameters.federatedServiceConnection }}
     scriptType: 'pscore'
@@ -25,4 +39,4 @@ steps:
         exit 1
       }
       Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
-      Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken"
+      Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken"

eng/common/core-templates/steps/install-microbuild.yml

@@ -0,0 +1,73 @@
+parameters:
+  # Enable install tasks for MicroBuild
+  enableMicrobuild: false
+  # Enable install tasks for MicroBuild on Mac and Linux
+  # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
+  enableMicrobuildForMacAndLinux: false
+  # Location of the MicroBuild output folder
+  microBuildOutputFolder: '$(Agent.TempDirectory)'
+  continueOnError: false
+
+steps:
+  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+    - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}:
+      # Install Python 3.12.x on when Python > 3.12.x is installed - https://github.com/dotnet/source-build/issues/4802
+      - script: |
+          version=$(python3 --version | awk '{print $2}')
+          major=$(echo $version | cut -d. -f1)
+          minor=$(echo $version | cut -d. -f2)
+
+          installPython=false
+          if [ "$major" -gt 3 ] || { [ "$major" -eq 3 ] && [ "$minor" -gt 12 ]; }; then
+            installPython=true
+          fi
+
+          echo "Python version: $version."
+          echo "Install Python 3.12.x: $installPython."
+          echo "##vso[task.setvariable variable=installPython;isOutput=true]$installPython"
+        name: InstallPython
+        displayName: 'Determine Python installation'
+        condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
+
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: '3.12.x'
+        displayName: 'Use Python 3.12.x'
+        condition: and(succeeded(), eq(variables['InstallPython.installPython'], 'true'), ne(variables['Agent.Os'], 'Windows_NT'))
+
+      # Needed to download the MicroBuild plugin nupkgs on Mac and Linux when nuget.exe is unavailable
+      - task: UseDotNet@2
+        displayName: Install .NET 8.0 SDK for MicroBuild Plugin
+        inputs:
+          packageType: sdk
+          version: 8.0.x
+          installationPath: ${{ parameters.microBuildOutputFolder }}/dotnet
+          workingDirectory: ${{ parameters.microBuildOutputFolder }}
+        condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
+
+    - task: MicroBuildSigningPlugin@4
+      displayName: Install MicroBuild plugin
+      inputs:
+        signType: $(_SignType)
+        zipSources: false
+        feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+        ${{ if and(eq(parameters.enableMicrobuildForMacAndLinux, 'true'), ne(variables['Agent.Os'], 'Windows_NT')) }}:
+          azureSubscription: 'MicroBuild Signing Task (DevDiv)'
+      env:
+        TeamName: $(_TeamName)
+        MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }}
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+      continueOnError: ${{ parameters.continueOnError }}
+      condition: and(
+        succeeded(),
+        or(
+          and(
+            eq(variables['Agent.Os'], 'Windows_NT'),
+            in(variables['_SignType'], 'real', 'test')
+          ),
+          and(
+            ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
+            ne(variables['Agent.Os'], 'Windows_NT'),
+            eq(variables['_SignType'], 'real')
+          )
+        ))