Changes for internal runtime build support for .NET 6 (#11422)

- Cherry pick the internal runtime download changes over
- Cherry pick changes to the arcade SB template over (making the next update a noop and unblocking P7+)
- Hoist out the variable groups and parameters needed for the internal download to the top level yaml file.
- Remove the old DownloadFile task and replace this with the Arcade version. Specifically this allows us to remove
  set/use of the DOTNETCLIMSRC_READ_SAS_TOKEN environment variable and instead rely on that task's ability to decode and
  use a base64 encoded SAS token. The reason for the environment variable usage before was that the
  non-encoded SAS token was getting mangled by msbuild/bash/etc. on non-Windows OSs.
- Update the source build tarball template with support for internal runtimes

Update to source build template

.vsts-ci.yml

@@ -31,6 +31,15 @@ variables:
   - name: _NonWindowsTestArg
     value: ''
 
+- name: _InternalRuntimeDownloadArgs
+  value: ''
+
+- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+  - group: DotNet-MSRC-Storage
+  - name: _InternalRuntimeDownloadArgs
+    value: /p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet
+      /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64)
+
 stages:
 - stage: build
   jobs:

eng/build.yml

@@ -54,11 +54,6 @@ phases:
       - _TeamName: Roslyn-Project-System
       - _SignType: test
       - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)'
-      - _DOTNETCLIMSRC_READ_SAS_TOKEN: ''
-
-      - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        - group: DotNet-MSRC-Storage
-        - _DOTNETCLIMSRC_READ_SAS_TOKEN: $(dotnetclimsrc-read-sas-token)
 
       - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
         - group: DotNet-Symbol-Server-PATs
@@ -92,10 +87,10 @@ phases:
                   -Architecture $(_BuildArchitecture)
                   $(_BuildArgs)
                   $(_AdditionalBuildParameters)
+                  $(_InternalRuntimeDownloadArgs)
         displayName: Build
         env:
           DOTNET_CLI_UI_LANGUAGE: $(_DOTNET_CLI_UI_LANGUAGE)
-          DOTNETCLIMSRC_READ_SAS_TOKEN: $(_DOTNETCLIMSRC_READ_SAS_TOKEN)
 
     - ${{ if ne(parameters.agentOs, 'Windows_NT') }}:
       - ${{ if ne(variables['System.TeamProject'], 'public') }}:
@@ -118,9 +113,8 @@ phases:
                   $(_RuntimeIdentifier)
                   $(_BuildArgs)
                   $(_AdditionalBuildParameters)
+                  $(_InternalRuntimeDownloadArgs)
         displayName: Build
-        env:
-          DOTNETCLIMSRC_READ_SAS_TOKEN: $(_DOTNETCLIMSRC_READ_SAS_TOKEN)
 
     - ${{ if or(eq(parameters.agentOs, 'Darwin'), eq(parameters.agentOs, 'FreeBSD')) }}:
       - script: ./build.sh
@@ -132,9 +126,8 @@ phases:
                   $(_RuntimeIdentifier)
                   $(_BuildArgs)
                   $(_AdditionalBuildParameters)
+                  $(_InternalRuntimeDownloadArgs)
         displayName: Build
-        env:
-          DOTNETCLIMSRC_READ_SAS_TOKEN: $(_DOTNETCLIMSRC_READ_SAS_TOKEN)
 
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.agentOs, 'Windows_NT'), ne(variables['PostBuildSign'], 'true')) }}:
       - task: NuGetCommand@2

eng/common/templates/steps/source-build.yml

@@ -18,6 +18,36 @@ steps:
     set -x
     df -h
 
+    # If building on the internal project, the artifact feeds variable may be available (usually only if needed)
+    # In that case, call the feed setup script to add internal feeds corresponding to public ones.
+    # In addition, add an msbuild argument to copy the WIP from the repo to the target build location.
+    # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those
+    # changes.
+    $internalRestoreArgs=
+    if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then
+      # Temporarily work around https://github.com/dotnet/arcade/issues/7709
+      chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+      $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw)
+      internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true'
+
+      # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo.
+      # This only works if there is a username/email configured, which won't be the case in most CI runs.
+      git config --get user.email
+      if [ $? -ne 0 ]; then
+        git config user.email dn-bot@microsoft.com
+        git config user.name dn-bot
+        git config core.autocrlf false
+      fi
+    fi
+
+    # If building on the internal project, the internal storage variable may be available (usually only if needed)
+    # In that case, add variables to allow the download of internal runtimes if the specified versions are not found
+    # in the default public locations.
+    internalRuntimeDownloadArgs=
+    if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then
+      internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64) --runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)'
+    fi
+
     buildConfig=Release
     # Check if AzDO substitutes in a build config from a variable, and use it if so.
     if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
@@ -29,11 +59,6 @@ steps:
       officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
     fi
 
-    internalRuntimeDownloadArgs=
-    if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then
-      internalRuntimeDownloadArgs='--runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)'
-    fi
-
     targetRidArgs=
     if [ '${{ parameters.platform.targetRID }}' != '' ]; then
       targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
@@ -49,6 +74,7 @@ steps:
       --restore --build --pack $publishArgs -bl \
       $officialBuildArgs \
       $internalRuntimeDownloadArgs \
+      $internalRestoreArgs \
       $targetRidArgs \
       /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
       /p:ArcadeBuildFromSource=true

eng/dockerrun.sh

@@ -169,7 +169,6 @@ docker run $INTERACTIVE -t --rm --sig-proxy=true \
     -e POSTBUILDSIGN \
     -e SYSTEM_DEFINITIONID \
     -e SYSTEM_TEAMFOUNDATIONCOLLECTIONURI \
-    -e DOTNETCLIMSRC_READ_SAS_TOKEN \
     -e AGENT_JOBNAME \
     -e AGENT_OS \
     -e VSS_NUGET_URI_PREFIXES \

src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml

@@ -21,12 +21,21 @@ jobs:
         officialBuildArgs='/p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
       fi
 
+      # If building on the internal project, the internal storage variable may be available (usually only if needed)
+      # In that case, add variables to allow the download of internal runtimes if the specified versions are not found
+      # in the default public locations.
+      internalRuntimeDownloadArgs=
+      if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then
+        internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64) --runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)'
+      fi
+
       ./build.sh \
         --ci \
         --configuration $(_BuildConfig) \
         --publish \
         -bl \
         $officialBuildArgs \
+        $internalRuntimeDownloadArgs \
         /p:DotNetPublishUsingPipelines=true \
         /p:ArcadeBuildTarball=true
     displayName: Create Tarball

src/core-sdk-tasks/DownloadFile.cs

@@ -1,138 +0,0 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-using System;
-using System.IO;
-using System.Net;
-using System.Net.Http;
-using System.Threading.Tasks;
-using System.Collections.Generic;
-using Microsoft.Build.Framework;
-using Microsoft.Build.Utilities;
-
-namespace Microsoft.DotNet.Cli.Build
-{
-    public class DownloadFile : Microsoft.Build.Utilities.Task
-    {
-        [Required]
-        public string Uri { get; set; }
-
-        /// <summary>
-        /// If this field is set and the task fail to download the file from `Uri`, with a NotFound
-        /// status, it will try to download the file from `PrivateUri`.
-        /// </summary>
-        public string PrivateUri { get; set; }
-
-        public int MaxRetries { get; set; } = 5;
-
-        [Required]
-        public string DestinationPath { get; set; }
-
-        public bool Overwrite { get; set; }
-
-        public override bool Execute()
-        {
-            return ExecuteAsync().GetAwaiter().GetResult();
-        }
-
-        private async System.Threading.Tasks.Task<bool> ExecuteAsync()
-        {
-            string destinationDir = Path.GetDirectoryName(DestinationPath);
-            if (!Directory.Exists(destinationDir))
-            {
-                Directory.CreateDirectory(destinationDir);
-            }
-
-            if (File.Exists(DestinationPath) && !Overwrite)
-            {
-                return true;
-            }
-
-            const string FileUriProtocol = "file://";
-
-            if (Uri.StartsWith(FileUriProtocol, StringComparison.Ordinal))
-            {
-                var filePath = Uri.Substring(FileUriProtocol.Length);
-                Log.LogMessage($"Copying '{filePath}' to '{DestinationPath}'");
-                File.Copy(filePath, DestinationPath);
-                return true;
-            }
-
-            List<string> errorMessages = new List<string>();
-            bool? downloadStatus = await DownloadWithRetriesAsync(Uri, DestinationPath, errorMessages);
-
-            if (downloadStatus == false && !string.IsNullOrEmpty(PrivateUri))
-            {
-                downloadStatus = await DownloadWithRetriesAsync(PrivateUri, DestinationPath, errorMessages);
-            }
-
-            if (downloadStatus != true)
-            {
-                foreach (var error in errorMessages)
-                {
-                    Log.LogError(error);
-                }
-            }
-
-            return downloadStatus == true;
-        }
-
-        /// <summary>
-        /// Attempt to download file from `source` with retries when response error is different of FileNotFound and Success.
-        /// </summary>
-        /// <param name="source">URL to the file to be downloaded.</param>
-        /// <param name="target">Local path where to put the downloaded file.</param>
-        /// <returns>true: Download Succeeded. false: Download failed with 404. null: Download failed but is retriable.</returns>
-        private async Task<bool?> DownloadWithRetriesAsync(string source, string target, List<string> errorMessages)
-        {
-            Random rng = new Random();
-
-            Log.LogMessage(MessageImportance.High, $"Attempting download '{source}' to '{target}'");
-
-            using (var httpClient = new HttpClient())
-            {
-                for (int retryNumber = 0; retryNumber < MaxRetries; retryNumber++)
-                {
-                    try
-                    {
-                        var httpResponse = await httpClient.GetAsync(source);
-
-                        Log.LogMessage(MessageImportance.Normal, $"{source} -> {httpResponse.StatusCode}");
-
-                        // The Azure Storage REST API returns '400 - Bad Request' in some cases
-                        // where the resource is not found on the storage.
-                        // https://docs.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
-                        if (httpResponse.StatusCode == HttpStatusCode.NotFound ||
-                            httpResponse.ReasonPhrase.IndexOf("The requested URI does not represent any resource on the server.", StringComparison.OrdinalIgnoreCase) == 0)
-                        {
-                            errorMessages.Add($"Problems downloading file from '{source}'. Does the resource exist on the storage? {httpResponse.StatusCode} : {httpResponse.ReasonPhrase}");
-                            return false;
-                        }
-
-                        httpResponse.EnsureSuccessStatusCode();
-
-                        using (var outStream = File.Create(target))
-                        {
-                            await httpResponse.Content.CopyToAsync(outStream);
-                        }
-
-                        Log.LogMessage(MessageImportance.Normal, $"returning true {source} -> {httpResponse.StatusCode}");
-                        return true;
-                    }
-                    catch (Exception e)
-                    {
-                        Log.LogMessage(MessageImportance.High, $"returning error in {source} ");
-                        errorMessages.Add($"Problems downloading file from '{source}'. {e.Message} {e.StackTrace}");
-                        File.Delete(target);
-                    }
-
-                    await System.Threading.Tasks.Task.Delay(rng.Next(1000, 10000));
-                }
-            }
-
-            Log.LogMessage(MessageImportance.High, $"giving up {source} ");
-            errorMessages.Add($"Giving up downloading the file from '{source}' after {MaxRetries} retries.");
-            return null;
-        }
-    }
-}

src/redist/targets/BuildCoreSdkTasks.targets

@@ -20,7 +20,7 @@
   </Target>
 
   <UsingTask TaskName="CalculateTemplateVersions" AssemblyFile="$(CoreSdkTaskDll)" />
-  <UsingTask TaskName="DownloadFile" AssemblyFile="$(CoreSdkTaskDll)" />
+  <UsingTask TaskName="DownloadFile" AssemblyFile="$(ArcadeSdkBuildTasksAssembly)" />
   <UsingTask TaskName="ExtractArchiveToDirectory" AssemblyFile="$(CoreSdkTaskDll)" />
   <UsingTask TaskName="ZipFileCreateFromDirectory" AssemblyFile="$(CoreSdkTaskDll)" />
   <UsingTask TaskName="ZipFileExtractToDirectory" AssemblyFile="$(CoreSdkTaskDll)" />

src/redist/targets/GenerateLayout.targets

@@ -331,16 +331,26 @@
         <PrivateBaseUrl Condition="'$(InternalBuild)' == 'true'">$([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetExtensionsBlobRootUrl)', '$(InternalBaseURL)'))</PrivateBaseUrl>
         <PrivateBaseUrl Condition="'$(InternalBuild)' == 'true'">$([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetToolsetBlobRootUrl)', '$(InternalBaseURL)'))</PrivateBaseUrl>
       </ComponentToDownload>
+
+      <!-- Now transform the group of components into a list of Uris,
+           with shared metadata DownloadDestination and potentially the encoded key token for the private url.
+           Then use task batching to send batched itemgroups to the DownloadFile task that have the same
+           DownloadDestination. -->
+      
+      <UrisToDownload Include="%(ComponentToDownload.BaseUrl)/%(ComponentToDownload.DownloadFileName)" Condition="'%(ComponentToDownload.ShouldDownload)' == 'true'">
+        <ShouldDownload>%(ComponentToDownload.ShouldDownload)</ShouldDownload>
+        <DownloadDestination>%(ComponentToDownload.DownloadDestination)</DownloadDestination>
+      </UrisToDownload>
+      <UrisToDownload Include="%(ComponentToDownload.PrivateBaseUrl)/%(ComponentToDownload.DownloadFileName)" Condition="'%(ComponentToDownload.ShouldDownload)' == 'true' and '$(DotNetRuntimeSourceFeedKey)' != ''">
+        <ShouldDownload>%(ComponentToDownload.ShouldDownload)</ShouldDownload>
+        <DownloadDestination>%(ComponentToDownload.DownloadDestination)</DownloadDestination>
+        <token>$(DotNetRuntimeSourceFeedKey)</token>
+      </UrisToDownload>
     </ItemGroup>
     
-    <PropertyGroup>
-      <DOTNETCLIMSRC_READ_SAS_TOKEN>$([System.Environment]::GetEnvironmentVariable('DOTNETCLIMSRC_READ_SAS_TOKEN'))</DOTNETCLIMSRC_READ_SAS_TOKEN>
-    </PropertyGroup>
-
-    <DownloadFile Condition=" '@(ComponentToDownload)' != '' And '%(ComponentToDownload.ShouldDownload)' == 'true'"
-                  Uri="%(ComponentToDownload.BaseUrl)/%(ComponentToDownload.DownloadFileName)"
-                  PrivateUri="%(ComponentToDownload.PrivateBaseUrl)/%(ComponentToDownload.DownloadFileName)$(DOTNETCLIMSRC_READ_SAS_TOKEN)"
-                  DestinationPath="%(ComponentToDownload.DownloadDestination)" />
+    <DownloadFile Condition=" '@(UrisToDownload)' != '' and %(ShouldDownload)"
+                  Uris="@(UrisToDownload)"
+                  DestinationPath="%(DownloadDestination)" />
 
     <ItemGroup>
       <BundledLayoutPackageDownloadProject Include="$(MSBuildThisFileDirectory)DownloadPackage.csproj">