diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 7b038f224..1e5d376c1 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -31,6 +31,15 @@ variables: - name: _NonWindowsTestArg value: '' +- name: _InternalRuntimeDownloadArgs + value: '' + +- ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNet-MSRC-Storage + - name: _InternalRuntimeDownloadArgs + value: /p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet + /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64) + stages: - stage: build jobs: diff --git a/eng/build.yml b/eng/build.yml index a46926894..c23bbb11b 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -54,11 +54,6 @@ phases: - _TeamName: Roslyn-Project-System - _SignType: test - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)' - - _DOTNETCLIMSRC_READ_SAS_TOKEN: '' - - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNet-MSRC-Storage - - _DOTNETCLIMSRC_READ_SAS_TOKEN: $(dotnetclimsrc-read-sas-token) - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: DotNet-Symbol-Server-PATs @@ -92,10 +87,10 @@ phases: -Architecture $(_BuildArchitecture) $(_BuildArgs) $(_AdditionalBuildParameters) + $(_InternalRuntimeDownloadArgs) displayName: Build env: DOTNET_CLI_UI_LANGUAGE: $(_DOTNET_CLI_UI_LANGUAGE) - DOTNETCLIMSRC_READ_SAS_TOKEN: $(_DOTNETCLIMSRC_READ_SAS_TOKEN) - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - ${{ if ne(variables['System.TeamProject'], 'public') }}: @@ -118,9 +113,8 @@ phases: $(_RuntimeIdentifier) $(_BuildArgs) $(_AdditionalBuildParameters) + $(_InternalRuntimeDownloadArgs) displayName: Build - env: - DOTNETCLIMSRC_READ_SAS_TOKEN: $(_DOTNETCLIMSRC_READ_SAS_TOKEN) - ${{ if or(eq(parameters.agentOs, 'Darwin'), eq(parameters.agentOs, 'FreeBSD')) }}: - script: ./build.sh @@ -132,9 +126,8 @@ phases: $(_RuntimeIdentifier) $(_BuildArgs) $(_AdditionalBuildParameters) + $(_InternalRuntimeDownloadArgs) displayName: Build - env: - DOTNETCLIMSRC_READ_SAS_TOKEN: $(_DOTNETCLIMSRC_READ_SAS_TOKEN) - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.agentOs, 'Windows_NT'), ne(variables['PostBuildSign'], 'true')) }}: - task: NuGetCommand@2 diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 705b7a1c8..a575fb8cc 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -18,6 +18,36 @@ steps: set -x df -h + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + $internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + git config core.autocrlf false + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64) --runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)' + fi + buildConfig=Release # Check if AzDO substitutes in a build config from a variable, and use it if so. if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then @@ -29,11 +59,6 @@ steps: officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' fi - internalRuntimeDownloadArgs= - if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then - internalRuntimeDownloadArgs='--runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)' - fi - targetRidArgs= if [ '${{ parameters.platform.targetRID }}' != '' ]; then targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' @@ -49,6 +74,7 @@ steps: --restore --build --pack $publishArgs -bl \ $officialBuildArgs \ $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ $targetRidArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true diff --git a/eng/dockerrun.sh b/eng/dockerrun.sh index 69e7d5471..775ca367d 100755 --- a/eng/dockerrun.sh +++ b/eng/dockerrun.sh @@ -169,7 +169,6 @@ docker run $INTERACTIVE -t --rm --sig-proxy=true \ -e POSTBUILDSIGN \ -e SYSTEM_DEFINITIONID \ -e SYSTEM_TEAMFOUNDATIONCOLLECTIONURI \ - -e DOTNETCLIMSRC_READ_SAS_TOKEN \ -e AGENT_JOBNAME \ -e AGENT_OS \ -e VSS_NUGET_URI_PREFIXES \ diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml index 74e4cfaa4..57bdbc4c2 100644 --- a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml +++ b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml @@ -21,12 +21,21 @@ jobs: officialBuildArgs='/p:OfficialBuildId=$(BUILD.BUILDNUMBER)' fi + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64) --runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)' + fi + ./build.sh \ --ci \ --configuration $(_BuildConfig) \ --publish \ -bl \ $officialBuildArgs \ + $internalRuntimeDownloadArgs \ /p:DotNetPublishUsingPipelines=true \ /p:ArcadeBuildTarball=true displayName: Create Tarball diff --git a/src/core-sdk-tasks/DownloadFile.cs b/src/core-sdk-tasks/DownloadFile.cs deleted file mode 100644 index 382d5cffb..000000000 --- a/src/core-sdk-tasks/DownloadFile.cs +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.IO; -using System.Net; -using System.Net.Http; -using System.Threading.Tasks; -using System.Collections.Generic; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Microsoft.DotNet.Cli.Build -{ - public class DownloadFile : Microsoft.Build.Utilities.Task - { - [Required] - public string Uri { get; set; } - - /// - /// If this field is set and the task fail to download the file from `Uri`, with a NotFound - /// status, it will try to download the file from `PrivateUri`. - /// - public string PrivateUri { get; set; } - - public int MaxRetries { get; set; } = 5; - - [Required] - public string DestinationPath { get; set; } - - public bool Overwrite { get; set; } - - public override bool Execute() - { - return ExecuteAsync().GetAwaiter().GetResult(); - } - - private async System.Threading.Tasks.Task ExecuteAsync() - { - string destinationDir = Path.GetDirectoryName(DestinationPath); - if (!Directory.Exists(destinationDir)) - { - Directory.CreateDirectory(destinationDir); - } - - if (File.Exists(DestinationPath) && !Overwrite) - { - return true; - } - - const string FileUriProtocol = "file://"; - - if (Uri.StartsWith(FileUriProtocol, StringComparison.Ordinal)) - { - var filePath = Uri.Substring(FileUriProtocol.Length); - Log.LogMessage($"Copying '{filePath}' to '{DestinationPath}'"); - File.Copy(filePath, DestinationPath); - return true; - } - - List errorMessages = new List(); - bool? downloadStatus = await DownloadWithRetriesAsync(Uri, DestinationPath, errorMessages); - - if (downloadStatus == false && !string.IsNullOrEmpty(PrivateUri)) - { - downloadStatus = await DownloadWithRetriesAsync(PrivateUri, DestinationPath, errorMessages); - } - - if (downloadStatus != true) - { - foreach (var error in errorMessages) - { - Log.LogError(error); - } - } - - return downloadStatus == true; - } - - /// - /// Attempt to download file from `source` with retries when response error is different of FileNotFound and Success. - /// - /// URL to the file to be downloaded. - /// Local path where to put the downloaded file. - /// true: Download Succeeded. false: Download failed with 404. null: Download failed but is retriable. - private async Task DownloadWithRetriesAsync(string source, string target, List errorMessages) - { - Random rng = new Random(); - - Log.LogMessage(MessageImportance.High, $"Attempting download '{source}' to '{target}'"); - - using (var httpClient = new HttpClient()) - { - for (int retryNumber = 0; retryNumber < MaxRetries; retryNumber++) - { - try - { - var httpResponse = await httpClient.GetAsync(source); - - Log.LogMessage(MessageImportance.Normal, $"{source} -> {httpResponse.StatusCode}"); - - // The Azure Storage REST API returns '400 - Bad Request' in some cases - // where the resource is not found on the storage. - // https://docs.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes - if (httpResponse.StatusCode == HttpStatusCode.NotFound || - httpResponse.ReasonPhrase.IndexOf("The requested URI does not represent any resource on the server.", StringComparison.OrdinalIgnoreCase) == 0) - { - errorMessages.Add($"Problems downloading file from '{source}'. Does the resource exist on the storage? {httpResponse.StatusCode} : {httpResponse.ReasonPhrase}"); - return false; - } - - httpResponse.EnsureSuccessStatusCode(); - - using (var outStream = File.Create(target)) - { - await httpResponse.Content.CopyToAsync(outStream); - } - - Log.LogMessage(MessageImportance.Normal, $"returning true {source} -> {httpResponse.StatusCode}"); - return true; - } - catch (Exception e) - { - Log.LogMessage(MessageImportance.High, $"returning error in {source} "); - errorMessages.Add($"Problems downloading file from '{source}'. {e.Message} {e.StackTrace}"); - File.Delete(target); - } - - await System.Threading.Tasks.Task.Delay(rng.Next(1000, 10000)); - } - } - - Log.LogMessage(MessageImportance.High, $"giving up {source} "); - errorMessages.Add($"Giving up downloading the file from '{source}' after {MaxRetries} retries."); - return null; - } - } -} diff --git a/src/redist/targets/BuildCoreSdkTasks.targets b/src/redist/targets/BuildCoreSdkTasks.targets index c87b22094..f89f2297f 100644 --- a/src/redist/targets/BuildCoreSdkTasks.targets +++ b/src/redist/targets/BuildCoreSdkTasks.targets @@ -20,7 +20,7 @@ - + diff --git a/src/redist/targets/GenerateLayout.targets b/src/redist/targets/GenerateLayout.targets index 4e0a507d6..e76b3ba1f 100644 --- a/src/redist/targets/GenerateLayout.targets +++ b/src/redist/targets/GenerateLayout.targets @@ -331,16 +331,26 @@ $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetExtensionsBlobRootUrl)', '$(InternalBaseURL)')) $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetToolsetBlobRootUrl)', '$(InternalBaseURL)')) + + + + + %(ComponentToDownload.ShouldDownload) + %(ComponentToDownload.DownloadDestination) + + + %(ComponentToDownload.ShouldDownload) + %(ComponentToDownload.DownloadDestination) + $(DotNetRuntimeSourceFeedKey) + - - - $([System.Environment]::GetEnvironmentVariable('DOTNETCLIMSRC_READ_SAS_TOKEN')) - - - + +