From e58843bebcdaf541c845ecbfe28672a28894d167 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 26 May 2021 15:13:19 -0700 Subject: [PATCH] Remove Copy Blobs To Latest functionality and clean up some publishing (#10773) - The stage has been subsumed by the aka.ms functionality and now can be removed. - Clean up dead properties and functionality no longer needed because of it. - Removing publishing of nupkg files as blobs. This used to be useful before because we often needed to get at the nupkgs in a flat, easy to copy format for pushing to the VS feed. But now the staging pipeline takes care of this, and moreover the use of post-build signing means that these files are not useful straight out of the build anyway. This should reduce publishing time and some post-build signing time. Remove nupkg install in publishing.props --- .vsts-ci.yml | 30 ---- eng/Publishing.props | 12 +- eng/build.yml | 2 - src/CopyToLatest/CopyToLatest.csproj | 14 -- src/CopyToLatest/targets/FinishBuild.targets | 18 --- src/core-sdk-tasks/CopyBlobsToLatest.cs | 137 ------------------- src/redist/targets/BuildCoreSdkTasks.targets | 1 - 7 files changed, 1 insertion(+), 213 deletions(-) delete mode 100644 src/CopyToLatest/CopyToLatest.csproj delete mode 100644 src/CopyToLatest/targets/FinishBuild.targets delete mode 100644 src/core-sdk-tasks/CopyBlobsToLatest.cs diff --git a/.vsts-ci.yml b/.vsts-ci.yml index db8b04ad3..2292860fd 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -9,12 +9,6 @@ trigger: - internal/release/5.* variables: -# These two variables are used by CopyToLatest. -# when CopyToLatest goes away in favor of improvements to dotnet-install, this can be removed. -- name: _PublishBlobFeedUrl - value: https://dotnetcli.blob.core.windows.net/dotnet/index.json -- name: _PublishChecksumsBlobFeedUrl - value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json - name: _PublishUsingPipelines value: false - name: PostBuildSign @@ -390,27 +384,3 @@ stages: -TsaRepositoryName "dotnet-installer" -TsaCodebaseName "dotnet-installer" -TsaPublish $True' - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), not(contains(variables['Build.SourceBranch'], 'refs/heads/internal/'))) }}: - - stage: copy_to_latest - displayName: Copy to latest - dependsOn: - # This will run only after all the publishing stages have run. - # These stages are introduced in the eng/common/templates/post-build/channels YAML templates - - publish_using_darc - jobs: - - job: Copy_SDK_To_Latest - pool: - name: Hosted VS2017 - condition: succeeded() - variables: - - group: DotNet-DotNetCli-Storage - steps: - - script: eng/CopyToLatest.cmd - /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - /p:DotnetPublishSdkAssetsBlobFeedUrl=https://dotnetcli.blob.core.windows.net/dotnet/index.json - /p:DotNetPublishSdkAssetsBlobFeedKey=$(dotnetcli-storage-key) - /p:DotnetPublishChecksumsBlobFeedUrl=https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json - /p:DotNetPublishChecksumsBlobFeedKey=$(dotnetclichecksums-storage-key) - /p:SourceBranch=$(BUILD.SOURCEBRANCH) - displayName: Copy to latest diff --git a/eng/Publishing.props b/eng/Publishing.props index d4f6a188e..cab8f45f0 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -8,11 +8,6 @@ Sdk $(Product) - $(DotnetPublishChecksumsBlobFeedUrl) - $(DotnetPublishSdkAssetsBlobFeedUrl) - - false - true true @@ -70,7 +65,6 @@ - @@ -79,7 +73,6 @@ - @@ -94,8 +87,7 @@ + BeforeTargets="Publish"> - - diff --git a/eng/build.yml b/eng/build.yml index 63e5254a3..a46926894 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -68,8 +68,6 @@ phases: - _PushToVSFeed: true - _SignType: real - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - /p:DotnetPublishSdkAssetsBlobFeedUrl=$(_PublishBlobFeedUrl) - /p:DotnetPublishChecksumsBlobFeedUrl=$(_PublishChecksumsBlobFeedUrl) /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) diff --git a/src/CopyToLatest/CopyToLatest.csproj b/src/CopyToLatest/CopyToLatest.csproj deleted file mode 100644 index a61ea4a61..000000000 --- a/src/CopyToLatest/CopyToLatest.csproj +++ /dev/null @@ -1,14 +0,0 @@ - - - $(CoreSdkTargetFramework) - true - false - $(SourceBranch.Replace('refs/heads/', '')) - - - - - - - - \ No newline at end of file diff --git a/src/CopyToLatest/targets/FinishBuild.targets b/src/CopyToLatest/targets/FinishBuild.targets deleted file mode 100644 index 880487ef7..000000000 --- a/src/CopyToLatest/targets/FinishBuild.targets +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - diff --git a/src/core-sdk-tasks/CopyBlobsToLatest.cs b/src/core-sdk-tasks/CopyBlobsToLatest.cs deleted file mode 100644 index a62000ba7..000000000 --- a/src/core-sdk-tasks/CopyBlobsToLatest.cs +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -#if !SOURCE_BUILD -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using System; -using System.IO; -using System.Linq; -using System.Text.RegularExpressions; - -namespace Microsoft.DotNet.Cli.Build -{ - public class CopyBlobsToLatest : Task - { - private const string feedRegex = @"(?https:\/\/(?[^\.-]+)(?[^\/]*)\/((?[a-zA-Z0-9+\/]*?\/\d{4}-\d{2}-\d{2})\/)?(?[^\/]+)\/(?.*\/)?)index\.json"; - - private AzurePublisher _azurePublisher; - - [Required] - public string FeedUrl { get; set; } - - [Required] - public string AccountKey { get; set; } - - [Required] - public string Channel { get; set; } - - [Required] - public string CommitHash { get; set; } - - [Required] - public string NugetVersion { get; set; } - - private string ContainerName { get; set; } - - private AzurePublisher AzurePublisherTool - { - get - { - if (_azurePublisher == null) - { - Match m = Regex.Match(FeedUrl, feedRegex); - if (m.Success) - { - string accountName = m.Groups["accountname"].Value; - string ContainerName = m.Groups["containername"].Value; - - _azurePublisher = new AzurePublisher( - accountName, - AccountKey, - ContainerName); - } - else - { - throw new Exception( - "Unable to parse expected feed. Please check ExpectedFeedUrl."); - } - } - - return _azurePublisher; - } - } - - public override bool Execute() - { - if (!(Channel.Equals("master") || Channel.Equals("main") || Channel.StartsWith("release"))) - { - return true; // Skip copy to latest, we don't want to publish to arbitrary places - } - - string targetFolder = $"{AzurePublisher.Product.Sdk}/{Channel}"; - - string targetVersionFile = $"{targetFolder}/{CommitHash}"; - string semaphoreBlob = $"{targetFolder}/publishSemaphore"; - AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob); - string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob); - - // Prevent race conditions by dropping a version hint of what version this is. If we see this file - // and it is the same as our version then we know that a race happened where two+ builds finished - // at the same time and someone already took care of publishing and we have no work to do. - if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile)) - { - AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); - return true; - } - else - { - Regex versionFileRegex = new Regex(@"(?[\w\d]{40})"); - - // Delete old version files - AzurePublisherTool.ListBlobs(targetFolder) - .Where(s => versionFileRegex.IsMatch(s)) - .ToList() - .ForEach(f => AzurePublisherTool.TryDeleteBlob(f)); - - // Drop the version file signaling such for any race-condition builds (see above comment). - AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile); - } - - try - { - CopyBlobs(targetFolder); - - string cliVersion = GetVersionFileContent(CommitHash, NugetVersion); - AzurePublisherTool.PublishStringToBlob($"{targetFolder}/latest.version", cliVersion); - } - finally - { - AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); - } - - return true; - } - - private void CopyBlobs(string destinationFolder) - { - Log.LogMessage("Copying blobs to {0}/{1}", ContainerName, destinationFolder); - - foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion)) - { - string targetName = Path.GetFileName(blob) - .Replace(NugetVersion, "latest"); - - string target = $"{destinationFolder}/{targetName}"; - - AzurePublisherTool.CopyBlob(blob, target); - } - } - - private string GetVersionFileContent(string commitHash, string version) - { - return $@"{commitHash}{Environment.NewLine}{version}{Environment.NewLine}"; - } - } -} -#endif diff --git a/src/redist/targets/BuildCoreSdkTasks.targets b/src/redist/targets/BuildCoreSdkTasks.targets index a3ccc3880..d7692cebc 100644 --- a/src/redist/targets/BuildCoreSdkTasks.targets +++ b/src/redist/targets/BuildCoreSdkTasks.targets @@ -33,7 +33,6 @@ -