Remove Copy Blobs To Latest functionality and clean up some publishing (#10773)

- The stage has been subsumed by the aka.ms functionality and now can be removed.
- Clean up dead properties and functionality no longer needed because of it.
- Removing publishing of nupkg files as blobs. This used to be useful before because we often needed to get at the nupkgs in a flat, easy to copy format for pushing to the VS feed. But now the staging pipeline takes care of this, and moreover the use of post-build signing means that these files are not useful straight out of the build anyway. This should reduce publishing time and some post-build signing time.

Remove nupkg install in publishing.props

.vsts-ci.yml

@@ -9,12 +9,6 @@ trigger:
     - internal/release/5.*
 
 variables:
-# These two variables are used by CopyToLatest.
-# when CopyToLatest goes away in favor of improvements to dotnet-install, this can be removed.
-- name: _PublishBlobFeedUrl
-  value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
-- name: _PublishChecksumsBlobFeedUrl
-  value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
 - name: _PublishUsingPipelines
   value: false
 - name: PostBuildSign
@@ -390,27 +384,3 @@ stages:
         -TsaRepositoryName "dotnet-installer"
         -TsaCodebaseName "dotnet-installer"
         -TsaPublish $True'
-
-- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), not(contains(variables['Build.SourceBranch'], 'refs/heads/internal/'))) }}:
-  - stage: copy_to_latest
-    displayName: Copy to latest
-    dependsOn:
-      # This will run only after all the publishing stages have run.
-      # These stages are introduced in the eng/common/templates/post-build/channels YAML templates
-      - publish_using_darc
-    jobs:
-    - job: Copy_SDK_To_Latest
-      pool:
-        name: Hosted VS2017
-      condition: succeeded()
-      variables:
-        - group: DotNet-DotNetCli-Storage
-      steps:
-        - script: eng/CopyToLatest.cmd
-                  /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
-                  /p:DotnetPublishSdkAssetsBlobFeedUrl=https://dotnetcli.blob.core.windows.net/dotnet/index.json
-                  /p:DotNetPublishSdkAssetsBlobFeedKey=$(dotnetcli-storage-key)
-                  /p:DotnetPublishChecksumsBlobFeedUrl=https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
-                  /p:DotNetPublishChecksumsBlobFeedKey=$(dotnetclichecksums-storage-key)
-                  /p:SourceBranch=$(BUILD.SOURCEBRANCH)
-          displayName: Copy to latest

eng/Publishing.props

@@ -8,11 +8,6 @@
   <PropertyGroup>
     <Product>Sdk</Product>
     <BlobStoragePartialRelativePath>$(Product)</BlobStoragePartialRelativePath>
-    <ChecksumsFeedUrl>$(DotnetPublishChecksumsBlobFeedUrl)</ChecksumsFeedUrl>
-    <SdkAssetsFeedUrl>$(DotnetPublishSdkAssetsBlobFeedUrl)</SdkAssetsFeedUrl>
-
-    <PublishSdkAssetsAndChecksumsToBlob>false</PublishSdkAssetsAndChecksumsToBlob>
-    <PublishSdkAssetsAndChecksumsToBlob Condition=" '$(ChecksumsFeedUrl)' != '' and '$(SdkAssetsFeedUrl)' != '' ">true</PublishSdkAssetsAndChecksumsToBlob>
 
     <PublishBinariesAndBadge Condition=" '$(PublishBinariesAndBadge)' == '' ">true</PublishBinariesAndBadge>
   </PropertyGroup>
@@ -70,7 +65,6 @@
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.deb" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.rpm" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.msi" />
-    <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.nupkg" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.cab" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.svg" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
     <!-- Only publish this file from windows x64 so that we don't end up with duplicates -->
@@ -79,7 +73,6 @@
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)sdk-productVersion.txt"
         Condition=" '$(PublishBinariesAndBadge)' == 'true' and '$(OS)' == 'Windows_NT' and '$(Architecture)' == 'x64' and '$(PgoInstrument)' != 'true'" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)productCommit-*.txt" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
-    <SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.nupkg" />
     <SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.swr" />
     <SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.msi" />
     <SdkNonShippingAssetsToPublish Condition="'$(PublishBinariesAndBadge)' != 'false'" Include="$(ArtifactsNonShippingPackagesDir)*.tar.gz" />
@@ -94,8 +87,7 @@
   </ItemGroup>
 
   <Target Name="PublishSdkAssetsAndChecksums"
-          BeforeTargets="Publish"
+          BeforeTargets="Publish">
-          Condition=" '$(PublishSdkAssetsAndChecksumsToBlob)' == 'true' ">
 
     <ReadLinesFromFile File="$(ArtifactsTmpDir)FullNugetVersion.version">
       <Output
@@ -123,10 +115,8 @@
       <ItemsToSignPostBuildWithPaths Include="$(ArtifactsShippingPackagesDir)*.zip" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
       <ItemsToSignPostBuildWithPaths Include="$(ArtifactsShippingPackagesDir)*.exe" />
       <ItemsToSignPostBuildWithPaths Include="$(ArtifactsShippingPackagesDir)*.msi" />
-      <ItemsToSignPostBuildWithPaths Include="$(ArtifactsShippingPackagesDir)*.nupkg" />
       <ItemsToSignPostBuildWithPaths Include="$(ArtifactsNonShippingPackagesDir)*.msi" />
       <ItemsToSignPostBuildWithPaths Include="$(ArtifactsNonShippingPackagesDir)*.zip" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
-      <ItemsToSignPostBuildWithPaths Include="$(ArtifactsNonShippingPackagesDir)*.nupkg" />
       <ItemsToSignPostBuild Remove="@(ItemsToSignPostBuild)" />
       <ItemsToSignPostBuild Include="@(ItemsToSignPostBuildWithPaths->'%(Filename)%(Extension)')" />
     </ItemGroup>

eng/build.yml

@@ -68,8 +68,6 @@ phases:
         - _PushToVSFeed: true
         - _SignType: real
         - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
-                  /p:DotnetPublishSdkAssetsBlobFeedUrl=$(_PublishBlobFeedUrl)
-                  /p:DotnetPublishChecksumsBlobFeedUrl=$(_PublishChecksumsBlobFeedUrl)
                   /p:DotNetSignType=$(_SignType)
                   /p:TeamName=$(_TeamName)
                   /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)

src/CopyToLatest/CopyToLatest.csproj

@@ -1,14 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="15.0">
-   <PropertyGroup>
-    <TargetFramework>$(CoreSdkTargetFramework)</TargetFramework>
-    <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
-    <CopyBuildOutputToPublishDirectory>false</CopyBuildOutputToPublishDirectory>
-    <Channel>$(SourceBranch.Replace('refs/heads/', ''))</Channel>
-  </PropertyGroup>
-
-  <Import Project="..\redist\targets\BuildCoreSdkTasks.targets" />
-  <Import Project="..\redist\targets\GetRuntimeInformation.targets" />
-  <Import Project="..\redist\targets\Versions.targets" />
-  
-  <Import Project="targets\FinishBuild.targets" />
-</Project>

src/CopyToLatest/targets/FinishBuild.targets

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <Target Name="FinalizeBuild"
-          BeforeTargets="AfterBuild"
-          DependsOnTargets="GenerateFullNuGetVersion;GetCommitHash">
-    <CopyBlobsToLatest FeedUrl="$(DotnetPublishSdkAssetsBlobFeedUrl)"
-                       AccountKey="$(DotNetPublishSdkAssetsBlobFeedKey)"
-                       NugetVersion="$(FullNugetVersion)"
-                       Channel="$(Channel)"
-                       CommitHash="$(GitCommitHash)" />
-
-    <CopyBlobsToLatest FeedUrl="$(DotNetPublishChecksumsBlobFeedUrl)"
-                       AccountKey="$(DotNetPublishChecksumsBlobFeedKey)"
-                       NugetVersion="$(FullNugetVersion)"
-                       Channel="$(Channel)"
-                       CommitHash="$(GitCommitHash)" />
-  </Target>
-</Project>

src/core-sdk-tasks/CopyBlobsToLatest.cs

@@ -1,137 +0,0 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-#if !SOURCE_BUILD
-using Microsoft.Build.Framework;
-using Microsoft.Build.Utilities;
-using System;
-using System.IO;
-using System.Linq;
-using System.Text.RegularExpressions;
-
-namespace Microsoft.DotNet.Cli.Build
-{
-    public class CopyBlobsToLatest : Task
-    {
-        private const string feedRegex = @"(?<feedurl>https:\/\/(?<accountname>[^\.-]+)(?<domain>[^\/]*)\/((?<token>[a-zA-Z0-9+\/]*?\/\d{4}-\d{2}-\d{2})\/)?(?<containername>[^\/]+)\/(?<relativepath>.*\/)?)index\.json";
-        
-        private AzurePublisher _azurePublisher;
-
-        [Required]
-        public string FeedUrl { get; set; }
-
-        [Required]
-        public string AccountKey { get; set; }
-
-        [Required]
-        public string Channel { get; set; }
-
-        [Required]
-        public string CommitHash { get; set; }
-
-        [Required]
-        public string NugetVersion { get; set; }
-
-        private string ContainerName { get; set; }
-
-        private AzurePublisher AzurePublisherTool
-        {
-            get
-            {
-                if (_azurePublisher == null)
-                {
-                    Match m = Regex.Match(FeedUrl, feedRegex);
-                    if (m.Success)
-                    {
-                        string accountName = m.Groups["accountname"].Value;
-                        string ContainerName = m.Groups["containername"].Value;
-
-                        _azurePublisher = new AzurePublisher(
-                            accountName,
-                            AccountKey,
-                            ContainerName);
-                    }
-                    else
-                    {
-                        throw new Exception(
-                            "Unable to parse expected feed. Please check ExpectedFeedUrl.");
-                    }
-                }
-
-                return _azurePublisher;
-            }
-        }
-
-        public override bool Execute()
-        {
-            if (!(Channel.Equals("master") || Channel.Equals("main") || Channel.StartsWith("release")))
-            {
-                return true; // Skip copy to latest, we don't want to publish to arbitrary places
-            }
-
-            string targetFolder = $"{AzurePublisher.Product.Sdk}/{Channel}";
-
-            string targetVersionFile = $"{targetFolder}/{CommitHash}";
-            string semaphoreBlob = $"{targetFolder}/publishSemaphore";
-            AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob);
-            string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob);
-
-            // Prevent race conditions by dropping a version hint of what version this is. If we see this file
-            // and it is the same as our version then we know that a race happened where two+ builds finished 
-            // at the same time and someone already took care of publishing and we have no work to do.
-            if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile))
-            {
-                AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
-                return true;
-            }
-            else
-            {
-                Regex versionFileRegex = new Regex(@"(?<CommitHash>[\w\d]{40})");
-
-                // Delete old version files
-                AzurePublisherTool.ListBlobs(targetFolder)
-                    .Where(s => versionFileRegex.IsMatch(s))
-                    .ToList()
-                    .ForEach(f => AzurePublisherTool.TryDeleteBlob(f));
-
-                // Drop the version file signaling such for any race-condition builds (see above comment).
-                AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile);
-            }
-
-            try
-            {
-                CopyBlobs(targetFolder);
-
-                string cliVersion = GetVersionFileContent(CommitHash, NugetVersion);
-                AzurePublisherTool.PublishStringToBlob($"{targetFolder}/latest.version", cliVersion);
-            }
-            finally
-            {
-                AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
-            }
-
-            return true;
-        }
-
-        private void CopyBlobs(string destinationFolder)
-        {
-            Log.LogMessage("Copying blobs to {0}/{1}", ContainerName, destinationFolder);
-
-            foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion))
-            {
-                string targetName = Path.GetFileName(blob)
-                                        .Replace(NugetVersion, "latest");
-
-                string target = $"{destinationFolder}/{targetName}";
-
-                AzurePublisherTool.CopyBlob(blob, target);
-            }
-        }
-
-        private string GetVersionFileContent(string commitHash, string version)
-        {
-            return $@"{commitHash}{Environment.NewLine}{version}{Environment.NewLine}";
-        }
-    }
-}
-#endif

src/redist/targets/BuildCoreSdkTasks.targets

@@ -33,7 +33,6 @@
   <UsingTask TaskName="GenerateDefaultRuntimeFrameworkVersion" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="AddMetadataIsPE" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="Crossgen" AssemblyFile="$(CoreSdkTaskDll)"/>
-  <UsingTask TaskName="CopyBlobsToLatest" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="GetRuntimePackRids" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="GenerateMSBuildExtensionsSWR" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="GetLinuxNativeInstallerDependencyVersions" AssemblyFile="$(CoreSdkTaskDll)"/>