Copy latest to blob when all legs succeed.

.vsts-ci.yml

@@ -15,7 +15,7 @@ variables:
     _PublishType: blob
     _SignType: real
 
-phases:
+jobs:
 - template: /eng/build.yml
   parameters:
     agentOs: Windows_NT
@@ -237,6 +237,27 @@ phases:
 #         Build_Release:
 #           _BuildConfig: Release
 
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - job: Copy_SDK_To_Latest
+    dependsOn:
+      - Windows_NT
+      - Linux
+      - Darwin
+    pool:
+      name: Hosted VS2017
+    condition: succeeded()
+    steps:
+      - task: AzureKeyVault@1
+        inputs:
+          azureSubscription: 'DotNet-Engineering-Services_KeyVault'
+          KeyVaultName: EngKeyVault
+          SecretsFilter: 'dotnetfeed-storage-access-key-1'
+        condition: succeeded()
+
+      - script: eng/CopyToLatest.cmd
+                /p:DotNetPublishBlobFeedUrl=$(PB_PublishBlobFeedUrl)
+                /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
+
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - template: /eng/common/templates/phases/publish-build-assets.yml
     parameters:
@@ -244,5 +265,6 @@ phases:
         - Windows_NT
         - Linux
         - Darwin
+        - Copy_SDK_To_Latest
       queue:
         name: Hosted VS2017

eng/CopyToLatest.cmd

@@ -0,0 +1,7 @@
+@echo off
+
+REM Copyright (c) .NET Foundation and contributors. All rights reserved.
+REM Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+powershell -ExecutionPolicy Bypass -NoProfile -NoLogo -Command "& \"%~dp0common\build.ps1\" -restore -build /p:Projects=\"%~dp0..\src\CopyToLatest\CopyToLatest.csproj\" %*; exit $LastExitCode;"
+if %errorlevel% neq 0 exit /b %errorlevel%

eng/Publishing.props

@@ -9,7 +9,6 @@
     <BlobStoragePartialRelativePath>$(Product)</BlobStoragePartialRelativePath>
     <BlobStoragePartialRelativePath Condition="'$(IsNotOrchestratedPublish)' == 'false'">assets/$(Product)</BlobStoragePartialRelativePath>
     <PublishToBlobFeedFlatContainer>true</PublishToBlobFeedFlatContainer>
-    <VersionSvgTemplate>$(RepoRoot)/resources/images/version_badge.svg</VersionSvgTemplate>
   </PropertyGroup>
 
   <ItemGroup>

src/CopyToLatest/CopyToLatest.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="15.0">
+   <PropertyGroup>
+    <TargetFramework>$(CoreSdkTargetFramework)</TargetFramework>
+    <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
+    <CopyBuildOutputToPublishDirectory>false</CopyBuildOutputToPublishDirectory>
+  </PropertyGroup>
+
+  <Import Project="targets\BranchInfo.props" />
+  <Import Project="..\redist\targets\BuildCoreSdkTasks.targets" />
+  <Import Project="..\redist\targets\GetRuntimeInformation.targets" />
+  <Import Project="..\redist\targets\Versions.targets" />
+  <Import Project="..\redist\targets\SetBuildDefaults.targets" />
+  
+  <Import Project="targets\FinishBuild.targets" />
+</Project>

src/CopyToLatest/targets/BranchInfo.props

@@ -0,0 +1,5 @@
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Channel>dev/arcade</Channel>
+  </PropertyGroup>
+</Project>

src/CopyToLatest/targets/FinishBuild.targets

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Target Name="FinalizeBuild"
+          DependsOnTargets="SetSdkVersionInfo"
+          AfterTargets="Build">
+    <CopyBlobsToLatest FeedUrl="$(DotNetPublishBlobFeedUrl)"
+                       AccountKey="$(DotNetPublishBlobFeedKey)"
+                       NugetVersion="$(FullNugetVersion)"
+                       Channel="$(Channel)"
+                       CommitHash="$(GitCommitHash)" />
+  </Target>
+</Project>

src/core-sdk-tasks/AzurePublisher.cs

@@ -0,0 +1,220 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#if !SOURCE_BUILD
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Auth;
+using Microsoft.WindowsAzure.Storage.Blob;
+
+namespace Microsoft.DotNet.Cli.Build
+{
+    public class AzurePublisher
+    {
+        public enum Product
+        {
+            SharedFramework,
+            Host,
+            HostFxr,
+            Sdk,
+        }
+
+        private const string s_dotnetBlobContainerName = "dotnet";
+
+        private string _connectionString { get; set; }
+        private string _containerName { get; set; }
+        private CloudBlobContainer _blobContainer { get; set; }
+
+        public AzurePublisher(string accountName, string accountKey, string containerName = s_dotnetBlobContainerName)
+        {
+            _containerName = containerName;
+            _blobContainer = GetDotnetBlobContainer(accountName, accountKey, containerName);
+        }
+
+        private CloudBlobContainer GetDotnetBlobContainer(string accountName, string accountKey, string containerName)
+        {
+            var storageCredentials = new StorageCredentials(accountName, accountKey);
+            var storageAccount = new CloudStorageAccount(storageCredentials, true);
+            return GetDotnetBlobContainer(storageAccount, containerName);
+        }
+
+        private CloudBlobContainer GetDotnetBlobContainer(CloudStorageAccount storageAccount, string containerName)
+        {
+            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
+
+            return blobClient.GetContainerReference(containerName);
+        }
+
+        public string UploadFile(string file, Product product, string version)
+        {
+            string url = CalculateRelativePathForFile(file, product, version);
+            CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(url);
+            blob.UploadFromFileAsync(file).Wait();
+            SetBlobPropertiesBasedOnFileType(blob);
+            return url;
+        }
+
+        public void PublishStringToBlob(string blob, string content)
+        {
+            CloudBlockBlob blockBlob = _blobContainer.GetBlockBlobReference(blob);
+            blockBlob.UploadTextAsync(content).Wait();
+
+            SetBlobPropertiesBasedOnFileType(blockBlob);
+        }
+
+        public void CopyBlob(string sourceBlob, string targetBlob)
+        {
+            CloudBlockBlob source = _blobContainer.GetBlockBlobReference(sourceBlob);
+            CloudBlockBlob target = _blobContainer.GetBlockBlobReference(targetBlob);
+
+            // Create the empty blob
+            using (MemoryStream ms = new MemoryStream())
+            {
+                target.UploadFromStreamAsync(ms).Wait();
+            }
+
+            // Copy actual blob data
+            target.StartCopyAsync(source).Wait();
+        }
+
+        public void SetBlobPropertiesBasedOnFileType(string path)
+        {
+            CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(path);
+            SetBlobPropertiesBasedOnFileType(blob);
+        }
+
+        private void SetBlobPropertiesBasedOnFileType(CloudBlockBlob blockBlob)
+        {
+            if (Path.GetExtension(blockBlob.Uri.AbsolutePath.ToLower()) == ".svg")
+            {
+                blockBlob.Properties.ContentType = "image/svg+xml";
+                blockBlob.Properties.CacheControl = "no-cache";
+                blockBlob.SetPropertiesAsync().Wait();
+            }
+            else if (Path.GetExtension(blockBlob.Uri.AbsolutePath.ToLower()) == ".version")
+            {
+                blockBlob.Properties.ContentType = "text/plain";
+                blockBlob.Properties.CacheControl = "no-cache";
+                blockBlob.SetPropertiesAsync().Wait();
+            }
+        }
+
+        public IEnumerable<string> ListBlobs(Product product, string version)
+        {
+            string virtualDirectory = $"{product}/{version}";
+            return ListBlobs(virtualDirectory);
+        }
+
+        public IEnumerable<string> ListBlobs(string virtualDirectory)
+        {
+            CloudBlobDirectory blobDir = _blobContainer.GetDirectoryReference(virtualDirectory);
+            BlobContinuationToken continuationToken = new BlobContinuationToken();
+
+            var blobFiles = blobDir.ListBlobsSegmentedAsync(continuationToken).Result;
+            return blobFiles.Results.Select(bf => bf.Uri.PathAndQuery.Replace($"/{_containerName}/", string.Empty));
+        }
+
+        public string AcquireLeaseOnBlob(
+            string blob,
+            TimeSpan? maxWaitDefault = null,
+            TimeSpan? delayDefault = null)
+        {
+            TimeSpan maxWait = maxWaitDefault ?? TimeSpan.FromSeconds(120);
+            TimeSpan delay = delayDefault ?? TimeSpan.FromMilliseconds(500);
+
+            Stopwatch stopWatch = new Stopwatch();
+            stopWatch.Start();
+
+            // This will throw an exception with HTTP code 409 when we cannot acquire the lease
+            // But we should block until we can get this lease, with a timeout (maxWaitSeconds)
+            while (stopWatch.ElapsedMilliseconds < maxWait.TotalMilliseconds)
+            {
+                try
+                {
+                    CloudBlockBlob cloudBlob = _blobContainer.GetBlockBlobReference(blob);
+                    Task<string> task = cloudBlob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null);
+                    task.Wait();
+                    return task.Result;
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine($"Retrying lease acquisition on {blob}, {e.Message}");
+                    Thread.Sleep(delay);
+                }
+            }
+
+            throw new Exception($"Unable to acquire lease on {blob}");
+        }
+
+        public void ReleaseLeaseOnBlob(string blob, string leaseId)
+        {
+            CloudBlockBlob cloudBlob = _blobContainer.GetBlockBlobReference(blob);
+            AccessCondition ac = new AccessCondition() { LeaseId = leaseId };
+            cloudBlob.ReleaseLeaseAsync(ac).Wait();
+        }
+
+        public bool IsLatestSpecifiedVersion(string version)
+        {
+            Task<bool> task = _blobContainer.GetBlockBlobReference(version).ExistsAsync();
+            task.Wait();
+            return task.Result;
+        }
+
+        public void DropLatestSpecifiedVersion(string version)
+        {
+            CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(version);
+            using (MemoryStream ms = new MemoryStream())
+            {
+                blob.UploadFromStreamAsync(ms).Wait();
+            }
+        }
+
+        public void CreateBlobIfNotExists(string path)
+        {
+            Task<bool> task = _blobContainer.GetBlockBlobReference(path).ExistsAsync();
+            task.Wait();
+            if (!task.Result)
+            {
+                CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(path);
+                using (MemoryStream ms = new MemoryStream())
+                {
+                    blob.UploadFromStreamAsync(ms).Wait();
+                }
+            }
+        }
+
+        public bool TryDeleteBlob(string path)
+        {
+            try
+            {
+                DeleteBlob(path);
+
+                return true;
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine($"Deleting blob {path} failed with \r\n{e.Message}");
+
+                return false;
+            }
+        }
+
+        private void DeleteBlob(string path)
+        {
+            _blobContainer.GetBlockBlobReference(path).DeleteAsync().Wait();
+        }
+
+        private static string CalculateRelativePathForFile(string file, Product product, string version)
+        {
+            return $"{product}/{version}/{Path.GetFileName(file)}";
+        }
+    }
+}
+#endif

src/core-sdk-tasks/CopyBlobsToLatest.cs

@@ -0,0 +1,132 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#if !SOURCE_BUILD
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+
+namespace Microsoft.DotNet.Cli.Build
+{
+    public class CopyBlobsToLatest : Task
+    {
+        private const string feedRegex = @"(?<feedurl>https:\/\/(?<accountname>[^\.-]+)(?<domain>[^\/]*)\/((?<token>[a-zA-Z0-9+\/]*?\/\d{4}-\d{2}-\d{2})\/)?(?<containername>[^\/]+)\/(?<relativepath>.*\/)?)index\.json";
+        
+        private AzurePublisher _azurePublisher;
+
+        [Required]
+        public string FeedUrl { get; set; }
+
+        [Required]
+        public string AccountKey { get; set; }
+
+        [Required]
+        public string Channel { get; set; }
+
+        [Required]
+        public string CommitHash { get; set; }
+
+        [Required]
+        public string NugetVersion { get; set; }
+
+        private string ContainerName { get; set; }
+
+        private AzurePublisher AzurePublisherTool
+        {
+            get
+            {
+                if (_azurePublisher == null)
+                {
+                    Match m = Regex.Match(FeedUrl, feedRegex);
+                    if (m.Success)
+                    {
+                        string accountName = m.Groups["accountname"].Value;
+                        string ContainerName = m.Groups["containername"].Value;
+
+                        _azurePublisher = new AzurePublisher(
+                            accountName,
+                            AccountKey,
+                            ContainerName);
+                    }
+                    else
+                    {
+                        throw new Exception(
+                            "Unable to parse expected feed. Please check ExpectedFeedUrl.");
+                    }
+                }
+
+                return _azurePublisher;
+            }
+        }
+
+        public override bool Execute()
+        {
+            string targetFolder = $"{AzurePublisher.Product.Sdk}/{Channel}";
+
+            string targetVersionFile = $"{targetFolder}/{CommitHash}";
+            string semaphoreBlob = $"{targetFolder}/publishSemaphore";
+            AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob);
+            string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob);
+
+            // Prevent race conditions by dropping a version hint of what version this is. If we see this file
+            // and it is the same as our version then we know that a race happened where two+ builds finished 
+            // at the same time and someone already took care of publishing and we have no work to do.
+            if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile))
+            {
+                AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
+                return true;
+            }
+            else
+            {
+                Regex versionFileRegex = new Regex(@"(?<CommitHash>[\w\d]{40})");
+
+                // Delete old version files
+                AzurePublisherTool.ListBlobs(targetFolder)
+                    .Where(s => versionFileRegex.IsMatch(s))
+                    .ToList()
+                    .ForEach(f => AzurePublisherTool.TryDeleteBlob(f));
+
+                // Drop the version file signaling such for any race-condition builds (see above comment).
+                AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile);
+            }
+
+            try
+            {
+                CopyBlobs(targetFolder);
+
+                string cliVersion = GetVersionFileContent(CommitHash, NugetVersion);
+                AzurePublisherTool.PublishStringToBlob($"{targetFolder}/latest.version", cliVersion);
+            }
+            finally
+            {
+                AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
+            }
+
+            return true;
+        }
+
+        private void CopyBlobs(string destinationFolder)
+        {
+            Log.LogMessage("Copying blobs to {0}/{1}", ContainerName, destinationFolder);
+
+            foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion))
+            {
+                string targetName = Path.GetFileName(blob)
+                                        .Replace(NugetVersion, "latest");
+
+                string target = $"{destinationFolder}/{targetName}";
+
+                AzurePublisherTool.CopyBlob(blob, target);
+            }
+        }
+
+        private string GetVersionFileContent(string commitHash, string version)
+        {
+            return $@"{commitHash}{Environment.NewLine}{version}{Environment.NewLine}";
+        }
+    }
+}
+#endif

src/core-sdk-tasks/core-sdk-tasks.csproj

@@ -2,6 +2,7 @@
   <PropertyGroup>
     <TargetFrameworks>$(CoreSdkTargetFramework);net472</TargetFrameworks>
     <TargetFrameworks Condition="'$(OS)' != 'Windows_NT'">$(CoreSdkTargetFramework)</TargetFrameworks>
+    <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
   </PropertyGroup>
 
   <ItemGroup>
@@ -11,6 +12,7 @@
     <PackageReference Include="Newtonsoft.Json" Version="9.0.1" />
     <PackageReference Include="NuGet.Versioning" Version="4.3.0" />
     <PackageReference Include="System.Reflection.Metadata" Version="1.4.2" />
+    <PackageReference Include="WindowsAzure.Storage" Version="8.4.0"/>
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">

src/redist/redist.csproj

@@ -28,7 +28,7 @@
   <Import Project="targets\GenerateRPMs.targets" />
   <Import Project="targets\GeneratePKG.targets" />
   <Import Project="targets\GenerateInstallers.targets" />
+  <Import Project="targets\Badge.targets" />
 
   <Import Project="targets\Signing.targets" />
-
 </Project>

src/redist/targets/Badge.targets

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VersionSvgTemplate>$(RepoRoot)/resources/images/version_badge.svg</VersionSvgTemplate>
+  </PropertyGroup>
+
+  <Target Name="GenerateVersionBadge"
+          DependsOnTargets="SetBadgeProps"
+          AfterTargets="Build">
+    <Message Text="$(VersionBadge)"  />
+
+    <ReplaceFileContents
+      InputFiles="$(VersionSvgTemplate)"
+      DestinationFiles="$(VersionBadge)"
+      ReplacementPatterns="ver_number"
+      ReplacementStrings="$(SdkVersion)" />
+  </Target>
+
+  <Target Name="SetBadgeProps" DependsOnTargets="GetCurrentRuntimeInformation">
+    <PropertyGroup>
+      <VersionBadgeMoniker>$(OSName)_$(Architecture)</VersionBadgeMoniker>
+      <VersionBadgeMoniker Condition=" '$(Rid)' == 'rhel.6-x64' ">rhel.6_x64</VersionBadgeMoniker>
+      <VersionBadgeMoniker Condition=" '$(Rid)' == 'linux-musl-x64' ">linux_musl_x64</VersionBadgeMoniker>
+      <VersionBadgeMoniker Condition=" '$(IslinuxPortable)' == 'true' ">linux_$(Architecture)</VersionBadgeMoniker>
+      <VersionBadgeMoniker Condition=" '$(IsBuildingAndPublishingAllLinuxDistrosNativeInstallers)' == 'true' ">all_linux_distros_native_installer</VersionBadgeMoniker>
+
+      <VersionBadge>$(ArtifactsShippingPackagesDir)$(VersionBadgeMoniker)_$(Configuration)_version_badge.svg</VersionBadge>
+      <CoherentBadge>$(ArtifactsShippingPackagesDir)$(VersionBadgeMoniker)_$(Configuration)_coherent_badge.svg</CoherentBadge>
+    </PropertyGroup>
+  </Target>
+</Project>

src/redist/targets/BuildCoreSdkTasks.targets

@@ -34,5 +34,6 @@
   <UsingTask TaskName="GetUseBundledNETCoreAppPackageVersionAsDefaultNetCorePatchVersion" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="AddMetadataIsPE" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="Crossgen" AssemblyFile="$(CoreSdkTaskDll)"/>
+  <UsingTask TaskName="CopyBlobsToLatest" AssemblyFile="$(CoreSdkTaskDll)"/>
 
 </Project>