diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 7ace18f75..603f1b7b7 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -15,7 +15,7 @@ variables: _PublishType: blob _SignType: real -phases: +jobs: - template: /eng/build.yml parameters: agentOs: Windows_NT @@ -237,6 +237,27 @@ phases: # Build_Release: # _BuildConfig: Release +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - job: Copy_SDK_To_Latest + dependsOn: + - Windows_NT + - Linux + - Darwin + pool: + name: Hosted VS2017 + condition: succeeded() + steps: + - task: AzureKeyVault@1 + inputs: + azureSubscription: 'DotNet-Engineering-Services_KeyVault' + KeyVaultName: EngKeyVault + SecretsFilter: 'dotnetfeed-storage-access-key-1' + condition: succeeded() + + - script: eng/CopyToLatest.cmd + /p:DotNetPublishBlobFeedUrl=$(PB_PublishBlobFeedUrl) + /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1) + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/templates/phases/publish-build-assets.yml parameters: @@ -244,5 +265,6 @@ phases: - Windows_NT - Linux - Darwin + - Copy_SDK_To_Latest queue: name: Hosted VS2017 \ No newline at end of file diff --git a/eng/CopyToLatest.cmd b/eng/CopyToLatest.cmd new file mode 100644 index 000000000..67b0a5860 --- /dev/null +++ b/eng/CopyToLatest.cmd @@ -0,0 +1,7 @@ +@echo off + +REM Copyright (c) .NET Foundation and contributors. All rights reserved. +REM Licensed under the MIT license. See LICENSE file in the project root for full license information. + +powershell -ExecutionPolicy Bypass -NoProfile -NoLogo -Command "& \"%~dp0common\build.ps1\" -restore -build /p:Projects=\"%~dp0..\src\CopyToLatest\CopyToLatest.csproj\" %*; exit $LastExitCode;" +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/eng/Publish.props b/eng/Publishing.props similarity index 95% rename from eng/Publish.props rename to eng/Publishing.props index 0177bb8e7..7f46d61ce 100644 --- a/eng/Publish.props +++ b/eng/Publishing.props @@ -9,7 +9,6 @@ $(Product) assets/$(Product) true - $(RepoRoot)/resources/images/version_badge.svg diff --git a/src/CopyToLatest/CopyToLatest.csproj b/src/CopyToLatest/CopyToLatest.csproj new file mode 100644 index 000000000..9e7c7d5af --- /dev/null +++ b/src/CopyToLatest/CopyToLatest.csproj @@ -0,0 +1,15 @@ + + + $(CoreSdkTargetFramework) + true + false + + + + + + + + + + diff --git a/src/CopyToLatest/targets/BranchInfo.props b/src/CopyToLatest/targets/BranchInfo.props new file mode 100644 index 000000000..a4b92cde1 --- /dev/null +++ b/src/CopyToLatest/targets/BranchInfo.props @@ -0,0 +1,5 @@ + + + dev/arcade + + diff --git a/src/CopyToLatest/targets/FinishBuild.targets b/src/CopyToLatest/targets/FinishBuild.targets new file mode 100644 index 000000000..c9d76ae21 --- /dev/null +++ b/src/CopyToLatest/targets/FinishBuild.targets @@ -0,0 +1,12 @@ + + + + + + diff --git a/src/core-sdk-tasks/AzurePublisher.cs b/src/core-sdk-tasks/AzurePublisher.cs new file mode 100644 index 000000000..c537331be --- /dev/null +++ b/src/core-sdk-tasks/AzurePublisher.cs @@ -0,0 +1,220 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#if !SOURCE_BUILD +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Auth; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace Microsoft.DotNet.Cli.Build +{ + public class AzurePublisher + { + public enum Product + { + SharedFramework, + Host, + HostFxr, + Sdk, + } + + private const string s_dotnetBlobContainerName = "dotnet"; + + private string _connectionString { get; set; } + private string _containerName { get; set; } + private CloudBlobContainer _blobContainer { get; set; } + + public AzurePublisher(string accountName, string accountKey, string containerName = s_dotnetBlobContainerName) + { + _containerName = containerName; + _blobContainer = GetDotnetBlobContainer(accountName, accountKey, containerName); + } + + private CloudBlobContainer GetDotnetBlobContainer(string accountName, string accountKey, string containerName) + { + var storageCredentials = new StorageCredentials(accountName, accountKey); + var storageAccount = new CloudStorageAccount(storageCredentials, true); + return GetDotnetBlobContainer(storageAccount, containerName); + } + + private CloudBlobContainer GetDotnetBlobContainer(CloudStorageAccount storageAccount, string containerName) + { + CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); + + return blobClient.GetContainerReference(containerName); + } + + public string UploadFile(string file, Product product, string version) + { + string url = CalculateRelativePathForFile(file, product, version); + CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(url); + blob.UploadFromFileAsync(file).Wait(); + SetBlobPropertiesBasedOnFileType(blob); + return url; + } + + public void PublishStringToBlob(string blob, string content) + { + CloudBlockBlob blockBlob = _blobContainer.GetBlockBlobReference(blob); + blockBlob.UploadTextAsync(content).Wait(); + + SetBlobPropertiesBasedOnFileType(blockBlob); + } + + public void CopyBlob(string sourceBlob, string targetBlob) + { + CloudBlockBlob source = _blobContainer.GetBlockBlobReference(sourceBlob); + CloudBlockBlob target = _blobContainer.GetBlockBlobReference(targetBlob); + + // Create the empty blob + using (MemoryStream ms = new MemoryStream()) + { + target.UploadFromStreamAsync(ms).Wait(); + } + + // Copy actual blob data + target.StartCopyAsync(source).Wait(); + } + + public void SetBlobPropertiesBasedOnFileType(string path) + { + CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(path); + SetBlobPropertiesBasedOnFileType(blob); + } + + private void SetBlobPropertiesBasedOnFileType(CloudBlockBlob blockBlob) + { + if (Path.GetExtension(blockBlob.Uri.AbsolutePath.ToLower()) == ".svg") + { + blockBlob.Properties.ContentType = "image/svg+xml"; + blockBlob.Properties.CacheControl = "no-cache"; + blockBlob.SetPropertiesAsync().Wait(); + } + else if (Path.GetExtension(blockBlob.Uri.AbsolutePath.ToLower()) == ".version") + { + blockBlob.Properties.ContentType = "text/plain"; + blockBlob.Properties.CacheControl = "no-cache"; + blockBlob.SetPropertiesAsync().Wait(); + } + } + + public IEnumerable ListBlobs(Product product, string version) + { + string virtualDirectory = $"{product}/{version}"; + return ListBlobs(virtualDirectory); + } + + public IEnumerable ListBlobs(string virtualDirectory) + { + CloudBlobDirectory blobDir = _blobContainer.GetDirectoryReference(virtualDirectory); + BlobContinuationToken continuationToken = new BlobContinuationToken(); + + var blobFiles = blobDir.ListBlobsSegmentedAsync(continuationToken).Result; + return blobFiles.Results.Select(bf => bf.Uri.PathAndQuery.Replace($"/{_containerName}/", string.Empty)); + } + + public string AcquireLeaseOnBlob( + string blob, + TimeSpan? maxWaitDefault = null, + TimeSpan? delayDefault = null) + { + TimeSpan maxWait = maxWaitDefault ?? TimeSpan.FromSeconds(120); + TimeSpan delay = delayDefault ?? TimeSpan.FromMilliseconds(500); + + Stopwatch stopWatch = new Stopwatch(); + stopWatch.Start(); + + // This will throw an exception with HTTP code 409 when we cannot acquire the lease + // But we should block until we can get this lease, with a timeout (maxWaitSeconds) + while (stopWatch.ElapsedMilliseconds < maxWait.TotalMilliseconds) + { + try + { + CloudBlockBlob cloudBlob = _blobContainer.GetBlockBlobReference(blob); + Task task = cloudBlob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null); + task.Wait(); + return task.Result; + } + catch (Exception e) + { + Console.WriteLine($"Retrying lease acquisition on {blob}, {e.Message}"); + Thread.Sleep(delay); + } + } + + throw new Exception($"Unable to acquire lease on {blob}"); + } + + public void ReleaseLeaseOnBlob(string blob, string leaseId) + { + CloudBlockBlob cloudBlob = _blobContainer.GetBlockBlobReference(blob); + AccessCondition ac = new AccessCondition() { LeaseId = leaseId }; + cloudBlob.ReleaseLeaseAsync(ac).Wait(); + } + + public bool IsLatestSpecifiedVersion(string version) + { + Task task = _blobContainer.GetBlockBlobReference(version).ExistsAsync(); + task.Wait(); + return task.Result; + } + + public void DropLatestSpecifiedVersion(string version) + { + CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(version); + using (MemoryStream ms = new MemoryStream()) + { + blob.UploadFromStreamAsync(ms).Wait(); + } + } + + public void CreateBlobIfNotExists(string path) + { + Task task = _blobContainer.GetBlockBlobReference(path).ExistsAsync(); + task.Wait(); + if (!task.Result) + { + CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(path); + using (MemoryStream ms = new MemoryStream()) + { + blob.UploadFromStreamAsync(ms).Wait(); + } + } + } + + public bool TryDeleteBlob(string path) + { + try + { + DeleteBlob(path); + + return true; + } + catch (Exception e) + { + Console.WriteLine($"Deleting blob {path} failed with \r\n{e.Message}"); + + return false; + } + } + + private void DeleteBlob(string path) + { + _blobContainer.GetBlockBlobReference(path).DeleteAsync().Wait(); + } + + private static string CalculateRelativePathForFile(string file, Product product, string version) + { + return $"{product}/{version}/{Path.GetFileName(file)}"; + } + } +} +#endif \ No newline at end of file diff --git a/src/core-sdk-tasks/CopyBlobsToLatest.cs b/src/core-sdk-tasks/CopyBlobsToLatest.cs new file mode 100644 index 000000000..734898a73 --- /dev/null +++ b/src/core-sdk-tasks/CopyBlobsToLatest.cs @@ -0,0 +1,132 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#if !SOURCE_BUILD +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.Cli.Build +{ + public class CopyBlobsToLatest : Task + { + private const string feedRegex = @"(?https:\/\/(?[^\.-]+)(?[^\/]*)\/((?[a-zA-Z0-9+\/]*?\/\d{4}-\d{2}-\d{2})\/)?(?[^\/]+)\/(?.*\/)?)index\.json"; + + private AzurePublisher _azurePublisher; + + [Required] + public string FeedUrl { get; set; } + + [Required] + public string AccountKey { get; set; } + + [Required] + public string Channel { get; set; } + + [Required] + public string CommitHash { get; set; } + + [Required] + public string NugetVersion { get; set; } + + private string ContainerName { get; set; } + + private AzurePublisher AzurePublisherTool + { + get + { + if (_azurePublisher == null) + { + Match m = Regex.Match(FeedUrl, feedRegex); + if (m.Success) + { + string accountName = m.Groups["accountname"].Value; + string ContainerName = m.Groups["containername"].Value; + + _azurePublisher = new AzurePublisher( + accountName, + AccountKey, + ContainerName); + } + else + { + throw new Exception( + "Unable to parse expected feed. Please check ExpectedFeedUrl."); + } + } + + return _azurePublisher; + } + } + + public override bool Execute() + { + string targetFolder = $"{AzurePublisher.Product.Sdk}/{Channel}"; + + string targetVersionFile = $"{targetFolder}/{CommitHash}"; + string semaphoreBlob = $"{targetFolder}/publishSemaphore"; + AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob); + string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob); + + // Prevent race conditions by dropping a version hint of what version this is. If we see this file + // and it is the same as our version then we know that a race happened where two+ builds finished + // at the same time and someone already took care of publishing and we have no work to do. + if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile)) + { + AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); + return true; + } + else + { + Regex versionFileRegex = new Regex(@"(?[\w\d]{40})"); + + // Delete old version files + AzurePublisherTool.ListBlobs(targetFolder) + .Where(s => versionFileRegex.IsMatch(s)) + .ToList() + .ForEach(f => AzurePublisherTool.TryDeleteBlob(f)); + + // Drop the version file signaling such for any race-condition builds (see above comment). + AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile); + } + + try + { + CopyBlobs(targetFolder); + + string cliVersion = GetVersionFileContent(CommitHash, NugetVersion); + AzurePublisherTool.PublishStringToBlob($"{targetFolder}/latest.version", cliVersion); + } + finally + { + AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); + } + + return true; + } + + private void CopyBlobs(string destinationFolder) + { + Log.LogMessage("Copying blobs to {0}/{1}", ContainerName, destinationFolder); + + foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion)) + { + string targetName = Path.GetFileName(blob) + .Replace(NugetVersion, "latest"); + + string target = $"{destinationFolder}/{targetName}"; + + AzurePublisherTool.CopyBlob(blob, target); + } + } + + private string GetVersionFileContent(string commitHash, string version) + { + return $@"{commitHash}{Environment.NewLine}{version}{Environment.NewLine}"; + } + } +} +#endif diff --git a/src/core-sdk-tasks/core-sdk-tasks.csproj b/src/core-sdk-tasks/core-sdk-tasks.csproj index c2ee30446..0230cd854 100644 --- a/src/core-sdk-tasks/core-sdk-tasks.csproj +++ b/src/core-sdk-tasks/core-sdk-tasks.csproj @@ -2,6 +2,7 @@ $(CoreSdkTargetFramework);net472 $(CoreSdkTargetFramework) + true @@ -11,6 +12,7 @@ + diff --git a/src/redist/redist.csproj b/src/redist/redist.csproj index cfe5d4d97..3491fbe1d 100644 --- a/src/redist/redist.csproj +++ b/src/redist/redist.csproj @@ -28,7 +28,7 @@ + - diff --git a/src/redist/targets/Badge.targets b/src/redist/targets/Badge.targets new file mode 100644 index 000000000..31321b8db --- /dev/null +++ b/src/redist/targets/Badge.targets @@ -0,0 +1,31 @@ + + + + $(RepoRoot)/resources/images/version_badge.svg + + + + + + + + + + + $(OSName)_$(Architecture) + rhel.6_x64 + linux_musl_x64 + linux_$(Architecture) + all_linux_distros_native_installer + + $(ArtifactsShippingPackagesDir)$(VersionBadgeMoniker)_$(Configuration)_version_badge.svg + $(ArtifactsShippingPackagesDir)$(VersionBadgeMoniker)_$(Configuration)_coherent_badge.svg + + + diff --git a/src/redist/targets/BuildCoreSdkTasks.targets b/src/redist/targets/BuildCoreSdkTasks.targets index dd25cf282..2002eedb6 100644 --- a/src/redist/targets/BuildCoreSdkTasks.targets +++ b/src/redist/targets/BuildCoreSdkTasks.targets @@ -34,5 +34,6 @@ +