From 3ae14ab618e157e91d92cb68eb86e4274bdadbd5 Mon Sep 17 00:00:00 2001 From: Nate Amundson Date: Wed, 12 Oct 2016 14:25:06 -0500 Subject: [PATCH] Upload checksums for installation artifacts (#4191) The checksums are SHA-512 hashes, which users can use to verify file integrity and authenticity. --- build/Microsoft.DotNet.Cli.Publish.targets | 38 ++-- build/Microsoft.DotNet.Cli.tasks | 7 +- build/publish/FinishBuild.targets | 34 ++++ .../Microsoft.DotNet.Cli.Checksum.targets | 21 +++ build/publish/PublishContent.targets | 43 +++-- .../CheckIfAllBuildsHavePublished.cs | 86 +++++++++ .../dotnet-cli-build/CopyBlobsToLatest.cs | 108 ++++++++++++ .../dotnet-cli-build/FinalizeBuildTask.cs | 163 ------------------ .../dotnet-cli-build/GenerateChecksums.cs | 66 +++++++ .../dotnet-cli-build/UpdateVersionsRepo.cs | 27 +++ scripts/dockerrun.sh | 9 +- 11 files changed, 402 insertions(+), 200 deletions(-) create mode 100644 build/publish/FinishBuild.targets create mode 100644 build/publish/Microsoft.DotNet.Cli.Checksum.targets create mode 100644 build_projects/dotnet-cli-build/CheckIfAllBuildsHavePublished.cs create mode 100644 build_projects/dotnet-cli-build/CopyBlobsToLatest.cs delete mode 100644 build_projects/dotnet-cli-build/FinalizeBuildTask.cs create mode 100644 build_projects/dotnet-cli-build/GenerateChecksums.cs create mode 100644 build_projects/dotnet-cli-build/UpdateVersionsRepo.cs diff --git a/build/Microsoft.DotNet.Cli.Publish.targets b/build/Microsoft.DotNet.Cli.Publish.targets index 2a3feae93..a05512cf4 100644 --- a/build/Microsoft.DotNet.Cli.Publish.targets +++ b/build/Microsoft.DotNet.Cli.Publish.targets @@ -1,7 +1,9 @@ + + - - - - Sdk - $(STORAGE_CONTAINER) - $(STORAGE_KEY) - $(STORAGE_ACCOUNT) - https://$(CloudDropAccountName).blob.core.windows.net/$(ContainerName) + $(ARTIFACT_STORAGE_CONTAINER) + $(ARTIFACT_STORAGE_KEY) + $(ARTIFACT_STORAGE_ACCOUNT) + https://$(ArtifactCloudDropAccountName).blob.core.windows.net/$(ArtifactContainerName) + $(CHECKSUM_STORAGE_CONTAINER) + $(CHECKSUM_STORAGE_KEY) + $(CHECKSUM_STORAGE_ACCOUNT) @@ -94,16 +92,16 @@ \ No newline at end of file diff --git a/build/Microsoft.DotNet.Cli.tasks b/build/Microsoft.DotNet.Cli.tasks index c4e97c612..7de1a9a70 100644 --- a/build/Microsoft.DotNet.Cli.tasks +++ b/build/Microsoft.DotNet.Cli.tasks @@ -2,7 +2,10 @@ + + + @@ -11,8 +14,8 @@ - + @@ -23,7 +26,7 @@ - + diff --git a/build/publish/FinishBuild.targets b/build/publish/FinishBuild.targets new file mode 100644 index 000000000..307e23d30 --- /dev/null +++ b/build/publish/FinishBuild.targets @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + diff --git a/build/publish/Microsoft.DotNet.Cli.Checksum.targets b/build/publish/Microsoft.DotNet.Cli.Checksum.targets new file mode 100644 index 000000000..57e27b2a0 --- /dev/null +++ b/build/publish/Microsoft.DotNet.Cli.Checksum.targets @@ -0,0 +1,21 @@ + + + + + + + + + + %(ForPublishing.FullPath).sha + %(ForPublishing.RelativeBlobPath).sha + + + + diff --git a/build/publish/PublishContent.targets b/build/publish/PublishContent.targets index 63427baba..d186a9db4 100644 --- a/build/publish/PublishContent.targets +++ b/build/publish/PublishContent.targets @@ -13,29 +13,48 @@ - $([System.String]::Copy('%(RecursiveDir)%(Filename)%(Extension)').Replace('\' ,'/')) + $([System.String]::Copy('%(RecursiveDir)%(Filename)%(Extension)').Replace('\' ,'/')) - - - - - + + + + + + AccountKey="$(ArtifactCloudDropAccessToken)" + AccountName="$(ArtifactCloudDropAccountName)" + ContainerName="$(ArtifactContainerName)" /> + + + + + + + + + + + \ No newline at end of file diff --git a/build_projects/dotnet-cli-build/CheckIfAllBuildsHavePublished.cs b/build_projects/dotnet-cli-build/CheckIfAllBuildsHavePublished.cs new file mode 100644 index 000000000..cfeb125a1 --- /dev/null +++ b/build_projects/dotnet-cli-build/CheckIfAllBuildsHavePublished.cs @@ -0,0 +1,86 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Microsoft.DotNet.Cli.Build +{ + public class CheckIfAllBuildsHavePublished : Task + { + private AzurePublisher _azurePublisher; + + [Required] + public string AccountName { get; set; } + + [Required] + public string AccountKey { get; set; } + + [Required] + public string ContainerName { get; set; } + + [Required] + public string NugetVersion { get; set; } + + [Output] + public string HaveAllBuildsPublished { get; set; } + + private AzurePublisher AzurePublisherTool + { + get + { + if (_azurePublisher == null) + { + _azurePublisher = new AzurePublisher(AccountName, AccountKey, ContainerName); + } + + return _azurePublisher; + } + } + + public override bool Execute() + { + var badges = new Dictionary() + { + { "Windows_x86", false }, + { "Windows_x64", false }, + { "Ubuntu_x64", false }, + { "Ubuntu_16_04_x64", false }, + { "RHEL_x64", false }, + { "OSX_x64", false }, + { "Debian_x64", false }, + { "CentOS_x64", false }, + { "Fedora_23_x64", false }, + { "openSUSE_13_2_x64", false } + }; + + var versionBadgeName = $"{Monikers.GetBadgeMoniker()}"; + if (!badges.ContainsKey(versionBadgeName)) + { + throw new ArgumentException($"A new OS build '{versionBadgeName}' was added without adding the moniker to the {nameof(badges)} lookup"); + } + + IEnumerable blobs = AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion); + foreach (string file in blobs) + { + string name = Path.GetFileName(file); + foreach (string img in badges.Keys) + { + if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg"))) + { + badges[img] = true; + break; + } + } + } + + HaveAllBuildsPublished = badges.Values.All(v => v).ToString(); + + return true; + } + } +} diff --git a/build_projects/dotnet-cli-build/CopyBlobsToLatest.cs b/build_projects/dotnet-cli-build/CopyBlobsToLatest.cs new file mode 100644 index 000000000..54b8aefb8 --- /dev/null +++ b/build_projects/dotnet-cli-build/CopyBlobsToLatest.cs @@ -0,0 +1,108 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.Cli.Build +{ + public class CopyBlobsToLatest : Task + { + private AzurePublisher _azurePublisher; + + [Required] + public string AccountName { get; set; } + + [Required] + public string AccountKey { get; set; } + + [Required] + public string Channel { get; set; } + + [Required] + public string CommitHash { get; set; } + + [Required] + public string ContainerName { get; set; } + + [Required] + public string NugetVersion { get; set; } + + private AzurePublisher AzurePublisherTool + { + get + { + if (_azurePublisher == null) + { + _azurePublisher = new AzurePublisher(AccountName, AccountKey, ContainerName); + } + + return _azurePublisher; + } + } + + public override bool Execute() + { + string targetFolder = $"{AzurePublisher.Product.Sdk}/{Channel}"; + + string targetVersionFile = $"{targetFolder}/{CommitHash}"; + string semaphoreBlob = $"{targetFolder}/publishSemaphore"; + AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob); + string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob); + + // Prevent race conditions by dropping a version hint of what version this is. If we see this file + // and it is the same as our version then we know that a race happened where two+ builds finished + // at the same time and someone already took care of publishing and we have no work to do. + if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile)) + { + AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); + return true; + } + else + { + Regex versionFileRegex = new Regex(@"(?[\w\d]{40})"); + + // Delete old version files + AzurePublisherTool.ListBlobs(targetFolder) + .Where(s => versionFileRegex.IsMatch(s)) + .ToList() + .ForEach(f => AzurePublisherTool.TryDeleteBlob(f)); + + // Drop the version file signaling such for any race-condition builds (see above comment). + AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile); + } + + try + { + CopyBlobs(targetFolder); + + string cliVersion = Utils.GetVersionFileContent(CommitHash, NugetVersion); + AzurePublisherTool.PublishStringToBlob($"{targetFolder}/latest.version", cliVersion); + } + finally + { + AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); + } + + return true; + } + + private void CopyBlobs(string destinationFolder) + { + Log.LogMessage("Copying blobs to {0}/{1}", ContainerName, destinationFolder); + + foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion)) + { + string targetName = Path.GetFileName(blob) + .Replace(NugetVersion, "latest"); + + string target = $"{destinationFolder}/{targetName}"; + + AzurePublisherTool.CopyBlob(blob, target); + } + } + } +} diff --git a/build_projects/dotnet-cli-build/FinalizeBuildTask.cs b/build_projects/dotnet-cli-build/FinalizeBuildTask.cs deleted file mode 100644 index 7c143415c..000000000 --- a/build_projects/dotnet-cli-build/FinalizeBuildTask.cs +++ /dev/null @@ -1,163 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text.RegularExpressions; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Microsoft.DotNet.Cli.Build -{ - public class FinalizeBuild : Task - { - private AzurePublisher _azurePublisher; - - [Required] - public string AccountName { get; set; } - - [Required] - public string AccountKey { get; set; } - - [Required] - public string NugetVersion { get; set; } - - [Required] - public string Channel { get; set; } - - [Required] - public string CommitHash { get; set; } - - [Required] - public string BranchName { get; set; } - - private AzurePublisher AzurePublisherTool - { - get - { - if(_azurePublisher == null) - { - _azurePublisher = new AzurePublisher(AccountName, AccountKey); - } - - return _azurePublisher; - } - } - - public override bool Execute() - { - DoFinalizeBuild(); - - return true; - } - - private void DoFinalizeBuild() - { - if (CheckIfAllBuildsHavePublished()) - { - string targetContainer = $"{AzurePublisher.Product.Sdk}/{Channel}"; - string targetVersionFile = $"{targetContainer}/{CommitHash}"; - string semaphoreBlob = $"{targetContainer}/publishSemaphore"; - AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob); - string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob); - - // Prevent race conditions by dropping a version hint of what version this is. If we see this file - // and it is the same as our version then we know that a race happened where two+ builds finished - // at the same time and someone already took care of publishing and we have no work to do. - if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile)) - { - AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); - return; - } - else - { - Regex versionFileRegex = new Regex(@"(?[\w\d]{40})"); - - // Delete old version files - AzurePublisherTool.ListBlobs(targetContainer) - .Where(s => versionFileRegex.IsMatch(s)) - .ToList() - .ForEach(f => AzurePublisherTool.TryDeleteBlob(f)); - - // Drop the version file signaling such for any race-condition builds (see above comment). - AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile); - } - - try - { - CopyBlobsToLatest(targetContainer); - - string cliVersion = Utils.GetVersionFileContent(CommitHash, NugetVersion); - AzurePublisherTool.PublishStringToBlob($"{targetContainer}/latest.version", cliVersion); - - UpdateVersionsRepo(); - } - finally - { - AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); - } - } - } - - private bool CheckIfAllBuildsHavePublished() - { - var badges = new Dictionary() - { - { "Windows_x86", false }, - { "Windows_x64", false }, - { "Ubuntu_x64", false }, - { "Ubuntu_16_04_x64", false }, - { "RHEL_x64", false }, - { "OSX_x64", false }, - { "Debian_x64", false }, - { "CentOS_x64", false }, - { "Fedora_23_x64", false }, - { "openSUSE_13_2_x64", false } - }; - - var versionBadgeName = $"{Monikers.GetBadgeMoniker()}"; - if (!badges.ContainsKey(versionBadgeName)) - { - throw new ArgumentException($"A new OS build '{versionBadgeName}' was added without adding the moniker to the {nameof(badges)} lookup"); - } - - IEnumerable blobs = AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion); - foreach (string file in blobs) - { - string name = Path.GetFileName(file); - foreach (string img in badges.Keys) - { - if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg"))) - { - badges[img] = true; - break; - } - } - } - - return badges.Values.All(v => v); - } - - private void CopyBlobsToLatest(string destinationFolder) - { - foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion)) - { - string targetName = Path.GetFileName(blob) - .Replace(NugetVersion, "latest"); - - string target = $"{destinationFolder}/{targetName}"; - AzurePublisherTool.CopyBlob(blob, target); - } - } - - private void UpdateVersionsRepo() - { - string githubAuthToken = EnvVars.EnsureVariable("GITHUB_PASSWORD"); - string nupkgFilePath = Dirs.Packages; - string branchName = BranchName; - string versionsRepoPath = $"build-info/dotnet/cli/{branchName}/Latest"; - - VersionRepoUpdater repoUpdater = new VersionRepoUpdater(githubAuthToken); - repoUpdater.UpdatePublishedVersions(nupkgFilePath, versionsRepoPath).Wait(); - } - } -} \ No newline at end of file diff --git a/build_projects/dotnet-cli-build/GenerateChecksums.cs b/build_projects/dotnet-cli-build/GenerateChecksums.cs new file mode 100644 index 000000000..4bd1ef7c0 --- /dev/null +++ b/build_projects/dotnet-cli-build/GenerateChecksums.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.IO; +using System.Security.Cryptography; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class GenerateChecksums : Task + { + /// + /// An item collection of files for which to generate checksums. Each item must have metadata + /// 'DestinationPath' that specifies the path of the checksum file to create. + /// + [Required] + public ITaskItem[] Items { get; set; } + + public override bool Execute() + { + foreach (ITaskItem item in Items) + { + try + { + string destinationPath = item.GetMetadata("DestinationPath"); + if (string.IsNullOrEmpty(destinationPath)) + { + throw new Exception($"Metadata 'DestinationPath' is missing for item '{item.ItemSpec}'."); + } + + if (!File.Exists(item.ItemSpec)) + { + throw new Exception($"The file '{item.ItemSpec}' does not exist."); + } + + Log.LogMessage( + MessageImportance.High, + "Generating checksum for '{0}' into '{1}'...", + item.ItemSpec, + destinationPath); + + using (FileStream stream = File.OpenRead(item.ItemSpec)) + { + HashAlgorithm hashAlgorithm = SHA512.Create(); + byte[] hash = hashAlgorithm.ComputeHash(stream); + string checksum = BitConverter.ToString(hash).Replace("-", string.Empty); + File.WriteAllText(destinationPath, checksum); + } + } + catch (Exception e) + { + // We have 2 log calls because we want a nice error message but we also want to capture the + // callstack in the log. + Log.LogError("An exception occurred while trying to generate a checksum for '{0}'.", item.ItemSpec); + Log.LogMessage(MessageImportance.Low, e.ToString()); + return false; + } + } + + return true; + } + } +} diff --git a/build_projects/dotnet-cli-build/UpdateVersionsRepo.cs b/build_projects/dotnet-cli-build/UpdateVersionsRepo.cs new file mode 100644 index 000000000..b17b9bee7 --- /dev/null +++ b/build_projects/dotnet-cli-build/UpdateVersionsRepo.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Cli.Build +{ + public class UpdateVersionsRepo : Task + { + [Required] + public string BranchName { get; set; } + + public override bool Execute() + { + string githubAuthToken = EnvVars.EnsureVariable("GITHUB_PASSWORD"); + string nupkgFilePath = Dirs.Packages; + string branchName = BranchName; + string versionsRepoPath = $"build-info/dotnet/cli/{branchName}/Latest"; + + VersionRepoUpdater repoUpdater = new VersionRepoUpdater(githubAuthToken); + repoUpdater.UpdatePublishedVersions(nupkgFilePath, versionsRepoPath).Wait(); + + return true; + } + } +} diff --git a/scripts/dockerrun.sh b/scripts/dockerrun.sh index 3d4ccf21c..4c289cc94 100755 --- a/scripts/dockerrun.sh +++ b/scripts/dockerrun.sh @@ -126,8 +126,11 @@ docker run $INTERACTIVE -t --rm --sig-proxy=true \ -e NUGET_FEED_URL \ -e NUGET_API_KEY \ -e GITHUB_PASSWORD \ - -e STORAGE_KEY \ - -e STORAGE_ACCOUNT \ - -e STORAGE_CONTAINER \ + -e ARTIFACT_STORAGE_KEY \ + -e ARTIFACT_STORAGE_ACCOUNT \ + -e ARTIFACT_STORAGE_CONTAINER \ + -e CHECKSUM_STORAGE_KEY \ + -e CHECKSUM_STORAGE_ACCOUNT \ + -e CHECKSUM_STORAGE_CONTAINER \ $DOTNET_BUILD_CONTAINER_TAG \ $BUILD_COMMAND "$@"