Producing and publishing checksums. Also, publishing sdk assets ourselves, so that we can add metadata regarding where to push nupkgs.

.vsts-ci.yml

@@ -6,11 +6,13 @@ variables:
   ${{ if eq(variables['System.TeamProject'], 'public') }}:
     PB_PublishBlobFeedKey:
     PB_PublishBlobFeedUrl:
+    PB_PublishChecksumsBlobFeedUrl:
     _DotNetPublishToBlobFeed: false
     _PublishType: none
     _SignType: test
   ${{ if ne(variables['System.TeamProject'], 'public') }}:
     PB_PublishBlobFeedUrl: https://dotnetcli.blob.core.windows.net/dotnet/index.json
+    PB_PublishChecksumsBlobFeedUrl: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
     _DotNetPublishToBlobFeed: true
     _PublishType: blob
     _SignType: real
@@ -253,12 +255,14 @@ jobs:
         inputs:
           azureSubscription: 'DotNet-Engineering-Services_KeyVault'
           KeyVaultName: EngKeyVault
-          SecretsFilter: 'dotnetfeed-storage-access-key-1'
+          SecretsFilter: 'dotnetfeed-storage-access-key-1,dotnetclichecksums-storage-key'
         condition: succeeded()
 
       - script: eng/CopyToLatest.cmd
                 /p:DotNetPublishBlobFeedUrl=$(PB_PublishBlobFeedUrl)
                 /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
+                /p:DotnetPublishChecksumsBlobFeedUrl=$(PB_PublishChecksumsBlobFeedUrl)
+                /p:DotNetPublishChecksumsBlobFeedKey=$(dotnetclichecksums-storage-key)
 
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - template: /eng/common/templates/phases/publish-build-assets.yml

eng/Publishing.props

@@ -4,11 +4,20 @@
 
   <Import Project="$(RepoRoot)\src\redist\targets\Versions.targets" />
 
+  <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets"/>
+
   <PropertyGroup>
     <Product>Sdk</Product>
     <BlobStoragePartialRelativePath>$(Product)</BlobStoragePartialRelativePath>
     <BlobStoragePartialRelativePath Condition="'$(IsNotOrchestratedPublish)' == 'false'">assets/$(Product)</BlobStoragePartialRelativePath>
     <PublishToBlobFeedFlatContainer>true</PublishToBlobFeedFlatContainer>
+    <ChecksumsFeedUrl>$(DotnetPublishChecksumsBlobFeedUrl)</ChecksumsFeedUrl>
+    <ChecksumsAccountKey>$(DotNetPublishChecksumsBlobFeedKey)</ChecksumsAccountKey>
+    <SdkAssetsFeedUrl>$(DotnetPublishSdkAssetsBlobFeedUrl)</SdkAssetsFeedUrl>
+    <SdkAssetsAzureAccountKey>$(DotNetPublishSdkAssetsBlobFeedKey)</SdkAssetsAzureAccountKey>
+
+    <PublishSdkAssetsAndChecksumsToBlob>false</PublishSdkAssetsAndChecksumsToBlob>
+    <PublishSdkAssetsAndChecksumsToBlob Condition=" '$(ChecksumsFeedUrl)' != '' and '$(SdkAssetsFeedUrl)' != '' ">true</PublishSdkAssetsAndChecksumsToBlob>
   </PropertyGroup>
 
   <ItemGroup>
@@ -19,16 +28,50 @@
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.deb" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.rpm" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.msi" />
+    <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.nupkg" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.svg" />
+
+    <CheckSumsToPublish Include="$(ArtifactsShippingPackagesDir)*.sha" />
   </ItemGroup>
 
-  <Target Name="PrepareCoreSdkAssets" BeforeTargets="Publish" DependsOnTargets="SetSdkVersionInfo">
+  <Target Name="PublishSdkAssetsAndChecksums"
+          BeforeTargets="Publish"
+          DependsOnTargets="SetSdkVersionInfo"
+          Condition=" '$(PublishSdkAssetsAndChecksumsToBlob)' == 'true' ">
     <ItemGroup>
-      <ItemsToPushToBlobFeed Include="@(SdkAssetsToPublish)" IsShipping="true" >
+      <SdkAssetsToPushToBlobFeed Include="@(SdkAssetsToPublish)" IsShipping="true" >
         <RelativeBlobPath>$(BlobStoragePartialRelativePath)/$(SdkVersion)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/'))</RelativeBlobPath>
         <ManifestArtifactData>ShipInstaller=dotnetcli</ManifestArtifactData>
-      </ItemsToPushToBlobFeed>
-    </ItemGroup>
-  </Target>
+      </SdkAssetsToPushToBlobFeed>
 
+      <ChecksumsToPushToBlobFeed Include="@(CheckSumsToPublish)" IsShipping="true" >
+        <RelativeBlobPath>$(BlobStoragePartialRelativePath)/$(SdkVersion)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/'))</RelativeBlobPath>
+        <ManifestArtifactData>ShipInstaller=dotnetclichecksums</ManifestArtifactData>
+      </ChecksumsToPushToBlobFeed>
+    </ItemGroup>
+
+    <PushToBlobFeed
+      ExpectedFeedUrl="$(SdkAssetsFeedUrl)"
+      AccountKey="$(SdkAssetsAzureAccountKey)"
+      ItemsToPush="@(SdkAssetsToPushToBlobFeed)"
+      ManifestBuildData="Location=$(SdkAssetsFeedUrl)"
+      ManifestRepoUri="$(BUILD_REPOSITORY_URI)"
+      ManifestBranch="$(BUILD_SOURCEBRANCH)"
+      ManifestBuildId="$(BUILD_BUILDNUMBER)"
+      ManifestCommit="$(BUILD_SOURCEVERSION)"
+      AssetManifestPath="$(AssetManifestFilePath)"
+      PublishFlatContainer="true"/>
+
+    <PushToBlobFeed
+      ExpectedFeedUrl="$(ChecksumsFeedUrl)"
+      AccountKey="$(ChecksumsAccountKey)"
+      ItemsToPush="@(ChecksumsToPushToBlobFeed)"
+      ManifestBuildData="Location=$(ChecksumsFeedUrl)"
+      ManifestRepoUri="$(BUILD_REPOSITORY_URI)"
+      ManifestBranch="$(BUILD_SOURCEBRANCH)"
+      ManifestBuildId="$(BUILD_BUILDNUMBER)"
+      ManifestCommit="$(BUILD_SOURCEVERSION)"
+      AssetManifestPath="$(AssetManifestFilePath)"
+      PublishFlatContainer="true"/>
+  </Target>
 </Project>

eng/build.yml

@@ -30,12 +30,14 @@ phases:
       ${{ if ne(variables['System.TeamProject'], 'public') }}:
         _HelixSource: official/dotnet/core-sdk/$(Build.SourceBranch)
         _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
-        _PublishArgs: /p:DotNetPublishBlobFeedKey=$(dotnetcli-storage-key)
-                  /p:DotNetPublishBlobFeedUrl=$(PB_PublishBlobFeedUrl)
-                  /p:DotNetPublishToBlobFeed=$(_DotNetPublishToBlobFeed)
+        _PublishArgs: /p:DotNetPublishToBlobFeed=$(_DotNetPublishToBlobFeed)
+                  /p:DotnetPublishSdkAssetsBlobFeedUrl=$(PB_PublishBlobFeedUrl)
+                  /p:DotNetPublishSdkAssetsBlobFeedKey=$(dotnetcli-storage-key)
                   /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
                   /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
                   /p:PB_PublishType=$(_PublishType)
+                  /p:DotnetPublishChecksumsBlobFeedUrl=$(PB_PublishChecksumsBlobFeedUrl)
+                  /p:DotNetPublishChecksumsBlobFeedKey=$(dotnetclichecksums-storage-key)
         _SignArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName)
 
     steps:
@@ -57,7 +59,7 @@ phases:
         inputs:
           azureSubscription: 'DotNet-Engineering-Services_KeyVault'
           KeyVaultName: EngKeyVault
-          SecretsFilter: 'dotnetcli-storage-key,microsoft-symbol-server-pat,symweb-symbol-server-pat'
+          SecretsFilter: 'dotnetcli-storage-key,dotnetclichecksums-storage-key,microsoft-symbol-server-pat,symweb-symbol-server-pat'
         condition: succeeded()
       - task: AzureKeyVault@1
         inputs:

src/CopyToLatest/targets/FinishBuild.targets

@@ -8,5 +8,11 @@
                        NugetVersion="$(FullNugetVersion)"
                        Channel="$(Channel)"
                        CommitHash="$(GitCommitHash)" />
+
+    <CopyBlobsToLatest FeedUrl="$(DotNetPublishChecksumsBlobFeedUrl)"
+                       AccountKey="$(DotNetPublishChecksumsBlobFeedKey)"
+                       NugetVersion="$(FullNugetVersion)"
+                       Channel="$(Channel)"
+                       CommitHash="$(GitCommitHash)" />
   </Target>
 </Project>

src/core-sdk-tasks/GenerateChecksums.cs

@@ -0,0 +1,66 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.IO;
+using System.Security.Cryptography;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+    public class GenerateChecksums : Task
+    {
+        /// <summary>
+        /// An item collection of files for which to generate checksums.  Each item must have metadata
+        /// 'DestinationPath' that specifies the path of the checksum file to create.
+        /// </summary>
+        [Required]
+        public ITaskItem[] Items { get; set; }
+
+        public override bool Execute()
+        {
+            foreach (ITaskItem item in Items)
+            {
+                try
+                {
+                    string destinationPath = item.GetMetadata("DestinationPath");
+                    if (string.IsNullOrEmpty(destinationPath))
+                    {
+                        throw new Exception($"Metadata 'DestinationPath' is missing for item '{item.ItemSpec}'.");
+                    }
+
+                    if (!File.Exists(item.ItemSpec))
+                    {
+                        throw new Exception($"The file '{item.ItemSpec}' does not exist.");
+                    }
+
+                    Log.LogMessage(
+                        MessageImportance.High,
+                        "Generating checksum for '{0}' into '{1}'...",
+                        item.ItemSpec,
+                        destinationPath);
+
+                    using (FileStream stream = File.OpenRead(item.ItemSpec))
+                    {
+                        HashAlgorithm hashAlgorithm = SHA512.Create();
+                        byte[] hash = hashAlgorithm.ComputeHash(stream);
+                        string checksum = BitConverter.ToString(hash).Replace("-", string.Empty);
+                        File.WriteAllText(destinationPath, checksum);
+                    }
+                }
+                catch (Exception e)
+                {
+                    // We have 2 log calls because we want a nice error message but we also want to capture the
+                    // callstack in the log.
+                    Log.LogError("An exception occurred while trying to generate a checksum for '{0}'.", item.ItemSpec);
+                    Log.LogMessage(MessageImportance.Low, e.ToString());
+                    return false;
+                }
+            }
+
+            return true;
+        }
+    }
+}

src/redist/redist.csproj

@@ -29,6 +29,7 @@
   <Import Project="targets\GeneratePKG.targets" />
   <Import Project="targets\GenerateInstallers.targets" />
   <Import Project="targets\Badge.targets" />
+  <Import Project="targets\Checksum.targets" />
 
   <Import Project="targets\Signing.targets" />
 </Project>

src/redist/targets/BuildCoreSdkTasks.targets

@@ -35,5 +35,6 @@
   <UsingTask TaskName="AddMetadataIsPE" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="Crossgen" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="CopyBlobsToLatest" AssemblyFile="$(CoreSdkTaskDll)"/>
+  <UsingTask TaskName="GenerateChecksums" AssemblyFile="$(CoreSdkTaskDll)"/>
 
 </Project>

src/redist/targets/Checksum.targets

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Target Name="GenerateChecksums"
+          DependsOnTargets="InitializeChecksumItemGroup;"
+          AfterTargets="GenerateInstallers"
+          Inputs="@(ArtifactsForGeneratingChecksums)"
+          Outputs="%(ArtifactsForGeneratingChecksums.DestinationPath)">
+    <GenerateChecksums Items="@(ArtifactsForGeneratingChecksums)" />
+  </Target>
+
+  <Target Name="InitializeChecksumItemGroup">
+    <ItemGroup>
+      <FilesToExcludeForChecksum Include="$(ArtifactsShippingPackagesDir)*.svg" />
+      <FilesToExcludeForChecksum Include="$(ArtifactsShippingPackagesDir)*.nupkg" />
+      <FilesToExcludeForChecksum Include="$(ArtifactsShippingPackagesDir)*.sha" />
+      <ArtifactsForGeneratingChecksums Include="$(ArtifactsShippingPackagesDir)*.*" Exclude="@(FilesToExcludeForChecksum)" />
+
+      <ArtifactsForGeneratingChecksums>
+        <DestinationPath>%(ArtifactsForGeneratingChecksums.Identity).sha</DestinationPath>
+      </ArtifactsForGeneratingChecksums>
+    </ItemGroup>
+  </Target>
+</Project>