Uploading assets using msbuild.

Copying Microsoft.WindowsAzure.Storage.dll to the sdk stage0 folder to get around issue https://github.com/Microsoft/msbuild/issues/658.

build.proj

@@ -18,6 +18,7 @@
 
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
     <RepoRoot>$(MSBuildThisFileDirectory)</RepoRoot>
+    <ToolsDir>$(RepoRoot)/build_tools</ToolsDir>
 
     <CoreSetupChannel>preview</CoreSetupChannel>
     <SharedFrameworkName>Microsoft.NETCore.App</SharedFrameworkName>
@@ -46,32 +47,36 @@
     <Message Text="Dont remove this target" />
   </Target>
 
-  <Target Name="BuildDotnetCliBuildFramework" 
+  <Target Name="BuildDotnetCliBuildFramework"
-          Inputs="@(DotnetCliBuildFrameworkInputs)" 
+          Inputs="@(DotnetCliBuildFrameworkInputs)"
           Outputs="$(CLIBuildDll)"
           DependsOnTargets="MSBuildWorkaroundTarget">
     <PropertyGroup>
       <DotnetCliBuildDirectory>$(MSBuildThisFileDirectory)/build_projects/dotnet-cli-build</DotnetCliBuildDirectory>
     </PropertyGroup>
 
+    <ItemGroup>
+      <DotnetSdkDirectories Include="$([System.IO.Directory]::GetDirectories(&quot;$(Stage0Path)/sdk&quot;))" />
+      <AzureStorageToCopy Include="@(DotnetSdkDirectories)">
+        <Source>$(DotnetCliBuildDirectory)/bin/Microsoft.WindowsAzure.Storage.dll</Source>
+        <Destination>%(Identity)/Microsoft.WindowsAzure.Storage.dll</Destination>
+      </AzureStorageToCopy>
+    </ItemGroup>
+
     <Exec Command="$(DotnetStage0) restore" WorkingDirectory="$(DotnetCliBuildDirectory)"/>
 
-    <Exec Command="$(DotnetStage0) publish -o $(DotnetCliBuildDirectory)/bin --framework netcoreapp1.0" WorkingDirectory="$(DotnetCliBuildDirectory)"/>    
+    <Exec Command="$(DotnetStage0) publish -o $(DotnetCliBuildDirectory)/bin --framework netcoreapp1.0" WorkingDirectory="$(DotnetCliBuildDirectory)"/>
+
+    <!-- This is a work around to issue https://github.com/Microsoft/msbuild/issues/658 -->
+    <Copy SourceFiles="%(AzureStorageToCopy.Source)"
+          DestinationFiles="%(AzureStorageToCopy.Destination)" />
   </Target>
 
   <Target DependsOnTargets="$(CLITargets)" Name="BuildTheWholeCli"></Target>
 
-  <UsingTask TaskName="PublishTargets" AssemblyFile="$(CLIBuildDll)" />
-
-  <Target DependsOnTargets="BuildDotnetCliBuildFramework;GenerateVersionBadge" Name="Publish">
-    <PublishTargets />
-  </Target>
-
   <Import Project="build/Microsoft.DotNet.Cli.Prepare.targets" />
   <Import Project="build/Microsoft.DotNet.Cli.Compile.targets" />
   <Import Project="build/Microsoft.DotNet.Cli.Package.targets" />
   <Import Project="build/Microsoft.DotNet.Cli.Test.targets" /> 
-
+  <Import Project="build/Microsoft.DotNet.Cli.Publish.targets" />
-  <!-- This should be in the build/Microsoft.DotNet.Cli.Publish.targets -->
-  <Import Project="build/publish/Microsoft.DotNet.Cli.Badge.targets" />
 </Project>

build/Microsoft.DotNet.Cli.Publish.targets

@@ -0,0 +1,101 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(ToolsDir)/PublishContent.targets" />
+  <Import Project="$(MSBuildThisFileDirectory)/publish/Microsoft.DotNet.Cli.Badge.targets" />
+
+  <UsingTask TaskName="UploadToAzure" AssemblyFile="$(BuildToolsTaskDir)/Microsoft.DotNet.Build.CloudTestTasks.dll"/>
+  <UsingTask TaskName="FinalizeBuildTask" AssemblyFile="$(CLIBuildDll)" />
+
+  <!-- PUBLISH_TO_AZURE_BLOB env variable set by CI -->
+  <Target Name="Publish"
+          Condition=" '$(PUBLISH_TO_AZURE_BLOB)' != '' "
+          DependsOnTargets="Prepare;
+                            Package;
+                            PublishArtifacts;
+                            FinalizeBuild" />
+
+  <Target Name="PublishArtifacts" DependsOnTargets="SetupAzureBlobInformation;
+                                                    GenerateVersionBadge;
+                                                    GatherItemsForPattern;
+                                                    UploadToAzure;
+                                                    PublishDebFilesToDebianRepo;
+                                                    PublishCliVersionBadge" />
+
+  <Target Name="FinalizeBuild">
+    <FinalizeBuildTask AccountName="$(CloudDropAccountName)"
+                       AccountKey="$(CloudDropAccessToken)"
+                       NugetVersion="$(NugetVersion)"
+                       Channel="$(Channel)"
+                       CommitHash="$(CommitHash)"
+                       BranchName="$(BranchName)" />
+  </Target>
+
+  <Target Name="SetupAzureBlobInformation">
+    <PropertyGroup>
+      <Product>Sdk</Product>
+      <ContainerName>$(STORAGE_CONTAINER)</ContainerName>
+      <CloudDropAccessToken>$(STORAGE_KEY)</CloudDropAccessToken>
+      <CloudDropAccountName>$(STORAGE_ACCOUNT)</CloudDropAccountName>
+      <DotnetBlobRootUrl>https://$(CloudDropAccountName).blob.core.windows.net/$(ContainerName)</DotnetBlobRootUrl>
+    </PropertyGroup>    
+  </Target>
+
+  <Target Name="GatherItemsForPattern">
+    <ItemGroup>
+      <ForPublishing Include="@(GeneratedInstallers)" />
+      <ForPublishing Include="%(GenerateArchivesInputsOutputs.Outputs)" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ForPublishing>
+        <RelativeBlobPath>$(Product)/$(NugetVersion)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/'))</RelativeBlobPath> 
+      </ForPublishing>
+    </ItemGroup>
+  </Target>  
+
+  <Target Name="PublishDebFilesToDebianRepo" Condition=" '$(OSName)' == 'ubuntu' ">
+    <Error Condition="'$(REPO_ID)' == ''" Text="REPO_ID must be set as an environment variable for debian publishing." />
+
+    <ItemGroup>
+      <SdkInstallerFileItemGroup Include="$(SdkInstallerFile)" />
+    </ItemGroup>
+    <PropertyGroup>
+      <SdkDebianPackageName>dotnet-dev-$(NugetVersion)</SdkDebianPackageName>
+      <SdkDebianUploadUrl>$(DotnetBlobRootUrl)/$(Product)/$(NugetVersion)/%(SdkInstallerFileItemGroup.Filename)%(SdkInstallerFileItemGroup.Extension)</SdkDebianUploadUrl>
+      <DebianUploadJsonFile>$(PackageDirectory)/package_upload.json</DebianUploadJsonFile>
+      <DebianRevisionNumber>1</DebianRevisionNumber>
+
+      <DebianUploadJsonContent>
+        {
+          "name":"$(SdkDebianPackageName)",
+          "version":"$(NugetVersion)-$(DebianRevisionNumber)",
+          "repositoryId":"$(REPO_ID)",
+          "sourceUrl": "$(SdkDebianUploadUrl)"
+        }
+      </DebianUploadJsonContent>
+    </PropertyGroup>
+
+    <Delete Files="$(DebianUploadJsonFile)" />
+    <WriteLinesToFile File="$(DebianUploadJsonFile)" Lines="$(DebianUploadJsonContent)" />
+
+    <Exec Command="$(RepoRoot)/scripts/publish/repoapi_client.sh -addpkg $(DebianUploadJsonFile)" />
+  </Target>
+
+  <Target Name="PublishCliVersionBadge">
+    <ItemGroup>
+      <CliVersionBadgeToUpload Include="$(VersionBadge)" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <CliVersionBadgeToUpload>
+        <RelativeBlobPath>$(Product)/$(NugetVersion)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/'))</RelativeBlobPath>
+      </CliVersionBadgeToUpload>
+    </ItemGroup>
+
+    <UploadToAzure
+      AccountKey="$(CloudDropAccessToken)"
+      AccountName="$(CloudDropAccountName)"
+      ContainerName="$(ContainerName)"
+      Items="@(CliVersionBadgeToUpload)" />
+  </Target>
+</Project>

build/package/Microsoft.DotNet.Cli.Installer.DEB.targets

@@ -35,6 +35,11 @@
             <EndToEndTestDirectory>$(RepoRoot)/test/EndToEnd</EndToEndTestDirectory>
         </PropertyGroup>
 
+        <!-- Consumed By Publish -->
+        <ItemGroup>
+          <GeneratedInstallers Include="$(SdkInstallerFile)" />
+        </ItemGroup>        
+
         <ItemGroup>
             <SdkDebInputFiles Include="$(SdkLayoutOutputDirectory)/**/*" />
         </ItemGroup>

build/package/Microsoft.DotNet.Cli.Nupkg.targets

@@ -63,7 +63,7 @@
                 NoBuild="True"
                 Output="$(NupkgOutputDirectory)"
                 ProjectPath="%(ProjectsToPack.Identity)"
-                ToolPath="$(DotnetStage2)"
+                ToolPath="$(Stage2Directory)"
                 VersionSuffix="$(NupkgVersionSuffix)"
                 Configuration="$(Configuration)" />
   </Target>

build_projects/dotnet-cli-build/FinalizeBuildTask.cs

@@ -0,0 +1,162 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+namespace Microsoft.DotNet.Cli.Build
+{
+    public class FinalizeBuildTask : Task
+    {
+        private AzurePublisher _azurePublisher;
+
+        [Required]
+        public string AccountName { get; set; }
+
+        [Required]
+        public string AccountKey { get; set; }
+
+        [Required]
+        public string NugetVersion { get; set; }
+
+        [Required]
+        public string Channel { get; set; }
+
+        [Required]
+        public string CommitHash { get; set; }
+
+        [Required]
+        public string BranchName { get; set; }
+
+        private AzurePublisher AzurePublisherTool {
+            get
+            {
+                if(_azurePublisher == null)
+                {
+                    _azurePublisher = new AzurePublisher(AccountName, AccountKey);
+                }
+
+                return _azurePublisher;
+            }
+        }
+
+        public override bool Execute()
+        {
+            FinalizeBuild();
+
+            return true;
+        }
+
+        private void FinalizeBuild()
+        {
+            if (CheckIfAllBuildsHavePublished())
+            {
+                string targetContainer = $"{AzurePublisher.Product.Sdk}/{Channel}";
+                string targetVersionFile = $"{targetContainer}/{CommitHash}";
+                string semaphoreBlob = $"{targetContainer}/publishSemaphore";
+                AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob);
+                string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob);
+
+                // Prevent race conditions by dropping a version hint of what version this is. If we see this file
+                // and it is the same as our version then we know that a race happened where two+ builds finished 
+                // at the same time and someone already took care of publishing and we have no work to do.
+                if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile))
+                {
+                    AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
+                    return;
+                }
+                else
+                {
+                    Regex versionFileRegex = new Regex(@"(?<CommitHash>[\w\d]{40})");
+
+                    // Delete old version files
+                    AzurePublisherTool.ListBlobs(targetContainer)
+                        .Where(s => versionFileRegex.IsMatch(s))
+                        .ToList()
+                        .ForEach(f => AzurePublisherTool.TryDeleteBlob(f));
+
+                    // Drop the version file signaling such for any race-condition builds (see above comment).
+                    AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile);
+                }
+
+                try
+                {
+                    CopyBlobsToLatest(targetContainer);
+
+                    string cliVersion = Utils.GetVersionFileContent(CommitHash, NugetVersion);
+                    AzurePublisherTool.PublishStringToBlob($"{targetContainer}/latest.version", cliVersion);
+
+                    UpdateVersionsRepo();
+                }
+                finally
+                {
+                    AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
+                }
+            }
+        }
+
+        private bool CheckIfAllBuildsHavePublished()
+        {
+            var badges = new Dictionary<string, bool>()
+            {
+                { "Windows_x86", false },
+                { "Windows_x64", false },
+                { "Ubuntu_x64", false },
+                { "Ubuntu_16_04_x64", false },
+                { "RHEL_x64", false },
+                { "OSX_x64", false },
+                { "Debian_x64", false },
+                { "CentOS_x64", false },
+                { "Fedora_23_x64", false },
+                { "openSUSE_13_2_x64", false }
+            };
+
+            var versionBadgeName = $"{Monikers.GetBadgeMoniker()}";
+            if (!badges.ContainsKey(versionBadgeName))
+            {
+                throw new ArgumentException($"A new OS build '{versionBadgeName}' was added without adding the moniker to the {nameof(badges)} lookup");
+            }
+
+            IEnumerable<string> blobs = AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion);
+            foreach (string file in blobs)
+            {
+                string name = Path.GetFileName(file);
+                foreach (string img in badges.Keys)
+                {
+                    if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg")))
+                    {
+                        badges[img] = true;
+                        break;
+                    }
+                }
+            }
+
+            return badges.Values.All(v => v);
+        }
+
+        private void CopyBlobsToLatest(string destinationFolder)
+        {
+            foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion))
+            {
+                string targetName = Path.GetFileName(blob)
+                                        .Replace(NugetVersion, "latest");
+
+                string target = $"{destinationFolder}/{targetName}";
+                AzurePublisherTool.CopyBlob(blob, target);
+            }
+        }
+
+        private void UpdateVersionsRepo()
+        {
+            string githubAuthToken = EnvVars.EnsureVariable("GITHUB_PASSWORD");
+            string nupkgFilePath = Dirs.Packages;
+            string branchName = BranchName;
+            string versionsRepoPath = $"build-info/dotnet/cli/{branchName}/Latest";
+
+            VersionRepoUpdater repoUpdater = new VersionRepoUpdater(githubAuthToken);
+            repoUpdater.UpdatePublishedVersions(nupkgFilePath, versionsRepoPath).Wait();
+        }
+    }
+}

build_projects/shared-build-targets-utils/Publishing/AzurePublisher.cs

@@ -7,6 +7,7 @@ using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Auth;
 using Microsoft.WindowsAzure.Storage.Blob;
 
 namespace Microsoft.DotNet.Cli.Build
@@ -33,9 +34,27 @@ namespace Microsoft.DotNet.Cli.Build
             _blobContainer = GetDotnetBlobContainer(_connectionString);
         }
 
+        public AzurePublisher(string accountName, string accountKey)
+        {
+            _blobContainer = GetDotnetBlobContainer(accountName, accountKey);
+        }
+
         private CloudBlobContainer GetDotnetBlobContainer(string connectionString)
         {
             CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
+            
+            return GetDotnetBlobContainer(storageAccount);
+        }
+
+        private CloudBlobContainer GetDotnetBlobContainer(string accountName, string accountKey)
+        {
+            var storageCredentials = new StorageCredentials(accountName, accountKey);
+            var storageAccount = new CloudStorageAccount(storageCredentials, true);
+            return GetDotnetBlobContainer(storageAccount);
+        }
+
+        private CloudBlobContainer GetDotnetBlobContainer(CloudStorageAccount storageAccount)
+        {
             CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
             
             return blobClient.GetContainerReference(s_dotnetBlobContainerName);