Remove Copy Blobs To Latest functionality and clean up some publishing (#10773)
- The stage has been subsumed by the aka.ms functionality and now can be removed. - Clean up dead properties and functionality no longer needed because of it. - Removing publishing of nupkg files as blobs. This used to be useful before because we often needed to get at the nupkgs in a flat, easy to copy format for pushing to the VS feed. But now the staging pipeline takes care of this, and moreover the use of post-build signing means that these files are not useful straight out of the build anyway. This should reduce publishing time and some post-build signing time. Remove nupkg install in publishing.props
This commit is contained in:
parent
789b3b05e8
commit
e58843bebc
7 changed files with 1 additions and 213 deletions
30
.vsts-ci.yml
30
.vsts-ci.yml
|
@ -9,12 +9,6 @@ trigger:
|
|||
- internal/release/5.*
|
||||
|
||||
variables:
|
||||
# These two variables are used by CopyToLatest.
|
||||
# when CopyToLatest goes away in favor of improvements to dotnet-install, this can be removed.
|
||||
- name: _PublishBlobFeedUrl
|
||||
value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
|
||||
- name: _PublishChecksumsBlobFeedUrl
|
||||
value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
|
||||
- name: _PublishUsingPipelines
|
||||
value: false
|
||||
- name: PostBuildSign
|
||||
|
@ -390,27 +384,3 @@ stages:
|
|||
-TsaRepositoryName "dotnet-installer"
|
||||
-TsaCodebaseName "dotnet-installer"
|
||||
-TsaPublish $True'
|
||||
|
||||
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), not(contains(variables['Build.SourceBranch'], 'refs/heads/internal/'))) }}:
|
||||
- stage: copy_to_latest
|
||||
displayName: Copy to latest
|
||||
dependsOn:
|
||||
# This will run only after all the publishing stages have run.
|
||||
# These stages are introduced in the eng/common/templates/post-build/channels YAML templates
|
||||
- publish_using_darc
|
||||
jobs:
|
||||
- job: Copy_SDK_To_Latest
|
||||
pool:
|
||||
name: Hosted VS2017
|
||||
condition: succeeded()
|
||||
variables:
|
||||
- group: DotNet-DotNetCli-Storage
|
||||
steps:
|
||||
- script: eng/CopyToLatest.cmd
|
||||
/p:OfficialBuildId=$(BUILD.BUILDNUMBER)
|
||||
/p:DotnetPublishSdkAssetsBlobFeedUrl=https://dotnetcli.blob.core.windows.net/dotnet/index.json
|
||||
/p:DotNetPublishSdkAssetsBlobFeedKey=$(dotnetcli-storage-key)
|
||||
/p:DotnetPublishChecksumsBlobFeedUrl=https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
|
||||
/p:DotNetPublishChecksumsBlobFeedKey=$(dotnetclichecksums-storage-key)
|
||||
/p:SourceBranch=$(BUILD.SOURCEBRANCH)
|
||||
displayName: Copy to latest
|
||||
|
|
|
@ -8,11 +8,6 @@
|
|||
<PropertyGroup>
|
||||
<Product>Sdk</Product>
|
||||
<BlobStoragePartialRelativePath>$(Product)</BlobStoragePartialRelativePath>
|
||||
<ChecksumsFeedUrl>$(DotnetPublishChecksumsBlobFeedUrl)</ChecksumsFeedUrl>
|
||||
<SdkAssetsFeedUrl>$(DotnetPublishSdkAssetsBlobFeedUrl)</SdkAssetsFeedUrl>
|
||||
|
||||
<PublishSdkAssetsAndChecksumsToBlob>false</PublishSdkAssetsAndChecksumsToBlob>
|
||||
<PublishSdkAssetsAndChecksumsToBlob Condition=" '$(ChecksumsFeedUrl)' != '' and '$(SdkAssetsFeedUrl)' != '' ">true</PublishSdkAssetsAndChecksumsToBlob>
|
||||
|
||||
<PublishBinariesAndBadge Condition=" '$(PublishBinariesAndBadge)' == '' ">true</PublishBinariesAndBadge>
|
||||
</PropertyGroup>
|
||||
|
@ -70,7 +65,6 @@
|
|||
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.deb" />
|
||||
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.rpm" />
|
||||
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.msi" />
|
||||
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.nupkg" />
|
||||
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.cab" />
|
||||
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.svg" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
|
||||
<!-- Only publish this file from windows x64 so that we don't end up with duplicates -->
|
||||
|
@ -79,7 +73,6 @@
|
|||
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)sdk-productVersion.txt"
|
||||
Condition=" '$(PublishBinariesAndBadge)' == 'true' and '$(OS)' == 'Windows_NT' and '$(Architecture)' == 'x64' and '$(PgoInstrument)' != 'true'" />
|
||||
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)productCommit-*.txt" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
|
||||
<SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.nupkg" />
|
||||
<SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.swr" />
|
||||
<SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.msi" />
|
||||
<SdkNonShippingAssetsToPublish Condition="'$(PublishBinariesAndBadge)' != 'false'" Include="$(ArtifactsNonShippingPackagesDir)*.tar.gz" />
|
||||
|
@ -94,8 +87,7 @@
|
|||
</ItemGroup>
|
||||
|
||||
<Target Name="PublishSdkAssetsAndChecksums"
|
||||
BeforeTargets="Publish"
|
||||
Condition=" '$(PublishSdkAssetsAndChecksumsToBlob)' == 'true' ">
|
||||
BeforeTargets="Publish">
|
||||
|
||||
<ReadLinesFromFile File="$(ArtifactsTmpDir)FullNugetVersion.version">
|
||||
<Output
|
||||
|
@ -123,10 +115,8 @@
|
|||
<ItemsToSignPostBuildWithPaths Include="$(ArtifactsShippingPackagesDir)*.zip" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
|
||||
<ItemsToSignPostBuildWithPaths Include="$(ArtifactsShippingPackagesDir)*.exe" />
|
||||
<ItemsToSignPostBuildWithPaths Include="$(ArtifactsShippingPackagesDir)*.msi" />
|
||||
<ItemsToSignPostBuildWithPaths Include="$(ArtifactsShippingPackagesDir)*.nupkg" />
|
||||
<ItemsToSignPostBuildWithPaths Include="$(ArtifactsNonShippingPackagesDir)*.msi" />
|
||||
<ItemsToSignPostBuildWithPaths Include="$(ArtifactsNonShippingPackagesDir)*.zip" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
|
||||
<ItemsToSignPostBuildWithPaths Include="$(ArtifactsNonShippingPackagesDir)*.nupkg" />
|
||||
<ItemsToSignPostBuild Remove="@(ItemsToSignPostBuild)" />
|
||||
<ItemsToSignPostBuild Include="@(ItemsToSignPostBuildWithPaths->'%(Filename)%(Extension)')" />
|
||||
</ItemGroup>
|
||||
|
|
|
@ -68,8 +68,6 @@ phases:
|
|||
- _PushToVSFeed: true
|
||||
- _SignType: real
|
||||
- _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
|
||||
/p:DotnetPublishSdkAssetsBlobFeedUrl=$(_PublishBlobFeedUrl)
|
||||
/p:DotnetPublishChecksumsBlobFeedUrl=$(_PublishChecksumsBlobFeedUrl)
|
||||
/p:DotNetSignType=$(_SignType)
|
||||
/p:TeamName=$(_TeamName)
|
||||
/p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
|
||||
|
|
|
@ -1,14 +0,0 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="15.0">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>$(CoreSdkTargetFramework)</TargetFramework>
|
||||
<GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
|
||||
<CopyBuildOutputToPublishDirectory>false</CopyBuildOutputToPublishDirectory>
|
||||
<Channel>$(SourceBranch.Replace('refs/heads/', ''))</Channel>
|
||||
</PropertyGroup>
|
||||
|
||||
<Import Project="..\redist\targets\BuildCoreSdkTasks.targets" />
|
||||
<Import Project="..\redist\targets\GetRuntimeInformation.targets" />
|
||||
<Import Project="..\redist\targets\Versions.targets" />
|
||||
|
||||
<Import Project="targets\FinishBuild.targets" />
|
||||
</Project>
|
|
@ -1,18 +0,0 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Target Name="FinalizeBuild"
|
||||
BeforeTargets="AfterBuild"
|
||||
DependsOnTargets="GenerateFullNuGetVersion;GetCommitHash">
|
||||
<CopyBlobsToLatest FeedUrl="$(DotnetPublishSdkAssetsBlobFeedUrl)"
|
||||
AccountKey="$(DotNetPublishSdkAssetsBlobFeedKey)"
|
||||
NugetVersion="$(FullNugetVersion)"
|
||||
Channel="$(Channel)"
|
||||
CommitHash="$(GitCommitHash)" />
|
||||
|
||||
<CopyBlobsToLatest FeedUrl="$(DotNetPublishChecksumsBlobFeedUrl)"
|
||||
AccountKey="$(DotNetPublishChecksumsBlobFeedKey)"
|
||||
NugetVersion="$(FullNugetVersion)"
|
||||
Channel="$(Channel)"
|
||||
CommitHash="$(GitCommitHash)" />
|
||||
</Target>
|
||||
</Project>
|
|
@ -1,137 +0,0 @@
|
|||
// Copyright (c) .NET Foundation and contributors. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
#if !SOURCE_BUILD
|
||||
using Microsoft.Build.Framework;
|
||||
using Microsoft.Build.Utilities;
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build
|
||||
{
|
||||
public class CopyBlobsToLatest : Task
|
||||
{
|
||||
private const string feedRegex = @"(?<feedurl>https:\/\/(?<accountname>[^\.-]+)(?<domain>[^\/]*)\/((?<token>[a-zA-Z0-9+\/]*?\/\d{4}-\d{2}-\d{2})\/)?(?<containername>[^\/]+)\/(?<relativepath>.*\/)?)index\.json";
|
||||
|
||||
private AzurePublisher _azurePublisher;
|
||||
|
||||
[Required]
|
||||
public string FeedUrl { get; set; }
|
||||
|
||||
[Required]
|
||||
public string AccountKey { get; set; }
|
||||
|
||||
[Required]
|
||||
public string Channel { get; set; }
|
||||
|
||||
[Required]
|
||||
public string CommitHash { get; set; }
|
||||
|
||||
[Required]
|
||||
public string NugetVersion { get; set; }
|
||||
|
||||
private string ContainerName { get; set; }
|
||||
|
||||
private AzurePublisher AzurePublisherTool
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_azurePublisher == null)
|
||||
{
|
||||
Match m = Regex.Match(FeedUrl, feedRegex);
|
||||
if (m.Success)
|
||||
{
|
||||
string accountName = m.Groups["accountname"].Value;
|
||||
string ContainerName = m.Groups["containername"].Value;
|
||||
|
||||
_azurePublisher = new AzurePublisher(
|
||||
accountName,
|
||||
AccountKey,
|
||||
ContainerName);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception(
|
||||
"Unable to parse expected feed. Please check ExpectedFeedUrl.");
|
||||
}
|
||||
}
|
||||
|
||||
return _azurePublisher;
|
||||
}
|
||||
}
|
||||
|
||||
public override bool Execute()
|
||||
{
|
||||
if (!(Channel.Equals("master") || Channel.Equals("main") || Channel.StartsWith("release")))
|
||||
{
|
||||
return true; // Skip copy to latest, we don't want to publish to arbitrary places
|
||||
}
|
||||
|
||||
string targetFolder = $"{AzurePublisher.Product.Sdk}/{Channel}";
|
||||
|
||||
string targetVersionFile = $"{targetFolder}/{CommitHash}";
|
||||
string semaphoreBlob = $"{targetFolder}/publishSemaphore";
|
||||
AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob);
|
||||
string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob);
|
||||
|
||||
// Prevent race conditions by dropping a version hint of what version this is. If we see this file
|
||||
// and it is the same as our version then we know that a race happened where two+ builds finished
|
||||
// at the same time and someone already took care of publishing and we have no work to do.
|
||||
if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile))
|
||||
{
|
||||
AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
Regex versionFileRegex = new Regex(@"(?<CommitHash>[\w\d]{40})");
|
||||
|
||||
// Delete old version files
|
||||
AzurePublisherTool.ListBlobs(targetFolder)
|
||||
.Where(s => versionFileRegex.IsMatch(s))
|
||||
.ToList()
|
||||
.ForEach(f => AzurePublisherTool.TryDeleteBlob(f));
|
||||
|
||||
// Drop the version file signaling such for any race-condition builds (see above comment).
|
||||
AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
CopyBlobs(targetFolder);
|
||||
|
||||
string cliVersion = GetVersionFileContent(CommitHash, NugetVersion);
|
||||
AzurePublisherTool.PublishStringToBlob($"{targetFolder}/latest.version", cliVersion);
|
||||
}
|
||||
finally
|
||||
{
|
||||
AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private void CopyBlobs(string destinationFolder)
|
||||
{
|
||||
Log.LogMessage("Copying blobs to {0}/{1}", ContainerName, destinationFolder);
|
||||
|
||||
foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion))
|
||||
{
|
||||
string targetName = Path.GetFileName(blob)
|
||||
.Replace(NugetVersion, "latest");
|
||||
|
||||
string target = $"{destinationFolder}/{targetName}";
|
||||
|
||||
AzurePublisherTool.CopyBlob(blob, target);
|
||||
}
|
||||
}
|
||||
|
||||
private string GetVersionFileContent(string commitHash, string version)
|
||||
{
|
||||
return $@"{commitHash}{Environment.NewLine}{version}{Environment.NewLine}";
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
|
@ -33,7 +33,6 @@
|
|||
<UsingTask TaskName="GenerateDefaultRuntimeFrameworkVersion" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="AddMetadataIsPE" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="Crossgen" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="CopyBlobsToLatest" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="GetRuntimePackRids" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="GenerateMSBuildExtensionsSWR" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="GetLinuxNativeInstallerDependencyVersions" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
|
|
Loading…
Reference in a new issue