diff --git a/eng/Publishing.props b/eng/Publishing.props index bbbcbbf60..388deaf16 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -135,7 +135,12 @@ true + + + diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 115646d29..765452dd3 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -173,7 +173,9 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'toolset', 'VSSdkResolvers')) $([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'AssetManifests')) $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsAssetsDir)', 'Symbols')) $([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt')) $([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored')) diff --git a/src/SourceBuild/content/Directory.Build.targets b/src/SourceBuild/content/Directory.Build.targets index 3dff8e158..9ebfcf3bb 100644 --- a/src/SourceBuild/content/Directory.Build.targets +++ b/src/SourceBuild/content/Directory.Build.targets @@ -7,7 +7,7 @@ dotnet-sdk- - + $(BaseIntermediateOutputPath)$([System.IO.Path]::GetFileName('$(OriginalNuGetConfigFile)')) + $([MSBuild]::NormalizeDirectory('$(AssetManifestsIntermediateDir)', '$(RepositoryName)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-built-sdks')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsShippingPackagesDir)', '$(RepositoryName)')) $(ReferencePackagesDir) $([MSBuild]::NormalizeDirectory('$(ArtifactsNonShippingPackagesDir)', '$(RepositoryName)')) $(ReferencePackagesDir) - $([MSBuild]::ValueOrDefault('$(ARCADE_BOOTSTRAP_VERSION)', '$(ArcadeSdkVersion)')) @@ -86,6 +88,11 @@ $(BuildArgs) /p:RestoreConfigFile=$(NuGetConfigFile) $(BuildArgs) /p:SourceBuildUseMonoRuntime=$(SourceBuildUseMonoRuntime) + + $(BuildArgs) /p:SourceBuiltAssetsDir=$(ArtifactsAssetsDir) + $(BuildArgs) /p:SourceBuiltShippingPackagesDir=$(RepoArtifactsShippingPackagesDir) + $(BuildArgs) /p:SourceBuiltNonShippingPackagesDir=$(RepoArtifactsNonShippingPackagesDir) + $(BuildArgs) /p:SourceBuiltAssetManifestsDir=$(RepoAssetManifestsDir) diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets index 1253d4a2a..7431dc499 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.targets +++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets @@ -24,6 +24,7 @@ $(PackageReportDir)all-project-assets-json-files.zip $(PackageReportDir)prodcon-build.xml $([MSBuild]::NormalizePath('$(ProjectDirectory)', 'artifacts', 'RepoManifest.xml')) + $(ArtifactsAssetsSymbolsDir)Symbols.$(RepositoryName)$(ArchiveExtension) $(IntermediateSymbolsRootDir)$(RepositoryName) @@ -408,9 +409,13 @@ + @@ -601,8 +606,18 @@ + + + + + + + diff --git a/src/SourceBuild/content/repo-projects/aspnetcore.proj b/src/SourceBuild/content/repo-projects/aspnetcore.proj index 24dddd442..3cf6425a4 100644 --- a/src/SourceBuild/content/repo-projects/aspnetcore.proj +++ b/src/SourceBuild/content/repo-projects/aspnetcore.proj @@ -4,8 +4,6 @@ $(ProjectDirectory)eng\build$(ShellExtension) - - $(FlagParameterPrefix)restore $(FlagParameterPrefix)build $(FlagParameterPrefix)pack $(FlagParameterPrefix)restore $(FlagParameterPrefix)all $(FlagParameterPrefix)pack diff --git a/src/SourceBuild/content/repo-projects/installer.proj b/src/SourceBuild/content/repo-projects/installer.proj index 518ca8aad..62083c458 100644 --- a/src/SourceBuild/content/repo-projects/installer.proj +++ b/src/SourceBuild/content/repo-projects/installer.proj @@ -137,4 +137,15 @@ + + + + + + diff --git a/src/SourceBuild/content/repo-projects/source-build-reference-packages.proj b/src/SourceBuild/content/repo-projects/source-build-reference-packages.proj index c7bb4961a..62852f74a 100644 --- a/src/SourceBuild/content/repo-projects/source-build-reference-packages.proj +++ b/src/SourceBuild/content/repo-projects/source-build-reference-packages.proj @@ -31,7 +31,14 @@ SourcePath="$(LocalNuGetPackageCacheDirectory)" /> + ++ ++ ++ <_VmrBuild Condition="'$(DotNetBuildFromSourceFlavor)' == 'Product' or '$(DotNetBuildOrchestrator)' == 'true'">true ++ <_InnerRepoBuild Condition="'$(ArcadeInnerBuildFromSource)' == 'true' or '$(DotNetBuildPhase)' == 'InnerRepo'">true ++ <_ShouldRunPublish Condition="'$(_InnerRepoBuild)' == 'true' and '$(_VmrBuild)' == 'true'">true ++ <_ShouldRunPublish Condition="'$(_InnerRepoBuild)' != 'true' and '$(_VmrBuild)' != 'true'">true ++ ++ ++ + ++ Condition="'$(Publish)' == 'true' and '$(_ShouldRunPublish)' == 'true'"/> + + + +diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj +index 9fa35734..dc83aac9 100644 +--- a/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj ++++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj +@@ -21,6 +21,16 @@ + --> + + ++ ++ ++ ++ true ++ ++ + + + +@@ -41,9 +51,10 @@ + + + ++ + true ++ ('$(DotNetBuildSourceOnly)' != 'true' or ++ ('$(ArcadeInnerBuildFromSource)' == 'true' and '$(DotNetBuildFromSourceFlavor)' != 'Product'))">true + + $(OS) + +@@ -89,6 +100,15 @@ + + + ++ ++ ++ ++ ++ true ++ Symbols/%(Filename)%(Extension) ++ ++ ++ + + + +- +- ++ ++ ++ ++ ++ + NonShipping=true + DotNetReleaseShipping=true + +@@ -163,7 +186,7 @@ + +- ++ IsReleaseOnlyPackageVersion="$(IsReleaseOnlyPackageVersion)" ++ PushToLocalStorage="$(PushToLocalStorage)" ++ AssetsLocalStorageDir="$(SourceBuiltAssetsDir)" ++ ShippingPackagesLocalStorageDir="$(SourceBuiltShippingPackagesDir)" ++ NonShippingPackagesLocalStorageDir="$(SourceBuiltNonShippingPackagesDir)" ++ AssetManifestsLocalStorageDir="$(SourceBuiltAssetManifestsDir)" /> + + +- ++ + + + +@@ -11,22 +11,23 @@ + + + ++ ++ ++ ++ ++ ++ ++ + +- +- +- ++ PackSourceBuildIntermediateNupkgs" /> + + + +diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets +index 1fcd705b..41817c3a 100644 +--- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets ++++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets +@@ -46,6 +46,7 @@ + $([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildSourceDir)', 'artifacts')) + $([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildArtifactsDir)', 'log', '$(Configuration)')) + $([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildArtifactsDir)', 'packages', '$(Configuration)')) ++ $([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildArtifactsPackagesDir)', 'NonShipping')) + + source-build-int-nupkg-cache + +@@ -69,6 +70,7 @@ + <_DefaultNetFrameworkFilter>netstandard2.0%3bnetstandard2.1%3bnetcoreapp2.1%3bnetcoreapp3.1%3bnet5.0%3bnet6.0%3bnet7.0%3bnet8.0%3bnet9.0 + $(_DefaultNetFrameworkFilter) + $(ArtifactsDir)RepoManifest.xml ++ GetCategorizedIntermediateNupkgContents + + + +@@ -159,12 +161,12 @@ + --> + ++ DependsOnTargets="$(CreateRepoSymbolsArchiveDependsOn)"> + + $(CurrentRepoSourceBuildArtifactsDir) + + $(RepoRoot) +- $(CurrentRepoSourceBuildArtifactsPackagesDir) ++ $(CurrentRepoSourceBuildArtifactsNonShippingPackagesDir) + $([MSBuild]::EnsureTrailingSlash('$(PackageOutputPath)')) + $([MSBuild]::NormalizePath('$(SymbolsArchiveLocation)', 'symbols.lst')) + Symbols. +diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadeBuild.targets b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadeBuild.targets +index a4343829..0c38335c 100644 +--- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadeBuild.targets ++++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadeBuild.targets +@@ -80,6 +80,9 @@ + $(InnerBuildArgs) /p:DotNetPackageVersionPropsPath="$(DotNetPackageVersionPropsPath)" + + $(InnerBuildArgs) /p:FullAssemblySigningSupported=$(FullAssemblySigningSupported) ++ ++ ++ $(InnerBuildArgs) /p:DotNetPublishUsingPipelines=true + + + +diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadePublish.targets b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadePublish.targets +index 2b023696..fa81a575 100644 +--- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadePublish.targets ++++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadePublish.targets +@@ -3,23 +3,4 @@ + + + +- +- +- +- +- + +diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadeTools.targets b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadeTools.targets +index ed8e7dfe..180b2ae6 100644 +--- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadeTools.targets ++++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcadeTools.targets +@@ -9,6 +9,7 @@ + + + ++ + + + +@@ -25,10 +28,11 @@ + IsImplicitlyDefined="false" + PrivateAssets="all" + ExcludeAssets="runtime" +- VersionOverride="2.0.3" /> ++ VersionOverride="2.0.3" ++ Condition="'$(DotNetBuildSourceOnly)' != 'true'" /> + + +- ++ + +@@ -41,7 +45,7 @@ + + + +- ++ + + + +@@ -49,4 +53,35 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/build/Microsoft.DotNet.Build.Tasks.Feed.targets b/src/Microsoft.DotNet.Build.Tasks.Feed/build/Microsoft.DotNet.Build.Tasks.Feed.targets +index 7110f878..c58f5a4d 100644 +--- a/src/Microsoft.DotNet.Build.Tasks.Feed/build/Microsoft.DotNet.Build.Tasks.Feed.targets ++++ b/src/Microsoft.DotNet.Build.Tasks.Feed/build/Microsoft.DotNet.Build.Tasks.Feed.targets +@@ -47,7 +47,7 @@ + <_MicrosoftDotNetBuildTasksFeedTaskDir Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)../tools/net9.0/ + + +- ++ + + + +diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +index ccd417c0..1de07e42 100644 +--- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs ++++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs +@@ -29,6 +29,7 @@ + using Newtonsoft.Json; + using NuGet.Versioning; + using static Microsoft.DotNet.Build.Tasks.Feed.GeneralUtils; ++using static Microsoft.DotNet.Build.CloudTestTasks.AzureStorageUtils; + using MsBuildUtils = Microsoft.Build.Utilities; + + namespace Microsoft.DotNet.Build.Tasks.Feed +@@ -1376,10 +1377,10 @@ public void SplitArtifactsInCategories(BuildModel buildModel) + ) + { + // Using these callbacks we can mock up functionality when testing. +- CompareLocalPackageToFeedPackageCallBack ??= GeneralUtils.CompareLocalPackageToFeedPackage; ++ CompareLocalPackageToFeedPackageCallBack ??= CompareLocalPackageToFeedPackage; + RunProcessAndGetOutputsCallBack ??= GeneralUtils.RunProcessAndGetOutputsAsync; + ProcessExecutionResult nugetResult = null; +- var packageStatus = GeneralUtils.PackageFeedStatus.Unknown; ++ var packageStatus = PackageFeedStatus.Unknown; + + try + { +@@ -1395,7 +1396,7 @@ public void SplitArtifactsInCategories(BuildModel buildModel) + if (nugetResult.ExitCode == 0) + { + // We have just pushed this package so we know it exists and is identical to our local copy +- packageStatus = GeneralUtils.PackageFeedStatus.ExistsAndIdenticalToLocal; ++ packageStatus = PackageFeedStatus.ExistsAndIdenticalToLocal; + break; + } + +@@ -1406,12 +1407,12 @@ public void SplitArtifactsInCategories(BuildModel buildModel) + + switch (packageStatus) + { +- case GeneralUtils.PackageFeedStatus.ExistsAndIdenticalToLocal: ++ case PackageFeedStatus.ExistsAndIdenticalToLocal: + { + Log.LogMessage(MessageImportance.Normal, $"Package '{localPackageLocation}' already exists on '{feedConfig.TargetURL}' but has the same content; skipping push"); + break; + } +- case GeneralUtils.PackageFeedStatus.ExistsAndDifferent: ++ case PackageFeedStatus.ExistsAndDifferent: + { + Log.LogError($"Package '{localPackageLocation}' already exists on '{feedConfig.TargetURL}' with different content."); + break; +@@ -1425,11 +1426,11 @@ public void SplitArtifactsInCategories(BuildModel buildModel) + } + } + } +- while (packageStatus != GeneralUtils.PackageFeedStatus.ExistsAndIdenticalToLocal && // Success +- packageStatus != GeneralUtils.PackageFeedStatus.ExistsAndDifferent && // Give up: Non-retriable error ++ while (packageStatus != PackageFeedStatus.ExistsAndIdenticalToLocal && // Success ++ packageStatus != PackageFeedStatus.ExistsAndDifferent && // Give up: Non-retriable error + attemptIndex <= MaxRetryCount); // Give up: Too many retries + +- if (packageStatus != GeneralUtils.PackageFeedStatus.ExistsAndIdenticalToLocal) ++ if (packageStatus != PackageFeedStatus.ExistsAndIdenticalToLocal) + { + Log.LogError($"Failed to publish package '{id}@{version}' to '{feedConfig.TargetURL}' after {MaxRetryCount} attempts. (Final status: {packageStatus})"); + } +@@ -1443,7 +1444,7 @@ public void SplitArtifactsInCategories(BuildModel buildModel) + Log.LogError($"Unexpected exception pushing package '{id}@{version}': {e.Message}"); + } + +- if (packageStatus != GeneralUtils.PackageFeedStatus.ExistsAndIdenticalToLocal && nugetResult?.ExitCode != 0) ++ if (packageStatus != PackageFeedStatus.ExistsAndIdenticalToLocal && nugetResult?.ExitCode != 0) + { + Log.LogError($"Output from nuget.exe: {Environment.NewLine}StdOut:{Environment.NewLine}{nugetResult.StandardOut}{Environment.NewLine}StdErr:{Environment.NewLine}{nugetResult.StandardError}"); + } +diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PushToAzureDevOpsArtifacts.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PushToBuildStorage.cs +similarity index 64% +rename from src/Microsoft.DotNet.Build.Tasks.Feed/src/PushToAzureDevOpsArtifacts.cs +rename to src/Microsoft.DotNet.Build.Tasks.Feed/src/PushToBuildStorage.cs +index 2c1825c2..05c853c7 100644 +--- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PushToAzureDevOpsArtifacts.cs ++++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PushToBuildStorage.cs +@@ -3,6 +3,7 @@ + + using Microsoft.Arcade.Common; + using Microsoft.Build.Framework; ++using Microsoft.Build.Utilities; + using Microsoft.DotNet.VersionTools.Automation; + using Microsoft.DotNet.VersionTools.BuildManifest.Model; + using Microsoft.Extensions.DependencyInjection; +@@ -14,7 +15,7 @@ + + namespace Microsoft.DotNet.Build.Tasks.Feed + { +- public class PushToAzureDevOpsArtifacts : MSBuildTaskBase ++ public class PushToBuildStorage : MSBuildTaskBase + { + [Required] + public ITaskItem[] ItemsToPush { get; set; } +@@ -57,12 +58,29 @@ public class PushToAzureDevOpsArtifacts : MSBuildTaskBase + + public bool IsReleaseOnlyPackageVersion { get; set; } + ++ public string AssetsLocalStorageDir { get; set; } ++ ++ public string ShippingPackagesLocalStorageDir { get; set; } ++ ++ public string NonShippingPackagesLocalStorageDir { get; set; } ++ ++ public string AssetManifestsLocalStorageDir { get; set; } ++ ++ public bool PushToLocalStorage { get; set; } ++ + /// + /// Which version should the build manifest be tagged with. + /// By default he latest version is used. + /// + public string PublishingVersion { get; set; } + ++ public enum ItemType ++ { ++ AssetManifest = 0, ++ PackageArtifact, ++ BlobArtifact ++ } ++ + public override void ConfigureServices(IServiceCollection collection) + { + collection.TryAddSingleton(); +@@ -83,7 +101,22 @@ public override void ConfigureServices(IServiceCollection collection) + { + try + { +- Log.LogMessage(MessageImportance.High, "Performing push to Azure DevOps artifacts storage."); ++ if (PushToLocalStorage) ++ { ++ if (string.IsNullOrEmpty(AssetsLocalStorageDir) || ++ string.IsNullOrEmpty(ShippingPackagesLocalStorageDir) || ++ string.IsNullOrEmpty(NonShippingPackagesLocalStorageDir) || ++ string.IsNullOrEmpty(AssetManifestsLocalStorageDir)) ++ { ++ throw new Exception($"AssetsLocalStorageDir, ShippingPackagesLocalStorageDir, NonShippingPackagesLocalStorageDir and AssetManifestsLocalStorageDir need to be specified if PublishToLocalStorage is set to true"); ++ } ++ ++ Log.LogMessage(MessageImportance.High, "Performing push to local artifacts storage."); ++ } ++ else ++ { ++ Log.LogMessage(MessageImportance.High, "Performing push to Azure DevOps artifacts storage."); ++ } + + if (!string.IsNullOrWhiteSpace(AssetsTemporaryDirectory)) + { +@@ -116,8 +149,7 @@ public override void ConfigureServices(IServiceCollection collection) + continue; + } + +- Log.LogMessage(MessageImportance.High, +- $"##vso[artifact.upload containerfolder=BlobArtifacts;artifactname=BlobArtifacts]{blobItem.ItemSpec}"); ++ PushToLocalStorageOrAzDO(ItemType.BlobArtifact, blobItem); + } + } + else +@@ -159,8 +191,7 @@ public override void ConfigureServices(IServiceCollection collection) + continue; + } + +- Log.LogMessage(MessageImportance.High, +- $"##vso[artifact.upload containerfolder=PackageArtifacts;artifactname=PackageArtifacts]{packagePath.ItemSpec}"); ++ PushToLocalStorageOrAzDO(ItemType.PackageArtifact, packagePath); + } + + foreach (var blobItem in blobItems) +@@ -171,8 +202,7 @@ public override void ConfigureServices(IServiceCollection collection) + continue; + } + +- Log.LogMessage(MessageImportance.High, +- $"##vso[artifact.upload containerfolder=BlobArtifacts;artifactname=BlobArtifacts]{blobItem.ItemSpec}"); ++ PushToLocalStorageOrAzDO(ItemType.BlobArtifact, blobItem); + } + + packageArtifacts = packageItems.Select(packageArtifactModelFactory.CreatePackageArtifactModel); +@@ -206,8 +236,7 @@ public override void ConfigureServices(IServiceCollection collection) + IsReleaseOnlyPackageVersion, + signingInformationModel: signingInformationModel); + +- Log.LogMessage(MessageImportance.High, +- $"##vso[artifact.upload containerfolder=AssetManifests;artifactname=AssetManifests]{AssetManifestPath}"); ++ PushToLocalStorageOrAzDO(ItemType.AssetManifest, new TaskItem(AssetManifestPath)); + } + } + catch (Exception e) +@@ -217,5 +246,73 @@ public override void ConfigureServices(IServiceCollection collection) + + return !Log.HasLoggedErrors; + } ++ ++ private void PushToLocalStorageOrAzDO(ItemType itemType, ITaskItem item) ++ { ++ string path = item.ItemSpec; ++ ++ if (PushToLocalStorage) ++ { ++ string filename = Path.GetFileName(path); ++ switch (itemType) ++ { ++ case ItemType.AssetManifest: ++ Directory.CreateDirectory(AssetManifestsLocalStorageDir); ++ File.Copy(path, Path.Combine(AssetManifestsLocalStorageDir, filename), true); ++ break; ++ ++ case ItemType.PackageArtifact: ++ if (string.Equals(item.GetMetadata("IsShipping"), "true", StringComparison.OrdinalIgnoreCase)) ++ { ++ Directory.CreateDirectory(ShippingPackagesLocalStorageDir); ++ File.Copy(path, Path.Combine(ShippingPackagesLocalStorageDir, filename), true); ++ } ++ else ++ { ++ Directory.CreateDirectory(NonShippingPackagesLocalStorageDir); ++ File.Copy(path, Path.Combine(NonShippingPackagesLocalStorageDir, filename), true); ++ } ++ break; ++ ++ case ItemType.BlobArtifact: ++ string relativeBlobPath = item.GetMetadata("RelativeBlobPath"); ++ string destinationPath = Path.Combine( ++ AssetsLocalStorageDir, ++ string.IsNullOrEmpty(relativeBlobPath) ? filename : relativeBlobPath); ++ ++ Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)); ++ File.Copy(path, destinationPath, true); ++ break; ++ ++ default: ++ throw new ArgumentOutOfRangeException(nameof(itemType)); ++ } ++ } ++ else ++ { ++ // Push to AzDO artifacts storage ++ ++ switch (itemType) ++ { ++ case ItemType.AssetManifest: ++ Log.LogMessage(MessageImportance.High, ++ $"##vso[artifact.upload containerfolder=AssetManifests;artifactname=AssetManifests]{path}"); ++ break; ++ ++ case ItemType.PackageArtifact: ++ Log.LogMessage(MessageImportance.High, ++ $"##vso[artifact.upload containerfolder=PackageArtifacts;artifactname=PackageArtifacts]{path}"); ++ break; ++ ++ case ItemType.BlobArtifact: ++ Log.LogMessage(MessageImportance.High, ++ $"##vso[artifact.upload containerfolder=BlobArtifacts;artifactname=BlobArtifacts]{path}"); ++ break; ++ ++ default: ++ throw new ArgumentOutOfRangeException(nameof(itemType)); ++ } ++ } ++ } + } + } +diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/AzureStorageUtils.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/AzureStorageUtils.cs +index b126fa4c..c805c6b4 100644 +--- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/AzureStorageUtils.cs ++++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/AzureStorageUtils.cs +@@ -8,6 +8,8 @@ + using Azure.Storage.Blobs.Specialized; + using Azure.Storage.Sas; + using Microsoft.Arcade.Common; ++using Microsoft.Build.Framework; ++using MsBuildUtils = Microsoft.Build.Utilities; + using System; + using System.Collections.Generic; + using System.IO; +@@ -33,6 +35,17 @@ public class AzureStorageUtils + {".svg", "no-cache"} + }; + ++ /// ++ /// Enum describing the states of a given package on a feed ++ /// ++ public enum PackageFeedStatus ++ { ++ DoesNotExist, ++ ExistsAndIdenticalToLocal, ++ ExistsAndDifferent, ++ Unknown ++ } ++ + // Save the credential so we can sign SAS tokens + private readonly StorageSharedKeyCredential _credential; + +@@ -168,5 +181,124 @@ public static BlobHttpHeaders GetBlobHeadersByExtension(string filePath) + + return headers; + } ++ ++ /// ++ /// Determine whether a local package is the same as a package on an AzDO feed. ++ /// ++ /// ++ /// ++ /// ++ /// ++ /// ++ /// ++ /// Open a stream to the local file and an http request to the package. There are a couple possibilities: ++ /// - The returned headers include a content MD5 header, in which case we can ++ /// hash the local file and just compare those. ++ /// - No content MD5 hash, and the streams must be compared in blocks. This is a bit trickier to do efficiently, ++ /// since we do not necessarily want to read all bytes if we can help it. Thus, we should compare in blocks. However, ++ /// the streams make no guarantee that they will return a full block each time when read operations are performed, so we ++ /// must be sure to only compare the minimum number of bytes returned. ++ /// ++ public static async Task CompareLocalPackageToFeedPackage( ++ string localPackageFullPath, ++ string packageContentUrl, ++ HttpClient client, ++ MsBuildUtils.TaskLoggingHelper log) ++ { ++ return await CompareLocalPackageToFeedPackage( ++ localPackageFullPath, ++ packageContentUrl, ++ client, ++ log, ++ GeneralUtils.CreateDefaultRetryHandler()); ++ } ++ ++ /// ++ /// Determine whether a local package is the same as a package on an AzDO feed. ++ /// ++ /// ++ /// ++ /// ++ /// ++ /// ++ /// ++ /// ++ /// Open a stream to the local file and an http request to the package. There are a couple possibilities: ++ /// - The returned headers include a content MD5 header, in which case we can ++ /// hash the local file and just compare those. ++ /// - No content MD5 hash, and the streams must be compared in blocks. This is a bit trickier to do efficiently, ++ /// since we do not necessarily want to read all bytes if we can help it. Thus, we should compare in blocks. However, ++ /// the streams make no guarantee that they will return a full block each time when read operations are performed, so we ++ /// must be sure to only compare the minimum number of bytes returned. ++ /// ++ public static async Task CompareLocalPackageToFeedPackage( ++ string localPackageFullPath, ++ string packageContentUrl, ++ HttpClient client, ++ MsBuildUtils.TaskLoggingHelper log, ++ IRetryHandler retryHandler) ++ { ++ log.LogMessage($"Getting package content from {packageContentUrl} and comparing to {localPackageFullPath}"); ++ ++ PackageFeedStatus result = PackageFeedStatus.Unknown; ++ ++ bool success = await retryHandler.RunAsync(async attempt => ++ { ++ try ++ { ++ using (Stream localFileStream = File.OpenRead(localPackageFullPath)) ++ using (HttpResponseMessage response = await client.GetAsync(packageContentUrl)) ++ { ++ response.EnsureSuccessStatusCode(); ++ ++ // Check the headers for content length and md5 ++ bool md5HeaderAvailable = response.Headers.TryGetValues("Content-MD5", out var md5); ++ bool lengthHeaderAvailable = response.Headers.TryGetValues("Content-Length", out var contentLength); ++ ++ if (lengthHeaderAvailable && long.Parse(contentLength.Single()) != localFileStream.Length) ++ { ++ log.LogMessage(MessageImportance.Low, $"Package '{localPackageFullPath}' has different length than remote package '{packageContentUrl}'."); ++ result = PackageFeedStatus.ExistsAndDifferent; ++ return true; ++ } ++ ++ if (md5HeaderAvailable) ++ { ++ var localMD5 = AzureStorageUtils.CalculateMD5(localPackageFullPath); ++ if (!localMD5.Equals(md5.Single(), StringComparison.OrdinalIgnoreCase)) ++ { ++ log.LogMessage(MessageImportance.Low, $"Package '{localPackageFullPath}' has different MD5 hash than remote package '{packageContentUrl}'."); ++ } ++ ++ result = PackageFeedStatus.ExistsAndDifferent; ++ return true; ++ } ++ ++ const int BufferSize = 64 * 1024; ++ ++ // Otherwise, compare the streams ++ var remoteStream = await response.Content.ReadAsStreamAsync(); ++ var streamsMatch = await GeneralUtils.CompareStreamsAsync(localFileStream, remoteStream, BufferSize); ++ result = streamsMatch ? PackageFeedStatus.ExistsAndIdenticalToLocal : PackageFeedStatus.ExistsAndDifferent; ++ return true; ++ } ++ } ++ // String based comparison because the status code isn't exposed in HttpRequestException ++ // see here: https://github.com/dotnet/runtime/issues/23648 ++ catch (Exception e) when (e is HttpRequestException || e is TaskCanceledException) ++ { ++ if (e.Message.Contains("404 (Not Found)")) ++ { ++ result = PackageFeedStatus.DoesNotExist; ++ return true; ++ } ++ ++ // Retry this. Could be an http client timeout, 500, etc. ++ return false; ++ } ++ }); ++ ++ return result; ++ } + } + } +diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs +index ab8b35f7..a361aa45 100644 +--- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs ++++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/common/GeneralUtils.cs +@@ -4,7 +4,9 @@ + using Microsoft.Arcade.Common; + using Microsoft.Build.Framework; + using Microsoft.Build.Utilities; ++#if !DOTNET_BUILD_SOURCE_ONLY + using Microsoft.DotNet.Build.CloudTestTasks; ++#endif + using System; + using System.Collections.Generic; + using System.Diagnostics; +@@ -31,17 +33,6 @@ public static ExponentialRetry CreateDefaultRetryHandler() + MaxAttempts = 5 + }; + +- /// +- /// Enum describing the states of a given package on a feed +- /// +- public enum PackageFeedStatus +- { +- DoesNotExist, +- ExistsAndIdenticalToLocal, +- ExistsAndDifferent, +- Unknown +- } +- + /// + /// Compare a local stream and a remote stream for quality + /// +@@ -119,125 +110,6 @@ public static async Task CompareStreamsAsync(Stream localFileStream, Strea + while (true); + } + +- /// +- /// Determine whether a local package is the same as a package on an AzDO feed. +- /// +- /// +- /// +- /// +- /// +- /// +- /// +- /// Open a stream to the local file and an http request to the package. There are a couple possibilities: +- /// - The returned headers include a content MD5 header, in which case we can +- /// hash the local file and just compare those. +- /// - No content MD5 hash, and the streams must be compared in blocks. This is a bit trickier to do efficiently, +- /// since we do not necessarily want to read all bytes if we can help it. Thus, we should compare in blocks. However, +- /// the streams make no guarantee that they will return a full block each time when read operations are performed, so we +- /// must be sure to only compare the minimum number of bytes returned. +- /// +- public static async Task CompareLocalPackageToFeedPackage( +- string localPackageFullPath, +- string packageContentUrl, +- HttpClient client, +- TaskLoggingHelper log) +- { +- return await CompareLocalPackageToFeedPackage( +- localPackageFullPath, +- packageContentUrl, +- client, +- log, +- CreateDefaultRetryHandler()); +- } +- +- /// +- /// Determine whether a local package is the same as a package on an AzDO feed. +- /// +- /// +- /// +- /// +- /// +- /// +- /// +- /// +- /// Open a stream to the local file and an http request to the package. There are a couple possibilities: +- /// - The returned headers include a content MD5 header, in which case we can +- /// hash the local file and just compare those. +- /// - No content MD5 hash, and the streams must be compared in blocks. This is a bit trickier to do efficiently, +- /// since we do not necessarily want to read all bytes if we can help it. Thus, we should compare in blocks. However, +- /// the streams make no guarantee that they will return a full block each time when read operations are performed, so we +- /// must be sure to only compare the minimum number of bytes returned. +- /// +- public static async Task CompareLocalPackageToFeedPackage( +- string localPackageFullPath, +- string packageContentUrl, +- HttpClient client, +- TaskLoggingHelper log, +- IRetryHandler retryHandler) +- { +- log.LogMessage($"Getting package content from {packageContentUrl} and comparing to {localPackageFullPath}"); +- +- PackageFeedStatus result = PackageFeedStatus.Unknown; +- +- bool success = await retryHandler.RunAsync(async attempt => +- { +- try +- { +- using (Stream localFileStream = File.OpenRead(localPackageFullPath)) +- using (HttpResponseMessage response = await client.GetAsync(packageContentUrl)) +- { +- response.EnsureSuccessStatusCode(); +- +- // Check the headers for content length and md5 +- bool md5HeaderAvailable = response.Headers.TryGetValues("Content-MD5", out var md5); +- bool lengthHeaderAvailable = response.Headers.TryGetValues("Content-Length", out var contentLength); +- +- if (lengthHeaderAvailable && long.Parse(contentLength.Single()) != localFileStream.Length) +- { +- log.LogMessage(MessageImportance.Low, $"Package '{localPackageFullPath}' has different length than remote package '{packageContentUrl}'."); +- result = PackageFeedStatus.ExistsAndDifferent; +- return true; +- } +- +- if (md5HeaderAvailable) +- { +- var localMD5 = AzureStorageUtils.CalculateMD5(localPackageFullPath); +- if (!localMD5.Equals(md5.Single(), StringComparison.OrdinalIgnoreCase)) +- { +- log.LogMessage(MessageImportance.Low, $"Package '{localPackageFullPath}' has different MD5 hash than remote package '{packageContentUrl}'."); +- } +- +- result = PackageFeedStatus.ExistsAndDifferent; +- return true; +- } +- +- const int BufferSize = 64 * 1024; +- +- // Otherwise, compare the streams +- var remoteStream = await response.Content.ReadAsStreamAsync(); +- var streamsMatch = await GeneralUtils.CompareStreamsAsync(localFileStream, remoteStream, BufferSize); +- result = streamsMatch ? PackageFeedStatus.ExistsAndIdenticalToLocal : PackageFeedStatus.ExistsAndDifferent; +- return true; +- } +- } +- // String based comparison because the status code isn't exposed in HttpRequestException +- // see here: https://github.com/dotnet/runtime/issues/23648 +- catch (Exception e) when (e is HttpRequestException || e is TaskCanceledException) +- { +- if (e.Message.Contains("404 (Not Found)")) +- { +- result = PackageFeedStatus.DoesNotExist; +- return true; +- } +- +- // Retry this. Could be an http client timeout, 500, etc. +- return false; +- } +- }); +- +- return result; +- } +- + /// + /// Determine whether the feed is public or private. + /// +diff --git a/src/Microsoft.DotNet.VersionTools/Directory.Build.props b/src/Microsoft.DotNet.VersionTools/Directory.Build.props +deleted file mode 100644 +index 2dc829c3..00000000 +--- a/src/Microsoft.DotNet.VersionTools/Directory.Build.props ++++ /dev/null +@@ -1,10 +0,0 @@ +- +- +- +- +- +- +- true +- +- +- +diff --git a/src/Microsoft.DotNet.VersionTools/lib/src/Automation/NupkgInfo.cs b/src/Microsoft.DotNet.VersionTools/lib/src/Automation/NupkgInfo.cs +index c3646c58..4e4aed0a 100644 +--- a/src/Microsoft.DotNet.VersionTools/lib/src/Automation/NupkgInfo.cs ++++ b/src/Microsoft.DotNet.VersionTools/lib/src/Automation/NupkgInfo.cs +@@ -1,7 +1,7 @@ + // Licensed to the .NET Foundation under one or more agreements. + // The .NET Foundation licenses this file to you under the MIT license. + +-using NuGet.Packaging.Core; ++using System; + + namespace Microsoft.DotNet.VersionTools.Automation + { +@@ -10,14 +10,40 @@ public class NupkgInfo + public NupkgInfo(PackageIdentity identity) + { + Id = identity.Id; +- Version = identity.Version.ToString(); +- Prerelease = identity.Version.Release; ++ Version = identity.Version; + } + + public string Id { get; } + public string Version { get; } +- public string Prerelease { get; } ++ public string Prerelease { get { throw new NotImplementedException();} } + + public static bool IsSymbolPackagePath(string path) => path.EndsWith(".symbols.nupkg"); + } ++ ++ public class PackageIdentity ++ { ++ private readonly string _id; ++ private readonly string _version; ++ ++ public PackageIdentity(string id, string version) ++ { ++ if (id == null) ++ { ++ throw new ArgumentNullException(nameof(id)); ++ } ++ ++ _id = id; ++ _version = version; ++ } ++ ++ public string Id ++ { ++ get { return _id; } ++ } ++ ++ public string Version ++ { ++ get { return _version; } ++ } ++ } + } +diff --git a/src/Microsoft.DotNet.VersionTools/lib/src/Automation/NupkgInfoFactory.cs b/src/Microsoft.DotNet.VersionTools/lib/src/Automation/NupkgInfoFactory.cs +index ddf8dd99..eab28d72 100644 +--- a/src/Microsoft.DotNet.VersionTools/lib/src/Automation/NupkgInfoFactory.cs ++++ b/src/Microsoft.DotNet.VersionTools/lib/src/Automation/NupkgInfoFactory.cs +@@ -1,8 +1,12 @@ + // Licensed to the .NET Foundation under one or more agreements. + // The .NET Foundation licenses this file to you under the MIT license. + +-using NuGet.Packaging; +-using NuGet.Packaging.Core; ++using System; ++using System.Globalization; ++using System.IO; ++using System.IO.Compression; ++using System.Linq; ++using System.Xml.Linq; + + namespace Microsoft.DotNet.VersionTools.Automation + { +@@ -22,10 +26,59 @@ public NupkgInfoFactory(IPackageArchiveReaderFactory packageArchiveReaderFactory + + public NupkgInfo CreateNupkgInfo(string path) + { +- using PackageArchiveReader archiveReader = _packageArchiveReaderFactory.CreatePackageArchiveReader(path); +- PackageIdentity identity = archiveReader.GetIdentity(); ++ if (path == null) ++ { ++ throw new ArgumentNullException(nameof(path)); ++ } + +- return new NupkgInfo(identity); ++ Stream stream = null; ++ Stream ZipReadStream = null; ++ ZipArchive _zipArchive; ++ try ++ { ++ stream = File.OpenRead(path); ++ ZipReadStream = stream; ++ _zipArchive = new ZipArchive(stream, ZipArchiveMode.Read); ++ string nuspecFile = Path.GetTempFileName(); ++ foreach (ZipArchiveEntry entry in _zipArchive.Entries) ++ { ++ if (entry.Name.EndsWith(".nuspec")) ++ { ++ Directory.CreateDirectory(Path.GetDirectoryName(nuspecFile)); ++ entry.ExtractToFile(nuspecFile, true); ++ } ++ } ++ ++ if (!File.Exists(nuspecFile)) ++ { ++ throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, "Did not extract nuspec file from package: {0}", path)); ++ } ++ ++ PackageIdentity identity = GetIdentity(nuspecFile); ++ return new NupkgInfo(identity); ++ } ++ catch (Exception ex) ++ { ++ stream?.Dispose(); ++ throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, "Invalid package", path), ex); ++ } ++ } ++ ++ private static PackageIdentity GetIdentity(string nuspecFile) ++ { ++ if (nuspecFile == null) ++ { ++ throw new ArgumentNullException(nameof(nuspecFile)); ++ } ++ ++ XDocument doc = XDocument.Load(nuspecFile, LoadOptions.PreserveWhitespace); ++ XElement metadataElement = GetSingleElement(doc.Root, "metadata"); ++ return new PackageIdentity(GetSingleElement(metadataElement, "id").Value, GetSingleElement(metadataElement, "version").Value); ++ } ++ ++ private static XElement GetSingleElement(XElement el, string name) ++ { ++ return el.Descendants().First(c => c.Name.LocalName.ToString() == name); + } + } + } +diff --git a/src/Microsoft.DotNet.VersionTools/tasks/Microsoft.DotNet.VersionTools.Tasks.csproj b/src/Microsoft.DotNet.VersionTools/tasks/Microsoft.DotNet.VersionTools.Tasks.csproj +index 2013e9da..72277ea5 100644 +--- a/src/Microsoft.DotNet.VersionTools/tasks/Microsoft.DotNet.VersionTools.Tasks.csproj ++++ b/src/Microsoft.DotNet.VersionTools/tasks/Microsoft.DotNet.VersionTools.Tasks.csproj +@@ -7,6 +7,11 @@ + MSBuildSdk + + ++ ++ ++ true ++ ++ + + + +diff --git a/src/Microsoft.DotNet.VersionTools/tools/Microsoft.DotNet.VersionTools.Cli.Tests/VersionTrimmingOperationTests.cs b/src/Microsoft.DotNet.VersionTools/tools/Microsoft.DotNet.VersionTools.Cli.Tests/VersionTrimmingOperationTests.cs +index 7a1b83c1..fb1be051 100644 +--- a/src/Microsoft.DotNet.VersionTools/tools/Microsoft.DotNet.VersionTools.Cli.Tests/VersionTrimmingOperationTests.cs ++++ b/src/Microsoft.DotNet.VersionTools/tools/Microsoft.DotNet.VersionTools.Cli.Tests/VersionTrimmingOperationTests.cs +@@ -4,7 +4,6 @@ + using FluentAssertions; + using Microsoft.DotNet.VersionTools.Automation; + using Moq; +-using NuGet.Packaging.Core; + using NuGet.Versioning; + using System.IO; + using System; +@@ -22,7 +21,7 @@ public void TestRemoveVersionFromFileNames() + { + var nupkgInfoFactory = new Mock(); + nupkgInfoFactory.Setup(m => m.CreateNupkgInfo(It.IsAny())) +- .Returns(new NupkgInfo(new PackageIdentity("id", new NuGetVersion("8.0.0-dev")))); ++ .Returns(new NupkgInfo(new PackageIdentity("id", "8.0.0-dev"))); + + var fileProxy = new Mock(); + fileProxy.Setup(m => m.Move(It.IsAny(), It.IsAny())).Verifiable(); +diff --git a/src/Microsoft.DotNet.VersionTools/tools/Microsoft.DotNet.VersionTools.Cli/Microsoft.DotNet.VersionTools.Cli.csproj b/src/Microsoft.DotNet.VersionTools/tools/Microsoft.DotNet.VersionTools.Cli/Microsoft.DotNet.VersionTools.Cli.csproj +index 3f28b805..a2f9360e 100644 +--- a/src/Microsoft.DotNet.VersionTools/tools/Microsoft.DotNet.VersionTools.Cli/Microsoft.DotNet.VersionTools.Cli.csproj ++++ b/src/Microsoft.DotNet.VersionTools/tools/Microsoft.DotNet.VersionTools.Cli/Microsoft.DotNet.VersionTools.Cli.csproj +@@ -9,6 +9,11 @@ + This package provides a CLI interface to the Microsoft.DotNet.VersionTools library. + + ++ ++ ++ true ++ ++ + + + diff --git a/src/SourceBuild/patches/aspnetcore/0001-Enable-publishing-in-VMR.patch b/src/SourceBuild/patches/aspnetcore/0001-Enable-publishing-in-VMR.patch new file mode 100644 index 000000000..9c3195f46 --- /dev/null +++ b/src/SourceBuild/patches/aspnetcore/0001-Enable-publishing-in-VMR.patch @@ -0,0 +1,87 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Nikola Milosavljevic +Date: Thu, 7 Mar 2024 21:11:28 +0000 +Subject: [PATCH] Enable publishing in VMR + +Backport: https://github.com/dotnet/aspnetcore/pull/54422 +--- + eng/Publishing.props | 10 +++++++++- + eng/build.sh | 9 +++++++++ + 2 files changed, 18 insertions(+), 1 deletion(-) + +diff --git a/eng/Publishing.props b/eng/Publishing.props +index 7edbb21d21..d00d8d91d2 100644 +--- a/eng/Publishing.props ++++ b/eng/Publishing.props +@@ -38,6 +38,8 @@ + + <_InstallersToPublish Remove="@(WixPacksToPublish)" Condition="'$(PostBuildSign)' != 'true'" /> + <_ChecksumsToPublish Include="$(ArtifactsDir)installers\**\*.sha512" /> ++ ++ <_VersionFileToPublish Include="$(ArtifactsDir)installers\**\aspnetcore_base_runtime.version" UploadPathSegment="Runtime" Condition="'$(DotNetBuildRepo)' == 'true'" /> + + + +- ++ + + + NonShipping=true +@@ -72,6 +74,12 @@ + $(_UploadPathRoot)/Runtime/$(_PackageVersion)/%(Filename)%(Extension) + + ++ ++ NonShipping=true ++ true ++ $(_UploadPathRoot)/Runtime/$(_PackageVersion)/%(Filename)%(Extension) ++ ++ + + NonShipping=false + true +diff --git a/eng/build.sh b/eng/build.sh +index 51a2cd6ecd..cfc42f1ceb 100755 +--- a/eng/build.sh ++++ b/eng/build.sh +@@ -18,6 +18,7 @@ verbosity='minimal' + run_restore='' + run_build=true + run_pack=false ++run_publish=false + run_tests=false + build_all=false + build_deps=true +@@ -62,6 +63,7 @@ Options: + --[no-]build Compile projects. (Implies --no-restore) + --[no-]pack Produce packages. + --[no-]test Run tests. ++ --[no-]publish Run publish. + + --projects A list of projects to build. (Must be an absolute path.) + Globbing patterns are supported, such as \"$(pwd)/**/*.csproj\". +@@ -152,6 +154,12 @@ while [[ $# -gt 0 ]]; do + -no-pack|-nopack) + run_pack=false + ;; ++ -publish) ++ run_publish=true ++ ;; ++ -no-publish|-nopublish) ++ run_publish=false ++ ;; + -test|-t) + run_tests=true + ;; +@@ -286,6 +294,7 @@ if [ "$run_build" = false ]; then + msbuild_args[${#msbuild_args[*]}]="-p:NoBuild=true" + fi + msbuild_args[${#msbuild_args[*]}]="-p:Pack=$run_pack" ++msbuild_args[${#msbuild_args[*]}]="-p:Publish=$run_publish" + msbuild_args[${#msbuild_args[*]}]="-p:Test=$run_tests" + + msbuild_args[${#msbuild_args[*]}]="-p:TargetArchitecture=$target_arch" diff --git a/src/SourceBuild/patches/fsharp/0001-Enable-publishing-in-VMR.patch b/src/SourceBuild/patches/fsharp/0001-Enable-publishing-in-VMR.patch new file mode 100644 index 000000000..ec363d7c4 --- /dev/null +++ b/src/SourceBuild/patches/fsharp/0001-Enable-publishing-in-VMR.patch @@ -0,0 +1,42 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Nikola Milosavljevic +Date: Thu, 7 Mar 2024 21:41:56 +0000 +Subject: [PATCH] Enable publishing in VMR + +Backport: https://github.com/dotnet/fsharp/pull/16838 +--- + eng/Publishing.props | 22 ++++++++++++++++++++++ + 1 file changed, 22 insertions(+) + +diff --git a/eng/Publishing.props b/eng/Publishing.props +index 18483e92a..90a2b9cbf 100644 +--- a/eng/Publishing.props ++++ b/eng/Publishing.props +@@ -1,5 +1,27 @@ + + + 3 ++ $(PublishDependsOnTargets);_PublishPackages + ++ ++ ++ <_PackagesToPublish Remove="@(_PackagesToPublish)" /> ++ <_PackagesToPublish Include="$(ArtifactsPackagesDir)**\*.nupkg" UploadPathSegment="Runtime" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ ++ ++ ++ ++ ++ ++ ++ ++ true ++ ++ ++ ++ + diff --git a/src/SourceBuild/patches/nuget-client/0001-Enable-publishing-in-VMR.patch b/src/SourceBuild/patches/nuget-client/0001-Enable-publishing-in-VMR.patch new file mode 100644 index 000000000..ba245c4ce --- /dev/null +++ b/src/SourceBuild/patches/nuget-client/0001-Enable-publishing-in-VMR.patch @@ -0,0 +1,109 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Nikola Milosavljevic +Date: Thu, 7 Mar 2024 21:20:16 +0000 +Subject: [PATCH] Enable publishing in VMR + +Backport: https://github.com/NuGet/NuGet.Client/pull/5673 +--- + Directory.Packages.props | 1 + + eng/Publishing.props | 31 ++++++++++++++++++++++++++ + eng/source-build/source-build.proj | 35 ++++++++++++++++++++++++++++++ + 3 files changed, 67 insertions(+) + create mode 100644 eng/Publishing.props + +diff --git a/Directory.Packages.props b/Directory.Packages.props +index 5de784200..789f9493f 100644 +--- a/Directory.Packages.props ++++ b/Directory.Packages.props +@@ -179,6 +179,7 @@ + <_allowBuildFromSourcePackage Include="Microsoft.Build.Utilities.Core" /> + <_allowBuildFromSourcePackage Include="Microsoft.Build" /> + <_allowBuildFromSourcePackage Include="Microsoft.CSharp" /> ++ <_allowBuildFromSourcePackage Include="Microsoft.DotNet.Build.Tasks.Feed" /> + <_allowBuildFromSourcePackage Include="Microsoft.Extensions.CommandLineUtils.Sources" /> + <_allowBuildFromSourcePackage Include="Microsoft.Extensions.FileProviders.Abstractions" /> + <_allowBuildFromSourcePackage Include="Microsoft.Extensions.FileSystemGlobbing" /> +diff --git a/eng/Publishing.props b/eng/Publishing.props +new file mode 100644 +index 000000000..81f0204a7 +--- /dev/null ++++ b/eng/Publishing.props +@@ -0,0 +1,31 @@ ++ ++ ++ ++ 3 ++ true ++ $(PublishDependsOnTargets);_PublishPackages ++ ++ ++ ++ <_PackagesToPublish Remove="@(_PackagesToPublish)" /> ++ <_PackagesToPublish Include="$(ArtifactsDir)nupkgs/*.nupkg" UploadPathSegment="nuget-client" Condition="'$(DotNetBuildFromSource)' == 'true'" /> ++ <_SymbolsPackages Include="$(ArtifactsDir)nupkgs/*.symbols.nupkg" /> ++ <_PackagesToPublish Remove="@(_SymbolsPackages)" Condition="'$(DotNetBuildSourceOnly)' == 'true'" /> ++ ++ ++ ++ ++ ++ ++ ++ ++ true ++ ++ ++ ++ ++ +\ No newline at end of file +diff --git a/eng/source-build/source-build.proj b/eng/source-build/source-build.proj +index 612811d81..3485a4212 100644 +--- a/eng/source-build/source-build.proj ++++ b/eng/source-build/source-build.proj +@@ -62,6 +62,41 @@ + Properties="Configuration=$(BuildConfiguration);DotNetBuildFromSource=true" + Targets="PackXPlat" /> + ++ ++ <_AfterSourceBuildProperties Include="_NETCORE_ENGINEERING_TELEMETRY=AfterSourceBuild" /> ++ <_AfterSourceBuildProperties Include="ArcadeBuildFromSource=true"/> ++ <_AfterSourceBuildProperties Include="ArcadeInnerBuildFromSource=true"/> ++ ++ ++ ++ ++ ++ <_PublishProperties Include="_NETCORE_ENGINEERING_TELEMETRY=Publish" /> ++ <_PublishProperties Include="Configuration=$(Configuration)" /> ++ <_PublishProperties Include="ArcadeBuildFromSource=$(ArcadeBuildFromSource)" /> ++ <_PublishProperties Include="ArcadeInnerBuildFromSource=true" /> ++ <_PublishProperties Include="DotNetBuildFromSource=$(DotNetBuildFromSource)" /> ++ <_PublishProperties Include="DotNetBuildFromSourceFlavor=$(DotNetBuildFromSourceFlavor)" /> ++ <_PublishProperties Include="DotNetBuildInnerRepo=true" /> ++ <_PublishProperties Include="DotNetBuildOrchestrator=$(DotNetBuildOrchestrator)" /> ++ <_PublishProperties Include="DotNetBuildPhase=InnerRepo" /> ++ <_PublishProperties Include="DotNetBuildRepo=$(DotNetBuildRepo)" /> ++ <_PublishProperties Include="DotNetBuildSourceOnly=$(DotNetBuildSourceOnly)" /> ++ <_PublishProperties Include="DotNetPublishUsingPipelines=true" /> ++ <_PublishProperties Include="PublishToSymbolServer=false" /> ++ <_PublishProperties Include="AssetsLocalStorageDir=$(SourceBuiltAssetsDir)" /> ++ <_PublishProperties Include="ShippingPackagesLocalStorageDir=$(SourceBuiltShippingPackagesDir)" /> ++ <_PublishProperties Include="NonShippingPackagesLocalStorageDir=$(SourceBuiltNonShippingPackagesDir)" /> ++ <_PublishProperties Include="AssetManifestsLocalStorageDir=$(SourceBuiltAssetManifestsDir)" /> ++ ++ ++ ++ + + + diff --git a/src/SourceBuild/patches/roslyn-analyzers/0001-Enable-publishing-in-VMR.patch b/src/SourceBuild/patches/roslyn-analyzers/0001-Enable-publishing-in-VMR.patch new file mode 100644 index 000000000..034499bc5 --- /dev/null +++ b/src/SourceBuild/patches/roslyn-analyzers/0001-Enable-publishing-in-VMR.patch @@ -0,0 +1,50 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Nikola Milosavljevic +Date: Thu, 7 Mar 2024 21:52:02 +0000 +Subject: [PATCH] Enable publishing in VMR + +Backport: https://github.com/dotnet/roslyn-analyzers/pull/7233 +--- + eng/Publishing.props | 32 +++++++++++++++++++++++++++----- + 1 file changed, 27 insertions(+), 5 deletions(-) + +diff --git a/eng/Publishing.props b/eng/Publishing.props +index 579a1360d..c71638ecb 100644 +--- a/eng/Publishing.props ++++ b/eng/Publishing.props +@@ -1,7 +1,29 @@ + + +- +- 3 +- true +- +- ++ ++ 3 ++ true ++ $(PublishDependsOnTargets);_PublishPackages ++ ++ ++ ++ <_PackagesToPublish Remove="@(_PackagesToPublish)" /> ++ <_PackagesToPublish Include="$(ArtifactsPackagesDir)**\*.nupkg" UploadPathSegment="Roslyn-analyzers" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ ++ ++ ++ ++ ++ ++ ++ ++ true ++ ++ ++ ++ ++ +\ No newline at end of file diff --git a/src/SourceBuild/patches/runtime/0001-Enable-publishing-in-VMR.patch b/src/SourceBuild/patches/runtime/0001-Enable-publishing-in-VMR.patch new file mode 100644 index 000000000..978aaf745 --- /dev/null +++ b/src/SourceBuild/patches/runtime/0001-Enable-publishing-in-VMR.patch @@ -0,0 +1,104 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Nikola Milosavljevic +Date: Thu, 7 Mar 2024 21:59:41 +0000 +Subject: [PATCH] Enable publishing in VMR + +Backport: https://github.com/dotnet/runtime/pull/99432 +--- + eng/DotNetBuild.props | 10 ++++++++ + eng/Publishing.props | 54 +++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 64 insertions(+) + +diff --git a/eng/DotNetBuild.props b/eng/DotNetBuild.props +index a6350c7fea9..c7ee305fecc 100644 +--- a/eng/DotNetBuild.props ++++ b/eng/DotNetBuild.props +@@ -45,6 +45,10 @@ + ++ ++ ++ $(InnerBuildArgs) $(FlagParameterPrefix)restore $(FlagParameterPrefix)build $(FlagParameterPrefix)publish ++ + $(InnerBuildArgs) $(FlagParameterPrefix)arch $(TargetArch) + $(InnerBuildArgs) $(FlagParameterPrefix)os $(TargetOS) + $(InnerBuildArgs) $(FlagParameterPrefix)cross +@@ -74,6 +78,12 @@ + $(InnerBuildArgs) /p:ContinuousIntegrationBuild=$(ContinuousIntegrationBuild) + $(InnerBuildArgs) /p:PortableBuild=$(PortableBuild) + $(InnerBuildArgs) /p:RestoreConfigFile=$(RestoreConfigFile) ++ ++ ++ $(InnerBuildArgs) /p:SourceBuiltAssetsDir=$(SourceBuiltAssetsDir) ++ $(InnerBuildArgs) /p:SourceBuiltShippingPackagesDir=$(SourceBuiltShippingPackagesDir) ++ $(InnerBuildArgs) /p:SourceBuiltNonShippingPackagesDir=$(SourceBuiltNonShippingPackagesDir) ++ $(InnerBuildArgs) /p:SourceBuiltAssetManifestsDir=$(SourceBuiltAssetManifestsDir) + + + +diff --git a/eng/Publishing.props b/eng/Publishing.props +index 8b796225f82..348da87600d 100644 +--- a/eng/Publishing.props ++++ b/eng/Publishing.props +@@ -2,6 +2,60 @@ + + + true ++ $(PublishDependsOnTargets);_PublishInstallers + + ++ ++ <_InstallersToPublish Remove="@(_InstallersToPublish)" /> ++ <_InstallersToPublish Include="$(ArtifactsPackagesDir)**\*.tar.gz" UploadPathSegment="Runtime" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ <_InstallersToPublish Include="$(ArtifactsPackagesDir)**\*.zip" UploadPathSegment="Runtime" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ <_InstallersToPublish Include="$(ArtifactsPackagesDir)**\*.deb" UploadPathSegment="Runtime" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ <_InstallersToPublish Include="$(ArtifactsPackagesDir)**\*.rpm" UploadPathSegment="Runtime" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ <_InstallersToPublish Include="$(ArtifactsPackagesDir)**\*.exe" UploadPathSegment="Runtime" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ <_InstallersToPublish Include="$(ArtifactsPackagesDir)**\*.msi" UploadPathSegment="Runtime" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ ++ ++ ++ ++ ++ <_SymbolsArchive Include="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz" Condition="'$(DotNetBuildFromSource)' == 'true'" /> ++ ++ <_InstallersToPublish Remove="@(_SymbolsArchive)" Condition="'$(DotNetBuildFromSource)' == 'true'" /> ++ ++ ++ ++ ++ <_RuntimeFilenamePrefix>dotnet-runtime- ++ <_PackageRid>$(OutputRID) ++ ++ <_PackageRid Condition="'$(_PackageRid)' == ''">$(TargetRid) ++ ++ ++ <_RuntimeArchiveItem Include="$(ArtifactsPackagesDir)**\$(_RuntimeFilenamePrefix)*$(_PackageRid)$(ArchiveExtension)" /> ++ <_RuntimeInternalArchiveItem Include="$(ArtifactsPackagesDir)**\dotnet-runtime-internal*$(_PackageRid)$(ArchiveExtension)" /> ++ <_RuntimeArchiveItem Remove="@(_RuntimeInternalArchiveItem)" /> ++ ++ ++ ++ ++ <_RuntimeArchiveFilename>%(_RuntimeArchiveItem.Filename)%(_RuntimeArchiveItem.Extension) ++ <_RuntimeVersion>$(_RuntimeArchiveFilename.Replace('$(_RuntimeFilenamePrefix)','').Replace('-$(_PackageRid)$(ArchiveExtension)','')) ++ ++ ++ ++ ++ ++ ++ ++ NonShipping=false ++ true ++ %(_InstallersToPublish.UploadPathSegment)/$(_RuntimeVersion)/%(Filename)%(Extension) ++ ++ ++ ++ + diff --git a/src/SourceBuild/patches/sdk/0001-Enable-publishing-in-VMR.patch b/src/SourceBuild/patches/sdk/0001-Enable-publishing-in-VMR.patch new file mode 100644 index 000000000..151924651 --- /dev/null +++ b/src/SourceBuild/patches/sdk/0001-Enable-publishing-in-VMR.patch @@ -0,0 +1,83 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Nikola Milosavljevic +Date: Tue, 12 Mar 2024 22:54:09 +0000 +Subject: [PATCH] Enable publishing in VMR + +Backport: https://github.com/dotnet/sdk/pull/39311 +--- + eng/Publishing.props | 37 ++++++++++++++++++++++++++++++++++--- + 1 file changed, 34 insertions(+), 3 deletions(-) + +diff --git a/eng/Publishing.props b/eng/Publishing.props +index 0e6e2ec2fb..e95cf5b80b 100644 +--- a/eng/Publishing.props ++++ b/eng/Publishing.props +@@ -15,7 +15,7 @@ + $(OS)-$(PlatformName)-SdkAssets.xml + $(ArtifactsLogDir)AssetManifest\$(SdkAssetManifestFileName) + +- $(ArtifactsDir)\..\AssetsTmpDir\$([System.Guid]::NewGuid()) ++ $(ArtifactsDir)\AssetsTmpDir\$([System.Guid]::NewGuid()) + + + + +@@ -98,11 +98,15 @@ + false + true + +- + + + ++ + + ++ ++ + diff --git a/src/SourceBuild/patches/source-build-externals/0001-Enable-publishing-in-VMR.patch b/src/SourceBuild/patches/source-build-externals/0001-Enable-publishing-in-VMR.patch new file mode 100644 index 000000000..f4235006a --- /dev/null +++ b/src/SourceBuild/patches/source-build-externals/0001-Enable-publishing-in-VMR.patch @@ -0,0 +1,42 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Nikola Milosavljevic +Date: Thu, 7 Mar 2024 22:10:43 +0000 +Subject: [PATCH] Enable publishing in VMR + +Backport: https://github.com/dotnet/source-build-externals/pull/275 +--- + eng/Publishing.props | 22 ++++++++++++++++++++++ + 1 file changed, 22 insertions(+) + +diff --git a/eng/Publishing.props b/eng/Publishing.props +index d3aa836..3629bb8 100644 +--- a/eng/Publishing.props ++++ b/eng/Publishing.props +@@ -2,5 +2,27 @@ + + + 3 ++ $(PublishDependsOnTargets);_PublishPackages + ++ ++ ++ <_PackagesToPublish Remove="@(_PackagesToPublish)" /> ++ <_PackagesToPublish Include="$(ArtifactsPackagesDir)**\*.nupkg" UploadPathSegment="Runtime" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ ++ ++ ++ ++ ++ ++ ++ ++ true ++ ++ ++ ++ + diff --git a/src/SourceBuild/patches/symreader/0001-Enable-publishing-in-VMR.patch b/src/SourceBuild/patches/symreader/0001-Enable-publishing-in-VMR.patch new file mode 100644 index 000000000..e17b3f1ff --- /dev/null +++ b/src/SourceBuild/patches/symreader/0001-Enable-publishing-in-VMR.patch @@ -0,0 +1,45 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Nikola Milosavljevic +Date: Thu, 7 Mar 2024 22:15:41 +0000 +Subject: [PATCH] Enable publishing in VMR + +Backport: https://github.com/dotnet/symreader/pull/312 +--- + eng/Publishing.props | 28 ++++++++++++++++++++++++++++ + 1 file changed, 28 insertions(+) + create mode 100644 eng/Publishing.props + +diff --git a/eng/Publishing.props b/eng/Publishing.props +new file mode 100644 +index 0000000..3629bb8 +--- /dev/null ++++ b/eng/Publishing.props +@@ -0,0 +1,28 @@ ++ ++ ++ ++ 3 ++ $(PublishDependsOnTargets);_PublishPackages ++ ++ ++ ++ <_PackagesToPublish Remove="@(_PackagesToPublish)" /> ++ <_PackagesToPublish Include="$(ArtifactsPackagesDir)**\*.nupkg" UploadPathSegment="Runtime" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ ++ ++ ++ ++ ++ ++ ++ ++ true ++ ++ ++ ++ ++ diff --git a/src/SourceBuild/patches/windowsdesktop/0001-Enable-publishing-in-VMR.patch b/src/SourceBuild/patches/windowsdesktop/0001-Enable-publishing-in-VMR.patch new file mode 100644 index 000000000..de5fb0167 --- /dev/null +++ b/src/SourceBuild/patches/windowsdesktop/0001-Enable-publishing-in-VMR.patch @@ -0,0 +1,57 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Nikola Milosavljevic +Date: Thu, 7 Mar 2024 22:19:59 +0000 +Subject: [PATCH] Enable publishing in VMR + +Backport: https://github.com/dotnet/windowsdesktop/pull/4215 +--- + eng/Publishing.props | 40 ++++++++++++++++++++++++++++++++++++---- + 1 file changed, 36 insertions(+), 4 deletions(-) + +diff --git a/eng/Publishing.props b/eng/Publishing.props +index a2ad9884..0426da4f 100644 +--- a/eng/Publishing.props ++++ b/eng/Publishing.props +@@ -1,5 +1,37 @@ + +- +- true +- +- +\ No newline at end of file ++ ++ ++ true ++ $(PublishDependsOnTargets);_PublishInstallers ++ ++ ++ ++ <_InstallersToPublish Remove="@(_InstallersToPublish)" /> ++ <_InstallersToPublish Include="$(ArtifactsPackagesDir)**\*.zip" UploadPathSegment="WindowsDesktop" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ <_InstallersToPublish Include="$(ArtifactsPackagesDir)**\*.exe" UploadPathSegment="WindowsDesktop" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ <_InstallersToPublish Include="$(ArtifactsPackagesDir)**\*.msi" UploadPathSegment="WindowsDesktop" Condition="'$(DotNetBuildRepo)' == 'true'" /> ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ NonShipping=false ++ true ++ %(_InstallersToPublish.UploadPathSegment)/$(MicrosoftWindowsDesktopAppRuntimewinx64PackageVersion)/%(Filename)%(Extension) ++ ++ ++ ++ ++ diff --git a/src/redist/targets/GenerateLayout.targets b/src/redist/targets/GenerateLayout.targets index 8d502588a..72344a894 100644 --- a/src/redist/targets/GenerateLayout.targets +++ b/src/redist/targets/GenerateLayout.targets @@ -102,23 +102,18 @@ - $(PublicBaseURL)Runtime/$(NetRuntimeBlobVersion) - $(PublicBaseURL) $(OfficialBaseURL)Runtime/$(NETStandardTargetingPackBlobVersion) $(PublicBaseURL)aspnetcore/Runtime/$(AspNetCoreBlobVersion) - $(PublicBaseURL) $(FallbackPublicBaseURL)aspnetcore/Runtime/$(AspNetCoreBlobVersion) $(PublicBaseURL)WindowsDesktop/$(WindowsDesktopBlobVersion) - $(PublicBaseURL) $(PublicBaseURL)Sdk/$(MicrosoftDotnetToolsetInternalPackageVersion) - $(PublicBaseURL) $(IntermediateDirectory)/coreSetupDownload/$(MicrosoftNETCoreAppRuntimePackageVersion) $(NetRuntimeDownloadDirectory)/combinedSharedHostAndFrameworkArchive$(ArchiveExtension)