Add SdkArchiveDiff task to verify the sdk archive has all the expected files (#18748)

eng/pipelines/templates/jobs/vmr-build.yml

@@ -208,7 +208,7 @@ jobs:
           exit 1
         fi
       displayName: Setup Previously Source-Built SDK
-  
+
   - ${{ if eq(parameters.targetOS, 'windows') }}:
     - script: |
         call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine ${{ parameters.extraProperties }}
@@ -305,7 +305,7 @@ jobs:
           for envVar in $customEnvVars; do
             customDockerRunArgs="$customDockerRunArgs -e $envVar"
           done
-          
+
           if [[ '${{ parameters.runOnline }}' == 'False' ]]; then
             customDockerRunArgs="$customDockerRunArgs --network none"
           fi
@@ -342,10 +342,10 @@ jobs:
     # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph.
     - powershell: |
         function CopyWithRelativeFolders($sourcePath, $targetFolder, $filter) {
-          Get-ChildItem -Path $sourcePath -Filter $filter -Recurse | ForEach-Object { 
+          Get-ChildItem -Path $sourcePath -Filter $filter -Recurse | ForEach-Object {
             $targetPath = Join-Path $targetFolder (Resolve-Path -Relative $_.FullName)
             New-Item -ItemType Directory -Path (Split-Path -Parent $targetPath) -Force | Out-Null
-            Copy-Item $_.FullName -Destination $targetPath -Force 
+            Copy-Item $_.FullName -Destination $targetPath -Force
           }
         }
 
@@ -356,6 +356,7 @@ jobs:
 
         CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.binlog"
         CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.log"
+        CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.diff"
         CopyWithRelativeFolders "src/"                       $targetFolder "*.binlog"
         CopyWithRelativeFolders "src/"                       $targetFolder "*.log"
         CopyWithRelativeFolders "test/"                      $targetFolder "*.binlog"
@@ -382,6 +383,7 @@ jobs:
         cd "$(sourcesPath)"
         find artifacts/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
         find artifacts/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
         if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then
           find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
         fi

src/SourceBuild/content/Directory.Build.props

@@ -203,6 +203,7 @@
   <PropertyGroup>
     <XPlatSourceBuildTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat.dll'))</XPlatSourceBuildTasksAssembly>
     <LeakDetectionTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll'))</LeakDetectionTasksAssembly>
+    <SdkArchiveDiffTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.dll'))</SdkArchiveDiffTasksAssembly>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(EnablePoison)' == 'true'">

src/SourceBuild/content/build.proj

@@ -24,6 +24,8 @@
   </Target>
 
   <Import Project="$(RepositoryEngineeringDir)build.sourcebuild.targets" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+  <!-- ShortStack doesn't produce an SDK, and GetClosestOfficialSdk doesn't support finding non-portable SDKs -->
+  <Import Project="$(RepositoryEngineeringDir)sdkArchiveDiff.targets" Condition="'$(ShortStack)' != 'true' and '$(PortableBuild)' == 'true'" />
 
   <!-- Intentionally below the import to appear at the end. -->
   <Target Name="LogBuildOutputFolders"

src/SourceBuild/content/eng/build.sourcebuild.targets

@@ -35,10 +35,6 @@
   <Target Name="CreateSdkSymbolsTarball"
           AfterTargets="Build"
           DependsOnTargets="RepackageSymbols">
-    <ItemGroup>
-      <SdkTarballItem Include="$(ArtifactsAssetsDir)dotnet-sdk-*$(ArchiveExtension)" />
-    </ItemGroup>
-
     <PropertyGroup>
       <SdkSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
 

src/SourceBuild/content/eng/sdkArchiveDiff.targets

@@ -0,0 +1,64 @@
+<Project>
+
+  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="GetValidArchiveItems" />
+  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="GetClosestOfficialSdk" />
+  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="FindArchiveDiffs" />
+
+  <Target Name="ReportSdkArchiveDiffs"
+          AfterTargets="Build"
+          DependsOnTargets="DetermineSourceBuiltSdkVersion">
+
+      <Message Text="Comparing built SDK against closest official build"
+               Importance="High"/>
+
+      <GetValidArchiveItems ArchiveItems="@(SdkTarballItem)"
+                            ArchiveName="dotnet-sdk">
+        <Output TaskParameter="ValidArchiveItems"
+                ItemName="_BuiltSdkArchivePath"/>
+      </GetValidArchiveItems>
+
+      <!-- There should only be 1 SDK archive -->
+      <Error Text="Multiple valid dotnet-sdk archives found."
+             Condition="'@(_BuiltSdkArchivePath->Count())' != '1'" />
+
+      <GetClosestOfficialSdk BuiltArchivePath="@(_BuiltSdkArchivePath)">
+        <Output TaskParameter="ClosestOfficialArchivePath"
+                PropertyName="_ClosestOfficialSdkPath" />
+      </GetClosestOfficialSdk>
+
+      <FindArchiveDiffs BaselineArchive="@(_BuiltSdkArchivePath)"
+                        TestArchive="$(_ClosestOfficialSdkPath)">
+        <Output TaskParameter="ContentDifferences"
+                ItemName="_ContentDifferences" />
+      </FindArchiveDiffs>
+
+      <ItemGroup>
+        <_changedFiles Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' != 'Unchanged'" />
+        <_sdkFilesDiff Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' == 'Added'" >
+          <DiffIndicator>+</DiffIndicator>
+        </_sdkFilesDiff>
+        <_sdkFilesDiff Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' == 'Removed'" >
+          <DiffIndicator>-</DiffIndicator>
+        </_sdkFilesDiff>
+        <_sdkFilesDiff Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' == 'Unchanged'" >
+          <DiffIndicator> </DiffIndicator>
+        </_sdkFilesDiff>
+      </ItemGroup>
+
+      <PropertyGroup>
+        <SdkArchiveDiffsReport>$(ArtifactsLogDir)SdkArchiveContent.diff</SdkArchiveDiffsReport>
+      </PropertyGroup>
+
+      <WriteLinesToFile File="$(SdkArchiveDiffsReport)" Lines="@(_sdkFilesDiff->'%(DiffIndicator) %(Identity)')" Overwrite="true" WriteOnlyWhenDifferent="true" />
+
+      <Message Text="Difference in sdk archive: %(_changedFiles.Kind): %(_changedFiles.Identity)"
+               Importance="High"
+               Condition="'@(_changedFiles->Count())' != '0'"/>
+      <Message Text="No differences in sdk archive file contents"
+               Importance="High"
+               Condition="'@(_changedFiles->Count())' == '0'" />
+
+      <Delete Files="$(_ClosestOfficialSdkPath)" />
+  </Target>
+
+</Project>

src/SourceBuild/content/eng/tools/init-build.proj

@@ -15,6 +15,7 @@
       UnpackTarballs;
       BuildXPlatTasks;
       BuildMSBuildSdkResolver;
+      BuildSdkArchiveDiff;
       BuildLeakDetection;
       ExtractToolPackage;
       GenerateRootFs;
@@ -116,6 +117,15 @@
     </Touch>
   </Target>
 
+  <Target Name="BuildSdkArchiveDiff"
+          Condition="'$(ShortStack)' != 'true' and '$(PortableBuild)' == 'true'" >
+    <MSBuild Projects="tasks\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj"
+             Targets="Restore"
+             Properties="MSBuildRestoreSessionId=$([System.Guid]::NewGuid())" />
+    <MSBuild Projects="tasks\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj"
+             Targets="Build" />
+  </Target>
+
   <Target Name="GenerateRootFs"
           Condition="'$(BuildOS)' != 'windows' and '$(CrossBuild)' == 'true' and '$(ROOTFS_DIR)' == ''">
     <PropertyGroup>

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs

@@ -0,0 +1,136 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Formats.Tar;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+public abstract class Archive : IDisposable
+{
+    public static async Task<Archive> Create(string path, CancellationToken cancellationToken = default)
+    {
+        if (path.EndsWith(".tar.gz"))
+            return await TarArchive.Create(path, cancellationToken);
+        else if (path.EndsWith(".zip"))
+            return ZipFileArchive.Create(path);
+        else
+            throw new NotSupportedException("Unsupported archive type");
+    }
+
+    public abstract string[] GetFileNames();
+
+    public abstract bool Contains(string relativePath);
+
+    public abstract void Dispose();
+
+    public class TarArchive : Archive
+    {
+        private string _extractedFolder;
+
+        private TarArchive(string extractedFolder)
+        {
+            _extractedFolder = extractedFolder;
+        }
+
+        public static new async Task<TarArchive> Create(string path, CancellationToken cancellationToken = default)
+        {
+            var tmpFolder = Directory.CreateTempSubdirectory(nameof(FindArchiveDiffs));
+            using (var gzStream = File.OpenRead(path))
+            using (var gzipStream = new GZipStream(gzStream, CompressionMode.Decompress))
+            {
+                await TarFile.ExtractToDirectoryAsync(gzipStream, tmpFolder.FullName, true, cancellationToken);
+            }
+            return new TarArchive(tmpFolder.FullName);
+        }
+
+        public override bool Contains(string relativePath)
+        {
+            return File.Exists(Path.Combine(_extractedFolder, relativePath));
+        }
+
+        public override string[] GetFileNames()
+        {
+            return Directory.GetFiles(_extractedFolder, "*", SearchOption.AllDirectories).Select(f => f.Substring(_extractedFolder.Length + 1)).ToArray();
+        }
+
+        public override void Dispose()
+        {
+            if (Directory.Exists(_extractedFolder))
+                Directory.Delete(_extractedFolder, true);
+        }
+    }
+
+    public class ZipFileArchive : Archive
+    {
+        private ZipArchive _archive;
+
+        private ZipFileArchive(ZipArchive archive)
+        {
+            _archive = archive;
+        }
+
+        public static ZipFileArchive Create(string path)
+        {
+            return new ZipFileArchive(new ZipArchive(File.OpenRead(path)));
+        }
+
+        public override bool Contains(string relativePath)
+        {
+            return _archive.GetEntry(relativePath) != null;
+        }
+
+        public override string[] GetFileNames()
+        {
+            return _archive.Entries.Select(e => e.FullName).ToArray();
+        }
+
+        public override void Dispose()
+        {
+            _archive.Dispose();
+        }
+    }
+
+    private static string GetArchiveExtension(string path)
+    {
+        if (path.EndsWith(".tar.gz"))
+        {
+            return ".tar.gz";
+        }
+        else if (path.EndsWith(".zip"))
+        {
+            return ".zip";
+        }
+        else
+        {
+            throw new ArgumentException($"Invalid archive extension '{path}': must end with .tar.gz or .zip");
+        }
+    }
+
+    public static (string Version, string Rid, string extension) GetInfoFromFileName(string filename, string packageName)
+    {
+        var extension = GetArchiveExtension(filename);
+        var Version = VersionIdentifier.GetVersion(filename);
+        if (Version is null)
+            throw new ArgumentException("Invalid archive file name '{filename}': No valid version found in file name.");
+        // Once we've removed the version, package name, and extension, we should be left with the RID
+        var Rid = filename
+            .Replace(extension, "")
+            .Replace(Version, "")
+            .Replace(packageName, "")
+            .Trim('-', '.', '_');
+
+        // A RID with '.' must have a version number after the first '.' in each part of the RID. For example, alpine.3.10-arm64.
+        // Otherwise, it's likely an archive of another type of file that we don't handle here, for example, .msi.wixpack.zip.
+        var ridParts = Rid.Split('-');
+        foreach(var item in ridParts.SelectMany(p => p.Split('.').Skip(1)))
+        {
+            if (!int.TryParse(item, out _))
+                throw new ArgumentException($"Invalid Rid '{Rid}' in archive file name '{filename}'. Expected RID with '.' to be part of a version. This likely means the file is an archive of a different file type.");
+        }
+        return (Version, Rid, extension);
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs

@@ -0,0 +1,126 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+static class Diff
+{
+    public static ITaskItem TaskItemFromDiff((string, DifferenceKind) diff)
+    {
+        var item = new TaskItem(diff.Item1);
+        item.SetMetadata("Kind", Enum.GetName(diff.Item2));
+        return item;
+    }
+
+    public enum DifferenceKind
+    {
+        /// <summary>
+        /// Present in the test but not in the baseline
+        /// </summary>
+        Added,
+
+        /// <summary>
+        /// Present in the baseline but not in the test
+        /// </summary>
+        Removed,
+
+        /// <summary>
+        /// Present in both the baseline and test
+        /// </summary>
+        Unchanged
+    }
+
+    /// <summary>
+    /// Uses the Longest Common Subsequence algorithm (as used in 'git diff') to find the differences between two lists of strings.
+    /// Returns a list of the joined lists with the differences marked as either added or removed.
+    /// </summary>
+    public static List<(string, DifferenceKind DifferenceKind)> GetDiffs(
+        Span<string> baselineSequence,
+        Span<string> testSequence,
+        Func<string, string, bool> equalityComparer,
+        Func<string, string>? formatter = null,
+        CancellationToken cancellationToken = default)
+    {
+        // Edit distance algorithm: https://en.wikipedia.org/wiki/Longest_common_subsequence
+        // cancellationToken.ThrowIfCancellationRequested();
+        formatter ??= static s => s;
+
+        // Optimization: remove common prefix
+        int i = 0;
+        List<(string, DifferenceKind)> prefix = [];
+        while (i < baselineSequence.Length && i < testSequence.Length && equalityComparer(baselineSequence[i], testSequence[i]))
+        {
+            prefix.Add((formatter(baselineSequence[i]), DifferenceKind.Unchanged));
+            i++;
+        }
+
+        baselineSequence = baselineSequence[i..];
+        testSequence = testSequence[i..];
+
+        // Initialize first row and column
+        int[,] m = new int[baselineSequence.Length + 1, testSequence.Length + 1];
+        for (i = 0; i <= baselineSequence.Length; i++)
+        {
+            m[i, 0] = i;
+        }
+        for (i = 0; i <= testSequence.Length; i++)
+        {
+            m[0, i] = i;
+        }
+
+        // Compute edit distance
+        for (i = 1; i <= baselineSequence.Length; i++)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            for (int j = 1; j <= testSequence.Length; j++)
+            {
+                if (equalityComparer(baselineSequence[i - 1], testSequence[j - 1]))
+                {
+                    m[i, j] = m[i - 1, j - 1];
+                }
+                else
+                {
+                    m[i, j] = 1 + Math.Min(m[i - 1, j], m[i, j - 1]);
+                }
+            }
+        }
+
+        // Trace back the edits
+        int row = baselineSequence.Length;
+        int col = testSequence.Length;
+        List<(string, DifferenceKind)> diff = [];
+        while (row > 0 || col > 0)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            if (row > 0 && col > 0 && equalityComparer(baselineSequence[row - 1], testSequence[col - 1]))
+            {
+                diff.Add((formatter(baselineSequence[row - 1]), DifferenceKind.Unchanged));
+                row--;
+                col--;
+            }
+            else if (col > 0 && (row == 0 || m[row, col - 1] <= m[row - 1, col]))
+            {
+                diff.Add((formatter(testSequence[col - 1]), DifferenceKind.Added));
+                col--;
+            }
+            else if (row > 0 && (col == 0 || m[row, col - 1] > m[row - 1, col]))
+            {
+                diff.Add((formatter(baselineSequence[row - 1]), DifferenceKind.Removed));
+                row--;
+            }
+            else
+            {
+                throw new UnreachableException();
+            }
+        }
+        diff.Reverse();
+        prefix.AddRange(diff);
+        return prefix;
+    }
+
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs

@@ -0,0 +1,49 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Build.Framework;
+using Task = System.Threading.Tasks.Task;
+
+public class FindArchiveDiffs : Microsoft.Build.Utilities.Task, ICancelableTask
+{
+    [Required]
+    public required ITaskItem BaselineArchive { get; init; }
+
+    [Required]
+    public required ITaskItem TestArchive { get; init; }
+
+    [Output]
+    public ITaskItem[] ContentDifferences { get; set; } = [];
+
+    private CancellationTokenSource _cancellationTokenSource = new();
+    private CancellationToken cancellationToken => _cancellationTokenSource.Token;
+
+    public override bool Execute()
+    {
+        return Task.Run(ExecuteAsync).Result;
+    }
+
+    public async Task<bool> ExecuteAsync()
+    {
+        var baselineTask = Archive.Create(BaselineArchive.ItemSpec);
+        var testTask = Archive.Create(TestArchive.ItemSpec);
+        Task.WaitAll([baselineTask, testTask], cancellationToken);
+        using var baseline = await baselineTask;
+        using var test = await testTask;
+        var baselineFiles = baseline.GetFileNames();
+        var testFiles = test.GetFileNames();
+        ContentDifferences =
+            Diff.GetDiffs(baselineFiles, testFiles, VersionIdentifier.AreVersionlessEqual, static p => VersionIdentifier.RemoveVersions(p, "{VERSION}"), cancellationToken)
+            .Select(Diff.TaskItemFromDiff)
+            .ToArray();
+        return true;
+    }
+
+    public void Cancel()
+    {
+        _cancellationTokenSource.Cancel();
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs

@@ -0,0 +1,94 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Build.Framework;
+
+public abstract class GetClosestArchive : Microsoft.Build.Utilities.Task, ICancelableTask
+{
+    [Required]
+    public required string BuiltArchivePath { get; init; }
+
+    [Output]
+    public string ClosestOfficialArchivePath { get; set; } = "";
+
+    private string? _builtVersion;
+    protected string BuiltVersion
+    {
+        get => _builtVersion ?? throw new InvalidOperationException();
+        private set => _builtVersion = value;
+    }
+
+    private string? _builtRid;
+    protected string BuiltRid
+    {
+        get => _builtRid ?? throw new InvalidOperationException();
+        private set => _builtRid = value;
+    }
+
+    private string? _archiveExtension;
+    protected string ArchiveExtension
+    {
+        get => _archiveExtension ?? throw new InvalidOperationException();
+        private set => _archiveExtension = value;
+    }
+
+    /// <summary>
+    /// The name of the package to find the closest official archive for. For example, "dotnet-sdk" or "aspnetcore-runtime".
+    /// </summary>
+    protected abstract string ArchiveName { get; }
+
+    private CancellationTokenSource _cancellationTokenSource = new();
+    protected CancellationToken CancellationToken => _cancellationTokenSource.Token;
+    public void Cancel()
+    {
+        _cancellationTokenSource.Cancel();
+    }
+
+    /// <summary>
+    /// Get the URL of the latest official archive for the given version string and RID.
+    /// </summary>
+    public abstract Task<string?> GetClosestOfficialArchiveUrl();
+
+    public abstract Task<string?> GetClosestOfficialArchiveVersion();
+
+    public override bool Execute()
+    {
+        return Task.Run(ExecuteAsync).Result;
+    }
+
+    public async Task<bool> ExecuteAsync()
+    {
+        CancellationToken.ThrowIfCancellationRequested();
+        var filename = Path.GetFileName(BuiltArchivePath);
+        (BuiltVersion, BuiltRid, ArchiveExtension) = Archive.GetInfoFromFileName(filename, ArchiveName);
+        Log.LogMessage($"Finding closest official archive for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'");
+
+        string? downloadUrl = await GetClosestOfficialArchiveUrl();
+        if (downloadUrl == null)
+        {
+            Log.LogError($"Failed to find a download URL for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'");
+            return false;
+        }
+
+        HttpClient client = new HttpClient();
+
+        Log.LogMessage(MessageImportance.High, $"Downloading {downloadUrl}");
+        HttpResponseMessage packageResponse = await client.GetAsync(downloadUrl, CancellationToken);
+
+        var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath;
+
+        ClosestOfficialArchivePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + "." + Path.GetFileName(packageUriPath));
+        Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialArchivePath}");
+        using (var file = File.Create(ClosestOfficialArchivePath))
+        {
+            await packageResponse.Content.CopyToAsync(file, CancellationToken);
+        }
+
+        return true;
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs

@@ -0,0 +1,44 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.Build.Framework;
+
+public class GetClosestOfficialSdk : GetClosestArchive
+{
+    protected override string ArchiveName => "dotnet-sdk";
+
+    HttpClient client = new HttpClient(new HttpClientHandler() { AllowAutoRedirect = false });
+
+    private string? closestVersion;
+    private string? closestUrl;
+
+    public override async Task<string?> GetClosestOfficialArchiveUrl()
+    {
+        // Channel in the form of 9.0.1xx
+        var channel = BuiltVersion[..5] + "xx";
+        var akaMsUrl = $"https://aka.ms/dotnet/{channel}/daily/{ArchiveName}-{BuiltRid}{ArchiveExtension}";
+        var redirectResponse = await client.GetAsync(akaMsUrl, CancellationToken);
+        // aka.ms returns a 301 for valid redirects and a 302 to Bing for invalid URLs
+        if (redirectResponse.StatusCode != HttpStatusCode.Moved)
+        {
+            Log.LogMessage(MessageImportance.High, $"Failed to find package at '{akaMsUrl}': invalid aka.ms URL");
+            return null;
+        }
+        closestUrl = redirectResponse.Headers.Location!.ToString();
+        closestVersion = VersionIdentifier.GetVersion(closestUrl);
+        return closestUrl;
+    }
+
+    public override async Task<string?> GetClosestOfficialArchiveVersion()
+    {
+        if (closestUrl is not null)
+        {
+            return closestVersion;
+        }
+        _ = await GetClosestOfficialArchiveUrl();
+        return closestVersion;
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs

@@ -0,0 +1,57 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Microsoft.Build.Framework;
+
+public class GetValidArchiveItems : Microsoft.Build.Utilities.Task
+{
+    [Required]
+    public required ITaskItem[] ArchiveItems { get; init; }
+
+    [Required]
+    public required string ArchiveName { get; init; }
+
+    [Output]
+    public ITaskItem[] ValidArchiveItems { get; set; } = [];
+
+    public override bool Execute()
+    {
+        List<ITaskItem> archiveItems = new();
+        foreach (var item in ArchiveItems)
+        {
+            var filename = Path.GetFileName(item.ItemSpec);
+            try
+            {
+                // Ensure the version and RID info can be parsed from the item
+                _ = Archive.GetInfoFromFileName(filename, ArchiveName);
+                archiveItems.Add(item);
+            }
+            catch (ArgumentException e)
+            {
+                Log.LogMessage($"'{item.ItemSpec}' is not a valid archive name: '{e.Message}'");
+                continue;
+            }
+        }
+        switch (archiveItems.Count)
+        {
+            case 0:
+                Log.LogMessage(MessageImportance.High, "No valid archive items found");
+                ValidArchiveItems = [];
+                return false;
+            case 1:
+                Log.LogMessage(MessageImportance.High, $"{archiveItems[0]} is the only valid archive item found");
+                ValidArchiveItems = archiveItems.ToArray();
+                break;
+            default:
+                archiveItems.Sort((a, b) => a.ItemSpec.Length - b.ItemSpec.Length);
+                Log.LogMessage(MessageImportance.High, $"Multiple valid archive items found: '{string.Join("', '", archiveItems)}'");
+                ValidArchiveItems = archiveItems.ToArray();
+                break;
+        }
+        return true;
+    }
+
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Build.Utilities.Core" PrivateAssets="all" ExcludeAssets="Runtime" Version="$(MicrosoftBuildVersion)" />
+  </ItemGroup>
+
+</Project>

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs

@@ -0,0 +1,257 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+// Copied from https://github.com/dotnet/arcade/blob/db45698020f58f88eef75b23b2598a59872918f6/src/Microsoft.DotNet.VersionTools/lib/src/BuildManifest/VersionIdentifier.cs
+// Conflicting MSBuild versions and some customizations make it difficult to use the Arcade assembly.
+public static class VersionIdentifier
+{
+    private static readonly HashSet<string> _knownTags = new HashSet<string>
+        {
+            "alpha",
+            "beta",
+            "preview",
+            "prerelease",
+            "servicing",
+            "rtm",
+            "rc"
+        };
+
+    private static readonly SortedDictionary<string, string> _sequencesToReplace =
+        new SortedDictionary<string, string>
+        {
+                { "-.", "." },
+                { "..", "." },
+                { "--", "-" },
+                { "//", "/" },
+                { "_.", "." }
+        };
+
+    private const string _finalSuffix = "final";
+
+    private static readonly char[] _delimiters = new char[] { '.', '-', '_' };
+
+    /// <summary>
+    /// Identify the version of an asset.
+    ///
+    /// Asset names can come in two forms:
+    /// - Blobs that include the full path
+    /// - Packages that do not include any path elements.
+    ///
+    /// There may be multiple different version numbers in a blob path.
+    /// This method starts at the last segment of the path and works backward to find a version number.
+    /// </summary>
+    /// <param name="assetName">Asset name</param>
+    /// <returns>Version, or null if none is found.</returns>
+    public static string? GetVersion(string assetName)
+    {
+        string[] pathSegments = assetName.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
+        string? potentialVersion = null;
+        for (int i = pathSegments.Length - 1; i >= 0; i--)
+        {
+            potentialVersion = GetVersionForSingleSegment(pathSegments[i]);
+            if (potentialVersion != null)
+            {
+                return potentialVersion;
+            }
+        }
+
+        return potentialVersion;
+    }
+
+    /// <summary>
+    /// Identify the version number of an asset segment.
+    /// </summary>
+    /// <param name="assetPathSegment">Asset segment</param>
+    /// <returns>Version number, or null if none was found</returns>
+    /// <remarks>
+    /// Identifying versions is not particularly easy. To constrain the problem,
+    /// we apply the following assumptions which are generally valid for .NET Core.
+    /// - We always have major.minor.patch, and it always begins the version string.
+    /// - The only pre-release or build metadata labels we use begin with the _knownTags shown above.
+    /// - We use additional numbers in our version numbers after the initial
+    ///   major.minor.patch-prereleaselabel.prereleaseiteration segment,
+    ///   but any non-numeric element will end the version string.
+    /// - The <see cref="_delimiters"/> we use in versions and file names are ., -, and _.
+    /// </remarks>
+    private static string? GetVersionForSingleSegment(string assetPathSegment)
+    {
+
+        // Find the start of the version number by finding the major.minor.patch.
+        // Scan the string forward looking for a digit preceded by one of the delimiters,
+        // then look for a minor.patch, completing the major.minor.patch.  Continue to do so until we get
+        // to something that is NOT major.minor.patch (this is necessary because we sometimes see things like:
+        // VS.Redist.Common.NetCore.Templates.x86.2.2.3.0.101-servicing-014320.nupkg
+        // Continue iterating until we find ALL potential versions. Return the one that is the latest in the segment
+        // This is to deal with files with multiple major.minor.patchs in the file name, for example:
+        // Microsoft.NET.Workload.Mono.ToolChain.Manifest-6.0.100.Msi.x64.6.0.0-rc.1.21380.2.symbols.nupkg
+
+        int currentIndex = 0;
+        // Stack of major.minor.patch.
+        Stack<(int versionNumber, int index)> majorMinorPatchStack = new Stack<(int, int)>(3);
+        string? majorMinorPatch = null;
+        int majorMinorPatchIndex = 0;
+        StringBuilder versionSuffix = new StringBuilder();
+        char prevDelimiterCharacter = char.MinValue;
+        char nextDelimiterCharacter = char.MinValue;
+        Dictionary<int, string> majorMinorPatchDictionary = new Dictionary<int, string>();
+        while (true)
+        {
+            string nextSegment;
+            prevDelimiterCharacter = nextDelimiterCharacter;
+            int nextDelimiterIndex = assetPathSegment.IndexOfAny(_delimiters, currentIndex);
+            if (nextDelimiterIndex != -1)
+            {
+                nextDelimiterCharacter = assetPathSegment[nextDelimiterIndex];
+                nextSegment = assetPathSegment.Substring(currentIndex, nextDelimiterIndex - currentIndex);
+            }
+            else
+            {
+                nextSegment = assetPathSegment.Substring(currentIndex);
+            }
+
+            // If we have not yet found the major/minor/patch, then there are four cases:
+            // - There have been no potential major/minor/patch numbers found and the current segment is a number. Push onto the majorMinorPatch stack
+            //   and continue.
+            // - There has been at least one number found, but less than 3, and the current segment not a number or not preceded by '.'. In this case,
+            //   we should clear out the stack and continue the search.
+            // - There have been at least 2 numbers found and the current segment is a number and preceded by '.'. Push onto the majorMinorPatch stack and continue
+            // - There have been at least 3 numbers found and the current segment is not a number or not preceded by '-'. In this case, we can call this the major minor
+            //   patch number and no longer need to continue searching
+            if (majorMinorPatch == null)
+            {
+                bool isNumber = int.TryParse(nextSegment, out int potentialVersionSegment);
+                if ((majorMinorPatchStack.Count == 0 && isNumber) ||
+                    (majorMinorPatchStack.Count > 0 && prevDelimiterCharacter == '.' && isNumber))
+                {
+                    majorMinorPatchStack.Push((potentialVersionSegment, currentIndex));
+                }
+                // Check for partial major.minor.patch cases, like: 2.2.bar or 2.2-100.bleh
+                else if (majorMinorPatchStack.Count > 0 && majorMinorPatchStack.Count < 3 &&
+                         (prevDelimiterCharacter != '.' || !isNumber))
+                {
+                    majorMinorPatchStack.Clear();
+                }
+
+                // Determine whether we are done with major.minor.patch after this update.
+                if (majorMinorPatchStack.Count >= 3 && (prevDelimiterCharacter != '.' || !isNumber || nextDelimiterIndex == -1))
+                {
+                    // Done with major.minor.patch, found. Pop the top 3 elements off the stack.
+                    (int patch, int patchIndex) = majorMinorPatchStack.Pop();
+                    (int minor, int minorIndex) = majorMinorPatchStack.Pop();
+                    (int major, int majorIndex) = majorMinorPatchStack.Pop();
+                    majorMinorPatch = $"{major}.{minor}.{patch}";
+                    majorMinorPatchIndex = majorIndex;
+                }
+            }
+
+            // Don't use else, so that we don't miss segments
+            // in case we are just deciding that we've finished major minor patch.
+            if (majorMinorPatch != null)
+            {
+                // Now look at the next segment. If it looks like it could be part of a version, append to what we have
+                // and continue. If it can't, then we're done.
+                //
+                // Cases where we should break out and be done:
+                // - We have an empty pre-release label and the delimiter is not '-'.
+                // - We have an empty pre-release label and the next segment does not start with a known tag.
+                // - We have a non-empty pre-release label and the current segment is not a number and also not 'final'
+                //      A corner case of versioning uses .final to represent a non-date suffixed final pre-release version:
+                //      3.1.0-preview.10.final
+                if (versionSuffix.Length == 0 &&
+                    (prevDelimiterCharacter != '-' || !_knownTags.Any(tag => nextSegment.StartsWith(tag, StringComparison.OrdinalIgnoreCase))))
+                {
+                    majorMinorPatchDictionary.Add(majorMinorPatchIndex, majorMinorPatch);
+                    majorMinorPatch = null;
+                    versionSuffix = new StringBuilder();
+                }
+                else if (versionSuffix.Length != 0 && !int.TryParse(nextSegment, out int potentialVersionSegment) && nextSegment != _finalSuffix)
+                {
+                    majorMinorPatchDictionary.Add(majorMinorPatchIndex, $"{majorMinorPatch}{versionSuffix.ToString()}");
+                    majorMinorPatch = null;
+                    versionSuffix = new StringBuilder();
+                }
+                else
+                {
+                    // Append the delimiter character and then the current segment
+                    versionSuffix.Append(prevDelimiterCharacter);
+                    versionSuffix.Append(nextSegment);
+                }
+            }
+
+            if (nextDelimiterIndex != -1)
+            {
+                currentIndex = nextDelimiterIndex + 1;
+            }
+            else
+            {
+                break;
+            }
+        }
+
+        if (majorMinorPatch != null)
+        {
+            majorMinorPatchDictionary.Add(majorMinorPatchIndex, $"{majorMinorPatch}{versionSuffix.ToString()}");
+        }
+
+        if (!majorMinorPatchDictionary.Any())
+        {
+            return null;
+        }
+
+        int maxKey = majorMinorPatchDictionary.Keys.Max();
+        return majorMinorPatchDictionary[maxKey];
+    }
+
+    /// <summary>
+    ///     Given an asset name, remove all .NET Core version numbers (as defined by GetVersionForSingleSegment)
+    ///     from the string
+    /// </summary>
+    /// <param name="assetName">Asset</param>
+    /// <returns>Asset name without versions</returns>
+    public static string RemoveVersions(string assetName, string replacement = "")
+    {
+        string[] pathSegments = assetName.Split('/');
+
+        // Remove the version number from each segment, then join back together and
+        // remove any useless character sequences.
+
+        for (int i = 0; i < pathSegments.Length; i++)
+        {
+            if (!string.IsNullOrEmpty(pathSegments[i]))
+            {
+                string? versionForSegment = GetVersionForSingleSegment(pathSegments[i]);
+                if (versionForSegment != null)
+                {
+                    pathSegments[i] = pathSegments[i].Replace(versionForSegment, replacement);
+                }
+            }
+        }
+
+        // Continue replacing things until there is nothing left to replace.
+        string assetWithoutVersions = string.Join("/", pathSegments);
+        bool anyReplacements = true;
+        while (anyReplacements)
+        {
+            string replacementIterationResult = assetWithoutVersions;
+            foreach (var sequence in _sequencesToReplace)
+            {
+                replacementIterationResult = replacementIterationResult.Replace(sequence.Key, sequence.Value);
+            }
+            anyReplacements = replacementIterationResult != assetWithoutVersions;
+            assetWithoutVersions = replacementIterationResult;
+        }
+
+        return assetWithoutVersions;
+    }
+
+
+    public static bool AreVersionlessEqual(string assetName1, string assetName2)
+    {
+        return RemoveVersions(assetName1) == RemoveVersions(assetName2);
+    }
+}