PR Feedback:

- Extract common archive diffing into a base class
- Move Diff utils to a separate class
- Return all valid archives and assert only one valid SDK archive
- Use Arcade VersionIdentifier util class

src/SourceBuild/content/eng/build.targets

@@ -1,28 +1,41 @@
 <Project>
-  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="FindArchiveDiffs" />
+  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="GetValidArchiveItems" />
   <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="GetClosestOfficialSdk" />
-  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="GetSingleArchiveItem" />
+  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="FindArchiveDiffs" />
 
   <Target Name="ReportSdkArchiveDiffs"
           AfterTargets="Build"
           DependsOnTargets="DetermineSourceBuiltSdkVersion" >
 
-      <GetSingleArchiveItem SdkArchiveItems="@(SdkTarballItem)">
-        <Output TaskParameter="BestSdkArchiveItem" PropertyName="_BuiltSdkArchivePath"/>
-      </GetSingleArchiveItem>
+      <GetValidArchiveItems ArchiveItems="@(SdkTarballItem)"
+                            ArchiveName="dotnet-sdk"
+                            Condition="'@(SdkTarballItem->Count())' != '0'">
+        <Output TaskParameter="ValidArchiveItems"
+                ItemName="_BuiltSdkArchivePath"/>
+      </GetValidArchiveItems>
 
-      <GetClosestOfficialSdk BuiltSdkPath="$(_BuiltSdkArchivePath)" Condition="'$(_BuiltSdkArchivePath)' != ''">
-        <Output TaskParameter="ClosestOfficialSdkPath" PropertyName="_ClosestOfficialSdkPath" />
+      <Error Message="Multiple valid dotnet-sdk archives found."
+             Condition="'@(_BuiltSdkArchivePath->Count())' != '1'" />
+
+      <GetClosestOfficialSdk BuiltArchivePath="@(_BuiltSdkArchivePath)"
+                             Condition="'@(_BuiltSdkArchivePath)' != ''">
+        <Output TaskParameter="ClosestOfficialArchivePath"
+                PropertyName="_ClosestOfficialSdkPath" />
       </GetClosestOfficialSdk>
 
-      <Message Text="Failed to find closest official SDK archive." Importance="High" Condition="'$(_BuiltSdkArchivePath)' != '' AND '$(_ClosestOfficialSdkPath)' == ''" />
-
-      <FindArchiveDiffs BaselineArchive="$(_BuiltSdkArchivePath)" TestArchive="$(_ClosestOfficialSdkPath)" Condition="'$(_BuiltSdkArchivePath)' != '' AND '$(_ClosestOfficialSdkPath)' != ''">
-        <Output TaskParameter="ContentDifferences" ItemName="_ContextDifferences" />
+      <FindArchiveDiffs BaselineArchive="@(_BuiltSdkArchivePath)"
+                        TestArchive="$(_ClosestOfficialSdkPath)"
+                        Condition="'@(_BuiltSdkArchivePath)' != '' AND '$(_ClosestOfficialSdkPath)' != ''">
+        <Output TaskParameter="ContentDifferences"
+                ItemName="_ContentDifferences" />
       </FindArchiveDiffs>
 
-      <Message Text="Difference in sdk archive: %(_ContextDifferences.Kind): %(_ContextDifferences.Identity)" Importance="High" Condition="'%(_ContextDifferences.Identity)' != '' AND '%(_ContextDifferences.Kind)' != 'Unchanged'"/>
-      <Delete Files="$(_ClosestOfficialSdkPath)" Condition="'$(_ClosestOfficialSdkPath)' == ''"/>
+      <Message Text="Difference in sdk archive: %(_ContentDifferences.Kind): %(_ContentDifferences.Identity)"
+               Importance="High"
+               Condition="'%(_ContentDifferences.Identity)' != '' AND '%(_ContentDifferences.Kind)' != 'Unchanged'"/>
+
+      <Delete Files="$(_ClosestOfficialSdkPath)"
+              Condition="'$(_ClosestOfficialSdkPath)' == ''"/>
 
   </Target>
 

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs

@@ -39,8 +39,8 @@ public abstract class Archive : IDisposable
         public static new async Task<TarArchive> Create(string path, CancellationToken cancellationToken = default)
         {
             var tmpFolder = Directory.CreateTempSubdirectory(nameof(FindArchiveDiffs));
-            using (var gzStream = File.OpenRead (path))
-            using (var gzipStream = new GZipStream (gzStream, CompressionMode.Decompress))
+            using (var gzStream = File.OpenRead(path))
+            using (var gzipStream = new GZipStream(gzStream, CompressionMode.Decompress))
             {
                 await TarFile.ExtractToDirectoryAsync(gzipStream, tmpFolder.FullName, true, cancellationToken);
             }
@@ -94,31 +94,34 @@ public abstract class Archive : IDisposable
         }
     }
 
-    public static (string Version, string Rid, string extension) GetInfoFromArchivePath(string path)
+    private static string GetArchiveExtension(string path)
     {
-        string extension;
         if (path.EndsWith(".tar.gz"))
         {
-            extension = ".tar.gz";
+            return ".tar.gz";
         }
         else if (path.EndsWith(".zip"))
         {
-            extension = ".zip";
+            return ".zip";
         }
         else
         {
             throw new ArgumentException($"Invalid archive extension '{path}': must end with .tar.gz or .zip");
         }
+    }
 
-        string filename = Path.GetFileName(path)[..^extension.Length];
-        var dashDelimitedParts = filename.Split('-');
-        var (rid, versionString) = dashDelimitedParts switch
-        {
-            ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(first) => (third + '-' + fourth, first + '-' + second),
-            ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(third) => (first + '-' + second, third + '-' + fourth),
-            _ => throw new ArgumentException($"Invalid archive file name '{filename}': file name should include full build version and rid in the format dotnet-sdk-<version>-<rid>{extension} or dotnet-sdk-<rid>-<version>{extension}")
-        };
-
-        return (versionString, rid, extension);
+    public static (string Version, string Rid, string extension) GetInfoFromFileName(string filename, string packageName)
+    {
+        var extension = GetArchiveExtension(filename);
+        var Version = VersionIdentifier.GetVersion(filename);
+        if (Version is null)
+            throw new ArgumentException("Invalid archive file name '{filename}': No valid version found in file name.");
+        // Once we've removed the version, package name, and extension, we should be left with the RID
+        var Rid = filename
+            .Replace(extension, "")
+            .Replace(Version, "")
+            .Replace(packageName, "")
+            .Trim('-', '.', '_');
+        return (Version, Rid, extension);
     }
 }

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs

@@ -0,0 +1,127 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+static class Diff
+{
+    public static ITaskItem TaskItemFromDiff((string, DifferenceKind) diff)
+    {
+        var item = new TaskItem(diff.Item1);
+        item.SetMetadata("Kind", Enum.GetName(diff.Item2));
+        return item;
+    }
+
+    public enum DifferenceKind
+    {
+        /// <summary>
+        /// Present in the test but not in the baseline
+        /// </summary>
+        Added,
+
+        /// <summary>
+        /// Present in the baseline but not in the test
+        /// </summary>
+        Removed,
+
+        /// <summary>
+        /// Present in both the baseline and test
+        /// </summary>
+        Unchanged
+    }
+
+    /// <summary>
+    /// Uses the Longest Common Subsequence algorithm (as used in 'git diff') to find the differences between two lists of strings.
+    /// Returns a list of the joined lists with the differences marked as either added or removed.
+    /// </summary>
+    public static List<(string, DifferenceKind DifferenceKind)> GetDiffs(
+        Span<string> baselineSequence,
+        Span<string> testSequence,
+        Func<string, string, bool> equalityComparer,
+        Func<string, string>? formatter = null,
+        CancellationToken cancellationToken = default)
+    {
+        // Edit distance algorithm: https://en.wikipedia.org/wiki/Longest_common_subsequence
+        // cancellationToken.ThrowIfCancellationRequested();
+        formatter ??= static s => s;
+        List<(string, DifferenceKind)> diff = [];
+
+        // Optimization: remove common prefix
+        int i = 0;
+        while (i < baselineSequence.Length && i < testSequence.Length && equalityComparer(baselineSequence[i], testSequence[i]))
+        {
+            diff.Add((formatter(baselineSequence[i]), DifferenceKind.Unchanged));
+            i++;
+        }
+
+        baselineSequence = baselineSequence[i..];
+        testSequence = testSequence[i..];
+
+        // Initialize first row and column
+        int[,] m = new int[baselineSequence.Length + 1, testSequence.Length + 1];
+        for (i = 0; i <= baselineSequence.Length; i++)
+        {
+            m[i, 0] = i;
+        }
+        for (i = 0; i <= testSequence.Length; i++)
+        {
+            m[0, i] = i;
+        }
+
+        // Compute edit distance
+        for (i = 1; i <= baselineSequence.Length; i++)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            for (int j = 1; j <= testSequence.Length; j++)
+            {
+                if (equalityComparer(baselineSequence[i - 1], testSequence[j - 1]))
+                {
+                    m[i, j] = m[i - 1, j - 1];
+                }
+                else
+                {
+                    m[i, j] = 1 + Math.Min(m[i - 1, j], m[i, j - 1]);
+                }
+            }
+        }
+
+        // Trace back the edits
+        int row = baselineSequence.Length;
+        int col = testSequence.Length;
+
+        while (row > 0 || col > 0)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            var baselineItem = baselineSequence[row - 1];
+            var testItem = testSequence[col - 1];
+            if (row > 0 && col > 0 && equalityComparer(baselineItem, testItem))
+            {
+                diff.Add((formatter(baselineSequence[row - 1]), DifferenceKind.Unchanged));
+                row--;
+                col--;
+            }
+            else if (col > 0 && (row == 0 || m[row, col - 1] <= m[row - 1, col]))
+            {
+                diff.Add((formatter(testSequence[col - 1]), DifferenceKind.Added));
+                col--;
+            }
+            else if (row > 0 && (col == 0 || m[row, col - 1] > m[row - 1, col]))
+            {
+                diff.Add((formatter(baselineSequence[row - 1]), DifferenceKind.Removed));
+                row--;
+            }
+            else
+            {
+                throw new UnreachableException();
+            }
+        }
+        diff.Reverse();
+        return diff;
+    }
+
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs

@@ -1,23 +1,14 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Build.Framework;
-using Microsoft.Build.Utilities;
 using Task = System.Threading.Tasks.Task;
 
 public class FindArchiveDiffs : Microsoft.Build.Utilities.Task, ICancelableTask
 {
-    public class ArchiveItem
-    {
-        public required string Path { get; init; }
-    }
-
     [Required]
     public required ITaskItem BaselineArchive { get; init; }
 
@@ -29,10 +20,6 @@ public class FindArchiveDiffs : Microsoft.Build.Utilities.Task, ICancelableTask
 
     private CancellationTokenSource _cancellationTokenSource = new();
     private CancellationToken cancellationToken => _cancellationTokenSource.Token;
-    public void Cancel()
-    {
-        _cancellationTokenSource.Cancel();
-    }
 
     public override bool Execute()
     {
@@ -49,98 +36,14 @@ public class FindArchiveDiffs : Microsoft.Build.Utilities.Task, ICancelableTask
         var baselineFiles = baseline.GetFileNames();
         var testFiles = test.GetFileNames();
         ContentDifferences =
-            GetDiffs(baselineFiles, testFiles, PathWithVersions.Equal, PathWithVersions.GetVersionlessPath, cancellationToken)
-            .Select(FromDiff)
+            Diff.GetDiffs(baselineFiles, testFiles, VersionIdentifier.AreVersionlessEqual, static p => VersionIdentifier.RemoveVersions(p, "{VERSION}"), cancellationToken)
+            .Select(Diff.TaskItemFromDiff)
             .ToArray();
         return true;
     }
 
-    static ITaskItem FromDiff((string, DifferenceKind) diff)
+    public void Cancel()
     {
-        var item = new TaskItem(diff.Item1);
-        item.SetMetadata("Kind", Enum.GetName(diff.Item2));
-        return item;
-    }
-
-    public enum DifferenceKind
-    {
-        Added,
-        Removed,
-        Unchanged
-    }
-
-    public static List<(string, DifferenceKind DifferenceKind)> GetDiffs(
-        string[] originalPathsWithVersions,
-        string[] modifiedPathsWithVersions,
-        Func<string, string, bool> equalityComparer,
-        Func<string, string>? formatter = null,
-        CancellationToken cancellationToken = default)
-    {
-        cancellationToken.ThrowIfCancellationRequested();
-        formatter ??= static s => s;
-        // Edit distance algorithm: https://en.wikipedia.org/wiki/Longest_common_subsequence
-
-        int[,] dp = new int[originalPathsWithVersions.Length + 1, modifiedPathsWithVersions.Length + 1];
-
-        // Initialize first row and column
-        for (int i = 0; i <= originalPathsWithVersions.Length; i++)
-        {
-            dp[i, 0] = i;
-        }
-        for (int j = 0; j <= modifiedPathsWithVersions.Length; j++)
-        {
-            dp[0, j] = j;
-        }
-
-        // Compute edit distance
-        for (int i = 1; i <= originalPathsWithVersions.Length; i++)
-        {
-            cancellationToken.ThrowIfCancellationRequested();
-            for (int j = 1; j <= modifiedPathsWithVersions.Length; j++)
-            {
-                if (equalityComparer(originalPathsWithVersions[i - 1], modifiedPathsWithVersions[j - 1]))
-                {
-                    dp[i, j] = dp[i - 1, j - 1];
-                }
-                else
-                {
-                    dp[i, j] = 1 + Math.Min(dp[i - 1, j], dp[i, j - 1]);
-                }
-            }
-        }
-
-        // Trace back the edits
-        int row = originalPathsWithVersions.Length;
-        int col = modifiedPathsWithVersions.Length;
-
-        List<(string, DifferenceKind)> formattedDiff = [];
-        while (row > 0 || col > 0)
-        {
-            cancellationToken.ThrowIfCancellationRequested();
-            var baselineItem = originalPathsWithVersions[row - 1];
-            var testItem = modifiedPathsWithVersions[col - 1];
-            if (row > 0 && col > 0 && PathWithVersions.Equal(baselineItem, testItem))
-            {
-                formattedDiff.Add((formatter(originalPathsWithVersions[row - 1]), DifferenceKind.Unchanged));
-                row--;
-                col--;
-            }
-            else if (col > 0 && (row == 0 || dp[row, col - 1] <= dp[row - 1, col]))
-            {
-                formattedDiff.Add((formatter(modifiedPathsWithVersions[col - 1]), DifferenceKind.Added));
-                col--;
-            }
-            else if (row > 0 && (col == 0 || dp[row, col - 1] > dp[row - 1, col]))
-            {
-                formattedDiff.Add((formatter(originalPathsWithVersions[row - 1]), DifferenceKind.Removed));
-                row--;
-            }
-            else
-            {
-                throw new UnreachableException();
-            }
-        }
-        formattedDiff.Reverse();
-        return formattedDiff;
+        _cancellationTokenSource.Cancel();
     }
 }

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchiveVersion.cs

@@ -0,0 +1,94 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Build.Framework;
+
+public abstract class GetClosestArchive : Microsoft.Build.Utilities.Task, ICancelableTask
+{
+    [Required]
+    public required string BuiltArchivePath { get; init; }
+
+    [Output]
+    public string ClosestOfficialArchivePath { get; set; } = "";
+
+    private string? _builtVersion;
+    protected string BuiltVersion
+    {
+        get => _builtVersion ?? throw new InvalidOperationException();
+        private set => _builtVersion = value;
+    }
+
+    private string? _builtRid;
+    protected string BuiltRid
+    {
+        get => _builtRid ?? throw new InvalidOperationException();
+        private set => _builtRid = value;
+    }
+
+    private string? _archiveExtension;
+    protected string ArchiveExtension
+    {
+        get => _archiveExtension ?? throw new InvalidOperationException();
+        private set => _archiveExtension = value;
+    }
+
+    /// <summary>
+    /// The name of the package to find the closest official archive for. For example, "dotnet-sdk" or "aspnetcore-runtime".
+    /// </summary>
+    protected abstract string ArchiveName { get; }
+
+    private CancellationTokenSource _cancellationTokenSource = new();
+    protected CancellationToken CancellationToken => _cancellationTokenSource.Token;
+    public void Cancel()
+    {
+        _cancellationTokenSource.Cancel();
+    }
+
+    /// <summary>
+    /// Get the URL of the latest official archive for the given version string and RID.
+    /// </summary>
+    public abstract Task<string?> GetLatestOfficialArchiveUrl();
+
+    public abstract Task<string?> GetClosestOfficialArchiveVersion();
+
+    public override bool Execute()
+    {
+        return Task.Run(ExecuteAsync).Result;
+    }
+
+    public async Task<bool> ExecuteAsync()
+    {
+        CancellationToken.ThrowIfCancellationRequested();
+        var filename = Path.GetFileName(BuiltArchivePath);
+        (BuiltVersion, BuiltRid, ArchiveExtension) = Archive.GetInfoFromFileName(filename, ArchiveName);
+        Log.LogMessage($"Finding closest official archive for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'");
+
+        string? downloadUrl = await GetLatestOfficialArchiveUrl();
+        if (downloadUrl == null)
+        {
+            Log.LogError($"Failed to find a download URL for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'");
+            return false;
+        }
+
+        HttpClient client = new HttpClient();
+
+        Log.LogMessage(MessageImportance.High, $"Downloading {downloadUrl}");
+        HttpResponseMessage packageResponse = await client.GetAsync(downloadUrl, CancellationToken);
+
+        var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath;
+
+        ClosestOfficialArchivePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + $".{ArchiveName}-{BuiltVersion}-{BuiltRid}.closest.{ArchiveExtension}");
+        Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialArchivePath}");
+        using (var file = File.Create(ClosestOfficialArchivePath))
+        {
+            await packageResponse.Content.CopyToAsync(file, CancellationToken);
+        }
+
+        return true;
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs

@@ -1,73 +1,44 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.IO;
 using System.Net;
 using System.Net.Http;
-using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Build.Framework;
 
-public class GetClosestOfficialSdk : Microsoft.Build.Utilities.Task, ICancelableTask
+public class GetClosestOfficialSdk : GetClosestArchive
 {
-    [Required]
-    public required string BuiltSdkPath { get; init; }
+    protected override string ArchiveName => "dotnet-sdk";
 
-    [Output]
-    public string ClosestOfficialSdkPath { get; set; } = "";
+    HttpClient client = new HttpClient(new HttpClientHandler() { AllowAutoRedirect = false });
 
-    public override bool Execute()
+    private string? closestVersion;
+    private string? closestUrl;
+
+    public override async Task<string?> GetLatestOfficialArchiveUrl()
     {
-        return Task.Run(ExecuteAsync).Result;
-    }
-
-    private CancellationTokenSource _cancellationTokenSource = new();
-    private CancellationToken cancellationToken => _cancellationTokenSource.Token;
-    public void Cancel()
-    {
-        _cancellationTokenSource.Cancel();
-    }
-
-    public async Task<bool> ExecuteAsync()
-    {
-        cancellationToken.ThrowIfCancellationRequested();
-        var (versionString, rid, extension) = Archive.GetInfoFromArchivePath(BuiltSdkPath);
-
-        string downloadUrl = GetLatestOfficialSdkUrl(versionString, rid, extension);
-
-        Log.LogMessage(MessageImportance.High, $"Downloading {downloadUrl}");
-        var handler = new HttpClientHandler()
-        {
-            AllowAutoRedirect = false
-        };
-        var client = new HttpClient(handler);
-        var redirectResponse = await client.GetAsync(downloadUrl, cancellationToken);
+        // Channel in the form of 9.0.1xx
+        var channel = BuiltVersion[..5] + "xx";
+        var akaMsUrl = $"https://aka.ms/dotnet/{channel}/daily/{ArchiveName}-{BuiltRid}{ArchiveExtension}";
+        var redirectResponse = await client.GetAsync(akaMsUrl, CancellationToken);
         // aka.ms returns a 301 for valid redirects and a 302 to Bing for invalid URLs
         if (redirectResponse.StatusCode != HttpStatusCode.Moved)
         {
-            Log.LogMessage(MessageImportance.High, $"Failed to download '{downloadUrl}': invalid aka.ms URL");
-            return true;
+            Log.LogMessage(MessageImportance.High, $"Failed to find package at '{akaMsUrl}': invalid aka.ms URL");
+            return null;
         }
-        var packageResponse = await client.GetAsync(redirectResponse.Headers.Location!, cancellationToken);
-
-        var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath;
-        string downloadedVersion = PathWithVersions.GetVersionInPath(packageUriPath).ToString();
-
-        ClosestOfficialSdkPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + $".dotnet-sdk-{downloadedVersion}-{rid}{extension}");
-        Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialSdkPath}");
-        using (var file = File.Create(ClosestOfficialSdkPath))
-        {
-            await packageResponse.Content.CopyToAsync(file, cancellationToken);
-        }
-
-        return true;
+        closestUrl = redirectResponse.Headers.Location!.ToString();
+        closestVersion = VersionIdentifier.GetVersion(closestUrl);
+        return closestUrl;
     }
 
-    string GetLatestOfficialSdkUrl(string versionString, string rid, string extension)
+    public override async Task<string?> GetClosestOfficialArchiveVersion()
     {
-        // Channel in the form of 9.0.1xx
-        var channel = versionString[..5] + "xx";
-        return $"https://aka.ms/dotnet/{channel}/daily/dotnet-sdk-{rid}{extension}";
+        if (closestUrl is not null)
+        {
+            return closestVersion;
+        }
+        _ = await GetLatestOfficialArchiveUrl();
+        return closestVersion;
     }
-
 }

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs

@@ -5,24 +5,27 @@ using System;
 using System.Collections.Generic;
 using Microsoft.Build.Framework;
 
-public class GetSingleArchiveItem : Microsoft.Build.Utilities.Task
+public class GetValidArchiveItems : Microsoft.Build.Utilities.Task
 {
     [Required]
-    public required ITaskItem[] SdkArchiveItems { get; init; }
+    public required ITaskItem[] ArchiveItems { get; init; }
+
+    [Required]
+    public required string ArchiveName { get; init; }
 
     [Output]
-    public string BestSdkArchiveItem { get; set; } = "";
+    public ITaskItem[] ValidArchiveItems { get; set; } = [];
 
     public override bool Execute()
     {
-        List<string> archiveItems = new ();
-        foreach(var item in SdkArchiveItems)
+        List<ITaskItem> archiveItems = new();
+        foreach (var item in ArchiveItems)
         {
             try
             {
                 // Ensure the version and RID info can be parsed from the item
-                _ = Archive.GetInfoFromArchivePath(item.ItemSpec);
-                archiveItems.Add(item.ItemSpec);
+                _ = Archive.GetInfoFromFileName(item.ItemSpec, ArchiveName);
+                archiveItems.Add(item);
             }
             catch (ArgumentException e)
             {
@@ -30,20 +33,20 @@ public class GetSingleArchiveItem : Microsoft.Build.Utilities.Task
                 continue;
             }
         }
-        switch (archiveItems.Count){
+        switch (archiveItems.Count)
+        {
             case 0:
                 Log.LogMessage(MessageImportance.High, "No valid archive items found");
-                BestSdkArchiveItem = "";
-                break;
+                ValidArchiveItems = [];
+                return false;
             case 1:
                 Log.LogMessage(MessageImportance.High, $"{archiveItems[0]} is the only valid archive item found");
-                BestSdkArchiveItem = archiveItems[0];
+                ValidArchiveItems = archiveItems.ToArray();
                 break;
             default:
-                archiveItems.Sort((a,b) => a.Length - b.Length);
+                archiveItems.Sort((a, b) => a.ItemSpec.Length - b.ItemSpec.Length);
                 Log.LogMessage(MessageImportance.High, $"Multiple valid archive items found: '{string.Join("', '", archiveItems)}'");
-                BestSdkArchiveItem = archiveItems[0];
-                Log.LogMessage(MessageImportance.High, $"Choosing '{BestSdkArchiveItem}");
+                ValidArchiveItems = archiveItems.ToArray();
                 break;
         }
         return true;

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs

@@ -1,112 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-
-using System;
-using System.IO;
-using System.Text;
-
-public static class PathWithVersions
-{
-    public const string VersionPlaceholder = "{VERSION}";
-
-    public static bool Equal(string path1, string path2)
-    {
-        if (path1 == path2)
-        {
-            return true;
-        }
-
-        ReadOnlySpan<char> directory = path1;
-        ReadOnlySpan<char> directory2 = path2;
-        while (TryGetPathLeaf(directory, out var root, out var directoryPart) && TryGetPathLeaf(directory2, out var root2, out var directoryPart2))
-        {
-            if (!ReplaceVersionString(directoryPart).SequenceEqual(ReplaceVersionString(directoryPart2)))
-            {
-                return false;
-            }
-            directory= Path.GetDirectoryName(directory);
-            directory2= Path.GetDirectoryName(directory2);
-        }
-        if (!directory.IsEmpty || !directory2.IsEmpty)
-        {
-            return false;
-        }
-        return true;
-    }
-
-    public static bool IsVersionString(ReadOnlySpan<char> directoryPart)
-    {
-        return directoryPart.Length >= 6
-            && char.IsDigit(directoryPart[0])
-            && directoryPart[1] == '.'
-            && char.IsDigit(directoryPart[2])
-            && directoryPart[3] == '.'
-            && char.IsDigit(directoryPart[4])
-            && ((char.IsDigit(directoryPart[5]) && char.IsDigit(directoryPart[6])) || directoryPart[5] == '-');
-    }
-
-    static ReadOnlySpan<char> ReplaceVersionString(ReadOnlySpan<char> directoryPart)
-    {
-        if (IsVersionString(directoryPart))
-        {
-            return VersionPlaceholder;
-        }
-        else
-        {
-            return directoryPart;
-        }
-    }
-
-    static bool TryGetPathLeaf(ReadOnlySpan<char> path, out ReadOnlySpan<char> root, out ReadOnlySpan<char> leaf)
-    {
-        if (path.IsEmpty)
-        {
-            root = default;
-            leaf = default;
-            return false;
-        }
-        leaf = Path.GetFileName(path);
-        root = Path.GetDirectoryName(path);
-        return true;
-    }
-
-    public static string GetVersionlessPath(string path)
-    {
-        return GetVersionlessPath(path.AsSpan()).ToString();
-    }
-
-    public static ReadOnlySpan<char> GetVersionlessPath(ReadOnlySpan<char> path)
-    {
-        StringBuilder sb = new StringBuilder();
-        bool altered = false;
-        ReadOnlySpan<char> myPath = path;
-        while (TryGetPathLeaf(myPath, out var root, out var leaf))
-        {
-            var versionOrDirectory = ReplaceVersionString(leaf);
-            if (versionOrDirectory == VersionPlaceholder)
-            {
-                altered = true;
-            }
-            sb = sb.Insert(0, versionOrDirectory);
-            sb = sb.Insert(0, Path.DirectorySeparatorChar);
-            myPath = root;
-        }
-        if (!altered)
-            return path;
-        return sb.ToString();
-    }
-
-    public static ReadOnlySpan<char> GetVersionInPath(ReadOnlySpan<char> path)
-    {
-        ReadOnlySpan<char> myPath = path;
-        while (TryGetPathLeaf(myPath, out var directory, out var directoryPart))
-        {
-            if (IsVersionString(directoryPart))
-            {
-                return directoryPart;
-            }
-            myPath = directory;
-        }
-        throw new ArgumentException("Path does not contain a version");
-    }
-}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs

@@ -0,0 +1,257 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+// Copied from https://github.com/dotnet/arcade/blob/903d35c426248ade56719db5bc0947cd7634b297/eng/common/native/init-distro-rid.sh
+// Conflicting MSBuild versions and some customizations make it difficult to use the Arcade assembly.
+public static class VersionIdentifier
+{
+    private static readonly HashSet<string> _knownTags = new HashSet<string>
+        {
+            "alpha",
+            "beta",
+            "preview",
+            "prerelease",
+            "servicing",
+            "rtm",
+            "rc"
+        };
+
+    private static readonly SortedDictionary<string, string> _sequencesToReplace =
+        new SortedDictionary<string, string>
+        {
+                { "-.", "." },
+                { "..", "." },
+                { "--", "-" },
+                { "//", "/" },
+                { "_.", "." }
+        };
+
+    private const string _finalSuffix = "final";
+
+    private static readonly char[] _delimiters = new char[] { '.', '-', '_' };
+
+    /// <summary>
+    /// Identify the version of an asset.
+    ///
+    /// Asset names can come in two forms:
+    /// - Blobs that include the full path
+    /// - Packages that do not include any path elements.
+    ///
+    /// There may be multiple different version numbers in a blob path.
+    /// This method starts at the last segment of the path and works backward to find a version number.
+    /// </summary>
+    /// <param name="assetName">Asset name</param>
+    /// <returns>Version, or null if none is found.</returns>
+    public static string? GetVersion(string assetName)
+    {
+        string[] pathSegments = assetName.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
+        string? potentialVersion = null;
+        for (int i = pathSegments.Length - 1; i >= 0; i--)
+        {
+            potentialVersion = GetVersionForSingleSegment(pathSegments[i]);
+            if (potentialVersion != null)
+            {
+                return potentialVersion;
+            }
+        }
+
+        return potentialVersion;
+    }
+
+    /// <summary>
+    /// Identify the version number of an asset segment.
+    /// </summary>
+    /// <param name="assetPathSegment">Asset segment</param>
+    /// <returns>Version number, or null if none was found</returns>
+    /// <remarks>
+    /// Identifying versions is not particularly easy. To constrain the problem,
+    /// we apply the following assumptions which are generally valid for .NET Core.
+    /// - We always have major.minor.patch, and it always begins the version string.
+    /// - The only pre-release or build metadata labels we use begin with the _knownTags shown above.
+    /// - We use additional numbers in our version numbers after the initial
+    ///   major.minor.patch-prereleaselabel.prereleaseiteration segment,
+    ///   but any non-numeric element will end the version string.
+    /// - The <see cref="_delimiters"/> we use in versions and file names are ., -, and _.
+    /// </remarks>
+    private static string? GetVersionForSingleSegment(string assetPathSegment)
+    {
+
+        // Find the start of the version number by finding the major.minor.patch.
+        // Scan the string forward looking for a digit preceded by one of the delimiters,
+        // then look for a minor.patch, completing the major.minor.patch.  Continue to do so until we get
+        // to something that is NOT major.minor.patch (this is necessary because we sometimes see things like:
+        // VS.Redist.Common.NetCore.Templates.x86.2.2.3.0.101-servicing-014320.nupkg
+        // Continue iterating until we find ALL potential versions. Return the one that is the latest in the segment
+        // This is to deal with files with multiple major.minor.patchs in the file name, for example:
+        // Microsoft.NET.Workload.Mono.ToolChain.Manifest-6.0.100.Msi.x64.6.0.0-rc.1.21380.2.symbols.nupkg
+
+        int currentIndex = 0;
+        // Stack of major.minor.patch.
+        Stack<(int versionNumber, int index)> majorMinorPatchStack = new Stack<(int, int)>(3);
+        string? majorMinorPatch = null;
+        int majorMinorPatchIndex = 0;
+        StringBuilder versionSuffix = new StringBuilder();
+        char prevDelimiterCharacter = char.MinValue;
+        char nextDelimiterCharacter = char.MinValue;
+        Dictionary<int, string> majorMinorPatchDictionary = new Dictionary<int, string>();
+        while (true)
+        {
+            string nextSegment;
+            prevDelimiterCharacter = nextDelimiterCharacter;
+            int nextDelimiterIndex = assetPathSegment.IndexOfAny(_delimiters, currentIndex);
+            if (nextDelimiterIndex != -1)
+            {
+                nextDelimiterCharacter = assetPathSegment[nextDelimiterIndex];
+                nextSegment = assetPathSegment.Substring(currentIndex, nextDelimiterIndex - currentIndex);
+            }
+            else
+            {
+                nextSegment = assetPathSegment.Substring(currentIndex);
+            }
+
+            // If we have not yet found the major/minor/patch, then there are four cases:
+            // - There have been no potential major/minor/patch numbers found and the current segment is a number. Push onto the majorMinorPatch stack
+            //   and continue.
+            // - There has been at least one number found, but less than 3, and the current segment not a number or not preceded by '.'. In this case,
+            //   we should clear out the stack and continue the search.
+            // - There have been at least 2 numbers found and the current segment is a number and preceded by '.'. Push onto the majorMinorPatch stack and continue
+            // - There have been at least 3 numbers found and the current segment is not a number or not preceded by '-'. In this case, we can call this the major minor
+            //   patch number and no longer need to continue searching
+            if (majorMinorPatch == null)
+            {
+                bool isNumber = int.TryParse(nextSegment, out int potentialVersionSegment);
+                if ((majorMinorPatchStack.Count == 0 && isNumber) ||
+                    (majorMinorPatchStack.Count > 0 && prevDelimiterCharacter == '.' && isNumber))
+                {
+                    majorMinorPatchStack.Push((potentialVersionSegment, currentIndex));
+                }
+                // Check for partial major.minor.patch cases, like: 2.2.bar or 2.2-100.bleh
+                else if (majorMinorPatchStack.Count > 0 && majorMinorPatchStack.Count < 3 &&
+                         (prevDelimiterCharacter != '.' || !isNumber))
+                {
+                    majorMinorPatchStack.Clear();
+                }
+
+                // Determine whether we are done with major.minor.patch after this update.
+                if (majorMinorPatchStack.Count >= 3 && (prevDelimiterCharacter != '.' || !isNumber || nextDelimiterIndex == -1))
+                {
+                    // Done with major.minor.patch, found. Pop the top 3 elements off the stack.
+                    (int patch, int patchIndex) = majorMinorPatchStack.Pop();
+                    (int minor, int minorIndex) = majorMinorPatchStack.Pop();
+                    (int major, int majorIndex) = majorMinorPatchStack.Pop();
+                    majorMinorPatch = $"{major}.{minor}.{patch}";
+                    majorMinorPatchIndex = majorIndex;
+                }
+            }
+
+            // Don't use else, so that we don't miss segments
+            // in case we are just deciding that we've finished major minor patch.
+            if (majorMinorPatch != null)
+            {
+                // Now look at the next segment. If it looks like it could be part of a version, append to what we have
+                // and continue. If it can't, then we're done.
+                //
+                // Cases where we should break out and be done:
+                // - We have an empty pre-release label and the delimiter is not '-'.
+                // - We have an empty pre-release label and the next segment does not start with a known tag.
+                // - We have a non-empty pre-release label and the current segment is not a number and also not 'final'
+                //      A corner case of versioning uses .final to represent a non-date suffixed final pre-release version:
+                //      3.1.0-preview.10.final
+                if (versionSuffix.Length == 0 &&
+                    (prevDelimiterCharacter != '-' || !_knownTags.Any(tag => nextSegment.StartsWith(tag, StringComparison.OrdinalIgnoreCase))))
+                {
+                    majorMinorPatchDictionary.Add(majorMinorPatchIndex, majorMinorPatch);
+                    majorMinorPatch = null;
+                    versionSuffix = new StringBuilder();
+                }
+                else if (versionSuffix.Length != 0 && !int.TryParse(nextSegment, out int potentialVersionSegment) && nextSegment != _finalSuffix)
+                {
+                    majorMinorPatchDictionary.Add(majorMinorPatchIndex, $"{majorMinorPatch}{versionSuffix.ToString()}");
+                    majorMinorPatch = null;
+                    versionSuffix = new StringBuilder();
+                }
+                else
+                {
+                    // Append the delimiter character and then the current segment
+                    versionSuffix.Append(prevDelimiterCharacter);
+                    versionSuffix.Append(nextSegment);
+                }
+            }
+
+            if (nextDelimiterIndex != -1)
+            {
+                currentIndex = nextDelimiterIndex + 1;
+            }
+            else
+            {
+                break;
+            }
+        }
+
+        if (majorMinorPatch != null)
+        {
+            majorMinorPatchDictionary.Add(majorMinorPatchIndex, $"{majorMinorPatch}{versionSuffix.ToString()}");
+        }
+
+        if (!majorMinorPatchDictionary.Any())
+        {
+            return null;
+        }
+
+        int maxKey = majorMinorPatchDictionary.Keys.Max();
+        return majorMinorPatchDictionary[maxKey];
+    }
+
+    /// <summary>
+    ///     Given an asset name, remove all .NET Core version numbers (as defined by GetVersionForSingleSegment)
+    ///     from the string
+    /// </summary>
+    /// <param name="assetName">Asset</param>
+    /// <returns>Asset name without versions</returns>
+    public static string RemoveVersions(string assetName, string replacement = "")
+    {
+        string[] pathSegments = assetName.Split('/');
+
+        // Remove the version number from each segment, then join back together and
+        // remove any useless character sequences.
+
+        for (int i = 0; i < pathSegments.Length; i++)
+        {
+            if (!string.IsNullOrEmpty(pathSegments[i]))
+            {
+                string? versionForSegment = GetVersionForSingleSegment(pathSegments[i]);
+                if (versionForSegment != null)
+                {
+                    pathSegments[i] = pathSegments[i].Replace(versionForSegment, replacement);
+                }
+            }
+        }
+
+        // Continue replacing things until there is nothing left to replace.
+        string assetWithoutVersions = string.Join("/", pathSegments);
+        bool anyReplacements = true;
+        while (anyReplacements)
+        {
+            string replacementIterationResult = assetWithoutVersions;
+            foreach (var sequence in _sequencesToReplace)
+            {
+                replacementIterationResult = replacementIterationResult.Replace(sequence.Key, sequence.Value);
+            }
+            anyReplacements = replacementIterationResult != assetWithoutVersions;
+            assetWithoutVersions = replacementIterationResult;
+        }
+
+        return assetWithoutVersions;
+    }
+
+
+    public static bool AreVersionlessEqual(string assetName1, string assetName2)
+    {
+        return RemoveVersions(assetName1) == RemoveVersions(assetName2);
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs

@@ -1,78 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-
-using System;
-using System.IO;
-using System.IO.Compression;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-static class ArchiveExtensions
-{
-    public static string[] Lines(this ZipArchiveEntry entry, Encoding? encoding = null)
-    {
-        return entry.ReadToString(encoding).Replace("\r\n", "\n").Split('\n').ToArray();
-    }
-
-    public static string ReadToString(this ZipArchiveEntry entry, Encoding? encoding = null)
-    {
-        Stream stream = entry.Open();
-        byte[] buffer = stream.ReadToEnd();
-        // Remove UTF-8 BOM if present
-        int index = 0;
-        if (buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF)
-        {
-            index = 3;
-        }
-        encoding ??= Encoding.UTF8;
-        string fileText = encoding.GetString(buffer, index, buffer.Length - index);
-        return fileText;
-    }
-
-    public static byte[] ReadToEnd(this Stream stream)
-    {
-        int bufferSize = 2048;
-        byte[] buffer = new byte[bufferSize];
-        int offset = 0;
-        while (true)
-        {
-            int bytesRead = stream.Read(buffer, offset, bufferSize - offset);
-            offset += bytesRead;
-            if (bytesRead == 0)
-            {
-                break;
-            }
-            if (offset == bufferSize)
-            {
-                Array.Resize(ref buffer, bufferSize * 2);
-                bufferSize *= 2;
-            }
-        }
-        Array.Resize(ref buffer, offset);
-        return buffer;
-    }
-
-    public static async Task<byte[]> ReadToEndAsync(this Stream stream)
-    {
-        int bufferSize = 2048;
-        byte[] buffer = new byte[bufferSize];
-        int offset = 0;
-        while (true)
-        {
-            int bytesRead = await stream.ReadAsync(buffer, offset, bufferSize - offset);
-            offset += bytesRead;
-            if (bytesRead == 0)
-            {
-                break;
-            }
-            if (offset == bufferSize)
-            {
-                Array.Resize(ref buffer, bufferSize * 2);
-                bufferSize *= 2;
-            }
-        }
-        Array.Resize(ref buffer, offset);
-        return buffer;
-    }
-}