Add SdkArchiveDiff task to verify the sdk archive has all the expected outputs

src/SourceBuild/content/Directory.Build.props

@@ -199,6 +199,7 @@
   <PropertyGroup>
     <XPlatSourceBuildTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat.dll'))</XPlatSourceBuildTasksAssembly>
     <LeakDetectionTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll'))</LeakDetectionTasksAssembly>
+    <SdkArchiveDiffTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.dll'))</SdkArchiveDiffTasksAssembly>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(EnablePoison)' == 'true'">

src/SourceBuild/content/build.proj

@@ -20,5 +20,6 @@
   </Target>
 
   <Import Project="$(RepositoryEngineeringDir)build.sourcebuild.targets" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+  <Import Project="$(RepositoryEngineeringDir)build.targets" />
 
 </Project>

src/SourceBuild/content/eng/build.targets

@@ -0,0 +1,27 @@
+<Project>
+  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="FindArchiveDiffs" />
+  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="GetClosestOfficialSdk" />
+
+  <Target Name="ReportSdkArchiveDiffs"
+          AfterTargets="Build"
+          DependsOnTargets="DetermineSourceBuiltSdkVersion"
+          Condition="'$(ReportSdkArchiveDiffs)' == 'true'">
+
+      <PropertyGroup>
+          <SdkArchiveDiffOutputPath>$(OutDir)</SdkArchiveDiffOutputPath>
+      </PropertyGroup>
+
+      <GetClosestOfficialSdk BuiltSdkPath="@(SdkTarballItem)">
+        <Output TaskParameter="ClosestOfficialSdkPath" PropertyName="ClosestOfficialSdkPath" />
+      </GetClosestOfficialSdk>
+
+      <FindArchiveDiffs BaselineArchive="@(SdkTarballItem)" TestArchive="$(ClosestOfficialSdkPath)">
+        <Output TaskParameter="ContentDifferences" ItemName="ContentDifferences" />
+      </FindArchiveDiffs>
+
+      <Message Text="Difference in sdk archive: %(ContentDifferences.Kind): %(ContentDifferences.Identity)" Importance="High" Condition="'%(ContentDifferences.Kind)' != 'Unchanged'"/>
+      <Delete Files="$(ClosestOfficialSdkPath)" />
+
+  </Target>
+
+</Project>

src/SourceBuild/content/eng/tools/init-build.proj

@@ -15,6 +15,7 @@
       UnpackTarballs;
       BuildXPlatTasks;
       BuildMSBuildSdkResolver;
+      BuildTarballDiff;
       BuildLeakDetection;
       ExtractToolPackage;
       GenerateRootFs;
@@ -116,6 +117,14 @@
     </Touch>
   </Target>
 
+  <Target Name="BuildSdkArchiveDiff" >
+    <MSBuild Projects="tasks\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj"
+             Targets="Restore"
+             Properties="MSBuildRestoreSessionId=$([System.Guid]::NewGuid())" />
+    <MSBuild Projects="tasks\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj"
+             Targets="Build" />
+  </Target>
+
   <Target Name="GenerateRootFs"
           Condition="'$(BuildOS)' != 'windows' and '$(CrossBuild)' == 'true' and '$(ROOTFS_DIR)' == ''">
     <PropertyGroup>

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs

@@ -0,0 +1,137 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Immutable;
+using System.Formats.Tar;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Reflection;
+using System.Reflection.Metadata;
+using System.Reflection.PortableExecutable;
+using System.Threading;
+using System.Threading.Tasks;
+using static ArchiveExtensions;
+
+public abstract class Archive : IDisposable
+{
+    public static async Task<Archive> Create(string path)
+    {
+        if (path.EndsWith(".tar.gz"))
+            return await TarArchive.Create(path);
+        else if (path.EndsWith(".zip"))
+            return ZipFileArchive.Create(path);
+        else
+            throw new NotSupportedException("Unsupported archive type");
+    }
+
+    public abstract bool Contains(string relativePath);
+
+    public abstract string[] GetFileNames();
+
+    public abstract string[] GetFileLines(string relativePath);
+
+    public abstract Task<byte[]> GetFileBytesAsync(string relativePath);
+
+    public abstract void Dispose();
+
+    public class TarArchive : Archive
+    {
+        private string _extractedFolder;
+
+        private TarArchive(string extractedFolder)
+        {
+            _extractedFolder = extractedFolder;
+        }
+
+        public static async Task<TarArchive> Create(string path, CancellationToken cancellationToken = default)
+        {
+            var tmpFolder = Directory.CreateTempSubdirectory(nameof(FindArchiveDiffs));
+            using (var gzStream = File.OpenRead (path))
+            using (var gzipStream = new GZipStream (gzStream, CompressionMode.Decompress))
+            {
+                await TarFile.ExtractToDirectoryAsync(gzipStream, tmpFolder.FullName, true, cancellationToken);
+            }
+            return new TarArchive(tmpFolder.FullName);
+        }
+
+        public override bool Contains(string relativePath)
+        {
+            return File.Exists(Path.Combine(_extractedFolder, relativePath));
+        }
+
+        public override string[] GetFileNames()
+        {
+            return Directory.GetFiles(_extractedFolder, "*", SearchOption.AllDirectories).Select(f => f.Substring(_extractedFolder.Length + 1)).ToArray();
+        }
+
+        public override string[] GetFileLines(string relativePath)
+        {
+            return File.ReadAllLines(Path.Combine(_extractedFolder, relativePath));
+        }
+
+        public override Task<byte[]> GetFileBytesAsync(string relativePath)
+        {
+            var filePath = Path.Combine(_extractedFolder, relativePath);
+            if (!File.Exists(filePath))
+                return Task.FromResult<byte[]>([]);
+            return File.ReadAllBytesAsync(Path.Combine(_extractedFolder, relativePath));
+        }
+
+        public override void Dispose()
+        {
+            if (Directory.Exists(_extractedFolder))
+                Directory.Delete(_extractedFolder, true);
+        }
+    }
+
+    public class ZipFileArchive : Archive
+    {
+        private ZipArchive _archive;
+
+        private ZipFileArchive(ZipArchive archive)
+        {
+            _archive = archive;
+        }
+
+        public static new ZipFileArchive Create(string path)
+        {
+            return new ZipFileArchive(new ZipArchive(File.OpenRead(path)));
+        }
+
+        public override bool Contains(string relativePath)
+        {
+            return _archive.GetEntry(relativePath) != null;
+        }
+
+        public override string[] GetFileNames()
+        {
+            return _archive.Entries.Select(e => e.FullName).ToArray();
+        }
+
+        public override string[] GetFileLines(string relativePath)
+        {
+            var entry = _archive.GetEntry(relativePath);
+            if (entry == null)
+                throw new ArgumentException("File not found");
+            return entry.Lines();
+        }
+        public override Task<byte[]> GetFileBytesAsync(string relativePath)
+        {
+            using (var entry = _archive.GetEntry(relativePath)?.Open())
+            {
+                if (entry == null)
+                {
+                    return Task.FromResult<byte[]>([]);
+                }
+                return entry.ReadToEndAsync();
+            }
+        }
+
+        public override void Dispose()
+        {
+            _archive.Dispose();
+        }
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs

@@ -0,0 +1,134 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using Task = System.Threading.Tasks.Task;
+
+public class FindArchiveDiffs : Microsoft.Build.Utilities.Task
+{
+    public class ArchiveItem
+    {
+        public required string Path { get; init; }
+    }
+
+    [Required]
+    public required ITaskItem BaselineArchive { get; init; }
+
+    [Required]
+    public required ITaskItem TestArchive { get; init; }
+
+    [Output]
+    public ITaskItem[] ContentDifferences { get; set; } = [];
+
+    public override bool Execute()
+    {
+        return Task.Run(ExecuteAsync).Result;
+    }
+
+    public async Task<bool> ExecuteAsync()
+    {
+        var baselineTask = Archive.Create(BaselineArchive.ItemSpec);
+        var testTask = Archive.Create(TestArchive.ItemSpec);
+        Task.WaitAll(baselineTask, testTask);
+        using var baseline = await baselineTask;
+        using var test = await testTask;
+        var baselineFiles = baseline.GetFileNames();
+        var testFiles = test.GetFileNames();
+        ContentDifferences =
+            GetDiffs(baselineFiles, testFiles, PathWithVersions.Equal, PathWithVersions.GetVersionAnonymousPath)
+            .Select(FromDiff)
+            .ToArray();
+        return true;
+    }
+
+    static ITaskItem FromDiff((string, DifferenceKind) diff)
+    {
+        var item = new TaskItem(diff.Item1);
+        item.SetMetadata("Kind", Enum.GetName(diff.Item2));
+        return item;
+    }
+
+    public enum DifferenceKind
+    {
+        Added,
+        Removed,
+        Unchanged
+    }
+
+    public static List<(string, DifferenceKind DifferenceKind)> GetDiffs(
+        string[] originalPathsWithVersions,
+        string[] modifiedPathsWithVersions,
+        Func<string, string, bool> equalityComparer,
+        Func<string, string>? formatter = null)
+    {
+        formatter ??= static s => s;
+        // Edit distance algorithm: https://en.wikipedia.org/wiki/Longest_common_subsequence
+
+        int[,] dp = new int[originalPathsWithVersions.Length + 1, modifiedPathsWithVersions.Length + 1];
+
+        // Initialize first row and column
+        for (int i = 0; i <= originalPathsWithVersions.Length; i++)
+        {
+            dp[i, 0] = i;
+        }
+        for (int j = 0; j <= modifiedPathsWithVersions.Length; j++)
+        {
+            dp[0, j] = j;
+        }
+
+        // Compute edit distance
+        for (int i = 1; i <= originalPathsWithVersions.Length; i++)
+        {
+            for (int j = 1; j <= modifiedPathsWithVersions.Length; j++)
+            {
+                if (equalityComparer(originalPathsWithVersions[i - 1], modifiedPathsWithVersions[j - 1]))
+                {
+                    dp[i, j] = dp[i - 1, j - 1];
+                }
+                else
+                {
+                    dp[i, j] = 1 + Math.Min(dp[i - 1, j], dp[i, j - 1]);
+                }
+            }
+        }
+
+        // Trace back the edits
+        int row = originalPathsWithVersions.Length;
+        int col = modifiedPathsWithVersions.Length;
+
+        List<(string, DifferenceKind)> formattedDiff = [];
+        while (row > 0 || col > 0)
+        {
+            var baselineItem = originalPathsWithVersions[row - 1];
+            var testItem = modifiedPathsWithVersions[col - 1];
+            if (row > 0 && col > 0 && PathWithVersions.Equal(baselineItem, testItem))
+            {
+                formattedDiff.Add((formatter(originalPathsWithVersions[row - 1]), DifferenceKind.Unchanged));
+                row--;
+                col--;
+            }
+            else if (col > 0 && (row == 0 || dp[row, col - 1] <= dp[row - 1, col]))
+            {
+                formattedDiff.Add((formatter(modifiedPathsWithVersions[col - 1]), DifferenceKind.Added));
+                col--;
+            }
+            else if (row > 0 && (col == 0 || dp[row, col - 1] > dp[row - 1, col]))
+            {
+                formattedDiff.Add((formatter(originalPathsWithVersions[row - 1]), DifferenceKind.Removed));
+                row--;
+            }
+            else
+            {
+                throw new UnreachableException();
+            }
+        }
+        formattedDiff.Reverse();
+        return formattedDiff;
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs

@@ -0,0 +1,82 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+using Microsoft.Build.Framework;
+public class GetClosestOfficialSdk : Microsoft.Build.Utilities.Task
+{
+    [Required]
+    public required string BuiltSdkPath { get; init; }
+
+    [Output]
+    public string ClosestOfficialSdkPath { get; set; } = "";
+
+    public override bool Execute()
+    {
+        return Task.Run(ExecuteAsync).Result;
+    }
+
+    public async Task<bool> ExecuteAsync()
+    {
+        var (versionString, rid, extension) = ExtractFromFilePath(BuiltSdkPath);
+
+        string downloadUrl = GetLatestOfficialSdkUrl(versionString, rid, extension);
+
+        Log.LogMessage($"Downloading {downloadUrl}");
+        var packageResponse = await new HttpClient().GetAsync(downloadUrl);
+        packageResponse.EnsureSuccessStatusCode();
+
+        var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath;
+        string downloadedVersion = PathWithVersions.GetVersionInPath(packageUriPath).ToString();
+
+        ClosestOfficialSdkPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + $".dotnet-sdk-{downloadedVersion}-{rid}{extension}");
+        Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialSdkPath}");
+        using (var file = File.Create(ClosestOfficialSdkPath))
+        {
+            await packageResponse.Content.CopyToAsync(file);
+        }
+
+        return true;
+    }
+
+    string GetLatestOfficialSdkUrl(string versionString, string rid, string extension)
+    {
+        // Channel in the form of 9.0.1xx
+        var channel = versionString[..5] + "xx";
+        return $"https://aka.ms/dotnet/{channel}/daily/dotnet-sdk-{rid}{extension}";
+    }
+
+    static (string Version, string Rid, string extension) ExtractFromFilePath(string path)
+    {
+        string extension;
+        if (path.EndsWith(".tar.gz"))
+        {
+            extension = ".tar.gz";
+        }
+        else if (path.EndsWith(".zip"))
+        {
+            extension = ".zip";
+        }
+        else
+        {
+            throw new ArgumentException($"Invalid archive extension '{path}': must end with .tar.gz or .zip");
+        }
+
+        string filename = Path.GetFileName(path)[..^extension.Length];
+        var dashDelimitedParts = filename.Split('-');
+        var (rid, versionString) = dashDelimitedParts switch
+        {
+            ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(first) => (third + '-' + fourth, first + '-' + second),
+            ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(third) => (first + '-' + second, third + '-' + fourth),
+            _ => throw new ArgumentException($"Invalid archive file name '{filename}': file name should include full build full build full build full build full build full build full build full build full build version and rid")
+        };
+
+        return (versionString, rid, extension);
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <!-- <PackageReference Include="Microsoft.Build.Utilities.Core" PrivateAssets="all" ExcludeAssets="Runtime" Version="$(MicrosoftBuildVersion)" /> -->
+    <PackageReference Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildVersion)" />
+  </ItemGroup>
+</Project>

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs

@@ -0,0 +1,113 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+public static class PathWithVersions
+{
+    public const string VersionPlaceholder = "{VERSION}";
+
+    public static bool Equal(string path1, string path2)
+    {
+        if (path1 == path2)
+        {
+            return true;
+        }
+
+        ReadOnlySpan<char> directory = path1;
+        ReadOnlySpan<char> directory2 = path2;
+        while (TryGetPathLeaf(directory, out var root, out var directoryPart) && TryGetPathLeaf(directory2, out var root2, out var directoryPart2))
+        {
+            if (!ReplaceVersionString(directoryPart).SequenceEqual(ReplaceVersionString(directoryPart2)))
+            {
+                return false;
+            }
+            directory= Path.GetDirectoryName(directory);
+            directory2= Path.GetDirectoryName(directory2);
+        }
+        if (!directory.IsEmpty || !directory2.IsEmpty)
+        {
+            return false;
+        }
+        return true;
+    }
+
+    public static bool IsVersionString(ReadOnlySpan<char> directoryPart)
+    {
+        return directoryPart.Length >= 6
+            && char.IsDigit(directoryPart[0])
+            && directoryPart[1] == '.'
+            && char.IsDigit(directoryPart[2])
+            && directoryPart[3] == '.'
+            && char.IsDigit(directoryPart[4])
+            && ((char.IsDigit(directoryPart[5]) && char.IsDigit(directoryPart[6])) || directoryPart[5] == '-');
+    }
+
+    static ReadOnlySpan<char> ReplaceVersionString(ReadOnlySpan<char> directoryPart)
+    {
+        if (IsVersionString(directoryPart))
+        {
+            return VersionPlaceholder;
+        }
+        else
+        {
+            return directoryPart;
+        }
+    }
+
+    static bool TryGetPathLeaf(ReadOnlySpan<char> path, out ReadOnlySpan<char> root, out ReadOnlySpan<char> leaf)
+    {
+        if (path.IsEmpty)
+        {
+            root = default;
+            leaf = default;
+            return false;
+        }
+        leaf = Path.GetFileName(path);
+        root = Path.GetDirectoryName(path);
+        return true;
+    }
+
+    public static string GetVersionAnonymousPath(string path)
+    {
+        return GetVersionAnonymousPath(path).ToString();
+    }
+
+    public static ReadOnlySpan<char> GetVersionAnonymousPath(ReadOnlySpan<char> path)
+    {
+        StringBuilder sb = new StringBuilder();
+        bool altered = false;
+        ReadOnlySpan<char> myPath = path;
+        while (TryGetPathLeaf(myPath, out var directory, out var directoryPart))
+        {
+            sb = sb.Insert(0, Path.DirectorySeparatorChar);
+            var versionOrDirectory = ReplaceVersionString(directoryPart);
+            if (versionOrDirectory == VersionPlaceholder)
+            {
+                altered = true;
+            }
+            sb = sb.Insert(0, versionOrDirectory);
+            myPath = directory;
+        }
+        if (!altered)
+            return path;
+        return sb.ToString();
+    }
+
+    public static ReadOnlySpan<char> GetVersionInPath(ReadOnlySpan<char> path)
+    {
+        ReadOnlySpan<char> myPath = path;
+        while (TryGetPathLeaf(myPath, out var directory, out var directoryPart))
+        {
+            if (IsVersionString(directoryPart))
+            {
+                return directoryPart;
+            }
+            myPath = directory;
+        }
+        throw new ArgumentException("Path does not contain a version");
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs

@@ -0,0 +1,79 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+static class ArchiveExtensions
+{
+    public static string[] Lines(this ZipArchiveEntry entry, Encoding? encoding = null)
+    {
+        return entry.ReadToString(encoding).Replace("\r\n", "\n").Split('\n').ToArray();
+    }
+
+    public static string ReadToString(this ZipArchiveEntry entry, Encoding? encoding = null)
+    {
+        Stream stream = entry.Open();
+        byte[] buffer = stream.ReadToEnd();
+        // Remove UTF-8 BOM if present
+        int index = 0;
+        if (buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF)
+        {
+            index = 3;
+        }
+        encoding ??= Encoding.UTF8;
+        string fileText = encoding.GetString(buffer, index, buffer.Length - index);
+        return fileText;
+    }
+
+    public static byte[] ReadToEnd(this Stream stream)
+    {
+        int bufferSize = 2048;
+        byte[] buffer = new byte[bufferSize];
+        int offset = 0;
+        while (true)
+        {
+            int bytesRead = stream.Read(buffer, offset, bufferSize - offset);
+            offset += bytesRead;
+            if (bytesRead == 0)
+            {
+                break;
+            }
+            if (offset == bufferSize)
+            {
+                Array.Resize(ref buffer, bufferSize * 2);
+                bufferSize *= 2;
+            }
+        }
+        Array.Resize(ref buffer, offset);
+        return buffer;
+    }
+
+    public static async Task<byte[]> ReadToEndAsync(this Stream stream)
+    {
+        int bufferSize = 2048;
+        byte[] buffer = new byte[bufferSize];
+        int offset = 0;
+        while (true)
+        {
+            int bytesRead = await stream.ReadAsync(buffer, offset, bufferSize - offset);
+            offset += bytesRead;
+            if (bytesRead == 0)
+            {
+                break;
+            }
+            if (offset == bufferSize)
+            {
+                Array.Resize(ref buffer, bufferSize * 2);
+                bufferSize *= 2;
+            }
+        }
+        Array.Resize(ref buffer, offset);
+        return buffer;
+    }
+}