Add Readme, just warn on baseline mismatch, log test info

src/SourceBuild/content/eng/unifiedBuildValidation.targets

@@ -12,7 +12,7 @@
     <MSBuild Projects="$(UnifiedBuildValidationTestsProject)"
              Targets="VSTest"
              ContinueOnError="WarnAndContinue"
-             Properties="ExcludeFromDotNetBuild=false;VsTestUseMSBuildOutput=true" />
+             Properties="ExcludeFromDotNetBuild=false;VsTestUseMSBuildOutput=false" />
 
   </Target>
 

src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Config.cs

@@ -14,7 +14,17 @@ using Xunit.Sdk;
 namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 public class Config : IDisposable
 {
+    public string MsftSdkArchivePath { get; }
+    public string UbBuildVersion { get; }
+    public string PortableRid { get; }
+    public string UbSdkArchivePath { get; }
+    public string TargetRid { get; }
+    public string TargetArchitecture { get; }
+    public bool WarnOnSdkContentDiffs { get; }
+
+    string? _downloadedMsftSdkPath = null;
     IMessageSink _sink;
+
     public Config(IMessageSink sink)
     {
         _sink = sink;
@@ -23,24 +33,34 @@ public class Config : IDisposable
         PortableRid = (string)(AppContext.GetData(PortableRidSwitch) ?? throw new InvalidOperationException("Portable RID must be specified"));
         UbSdkArchivePath = (string)(AppContext.GetData(UbSdkArchivePathSwitch) ?? throw new InvalidOperationException("Unified Build SDK archive path must be specified"));
         TargetArchitecture = TargetRid.Split('-')[1];
-        MsftSdkArchivePath = AppContext.GetData(MsftSdkArchivePathSwitch) as string ?? DownloadMsftSdkArchive().Result;
+        WarnOnSdkContentDiffs = bool.Parse((string)(AppContext.GetData(WarnOnSdkContentDiffsSwitch) ?? "false"));
+        MsftSdkArchivePath = (string)AppContext.GetData(MsftSdkArchivePathSwitch)!;
+        if (string.IsNullOrEmpty(MsftSdkArchivePath))
+        {
+            MsftSdkArchivePath = DownloadMsftSdkArchive().Result;
+        }
+        else {
+            sink.OnMessage(new DiagnosticMessage($"Skipping downloading latest SDK. Using provided sdk archive: '{MsftSdkArchivePath}'"));
+        }
+        sink.OnMessage(new DiagnosticMessage($$"""
+            Test config values:
+            {{nameof(UbBuildVersion)}}='{{UbBuildVersion}}'
+            {{nameof(TargetRid)}}='{{TargetRid}}'
+            {{nameof(PortableRid)}}='{{PortableRid}}'
+            {{nameof(UbSdkArchivePath)}}='{{UbSdkArchivePath}}'
+            {{nameof(TargetArchitecture)}}='{{TargetArchitecture}}'
+            {{nameof(WarnOnSdkContentDiffs)}}='{{WarnOnSdkContentDiffs}}'
+            {{nameof(MsftSdkArchivePath)}}='{{MsftSdkArchivePath}}'
+            """));
     }
 
-    public const string ConfigSwitchPrefix = "Microsoft.DotNet.UnifiedBuild.Tests.";
-    public const string BuildVersionSwitch = ConfigSwitchPrefix + nameof(UbBuildVersion);
-    public const string TargetRidSwitch = ConfigSwitchPrefix + nameof(TargetRid);
-    public const string PortableRidSwitch = ConfigSwitchPrefix + nameof(PortableRid);
-    public const string UbSdkArchivePathSwitch = ConfigSwitchPrefix + nameof(UbSdkArchivePath);
-    public const string MsftSdkArchivePathSwitch = ConfigSwitchPrefix + nameof(MsftSdkArchivePath);
-
-    public string? MsftSdkArchivePath { get; }
-    public string UbBuildVersion { get; }
-    public string PortableRid { get; }
-    public string UbSdkArchivePath { get; }
-    public string TargetRid { get; }
-    public string TargetArchitecture { get; }
-    public bool WarnOnSdkContentDiffs { get; }
-    string? _downloadedMsftSdkPath = null;
+    const string ConfigSwitchPrefix = "Microsoft.DotNet.UnifiedBuild.Tests.";
+    const string BuildVersionSwitch = ConfigSwitchPrefix + nameof(UbBuildVersion);
+    const string TargetRidSwitch = ConfigSwitchPrefix + nameof(TargetRid);
+    const string PortableRidSwitch = ConfigSwitchPrefix + nameof(PortableRid);
+    const string UbSdkArchivePathSwitch = ConfigSwitchPrefix + nameof(UbSdkArchivePath);
+    const string MsftSdkArchivePathSwitch = ConfigSwitchPrefix + nameof(MsftSdkArchivePath);
+    const string WarnOnSdkContentDiffsSwitch = ConfigSwitchPrefix + nameof(WarnOnSdkContentDiffs);
 
     static string GetArchiveExtension(string path)
     {

src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Microsoft.DotNet.UnifiedBuild.Tests.csproj

@@ -6,6 +6,7 @@
     <TargetFramework>$(NetCurrent)</TargetFramework>
     <Nullable>enable</Nullable>
     <DefaultExcludesInProjectFolder>$(DefaultExcludesInProjectFolder);assets/**/*</DefaultExcludesInProjectFolder>
+    <UBTestsWarnOnSdkContentDiffs>true</UBTestsWarnOnSdkContentDiffs>
   </PropertyGroup>
 
   <ItemGroup>
@@ -33,6 +34,13 @@
       <RuntimeHostConfigurationOption Include="Microsoft.DotNet.UnifiedBuild.Tests.UbBuildVersion">
         <Value>$(SourceBuiltSdkVersion)</Value>
       </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="Microsoft.DotNet.UnifiedBuild.Tests.WarnOnSdkContentDiffs">
+        <Value>$(UBTestsWarnOnSdkContentDiffs)</Value>
+      </RuntimeHostConfigurationOption>
+      <!-- Set for local testing to avoid downloading a new SDK each test -->
+      <RuntimeHostConfigurationOption Include="Microsoft.DotNet.UnifiedBuild.Tests.MsftSdkArchivePath">
+        <Value>$(UBTestsMsftSdkArchivePath)</Value>
+      </RuntimeHostConfigurationOption>
     </ItemGroup>
   </Target>
 

src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Readme.md

@@ -0,0 +1,26 @@
+# Unified Build Validation Tests
+
+This project compares the output of the Unified Build system with the official Microsoft Build. There are tests to compare the list of files in the SDK archive, and the versions of each managed assembly in each SDK.
+Files can be excluded from each comparison in by placing the file name in one of the files in the `assets/` folder. Each file will apply to a all or a subset of RIDs depending on the suffix of the filename. For example, filenames in SdkFileDiffExclusions-linux-any.txt will be excluded from all linux SDK comparisons.
+
+| File Name                            | Purpose                                                                                                                |
+|--------------------------------------|------------------------------------------------------------------------------------------------------------------------|
+| SdkFileDiffExclusions.txt            | Files that should not be included in any file comparisons                                                              |
+| NativeDlls.txt                       | Files that end in .dll or .exe that are not managed assemblies and should be excluded from assembly version validation |
+| SdkAssemblyVersionDiffExclusions.txt | Files that should not be included in the assembly version validation for any other reason                              |
+
+## SDK archive file list comparison
+
+The files of each sdk archive iterated through, filtered and the name is written to `msftSdkFiles.txt` or `ubSdkFiles.txt` in the same directory as the test assembly. Then the test runs `git diff --no-index msftSdkFiles.txt ubSdkFiles.txt` and writes stdout to `$(OutDir)/log/UpdatedMsftToUbSdkFiles-{Rid}.diff`. This diff is then compared with the expected baseline in `./assets/baselines/MsftToUbSdkFiles-{Rid}.diff`. If the baselines are not identical, the test fails.
+
+### Updating the baseline
+
+If the test fails, but you've inspected the `UpdatedMsftToUbSdkFiles-{RID}.diff` and the changes are expected, then overwrite `./assets/baselines/MsftToUbSdkFiles-{Rid}.diff` with the updated baseline diff.
+
+## Sdk assembly version comparison
+
+This is done the similarly to the file comparison, but only looks at files that end in `.dll` or `.exe`, and filters out filename in `SdkFileDiffExclusions.txt`, `NativeDlls.txt`, and `SdkAssemblyVersionDiffExclusions.txt`. Each file name and the assembly version are written to `ub_assemblyversions.txt` or `msft_assemblyversions.txt`. Those files are then `git diff`'ed, and stdout is written to `$(OutDir)/log/UpdatedMsftToUbSdkAssemblyVersions-{Rid}.diff`. Then, the test diffs that with the baseline.
+
+### Updating the baseline
+
+If the test fails and the changes to the baseline are expected, the overwrite `./assets/baselines/MsftToUbSdkFiles-{RID}.diff` with the updated baseline.