Refactor SdkContentTests to warn on diffs when run in CI (#13432)

src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml

@@ -91,7 +91,7 @@ steps:
       cp $(Build.SourcesDirectory)/NuGet.config ${{ parameters.tarballDir }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config
 
       dockerVolumeArgs="-v ${{ parameters.tarballDir }}:/tarball"
-      dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=${{ parameters.excludeOmniSharpTests}}"
+      dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=${{ parameters.excludeOmniSharpTests}} -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true"
 
       if [[ '${{ parameters.isBootstrapped }}' != 'true' && '${{ parameters.installerBuildResourceId }}' != 'current' ]]; then
         dockerVolumeArgs+=" -v $(PIPELINE.WORKSPACE)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/:/BlobArtifacts"
@@ -99,7 +99,7 @@ steps:
         dockerEnvArgs+=" -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=/BlobArtifacts/$msftSdkTarballName"
       fi
 
-      docker run --rm $dockerVolumeArgs -w /tarball $dockerEnvArgs ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }}
+      docker run --rm $dockerVolumeArgs -w /tarball $dockerEnvArgs ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }} -- /p:SmokeTestConsoleVerbosity=detailed
     displayName: Run Tests
 
   # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph.

src/SourceBuild/tarball/content/build.proj

@@ -93,9 +93,11 @@
 
     <PropertyGroup>
       <SdkTarballPath>%(SdkTarballItem.Identity)</SdkTarballPath>
+      <SmokeTestConsoleVerbosity Condition="'$(SmokeTestConsoleVerbosity)' == ''">normal</SmokeTestConsoleVerbosity>
     </PropertyGroup>
 
-    <Exec Command="$(DotnetToolCommand) test $(SmokeTestsDir) --logger:trx -c $(Configuration)"
+    <!-- Multiple loggers are specified so that results are captured in trx and pipelines can fail with AzDO pipeline warnings -->
+    <Exec Command="$(DotnetToolCommand) test $(SmokeTestsDir) --logger:trx --logger:'console;verbosity=$(SmokeTestConsoleVerbosity)' -c $(Configuration)"
           EnvironmentVariables="
             SMOKE_TESTS_SDK_TARBALL_PATH=$(SdkTarballPath);
             SMOKE_TESTS_TARGET_RID=$(TargetRid);

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BaselineHelper.cs

@@ -34,17 +34,17 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests
             Assert.Null(message);
         }
 
-        public static void CompareContents(string baselineFileName, string actualContents, ITestOutputHelper outputHelper)
+        public static void CompareContents(string baselineFileName, string actualContents, ITestOutputHelper outputHelper, bool warnOnDiffs = false)
         {
             string baselineFilePath = GetBaselineFilePath(baselineFileName);
 
             string actualFilePath = Path.Combine(Environment.CurrentDirectory, $"{baselineFileName}");
             File.WriteAllText(actualFilePath, actualContents);
 
-            CompareFiles(baselineFilePath, actualFilePath, outputHelper);
+            CompareFiles(baselineFilePath, actualFilePath, outputHelper, warnOnDiffs);
         }
 
-        public static void CompareFiles(string baselineFilePath, string actualFilePath, ITestOutputHelper outputHelper)
+        public static void CompareFiles(string baselineFilePath, string actualFilePath, ITestOutputHelper outputHelper, bool warnOnDiffs = false)
         {
             string baselineFileText = File.ReadAllText(baselineFilePath);
             string actualFileText = File.ReadAllText(actualFilePath);
@@ -55,11 +55,21 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests
             {
                 // Retrieve a diff in order to provide a UX which calls out the diffs.
                 string diff = DiffFiles(baselineFilePath, actualFilePath, outputHelper);
-                message = $"{Environment.NewLine}Baseline '{baselineFilePath}' does not match actual '{actualFilePath}`.  {Environment.NewLine}"
+                string prefix = warnOnDiffs ? "##vso[task.logissue type=warning;]" : string.Empty;
+                message = $"{Environment.NewLine}{prefix}Baseline '{baselineFilePath}' does not match actual '{actualFilePath}`.  {Environment.NewLine}"
                     + $"{diff}{Environment.NewLine}";
+
+                if (warnOnDiffs)
+                {
+                    outputHelper.WriteLine(message);
+                    outputHelper.WriteLine("##vso[task.complete result=SucceededWithIssues;]");
+                }
             }
 
-            Assert.Null(message);
+            if (!warnOnDiffs)
+            {
+                Assert.Null(message);
+            }
         }
 
         public static string DiffFiles(string file1Path, string file2Path, ITestOutputHelper outputHelper)

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs

@@ -15,6 +15,7 @@ internal static class Config
     public const string PrereqsPathEnv = "SMOKE_TESTS_PREREQS_PATH";
     public const string SdkTarballPathEnv = "SMOKE_TESTS_SDK_TARBALL_PATH";
     public const string TargetRidEnv = "SMOKE_TESTS_TARGET_RID";
+    public const string WarnSdkContentDiffsEnv = "SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS";
 
     public static string DotNetDirectory { get; } =
         Environment.GetEnvironmentVariable(DotNetDirectoryEnv) ?? Path.Combine(Directory.GetCurrentDirectory(), ".dotnet");
@@ -23,4 +24,6 @@ internal static class Config
     public static string? SdkTarballPath { get; } = Environment.GetEnvironmentVariable(SdkTarballPathEnv);
     public static string TargetRid { get; } = Environment.GetEnvironmentVariable(TargetRidEnv) ??
         throw new InvalidOperationException($"'{Config.TargetRidEnv}' must be specified");
+    public static bool WarnOnSdkContentDiffs { get; } =
+        bool.TryParse(Environment.GetEnvironmentVariable(WarnSdkContentDiffsEnv), out bool excludeOnlineTests) && excludeOnlineTests;
 }

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs

@@ -22,7 +22,7 @@ public class SdkContentTests : SmokeTests
     /// This makes the baseline durable between releases.  This does mean however, entries
     /// in the baseline may appear identical if the diff is version specific.
     /// </Summary>
-    [SkippableFact(new[] { Config.MsftSdkTarballPathEnv, Config.MsftSdkTarballPathEnv }, skipOnNullOrWhiteSpace: true)]
+    [SkippableFact(new[] { Config.MsftSdkTarballPathEnv, Config.SdkTarballPathEnv }, skipOnNullOrWhiteSpace: true)]
     public void CompareMsftToSb()
     {
         const string msftFileListingFileName = "msftSdkFiles.txt";
@@ -34,7 +34,7 @@ public class SdkContentTests : SmokeTests
         diff = RemoveVersionedPaths(diff);
         diff = RemoveDiffMarkers(diff);
         diff = RemoveRids(diff);
-        BaselineHelper.CompareContents("MsftToSbSdk.diff", diff, OutputHelper);
+        BaselineHelper.CompareContents("MsftToSbSdk.diff", diff, OutputHelper, Config.WarnOnSdkContentDiffs);
     }
 
     private void WriteTarballFileList(string? tarballPath, string outputFileName)