mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Refactor SdkContentTests to warn on diffs when run in CI (#13432)

Browse source
This commit is contained in:
Michael Simons 2022-03-23 12:25:05 -05:00 committed by GitHub
parent a9d43d1d3d
commit 211912bd41
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 25 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml
View file

@ -91,7 +91,7 @@ steps:
cp $(Build.SourcesDirectory)/NuGet.config ${{ parameters.tarballDir }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config
dockerVolumeArgs="-v ${{ parameters.tarballDir }}:/tarball"
dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=${{ parameters.excludeOmniSharpTests}}"
dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=${{ parameters.excludeOmniSharpTests}} -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true"
if [[ '${{ parameters.isBootstrapped }}' != 'true' && '${{ parameters.installerBuildResourceId }}' != 'current' ]]; then
dockerVolumeArgs+=" -v $(PIPELINE.WORKSPACE)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/:/BlobArtifacts"
@ -99,7 +99,7 @@ steps:
dockerEnvArgs+=" -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=/BlobArtifacts/$msftSdkTarballName"
fi
docker run --rm $dockerVolumeArgs -w /tarball $dockerEnvArgs ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }}
docker run --rm $dockerVolumeArgs -w /tarball $dockerEnvArgs ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }} -- /p:SmokeTestConsoleVerbosity=detailed
displayName: Run Tests
# Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph.

4
src/SourceBuild/tarball/content/build.proj
View file

@ -93,9 +93,11 @@
<PropertyGroup>
<SdkTarballPath>%(SdkTarballItem.Identity)</SdkTarballPath>
<SmokeTestConsoleVerbosity Condition="'$(SmokeTestConsoleVerbosity)' == ''">normal</SmokeTestConsoleVerbosity>
</PropertyGroup>
<Exec Command="$(DotnetToolCommand) test $(SmokeTestsDir) --logger:trx -c $(Configuration)"
<!-- Multiple loggers are specified so that results are captured in trx and pipelines can fail with AzDO pipeline warnings -->
<Exec Command="$(DotnetToolCommand) test $(SmokeTestsDir) --logger:trx --logger:'console;verbosity=$(SmokeTestConsoleVerbosity)' -c $(Configuration)"
EnvironmentVariables="
SMOKE_TESTS_SDK_TARBALL_PATH=$(SdkTarballPath);
SMOKE_TESTS_TARGET_RID=$(TargetRid);

20
src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BaselineHelper.cs
View file

@ -34,17 +34,17 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests
Assert.Null(message);
}
public static void CompareContents(string baselineFileName, string actualContents, ITestOutputHelper outputHelper)
public static void CompareContents(string baselineFileName, string actualContents, ITestOutputHelper outputHelper, bool warnOnDiffs = false)
{
string baselineFilePath = GetBaselineFilePath(baselineFileName);
string actualFilePath = Path.Combine(Environment.CurrentDirectory, $"{baselineFileName}");
File.WriteAllText(actualFilePath, actualContents);
CompareFiles(baselineFilePath, actualFilePath, outputHelper);
CompareFiles(baselineFilePath, actualFilePath, outputHelper, warnOnDiffs);
}
public static void CompareFiles(string baselineFilePath, string actualFilePath, ITestOutputHelper outputHelper)
public static void CompareFiles(string baselineFilePath, string actualFilePath, ITestOutputHelper outputHelper, bool warnOnDiffs = false)
{
string baselineFileText = File.ReadAllText(baselineFilePath);
string actualFileText = File.ReadAllText(actualFilePath);
@ -55,11 +55,21 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests
{
// Retrieve a diff in order to provide a UX which calls out the diffs.
string diff = DiffFiles(baselineFilePath, actualFilePath, outputHelper);
message = $"{Environment.NewLine}Baseline '{baselineFilePath}' does not match actual '{actualFilePath}`. {Environment.NewLine}"
string prefix = warnOnDiffs ? "##vso[task.logissue type=warning;]" : string.Empty;
message = $"{Environment.NewLine}{prefix}Baseline '{baselineFilePath}' does not match actual '{actualFilePath}`. {Environment.NewLine}"
+ $"{diff}{Environment.NewLine}";
if (warnOnDiffs)
{
outputHelper.WriteLine(message);
outputHelper.WriteLine("##vso[task.complete result=SucceededWithIssues;]");
}
}
Assert.Null(message);
if (!warnOnDiffs)
{
Assert.Null(message);
}
}
public static string DiffFiles(string file1Path, string file2Path, ITestOutputHelper outputHelper)

3
src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs
View file

@ -15,6 +15,7 @@ internal static class Config
public const string PrereqsPathEnv = "SMOKE_TESTS_PREREQS_PATH";
public const string SdkTarballPathEnv = "SMOKE_TESTS_SDK_TARBALL_PATH";
public const string TargetRidEnv = "SMOKE_TESTS_TARGET_RID";
public const string WarnSdkContentDiffsEnv = "SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS";
public static string DotNetDirectory { get; } =
Environment.GetEnvironmentVariable(DotNetDirectoryEnv) ?? Path.Combine(Directory.GetCurrentDirectory(), ".dotnet");
@ -23,4 +24,6 @@ internal static class Config
public static string? SdkTarballPath { get; } = Environment.GetEnvironmentVariable(SdkTarballPathEnv);
public static string TargetRid { get; } = Environment.GetEnvironmentVariable(TargetRidEnv) ??
throw new InvalidOperationException($"'{Config.TargetRidEnv}' must be specified");
public static bool WarnOnSdkContentDiffs { get; } =
bool.TryParse(Environment.GetEnvironmentVariable(WarnSdkContentDiffsEnv), out bool excludeOnlineTests) && excludeOnlineTests;
}

4
src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs
View file

@ -22,7 +22,7 @@ public class SdkContentTests : SmokeTests
/// This makes the baseline durable between releases. This does mean however, entries
/// in the baseline may appear identical if the diff is version specific.
/// </Summary>
[SkippableFact(new[] { Config.MsftSdkTarballPathEnv, Config.MsftSdkTarballPathEnv }, skipOnNullOrWhiteSpace: true)]
[SkippableFact(new[] { Config.MsftSdkTarballPathEnv, Config.SdkTarballPathEnv }, skipOnNullOrWhiteSpace: true)]
public void CompareMsftToSb()
{
const string msftFileListingFileName = "msftSdkFiles.txt";
@ -34,7 +34,7 @@ public class SdkContentTests : SmokeTests
diff = RemoveVersionedPaths(diff);
diff = RemoveDiffMarkers(diff);
diff = RemoveRids(diff);
BaselineHelper.CompareContents("MsftToSbSdk.diff", diff, OutputHelper);
BaselineHelper.CompareContents("MsftToSbSdk.diff", diff, OutputHelper, Config.WarnOnSdkContentDiffs);
}
private void WriteTarballFileList(string? tarballPath, string outputFileName)