Add source build artifacts size tests (#17233)

This commit is contained in:
Ella Hathaway 2023-09-12 14:26:19 -07:00 committed by GitHub
commit 67d98b6ccc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 4493 additions and 58 deletions

View file

@ -26,6 +26,7 @@ jobs:
targetRid: centos.8-x64
architecture: x64
dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}
includeArtifactsSize: true
- template: templates/jobs/sdk-diff-tests.yml
parameters:

View file

@ -11,6 +11,10 @@ parameters:
- name: dotnetDotnetRunId
type: string
- name: includeArtifactsSize
type: boolean
default: false
jobs:
- job: ${{ parameters.buildName }}_${{ parameters.architecture }}
timeoutInMinutes: 150
@ -51,36 +55,25 @@ jobs:
env:
AZURE_DEVOPS_EXT_PAT: $(System.AccessToken)
- task: DownloadPipelineArtifact@2
displayName: Download MSFT SDK
inputs:
buildType: specific
buildVersionToDownload: specific
project: internal
pipeline: $(INSTALLER_OFFICIAL_CI_PIPELINE_ID)
buildId: $(InstallerBuildId)
artifact: BlobArtifacts
patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz'
allowPartiallySucceededBuilds: true
allowFailedBuilds: true
downloadPath: $(Pipeline.Workspace)/Artifacts
checkDownloadedFiles: true
- template: ../steps/download-pipeline-artifact.yml
parameters:
patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz'
displayName: Download MSFT SDK
- task: DownloadPipelineArtifact@2
displayName: Download Source Build SDK
inputs:
buildType: specific
buildVersionToDownload: specific
project: internal
pipeline: $(DOTNET_DOTNET_CI_PIPELINE_ID)
buildId: $(DotnetDotnetBuildId)
artifact: ${{ parameters.buildName }}_${{ parameters.architecture }}_Artifacts
- template: ../steps/download-vmr-artifact.yml
parameters:
buildName: ${{ parameters.buildName }}
architecture: ${{ parameters.architecture }}
patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-${{ parameters.targetRid }}.tar.gz'
allowPartiallySucceededBuilds: true
allowFailedBuilds: true
downloadPath: $(Pipeline.Workspace)/Artifacts
checkDownloadedFiles: true
displayName: Download Source Build SDK
- template: ../steps/download-vmr-artifact.yml
parameters:
buildName: ${{ parameters.buildName }}
architecture: ${{ parameters.architecture }}
patterns: '**/Private.SourceBuilt.Artifacts.+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*).${{ parameters.targetRid }}.tar.gz'
displayName: Download Source Built Artifacts
- script: |
msft_sdk_tarball_name=$(find "$(Pipeline.Workspace)/Artifacts" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;)
@ -96,10 +89,18 @@ jobs:
exit 1
fi
artifacts_path=$(find "$(Pipeline.Workspace)/Artifacts" -name "Private.SourceBuilt.Artifacts.*.${{ parameters.targetRid }}.tar.gz" -exec basename {} \;)
if [[ -z "$artifacts_path" ]]; then
echo "Source-build artifacts path does not exist in '$(Pipeline.Workspace)/Artifacts'. The associated build https://dev.azure.com/dnceng/internal/_build/results?buildId=$(DotnetDotnetBuildId) might have failed"
exit 1
fi
eng/common/build.sh -bl --projects $(Build.SourcesDirectory)/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj --restore
echo "##vso[task.setvariable variable=MsftSdkTarballPath]$(Pipeline.Workspace)/Artifacts/$msft_sdk_tarball_name"
echo "##vso[task.setvariable variable=SdkTarballPath]$(Pipeline.Workspace)/Artifacts/$sdk_tarball_name"
echo "##vso[task.setvariable variable=SourceBuiltArtifactsPath]$(Pipeline.Workspace)/Artifacts/$artifacts_path"
displayName: Prepare Tests
workingDirectory: $(Build.SourcesDirectory)
@ -115,12 +116,13 @@ jobs:
-clp:v=m
-e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=$(MsftSdkTarballPath)
-e SMOKE_TESTS_SDK_TARBALL_PATH=$(SdkTarballPath)
-e SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH=
-e SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH=$(SourceBuiltArtifactsPath)
-e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=false
-e SMOKE_TESTS_RUNNING_IN_CI=true
-e SMOKE_TESTS_TARGET_RID=${{ parameters.targetRid }}
-e SMOKE_TESTS_PORTABLE_RID=linux-${{ parameters.architecture }}
-e SMOKE_TESTS_CUSTOM_PACKAGES_PATH=
-e SMOKE_TESTS_CUSTOM_PACKAGES_PATH=
-e SMOKE_TESTS_INCLUDE_ARTIFACTSSIZE=${{ parameters.includeArtifactsSize }}
displayName: Run Tests
workingDirectory: $(Build.SourcesDirectory)

View file

@ -0,0 +1,35 @@
parameters:
- name: pipeline
type: string
default: $(INSTALLER_OFFICIAL_CI_PIPELINE_ID)
- name: buildId
type: string
default: $(InstallerBuildId)
- name: artifact
type: string
default: BlobArtifacts
- name: patterns
type: string
- name: displayName
type: string
default: Download Pipeline Artifact
steps:
- task: DownloadPipelineArtifact@2
inputs:
buildType: specific
buildVersionToDownload: specific
project: internal
pipeline: ${{ parameters.pipeline }}
buildId: ${{ parameters.buildId }}
artifact: ${{ parameters.artifact }}
patterns: ${{ parameters.patterns }}
allowPartiallySucceededBuilds: true
allowFailedBuilds: true
downloadPath: $(Pipeline.Workspace)/Artifacts
checkDownloadedFiles: true
displayName: ${{ parameters.displayName }}

View file

@ -0,0 +1,22 @@
parameters:
- name: buildName
type: string
- name: architecture
type: string
- name: patterns
type: string
- name: displayName
type: string
default: Download VMR Artifact
steps:
- template: ../steps/download-pipeline-artifact.yml
parameters:
pipeline: $(DOTNET_DOTNET_CI_PIPELINE_ID)
buildId: $(DotnetDotnetBuildId)
artifact: ${{ parameters.buildName }}_${{ parameters.architecture }}_Artifacts
patterns: ${{ parameters.patterns }}
displayName: ${{ parameters.displayName }}

View file

@ -0,0 +1,162 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.IO;
using System.Linq;
using System.Collections;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.Formats.Tar;
using System.Threading.Tasks;
using Xunit;
using Xunit.Abstractions;
namespace Microsoft.DotNet.SourceBuild.SmokeTests;
[Trait("Category", "SdkContent")]
public class ArtifactsSizeTest : SmokeTests
{
private const int SizeThresholdPercentage = 25;
private static readonly string BaselineFilePath = BaselineHelper.GetBaselineFilePath($"ArtifactsSizes/{Config.TargetRid}.txt");
private readonly Dictionary<string, long> BaselineFileContent = new();
private Dictionary<string, int> FilePathCountMap = new();
public ArtifactsSizeTest(ITestOutputHelper outputHelper) : base(outputHelper)
{
if (File.Exists(BaselineFilePath))
{
string[] baselineFileContent = File.ReadAllLines(BaselineFilePath);
foreach (string entry in baselineFileContent)
{
string[] splitEntry = entry.Split(':', StringSplitOptions.TrimEntries);
BaselineFileContent[splitEntry[0]] = long.Parse(splitEntry[1]);
}
}
else
{
Assert.False(true, $"Baseline file `{BaselineFilePath}' does not exist. Please create the baseline file then rerun the test.");
}
}
[SkippableFact(Config.IncludeArtifactsSizeEnv, skipOnFalseEnv: true)]
public void CompareArtifactsToBaseline()
{
Utilities.ValidateNotNullOrWhiteSpace(Config.SourceBuiltArtifactsPath, Config.SourceBuiltArtifactsPathEnv);
Utilities.ValidateNotNullOrWhiteSpace(Config.SdkTarballPath, Config.SdkTarballPathEnv);
Utilities.ValidateNotNullOrWhiteSpace(Config.TargetRid, Config.TargetRidEnv);
var tarEntries = ProcessSdkAndArtifactsTarballs();
foreach (var entry in tarEntries)
{
if (!BaselineFileContent.TryGetValue(entry.FilePath, out long baselineBytes))
{
OutputHelper.LogWarningMessage($"{entry.FilePath} does not exist in baseline. Adding it to the baseline file");
}
else
{
CompareFileSizes(entry.FilePath, entry.Bytes, baselineBytes);
}
}
try
{
string actualFilePath = Path.Combine(DotNetHelper.LogsDirectory, $"UpdatedArtifactsSizes_{Config.TargetRid}.txt");
File.WriteAllLines(actualFilePath, tarEntries.Select(entry => $"{entry.FilePath}: {entry.Bytes}"));
}
catch (IOException ex)
{
throw new InvalidOperationException($"An error occurred while copying the baselines file: {BaselineFilePath}", ex);
}
}
private (string FilePath, long Bytes)[] ProcessSdkAndArtifactsTarballs()
{
string tempTarballDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(tempTarballDir);
Utilities.ExtractTarball(Config.SdkTarballPath, tempTarballDir, OutputHelper);
Utilities.ExtractTarball(Config.SourceBuiltArtifactsPath, tempTarballDir, OutputHelper);
(string FilePath, long Bytes)[] tarEntries = Directory.EnumerateFiles(tempTarballDir, "*", SearchOption.AllDirectories)
.Where(filepath => !filepath.Contains("SourceBuildReferencePackages"))
.Select(filePath =>
{
string result = filePath.Substring(tempTarballDir.Length + 1);
result = ProcessFilePath(result);
return (FilePath: result, Bytes: new FileInfo(filePath).Length);
})
.OrderBy(entry => entry.FilePath)
.ToArray();
Directory.Delete(tempTarballDir, true);
return tarEntries;
}
private string ProcessFilePath(string originalPath)
{
string result = BaselineHelper.RemoveRids(originalPath);
result = BaselineHelper.RemoveVersions(result);
return AddDifferenciatingSuffix(result);
}
// Because version numbers are abstracted, it is possible to have duplicate FilePath entries.
// This code adds a numeric suffix to differentiate duplicate FilePath entries.
private string AddDifferenciatingSuffix(string filePath)
{
string[] patterns = {@"x\.y\.z", @"x\.y(?!\.z)"};
int matchIndex = -1;
string matchPattern = "";
foreach (string pattern in patterns)
{
MatchCollection matches = Regex.Matches(filePath, pattern);
if (matches.Count > 0)
{
if (matches[matches.Count - 1].Index > matchIndex)
{
matchIndex = matches[matches.Count - 1].Index;
matchPattern = matches[matches.Count - 1].Value;
}
}
}
if (matchIndex != -1)
{
int count = FilePathCountMap.TryGetValue(filePath, out count) ? count : 0;
FilePathCountMap[filePath] = count + 1;
if (count > 0)
{
return filePath.Substring(0, matchIndex) + $"{matchPattern}-{count}" + filePath.Substring(matchIndex + matchPattern.Length);
}
}
return filePath;
}
private void CompareFileSizes(string filePath, long fileSize, long baselineSize)
{
if (fileSize == 0 && baselineSize != 0)
{
OutputHelper.LogWarningMessage($"'{filePath}' is now 0 bytes. It was {baselineSize} bytes");
}
else if (fileSize != 0 && baselineSize == 0)
{
OutputHelper.LogWarningMessage($"'{filePath}' is no longer 0 bytes. It is now {fileSize} bytes");
}
else if (baselineSize != 0 && (((fileSize - baselineSize) / (double)baselineSize) * 100) >= SizeThresholdPercentage)
{
OutputHelper.LogWarningMessage($"'{filePath}' increased in size by more than {SizeThresholdPercentage}%. It was originally {baselineSize} bytes and is now {fileSize} bytes");
}
else if (baselineSize != 0 && (((baselineSize - fileSize) / (double)baselineSize) * 100) >= SizeThresholdPercentage)
{
OutputHelper.LogWarningMessage($"'{filePath}' decreased in size by more than {SizeThresholdPercentage}%. It was originally {baselineSize} bytes and is now {fileSize} bytes");
}
}
}

View file

@ -16,10 +16,10 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests
{
internal class BaselineHelper
{
private const string VersionPlaceholder = "x.y.z";
private const string VersionPlaceholderMatchingPattern = "*.*.*"; // wildcard pattern used to match on the version represented by the placeholder
private const string NetTfmPlaceholder = "netx.y";
private const string NetTfmPlaceholderMatchingPattern = "net*.*"; // wildcard pattern used to match on the version represented by the placeholder
private const string SemanticVersionPlaceholder = "x.y.z";
private const string SemanticVersionPlaceholderMatchingPattern = "*.*.*"; // wildcard pattern used to match on the version represented by the placeholder
private const string NonSemanticVersionPlaceholder = "x.y";
private const string NonSemanticVersionPlaceholderMatchingPattern = "*.*"; // wildcard pattern used to match on the version represented by the placeholder
public static void CompareEntries(string baselineFileName, IOrderedEnumerable<string> actualEntries)
{
@ -89,27 +89,32 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests
public static string GetBaselineFilePath(string baselineFileName) => Path.Combine(GetAssetsDirectory(), "baselines", baselineFileName);
public static string RemoveNetTfmPaths(string source)
{
string pathSeparator = Regex.Escape(Path.DirectorySeparatorChar.ToString());
Regex netTfmRegex = new($"{pathSeparator}net[1-9]+\\.[0-9]+{pathSeparator}");
return netTfmRegex.Replace(source, $"{Path.DirectorySeparatorChar}{NetTfmPlaceholder}{Path.DirectorySeparatorChar}");
}
public static string RemoveRids(string diff, bool isPortable = false) =>
isPortable ? diff.Replace(Config.PortableRid, "portable-rid") : diff.Replace(Config.TargetRid, "banana-rid");
public static string RemoveVersions(string source)
{
// Remove version numbers for examples like "roslyn4.1", "net8.0", and "netstandard2.1".
string pathSeparator = Regex.Escape(Path.DirectorySeparatorChar.ToString());
string result = Regex.Replace(source, $@"{pathSeparator}(net|roslyn)[1-9]+\.[0-9]+{pathSeparator}", match =>
{
string wordPart = match.Groups[1].Value;
return $"{Path.DirectorySeparatorChar}{wordPart}{NonSemanticVersionPlaceholder}{Path.DirectorySeparatorChar}";
});
// Remove semantic versions
// Regex source: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
// The regex from https://semver.org has been modified to account for the following:
// - The version should be preceded by a path separator, '.', '-', or '/'
// - The version should match a release identifier that begins with '.' or '-'
// - The version may have one or more release identifiers that begin with '.' or '-'
// - The version should end before a path separator, '.', '-', or '/'
Regex semanticVersionRegex = new(
$"(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)"
+ $"(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))"
+ $"?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?");
string result = semanticVersionRegex.Replace(source, VersionPlaceholder);
return RemoveNetTfmPaths(result);
@"(?<=[./-])(0|[1-9]\d*)\.(0|[1-9]\d*)(\.(0|[1-9]\d*))+"
+ @"(((?:[-.]((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)))+"
+ @"(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
+ @"(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?(?=[/.-])");
return semanticVersionRegex.Replace(result, SemanticVersionPlaceholder);
}
/// <summary>
@ -119,8 +124,8 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests
public static Matcher GetFileMatcherFromPath(string path)
{
path = path
.Replace(VersionPlaceholder, VersionPlaceholderMatchingPattern)
.Replace(NetTfmPlaceholder, NetTfmPlaceholderMatchingPattern);
.Replace(SemanticVersionPlaceholder, SemanticVersionPlaceholderMatchingPattern)
.Replace(NonSemanticVersionPlaceholder, NonSemanticVersionPlaceholderMatchingPattern);
Matcher matcher = new();
matcher.AddInclude(path);
return matcher;

View file

@ -11,6 +11,7 @@ internal static class Config
{
public const string DotNetDirectoryEnv = "SMOKE_TESTS_DOTNET_DIR";
public const string ExcludeOmniSharpEnv = "SMOKE_TESTS_EXCLUDE_OMNISHARP";
public const string IncludeArtifactsSizeEnv = "SMOKE_TESTS_INCLUDE_ARTIFACTSSIZE";
public const string MsftSdkTarballPathEnv = "SMOKE_TESTS_MSFT_SDK_TARBALL_PATH";
public const string PoisonReportPathEnv = "SMOKE_TESTS_POISON_REPORT_PATH";
public const string PortableRidEnv = "SMOKE_TESTS_PORTABLE_RID";

View file

@ -13,13 +13,13 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
/// </summary>
internal class SkippableFactAttribute : FactAttribute
{
public SkippableFactAttribute(string envName, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, string[] skipArchitectures = null) =>
EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipArchitectures, (skip) => Skip = skip, envName);
public SkippableFactAttribute(string envName, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) =>
EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envName);
public SkippableFactAttribute(string[] envNames, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, string[] skipArchitectures = null) =>
EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipArchitectures, (skip) => Skip = skip, envNames);
public SkippableFactAttribute(string[] envNames, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) =>
EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envNames);
public static void EvaluateSkips(bool skipOnNullOrWhiteSpaceEnv, bool skipOnTrueEnv, string[] skipArchitectures, Action<string> setSkip, params string[] envNames)
public static void EvaluateSkips(bool skipOnNullOrWhiteSpaceEnv, bool skipOnTrueEnv, bool skipOnFalseEnv, string[] skipArchitectures, Action<string> setSkip, params string[] envNames)
{
foreach (string envName in envNames)
{
@ -35,6 +35,11 @@ internal class SkippableFactAttribute : FactAttribute
setSkip($"Skipping because `{envName}` is set to True");
break;
}
else if (skipOnFalseEnv && (!bool.TryParse(envValue, out boolValue) || !boolValue))
{
setSkip($"Skipping because `{envName}` is set to False or an invalid value");
break;
}
}
if (skipArchitectures != null) {

View file

@ -11,9 +11,9 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
/// </summary>
internal class SkippableTheoryAttribute : TheoryAttribute
{
public SkippableTheoryAttribute(string envName, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, string[] skipArchitectures = null) =>
SkippableFactAttribute.EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipArchitectures, (skip) => Skip = skip, envName);
public SkippableTheoryAttribute(string envName, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) =>
SkippableFactAttribute.EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envName);
public SkippableTheoryAttribute(string[] envNames, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, string[] skipArchitectures = null) =>
SkippableFactAttribute.EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipArchitectures, (skip) => Skip = skip, envNames);
public SkippableTheoryAttribute(string[] envNames, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) =>
SkippableFactAttribute.EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envNames);
}

View file

@ -8,8 +8,10 @@ using System.Collections.Generic;
using System.Formats.Tar;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Xunit;
using Xunit.Abstractions;
namespace Microsoft.DotNet.SourceBuild.SmokeTests;
@ -104,4 +106,20 @@ public static class Utilities
exception = await executor();
}
}
public static void LogWarningMessage(this ITestOutputHelper outputHelper, string message)
{
string prefix = "##vso[task.logissue type=warning;]";
outputHelper.WriteLine($"{Environment.NewLine}{prefix}{message}.{Environment.NewLine}");
outputHelper.WriteLine("##vso[task.complete result=SucceededWithIssues;]");
}
public static void ValidateNotNullOrWhiteSpace(string? variable, string variableName)
{
if (string.IsNullOrWhiteSpace(variable))
{
throw new ArgumentException($"{variableName} is null, empty, or whitespace.");
}
}
}