From 04b8e88747b3bfec458153527984841214d15e2a Mon Sep 17 00:00:00 2001 From: Ella Hathaway <67609881+ellahathaway@users.noreply.github.com> Date: Wed, 3 Apr 2024 11:25:00 -0700 Subject: [PATCH] Add Exclusion File Validation & Consistency (#19243) --- eng/pipelines/vmr-license-scan.yml | 2 + .../ExclusionsHelper.cs | 170 ++++++++++++++++++ .../LicenseScanTests.cs | 67 ++----- .../SdkContentTests.cs | 36 ++-- .../Utilities.cs | 31 ---- .../assets/LicenseExclusions.txt | 4 - .../SdkAssemblyVersionDiffExclusions.txt | 52 +----- .../assets/SdkFileDiffExclusions.txt | 136 +++++++------- .../assets/baselines/MsftToSbSdkFiles.diff | 18 +- 9 files changed, 279 insertions(+), 237 deletions(-) create mode 100644 src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExclusionsHelper.cs diff --git a/eng/pipelines/vmr-license-scan.yml b/eng/pipelines/vmr-license-scan.yml index 1a79d9d32..d1f33c6b4 100644 --- a/eng/pipelines/vmr-license-scan.yml +++ b/eng/pipelines/vmr-license-scan.yml @@ -109,6 +109,8 @@ jobs: find artifacts/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \; find artifacts/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \; echo "Updated:" + find test/ -type f -name "UpdatedLicenseExclusions*.txt" + find test/ -type f -name "UpdatedLicenseExclusions*.txt" -exec cp {} --parents -t ${targetFolder} \; find test/ -type f -name "Updated*.json" find test/ -type f -name "Updated*.json" -exec cp {} --parents -t ${targetFolder} \; echo "Results:" diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExclusionsHelper.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExclusionsHelper.cs new file mode 100644 index 000000000..19c1badac --- /dev/null +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExclusionsHelper.cs @@ -0,0 +1,170 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.Extensions.FileSystemGlobbing; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.DotNet.SourceBuild.SmokeTests; + +internal class ExclusionsHelper +{ + private const string NullSuffix = "NULL_SUFFIX"; + + private readonly string _exclusionsFileName; + + // Use this to narrow down the scope of exclusions to a specific category. + // For instance, setting this to "test-templates" will consider + // "src/test-templates/exclusions.txt" but not "src/arcade/exclusions.txt". + private readonly Regex? _exclusionRegex; + + private readonly Dictionary> _suffixToExclusions; + + private readonly Dictionary> _suffixToUnusedExclusions; + + public ExclusionsHelper(string exclusionsFileName, string? exclusionRegexString = null) + { + if (exclusionsFileName is null) + { + throw new ArgumentNullException(nameof(exclusionsFileName)); + } + + _exclusionsFileName = exclusionsFileName; + _exclusionRegex = string.IsNullOrWhiteSpace(exclusionRegexString) ? null : new Regex(exclusionRegexString); + _suffixToExclusions = ParseExclusionsFile(_exclusionsFileName); + _suffixToUnusedExclusions = new Dictionary>( + _suffixToExclusions.ToDictionary(pair => pair.Key, pair => new HashSet(pair.Value))); + } + + internal bool IsFileExcluded(string filePath, string suffix = NullSuffix) + { + if (suffix is null) + { + throw new ArgumentNullException(nameof(suffix)); + } + + // If a specific suffix is provided, check that first. If it is not found, check the default suffix. + return CheckAndRemoveIfExcluded(filePath, suffix) || + (suffix != NullSuffix && CheckAndRemoveIfExcluded(filePath, NullSuffix)); + } + + internal void GenerateNewBaselineFile(string? updatedFileTag = null) + { + string exclusionsFilePath = Path.Combine(BaselineHelper.GetAssetsDirectory(), _exclusionsFileName); + + string[] lines = File.ReadAllLines(exclusionsFilePath); + + var newLines = lines + .Select(line => UpdateExclusionsLine(line)) + .Where(line => line is not null); + + string updatedFileName = updatedFileTag is null + ? $"Updated{_exclusionsFileName}" + : $"Updated{Path.GetFileNameWithoutExtension(_exclusionsFileName)}.{updatedFileTag}{Path.GetExtension(_exclusionsFileName)}"; + string actualFilePath = Path.Combine(TestBase.LogsDirectory, updatedFileName); + File.WriteAllLines(actualFilePath, newLines!); + } + + private bool CheckAndRemoveIfExcluded(string filePath, string suffix = NullSuffix) + { + if (_suffixToExclusions.TryGetValue(suffix, out HashSet? suffixExclusionList)) + { + foreach (string exclusion in suffixExclusionList) + { + Matcher matcher = new(); + matcher.AddInclude(exclusion); + if (matcher.Match(filePath).HasMatches) + { + RemoveUsedExclusion(exclusion, suffix); + return true; + } + } + } + return false; + } + + private Dictionary> ParseExclusionsFile(string exclusionsFileName) + { + string exclusionsFilePath = Path.Combine(BaselineHelper.GetAssetsDirectory(), exclusionsFileName); + return File.ReadAllLines(exclusionsFilePath) + .Select(line => + { + // Ignore comments + var index = line.IndexOf('#'); + return index >= 0 ? line[..index].TrimEnd() : line; + }) + .Where(line => !string.IsNullOrEmpty(line)) + .Select(line => line.Split('|')) + .Where(parts => + { + // Only include exclusions that match the exclusion regex + return _exclusionRegex is null || _exclusionRegex.IsMatch(parts[0]); + }) + .SelectMany(parts => + { + // Create a new object for each suffix + return parts.Length == 1 + ? new[] { new { Exclusion = parts[0], Suffix = NullSuffix } } + : parts[1].Split(',').Select(suffix => new { Exclusion = parts[0], Suffix = suffix.Trim() }); + }) + .GroupBy( + parts => parts.Suffix, + parts => parts.Exclusion + ) + .ToDictionary( + group => group.Key, + group => new HashSet(group) + ); + } + + private void RemoveUsedExclusion(string exclusion, string suffix) + { + if (_suffixToUnusedExclusions.TryGetValue(suffix, out HashSet? exclusions)) + { + exclusions.Remove(exclusion); + } + } + + private string? UpdateExclusionsLine(string line) + { + string[] parts = line.Split('|'); + string exclusion = parts[0]; + var unusedSuffixes = _suffixToUnusedExclusions.Where(pair => pair.Value.Contains(exclusion)).Select(pair => pair.Key).ToList(); + + if (!unusedSuffixes.Any()) + { + // Exclusion is used in all suffixes, so we can keep it as is + return line; + } + + if (parts.Length == 1) + { + if (unusedSuffixes.Contains(NullSuffix)) + { + // Exclusion is unused in the default suffix, so we can remove it entirely + return null; + } + // Line is duplicated for other suffixes, but null suffix is used so we can keep it as is + return line; + } + + string suffixString = parts[1].Split('#')[0]; + var originalSuffixes = suffixString.Split(',').Select(suffix => suffix.Trim()).ToList(); + var newSuffixes = originalSuffixes.Except(unusedSuffixes).ToList(); + + if (newSuffixes.Count == 0) + { + // All suffixes were unused, so we can remove the line entirely + return null; + } + + return line.Replace(suffixString, string.Join(",", newSuffixes)); + } +} \ No newline at end of file diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs index 4cab87bc8..79e0111ea 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs @@ -112,10 +112,16 @@ public class LicenseScanTests : TestBase private readonly string _targetRepo; + private readonly string _relativeRepoPath; + public LicenseScanTests(ITestOutputHelper outputHelper) : base(outputHelper) { Assert.NotNull(Config.LicenseScanPath); _targetRepo = new DirectoryInfo(Config.LicenseScanPath).Name; + + Match relativeRepoPathMatch = Regex.Match(Config.LicenseScanPath, @"(src/)[^/]+"); + Assert.True(relativeRepoPathMatch.Success); + _relativeRepoPath = relativeRepoPathMatch.Value; } [SkippableFact(Config.LicenseScanPathEnv, skipOnNullOrWhiteSpaceEnv: true)] @@ -169,43 +175,8 @@ public class LicenseScanTests : TestBase BaselineHelper.CompareFiles(expectedFilePath, actualFilePath, OutputHelper, Config.WarnOnLicenseScanDiffs); } - private LicenseExclusion ParseLicenseExclusion(string rawExclusion) - { - string[] parts = rawExclusion.Split('|', StringSplitOptions.RemoveEmptyEntries); - - Match repoNameMatch = Regex.Match(parts[0], @"(?<=src/)[^/]+"); - - Assert.True(repoNameMatch.Success); - - // The path in the exclusion file is rooted from the VMR. But the path in the scancode results is rooted from the - // target repo within the VMR. So we need to strip off the beginning part of the path. - Match restOfPathMatch = Regex.Match(parts[0], @"(?<=src/[^/]+/).*"); - string path = restOfPathMatch.Value; - - if (parts.Length == 0 || parts.Length > 2) - { - throw new Exception($"Invalid license exclusion: '{rawExclusion}'"); - } - - if (parts.Length > 1) - { - string[] licenseExpressions = parts[1].Split(',', StringSplitOptions.RemoveEmptyEntries); - return new LicenseExclusion(repoNameMatch.Value, path, licenseExpressions); - } - else - { - return new LicenseExclusion(repoNameMatch.Value, path, Enumerable.Empty()); - } - } - private void FilterFiles(ScancodeResults scancodeResults) { - IEnumerable rawExclusions = Utilities.ParseExclusionsFile("LicenseExclusions.txt"); - IEnumerable exclusions = rawExclusions - .Select(exclusion => ParseLicenseExclusion(exclusion)) - .Where(exclusion => exclusion.Repo == _targetRepo) - .ToList(); - // This will filter out files that we don't want to include in the baseline. // Filtering can happen in two ways: // 1. There are a set of allowed license expressions that apply to all files. If a file has a match on one of those licenses, @@ -215,6 +186,9 @@ public class LicenseScanTests : TestBase // In that case, the baseline will list all of the licenses for that file, even if some were originally excluded during this processing. // In other words, the baseline will be fully representative of the licenses that apply to the files that are listed there. + // We only care about the license expressions that are in the target repo. + ExclusionsHelper exclusionsHelper = new("LicenseExclusions.txt", _targetRepo); + for (int i = scancodeResults.Files.Count - 1; i >= 0; i--) { ScancodeFileResult file = scancodeResults.Files[i]; @@ -244,30 +218,21 @@ public class LicenseScanTests : TestBase { // There are some licenses that are not allowed. Now check whether the file is excluded. - IEnumerable matchingExclusions = - Utilities.GetMatchingFileExclusions(file.Path, exclusions, exclusion => exclusion.Path); + // The path in the exclusion file is rooted from the VMR. But the path in the scancode results is rooted from the + // target repo within the VMR. So we need to add back the beginning part of the path. + string fullRelativePath = Path.Combine(_relativeRepoPath, file.Path); - IEnumerable excludedLicenses = matchingExclusions.SelectMany(exclusion => exclusion.LicenseExpressions); - // If no licenses are explicitly specified, it means they're all excluded. - if (matchingExclusions.Any() && !excludedLicenses.Any()) + var remainingLicenses = disallowedLicenses.Where(license => !exclusionsHelper.IsFileExcluded(fullRelativePath, license)); + + if (!remainingLicenses.Any()) { scancodeResults.Files.Remove(file); } - else - { - IEnumerable remainingLicenses = disallowedLicenses.Except(excludedLicenses); - - if (!remainingLicenses.Any()) - { - scancodeResults.Files.Remove(file); - } - } } } + exclusionsHelper.GenerateNewBaselineFile(_targetRepo); } - private record LicenseExclusion(string Repo, string Path, IEnumerable LicenseExpressions); - private class ScancodeResults { [JsonPropertyName("files")] diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs index 29bb7dbc5..7c8559fdd 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs @@ -36,12 +36,16 @@ public class SdkContentTests : SdkTests { const string msftFileListingFileName = "msftSdkFiles.txt"; const string sbFileListingFileName = "sbSdkFiles.txt"; - WriteTarballFileList(Config.MsftSdkTarballPath, msftFileListingFileName, isPortable: true, MsftSdkType); - WriteTarballFileList(Config.SdkTarballPath, sbFileListingFileName, isPortable: false, SourceBuildSdkType); + ExclusionsHelper exclusionsHelper = new ExclusionsHelper("SdkFileDiffExclusions.txt"); + + WriteTarballFileList(Config.MsftSdkTarballPath, msftFileListingFileName, isPortable: true, MsftSdkType, exclusionsHelper); + WriteTarballFileList(Config.SdkTarballPath, sbFileListingFileName, isPortable: false, SourceBuildSdkType, exclusionsHelper); string diff = BaselineHelper.DiffFiles(msftFileListingFileName, sbFileListingFileName, OutputHelper); diff = RemoveDiffMarkers(diff); BaselineHelper.CompareBaselineContents("MsftToSbSdkFiles.diff", diff, OutputHelper, Config.WarnOnSdkContentDiffs); + + exclusionsHelper.GenerateNewBaselineFile("FileList"); } [SkippableFact(new[] { Config.MsftSdkTarballPathEnv, Config.SdkTarballPathEnv }, skipOnNullOrWhiteSpaceEnv: true)] @@ -80,14 +84,12 @@ public class SdkContentTests : SdkTests } } - private static void RemoveExcludedAssemblyVersionPaths(Dictionary sbSdkAssemblyVersions, Dictionary msftSdkAssemblyVersions) + private void RemoveExcludedAssemblyVersionPaths(Dictionary sbSdkAssemblyVersions, Dictionary msftSdkAssemblyVersions) { - IEnumerable assemblyVersionDiffFilters = GetSdkAssemblyVersionDiffExclusionFilters() - .Select(filter => filter.TrimStart("./".ToCharArray())); - // Remove any excluded files as long as SB SDK's file has the same or greater assembly version compared to the corresponding // file in the MSFT SDK. If the version is less, the file will show up in the results as this is not a scenario // that is valid for shipping. + ExclusionsHelper exclusionsHelper = new ExclusionsHelper("SdkAssemblyVersionDiffExclusions.txt"); string[] sbSdkFileArray = sbSdkAssemblyVersions.Keys.ToArray(); for (int i = sbSdkFileArray.Length - 1; i >= 0; i--) { @@ -98,12 +100,13 @@ public class SdkContentTests : SdkTests if (sbVersion is not null && msftVersion is not null && sbVersion >= msftVersion && - Utilities.IsFileExcluded(assemblyPath, assemblyVersionDiffFilters)) + exclusionsHelper.IsFileExcluded(assemblyPath)) { sbSdkAssemblyVersions.Remove(assemblyPath); msftSdkAssemblyVersions.Remove(assemblyPath); } } + exclusionsHelper.GenerateNewBaselineFile(); } private static void WriteAssemblyVersionsToFile(Dictionary assemblyVersions, string outputPath) @@ -169,8 +172,7 @@ public class SdkContentTests : SdkTests private Dictionary GetSbSdkAssemblyVersions(string sbSdkPath) { - IEnumerable exclusionFilters = GetSdkDiffExclusionFilters(SourceBuildSdkType) - .Select(filter => filter.TrimStart("./".ToCharArray())); + ExclusionsHelper exclusionsHelper = new("SdkFileDiffExclusions.txt"); Dictionary sbSdkAssemblyVersions = new(); foreach (string file in Directory.EnumerateFiles(sbSdkPath, "*", SearchOption.AllDirectories)) { @@ -182,16 +184,17 @@ public class SdkContentTests : SdkTests string relativePath = Path.GetRelativePath(sbSdkPath, file); string normalizedPath = BaselineHelper.RemoveVersions(relativePath); - if (!Utilities.IsFileExcluded(normalizedPath, exclusionFilters)) + if(!exclusionsHelper.IsFileExcluded(normalizedPath, SourceBuildSdkType)) { sbSdkAssemblyVersions.Add(normalizedPath, GetVersion(assemblyName)); } } } + exclusionsHelper.GenerateNewBaselineFile("AssemblyVersions"); return sbSdkAssemblyVersions; } - private void WriteTarballFileList(string? tarballPath, string outputFileName, bool isPortable, string sdkType) + private void WriteTarballFileList(string? tarballPath, string outputFileName, bool isPortable, string sdkType, ExclusionsHelper exclusionsHelper) { if (!File.Exists(tarballPath)) { @@ -202,20 +205,11 @@ public class SdkContentTests : SdkTests fileListing = BaselineHelper.RemoveRids(fileListing, isPortable); fileListing = BaselineHelper.RemoveVersions(fileListing); IEnumerable files = fileListing.Split(Environment.NewLine).OrderBy(path => path); - files = RemoveExclusions(files, GetSdkDiffExclusionFilters(sdkType)); + files = files.Where(item => !exclusionsHelper.IsFileExcluded(item, sdkType)); File.WriteAllLines(outputFileName, files); } - private static IEnumerable RemoveExclusions(IEnumerable files, IEnumerable exclusions) => - files.Where(item => !Utilities.IsFileExcluded(item, exclusions)); - - private static IEnumerable GetSdkDiffExclusionFilters(string sdkType) => - Utilities.ParseExclusionsFile("SdkFileDiffExclusions.txt", sdkType); - - private static IEnumerable GetSdkAssemblyVersionDiffExclusionFilters() => - Utilities.ParseExclusionsFile("SdkAssemblyVersionDiffExclusions.txt"); - private static string RemoveDiffMarkers(string source) { Regex indexRegex = new("^index .*", RegexOptions.Multiline); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs index 228a65a61..65c5cfd92 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs @@ -19,37 +19,6 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests; public static class Utilities { - /// - /// Returns whether the given file path is excluded by the given exclusions using glob file matching. - /// - public static bool IsFileExcluded(string filePath, IEnumerable exclusions) => - GetMatchingFileExclusions(filePath, exclusions, exclusion => exclusion).Any(); - - public static IEnumerable GetMatchingFileExclusions(string filePath, IEnumerable exclusions, Func getExclusionExpression) => - exclusions.Where(exclusion => FileSystemName.MatchesSimpleExpression(getExclusionExpression(exclusion), filePath)); - - /// - /// Parses a common file format in the test suite for listing file exclusions. - /// - /// Name of the exclusions file. - /// When specified, filters the exclusions to those that begin with the prefix value. - public static IEnumerable ParseExclusionsFile(string exclusionsFileName, string? prefix = null) - { - string exclusionsFilePath = Path.Combine(BaselineHelper.GetAssetsDirectory(), exclusionsFileName); - int prefixSkip = prefix?.Length + 1 ?? 0; - return File.ReadAllLines(exclusionsFilePath) - // process only specific exclusions if a prefix is provided - .Where(line => prefix is null || line.StartsWith(prefix + ",")) - .Select(line => - { - // Ignore comments - var index = line.IndexOf('#'); - return index >= 0 ? line[prefixSkip..index].TrimEnd() : line[prefixSkip..]; - }) - .Where(line => !string.IsNullOrEmpty(line)) - .ToList(); - } - public static void ExtractTarball(string tarballPath, string outputDir, ITestOutputHelper outputHelper) { // TarFile doesn't properly handle hard links (https://github.com/dotnet/runtime/pull/85378#discussion_r1221817490), diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt index f2f5e9e9c..ee0e7dfef 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt @@ -177,11 +177,7 @@ src/runtime/src/libraries/System.ServiceModel.Syndication/tests/BasicScenarioTes src/runtime/src/mono/mono/mini/mini-posix.c|unknown-license-reference src/runtime/src/mono/mono/mini/mini-windows.c|unknown-license-reference src/runtime/src/native/external/libunwind/doc/libunwind-ia64.*|generic-exception -src/runtime/src/tests/GC/Scenarios/GCBench/THIRD-PARTY-NOTICES|unknown-license-reference -src/runtime/src/tests/JIT/Performance/CodeQuality/Benchstones/BenchF/LLoops/THIRD-PARTY-NOTICES|unknown-license-reference -src/runtime/src/tests/JIT/Performance/CodeQuality/Benchstones/MDBenchF/MDLLoops/THIRD-PARTY-NOTICES|unknown-license-reference src/runtime/src/tests/JIT/Performance/CodeQuality/V8/Crypto/Crypto.cs|unknown-license-reference -src/runtime/src/tests/JIT/Performance/CodeQuality/V8/Richards/THIRD-PARTY-NOTICES|unknown-license-reference # Test data src/runtime/src/libraries/System.Private.Xml.Linq/tests/XDocument.Common/InputSpace.cs|other-permissive diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkAssemblyVersionDiffExclusions.txt b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkAssemblyVersionDiffExclusions.txt index ce7d9c74a..e55baa415 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkAssemblyVersionDiffExclusions.txt +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkAssemblyVersionDiffExclusions.txt @@ -5,15 +5,15 @@ # # This list is processed using FileSystemName.MatchesSimpleExpression # -# Examples -# 'folder/*' matches 'folder/' and 'folder/abc' -# 'folder/?*' matches 'folder/abc' but not 'folder/' +# '*' in exclusions match zero or more characters. +# '*' will match files and directory names but it will not match separator characters. +# '/' will be evaluated as '/**' if it is the last character. # -# We do not want to filter-out folder entries, therefore, we should use: '?*' and not just '*' +# Examples +# 'folder/*' matches all files and directories in 'folder/'. It will not match 'folder/abc/def' +# 'folder/' is equivalent to 'folder/**. It matches 'folder/', 'folder/abc', and 'folder/abc/def/' # Referenced 6.0/7.0 assemblies (https://github.com/dotnet/sdk/issues/34245) -./sdk/x.y.z/Containers/tasks/netx.y/runtimes/win/lib/netx.y/?* -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/System.Composition.* ./sdk/x.y.z/Microsoft.Extensions.FileProviders.Abstractions.dll ./sdk/x.y.z/Microsoft.Extensions.FileSystemGlobbing.dll ./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/source-generators/System.Collections.Immutable.dll @@ -24,43 +24,3 @@ ./sdk/**/System.Security.Cryptography.Pkcs.dll ./sdk/**/System.Security.Cryptography.ProtectedData.dll ./sdk/x.y.z/System.Security.Cryptography.Xml.dll - -# These assemblies are lifted to a higher version naturally via SB (https://github.com/dotnet/source-build/issues/3922) -./sdk/x.y.z/DotnetTools/dotnet-format/BuildHost-netcore/Humanizer.dll -./sdk/x.y.z/DotnetTools/dotnet-format/BuildHost-netcore/Microsoft.Build.Locator.dll -./sdk/x.y.z/DotnetTools/dotnet-format/BuildHost-netcore/Microsoft.Extensions.* -./sdk/x.y.z/DotnetTools/dotnet-format/BuildHost-netcore/System.Composition.* -./sdk/x.y.z/DotnetTools/dotnet-format/BuildHost-netcore/System.IO.Pipelines.dll -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/BuildHost-netcore/Humanizer.dll -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/BuildHost-netcore/Microsoft.Build.Locator.dll -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/BuildHost-netcore/Microsoft.Extensions.* -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/BuildHost-netcore/System.Composition.* -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/BuildHost-netcore/System.IO.Pipelines.dll - -# These assemblies are lifted to 9.0 (https://github.com/dotnet/source-build/issues/4013) -./sdk/x.y.z/DotnetTools/dotnet-format/BuildHost-netcore/runtimes/browser/lib/netx.y/System.Text.Encodings.Web.dll -./sdk/x.y.z/DotnetTools/dotnet-format/BuildHost-netcore/System.Collections.Immutable.dll -./sdk/x.y.z/DotnetTools/dotnet-format/BuildHost-netcore/System.Reflection.Metadata.dll -./sdk/x.y.z/DotnetTools/dotnet-format/BuildHost-netcore/System.Text.Encodings.Web.dll -./sdk/x.y.z/DotnetTools/dotnet-format/BuildHost-netcore/System.Text.Json.dll -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/BuildHost-netcore/System.Collections.Immutable.dll -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/BuildHost-netcore/System.Reflection.Metadata.dll - -# These assemblies are lifted to a higher version naturally via SB -./sdk/x.y.z/DotnetTools/dotnet-format/dotnet-format.dll -./sdk/x.y.z/DotnetTools/dotnet-format/*/dotnet-format.resources.dll -./sdk/x.y.z/DotnetTools/dotnet-format/*/Microsoft.CodeAnalysis.* -./sdk/x.y.z/DotnetTools/dotnet-format/Humanizer.dll -./sdk/x.y.z/DotnetTools/dotnet-format/Microsoft.Build.Locator.dll -./sdk/x.y.z/DotnetTools/dotnet-format/Microsoft.CodeAnalysis.* -./sdk/x.y.z/DotnetTools/dotnet-format/Microsoft.DiaSymReader.dll -./sdk/x.y.z/DotnetTools/dotnet-format/System.CodeDom.dll -./sdk/x.y.z/DotnetTools/dotnet-format/System.Composition.* -./sdk/x.y.z/DotnetTools/dotnet-format/System.IO.Pipelines.dll -./sdk/x.y.z/DotnetTools/dotnet-format/System.Resources.Extensions.dll -./sdk/x.y.z/DotnetTools/dotnet-format/System.Security.Cryptography.Xml.dll -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/Humanizer.dll -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/Microsoft.Build.Locator.dll -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/Microsoft.CodeAnalysis.AnalyzerUtilities.dll -./sdk/x.y.z/DotnetTools/dotnet-watch/x.y.z/tools/netx.y/any/Microsoft.DiaSymReader.dll -./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/source-generators/Microsoft.CodeAnalysis.ExternalAccess.RazorCompiler.dll diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkFileDiffExclusions.txt b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkFileDiffExclusions.txt index 45bf347ff..1924edfcd 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkFileDiffExclusions.txt +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/SdkFileDiffExclusions.txt @@ -1,95 +1,97 @@ # This list is processed using FileSystemName.MatchesSimpleExpression # # Format -# {msft|sb}, [# comment] -# msft = Microsoft built SDK -# sb = source-built SDK +# Exclude the path entirely: +# [# comment] +# Exclude a path from a specific sdk: +# |{msft|sb} [# comment] +# msft = Microsoft built SDK +# sb = source-built SDK +# +# '*' in exclusions match zero or more characters. +# '*' will match files and directory names but it will not match separator characters. +# '/' will be evaluated as '/**' if it is the last character. # # Examples -# 'folder/*' matches 'folder/' and 'folder/abc' -# 'folder/?*' matches 'folder/abc' but not 'folder/' -# -# We do not want to filter-out folder entries, therefore, we should use: '?*' and not just '*' +# 'folder/*' matches all files and directories in 'folder/'. It will not match 'folder/abc/def' +# 'folder/' is equivalent to 'folder/**. It matches 'folder/', 'folder/abc', and 'folder/abc/def/' -msft,./sdk/x.y.z/TestHostNetFramework/?* # Intentional - MSFT build includes test-host that targets netcoreapp3.1 -msft,./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.WindowsDesktop/?* # Intentional - explicitly excluded from source-build +./sdk/x.y.z/TestHostNetFramework/|msft # Intentional - MSFT build includes test-host that targets netcoreapp3.1 +./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.WindowsDesktop/|msft # Intentional - explicitly excluded from source-build # netfx tooling and tasks, not building in source-build - https://github.com/dotnet/source-build/issues/3514 -msft,./sdk/x.y.z/Sdks/Microsoft.Build.Tasks.Git/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.NET.Sdk/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.BlazorWebAssembly/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Publish/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/tasks/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.StaticWebAssets/tasks/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Web/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Web.ProjectSystem/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.WebAssembly/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Worker/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.SourceLink.AzureRepos.Git/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.SourceLink.Bitbucket.Git/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.SourceLink.Common/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.SourceLink.GitHub/tools/net472/* -msft,./sdk/x.y.z/Sdks/Microsoft.SourceLink.GitLab/tools/net472/* +./sdk/x.y.z/Sdks/Microsoft.Build.Tasks.Git/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.NET.Sdk/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.BlazorWebAssembly/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Publish/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/tasks/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.StaticWebAssets/tasks/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Web/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Web.ProjectSystem/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.WebAssembly/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Worker/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.SourceLink.AzureRepos.Git/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.SourceLink.Bitbucket.Git/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.SourceLink.Common/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.SourceLink.GitHub/tools/net472/|msft +./sdk/x.y.z/Sdks/Microsoft.SourceLink.GitLab/tools/net472/|msft # vstest localization is disabled in Linux builds - https://github.com/dotnet/source-build/issues/3517 -msft,./sdk/x.y.z/*?/Microsoft.CodeCoverage.IO.resources.dll +./sdk/x.y.z/*/Microsoft.CodeCoverage.IO.resources.dll|msft # nuget localization is not available for Linux builds - https://github.com/NuGet/Home/issues/12440 -msft,./sdk/x.y.z/*?/NuGet.*?.resources.dll -msft,./sdk/x.y.z/*?/Microsoft.Build.NuGetSdkResolver.resources.dll -msft,./sdk/x.y.z/*?/Test.Utility.resources.dll +./sdk/x.y.z/*/NuGet.*.resources.dll|msft +./sdk/x.y.z/*/Test.Utility.resources.dll|msft # ILMerge is not supported in Linux builds - excluding the whole NuGet.Build.Tasks.Pack directory, to avoid a noisy diff -msft,./sdk/x.y.z/Sdks/NuGet.Build.Tasks.Pack/*? -sb,./sdk/x.y.z/Sdks/NuGet.Build.Tasks.Pack/*? +./sdk/x.y.z/Sdks/NuGet.Build.Tasks.Pack/ # missing workload manifests - https://github.com/dotnet/source-build/issues/3242 -msft,./sdk-manifests/x.y.z/microsoft.net.sdk.android/* -msft,./sdk-manifests/x.y.z/microsoft.net.sdk.ios/* -msft,./sdk-manifests/x.y.z/microsoft.net.sdk.maccatalyst/* -msft,./sdk-manifests/x.y.z/microsoft.net.sdk.macos/* -msft,./sdk-manifests/x.y.z/microsoft.net.sdk.maui/* -msft,./sdk-manifests/x.y.z/microsoft.net.sdk.tvos/* +./sdk-manifests/x.y.z/microsoft.net.sdk.android/|msft +./sdk-manifests/x.y.z/microsoft.net.sdk.ios/|msft +./sdk-manifests/x.y.z/microsoft.net.sdk.maccatalyst/|msft +./sdk-manifests/x.y.z/microsoft.net.sdk.macos/|msft +./sdk-manifests/x.y.z/microsoft.net.sdk.maui/|msft +./sdk-manifests/x.y.z/microsoft.net.sdk.tvos/|msft # linux runtimes are included in source-build for self-contained apps - https://github.com/dotnet/source-build/issues/3507 -sb,./packs/Microsoft.AspNetCore.App.Runtime.*/* -sb,./packs/Microsoft.NETCore.App.Runtime.*/* - -# netfx tooling - dumpminitool - https://github.com/dotnet/source-build/issues/3289 -msft,./sdk/x.y.z/Extensions/dump/* - -# https://github.com/dotnet/msbuild/issues/9213 -msft,./sdk/x.y.z/**/System.Windows.Extensions.dll -msft,./sdk/x.y.z/**/System.Security.Permissions.dll +./packs/Microsoft.AspNetCore.App.Runtime.*/|sb +./packs/Microsoft.NETCore.App.Runtime.*/|sb # Exclude format and watch tools due to too much noise -msft,./sdk/x.y.z/DotnetTools/dotnet-format/** -sb,./sdk/x.y.z/DotnetTools/dotnet-format/** -msft,./sdk/x.y.z/DotnetTools/dotnet-watch/** -sb,./sdk/x.y.z/DotnetTools/dotnet-watch/** +./sdk/x.y.z/DotnetTools/dotnet-format/ +./sdk/x.y.z/DotnetTools/dotnet-watch/ + +./sdk/x.y.z/Extensions/cs/|msft +./sdk/x.y.z/Extensions/de/|msft +./sdk/x.y.z/Extensions/es/|msft +./sdk/x.y.z/Extensions/fr/|msft +./sdk/x.y.z/Extensions/it/|msft +./sdk/x.y.z/Extensions/ja/|msft +./sdk/x.y.z/Extensions/ko/|msft +./sdk/x.y.z/Extensions/pl/|msft +./sdk/x.y.z/Extensions/pt-BR/|msft +./sdk/x.y.z/Extensions/ru/|msft +./sdk/x.y.z/Extensions/tr/|msft +./sdk/x.y.z/Extensions/zh-Hans/|msft +./sdk/x.y.z/Extensions/zh-Hant/|msft + +./sdk/x.y.z/*/dump/|msft # netfx runtimes for fsharp - https://github.com/dotnet/source-build/issues/3290 -msft,./sdk/x.y.z/FSharp/Microsoft.VisualStudio.Setup.Configuration.Interop.dll -msft,./sdk/x.y.z/FSharp/runtimes/win/lib/netx.y/Microsoft.Win32.SystemEvents.dll -msft,./sdk/x.y.z/FSharp/runtimes/win/lib/netx.y/System.Drawing.Common.dll -msft,./sdk/x.y.z/FSharp/runtimes/win/lib/netx.y/System.Security.Cryptography.ProtectedData.dll - -# windows components - https://github.com/dotnet/source-build/issues/3526 -msft,./sdk/x.y.z/runtimes/win/lib/netx.y/Microsoft.Win32.SystemEvents.dll -msft,./sdk/x.y.z/runtimes/win/lib/netx.y/System.Drawing.Common.dll +./sdk/x.y.z/FSharp/Microsoft.VisualStudio.Setup.Configuration.Interop.dll|msft # runtime components in roslyn layout - https://github.com/dotnet/source-build/issues/4016 -sb,./sdk/x.y.z/Roslyn/bincore/System.Collections.Immutable.dll -sb,./sdk/x.y.z/Roslyn/bincore/System.Reflection.Metadata.dll +./sdk/x.y.z/Roslyn/bincore/System.Collections.Immutable.dll|sb +./sdk/x.y.z/Roslyn/bincore/System.Reflection.Metadata.dll|sb # https://github.com/dotnet/source-build/issues/4079 -sb,./sdk/x.y.z/*/Microsoft.TestPlatform.Extensions.BlameDataCollector.resources.dll -sb,./sdk/x.y.z/*/Microsoft.TestPlatform.Extensions.EventLogCollector.resources.dll -sb,./sdk/x.y.z/*/Microsoft.TestPlatform.TestHostRuntimeProvider.resources.dll -sb,./sdk/x.y.z/*/Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.resources.dll -sb,./sdk/x.y.z/*/Microsoft.VisualStudio.TestPlatform.Extensions.Trx.TestLogger.resources.dll -msft,./sdk/x.y.z/Extensions/*/* +./sdk/x.y.z/*/Microsoft.TestPlatform.Extensions.BlameDataCollector.resources.dll|sb +./sdk/x.y.z/*/Microsoft.TestPlatform.Extensions.EventLogCollector.resources.dll|sb +./sdk/x.y.z/*/Microsoft.TestPlatform.TestHostRuntimeProvider.resources.dll|sb +./sdk/x.y.z/*/Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.resources.dll|sb +./sdk/x.y.z/*/Microsoft.VisualStudio.TestPlatform.Extensions.Trx.TestLogger.resources.dll|sb # https://github.com/dotnet/source-build/issues/3510 -msft,./sdk/x.y.z/Containers/containerize/** -msft,./sdk/x.y.z/Containers/tasks/net472/** +./sdk/x.y.z/Containers/containerize/|msft +./sdk/x.y.z/Containers/tasks/net472/|msft diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdkFiles.diff b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdkFiles.diff index 955c36509..cfd17f75b 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdkFiles.diff +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdkFiles.diff @@ -51,20 +51,4 @@ index ------------ -./sdk/x.y.z/Microsoft.CodeCoverage.IO.dll ./sdk/x.y.z/Microsoft.Common.CrossTargeting.targets ./sdk/x.y.z/Microsoft.Common.CurrentVersion.targets - ./sdk/x.y.z/Microsoft.Common.overridetasks -@@ ------------ @@ - ./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.WebAssembly/tools/ - ./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.WebAssembly/tools/netx.y/ - ./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.WebAssembly/tools/netx.y/Microsoft.NET.Sdk.WebAssembly.Tasks.dll --./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.WindowsDesktop/ - ./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Worker/ - ./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Worker/Sdk/ - ./sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Worker/Sdk/Sdk.props -@@ ------------ @@ - ./sdk/x.y.z/testhost-latest.runtimeconfig.json - ./sdk/x.y.z/testhost.deps.json - ./sdk/x.y.z/testhost.dll --./sdk/x.y.z/TestHostNetFramework/ - ./sdk/x.y.z/tr/ - ./sdk/x.y.z/tr/dotnet.resources.dll - ./sdk/x.y.z/tr/Microsoft.Build.resources.dll \ No newline at end of file + ./sdk/x.y.z/Microsoft.Common.overridetasks \ No newline at end of file