diff --git a/eng/ManualVersions.props b/eng/ManualVersions.props index fb41f0bad..57a4b6415 100644 --- a/eng/ManualVersions.props +++ b/eng/ManualVersions.props @@ -9,11 +9,11 @@ Basically: In this file, choose the highest version when resolving merge conflicts. --> - 10.0.17763.29 - 10.0.18362.29 - 10.0.19041.29 - 10.0.20348.29 - 10.0.22000.29 - 10.0.22621.29 + 10.0.17763.31 + 10.0.18362.31 + 10.0.19041.31 + 10.0.20348.31 + 10.0.22000.31 + 10.0.22621.31 diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index f11af8c4b..041a47580 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -127,7 +127,7 @@ jobs: artifact: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts patterns: | **/Private.SourceBuilt.Artifacts.*.tar.gz - **/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])*.tar.gz + **/dotnet-sdk-*.tar.gz displayName: Download Previous Build - task: CopyFiles@2 diff --git a/eng/pipelines/vmr-sync.yml b/eng/pipelines/vmr-sync.yml index aa12adcbe..6659887f1 100644 --- a/eng/pipelines/vmr-sync.yml +++ b/eng/pipelines/vmr-sync.yml @@ -6,6 +6,10 @@ trigger: include: - main - release/* + exclude: + - release/*.0.2xx + - release/*.0.3xx + - release/*.0.4xx resources: repositories: diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index 577e7e9b2..22ba31b83 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -95,7 +95,7 @@ DiscoverSymbolsTarballs; ExtractSymbolsTarballs"> - $(OutputPath)dotnet-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz + $(OutputPath)dotnet-symbols-all-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz - + $(ArtifactsTmpDir)SdkSymbols - $(OutputPath)dotnet-sdk-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz + $(OutputPath)dotnet-symbols-sdk-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz $(ArtifactsTmpDir)Sdk %(SdkTarballItem.Identity) @@ -187,8 +186,7 @@ - + diff --git a/src/SourceBuild/content/eng/Versions.props b/src/SourceBuild/content/eng/Versions.props index e22e27fa3..07844e239 100644 --- a/src/SourceBuild/content/eng/Versions.props +++ b/src/SourceBuild/content/eng/Versions.props @@ -30,8 +30,8 @@ These URLs can't be composed from their base URL and version as we read them from the prep.sh and pipeline scripts, outside of MSBuild. --> - https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.8.0.100-rc.1.23455.1.centos.8-x64.tar.gz - https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-rc.1.23455.1-centos.8-x64.tar.gz + https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.8.0.100-rc.2.23502.1.centos.8-x64.tar.gz + https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-rc.2.23502.1-centos.8-x64.tar.gz https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Prebuilts.0.1.0-9.0.100-3.centos.8-x64.tar.gz diff --git a/src/SourceBuild/content/eng/bootstrap/OverrideBootstrapVersions.props b/src/SourceBuild/content/eng/bootstrap/OverrideBootstrapVersions.props index 2692610eb..358c00140 100644 --- a/src/SourceBuild/content/eng/bootstrap/OverrideBootstrapVersions.props +++ b/src/SourceBuild/content/eng/bootstrap/OverrideBootstrapVersions.props @@ -1,11 +1,23 @@ - - 7.0.4-servicing.23107.6 + + 7.0.4-servicing.23107.6 + + $(NonshippingRuntimeVersionFor700) - $(NonshippingRuntimeVersionFor700) - $(NonshippingRuntimeVersionFor700) - $(NonshippingRuntimeVersionFor700) + 8.0.0-rc.2.23479.6 + $(Msft80RC2RuntimeVersion) + $(Msft80RC2RuntimeVersion) + $(Msft80RC2RuntimeVersion) + $(Msft80RC2RuntimeVersion) + $(Msft80RC2RuntimeVersion) + $(Msft80RC2RuntimeVersion) + $(Msft80RC2RuntimeVersion) + $(Msft80RC2RuntimeVersion) + $(Msft80RC2RuntimeVersion) + $(Msft80RC2RuntimeVersion) + $(Msft80RC2RuntimeVersion) + $(Msft80RC2RuntimeVersion) diff --git a/src/SourceBuild/content/eng/bootstrap/buildBootstrapPreviouslySB.csproj b/src/SourceBuild/content/eng/bootstrap/buildBootstrapPreviouslySB.csproj index 56d828d22..d85e32ca7 100644 --- a/src/SourceBuild/content/eng/bootstrap/buildBootstrapPreviouslySB.csproj +++ b/src/SourceBuild/content/eng/bootstrap/buildBootstrapPreviouslySB.csproj @@ -47,6 +47,8 @@ + + reader.GetCustomAttribute(attrHandle)) + .Any(attr => IsAttributeSbrp(reader, attr)); + } + + private static bool IsAttributeSbrp(MetadataReader reader, CustomAttribute attr) + { + string attributeType = string.Empty; + + if (attr.Constructor.Kind == HandleKind.MemberReference) + { + MemberReference mref = reader.GetMemberReference((MemberReferenceHandle)attr.Constructor); + + if (mref.Parent.Kind == HandleKind.TypeReference) + { + TypeReference tref = reader.GetTypeReference((TypeReferenceHandle)mref.Parent); + attributeType = $"{reader.GetString(tref.Namespace)}.{reader.GetString(tref.Name)}"; + } + } + + if (attributeType == SbrpAttributeType) + { + BlobReader blobReader = reader.GetBlobReader(attr.Value); + string attributeValue = Encoding.UTF8.GetString(blobReader.ReadBytes(blobReader.Length)); + attributeValue = Regex.Replace(attributeValue, @"\p{C}+", string.Empty); + return Regex.IsMatch(attributeValue, SbrpAttributeValuePattern); + } + return false; + } + private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue futureFilesToCheck) { var poisonEntry = new PoisonedFileEntry(); diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs index 270fb11f9..32abcae71 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs @@ -155,6 +155,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection } File.Delete(p.ItemSpec); File.Move(poisonedPackagePath, p.ItemSpec); + Directory.Delete(packageTempPath, true); } } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs index de5c35961..4ac37e1cb 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs @@ -11,5 +11,6 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection Hash = 1, AssemblyAttribute = 2, NupkgFile = 4, + SourceBuildReferenceAssembly = 8, } } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/CreateSdkSymbolsLayout.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/CreateSdkSymbolsLayout.cs index 8802dd543..7e00bbf30 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/CreateSdkSymbolsLayout.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/CreateSdkSymbolsLayout.cs @@ -57,7 +57,7 @@ namespace Microsoft.DotNet.Build.Tasks private void LogErrorOrWarning(bool isError, string message) { - if (FailOnMissingPDBs) + if (isError) { Log.LogError(message); } @@ -104,14 +104,15 @@ namespace Microsoft.DotNet.Build.Tasks if (guid != string.Empty) { - if (!allPdbGuids.ContainsKey(guid)) + string debugId = GetDebugId(guid, file); + if (!allPdbGuids.ContainsKey(debugId)) { filesWithoutPDBs.Add(file.Substring(SdkLayoutPath.Length + 1)); } else { // Copy matching pdb to symbols path, preserving sdk binary's hierarchy - string sourcePath = (string)allPdbGuids[guid]!; + string sourcePath = (string)allPdbGuids[debugId]!; string destinationPath = file.Replace(SdkLayoutPath, SdkSymbolsLayoutPath) .Replace(Path.GetFileName(file), Path.GetFileName(sourcePath)); @@ -142,13 +143,21 @@ namespace Microsoft.DotNet.Build.Tasks var id = new BlobContentId(metadataReader.DebugMetadataHeader.Id); string guid = $"{id.Guid:N}"; - if (!string.IsNullOrEmpty(guid) && !allPdbGuids.ContainsKey(guid)) + string debugId = GetDebugId(guid, file); + if (!string.IsNullOrEmpty(guid) && !allPdbGuids.ContainsKey(debugId)) { - allPdbGuids.Add(guid, file); + allPdbGuids.Add(debugId, file); } } return allPdbGuids; } + + /// + /// Calculates a debug Id from debug guid and filename. We use this as a key + /// in PDB hashtable. Guid is not enough due to collisions in several PDBs. + /// + private string GetDebugId(string guid, string file) => + $"{guid}.{Path.GetFileNameWithoutExtension(file)}".ToLower(); } } diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs index 38779c74a..d8d6286bb 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs @@ -23,12 +23,6 @@ public class DotNetFormatTests : SdkTests // [Fact] public void FormatProject() { - if (Config.TargetRid.Contains("alpine")) - { - // Skipping this test on Alpine due to https://github.com/dotnet/format/issues/1945 - return; - } - string unformattedCsFilePath = Path.Combine(BaselineHelper.GetAssetsDirectory(), UnformattedFileName); string projectDirectory = DotNetHelper.ExecuteNew("console", nameof(FormatProject), "C#"); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs index 46efa0420..a4727104f 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs @@ -73,6 +73,7 @@ public class LicenseScanTests : TestBase "lgpl-2.0-plus", // https://opensource.org/license/lgpl-2-0/ "lgpl-2.1", // https://opensource.org/license/lgpl-2-1/ "lgpl-2.1-plus", // https://opensource.org/license/lgpl-2-1/ + "lzma-sdk-9.22", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/lzma-sdk-9.22.LICENSE "mit", // https://opensource.org/license/mit/ "mit-addition", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/mit-addition.LICENSE "ms-patent-promise", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/ms-patent-promise.LICENSE @@ -142,17 +143,16 @@ public class LicenseScanTests : TestBase { Assert.NotNull(Config.LicenseScanPath); - string OriginalScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-original.json"); - string FilteredScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-filtered.json"); + string scancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results.json"); // Scancode Doc: https://scancode-toolkit.readthedocs.io/en/latest/index.html string ignoreOptions = string.Join(" ", s_ignoredFilePatterns.Select(pattern => $"--ignore {pattern}")); ExecuteHelper.ExecuteProcessValidateExitCode( "scancode", - $"--license --strip-root --only-findings {ignoreOptions} --json-pp {OriginalScancodeResultsPath} {Config.LicenseScanPath}", + $"--license --strip-root --only-findings {ignoreOptions} --json-pp {scancodeResultsPath} {Config.LicenseScanPath}", OutputHelper); - JsonDocument doc = JsonDocument.Parse(File.ReadAllText(OriginalScancodeResultsPath)); + JsonDocument doc = JsonDocument.Parse(File.ReadAllText(scancodeResultsPath)); ScancodeResults? scancodeResults = doc.Deserialize(); Assert.NotNull(scancodeResults); @@ -163,7 +163,6 @@ public class LicenseScanTests : TestBase WriteIndented = true }; string json = JsonSerializer.Serialize(scancodeResults, options); - File.WriteAllText(FilteredScancodeResultsPath, json); string baselineName = $"Licenses.{_targetRepo}.json"; diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourcelinkTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourcelinkTests.cs index 1c20b57e2..996233830 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourcelinkTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourcelinkTests.cs @@ -14,6 +14,15 @@ using Xunit.Abstractions; namespace Microsoft.DotNet.SourceBuild.SmokeTests; +/// +/// Separate test collection for Sourcelink tests. This is needed due to intra-test parallelization, +/// which can cause less CPU time to be allocated to other tests. +/// This would make other tests run too long and fail due to timeouts. +/// +[CollectionDefinition(nameof(SourcelinkTestCollection), DisableParallelization = true)] +public class SourcelinkTestCollection { } + +[Collection(nameof(SourcelinkTestCollection))] public class SourcelinkTests : SdkTests { private static string SourcelinkRoot { get; } = Path.Combine(Directory.GetCurrentDirectory(), nameof(SourcelinkTests)); @@ -27,24 +36,40 @@ public class SourcelinkTests : SdkTests // [Fact] public void VerifySourcelinks() { - if (Directory.Exists(SourcelinkRoot)) + try + { + if (Directory.Exists(SourcelinkRoot)) + { + Directory.Delete(SourcelinkRoot, true); + } + Directory.CreateDirectory(SourcelinkRoot); + + string symbolsRoot = Directory.CreateDirectory(Path.Combine(SourcelinkRoot, "symbols")).FullName; + + // We are validating dotnet-symbols-all-*.tar.gz which contains all source-built symbols, including + // SDK-specific symbols that are also packaged in dotnet-symbols-sdk-*.tar.gz. + Utilities.ExtractTarball( + Utilities.GetFile(Path.GetDirectoryName(Config.SourceBuiltArtifactsPath), "dotnet-symbols-all-*.tar.gz"), + symbolsRoot, + OutputHelper); + + IList failedFiles = ValidateSymbols(symbolsRoot, InitializeSourcelinkTool()); + + if (failedFiles.Count > 0) + { + OutputHelper.WriteLine($"Sourcelink verification failed for the following files:"); + foreach (string file in failedFiles) + { + OutputHelper.WriteLine(file); + } + } + + Assert.True(failedFiles.Count == 0); + } + finally { Directory.Delete(SourcelinkRoot, true); } - Directory.CreateDirectory(SourcelinkRoot); - - IList failedFiles = ValidateSymbols(ExtractSymbolsPackages(GetAllSymbolsPackages()), InitializeSourcelinkTool()); - - if (failedFiles.Count > 0) - { - OutputHelper.WriteLine($"Sourcelink verification failed for the following files:"); - foreach (string file in failedFiles) - { - OutputHelper.WriteLine(file); - } - } - - Assert.True(failedFiles.Count == 0); } /// @@ -54,6 +79,8 @@ public class SourcelinkTests : SdkTests /// Path to sourcelink tool binary. private string InitializeSourcelinkTool() { + Assert.NotNull(Config.SourceBuiltArtifactsPath); + const string SourcelinkToolPackageNamePattern = "dotnet-sourcelink*nupkg"; const string SourcelinkToolBinaryFilename = "dotnet-sourcelink.dll"; @@ -66,38 +93,6 @@ public class SourcelinkTests : SdkTests return Utilities.GetFile(extractedToolPath, SourcelinkToolBinaryFilename); } - private IEnumerable GetAllSymbolsPackages() - { - /* - At the moment we validate sourcelinks from runtime symbols package. - The plan is to make symbols, from all repos, available in source-build artifacts. - Once that's available, this code will be modified to validate all available symbols. - Tracking issue: https://github.com/dotnet/source-build/issues/3612 - */ - - // Runtime symbols package lives in the same directory as PSB artifacts. - // i.e. /artifacts/x64/Release/runtime/dotnet-runtime-symbols-fedora.36-x64-8.0.0-preview.7.23355.7.tar.gz - yield return Utilities.GetFile(Path.GetDirectoryName(Config.SourceBuiltArtifactsPath), "dotnet-runtime-symbols-*.tar.gz"); - } - - /// - /// Extracts symbols packages to subdirectories of the common symbols root directory. - /// - /// Path to common symbols root directory. - private string ExtractSymbolsPackages(IEnumerable packages) - { - string symbolsRoot = Directory.CreateDirectory(Path.Combine(SourcelinkRoot, "symbols")).FullName; - - foreach (string package in packages) - { - Assert.True(package.EndsWith(".tar.gz"), $"Package extension is not supported: {package}"); - DirectoryInfo targetDirInfo = Directory.CreateDirectory(Path.Combine(symbolsRoot, Path.GetFileNameWithoutExtension(package))); - Utilities.ExtractTarball(package, targetDirInfo.FullName, OutputHelper); - } - - return symbolsRoot; - } - private IList ValidateSymbols(string path, string sourcelinkToolPath) { Assert.True(Directory.Exists(path), $"Path, with symbol files to validate, does not exist: {path}"); @@ -113,7 +108,7 @@ public class SourcelinkTests : SdkTests OutputHelper, logOutput: false, excludeInfo: true, // Exclude info messages, as there can be 1,000+ processes - millisecondTimeout: 5000, + millisecondTimeout: 60000, configureCallback: (process) => DotNetHelper.ConfigureProcess(process, null)); if (executeResult.Process.ExitCode != 0) diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt index e0efc77dd..9bcfbafc2 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt @@ -28,8 +28,9 @@ src/arcade/src/Microsoft.DotNet.Build.Tasks.Installers/build/wix/eula.rtf # aspnetcore # -# Line 1 is a generic statement about license applicability that is being detected as "unknown" +# A generic statement about license applicability that is being detected as "unknown" src/aspnetcore/src/Components/THIRD-PARTY-NOTICES.txt|unknown +src/aspnetcore/THIRD-PARTY-NOTICES.txt|unknown # Windows installer files that have a reference to a URL for license src/aspnetcore/src/Installers/Windows/**/*.wxl|unknown-license-reference @@ -167,7 +168,7 @@ src/runtime/src/installer/pkg/THIRD-PARTY-NOTICES.TXT # False positive src/runtime/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/LicFileLicenseProvider.cs|proprietary-license -src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|json +src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|embedthis-extension src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/Connect.cs|other-permissive src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/UdpClientTest.cs|other-permissive src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs|other-permissive @@ -226,10 +227,11 @@ src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library/2. src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library.ref/2.1.0/THIRD-PARTY-NOTICES.TXT|codesourcery-2004 src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/ThirdPartyNotices.rtf|json src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.netcore.*/1.*/ThirdPartyNotices.txt|unknown +src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/*/1033/System.Security.Permissions.xml|unknown-license-reference # Contains references to licenses which are not applicable to the source src/source-build-reference-packages/src/packageSourceGenerator/PackageSourceGeneratorTask/RewriteNuspec.cs|unknown-license-reference,ms-net-library-2018-11 -src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.0-preview-20230918.1/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license +src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license # # sourcelink diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdkFiles.diff b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdkFiles.diff index 434883537..5749c4e82 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdkFiles.diff +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdkFiles.diff @@ -48,6 +48,7 @@ index ------------ ./packs/NETStandard.Library.Ref/x.y.z/ref/netstandard2.1/System.Xml.XPath.XDocument.dll ./sdk-manifests/ ./sdk-manifests/x.y.z/ +-./sdk-manifests/x.y.z/ -./sdk-manifests/x.y.z/ ./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/ ./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/x.y.z/ diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/PoisonUsage.txt b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/PoisonUsage.txt index f1ab9ecc1..fee5ec5e8 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/PoisonUsage.txt +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/PoisonUsage.txt @@ -1 +1,14 @@ - \ No newline at end of file + + + SourceBuildReferenceAssembly + + + SourceBuildReferenceAssembly + + + SourceBuildReferenceAssembly + + + SourceBuildReferenceAssembly + + \ No newline at end of file diff --git a/src/core-sdk-tasks/ReplaceDuplicateFilesWithHardLinks.cs b/src/core-sdk-tasks/ReplaceFilesWithSymbolicLinks.cs similarity index 50% rename from src/core-sdk-tasks/ReplaceDuplicateFilesWithHardLinks.cs rename to src/core-sdk-tasks/ReplaceFilesWithSymbolicLinks.cs index 7fe6559fc..44c6ca31b 100644 --- a/src/core-sdk-tasks/ReplaceDuplicateFilesWithHardLinks.cs +++ b/src/core-sdk-tasks/ReplaceFilesWithSymbolicLinks.cs @@ -22,18 +22,24 @@ namespace Microsoft.DotNet.Build.Tasks /// /// Replaces files that have the same content with hard links. /// - public sealed class ReplaceDuplicateFilesWithHardLinks : Task + public sealed class ReplaceFilesWithSymbolicLinks : Task { /// - /// The path to the directory. + /// The path to the directory to recursively search for files to replace with symbolic links. /// [Required] public string Directory { get; set; } = ""; + /// + /// The path to the directory with files to link to. + /// + [Required] + public string LinkToFilesFrom { get; set; } = ""; + #if NETFRAMEWORK public override bool Execute() { - Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on .NET Framework."); + Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on .NET Framework."); return false; } #else @@ -41,7 +47,7 @@ namespace Microsoft.DotNet.Build.Tasks { if (OperatingSystem.IsWindows()) { - Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on Windows."); + Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on Windows."); return false; } @@ -51,52 +57,36 @@ namespace Microsoft.DotNet.Build.Tasks return false; } + if (!System.IO.Directory.Exists(LinkToFilesFrom)) + { + Log.LogError($"'{LinkToFilesFrom}' does not exist."); + return false; + } + // Find all non-empty, non-symbolic link files. - IEnumerable fse = new FileSystemEnumerable( - Directory, - (ref FileSystemEntry entry) => (FileInfo)entry.ToFileSystemInfo(), - new EnumerationOptions() - { - AttributesToSkip = FileAttributes.ReparsePoint, - RecurseSubdirectories = true - }) + string[] files = new FileSystemEnumerable( + Directory, + (ref FileSystemEntry entry) => entry.ToFullPath(), + new EnumerationOptions() + { + AttributesToSkip = FileAttributes.ReparsePoint, + RecurseSubdirectories = true + }) { ShouldIncludePredicate = (ref FileSystemEntry entry) => !entry.IsDirectory && entry.Length > 0 - }; + }.ToArray(); - // Group them by file size. - IEnumerable filesGroupedBySize = fse.GroupBy(file => file.Length, - file => file.FullName, - (size, files) => files.ToArray()); - - // Replace files with same content with hard link. - foreach (var files in filesGroupedBySize) + foreach (var file in files) { - for (int i = 0; i < files.Length; i++) + string fileName = Path.GetFileName(file); + + // Look for a file with the same name in LinkToFilesFrom + // and replace it with a symbolic link if it has the same content. + string targetFile = Path.Combine(LinkToFilesFrom, fileName); + if (File.Exists(targetFile) && FilesHaveSameContent(file, targetFile)) { - string? path1 = files[i]; - if (path1 is null) - { - continue; // already linked. - } - for (int j = i + 1; j < files.Length; j++) - { - string? path2 = files[j]; - if (path2 is null) - { - continue; // already linked. - } - - // note: There's no public API we can use to see if paths are already linked. - // We treat those paths as unlinked files, and link them again. - if (FilesHaveSameContent(path1, path2)) - { - ReplaceByLink(path1, path2); - - files[j] = null; - } - } + ReplaceByLinkTo(file, targetFile); } } @@ -138,34 +128,30 @@ namespace Microsoft.DotNet.Build.Tasks } } - void ReplaceByLink(string path1, string path2) + void ReplaceByLinkTo(string path, string pathToTarget) { // To link, the target mustn't exist. Make a backup, so we can restore it when linking fails. - string path2Backup = $"{path2}.pre_link_backup"; - File.Move(path2, path2Backup); + string backupFile = $"{path}.pre_link_backup"; + File.Move(path, backupFile); - int rv = SystemNative_Link(path1, path2); - if (rv != 0) + try { - var ex = new Win32Exception(); // Captures the LastError. + string relativePath = Path.GetRelativePath(Path.GetDirectoryName(path)!, pathToTarget); + File.CreateSymbolicLink(path, relativePath); - Log.LogError($"Unable to link '{path2}' to '{path1}.': {ex}"); + File.Delete(backupFile); - File.Move(path2Backup, path2); - - throw ex; + Log.LogMessage(MessageImportance.Normal, $"Linked '{path}' to '{relativePath}'."); } - else + catch (Exception ex) { - File.Delete(path2Backup); + Log.LogError($"Unable to link '{path}' to '{pathToTarget}.': {ex}"); - Log.LogMessage(MessageImportance.Normal, $"Linked '{path1}' and '{path2}'."); + File.Move(backupFile, path); + + throw; } } - - // This native method is used by the runtime to create hard links. It is not exposed through a public .NET API. - [DllImport("libSystem.Native", SetLastError = true)] - static extern int SystemNative_Link(string source, string link); #endif } } diff --git a/src/redist/targets/BuildCoreSdkTasks.targets b/src/redist/targets/BuildCoreSdkTasks.targets index 6268b743f..a0fd17dea 100644 --- a/src/redist/targets/BuildCoreSdkTasks.targets +++ b/src/redist/targets/BuildCoreSdkTasks.targets @@ -35,7 +35,7 @@ - + diff --git a/src/redist/targets/GenerateLayout.targets b/src/redist/targets/GenerateLayout.targets index a73c0922c..0e9a4664c 100644 --- a/src/redist/targets/GenerateLayout.targets +++ b/src/redist/targets/GenerateLayout.targets @@ -569,11 +569,12 @@ - - - + + + + + diff --git a/test/SdkTests/TestConfig.xml b/test/SdkTests/TestConfig.xml index 78e933a49..ed432ea13 100644 --- a/test/SdkTests/TestConfig.xml +++ b/test/SdkTests/TestConfig.xml @@ -245,6 +245,14 @@ Skip="true" Issue="" Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/> + +