Merge branch 'release/8.0.2xx'

eng/ManualVersions.props

@@ -9,11 +9,11 @@
      Basically: In this file, choose the highest version when resolving merge conflicts.
  -->
   <PropertyGroup>
-    <MicrosoftWindowsSDKNETRef10_0_17763PackageVersion>10.0.17763.29</MicrosoftWindowsSDKNETRef10_0_17763PackageVersion>
+    <MicrosoftWindowsSDKNETRef10_0_17763PackageVersion>10.0.17763.31</MicrosoftWindowsSDKNETRef10_0_17763PackageVersion>
-    <MicrosoftWindowsSDKNETRef10_0_18362PackageVersion>10.0.18362.29</MicrosoftWindowsSDKNETRef10_0_18362PackageVersion>
+    <MicrosoftWindowsSDKNETRef10_0_18362PackageVersion>10.0.18362.31</MicrosoftWindowsSDKNETRef10_0_18362PackageVersion>
-    <MicrosoftWindowsSDKNETRef10_0_19041PackageVersion>10.0.19041.29</MicrosoftWindowsSDKNETRef10_0_19041PackageVersion>
+    <MicrosoftWindowsSDKNETRef10_0_19041PackageVersion>10.0.19041.31</MicrosoftWindowsSDKNETRef10_0_19041PackageVersion>
-    <MicrosoftWindowsSDKNETRef10_0_20348PackageVersion>10.0.20348.29</MicrosoftWindowsSDKNETRef10_0_20348PackageVersion>
+    <MicrosoftWindowsSDKNETRef10_0_20348PackageVersion>10.0.20348.31</MicrosoftWindowsSDKNETRef10_0_20348PackageVersion>
-    <MicrosoftWindowsSDKNETRef10_0_22000PackageVersion>10.0.22000.29</MicrosoftWindowsSDKNETRef10_0_22000PackageVersion>
+    <MicrosoftWindowsSDKNETRef10_0_22000PackageVersion>10.0.22000.31</MicrosoftWindowsSDKNETRef10_0_22000PackageVersion>
-    <MicrosoftWindowsSDKNETRef10_0_22621PackageVersion>10.0.22621.29</MicrosoftWindowsSDKNETRef10_0_22621PackageVersion>
+    <MicrosoftWindowsSDKNETRef10_0_22621PackageVersion>10.0.22621.31</MicrosoftWindowsSDKNETRef10_0_22621PackageVersion>
   </PropertyGroup>
 </Project>

eng/pipelines/templates/jobs/vmr-build.yml

@@ -127,7 +127,7 @@ jobs:
       artifact: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts
       patterns: |
         **/Private.SourceBuilt.Artifacts.*.tar.gz
-        **/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])*.tar.gz
+        **/dotnet-sdk-*.tar.gz
       displayName: Download Previous Build
 
     - task: CopyFiles@2

eng/pipelines/vmr-sync.yml

@@ -6,6 +6,10 @@ trigger:
     include:
     - main
     - release/*
+    exclude:
+    - release/*.0.2xx
+    - release/*.0.3xx
+    - release/*.0.4xx
 
 resources:
   repositories:

src/SourceBuild/content/build.proj

@@ -95,7 +95,7 @@
             DiscoverSymbolsTarballs;
             ExtractSymbolsTarballs">
     <PropertyGroup>
-      <UnifiedSymbolsTarball>$(OutputPath)dotnet-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</UnifiedSymbolsTarball>
+      <UnifiedSymbolsTarball>$(OutputPath)dotnet-symbols-all-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</UnifiedSymbolsTarball>
     </PropertyGroup>
 
     <Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
@@ -109,13 +109,12 @@
           AfterTargets="Build"
           DependsOnTargets="RepackageSymbols">
     <ItemGroup>
-      <SdkTarballItem Include="$(OutputPath)dotnet-sdk-*$(TarBallExtension)"
+      <SdkTarballItem Include="$(OutputPath)dotnet-sdk-*$(TarBallExtension)" />
-                      Exclude="$(OutputPath)dotnet-sdk-symbols-*$(TarBallExtension)" />
     </ItemGroup>
 
     <PropertyGroup>
       <SdkSymbolsLayout>$(ArtifactsTmpDir)SdkSymbols</SdkSymbolsLayout>
-      <SdkSymbolsTarball>$(OutputPath)dotnet-sdk-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</SdkSymbolsTarball>
+      <SdkSymbolsTarball>$(OutputPath)dotnet-symbols-sdk-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</SdkSymbolsTarball>
       <SdkLayout>$(ArtifactsTmpDir)Sdk</SdkLayout>
       <SdkTarball>%(SdkTarballItem.Identity)</SdkTarball>
     </PropertyGroup>
@@ -187,8 +186,7 @@
 
   <Target Name="RunSmokeTest">
     <ItemGroup>
-      <SdkTarballItem Include="$(SourceBuiltTarBallPath)**/dotnet-sdk*$(TarBallExtension)"
+      <SdkTarballItem Include="$(SourceBuiltTarBallPath)**/dotnet-sdk*$(TarBallExtension)" />
-                      Exclude="$(SourceBuiltTarBallPath)**/dotnet-sdk-symbols*$(TarBallExtension)" />
       <SourceBuiltArtifactsItem Include="$(SourceBuiltTarBallPath)**/Private.SourceBuilt.Artifacts.*$(TarBallExtension)" />
     </ItemGroup>
 

src/SourceBuild/content/eng/Versions.props

@@ -30,8 +30,8 @@
       These URLs can't be composed from their base URL and version as we read them from the
       prep.sh and pipeline scripts, outside of MSBuild.
     -->
-    <PrivateSourceBuiltArtifactsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.8.0.100-rc.1.23455.1.centos.8-x64.tar.gz</PrivateSourceBuiltArtifactsUrl>
+    <PrivateSourceBuiltArtifactsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.8.0.100-rc.2.23502.1.centos.8-x64.tar.gz</PrivateSourceBuiltArtifactsUrl>
-    <PrivateSourceBuiltSdkUrl_CentOS8Stream>https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-rc.1.23455.1-centos.8-x64.tar.gz</PrivateSourceBuiltSdkUrl_CentOS8Stream>
+    <PrivateSourceBuiltSdkUrl_CentOS8Stream>https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-rc.2.23502.1-centos.8-x64.tar.gz</PrivateSourceBuiltSdkUrl_CentOS8Stream>
     <PrivateSourceBuiltPrebuiltsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Prebuilts.0.1.0-9.0.100-3.centos.8-x64.tar.gz</PrivateSourceBuiltPrebuiltsUrl>
   </PropertyGroup>
 </Project>

src/SourceBuild/content/eng/bootstrap/OverrideBootstrapVersions.props

@@ -1,11 +1,23 @@
 <Project>
   <PropertyGroup>
-     <!-- 7.0.0 produced improperly versioned runtime assets because the OfficialBuildId
+    <!-- 7.0.0 produced improperly versioned runtime assets because the OfficialBuildId
-          was not set correctly.  This is the actual shipping version that it should have been -->
+         was not set correctly.  This is the actual shipping version that it should have been -->
-     <NonshippingRuntimeVersionFor700>7.0.4-servicing.23107.6</NonshippingRuntimeVersionFor700>
+    <NonshippingRuntimeVersionFor700>7.0.4-servicing.23107.6</NonshippingRuntimeVersionFor700>
+    
+    <MicrosoftNETCoreTestHostVersion>$(NonshippingRuntimeVersionFor700)</MicrosoftNETCoreTestHostVersion>
 
-     <MicrosoftNETHostModelVersion>$(NonshippingRuntimeVersionFor700)</MicrosoftNETHostModelVersion>
+    <Msft80RC2RuntimeVersion>8.0.0-rc.2.23479.6</Msft80RC2RuntimeVersion>
-     <MicrosoftNETCoreTestHostVersion>$(NonshippingRuntimeVersionFor700)</MicrosoftNETCoreTestHostVersion>
+    <MicrosoftNETCoreAppCrossgen2Version>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreAppCrossgen2Version>
-     <RuntimeLinuxX64MicrosoftNETCoreTestHostVersion>$(NonshippingRuntimeVersionFor700)</RuntimeLinuxX64MicrosoftNETCoreTestHostVersion>
+    <MicrosoftNETCoreAppHostPackageVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreAppHostPackageVersion>
+    <MicrosoftNETCoreAppRuntimeVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreAppRuntimeVersion>
+    <MicrosoftNETILLinkTasksVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETILLinkTasksVersion>
+    <MicrosoftDotNetILCompilerVersion>$(Msft80RC2RuntimeVersion)</MicrosoftDotNetILCompilerVersion>
+    <MicrosoftNETCoreDotNetAppHostVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreDotNetAppHostVersion>
+    <MicrosoftNETCoreDotNetHostVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreDotNetHostVersion>
+    <MicrosoftNETCoreDotNetHostPolicyVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreDotNetHostPolicyVersion>
+    <MicrosoftNETCoreDotNetHostResolverVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreDotNetHostResolverVersion>
+    <MicrosoftNETCoreILAsmVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreILAsmVersion>
+    <MicrosoftNETCoreILDAsmVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreILDAsmVersion>
+    <RuntimeNativeSystemIOPortsVersion>$(Msft80RC2RuntimeVersion)</RuntimeNativeSystemIOPortsVersion>
   </PropertyGroup>
 </Project>

src/SourceBuild/content/eng/bootstrap/buildBootstrapPreviouslySB.csproj

@@ -47,6 +47,8 @@
     <!-- These packages don't actually exist -->
     <ExcludedPackage Include="runtime.linux-musl-x64.runtime.native.System.IO.Ports" />
     <ExcludedPackage Include="runtime.linux-musl-arm64.runtime.native.System.IO.Ports" />
+
+    <PackageDownload Include="Microsoft.NET.ILLink.Tasks" Version="[$(MicrosoftNETILLinkTasksVersion)]" />
   </ItemGroup>
 
   <Target Name="GetPackagesToDownload"

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs

@@ -12,8 +12,11 @@ using System.IO;
 using System.IO.Compression;
 using System.Linq;
 using System.Reflection;
+using System.Reflection.Metadata;
+using System.Reflection.PortableExecutable;
 using System.Security.Cryptography;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Xml;
 using System.Xml.Linq;
 
@@ -147,6 +150,10 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
 
         private const string PoisonMarker = "POISONED";
 
+        private const string SbrpAttributeType = "System.Reflection.AssemblyMetadataAttribute";
+
+        private const string SbrpAttributeValuePattern = "source\\s?source\\-build\\-reference\\-packages";
+
         private record CandidateFileEntry(string ExtractedPath, string DisplayPath);
 
         public override bool Execute()
@@ -298,7 +305,11 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
             try
             {
                 AssemblyName asm = AssemblyName.GetAssemblyName(fileToCheck);
-                if (IsAssemblyPoisoned(fileToCheck))
+                if (!candidate.DisplayPath.Contains("SourceBuildReferencePackages") && IsAssemblyFromSbrp(fileToCheck))
+                {
+                    poisonEntry.Type |= PoisonType.SourceBuildReferenceAssembly;
+                }
+                else if (IsAssemblyPoisoned(fileToCheck))
                 {
                     poisonEntry.Type |= PoisonType.AssemblyAttribute;
                 }
@@ -332,6 +343,41 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
             return false;
         }
 
+        private static bool IsAssemblyFromSbrp(string assemblyPath)
+        {
+            using var stream = new FileStream(assemblyPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+            using var peReader = new PEReader(stream);
+
+            MetadataReader reader = peReader.GetMetadataReader();
+            return reader.CustomAttributes.Select(attrHandle => reader.GetCustomAttribute(attrHandle))
+                    .Any(attr => IsAttributeSbrp(reader, attr));
+        }
+
+        private static bool IsAttributeSbrp(MetadataReader reader, CustomAttribute attr)
+        {
+            string attributeType = string.Empty;
+
+            if (attr.Constructor.Kind == HandleKind.MemberReference)
+            {
+                MemberReference mref = reader.GetMemberReference((MemberReferenceHandle)attr.Constructor);
+
+                if (mref.Parent.Kind == HandleKind.TypeReference)
+                {
+                    TypeReference tref = reader.GetTypeReference((TypeReferenceHandle)mref.Parent);
+                    attributeType = $"{reader.GetString(tref.Namespace)}.{reader.GetString(tref.Name)}";
+                }
+            }
+
+            if (attributeType == SbrpAttributeType)
+            {
+                BlobReader blobReader = reader.GetBlobReader(attr.Value);
+                string attributeValue = Encoding.UTF8.GetString(blobReader.ReadBytes(blobReader.Length));
+                attributeValue = Regex.Replace(attributeValue, @"\p{C}+", string.Empty);
+                return Regex.IsMatch(attributeValue, SbrpAttributeValuePattern);
+            }
+            return false;
+        }
+
         private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue<CandidateFileEntry> futureFilesToCheck)
         {
             var poisonEntry = new PoisonedFileEntry();

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs

@@ -155,6 +155,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
                     }
                     File.Delete(p.ItemSpec);
                     File.Move(poisonedPackagePath, p.ItemSpec);
+                    Directory.Delete(packageTempPath, true);
                 }
             }
 

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs

@@ -11,5 +11,6 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
         Hash = 1,
         AssemblyAttribute = 2,
         NupkgFile = 4,
+        SourceBuildReferenceAssembly = 8,
     }
 }

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/CreateSdkSymbolsLayout.cs

@@ -57,7 +57,7 @@ namespace Microsoft.DotNet.Build.Tasks
 
         private void LogErrorOrWarning(bool isError, string message)
         {
-            if (FailOnMissingPDBs)
+            if (isError)
             {
                 Log.LogError(message);
             }
@@ -104,14 +104,15 @@ namespace Microsoft.DotNet.Build.Tasks
 
                     if (guid != string.Empty)
                     {
-                        if (!allPdbGuids.ContainsKey(guid))
+                        string debugId = GetDebugId(guid, file);
+                        if (!allPdbGuids.ContainsKey(debugId))
                         {
                             filesWithoutPDBs.Add(file.Substring(SdkLayoutPath.Length + 1));
                         }
                         else
                         {
                             // Copy matching pdb to symbols path, preserving sdk binary's hierarchy
-                            string sourcePath = (string)allPdbGuids[guid]!;
+                            string sourcePath = (string)allPdbGuids[debugId]!;
                             string destinationPath =
                                 file.Replace(SdkLayoutPath, SdkSymbolsLayoutPath)
                                     .Replace(Path.GetFileName(file), Path.GetFileName(sourcePath));
@@ -142,13 +143,21 @@ namespace Microsoft.DotNet.Build.Tasks
 
                 var id = new BlobContentId(metadataReader.DebugMetadataHeader.Id);
                 string guid = $"{id.Guid:N}";
-                if (!string.IsNullOrEmpty(guid) && !allPdbGuids.ContainsKey(guid))
+                string debugId = GetDebugId(guid, file);
+                if (!string.IsNullOrEmpty(guid) && !allPdbGuids.ContainsKey(debugId))
                 {
-                    allPdbGuids.Add(guid, file);
+                    allPdbGuids.Add(debugId, file);
                 }
             }
 
             return allPdbGuids;
         }
+
+        /// <summary>
+        /// Calculates a debug Id from debug guid and filename. We use this as a key
+        /// in PDB hashtable. Guid is not enough due to collisions in several PDBs.
+        /// </summary>
+        private string GetDebugId(string guid, string file) =>
+            $"{guid}.{Path.GetFileNameWithoutExtension(file)}".ToLower();
     }
 }

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs

@@ -23,12 +23,6 @@ public class DotNetFormatTests : SdkTests
     // [Fact]
     public void FormatProject()
     {
-        if (Config.TargetRid.Contains("alpine"))
-        {
-            // Skipping this test on Alpine due to https://github.com/dotnet/format/issues/1945
-            return;
-        }
-
         string unformattedCsFilePath = Path.Combine(BaselineHelper.GetAssetsDirectory(), UnformattedFileName);
 
         string projectDirectory = DotNetHelper.ExecuteNew("console", nameof(FormatProject), "C#");

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs

@@ -73,6 +73,7 @@ public class LicenseScanTests : TestBase
         "lgpl-2.0-plus", // https://opensource.org/license/lgpl-2-0/
         "lgpl-2.1", // https://opensource.org/license/lgpl-2-1/
         "lgpl-2.1-plus", // https://opensource.org/license/lgpl-2-1/
+        "lzma-sdk-9.22", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/lzma-sdk-9.22.LICENSE
         "mit", // https://opensource.org/license/mit/
         "mit-addition", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/mit-addition.LICENSE
         "ms-patent-promise", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/ms-patent-promise.LICENSE
@@ -142,17 +143,16 @@ public class LicenseScanTests : TestBase
     {
         Assert.NotNull(Config.LicenseScanPath);
 
-        string OriginalScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-original.json");
+        string scancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results.json");
-        string FilteredScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-filtered.json");
 
         // Scancode Doc: https://scancode-toolkit.readthedocs.io/en/latest/index.html
         string ignoreOptions = string.Join(" ", s_ignoredFilePatterns.Select(pattern => $"--ignore {pattern}"));
         ExecuteHelper.ExecuteProcessValidateExitCode(
             "scancode",
-            $"--license --strip-root --only-findings {ignoreOptions} --json-pp {OriginalScancodeResultsPath} {Config.LicenseScanPath}",
+            $"--license --strip-root --only-findings {ignoreOptions} --json-pp {scancodeResultsPath} {Config.LicenseScanPath}",
             OutputHelper);
 
-        JsonDocument doc = JsonDocument.Parse(File.ReadAllText(OriginalScancodeResultsPath));
+        JsonDocument doc = JsonDocument.Parse(File.ReadAllText(scancodeResultsPath));
         ScancodeResults? scancodeResults = doc.Deserialize<ScancodeResults>();
         Assert.NotNull(scancodeResults);
 
@@ -163,7 +163,6 @@ public class LicenseScanTests : TestBase
             WriteIndented = true
         };
         string json = JsonSerializer.Serialize(scancodeResults, options);
-        File.WriteAllText(FilteredScancodeResultsPath, json);
 
         string baselineName = $"Licenses.{_targetRepo}.json";
 

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourcelinkTests.cs

@@ -14,6 +14,15 @@ using Xunit.Abstractions;
 
 namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 
+/// <summary>
+/// Separate test collection for Sourcelink tests. This is needed due to intra-test parallelization,
+/// which can cause less CPU time to be allocated to other tests.
+/// This would make other tests run too long and fail due to timeouts.
+/// </summary>
+[CollectionDefinition(nameof(SourcelinkTestCollection), DisableParallelization = true)]
+public class SourcelinkTestCollection { }
+
+[Collection(nameof(SourcelinkTestCollection))]
 public class SourcelinkTests : SdkTests
 {
     private static string SourcelinkRoot { get; } = Path.Combine(Directory.GetCurrentDirectory(), nameof(SourcelinkTests));
@@ -27,24 +36,40 @@ public class SourcelinkTests : SdkTests
     // [Fact]
     public void VerifySourcelinks()
     {
-        if (Directory.Exists(SourcelinkRoot))
+        try
+        {
+            if (Directory.Exists(SourcelinkRoot))
+            {
+                Directory.Delete(SourcelinkRoot, true);
+            }
+            Directory.CreateDirectory(SourcelinkRoot);
+
+            string symbolsRoot = Directory.CreateDirectory(Path.Combine(SourcelinkRoot, "symbols")).FullName;
+
+            // We are validating dotnet-symbols-all-*.tar.gz which contains all source-built symbols, including
+            // SDK-specific symbols that are also packaged in dotnet-symbols-sdk-*.tar.gz.
+            Utilities.ExtractTarball(
+                Utilities.GetFile(Path.GetDirectoryName(Config.SourceBuiltArtifactsPath), "dotnet-symbols-all-*.tar.gz"),
+                symbolsRoot,
+                OutputHelper);
+
+            IList<string> failedFiles = ValidateSymbols(symbolsRoot, InitializeSourcelinkTool());
+
+            if (failedFiles.Count > 0)
+            {
+                OutputHelper.WriteLine($"Sourcelink verification failed for the following files:");
+                foreach (string file in failedFiles)
+                {
+                    OutputHelper.WriteLine(file);
+                }
+            }
+
+            Assert.True(failedFiles.Count == 0);
+        }
+        finally
         {
             Directory.Delete(SourcelinkRoot, true);
         }
-        Directory.CreateDirectory(SourcelinkRoot);
-
-        IList<string> failedFiles = ValidateSymbols(ExtractSymbolsPackages(GetAllSymbolsPackages()), InitializeSourcelinkTool());
-
-        if (failedFiles.Count > 0)
-        {
-            OutputHelper.WriteLine($"Sourcelink verification failed for the following files:");
-            foreach (string file in failedFiles)
-            {
-                OutputHelper.WriteLine(file);
-            }
-        }
-
-        Assert.True(failedFiles.Count == 0);
     }
 
     /// <summary>
@@ -54,6 +79,8 @@ public class SourcelinkTests : SdkTests
     /// <returns>Path to sourcelink tool binary.</returns>
     private string InitializeSourcelinkTool()
     {
+        Assert.NotNull(Config.SourceBuiltArtifactsPath);
+        
         const string SourcelinkToolPackageNamePattern = "dotnet-sourcelink*nupkg";
         const string SourcelinkToolBinaryFilename = "dotnet-sourcelink.dll";
 
@@ -66,38 +93,6 @@ public class SourcelinkTests : SdkTests
         return Utilities.GetFile(extractedToolPath, SourcelinkToolBinaryFilename);
     }
 
-    private IEnumerable<string> GetAllSymbolsPackages()
-    {
-        /*
-            At the moment we validate sourcelinks from runtime symbols package.
-            The plan is to make symbols, from all repos, available in source-build artifacts.
-            Once that's available, this code will be modified to validate all available symbols.
-            Tracking issue: https://github.com/dotnet/source-build/issues/3612
-        */
-
-        // Runtime symbols package lives in the same directory as PSB artifacts.
-        // i.e. <repo-root>/artifacts/x64/Release/runtime/dotnet-runtime-symbols-fedora.36-x64-8.0.0-preview.7.23355.7.tar.gz
-        yield return Utilities.GetFile(Path.GetDirectoryName(Config.SourceBuiltArtifactsPath), "dotnet-runtime-symbols-*.tar.gz");
-    }
-
-    /// <summary>
-    /// Extracts symbols packages to subdirectories of the common symbols root directory.
-    /// </summary>
-    /// <returns>Path to common symbols root directory.</returns>
-    private string ExtractSymbolsPackages(IEnumerable<string> packages)
-    {
-        string symbolsRoot = Directory.CreateDirectory(Path.Combine(SourcelinkRoot, "symbols")).FullName;
-
-        foreach (string package in packages)
-        {
-            Assert.True(package.EndsWith(".tar.gz"), $"Package extension is not supported: {package}");
-            DirectoryInfo targetDirInfo = Directory.CreateDirectory(Path.Combine(symbolsRoot, Path.GetFileNameWithoutExtension(package)));
-            Utilities.ExtractTarball(package, targetDirInfo.FullName, OutputHelper);
-        }
-
-        return symbolsRoot;
-    }
-
     private IList<string> ValidateSymbols(string path, string sourcelinkToolPath)
     {
         Assert.True(Directory.Exists(path), $"Path, with symbol files to validate, does not exist: {path}");
@@ -113,7 +108,7 @@ public class SourcelinkTests : SdkTests
                 OutputHelper,
                 logOutput: false,
                 excludeInfo: true, // Exclude info messages, as there can be 1,000+ processes
-                millisecondTimeout: 5000,
+                millisecondTimeout: 60000,
                 configureCallback: (process) => DotNetHelper.ConfigureProcess(process, null));
 
             if (executeResult.Process.ExitCode != 0)

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt

@@ -28,8 +28,9 @@ src/arcade/src/Microsoft.DotNet.Build.Tasks.Installers/build/wix/eula.rtf
 # aspnetcore
 #
 
-# Line 1 is a generic statement about license applicability that is being detected as "unknown"
+# A generic statement about license applicability that is being detected as "unknown"
 src/aspnetcore/src/Components/THIRD-PARTY-NOTICES.txt|unknown
+src/aspnetcore/THIRD-PARTY-NOTICES.txt|unknown
 
 # Windows installer files that have a reference to a URL for license
 src/aspnetcore/src/Installers/Windows/**/*.wxl|unknown-license-reference
@@ -167,7 +168,7 @@ src/runtime/src/installer/pkg/THIRD-PARTY-NOTICES.TXT
 
 # False positive
 src/runtime/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/LicFileLicenseProvider.cs|proprietary-license
-src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|json
+src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|embedthis-extension
 src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/Connect.cs|other-permissive
 src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/UdpClientTest.cs|other-permissive
 src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs|other-permissive
@@ -226,10 +227,11 @@ src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library/2.
 src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library.ref/2.1.0/THIRD-PARTY-NOTICES.TXT|codesourcery-2004
 src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/ThirdPartyNotices.rtf|json
 src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.netcore.*/1.*/ThirdPartyNotices.txt|unknown
+src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/*/1033/System.Security.Permissions.xml|unknown-license-reference
 
 # Contains references to licenses which are not applicable to the source
 src/source-build-reference-packages/src/packageSourceGenerator/PackageSourceGeneratorTask/RewriteNuspec.cs|unknown-license-reference,ms-net-library-2018-11
-src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.0-preview-20230918.1/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license
+src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license
 
 #
 # sourcelink

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdkFiles.diff

@@ -48,6 +48,7 @@ index ------------
  ./packs/NETStandard.Library.Ref/x.y.z/ref/netstandard2.1/System.Xml.XPath.XDocument.dll
  ./sdk-manifests/
  ./sdk-manifests/x.y.z/
+-./sdk-manifests/x.y.z/
 -./sdk-manifests/x.y.z/
  ./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/
  ./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/x.y.z/

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/PoisonUsage.txt

@@ -1 +1,14 @@
-<PrebuiltLeakReport />
+<PrebuiltLeakReport>
+  <File Path="artifacts/x64/Release/dotnet-sdk-x.y.z-banana-rid.tar.gz/sdk/x.y.z/DotnetTools/dotnet-format/Microsoft.Bcl.AsyncInterfaces.dll">
+    <Type>SourceBuildReferenceAssembly</Type>
+  </File>
+  <File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/dotnet-format.x.y.z.nupkg/tools/netx.y/any/Microsoft.Bcl.AsyncInterfaces.dll">
+    <Type>SourceBuildReferenceAssembly</Type>
+  </File>
+  <File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/Microsoft.TestPlatform.CLI.x.y.z/contentFiles/any/netx.y/Microsoft.Extensions.DependencyModel.dll">
+    <Type>SourceBuildReferenceAssembly</Type>
+  </File>
+  <File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/Microsoft.TestPlatform.CLI.x.y.z/contentFiles/any/netx.y/Microsoft.Extensions.FileSystemGlobbing.dll">
+    <Type>SourceBuildReferenceAssembly</Type>
+  </File>
+</PrebuiltLeakReport>

src/core-sdk-tasks/ReplaceFilesWithSymbolicLinks.cs

@@ -22,18 +22,24 @@ namespace Microsoft.DotNet.Build.Tasks
     /// <summary>
     /// Replaces files that have the same content with hard links.
     /// </summary>
-    public sealed class ReplaceDuplicateFilesWithHardLinks  : Task
+    public sealed class ReplaceFilesWithSymbolicLinks  : Task
     {
         /// <summary>
-        /// The path to the directory.
+        /// The path to the directory to recursively search for files to replace with symbolic links.
         /// </summary>
         [Required]
         public string Directory { get; set; } = "";
 
+        /// <summary>
+        /// The path to the directory with files to link to.
+        /// </summary>
+        [Required]
+        public string LinkToFilesFrom { get; set; } = "";
+
 #if NETFRAMEWORK
         public override bool Execute()
         {
-            Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on .NET Framework.");
+            Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on .NET Framework.");
             return false;
         }
 #else
@@ -41,7 +47,7 @@ namespace Microsoft.DotNet.Build.Tasks
         {
             if (OperatingSystem.IsWindows())
             {
-                Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on Windows.");
+                Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on Windows.");
                 return false;
             }
 
@@ -51,52 +57,36 @@ namespace Microsoft.DotNet.Build.Tasks
                 return false;
             }
 
+            if (!System.IO.Directory.Exists(LinkToFilesFrom))
+            {
+                Log.LogError($"'{LinkToFilesFrom}' does not exist.");
+                return false;
+            }
+
             // Find all non-empty, non-symbolic link files.
-            IEnumerable<FileInfo> fse = new FileSystemEnumerable<FileInfo>(
+            string[] files = new FileSystemEnumerable<string>(
-                                            Directory,
+                                        Directory,
-                                            (ref FileSystemEntry entry) => (FileInfo)entry.ToFileSystemInfo(),
+                                        (ref FileSystemEntry entry) => entry.ToFullPath(),
-                                            new EnumerationOptions()
+                                        new EnumerationOptions()
-                                            {
+                                        {
-                                                AttributesToSkip = FileAttributes.ReparsePoint,
+                                            AttributesToSkip = FileAttributes.ReparsePoint,
-                                                RecurseSubdirectories = true
+                                            RecurseSubdirectories = true
-                                            })
+                                        })
             {
                 ShouldIncludePredicate = (ref FileSystemEntry entry) => !entry.IsDirectory
                                                                         && entry.Length > 0
-            };
+            }.ToArray();
 
-            // Group them by file size.
+            foreach (var file in files)
-            IEnumerable<string?[]> filesGroupedBySize = fse.GroupBy(file => file.Length,
-                                                                    file => file.FullName,
-                                                                    (size, files) => files.ToArray());
-
-            // Replace files with same content with hard link.
-            foreach (var files in filesGroupedBySize)
             {
-                for (int i = 0; i < files.Length; i++)
+                string fileName = Path.GetFileName(file);
+
+                // Look for a file with the same name in LinkToFilesFrom
+                // and replace it with a symbolic link if it has the same content.
+                string targetFile = Path.Combine(LinkToFilesFrom, fileName);
+                if (File.Exists(targetFile) && FilesHaveSameContent(file, targetFile))
                 {
-                    string? path1 = files[i];
+                    ReplaceByLinkTo(file, targetFile);
-                    if (path1 is null)
-                    {
-                        continue; // already linked.
-                    }
-                    for (int j = i + 1; j < files.Length; j++)
-                    {
-                        string? path2 = files[j];
-                        if (path2 is null)
-                        {
-                            continue; // already linked.
-                        }
-
-                        // note: There's no public API we can use to see if paths are already linked.
-                        //       We treat those paths as unlinked files, and link them again.
-                        if (FilesHaveSameContent(path1, path2))
-                        {
-                            ReplaceByLink(path1, path2);
-
-                            files[j] = null;
-                        }
-                    }
                 }
             }
 
@@ -138,34 +128,30 @@ namespace Microsoft.DotNet.Build.Tasks
             }
         }
 
-        void ReplaceByLink(string path1, string path2)
+        void ReplaceByLinkTo(string path, string pathToTarget)
         {
             // To link, the target mustn't exist. Make a backup, so we can restore it when linking fails.
-            string path2Backup = $"{path2}.pre_link_backup";
+            string backupFile = $"{path}.pre_link_backup";
-            File.Move(path2, path2Backup);
+            File.Move(path, backupFile);
 
-            int rv = SystemNative_Link(path1, path2);
+            try
-            if (rv != 0)
             {
-                var ex = new Win32Exception(); // Captures the LastError.
+                string relativePath = Path.GetRelativePath(Path.GetDirectoryName(path)!, pathToTarget);
+                File.CreateSymbolicLink(path, relativePath);
 
-                Log.LogError($"Unable to link '{path2}' to '{path1}.': {ex}");
+                File.Delete(backupFile);
 
-                File.Move(path2Backup, path2);
+                Log.LogMessage(MessageImportance.Normal, $"Linked '{path}' to '{relativePath}'.");
-
-                throw ex;
             }
-            else
+            catch (Exception ex)
             {
-                File.Delete(path2Backup);
+                Log.LogError($"Unable to link '{path}' to '{pathToTarget}.': {ex}");
 
-                Log.LogMessage(MessageImportance.Normal, $"Linked '{path1}' and '{path2}'.");
+                File.Move(backupFile, path);
+
+                throw;
             }
         }
-
-        // This native method is used by the runtime to create hard links. It is not exposed through a public .NET API.
-        [DllImport("libSystem.Native", SetLastError = true)]
-        static extern int SystemNative_Link(string source, string link);
 #endif
     }
 }

src/redist/targets/BuildCoreSdkTasks.targets

@@ -35,7 +35,7 @@
   <UsingTask TaskName="GetDependencyInfo" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="GetLinuxNativeInstallerDependencyVersions" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="GetRuntimePackRids" AssemblyFile="$(CoreSdkTaskDll)"/>
-  <UsingTask TaskName="ReplaceDuplicateFilesWithHardLinks" AssemblyFile="$(CoreSdkTaskDll)"/>
+  <UsingTask TaskName="ReplaceFilesWithSymbolicLinks" AssemblyFile="$(CoreSdkTaskDll)"/>
   <UsingTask TaskName="ReplaceFileContents" AssemblyFile="$(CoreSdkTaskDll)" />
   <UsingTask TaskName="TarGzFileCreateFromDirectory" AssemblyFile="$(CoreSdkTaskDll)" />
   <UsingTask TaskName="UpdateRuntimeConfig" AssemblyFile="$(CoreSdkTaskDll)" />

src/redist/targets/GenerateLayout.targets

@@ -569,11 +569,12 @@
     <ResolveAssemblyReference AssemblyFiles="@(AssembliesToResolve)" Silent="$(ResolveAssemblyReferencesSilent)" AssemblyInformationCacheOutputPath="$(RedistLayoutPath)sdk\$(Version)\SDKPrecomputedAssemblyReferences.cache" SearchPaths="{RawFileName}" WarnOrErrorOnTargetArchitectureMismatch="$(ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch)" />
   </Target>
 
-  <!-- Replace duplicate files with hard links so that when the same files from a runtime pack
+  <!-- Replace files from the runtime packs with symbolic links to the corresponding shared framework files (and hostfxr) to reduce the size of the runtime pack directories. -->
-       and the corresponding shared frameworks are included in a distro package their data is shared instead of duplicated. -->
+  <Target Name="ReplaceBundledRuntimePackFilesWithSymbolicLinks" DependsOnTargets="LayoutBundledComponents"
-  <Target Name="ReplaceDuplicateFilesWithHardLinks" DependsOnTargets="LayoutBundledComponents"
+                                                                 Condition="'$(BundleRuntimePacks)' == 'true' and  !$([MSBuild]::IsOSPlatform('WINDOWS'))">
-                                                    Condition="'$(BundleRuntimePacks)' == 'true' and  !$([MSBuild]::IsOSPlatform('WINDOWS'))">
+    <ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.NETCore.App/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
-    <ReplaceDuplicateFilesWithHardLinks Directory="$(RedistLayoutPath)" />
+    <ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/host/fxr/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
+    <ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.AspNetCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.AspNetCore.App/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" />
   </Target>
 
   <Target Name="GenerateLayout"
@@ -593,7 +594,7 @@
                             CrossgenLayout;
                             LayoutAppHostTemplate;
                             GeneratePrecomputedRarCache;
-                            ReplaceDuplicateFilesWithHardLinks"
+                            ReplaceBundledRuntimePackFilesWithSymbolicLinks"
           BeforeTargets="AfterBuild">
 
   </Target>

test/SdkTests/TestConfig.xml

@@ -245,6 +245,14 @@
             Skip="true"
             Issue=""
             Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
+    <Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToRunILLink.IsTrimmable_warns_when_expected_for_not_correctly_multitargeted_libraries"
+            Skip="true"
+            Issue=""
+            Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
+    <Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToPublishASingleFileApp.EnableSingleFile_warns_when_expected_for_not_correctly_multitargeted_libraries"
+            Skip="true"
+            Issue=""
+            Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
     <Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToRunILLink.ILLink_can_use_latest_with_unsupported_target_framework"
             Skip="true"
             Issue=""