Merge branch 'release/8.0.2xx'
This commit is contained in:
Jason Zhai
commit
2084a36452
21 changed files with 229 additions and 157 deletions
|
|
@ -9,11 +9,11 @@
|
|||
Basically: In this file, choose the highest version when resolving merge conflicts.
|
||||
-->
|
||||
<PropertyGroup>
|
||||
<MicrosoftWindowsSDKNETRef10_0_17763PackageVersion>10.0.17763.29</MicrosoftWindowsSDKNETRef10_0_17763PackageVersion>
|
||||
<MicrosoftWindowsSDKNETRef10_0_18362PackageVersion>10.0.18362.29</MicrosoftWindowsSDKNETRef10_0_18362PackageVersion>
|
||||
<MicrosoftWindowsSDKNETRef10_0_19041PackageVersion>10.0.19041.29</MicrosoftWindowsSDKNETRef10_0_19041PackageVersion>
|
||||
<MicrosoftWindowsSDKNETRef10_0_20348PackageVersion>10.0.20348.29</MicrosoftWindowsSDKNETRef10_0_20348PackageVersion>
|
||||
<MicrosoftWindowsSDKNETRef10_0_22000PackageVersion>10.0.22000.29</MicrosoftWindowsSDKNETRef10_0_22000PackageVersion>
|
||||
<MicrosoftWindowsSDKNETRef10_0_22621PackageVersion>10.0.22621.29</MicrosoftWindowsSDKNETRef10_0_22621PackageVersion>
|
||||
<MicrosoftWindowsSDKNETRef10_0_17763PackageVersion>10.0.17763.31</MicrosoftWindowsSDKNETRef10_0_17763PackageVersion>
|
||||
<MicrosoftWindowsSDKNETRef10_0_18362PackageVersion>10.0.18362.31</MicrosoftWindowsSDKNETRef10_0_18362PackageVersion>
|
||||
<MicrosoftWindowsSDKNETRef10_0_19041PackageVersion>10.0.19041.31</MicrosoftWindowsSDKNETRef10_0_19041PackageVersion>
|
||||
<MicrosoftWindowsSDKNETRef10_0_20348PackageVersion>10.0.20348.31</MicrosoftWindowsSDKNETRef10_0_20348PackageVersion>
|
||||
<MicrosoftWindowsSDKNETRef10_0_22000PackageVersion>10.0.22000.31</MicrosoftWindowsSDKNETRef10_0_22000PackageVersion>
|
||||
<MicrosoftWindowsSDKNETRef10_0_22621PackageVersion>10.0.22621.31</MicrosoftWindowsSDKNETRef10_0_22621PackageVersion>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -127,7 +127,7 @@ jobs:
|
|||
artifact: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts
|
||||
patterns: |
|
||||
**/Private.SourceBuilt.Artifacts.*.tar.gz
|
||||
**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])*.tar.gz
|
||||
**/dotnet-sdk-*.tar.gz
|
||||
displayName: Download Previous Build
|
||||
|
||||
- task: CopyFiles@2
|
||||
|
|
|
|||
|
|
@ -6,6 +6,10 @@ trigger:
|
|||
include:
|
||||
- main
|
||||
- release/*
|
||||
exclude:
|
||||
- release/*.0.2xx
|
||||
- release/*.0.3xx
|
||||
- release/*.0.4xx
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@
|
|||
DiscoverSymbolsTarballs;
|
||||
ExtractSymbolsTarballs">
|
||||
<PropertyGroup>
|
||||
<UnifiedSymbolsTarball>$(OutputPath)dotnet-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</UnifiedSymbolsTarball>
|
||||
<UnifiedSymbolsTarball>$(OutputPath)dotnet-symbols-all-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</UnifiedSymbolsTarball>
|
||||
</PropertyGroup>
|
||||
|
||||
<Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
|
||||
|
|
@ -109,13 +109,12 @@
|
|||
AfterTargets="Build"
|
||||
DependsOnTargets="RepackageSymbols">
|
||||
<ItemGroup>
|
||||
<SdkTarballItem Include="$(OutputPath)dotnet-sdk-*$(TarBallExtension)"
|
||||
Exclude="$(OutputPath)dotnet-sdk-symbols-*$(TarBallExtension)" />
|
||||
<SdkTarballItem Include="$(OutputPath)dotnet-sdk-*$(TarBallExtension)" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<SdkSymbolsLayout>$(ArtifactsTmpDir)SdkSymbols</SdkSymbolsLayout>
|
||||
<SdkSymbolsTarball>$(OutputPath)dotnet-sdk-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</SdkSymbolsTarball>
|
||||
<SdkSymbolsTarball>$(OutputPath)dotnet-symbols-sdk-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</SdkSymbolsTarball>
|
||||
<SdkLayout>$(ArtifactsTmpDir)Sdk</SdkLayout>
|
||||
<SdkTarball>%(SdkTarballItem.Identity)</SdkTarball>
|
||||
</PropertyGroup>
|
||||
|
|
@ -187,8 +186,7 @@
|
|||
|
||||
<Target Name="RunSmokeTest">
|
||||
<ItemGroup>
|
||||
<SdkTarballItem Include="$(SourceBuiltTarBallPath)**/dotnet-sdk*$(TarBallExtension)"
|
||||
Exclude="$(SourceBuiltTarBallPath)**/dotnet-sdk-symbols*$(TarBallExtension)" />
|
||||
<SdkTarballItem Include="$(SourceBuiltTarBallPath)**/dotnet-sdk*$(TarBallExtension)" />
|
||||
<SourceBuiltArtifactsItem Include="$(SourceBuiltTarBallPath)**/Private.SourceBuilt.Artifacts.*$(TarBallExtension)" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@
|
|||
These URLs can't be composed from their base URL and version as we read them from the
|
||||
prep.sh and pipeline scripts, outside of MSBuild.
|
||||
-->
|
||||
<PrivateSourceBuiltArtifactsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.8.0.100-rc.1.23455.1.centos.8-x64.tar.gz</PrivateSourceBuiltArtifactsUrl>
|
||||
<PrivateSourceBuiltSdkUrl_CentOS8Stream>https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-rc.1.23455.1-centos.8-x64.tar.gz</PrivateSourceBuiltSdkUrl_CentOS8Stream>
|
||||
<PrivateSourceBuiltArtifactsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.8.0.100-rc.2.23502.1.centos.8-x64.tar.gz</PrivateSourceBuiltArtifactsUrl>
|
||||
<PrivateSourceBuiltSdkUrl_CentOS8Stream>https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-rc.2.23502.1-centos.8-x64.tar.gz</PrivateSourceBuiltSdkUrl_CentOS8Stream>
|
||||
<PrivateSourceBuiltPrebuiltsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Prebuilts.0.1.0-9.0.100-3.centos.8-x64.tar.gz</PrivateSourceBuiltPrebuiltsUrl>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -4,8 +4,20 @@
|
|||
was not set correctly. This is the actual shipping version that it should have been -->
|
||||
<NonshippingRuntimeVersionFor700>7.0.4-servicing.23107.6</NonshippingRuntimeVersionFor700>
|
||||
|
||||
<MicrosoftNETHostModelVersion>$(NonshippingRuntimeVersionFor700)</MicrosoftNETHostModelVersion>
|
||||
<MicrosoftNETCoreTestHostVersion>$(NonshippingRuntimeVersionFor700)</MicrosoftNETCoreTestHostVersion>
|
||||
<RuntimeLinuxX64MicrosoftNETCoreTestHostVersion>$(NonshippingRuntimeVersionFor700)</RuntimeLinuxX64MicrosoftNETCoreTestHostVersion>
|
||||
|
||||
<Msft80RC2RuntimeVersion>8.0.0-rc.2.23479.6</Msft80RC2RuntimeVersion>
|
||||
<MicrosoftNETCoreAppCrossgen2Version>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreAppCrossgen2Version>
|
||||
<MicrosoftNETCoreAppHostPackageVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreAppHostPackageVersion>
|
||||
<MicrosoftNETCoreAppRuntimeVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreAppRuntimeVersion>
|
||||
<MicrosoftNETILLinkTasksVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETILLinkTasksVersion>
|
||||
<MicrosoftDotNetILCompilerVersion>$(Msft80RC2RuntimeVersion)</MicrosoftDotNetILCompilerVersion>
|
||||
<MicrosoftNETCoreDotNetAppHostVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreDotNetAppHostVersion>
|
||||
<MicrosoftNETCoreDotNetHostVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreDotNetHostVersion>
|
||||
<MicrosoftNETCoreDotNetHostPolicyVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreDotNetHostPolicyVersion>
|
||||
<MicrosoftNETCoreDotNetHostResolverVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreDotNetHostResolverVersion>
|
||||
<MicrosoftNETCoreILAsmVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreILAsmVersion>
|
||||
<MicrosoftNETCoreILDAsmVersion>$(Msft80RC2RuntimeVersion)</MicrosoftNETCoreILDAsmVersion>
|
||||
<RuntimeNativeSystemIOPortsVersion>$(Msft80RC2RuntimeVersion)</RuntimeNativeSystemIOPortsVersion>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -47,6 +47,8 @@
|
|||
<!-- These packages don't actually exist -->
|
||||
<ExcludedPackage Include="runtime.linux-musl-x64.runtime.native.System.IO.Ports" />
|
||||
<ExcludedPackage Include="runtime.linux-musl-arm64.runtime.native.System.IO.Ports" />
|
||||
|
||||
<PackageDownload Include="Microsoft.NET.ILLink.Tasks" Version="[$(MicrosoftNETILLinkTasksVersion)]" />
|
||||
</ItemGroup>
|
||||
|
||||
<Target Name="GetPackagesToDownload"
|
||||
|
|
|
|||
|
|
@ -12,8 +12,11 @@ using System.IO;
|
|||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Reflection.Metadata;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Xml;
|
||||
using System.Xml.Linq;
|
||||
|
||||
|
|
@ -147,6 +150,10 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
|
|||
|
||||
private const string PoisonMarker = "POISONED";
|
||||
|
||||
private const string SbrpAttributeType = "System.Reflection.AssemblyMetadataAttribute";
|
||||
|
||||
private const string SbrpAttributeValuePattern = "source\\s?source\\-build\\-reference\\-packages";
|
||||
|
||||
private record CandidateFileEntry(string ExtractedPath, string DisplayPath);
|
||||
|
||||
public override bool Execute()
|
||||
|
|
@ -298,7 +305,11 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
|
|||
try
|
||||
{
|
||||
AssemblyName asm = AssemblyName.GetAssemblyName(fileToCheck);
|
||||
if (IsAssemblyPoisoned(fileToCheck))
|
||||
if (!candidate.DisplayPath.Contains("SourceBuildReferencePackages") && IsAssemblyFromSbrp(fileToCheck))
|
||||
{
|
||||
poisonEntry.Type |= PoisonType.SourceBuildReferenceAssembly;
|
||||
}
|
||||
else if (IsAssemblyPoisoned(fileToCheck))
|
||||
{
|
||||
poisonEntry.Type |= PoisonType.AssemblyAttribute;
|
||||
}
|
||||
|
|
@ -332,6 +343,41 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
|
|||
return false;
|
||||
}
|
||||
|
||||
private static bool IsAssemblyFromSbrp(string assemblyPath)
|
||||
{
|
||||
using var stream = new FileStream(assemblyPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
|
||||
using var peReader = new PEReader(stream);
|
||||
|
||||
MetadataReader reader = peReader.GetMetadataReader();
|
||||
return reader.CustomAttributes.Select(attrHandle => reader.GetCustomAttribute(attrHandle))
|
||||
.Any(attr => IsAttributeSbrp(reader, attr));
|
||||
}
|
||||
|
||||
private static bool IsAttributeSbrp(MetadataReader reader, CustomAttribute attr)
|
||||
{
|
||||
string attributeType = string.Empty;
|
||||
|
||||
if (attr.Constructor.Kind == HandleKind.MemberReference)
|
||||
{
|
||||
MemberReference mref = reader.GetMemberReference((MemberReferenceHandle)attr.Constructor);
|
||||
|
||||
if (mref.Parent.Kind == HandleKind.TypeReference)
|
||||
{
|
||||
TypeReference tref = reader.GetTypeReference((TypeReferenceHandle)mref.Parent);
|
||||
attributeType = $"{reader.GetString(tref.Namespace)}.{reader.GetString(tref.Name)}";
|
||||
}
|
||||
}
|
||||
|
||||
if (attributeType == SbrpAttributeType)
|
||||
{
|
||||
BlobReader blobReader = reader.GetBlobReader(attr.Value);
|
||||
string attributeValue = Encoding.UTF8.GetString(blobReader.ReadBytes(blobReader.Length));
|
||||
attributeValue = Regex.Replace(attributeValue, @"\p{C}+", string.Empty);
|
||||
return Regex.IsMatch(attributeValue, SbrpAttributeValuePattern);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue<CandidateFileEntry> futureFilesToCheck)
|
||||
{
|
||||
var poisonEntry = new PoisonedFileEntry();
|
||||
|
|
|
|||
|
|
@ -155,6 +155,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
|
|||
}
|
||||
File.Delete(p.ItemSpec);
|
||||
File.Move(poisonedPackagePath, p.ItemSpec);
|
||||
Directory.Delete(packageTempPath, true);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,5 +11,6 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
|
|||
Hash = 1,
|
||||
AssemblyAttribute = 2,
|
||||
NupkgFile = 4,
|
||||
SourceBuildReferenceAssembly = 8,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
|
||||
private void LogErrorOrWarning(bool isError, string message)
|
||||
{
|
||||
if (FailOnMissingPDBs)
|
||||
if (isError)
|
||||
{
|
||||
Log.LogError(message);
|
||||
}
|
||||
|
|
@ -104,14 +104,15 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
|
||||
if (guid != string.Empty)
|
||||
{
|
||||
if (!allPdbGuids.ContainsKey(guid))
|
||||
string debugId = GetDebugId(guid, file);
|
||||
if (!allPdbGuids.ContainsKey(debugId))
|
||||
{
|
||||
filesWithoutPDBs.Add(file.Substring(SdkLayoutPath.Length + 1));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Copy matching pdb to symbols path, preserving sdk binary's hierarchy
|
||||
string sourcePath = (string)allPdbGuids[guid]!;
|
||||
string sourcePath = (string)allPdbGuids[debugId]!;
|
||||
string destinationPath =
|
||||
file.Replace(SdkLayoutPath, SdkSymbolsLayoutPath)
|
||||
.Replace(Path.GetFileName(file), Path.GetFileName(sourcePath));
|
||||
|
|
@ -142,13 +143,21 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
|
||||
var id = new BlobContentId(metadataReader.DebugMetadataHeader.Id);
|
||||
string guid = $"{id.Guid:N}";
|
||||
if (!string.IsNullOrEmpty(guid) && !allPdbGuids.ContainsKey(guid))
|
||||
string debugId = GetDebugId(guid, file);
|
||||
if (!string.IsNullOrEmpty(guid) && !allPdbGuids.ContainsKey(debugId))
|
||||
{
|
||||
allPdbGuids.Add(guid, file);
|
||||
allPdbGuids.Add(debugId, file);
|
||||
}
|
||||
}
|
||||
|
||||
return allPdbGuids;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates a debug Id from debug guid and filename. We use this as a key
|
||||
/// in PDB hashtable. Guid is not enough due to collisions in several PDBs.
|
||||
/// </summary>
|
||||
private string GetDebugId(string guid, string file) =>
|
||||
$"{guid}.{Path.GetFileNameWithoutExtension(file)}".ToLower();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,12 +23,6 @@ public class DotNetFormatTests : SdkTests
|
|||
// [Fact]
|
||||
public void FormatProject()
|
||||
{
|
||||
if (Config.TargetRid.Contains("alpine"))
|
||||
{
|
||||
// Skipping this test on Alpine due to https://github.com/dotnet/format/issues/1945
|
||||
return;
|
||||
}
|
||||
|
||||
string unformattedCsFilePath = Path.Combine(BaselineHelper.GetAssetsDirectory(), UnformattedFileName);
|
||||
|
||||
string projectDirectory = DotNetHelper.ExecuteNew("console", nameof(FormatProject), "C#");
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ public class LicenseScanTests : TestBase
|
|||
"lgpl-2.0-plus", // https://opensource.org/license/lgpl-2-0/
|
||||
"lgpl-2.1", // https://opensource.org/license/lgpl-2-1/
|
||||
"lgpl-2.1-plus", // https://opensource.org/license/lgpl-2-1/
|
||||
"lzma-sdk-9.22", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/lzma-sdk-9.22.LICENSE
|
||||
"mit", // https://opensource.org/license/mit/
|
||||
"mit-addition", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/mit-addition.LICENSE
|
||||
"ms-patent-promise", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/ms-patent-promise.LICENSE
|
||||
|
|
@ -142,17 +143,16 @@ public class LicenseScanTests : TestBase
|
|||
{
|
||||
Assert.NotNull(Config.LicenseScanPath);
|
||||
|
||||
string OriginalScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-original.json");
|
||||
string FilteredScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-filtered.json");
|
||||
string scancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results.json");
|
||||
|
||||
// Scancode Doc: https://scancode-toolkit.readthedocs.io/en/latest/index.html
|
||||
string ignoreOptions = string.Join(" ", s_ignoredFilePatterns.Select(pattern => $"--ignore {pattern}"));
|
||||
ExecuteHelper.ExecuteProcessValidateExitCode(
|
||||
"scancode",
|
||||
$"--license --strip-root --only-findings {ignoreOptions} --json-pp {OriginalScancodeResultsPath} {Config.LicenseScanPath}",
|
||||
$"--license --strip-root --only-findings {ignoreOptions} --json-pp {scancodeResultsPath} {Config.LicenseScanPath}",
|
||||
OutputHelper);
|
||||
|
||||
JsonDocument doc = JsonDocument.Parse(File.ReadAllText(OriginalScancodeResultsPath));
|
||||
JsonDocument doc = JsonDocument.Parse(File.ReadAllText(scancodeResultsPath));
|
||||
ScancodeResults? scancodeResults = doc.Deserialize<ScancodeResults>();
|
||||
Assert.NotNull(scancodeResults);
|
||||
|
||||
|
|
@ -163,7 +163,6 @@ public class LicenseScanTests : TestBase
|
|||
WriteIndented = true
|
||||
};
|
||||
string json = JsonSerializer.Serialize(scancodeResults, options);
|
||||
File.WriteAllText(FilteredScancodeResultsPath, json);
|
||||
|
||||
string baselineName = $"Licenses.{_targetRepo}.json";
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,15 @@ using Xunit.Abstractions;
|
|||
|
||||
namespace Microsoft.DotNet.SourceBuild.SmokeTests;
|
||||
|
||||
/// <summary>
|
||||
/// Separate test collection for Sourcelink tests. This is needed due to intra-test parallelization,
|
||||
/// which can cause less CPU time to be allocated to other tests.
|
||||
/// This would make other tests run too long and fail due to timeouts.
|
||||
/// </summary>
|
||||
[CollectionDefinition(nameof(SourcelinkTestCollection), DisableParallelization = true)]
|
||||
public class SourcelinkTestCollection { }
|
||||
|
||||
[Collection(nameof(SourcelinkTestCollection))]
|
||||
public class SourcelinkTests : SdkTests
|
||||
{
|
||||
private static string SourcelinkRoot { get; } = Path.Combine(Directory.GetCurrentDirectory(), nameof(SourcelinkTests));
|
||||
|
|
@ -26,6 +35,8 @@ public class SourcelinkTests : SdkTests
|
|||
// https://github.com/dotnet/source-build/issues/3668
|
||||
// [Fact]
|
||||
public void VerifySourcelinks()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(SourcelinkRoot))
|
||||
{
|
||||
|
|
@ -33,7 +44,16 @@ public class SourcelinkTests : SdkTests
|
|||
}
|
||||
Directory.CreateDirectory(SourcelinkRoot);
|
||||
|
||||
IList<string> failedFiles = ValidateSymbols(ExtractSymbolsPackages(GetAllSymbolsPackages()), InitializeSourcelinkTool());
|
||||
string symbolsRoot = Directory.CreateDirectory(Path.Combine(SourcelinkRoot, "symbols")).FullName;
|
||||
|
||||
// We are validating dotnet-symbols-all-*.tar.gz which contains all source-built symbols, including
|
||||
// SDK-specific symbols that are also packaged in dotnet-symbols-sdk-*.tar.gz.
|
||||
Utilities.ExtractTarball(
|
||||
Utilities.GetFile(Path.GetDirectoryName(Config.SourceBuiltArtifactsPath), "dotnet-symbols-all-*.tar.gz"),
|
||||
symbolsRoot,
|
||||
OutputHelper);
|
||||
|
||||
IList<string> failedFiles = ValidateSymbols(symbolsRoot, InitializeSourcelinkTool());
|
||||
|
||||
if (failedFiles.Count > 0)
|
||||
{
|
||||
|
|
@ -46,6 +66,11 @@ public class SourcelinkTests : SdkTests
|
|||
|
||||
Assert.True(failedFiles.Count == 0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(SourcelinkRoot, true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes sourcelink tool.
|
||||
|
|
@ -54,6 +79,8 @@ public class SourcelinkTests : SdkTests
|
|||
/// <returns>Path to sourcelink tool binary.</returns>
|
||||
private string InitializeSourcelinkTool()
|
||||
{
|
||||
Assert.NotNull(Config.SourceBuiltArtifactsPath);
|
||||
|
||||
const string SourcelinkToolPackageNamePattern = "dotnet-sourcelink*nupkg";
|
||||
const string SourcelinkToolBinaryFilename = "dotnet-sourcelink.dll";
|
||||
|
||||
|
|
@ -66,38 +93,6 @@ public class SourcelinkTests : SdkTests
|
|||
return Utilities.GetFile(extractedToolPath, SourcelinkToolBinaryFilename);
|
||||
}
|
||||
|
||||
private IEnumerable<string> GetAllSymbolsPackages()
|
||||
{
|
||||
/*
|
||||
At the moment we validate sourcelinks from runtime symbols package.
|
||||
The plan is to make symbols, from all repos, available in source-build artifacts.
|
||||
Once that's available, this code will be modified to validate all available symbols.
|
||||
Tracking issue: https://github.com/dotnet/source-build/issues/3612
|
||||
*/
|
||||
|
||||
// Runtime symbols package lives in the same directory as PSB artifacts.
|
||||
// i.e. <repo-root>/artifacts/x64/Release/runtime/dotnet-runtime-symbols-fedora.36-x64-8.0.0-preview.7.23355.7.tar.gz
|
||||
yield return Utilities.GetFile(Path.GetDirectoryName(Config.SourceBuiltArtifactsPath), "dotnet-runtime-symbols-*.tar.gz");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts symbols packages to subdirectories of the common symbols root directory.
|
||||
/// </summary>
|
||||
/// <returns>Path to common symbols root directory.</returns>
|
||||
private string ExtractSymbolsPackages(IEnumerable<string> packages)
|
||||
{
|
||||
string symbolsRoot = Directory.CreateDirectory(Path.Combine(SourcelinkRoot, "symbols")).FullName;
|
||||
|
||||
foreach (string package in packages)
|
||||
{
|
||||
Assert.True(package.EndsWith(".tar.gz"), $"Package extension is not supported: {package}");
|
||||
DirectoryInfo targetDirInfo = Directory.CreateDirectory(Path.Combine(symbolsRoot, Path.GetFileNameWithoutExtension(package)));
|
||||
Utilities.ExtractTarball(package, targetDirInfo.FullName, OutputHelper);
|
||||
}
|
||||
|
||||
return symbolsRoot;
|
||||
}
|
||||
|
||||
private IList<string> ValidateSymbols(string path, string sourcelinkToolPath)
|
||||
{
|
||||
Assert.True(Directory.Exists(path), $"Path, with symbol files to validate, does not exist: {path}");
|
||||
|
|
@ -113,7 +108,7 @@ public class SourcelinkTests : SdkTests
|
|||
OutputHelper,
|
||||
logOutput: false,
|
||||
excludeInfo: true, // Exclude info messages, as there can be 1,000+ processes
|
||||
millisecondTimeout: 5000,
|
||||
millisecondTimeout: 60000,
|
||||
configureCallback: (process) => DotNetHelper.ConfigureProcess(process, null));
|
||||
|
||||
if (executeResult.Process.ExitCode != 0)
|
||||
|
|
|
|||
|
|
@ -28,8 +28,9 @@ src/arcade/src/Microsoft.DotNet.Build.Tasks.Installers/build/wix/eula.rtf
|
|||
# aspnetcore
|
||||
#
|
||||
|
||||
# Line 1 is a generic statement about license applicability that is being detected as "unknown"
|
||||
# A generic statement about license applicability that is being detected as "unknown"
|
||||
src/aspnetcore/src/Components/THIRD-PARTY-NOTICES.txt|unknown
|
||||
src/aspnetcore/THIRD-PARTY-NOTICES.txt|unknown
|
||||
|
||||
# Windows installer files that have a reference to a URL for license
|
||||
src/aspnetcore/src/Installers/Windows/**/*.wxl|unknown-license-reference
|
||||
|
|
@ -167,7 +168,7 @@ src/runtime/src/installer/pkg/THIRD-PARTY-NOTICES.TXT
|
|||
|
||||
# False positive
|
||||
src/runtime/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/LicFileLicenseProvider.cs|proprietary-license
|
||||
src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|json
|
||||
src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|embedthis-extension
|
||||
src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/Connect.cs|other-permissive
|
||||
src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/UdpClientTest.cs|other-permissive
|
||||
src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs|other-permissive
|
||||
|
|
@ -226,10 +227,11 @@ src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library/2.
|
|||
src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library.ref/2.1.0/THIRD-PARTY-NOTICES.TXT|codesourcery-2004
|
||||
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/ThirdPartyNotices.rtf|json
|
||||
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.netcore.*/1.*/ThirdPartyNotices.txt|unknown
|
||||
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/*/1033/System.Security.Permissions.xml|unknown-license-reference
|
||||
|
||||
# Contains references to licenses which are not applicable to the source
|
||||
src/source-build-reference-packages/src/packageSourceGenerator/PackageSourceGeneratorTask/RewriteNuspec.cs|unknown-license-reference,ms-net-library-2018-11
|
||||
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.0-preview-20230918.1/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license
|
||||
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license
|
||||
|
||||
#
|
||||
# sourcelink
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ index ------------
|
|||
./packs/NETStandard.Library.Ref/x.y.z/ref/netstandard2.1/System.Xml.XPath.XDocument.dll
|
||||
./sdk-manifests/
|
||||
./sdk-manifests/x.y.z/
|
||||
-./sdk-manifests/x.y.z/
|
||||
-./sdk-manifests/x.y.z/
|
||||
./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/
|
||||
./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/x.y.z/
|
||||
|
|
|
|||
|
|
@ -1 +1,14 @@
|
|||
<PrebuiltLeakReport />
|
||||
<PrebuiltLeakReport>
|
||||
<File Path="artifacts/x64/Release/dotnet-sdk-x.y.z-banana-rid.tar.gz/sdk/x.y.z/DotnetTools/dotnet-format/Microsoft.Bcl.AsyncInterfaces.dll">
|
||||
<Type>SourceBuildReferenceAssembly</Type>
|
||||
</File>
|
||||
<File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/dotnet-format.x.y.z.nupkg/tools/netx.y/any/Microsoft.Bcl.AsyncInterfaces.dll">
|
||||
<Type>SourceBuildReferenceAssembly</Type>
|
||||
</File>
|
||||
<File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/Microsoft.TestPlatform.CLI.x.y.z/contentFiles/any/netx.y/Microsoft.Extensions.DependencyModel.dll">
|
||||
<Type>SourceBuildReferenceAssembly</Type>
|
||||
</File>
|
||||
<File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/Microsoft.TestPlatform.CLI.x.y.z/contentFiles/any/netx.y/Microsoft.Extensions.FileSystemGlobbing.dll">
|
||||
<Type>SourceBuildReferenceAssembly</Type>
|
||||
</File>
|
||||
</PrebuiltLeakReport>
|
||||
|
|
@ -22,18 +22,24 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
/// <summary>
|
||||
/// Replaces files that have the same content with hard links.
|
||||
/// </summary>
|
||||
public sealed class ReplaceDuplicateFilesWithHardLinks : Task
|
||||
public sealed class ReplaceFilesWithSymbolicLinks : Task
|
||||
{
|
||||
/// <summary>
|
||||
/// The path to the directory.
|
||||
/// The path to the directory to recursively search for files to replace with symbolic links.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string Directory { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// The path to the directory with files to link to.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string LinkToFilesFrom { get; set; } = "";
|
||||
|
||||
#if NETFRAMEWORK
|
||||
public override bool Execute()
|
||||
{
|
||||
Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on .NET Framework.");
|
||||
Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on .NET Framework.");
|
||||
return false;
|
||||
}
|
||||
#else
|
||||
|
|
@ -41,7 +47,7 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
{
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on Windows.");
|
||||
Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on Windows.");
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -51,10 +57,16 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
return false;
|
||||
}
|
||||
|
||||
if (!System.IO.Directory.Exists(LinkToFilesFrom))
|
||||
{
|
||||
Log.LogError($"'{LinkToFilesFrom}' does not exist.");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Find all non-empty, non-symbolic link files.
|
||||
IEnumerable<FileInfo> fse = new FileSystemEnumerable<FileInfo>(
|
||||
string[] files = new FileSystemEnumerable<string>(
|
||||
Directory,
|
||||
(ref FileSystemEntry entry) => (FileInfo)entry.ToFileSystemInfo(),
|
||||
(ref FileSystemEntry entry) => entry.ToFullPath(),
|
||||
new EnumerationOptions()
|
||||
{
|
||||
AttributesToSkip = FileAttributes.ReparsePoint,
|
||||
|
|
@ -63,40 +75,18 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
{
|
||||
ShouldIncludePredicate = (ref FileSystemEntry entry) => !entry.IsDirectory
|
||||
&& entry.Length > 0
|
||||
};
|
||||
}.ToArray();
|
||||
|
||||
// Group them by file size.
|
||||
IEnumerable<string?[]> filesGroupedBySize = fse.GroupBy(file => file.Length,
|
||||
file => file.FullName,
|
||||
(size, files) => files.ToArray());
|
||||
foreach (var file in files)
|
||||
{
|
||||
string fileName = Path.GetFileName(file);
|
||||
|
||||
// Replace files with same content with hard link.
|
||||
foreach (var files in filesGroupedBySize)
|
||||
// Look for a file with the same name in LinkToFilesFrom
|
||||
// and replace it with a symbolic link if it has the same content.
|
||||
string targetFile = Path.Combine(LinkToFilesFrom, fileName);
|
||||
if (File.Exists(targetFile) && FilesHaveSameContent(file, targetFile))
|
||||
{
|
||||
for (int i = 0; i < files.Length; i++)
|
||||
{
|
||||
string? path1 = files[i];
|
||||
if (path1 is null)
|
||||
{
|
||||
continue; // already linked.
|
||||
}
|
||||
for (int j = i + 1; j < files.Length; j++)
|
||||
{
|
||||
string? path2 = files[j];
|
||||
if (path2 is null)
|
||||
{
|
||||
continue; // already linked.
|
||||
}
|
||||
|
||||
// note: There's no public API we can use to see if paths are already linked.
|
||||
// We treat those paths as unlinked files, and link them again.
|
||||
if (FilesHaveSameContent(path1, path2))
|
||||
{
|
||||
ReplaceByLink(path1, path2);
|
||||
|
||||
files[j] = null;
|
||||
}
|
||||
}
|
||||
ReplaceByLinkTo(file, targetFile);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -138,34 +128,30 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
}
|
||||
}
|
||||
|
||||
void ReplaceByLink(string path1, string path2)
|
||||
void ReplaceByLinkTo(string path, string pathToTarget)
|
||||
{
|
||||
// To link, the target mustn't exist. Make a backup, so we can restore it when linking fails.
|
||||
string path2Backup = $"{path2}.pre_link_backup";
|
||||
File.Move(path2, path2Backup);
|
||||
string backupFile = $"{path}.pre_link_backup";
|
||||
File.Move(path, backupFile);
|
||||
|
||||
int rv = SystemNative_Link(path1, path2);
|
||||
if (rv != 0)
|
||||
try
|
||||
{
|
||||
var ex = new Win32Exception(); // Captures the LastError.
|
||||
string relativePath = Path.GetRelativePath(Path.GetDirectoryName(path)!, pathToTarget);
|
||||
File.CreateSymbolicLink(path, relativePath);
|
||||
|
||||
Log.LogError($"Unable to link '{path2}' to '{path1}.': {ex}");
|
||||
File.Delete(backupFile);
|
||||
|
||||
File.Move(path2Backup, path2);
|
||||
|
||||
throw ex;
|
||||
Log.LogMessage(MessageImportance.Normal, $"Linked '{path}' to '{relativePath}'.");
|
||||
}
|
||||
else
|
||||
catch (Exception ex)
|
||||
{
|
||||
File.Delete(path2Backup);
|
||||
Log.LogError($"Unable to link '{path}' to '{pathToTarget}.': {ex}");
|
||||
|
||||
Log.LogMessage(MessageImportance.Normal, $"Linked '{path1}' and '{path2}'.");
|
||||
File.Move(backupFile, path);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
// This native method is used by the runtime to create hard links. It is not exposed through a public .NET API.
|
||||
[DllImport("libSystem.Native", SetLastError = true)]
|
||||
static extern int SystemNative_Link(string source, string link);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
|
@ -35,7 +35,7 @@
|
|||
<UsingTask TaskName="GetDependencyInfo" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="GetLinuxNativeInstallerDependencyVersions" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="GetRuntimePackRids" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="ReplaceDuplicateFilesWithHardLinks" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="ReplaceFilesWithSymbolicLinks" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="ReplaceFileContents" AssemblyFile="$(CoreSdkTaskDll)" />
|
||||
<UsingTask TaskName="TarGzFileCreateFromDirectory" AssemblyFile="$(CoreSdkTaskDll)" />
|
||||
<UsingTask TaskName="UpdateRuntimeConfig" AssemblyFile="$(CoreSdkTaskDll)" />
|
||||
|
|
|
|||
|
|
@ -569,11 +569,12 @@
|
|||
<ResolveAssemblyReference AssemblyFiles="@(AssembliesToResolve)" Silent="$(ResolveAssemblyReferencesSilent)" AssemblyInformationCacheOutputPath="$(RedistLayoutPath)sdk\$(Version)\SDKPrecomputedAssemblyReferences.cache" SearchPaths="{RawFileName}" WarnOrErrorOnTargetArchitectureMismatch="$(ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch)" />
|
||||
</Target>
|
||||
|
||||
<!-- Replace duplicate files with hard links so that when the same files from a runtime pack
|
||||
and the corresponding shared frameworks are included in a distro package their data is shared instead of duplicated. -->
|
||||
<Target Name="ReplaceDuplicateFilesWithHardLinks" DependsOnTargets="LayoutBundledComponents"
|
||||
<!-- Replace files from the runtime packs with symbolic links to the corresponding shared framework files (and hostfxr) to reduce the size of the runtime pack directories. -->
|
||||
<Target Name="ReplaceBundledRuntimePackFilesWithSymbolicLinks" DependsOnTargets="LayoutBundledComponents"
|
||||
Condition="'$(BundleRuntimePacks)' == 'true' and !$([MSBuild]::IsOSPlatform('WINDOWS'))">
|
||||
<ReplaceDuplicateFilesWithHardLinks Directory="$(RedistLayoutPath)" />
|
||||
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.NETCore.App/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
|
||||
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/host/fxr/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
|
||||
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.AspNetCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.AspNetCore.App/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" />
|
||||
</Target>
|
||||
|
||||
<Target Name="GenerateLayout"
|
||||
|
|
@ -593,7 +594,7 @@
|
|||
CrossgenLayout;
|
||||
LayoutAppHostTemplate;
|
||||
GeneratePrecomputedRarCache;
|
||||
ReplaceDuplicateFilesWithHardLinks"
|
||||
ReplaceBundledRuntimePackFilesWithSymbolicLinks"
|
||||
BeforeTargets="AfterBuild">
|
||||
|
||||
</Target>
|
||||
|
|
|
|||
|
|
@ -245,6 +245,14 @@
|
|||
Skip="true"
|
||||
Issue=""
|
||||
Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
|
||||
<Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToRunILLink.IsTrimmable_warns_when_expected_for_not_correctly_multitargeted_libraries"
|
||||
Skip="true"
|
||||
Issue=""
|
||||
Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
|
||||
<Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToPublishASingleFileApp.EnableSingleFile_warns_when_expected_for_not_correctly_multitargeted_libraries"
|
||||
Skip="true"
|
||||
Issue=""
|
||||
Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
|
||||
<Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToRunILLink.ILLink_can_use_latest_with_unsupported_target_framework"
|
||||
Skip="true"
|
||||
Issue=""
|
||||
|
|
|
|||
Loading…
Reference in a new issue