From 0b8e35cfcba7710d9feb86221e79faadda820c97 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Thu, 26 Oct 2023 08:35:53 -0500 Subject: [PATCH 1/4] [source build] Remove resolved poisoned files from baseline (#17629) --- .../assets/baselines/PoisonUsage.txt | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/PoisonUsage.txt b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/PoisonUsage.txt index fee5ec5e8..f1ab9ecc1 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/PoisonUsage.txt +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/PoisonUsage.txt @@ -1,14 +1 @@ - - - SourceBuildReferenceAssembly - - - SourceBuildReferenceAssembly - - - SourceBuildReferenceAssembly - - - SourceBuildReferenceAssembly - - \ No newline at end of file + \ No newline at end of file From 7e4163a7e5200c2f07c631c3001ad7f53c71b1d0 Mon Sep 17 00:00:00 2001 From: Ella Hathaway <67609881+ellahathaway@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:12:43 -0700 Subject: [PATCH 2/4] Use DecodeValue to check attribute values (#17605) --- .../CheckForPoison.cs | 16 +++++---- .../DummyAttributeTypeProvider.cs | 34 +++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/DummyAttributeTypeProvider.cs diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs index cf16890ce..dab29fae4 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs @@ -152,8 +152,6 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection private const string SbrpAttributeType = "System.Reflection.AssemblyMetadataAttribute"; - private const string SbrpAttributeValuePattern = "source\\s?source\\-build\\-reference\\-packages"; - private record CandidateFileEntry(string ExtractedPath, string DisplayPath); public override bool Execute() @@ -370,11 +368,17 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection if (attributeType == SbrpAttributeType) { - BlobReader blobReader = reader.GetBlobReader(attr.Value); - string attributeValue = Encoding.UTF8.GetString(blobReader.ReadBytes(blobReader.Length)); - attributeValue = Regex.Replace(attributeValue, @"\p{C}+", string.Empty); - return Regex.IsMatch(attributeValue, SbrpAttributeValuePattern); + var decodedValue = attr.DecodeValue(DummyAttributeTypeProvider.Instance); + try + { + return decodedValue.FixedArguments[0].Value.ToString() == "source" && decodedValue.FixedArguments[1].Value.ToString() == "source-build-reference-packages"; + } + catch + { + throw new InvalidOperationException($"{SbrpAttributeType} is not formatted properly with a key, value pair."); + } } + return false; } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/DummyAttributeTypeProvider.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/DummyAttributeTypeProvider.cs new file mode 100644 index 000000000..6f352c88b --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/DummyAttributeTypeProvider.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + + // An empty ICustomAttributeTypeProvider implementation is necessary to read metadata attribute values. + internal class DummyAttributeTypeProvider : ICustomAttributeTypeProvider + { + public static readonly DummyAttributeTypeProvider Instance = new(); + + public Type GetPrimitiveType(PrimitiveTypeCode typeCode) => default(Type); + + public Type GetSystemType() => default(Type); + + public Type GetSZArrayType(Type elementType) => default(Type); + + public Type GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => default(Type); + + public Type GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => default(Type); + + public Type GetTypeFromSerializedName(string name) => default(Type); + + public PrimitiveTypeCode GetUnderlyingEnumType(Type type) => default(PrimitiveTypeCode); + + public bool IsSystemType(Type type) => default(bool); + } +} From 6f3eeae49dcd7496bdb2f3cef30dbff6fbb4eafe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Fri, 27 Oct 2023 17:56:49 +0200 Subject: [PATCH 3/4] Bump `darc` to fix package version updates (#17639) --- .config/dotnet-tools.json | 2 +- eng/Version.Details.xml | 8 +-- eng/Versions.props | 2 +- eng/vmr-sync.sh | 4 +- .../Tasks/GitFileManagerFactory.cs | 52 ------------------- .../Tasks/VirtualMonoRepo_Initialize.cs | 5 +- 6 files changed, 11 insertions(+), 62 deletions(-) delete mode 100644 src/VirtualMonoRepo/Tasks/GitFileManagerFactory.cs diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 2b7fecfce..3b88bcbde 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "microsoft.dotnet.darc": { - "version": "1.1.0-beta.23416.3", + "version": "1.1.0-beta.23527.3", "commands": [ "darc" ] diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 783b33db9..a7997a96e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -222,13 +222,13 @@ https://github.com/dotnet/arcade 39042b4048580366d35a7c1c4f4ce8fc0dbea4b4 - + https://github.com/dotnet/arcade-services - 5d63a226d022fda90cae2c239e882ad253baa758 + 6e08a501d23d7ca53cfe10d74d0a002f75040a4d - + https://github.com/dotnet/arcade-services - 5d63a226d022fda90cae2c239e882ad253baa758 + 6e08a501d23d7ca53cfe10d74d0a002f75040a4d https://github.com/dotnet/runtime diff --git a/eng/Versions.props b/eng/Versions.props index f08343dc9..7a2af12f7 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -44,7 +44,7 @@ - 1.1.0-beta.23416.3 + 1.1.0-beta.23527.3 diff --git a/eng/vmr-sync.sh b/eng/vmr-sync.sh index a753efdd8..db0f704e2 100755 --- a/eng/vmr-sync.sh +++ b/eng/vmr-sync.sh @@ -135,7 +135,7 @@ while [[ $# -gt 0 ]]; do recursive=true ;; --remote) - additional_remotes="$additional_remotes,$2" + additional_remotes="$additional_remotes $2" shift ;; --readme-template) @@ -265,9 +265,9 @@ fi $azdev_pat \ --$verbosity \ $recursive_arg \ + $additional_remotes \ --readme-template "$readme_template" \ --tpn-template "$tpn_template" \ - $additional_remotes \ "$repository" if [[ $? == 0 ]]; then diff --git a/src/VirtualMonoRepo/Tasks/GitFileManagerFactory.cs b/src/VirtualMonoRepo/Tasks/GitFileManagerFactory.cs deleted file mode 100644 index 2da3d540b..000000000 --- a/src/VirtualMonoRepo/Tasks/GitFileManagerFactory.cs +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using Microsoft.DotNet.DarcLib; -using Microsoft.DotNet.DarcLib.Helpers; -using Microsoft.DotNet.DarcLib.VirtualMonoRepo; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DotNet.VirtualMonoRepo.Tasks; - -public class GitFileManagerFactory : IGitFileManagerFactory -{ - private readonly IVmrInfo _vmrInfo; - private readonly VmrRemoteConfiguration _remoteConfiguration; - private readonly IProcessManager _processManager; - private readonly IVersionDetailsParser _versionDetailsParser; - private readonly ILoggerFactory _loggerFactory; - - public GitFileManagerFactory( - IVmrInfo vmrInfo, - VmrRemoteConfiguration remoteConfiguration, - IProcessManager processManager, - IVersionDetailsParser versionDetailsParser, - ILoggerFactory loggerFactory) - { - _vmrInfo = vmrInfo; - _remoteConfiguration = remoteConfiguration; - _processManager = processManager; - _versionDetailsParser = versionDetailsParser; - _loggerFactory = loggerFactory; - } - - public IGitFileManager Create(string repoUri) - => new GitFileManager(CreateGitRepo(repoUri), _versionDetailsParser, _loggerFactory.CreateLogger()); - - private IGitRepo CreateGitRepo(string repoUri) => GitRepoTypeParser.ParseFromUri(repoUri) switch - { - GitRepoType.AzureDevOps => throw new Exception("VMR initialization should not require Azure DevOps repositories"), - - GitRepoType.GitHub => new GitHubClient( - _processManager.GitExecutable, - _remoteConfiguration.GitHubToken, - _loggerFactory.CreateLogger(), - _vmrInfo.TmpPath, - // Caching not in use for Darc local client. - null), - - GitRepoType.Local => new LocalGitClient(_processManager.GitExecutable, _loggerFactory.CreateLogger()), - _ => throw new ArgumentException("Unknown git repository type", nameof(repoUri)), - }; -} diff --git a/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs b/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs index eae76d532..2a8c1d726 100644 --- a/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs +++ b/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs @@ -74,6 +74,8 @@ public class VirtualMonoRepo_Initialize : Build.Utilities.Task, ICancelableTask additionalRemotes, ReadmeTemplatePath, TpnTemplatePath, + generateCodeowners: false, + discardPatches: true, _cancellationToken.Token); return true; } @@ -82,7 +84,6 @@ public class VirtualMonoRepo_Initialize : Build.Utilities.Task, ICancelableTask private IServiceProvider CreateServiceProvider() => new ServiceCollection() .AddLogging(b => b.AddConsole().AddFilter(l => l >= LogLevel.Information)) - .AddTransient() - .AddVmrManagers(sp => sp.GetRequiredService(), "git", VmrPath, TmpPath, null, null) + .AddVmrManagers("git", VmrPath, TmpPath, null, null) .BuildServiceProvider(); } From d6d8155f4fdb548222e438e73a7fd93813b8af11 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Fri, 27 Oct 2023 17:11:48 -0500 Subject: [PATCH 4/4] Revert "Bump `darc` to fix package version updates" (#17644) --- .config/dotnet-tools.json | 2 +- eng/Version.Details.xml | 8 +-- eng/Versions.props | 2 +- eng/vmr-sync.sh | 4 +- .../Tasks/GitFileManagerFactory.cs | 52 +++++++++++++++++++ .../Tasks/VirtualMonoRepo_Initialize.cs | 5 +- 6 files changed, 62 insertions(+), 11 deletions(-) create mode 100644 src/VirtualMonoRepo/Tasks/GitFileManagerFactory.cs diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 3b88bcbde..2b7fecfce 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "microsoft.dotnet.darc": { - "version": "1.1.0-beta.23527.3", + "version": "1.1.0-beta.23416.3", "commands": [ "darc" ] diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a7997a96e..783b33db9 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -222,13 +222,13 @@ https://github.com/dotnet/arcade 39042b4048580366d35a7c1c4f4ce8fc0dbea4b4 - + https://github.com/dotnet/arcade-services - 6e08a501d23d7ca53cfe10d74d0a002f75040a4d + 5d63a226d022fda90cae2c239e882ad253baa758 - + https://github.com/dotnet/arcade-services - 6e08a501d23d7ca53cfe10d74d0a002f75040a4d + 5d63a226d022fda90cae2c239e882ad253baa758 https://github.com/dotnet/runtime diff --git a/eng/Versions.props b/eng/Versions.props index 7a2af12f7..f08343dc9 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -44,7 +44,7 @@ - 1.1.0-beta.23527.3 + 1.1.0-beta.23416.3 diff --git a/eng/vmr-sync.sh b/eng/vmr-sync.sh index db0f704e2..a753efdd8 100755 --- a/eng/vmr-sync.sh +++ b/eng/vmr-sync.sh @@ -135,7 +135,7 @@ while [[ $# -gt 0 ]]; do recursive=true ;; --remote) - additional_remotes="$additional_remotes $2" + additional_remotes="$additional_remotes,$2" shift ;; --readme-template) @@ -265,9 +265,9 @@ fi $azdev_pat \ --$verbosity \ $recursive_arg \ - $additional_remotes \ --readme-template "$readme_template" \ --tpn-template "$tpn_template" \ + $additional_remotes \ "$repository" if [[ $? == 0 ]]; then diff --git a/src/VirtualMonoRepo/Tasks/GitFileManagerFactory.cs b/src/VirtualMonoRepo/Tasks/GitFileManagerFactory.cs new file mode 100644 index 000000000..2da3d540b --- /dev/null +++ b/src/VirtualMonoRepo/Tasks/GitFileManagerFactory.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DotNet.VirtualMonoRepo.Tasks; + +public class GitFileManagerFactory : IGitFileManagerFactory +{ + private readonly IVmrInfo _vmrInfo; + private readonly VmrRemoteConfiguration _remoteConfiguration; + private readonly IProcessManager _processManager; + private readonly IVersionDetailsParser _versionDetailsParser; + private readonly ILoggerFactory _loggerFactory; + + public GitFileManagerFactory( + IVmrInfo vmrInfo, + VmrRemoteConfiguration remoteConfiguration, + IProcessManager processManager, + IVersionDetailsParser versionDetailsParser, + ILoggerFactory loggerFactory) + { + _vmrInfo = vmrInfo; + _remoteConfiguration = remoteConfiguration; + _processManager = processManager; + _versionDetailsParser = versionDetailsParser; + _loggerFactory = loggerFactory; + } + + public IGitFileManager Create(string repoUri) + => new GitFileManager(CreateGitRepo(repoUri), _versionDetailsParser, _loggerFactory.CreateLogger()); + + private IGitRepo CreateGitRepo(string repoUri) => GitRepoTypeParser.ParseFromUri(repoUri) switch + { + GitRepoType.AzureDevOps => throw new Exception("VMR initialization should not require Azure DevOps repositories"), + + GitRepoType.GitHub => new GitHubClient( + _processManager.GitExecutable, + _remoteConfiguration.GitHubToken, + _loggerFactory.CreateLogger(), + _vmrInfo.TmpPath, + // Caching not in use for Darc local client. + null), + + GitRepoType.Local => new LocalGitClient(_processManager.GitExecutable, _loggerFactory.CreateLogger()), + _ => throw new ArgumentException("Unknown git repository type", nameof(repoUri)), + }; +} diff --git a/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs b/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs index 2a8c1d726..eae76d532 100644 --- a/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs +++ b/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs @@ -74,8 +74,6 @@ public class VirtualMonoRepo_Initialize : Build.Utilities.Task, ICancelableTask additionalRemotes, ReadmeTemplatePath, TpnTemplatePath, - generateCodeowners: false, - discardPatches: true, _cancellationToken.Token); return true; } @@ -84,6 +82,7 @@ public class VirtualMonoRepo_Initialize : Build.Utilities.Task, ICancelableTask private IServiceProvider CreateServiceProvider() => new ServiceCollection() .AddLogging(b => b.AddConsole().AddFilter(l => l >= LogLevel.Information)) - .AddVmrManagers("git", VmrPath, TmpPath, null, null) + .AddTransient() + .AddVmrManagers(sp => sp.GetRequiredService(), "git", VmrPath, TmpPath, null, null) .BuildServiceProvider(); }