Merge in 'release/8.0.1xx' changes

This commit is contained in:
dotnet-bot 2024-06-17 17:38:03 +00:00
commit b0a6a3b296
10 changed files with 647 additions and 1 deletions

View file

@ -0,0 +1,13 @@
# Partially copied from https://github.com/dotnet/arcade/blob/dfc6882da43decb37f12e0d9011ce82b25225578/.vault-config/product-builds-dnceng-pipeline-secrets.yaml
secrets:
BotAccount-dotnet-sb-bot:
type: github-account
parameters:
Name: dotnet-sb-bot
BotAccount-dotnet-sb-bot-pat:
type: github-access-token
parameters:
gitHubBotAccountSecret: BotAccount-dotnet-sb-bot
gitHubBotAccountName: dotnet-sb-bot

View file

@ -33,6 +33,9 @@ parameters:
variables: variables:
- template: /src/installer/eng/pipelines/templates/variables/vmr-build.yml@self - template: /src/installer/eng/pipelines/templates/variables/vmr-build.yml@self
# GH access token for SB bot - BotAccount-dotnet-sb-bot-pat
- group: DotNet-Source-Build-Bot-Secrets-MVP
jobs: jobs:
- template: templates/jobs/sdk-diff-tests.yml - template: templates/jobs/sdk-diff-tests.yml
parameters: parameters:
@ -40,6 +43,7 @@ jobs:
targetRid: ${{ variables.centOSStreamX64Rid }} targetRid: ${{ variables.centOSStreamX64Rid }}
architecture: x64 architecture: x64
dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}
publishTestResultsPr: true
- template: templates/jobs/sdk-diff-tests.yml - template: templates/jobs/sdk-diff-tests.yml
parameters: parameters:

View file

@ -11,6 +11,10 @@ parameters:
- name: dotnetDotnetRunId - name: dotnetDotnetRunId
type: string type: string
- name: publishTestResultsPr
type: boolean
default: false
jobs: jobs:
- job: ${{ parameters.buildName }}_${{ parameters.architecture }} - job: ${{ parameters.buildName }}_${{ parameters.architecture }}
timeoutInMinutes: 150 timeoutInMinutes: 150
@ -161,3 +165,12 @@ jobs:
mergeTestResults: true mergeTestResults: true
publishRunAttachments: true publishRunAttachments: true
testRunTitle: $(Agent.JobName) testRunTitle: $(Agent.JobName)
- ${{ if and(eq(parameters.publishTestResultsPr, 'true'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release'))) }}:
- template: ../steps/create-baseline-update-pr.yml
parameters:
pipeline: sdk
repo: dotnet/installer
originalFilesDirectory: src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines
updatedFilesDirectory: $(Build.StagingDirectory)/BuildLogs
pullRequestTitle: Update Source-Build SDK Diff Tests Baselines and Exclusions

View file

@ -0,0 +1,48 @@
parameters:
# The pipeline that is being run
# Used to determine the correct baseline update tool to run
# Currently only supports "sdk" and "license"
- name: pipeline
type: string
# The GitHub repository to create the PR in.
# Should be in the form '<owner>/<repo-name>'
- name: repo
type: string
# Path to the directory containing the original test files
# Should be relative to the "repo" parameter
- name: originalFilesDirectory
type: string
# Path to the directory containing the updated test files
# Should be absolute or relative to the working directory of the tool
- name: updatedFilesDirectory
type: string
- name: pullRequestTitle
type: string
default: Update Test Baselines
steps:
- script: |
restoreSources="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json"
restoreSources+="%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries/nuget/v3/index.json"
branchName=$(echo "$(Build.SourceBranch)" | sed 's/refs\/heads\///g')
.dotnet/dotnet run \
--project eng/tools/CreateBaselineUpdatePR/ \
--property:RestoreSources="$restoreSources" \
"${{ parameters.pipeline }}" \
"${{ parameters.repo }}" \
"${{ parameters.originalFilesDirectory }}" \
"${{ parameters.updatedFilesDirectory }}" \
"$(Build.BuildId)" \
--title "${{ parameters.pullRequestTitle }}" \
--branch "$branchName"
displayName: Publish Test Results PR
workingDirectory: $(Build.SourcesDirectory)
condition: succeededOrFailed()
env:
GH_TOKEN: $(BotAccount-dotnet-sb-bot-pat)

View file

@ -20,7 +20,10 @@ parameters:
default: " " # Set it to an empty string to allow it be an optional parameter default: " " # Set it to an empty string to allow it be an optional parameter
variables: variables:
installerRoot: '$(Build.SourcesDirectory)/src/installer' # GH access token for SB bot - BotAccount-dotnet-sb-bot-pat
- group: DotNet-Source-Build-Bot-Secrets-MVP
- name: installerRoot
value: '$(Build.SourcesDirectory)/src/installer'
jobs: jobs:
- job: Setup - job: Setup
@ -136,3 +139,45 @@ jobs:
mergeTestResults: true mergeTestResults: true
publishRunAttachments: true publishRunAttachments: true
testRunTitle: $(Agent.JobName) testRunTitle: $(Agent.JobName)
- job: CreateBaselineUpdatePR
dependsOn: LicenseScan
condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release'))
pool:
name: NetCore1ESPool-Svc-Internal
demands: ImageOverride -equals 1es-ubuntu-2004
variables:
- template: templates/variables/pipelines.yml
steps:
- script: |
source ./eng/common/tools.sh
InitializeDotNetCli true
displayName: Install .NET SDK
workingDirectory: $(Build.SourcesDirectory)
- task: DownloadPipelineArtifact@2
inputs:
buildType: specific
buildVersionToDownload: specific
project: internal
buildId: $(Build.BuildId)
artifact: ''
patterns: '**/Updated*'
allowPartiallySucceededBuilds: true
allowFailedBuilds: true
downloadPath: $(Pipeline.Workspace)/Artifacts
checkDownloadedFiles: true
displayName: Download Updated Baselines
- script: |
find $(Pipeline.Workspace)/Artifacts -type f -exec mv {} $(Pipeline.Workspace)/Artifacts \;
displayName: Move Artifacts to root
- template: templates/steps/create-baseline-update-pr.yml
parameters:
pipeline: license
repo: dotnet/installer
originalFilesDirectory: src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/licenses
updatedFilesDirectory: $(Pipeline.Workspace)/Artifacts
pullRequestTitle: Update Source-Build License Scan Baselines and Exclusions

View file

@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.24209.3" />
<PackageReference Include="Octokit" Version="10.0.0" />
</ItemGroup>
</Project>

View file

@ -0,0 +1,68 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Extensions.Logging;
namespace CreateBaselineUpdatePR;
public static class Log
{
public static LogLevel Level = LogLevel.Information;
private static bool WarningLogged = false;
private static bool ErrorLogged = false;
private static readonly Lazy<ILogger> _logger = new Lazy<ILogger>(ConfigureLogger);
public static void LogDebug(string message)
{
_logger.Value.LogDebug(message);
}
public static void LogInformation(string message)
{
_logger.Value.LogInformation(message);
}
public static void LogWarning(string message)
{
_logger.Value.LogWarning(message);
WarningLogged = true;
}
public static void LogError(string message)
{
_logger.Value.LogError(message);
ErrorLogged = true;
}
private static ILogger ConfigureLogger()
{
using ILoggerFactory loggerFactory =
LoggerFactory.Create(builder =>
builder.AddSimpleConsole(options =>
{
options.SingleLine = true;
options.TimestampFormat = "HH:mm:ss ";
options.UseUtcTimestamp = true;
})
.SetMinimumLevel(Level));
return loggerFactory.CreateLogger(System.Reflection.Assembly.GetExecutingAssembly().GetName().Name!);
}
public static int GetExitCode()
{
if (ErrorLogged)
{
return 1;
}
if (WarningLogged)
{
return 2;
}
return 0;
}
}

View file

@ -0,0 +1,310 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
namespace CreateBaselineUpdatePR;
using Octokit;
using System.Text;
using System.Threading.Tasks;
using System.Collections.Generic;
public class PRCreator
{
private readonly string _repoOwner;
private readonly string _repoName;
private readonly GitHubClient _client;
private const string BuildLink = "https://dev.azure.com/dnceng/internal/_build/results?buildId=";
private const string TreeMode = "040000";
public PRCreator(string repo, string gitHubToken)
{
// Create a new GitHub client
_client = new GitHubClient(new ProductHeaderValue(System.Reflection.Assembly.GetExecutingAssembly().GetName().Name));
var authToken = new Credentials(gitHubToken);
_client.Credentials = authToken;
_repoOwner = repo.Split('/')[0];
_repoName = repo.Split('/')[1];
}
public async Task<int> ExecuteAsync(
string originalFilesDirectory,
string updatedFilesDirectory,
int buildId,
string title,
string targetBranch,
Pipelines pipeline)
{
DateTime startTime = DateTime.Now.ToUniversalTime();
Log.LogInformation($"Starting PR creation at {startTime} UTC for pipeline {pipeline}.");
var updatedTestsFiles = GetUpdatedFiles(updatedFilesDirectory);
// Create a new tree for the originalFilesDirectory based on the target branch
var originalTreeResponse = await _client.Git.Tree.GetRecursive(_repoOwner, _repoName, targetBranch);
var testResultsTreeItems = originalTreeResponse.Tree
.Where(file => file.Path.Contains(originalFilesDirectory) && file.Path != originalFilesDirectory)
.Select(file => new NewTreeItem
{
Path = Path.GetRelativePath(originalFilesDirectory, file.Path),
Mode = file.Mode,
Type = file.Type.Value,
Sha = file.Sha
})
.ToList();
// Update the test results tree
testResultsTreeItems = await UpdateAllFilesAsync(updatedTestsFiles, testResultsTreeItems);
var testResultsTreeResponse = await CreateTreeFromItemsAsync(testResultsTreeItems);
var parentTreeResponse = await CreateParentTreeAsync(testResultsTreeResponse, originalTreeResponse, originalFilesDirectory);
await CreateOrUpdatePullRequestAsync(parentTreeResponse, buildId, title, targetBranch);
return Log.GetExitCode();
}
// Return a dictionary using the filename without the
// "Updated" prefix and anything after the first '.' as the key
private Dictionary<string, HashSet<string>> GetUpdatedFiles(string updatedFilesDirectory) =>
Directory
.GetFiles(updatedFilesDirectory, "Updated*", SearchOption.AllDirectories)
.GroupBy(updatedTestsFile => ParseUpdatedFileName(updatedTestsFile).Split('.')[0])
.ToDictionary(
group => group.Key,
group => new HashSet<string>(group)
);
private async Task<List<NewTreeItem>> UpdateAllFilesAsync(Dictionary<string, HashSet<string>> updatedFiles, List<NewTreeItem> tree)
{
foreach (var updatedFile in updatedFiles)
{
foreach (var filePath in updatedFile.Value)
{
var content = File.ReadAllText(filePath);
string originalFileName = Path.GetFileName(ParseUpdatedFileName(filePath));
tree = await UpdateFileAsync(tree, content, originalFileName, originalFileName);
}
}
return tree;
}
private async Task<List<NewTreeItem>> UpdateFileAsync(List<NewTreeItem> tree, string content, string searchFileName, string updatedPath)
{
var originalTreeItem = tree
.Where(item => item.Path.Contains(searchFileName))
.FirstOrDefault();
if (originalTreeItem == null)
{
// Path not in the tree, add a new tree item
var blob = await CreateBlobAsync(content);
tree.Add(new NewTreeItem
{
Type = TreeType.Blob,
Mode = FileMode.File,
Path = updatedPath,
Sha = blob.Sha
});
}
else
{
// Path in the tree, update the sha and the content
var blob = await CreateBlobAsync(content);
originalTreeItem.Sha = blob.Sha;
}
return tree;
}
private async Task<BlobReference> CreateBlobAsync(string content)
{
var blob = new NewBlob
{
Content = content,
Encoding = EncodingType.Utf8
};
return await _client.Git.Blob.Create(_repoOwner, _repoName, blob);
}
private string ParseUpdatedFileName(string updatedFile) => updatedFile.Split("Updated")[1];
private async Task<TreeResponse> CreateTreeFromItemsAsync(List<NewTreeItem> items, string path = "")
{
var newTreeItems = new List<NewTreeItem>();
var groups = items.GroupBy(item => Path.GetDirectoryName(item.Path));
foreach (var group in groups)
{
if (string.IsNullOrEmpty(group.Key) || group.Key == path)
{
// These items are in the current directory, so add them to the new tree items
foreach (var item in group)
{
if(item.Type != TreeType.Tree)
{
newTreeItems.Add(new NewTreeItem
{
Path = path == string.Empty ? item.Path : Path.GetRelativePath(path, item.Path),
Mode = item.Mode,
Type = item.Type,
Sha = item.Sha
});
}
}
}
else
{
// These items are in a subdirectory, so recursively create a tree for them
var subtreeResponse = await CreateTreeFromItemsAsync(group.ToList(), group.Key);
newTreeItems.Add(new NewTreeItem
{
Path = group.Key,
Mode = TreeMode,
Type = TreeType.Tree,
Sha = subtreeResponse.Sha
});
}
}
var newTree = new NewTree();
foreach (var item in newTreeItems)
{
newTree.Tree.Add(item);
}
return await _client.Git.Tree.Create(_repoOwner, _repoName, newTree);
}
private async Task<TreeResponse> CreateParentTreeAsync(TreeResponse testResultsTreeResponse, TreeResponse originalTreeResponse, string originalFilesDirectory)
{
// Create a new tree for the parent directory
NewTree parentTree = new NewTree { BaseTree = originalTreeResponse.Sha };
// Connect the updated test results tree
parentTree.Tree.Add(new NewTreeItem
{
Path = originalFilesDirectory,
Mode = TreeMode,
Type = TreeType.Tree,
Sha = testResultsTreeResponse.Sha
});
return await _client.Git.Tree.Create(_repoOwner, _repoName, parentTree);
}
private async Task CreateOrUpdatePullRequestAsync(TreeResponse parentTreeResponse, int buildId, string title, string targetBranch)
{
var existingPullRequest = await GetExistingPullRequestAsync(title, targetBranch);
// Create the branch name and get the head reference
string newBranchName = string.Empty;
string headSha = await GetHeadShaAsync(targetBranch);
if (existingPullRequest == null)
{
string utcTime = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
newBranchName = $"pr-baseline-{utcTime}";
}
else
{
newBranchName = existingPullRequest.Head.Ref;
try
{
// Merge the target branch into the existing pull request
var merge = new NewMerge(newBranchName, headSha);
await _client.Repository.Merging.Create(_repoOwner, _repoName, merge);
}
catch (Exception e)
{
Log.LogWarning($"Failed to merge the target branch into the existing pull request: {e.Message}");
Log.LogWarning("Continuing with updating the existing pull request. You may need to resolve conflicts manually in the PR.");
}
headSha = await GetHeadShaAsync(newBranchName);
}
var commitSha = await CreateCommitAsync(parentTreeResponse.Sha, headSha, $"Update baselines for build {BuildLink}{buildId} (internal Microsoft link)");
if (await ShouldMakeUpdatesAsync(headSha, commitSha))
{
string pullRequestBody = $"This PR was created by the `CreateBaselineUpdatePR` tool for build {buildId}. \n\n" +
$"The updated test results can be found at {BuildLink}{buildId} (internal Microsoft link)";
if (existingPullRequest != null)
{
await UpdatePullRequestAsync(newBranchName, commitSha, pullRequestBody, existingPullRequest);
}
else
{
await CreatePullRequestAsync(newBranchName, commitSha, targetBranch, title, pullRequestBody);
}
}
}
private async Task<PullRequest?> GetExistingPullRequestAsync(string title, string targetBranch)
{
var request = new PullRequestRequest
{
Base = targetBranch
};
var existingPullRequest = await _client.PullRequest.GetAllForRepository(_repoOwner, _repoName, request);
return existingPullRequest.FirstOrDefault(pr => pr.Title == title);
}
private async Task<string> CreateCommitAsync(string newSha, string headSha, string commitMessage)
{
var newCommit = new NewCommit(commitMessage, newSha, headSha);
var commit = await _client.Git.Commit.Create(_repoOwner, _repoName, newCommit);
return commit.Sha;
}
private async Task<bool> ShouldMakeUpdatesAsync(string headSha, string commitSha)
{
var comparison = await _client.Repository.Commit.Compare(_repoOwner, _repoName, headSha, commitSha);
if (!comparison.Files.Any())
{
Log.LogInformation("No changes to commit. Skipping PR creation/updates.");
return false;
}
return true;
}
private async Task UpdatePullRequestAsync(string branchName, string commitSha, string body, PullRequest pullRequest)
{
await UpdateReferenceAsync(branchName, commitSha);
var pullRequestUpdate = new PullRequestUpdate
{
Body = body
};
await _client.PullRequest.Update(_repoOwner, _repoName, pullRequest.Number, pullRequestUpdate);
Log.LogInformation($"Updated existing pull request #{pullRequest.Number}. URL: {pullRequest.HtmlUrl}");
}
private async Task CreatePullRequestAsync(string newBranchName, string commitSha, string targetBranch, string title, string body)
{
await CreateReferenceAsync(newBranchName, commitSha);
var newPullRequest = new NewPullRequest(title, newBranchName, targetBranch)
{
Body = body
};
var pullRequest = await _client.PullRequest.Create(_repoOwner, _repoName, newPullRequest);
Log.LogInformation($"Created pull request #{pullRequest.Number}. URL: {pullRequest.HtmlUrl}");
}
private async Task<string> GetHeadShaAsync(string branchName)
{
var reference = await _client.Git.Reference.Get(_repoOwner, _repoName, $"heads/{branchName}");
return reference.Object.Sha;
}
private async Task UpdateReferenceAsync(string branchName, string commitSha)
{
var referenceUpdate = new ReferenceUpdate(commitSha);
await _client.Git.Reference.Update(_repoOwner, _repoName, $"heads/{branchName}", referenceUpdate);
}
private async Task CreateReferenceAsync(string branchName, string commitSha)
{
var newReference = new NewReference($"refs/heads/{branchName}", commitSha);
await _client.Git.Reference.Create(_repoOwner, _repoName, newReference);
}
}

View file

@ -0,0 +1,10 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
namespace CreateBaselineUpdatePR;
public enum Pipelines
{
Sdk,
License
}

View file

@ -0,0 +1,118 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System;
using System.CommandLine;
using Microsoft.Extensions.Logging;
namespace CreateBaselineUpdatePR;
public class Program
{
public static readonly CliArgument<string> Repo = new("repo")
{
Description = "The GitHub repository to create the PR in. Should be in the form '<owner>/<repo-name>'",
Arity = ArgumentArity.ExactlyOne
};
public static readonly CliArgument<string> OriginalFilesDirectory = new("original-files-directory")
{
Description = "The directory where the original test files are located. Should be relative to the repo",
Arity = ArgumentArity.ExactlyOne
};
public static readonly CliArgument<string> UpdatedFilesDirectory = new("updated-files-directory")
{
Description = "The directory containing the updated test files published by the associated test. Should be absolute or relative to the working directory of the tool.",
Arity = ArgumentArity.ExactlyOne
};
public static readonly CliArgument<int> BuildId = new("build-id")
{
Description = "The id of the build that published the updated test files.",
Arity = ArgumentArity.ExactlyOne
};
public static readonly CliOption<string> Title = new("--title", "-t")
{
Description = "The title of the PR.",
Arity = ArgumentArity.ZeroOrOne,
DefaultValueFactory = _ => "Update Test Baselines and Exclusions"
};
public static readonly CliOption<string> Branch = new("--branch", "-b")
{
Description = "The target branch of the PR.",
Arity = ArgumentArity.ZeroOrOne,
DefaultValueFactory = _ => "main"
};
public static readonly CliOption<string> GitHubToken = new("--github-token", "-g")
{
Description = "The GitHub token to use to create the PR.",
Arity = ArgumentArity.ZeroOrOne,
DefaultValueFactory = _ => Environment.GetEnvironmentVariable("GH_TOKEN") ?? throw new ArgumentException("GitHub token not provided.")
};
public static readonly CliOption<LogLevel> Level = new("--log-level", "-l")
{
Description = "The log level to run the tool in.",
Arity = ArgumentArity.ZeroOrOne,
DefaultValueFactory = _ => LogLevel.Information,
Recursive = true
};
public static int ExitCode = 0;
public static async Task<int> Main(string[] args)
{
var sdkDiffTestsCommand = CreateCommand("sdk", "Creates a PR that updates baselines and exclusion files published by the sdk diff tests.");
var licenseScanTestsCommand = CreateCommand("license", "Creates a PR that updates baselines and exclusion files published by the license scan tests.");
var rootCommand = new CliRootCommand("Tool for creating PRs that update baselines and exclusion files.")
{
Level,
sdkDiffTestsCommand,
licenseScanTestsCommand
};
SetCommandAction(sdkDiffTestsCommand, Pipelines.Sdk);
SetCommandAction(licenseScanTestsCommand, Pipelines.License);
await rootCommand.Parse(args).InvokeAsync();
return ExitCode;
}
private static CliCommand CreateCommand(string name, string description)
{
return new CliCommand(name, description)
{
Repo,
OriginalFilesDirectory,
UpdatedFilesDirectory,
BuildId,
Title,
Branch,
GitHubToken
};
}
private static void SetCommandAction(CliCommand command, Pipelines pipeline)
{
command.SetAction(async (result, CancellationToken) =>
{
Log.Level = result.GetValue(Level);
var creator = new PRCreator(result.GetValue(Repo)!, result.GetValue(GitHubToken)!);
ExitCode = await creator.ExecuteAsync(
result.GetValue(OriginalFilesDirectory)!,
result.GetValue(UpdatedFilesDirectory)!,
result.GetValue(BuildId)!,
result.GetValue(Title)!,
result.GetValue(Branch)!,
pipeline);
});
}
}