Add a VMR initialization (bootstrap) process (#14343)

Adds the possibility to initialize a fresh VMR off of the current `dotnet/installer` commit.
Similarly to the tarball generation, the process can be triggered by running:
```
./build.sh /p:InitializeVMR=true /p:VmrDir=[Path were VMR will be created]
```
This commit is contained in:
Přemek Vysoký 2022-08-23 21:06:59 +02:00 committed by GitHub
parent f3f108c573
commit b9635390c8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 424 additions and 18 deletions

View file

@ -1,12 +1,30 @@
<Project>
<ItemGroup Condition=" '$(ArcadeBuildTarball)' != 'true' ">
<ProjectToBuild Include="$(RepoRoot)Microsoft.DotNet.Cli.sln" />
<ProjectToBuild Condition="'$(OS)' == 'Windows_NT' And ('$(Architecture)' == 'x86' Or '$(Architecture)' == 'x64' Or '$(Architecture)' == 'arm64')"
Include="$(RepoRoot)eng\version.csproj;
$(RepoRoot)eng\native.proj" />
</ItemGroup>
<ItemGroup Condition=" '$(ArcadeBuildTarball)' == 'true' " >
<ProjectToBuild Include="$(RepoRoot)src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj" BuildInParallel="false" />
<ProjectToBuild Include="$(RepoRoot)src/SourceBuild/tarball/BuildSourceBuildTarball.proj" BuildInParallel="false" />
</ItemGroup>
<Choose>
<When Condition=" '$(ArcadeBuildTarball)' == 'true' ">
<!-- Source build / tarball generation -->
<ItemGroup>
<ProjectToBuild Include="$(RepoRoot)src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj" BuildInParallel="false" />
<ProjectToBuild Include="$(RepoRoot)src/SourceBuild/tarball/BuildSourceBuildTarball.proj" BuildInParallel="false" />
</ItemGroup>
</When>
<When Condition=" '$(InitializeVMR)' == 'true' ">
<!-- VMR bootstrap -->
<ItemGroup>
<ProjectToBuild Include="$(RepoRoot)src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj" BuildInParallel="false" />
<ProjectToBuild Include="$(RepoRoot)src/VirtualMonoRepo/Tasks/VirtualMonoRepo.Tasks.csproj" BuildInParallel="false" />
<ProjectToBuild Include="$(RepoRoot)src/VirtualMonoRepo/InitializeVMR.proj" BuildInParallel="false" />
</ItemGroup>
</When>
<Otherwise>
<!-- Regular build -->
<ItemGroup>
<ProjectToBuild Include="$(RepoRoot)Microsoft.DotNet.Cli.sln" />
<ProjectToBuild Condition="'$(OS)' == 'Windows_NT' And ('$(Architecture)' == 'x86' Or '$(Architecture)' == 'x64' Or '$(Architecture)' == 'arm64')"
Include="$(RepoRoot)eng\version.csproj;
$(RepoRoot)eng\native.proj" />
</ItemGroup>
</Otherwise>
</Choose>
</Project>

View file

@ -213,6 +213,14 @@
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>34dff939b4a91e4693f78a856e0e055c1a3f3fba</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.DarcLib" Version="1.1.0-beta.22422.1">
<Uri>https://github.com/dotnet/arcade-services</Uri>
<Sha>a27d5e56acf479b4a0944a0cf4511acb22d4a75b</Sha>
</Dependency>
<Dependency Name="Microsoft.Extensions.Logging.Console" Version="7.0.0-preview.7.22375.6">
<Uri>https://github.com/dotnet/runtime</Uri>
<Sha>eecb02807867cad56cd05badddef65e432248b75</Sha>
</Dependency>
<Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-reference-packages" Version="7.0.0-alpha.1.22416.1">
<Uri>https://github.com/dotnet/source-build-reference-packages</Uri>
<Sha>c1bc837421e41a673cb30bf670c0cdd83ac09b75</Sha>

View file

@ -28,6 +28,10 @@
<!-- Dependency from https://github.com/dotnet/arcade -->
<MicrosoftDotNetBuildTasksInstallersPackageVersion>7.0.0-beta.22419.1</MicrosoftDotNetBuildTasksInstallersPackageVersion>
</PropertyGroup>
<PropertyGroup>
<!-- Dependency from https://github.com/dotnet/arcade-services -->
<MicrosoftDotNetDarcLibVersion>1.1.0-beta.22422.1</MicrosoftDotNetDarcLibVersion>
</PropertyGroup>
<PropertyGroup>
<!-- Dependency from https://github.com/dotnet/winforms -->
<MicrosoftDotnetWinFormsProjectTemplatesPackageVersion>7.0.0-rc.1.22377.2</MicrosoftDotnetWinFormsProjectTemplatesPackageVersion>
@ -171,6 +175,7 @@
<VersionToolsVersion>2.2.0-beta.19072.10</VersionToolsVersion>
<DotnetDebToolVersion>2.0.0</DotnetDebToolVersion>
<MicrosoftNETTestSdkVersion>17.4.0-preview-20220813-01</MicrosoftNETTestSdkVersion>
<MicrosoftExtensionsLoggingConsoleVersion>7.0.0-preview.7.22375.6</MicrosoftExtensionsLoggingConsoleVersion>
</PropertyGroup>
<!-- dependencies for source-build tarball -->
<PropertyGroup>

View file

@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net5.0</TargetFrameworks>
<TargetFrameworks>net7.0</TargetFrameworks>
<EnableDefaultCompileItems>false</EnableDefaultCompileItems>
<RunAnalyzers>false</RunAnalyzers>
<Nullable>disable</Nullable>

View file

@ -2,6 +2,6 @@
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
<Project>
<PropertyGroup>
<SourceBuildTasksAssembly>$(RepoRoot)\artifacts\bin\SourceBuild.Tasks\$(Configuration)\net5.0\SourceBuild.Tasks.dll</SourceBuildTasksAssembly>
<SourceBuildTasksAssembly>$(RepoRoot)\artifacts\bin\SourceBuild.Tasks\$(Configuration)\net7.0\SourceBuild.Tasks.dll</SourceBuildTasksAssembly>
</PropertyGroup>
</Project>

View file

@ -14,7 +14,7 @@
<PropertyGroup>
<ArtifactsDir>$(RepoRoot)artifacts/</ArtifactsDir>
<TarballDir>$(ArtifactsDir)tarball/</TarballDir>
<TarballDir Condition=" '$(TarballDir)' == '' ">$(ArtifactsDir)tarball/</TarballDir>
<TarballRootDir>$([MSBuild]::EnsureTrailingSlash('$(TarballDir)'))</TarballRootDir>
<TarballSourceDir>$(TarballRootDir)src/</TarballSourceDir>
<TarballGitInfoDir>$(TarballRootDir)git-info/</TarballGitInfoDir>
@ -234,8 +234,8 @@
SourceBuildMetadataDir="$(TarballGitInfoDir)"
Dependencies="@(SourceBuildRepos)" />
<Message Text="--> Dependencies for $(TarballVersionDetailsFile):" Importance="High" />
<Message Text="--> [@(SourceBuildRepos)]" Importance="High" />
<Message Text="--> Dependencies for $(TarballVersionDetailsFile):" Importance="High" Condition=" '@(SourceBuildRepos)' != '' " />
<Message Text="--> %(SourceBuildRepos.SourceBuildRepoName) / %(SourceBuildRepos.Sha)" Importance="High" Condition=" '@(SourceBuildRepos)' != '' " />
<MSBuild Projects="$(MSBuildProjectFile)"
Condition=" '@(SourceBuildRepos)' != '' "
Targets="CloneRepoAndDependentsRecursive"
@ -285,7 +285,7 @@
<Message Text="Restoring text only packages..." Importance="High" />
<MSBuild Projects="../Arcade/tools/TextOnlyPackages.csproj"
<MSBuild Projects="$(RepoRoot)src/SourceBuild/Arcade/tools/TextOnlyPackages.csproj"
Properties="TargetPackagesPath=$(TextOnlyPackageRestoreDir)"
Targets="Restore" />
</Target>
@ -296,7 +296,7 @@
<EngCommonContent Include="$(RepoRoot)eng/common/**/*" />
</ItemGroup>
<Copy
<Copy Condition=" '$(InitializeVMR)' != 'true' "
SourceFiles="$(TarballGitInfoDir)runtime.props"
DestinationFiles="$(TarballGitInfoDir)runtime-portable.props" />
@ -428,4 +428,3 @@
</Target>
</Project>

View file

@ -0,0 +1,140 @@
{
"defaults": {
"defaultRef": "main",
"exclude": [
"**/*.dll",
"**/*.Dll",
"**/*.exe",
"**/*.pdb",
"**/*.mdb",
"**/*.zip",
"**/*.nupkg"
]
},
"mappings": [
{
"name": "arcade",
"defaultRemote": "https://github.com/dotnet/arcade",
"include": [
"*.*",
".*",
"eng/**/*",
"src/**/*",
"scripts/**/*"
],
"exclude": [
"**/tests/**/*",
"**/*.Tests/**/*"
]
},
{
"name": "aspnetcore",
"defaultRemote": "https://github.com/dotnet/aspnetcore"
},
{
"name": "clicommandlineparser",
"defaultRemote": "https://github.com/dotnet/clicommandlineparser"
},
{
"name": "command-line-api",
"defaultRemote": "https://github.com/dotnet/command-line-api"
},
{
"name": "deployment-tools",
"defaultRemote": "https://github.com/dotnet/deployment-tools"
},
{
"name": "diagnostics",
"defaultRemote": "https://github.com/dotnet/diagnostics"
},
{
"name": "format",
"defaultRemote": "https://github.com/dotnet/format"
},
{
"name": "fsharp",
"defaultRemote": "https://github.com/dotnet/fsharp"
},
{
"name": "installer",
"defaultRemote": "https://github.com/dotnet/installer",
"defaultRef": "vmr"
},
{
"name": "linker",
"defaultRemote": "https://github.com/dotnet/linker"
},
{
"name": "msbuild",
"defaultRemote": "https://github.com/dotnet/msbuild"
},
{
"name": "nuget-client",
"defaultRemote": "https://github.com/NuGet/NuGet.Client",
"defaultRef": "dev"
},
{
"name": "razor-compiler",
"defaultRemote": "https://github.com/dotnet/razor-compiler"
},
{
"name": "roslyn",
"defaultRemote": "https://github.com/dotnet/roslyn"
},
{
"name": "roslyn-analyzers",
"defaultRemote": "https://github.com/dotnet/roslyn-analyzers"
},
{
"name": "runtime",
"defaultRemote": "https://github.com/dotnet/runtime"
},
{
"name": "sdk",
"defaultRemote": "https://github.com/dotnet/sdk"
},
{
"name": "source-build",
"defaultRemote": "https://github.com/dotnet/source-build"
},
{
"name": "source-build-externals",
"defaultRemote": "https://github.com/dotnet/source-build-externals",
"exclude": [
"**/humanizer/samples/**/*.js"
]
},
{
"name": "source-build-reference-packages",
"defaultRemote": "https://github.com/dotnet/source-build-reference-packages"
},
{
"name": "sourcelink",
"defaultRemote": "https://github.com/dotnet/sourcelink"
},
{
"name": "symreader",
"defaultRemote": "https://github.com/dotnet/symreader"
},
{
"name": "templating",
"defaultRemote": "https://github.com/dotnet/templating"
},
{
"name": "test-templates",
"defaultRemote": "https://github.com/dotnet/test-templates"
},
{
"name": "vstest",
"defaultRemote": "https://github.com/microsoft/vstest"
},
{
"name": "xdt",
"defaultRemote": "https://github.com/dotnet/xdt"
},
{
"name": "xliff-tasks",
"defaultRemote": "https://github.com/dotnet/xliff-tasks"
}
]
}

View file

@ -0,0 +1,121 @@
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
<Project>
<PropertyGroup>
<ArtifactsDir>$(RepoRoot)artifacts/</ArtifactsDir>
<VirtualMonoRepoTasksAssembly>$(ArtifactsDir)bin/VirtualMonoRepo.Tasks/$(Configuration)/net7.0/VirtualMonoRepo.Tasks.dll</VirtualMonoRepoTasksAssembly>
<VmrDir Condition=" '$(VmrDir)' != '' ">$([MSBuild]::EnsureTrailingSlash('$(VmrDir)'))</VmrDir>
<TmpDir Condition=" '$(TmpDir)' == '' ">$(ArtifactsDir)tmp/</TmpDir>
<VmrDefaultBranch Condition=" '$(VmrDefaultBranch)' == '' ">main</VmrDefaultBranch>
<!-- We set this for the source build targets to work well -->
<TarballDir>$(VmrDir)</TarballDir>
<InitializeVMR>true</InitializeVMR>
</PropertyGroup>
<Import Project="../SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets" />
<!-- Pull back in some properties from the tarball defaults -->
<PropertyGroup>
<VmrGitInfoDir>$(TarballGitInfoDir)</VmrGitInfoDir>
<VmrSourceDir>$(TarballSourceDir)</VmrSourceDir>
</PropertyGroup>
<UsingTask TaskName="Microsoft.DotNet.VirtualMonoRepo.Tasks.VirtualMonoRepo_Initialize" AssemblyFile="$(VirtualMonoRepoTasksAssembly)" />
<Target Name="Build" DependsOnTargets="InitializeVMR" />
<!--
This target initializes a fresh VMR from scratch by assembling it similarly to the tarball.
Instead of cloning the repositories, we pull the sources via the `darc vmr initialize` command.
Most of the targets are reused from the tarball generation process (SourceBuildArcadeTarball.targets).
-->
<Target Name="InitializeVMR" DependsOnTargets="GenerateFullNuGetVersion;
GenerateVersionFile;
InitializeCleanVmr;
CopyTarballContent;
CommitInitialContent;
SetupSelfGithubInfo;
InitializeRepoAndDependentsRecursive;
CommitGitInfoFiles;
CopyTextOnlyPackages;
CommitTextOnlyPackages">
<Message Text="VMR was successfully initialized in '$(VmrDir)'" Importance="High" />
</Target>
<Target Name="InitializeCleanVmr">
<RemoveDir Directories='$(VmrDir)' Condition=" EXISTS('$(VmrDir)') " />
<MakeDir Directories="$(VmrDir)" />
<MakeDir Directories="$(VmrGitInfoDir)" Condition=" '$(VmrGitInfoDir)' != '' " />
<MakeDir Directories="$(TmpDir)" Condition=" !EXISTS('$(TmpDir)') " />
<Exec Command="git init -b $(VmrDefaultBranch)" WorkingDirectory="$(VmrDir)" />
</Target>
<Target Name="CommitInitialContent">
<Exec Command="git add -A" WorkingDirectory="$(VmrDir)" />
<Exec Command="git commit -m 'VMR initialized'" WorkingDirectory="$(VmrDir)" />
</Target>
<Target Name="InitializeRepoAndDependentsRecursive"
DependsOnTargets="GetSourceBuildIntermediateNupkgNameConvention">
<Message Text="--> Initializing repo $(SourceBuildRepoName)" Importance="High" />
<PropertyGroup>
<SourceDir>$(SourceBuildRepoName)/</SourceDir>
<IndividualRepoSourceDir>$(VmrSourceDir)$(SourceDir)</IndividualRepoSourceDir>
<IndividualRepoSourceEngDir>$(IndividualRepoSourceDir)eng/</IndividualRepoSourceEngDir>
<IndividualRepoVersionDetailsFile>$(IndividualRepoSourceEngDir)Version.Details.xml</IndividualRepoVersionDetailsFile>
</PropertyGroup>
<VirtualMonoRepo_Initialize
Repository="$(SourceBuildRepoName)"
Revision="$(RepoSha)"
VmrPath="$(VmrDir)"
TmpPath="$(TmpDir)" />
<Message Text=" -> Done initializing repo $(SourceBuildRepoName)" Importance="High" />
<Tarball_ReadSourceBuildIntermediateNupkgDependencies
VersionDetailsXmlFile="$([MSBuild]::NormalizePath($(IndividualRepoVersionDetailsFile)))"
SourceBuildIntermediateNupkgPrefix="$(SourceBuildIntermediateNupkgPrefix)"
SourceBuildIntermediateNupkgRid="$(SourceBuildIntermediateNupkgRid)"
ConvertInternalRepos="$(ConvertInternalRepos)">
<Output TaskParameter="Dependencies" ItemName="SourceBuildRepos" />
</Tarball_ReadSourceBuildIntermediateNupkgDependencies>
<!-- Remove repo if it has already be cloned at any sha. This results in
The commit sha for the cloned repo in the tarball being the one that
was first encountered. -->
<ItemGroup>
<SourceBuildRepos Remove="@(SourceBuildRepos)" Condition=" EXISTS('$(VmrGitInfoDir)%(SourceBuildRepoName).props')" />
</ItemGroup>
<Tarball_WriteSourceRepoProperties
SourceBuildMetadataDir="$(VmrGitInfoDir)"
Dependencies="@(SourceBuildRepos)" />
<Message Text="--> Dependencies for $(IndividualRepoVersionDetailsFile):" Importance="High" Condition=" '@(SourceBuildRepos)' != '' " />
<Message Text="--> %(SourceBuildRepos.SourceBuildRepoName) / %(SourceBuildRepos.Sha)" Importance="High" Condition=" '@(SourceBuildRepos)' != '' " />
<MSBuild Projects="$(MSBuildProjectFile)"
Condition=" '@(SourceBuildRepos)' != '' "
Targets="InitializeRepoAndDependentsRecursive"
Properties="SourceBuildRepoName=%(SourceBuildRepos.SourceBuildRepoName);RepoSha=%(SourceBuildRepos.Sha)" />
</Target>
<Target Name="CommitGitInfoFiles">
<Copy
SourceFiles="$(VmrGitInfoDir)runtime.props"
DestinationFiles="$(VmrGitInfoDir)runtime-portable.props" />
<Exec Command="git add git-info" WorkingDirectory="$(VmrDir)" />
<Exec Command="git commit -m 'Initialized git-info files'" WorkingDirectory="$(VmrDir)" />
</Target>
<Target Name="CommitTextOnlyPackages">
<Exec Command="git add packages/text-only" WorkingDirectory="$(VmrDir)" />
<Exec Command="git commit -m 'Initialized text-only packages'" WorkingDirectory="$(VmrDir)" />
</Target>
</Project>

View file

@ -0,0 +1,38 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System;
using System.Threading.Tasks;
using Microsoft.DotNet.DarcLib;
using Microsoft.DotNet.DarcLib.Helpers;
namespace Microsoft.DotNet.VirtualMonoRepo.Tasks;
internal class RemoteFactory : IRemoteFactory
{
private readonly IProcessManager _processManager;
private readonly string _tmpPath;
public RemoteFactory(IProcessManager processManager, string tmpPath)
{
_processManager = processManager;
_tmpPath = tmpPath;
}
public Task<IRemote> GetBarOnlyRemoteAsync(Extensions.Logging.ILogger logger)
{
throw new NotImplementedException();
}
public Task<IRemote> GetRemoteAsync(string repoUrl, Extensions.Logging.ILogger logger)
{
var githubClient = new DarcLib.GitHubClient(
_processManager.GitExecutable,
accessToken: null,
logger,
_tmpPath,
cache: null);
return System.Threading.Tasks.Task.FromResult<IRemote>(new Remote(githubClient, barClient: null, logger));
}
}

View file

@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net7.0</TargetFrameworks>
<LangVersion>preview</LangVersion>
<Nullable>disable</Nullable>
<RunAnalyzers>false</RunAnalyzers>
<SignAssembly>false</SignAssembly>
<IsPackable>false</IsPackable>
<IsShipping>false</IsShipping>
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Build.Framework" Version="$(MicrosoftBuildFrameworkVersion)" />
<PackageReference Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildUtilitiesCoreVersion)" />
<PackageReference Include="Microsoft.DotNet.DarcLib" Version="$(MicrosoftDotNetDarcLibVersion)" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
</ItemGroup>
</Project>

View file

@ -0,0 +1,57 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Build.Framework;
using Microsoft.DotNet.DarcLib;
using Microsoft.DotNet.DarcLib.Helpers;
using Microsoft.DotNet.DarcLib.VirtualMonoRepo;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace Microsoft.DotNet.VirtualMonoRepo.Tasks;
public class VirtualMonoRepo_Initialize : Build.Utilities.Task, ICancelableTask
{
private readonly Lazy<IServiceProvider> _serviceProvider;
private readonly CancellationTokenSource _cancellationToken = new();
[Required]
public string Repository { get; set; }
[Required]
public string VmrPath { get; set; }
[Required]
public string TmpPath { get; set; }
public string Revision { get; set; }
public VirtualMonoRepo_Initialize()
{
_serviceProvider = new(CreateServiceProvider);
}
public override bool Execute() => ExecuteAsync().GetAwaiter().GetResult();
private async Task<bool> ExecuteAsync()
{
var factory = _serviceProvider.Value.GetRequiredService<IVmrManagerFactory>();
var vmrManager = await factory.CreateVmrManager(_serviceProvider.Value, VmrPath, TmpPath);
await vmrManager.InitializeVmr(Repository, Revision, false, _cancellationToken.Token);
return true;
}
public void Cancel() => _cancellationToken.Cancel();
private IServiceProvider CreateServiceProvider() => new ServiceCollection()
.AddLogging(b => b.AddConsole().AddFilter(l => l >= LogLevel.Information))
.AddTransient<IProcessManager>(sp => ActivatorUtilities.CreateInstance<ProcessManager>(sp, "git"))
.AddSingleton<ISourceMappingParser, SourceMappingParser>()
.AddSingleton<IVmrManagerFactory, VmrManagerFactory>()
.AddSingleton<IRemoteFactory>(sp => ActivatorUtilities.CreateInstance<RemoteFactory>(sp, TmpPath))
.BuildServiceProvider();
}