mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove VMR and source-build infrastructure from installer (#19697)

Browse source
This commit is contained in:
Viktor Hofer 2024-05-07 16:44:18 +02:00 committed by GitHub
parent 6c72a489da
commit d3b25302eb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
290 changed files with 37 additions and 24912 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
.devcontainer/vmr-source-build/README.md
View file

@ -1,55 +0,0 @@
<!--
######## ######## ### ######## ######## ## ## #### ######
## ## ## ## ## ## ## ## ## ## ## ## ##
## ## ## ## ## ## ## ## ## ## ## ##
######## ###### ## ## ## ## ## ######### ## ######
## ## ## ######### ## ## ## ## ## ## ##
## ## ## ## ## ## ## ## ## ## ## ## ##
## ## ######## ## ## ######## ## ## ## #### ######
-->
This Codespace can help you debug the source build of .NET. In case you have run this from a
`dotnet/installer` PR branch, it will contain the VMR (`dotnet/dotnet`) checked out into
`/workspaces/dotnet` with the PR changes pulled into it. You can then attempt to source-build
the VMR which is what the VMR leg in the installer PR build doing. This build takes about 45
minutes and, after completion, produces an archived .NET SDK located in
`/workspaces/dotnet/artifacts/assets/Release`.
## Build the SDK
To build the VMR, run following:
```bash
cd /workspaces/dotnet
./build.sh --online
```
> Please note that, at this time, the build modifies some of the checked-in sources so it might
be preferential to rebuild the Codespace between attempts (or reset the working tree changes).
For more details, see the instructions at https://github.com/dotnet/dotnet.
## Synchronize your changes in locally
When debugging the build, you have two options how to test your changes in this environment.
### Making changes to the VMR directly
You can make the changes directly to the local checkout of the VMR at `/workspaces/dotnet`. You
can then try to build the VMR and see if the change works for you.
### Pull changes into the Codespace from your fork
You can also make a fix in the individual source repository (e.g. `dotnet/runtime`) and push the
fix into a branch; can be in your fork too. Once you have the commit pushed, you can pull this
version of the repository into the Codespace by running:
```
/workspaces/synchronize-vmr.sh \
--repository <repo>:<commit, tag or branch> \
--remote <repo>:<fork URI>
```
You can now proceed building the VMR in the Codespace using instructions above. You can repeat
this process and sync a new commit from your fork. Only note that, at this time, Source-Build
modifies some of the checked-in sources so you'll need to revert the working tree changes
between attempts.

25
.devcontainer/vmr-source-build/devcontainer.json
View file

@ -1,25 +0,0 @@
// Container suitable for investigating issues with source build
// Contains the VMR (dotnet/dotnet) with applied changes from the current PR
// The container supports source-building the SDK
{
"name": "VMR with PR changes",
"image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39",
"hostRequirements": {
// A completely source built .NET is >64 GB with all the repos/artifacts
"storage": "128gb"
},
"customizations": {
"vscode": {
"extensions": [
"ms-dotnettools.csharp"
]
},
"codespaces": {
"openFiles": [
"installer/.devcontainer/vmr-source-build/README.md"
]
}
},
"onCreateCommand": "${containerWorkspaceFolder}/installer/.devcontainer/vmr-source-build/init.sh",
"workspaceFolder": "/workspaces"
}

30
.devcontainer/vmr-source-build/init.sh
View file

@ -1,30 +0,0 @@
#!/usr/bin/env bash
set -ex
source="${BASH_SOURCE[0]}"
script_root="$( cd -P "$( dirname "$source" )" && pwd )"
installer_dir=$(realpath "$script_root/../..")
workspace_dir=$(realpath "$installer_dir/../")
tmp_dir=$(realpath "$workspace_dir/tmp")
vmr_dir=$(realpath "$workspace_dir/dotnet")
cp "$installer_dir/.devcontainer/vmr-source-build/synchronize-vmr.sh" "$workspace_dir"
mkdir -p "$tmp_dir"
# Codespaces performs a shallow fetch only
git -C "$installer_dir" fetch --all --unshallow
# We will try to figure out, which branch is the current (PR) branch based off of
# We need this to figure out, which VMR branch to use
vmr_branch=$(git -C "$installer_dir" log --pretty=format:'%D' HEAD^ \
| grep 'origin/' \
| head -n1 \
| sed 's@origin/@@' \
| sed 's@,.*@@')
"$workspace_dir/synchronize-vmr.sh" --branch "$vmr_branch" --debug
(cd "$vmr_dir" && ./prep-source-build.sh)

4
.devcontainer/vmr-source-build/synchronize-vmr.sh
View file

@ -1,4 +0,0 @@
#!/bin/bash
(cd /workspaces/installer \
&& ./eng/vmr-sync.sh --vmr /workspaces/dotnet --tmp /workspaces/tmp $*)

3
.vsts-ci.yml
View file

@ -372,9 +372,6 @@ extends:
linuxPortable: true
runTests: false
# Source Build
- template: /eng/common/templates-official/jobs/source-build.yml@self
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- stage: Publish
dependsOn:

2
.vsts-pr.yml
View file

@ -346,8 +346,6 @@ stages:
linuxPortable: true
runTests: false
- template: /eng/common/templates/jobs/source-build.yml
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- stage: Publish
dependsOn:

6
CODEOWNERS
View file

@ -1,8 +1,4 @@
# Users referenced in this file will automatically be requested as reviewers for PRs that modify the given paths.
# See https://help.github.com/articles/about-code-owners/
/.devcontainer/ @dotnet/source-build-internal
/eng/SourceBuild* @dotnet/source-build-internal
/src/snaps/ @rbhanda
/src/SourceBuild/ @dotnet/source-build-internal @dotnet/product-construction
/src/VirtualMonoRepo/ @dotnet/product-construction
/src/snaps/ @rbhanda

14
README.md
View file

@ -46,7 +46,6 @@ The repository contains native code project required for the Windows installer.
# Build .NET from source (source-build)
This repo also contains code to help you build the entire .NET product end-to-end from source (often referred to as source-build), even in disconnected/offline mode.
Please see the [dotnet/source-build](https://github.com/dotnet/source-build) repo for more information.
## Support
@ -55,25 +54,12 @@ Please see the [dotnet/source-build](https://github.com/dotnet/source-build) rep
For example, if .NET 6.0.1xx, 6.0.2xx, and 7.0.1xx feature updates are available from [dotnet.microsoft.com](https://dotnet.microsoft.com/en-us/download/dotnet/6.0), Source-Build will support 6.0.1xx and 7.0.1xx.
For the latest information about Source-Build support for new .NET versions, please check our [GitHub Discussions page](https://github.com/dotnet/source-build/discussions) for announcements.
## Prerequisites
The dependencies for building .NET from source can be found [here](https://github.com/dotnet/runtime/blob/main/docs/workflow/requirements/linux-requirements.md).
## Building .NET 8.0
.NET 8.0 (currently in prerelease) and newer will be built from the [dotnet/dotnet](https://github.com/dotnet/dotnet) repo.
Clone the dotnet/dotnet repo and check out the tag for the desired release.
Then, follow the instructions in [dotnet/dotnet's README](https://github.com/dotnet/dotnet/blob/main/README.md#dev-instructions) to build .NET from source.
### Codespaces
It is also possible to utilize [GitHub Codespaces](https://github.com/features/codespaces) and build .NET from the `dotnet/dotnet` repository from source that way.
You can either create a Codespace in `dotnet/dotnet` directly or you can also make one from a PR branch in `dotnet/installer`. This will give you an environment with the VMR checked out and containing all of new changes from the PR.
This can be especially valuable for investigations of source-build failures during PRs.
To create a Codespace for a `dotnet/installer` PR, use the `vmr-source-build` devcontainer configuration (select this when "newing the Codespace with options" under the three-dots-menu).
Further instructions on how to build inside of the Codespace will be available upon launch.
## Building .NET 7.0 and .NET 6.0
1. Create a .NET source tarball.

23
eng/Build.props
View file

@ -1,20 +1,9 @@
<Project>
<Choose>
<When Condition=" '$(InitializeVMR)' == 'true' ">
<!-- VMR bootstrap -->
<ItemGroup>
<ProjectToBuild Include="$(RepoRoot)src/VirtualMonoRepo/Tasks/VirtualMonoRepo.Tasks.csproj" BuildInParallel="false" />
<ProjectToBuild Include="$(RepoRoot)src/VirtualMonoRepo/InitializeVMR.proj" BuildInParallel="false" />
</ItemGroup>
</When>
<Otherwise>
<!-- Regular build -->
<ItemGroup>
<ProjectToBuild Include="$(RepoRoot)Microsoft.DotNet.Cli.sln" />
<ProjectToBuild Condition="'$(OS)' == 'Windows_NT' And ('$(Architecture)' == 'x86' Or '$(Architecture)' == 'x64' Or '$(Architecture)' == 'arm64')"
Include="$(RepoRoot)src\finalizer\finalizer-build.csproj" />
</ItemGroup>
</Otherwise>
</Choose>
<ItemGroup>
<ProjectToBuild Include="$(RepoRoot)Microsoft.DotNet.Cli.sln" />
<ProjectToBuild Condition="'$(OS)' == 'Windows_NT' And ('$(Architecture)' == 'x86' Or '$(Architecture)' == 'x64' Or '$(Architecture)' == 'arm64')"
Include="$(RepoRoot)src\finalizer\finalizer-build.csproj" />
</ItemGroup>
</Project>

32
eng/SourceBuild.props
View file

@ -1,32 +0,0 @@
<!-- Whenever altering this or other Source Build files, please include @dotnet/source-build-internal as a reviewer. -->
<Project>
<PropertyGroup>
<GitHubRepositoryName>installer</GitHubRepositoryName>
<SourceBuildManagedOnly>true</SourceBuildManagedOnly>
</PropertyGroup>
<Target Name="ConfigureInnerBuildArg" BeforeTargets="GetSourceBuildCommandConfiguration">
<PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
<InnerBuildArgs>$(InnerBuildArgs) /p:SkipBuildingInstallers=true</InnerBuildArgs>
<InnerBuildArgs>$(InnerBuildArgs) /p:IncludeAdditionalSharedFrameworks=false</InnerBuildArgs>
<InnerBuildArgs Condition="'$(SourceBuildUseMonoRuntime)' == 'true'">$(InnerBuildArgs) /p:DISABLE_CROSSGEN=true</InnerBuildArgs>
<InnerBuildArgs Condition="'$(PgoInstrument)' == 'true'">$(InnerBuildArgs) /p:PgoInstrument=true</InnerBuildArgs>
</PropertyGroup>
</Target>
<!-- This should be resolved/removed with https://github.com/dotnet/source-build/issues/4101 -->
<Target Name="AddInstallers"
BeforeTargets="GetCategorizedIntermediateNupkgContents">
<ItemGroup Condition="'$(DotNetBuildOrchestrator)' == 'true'">
<!-- Include installers when in product VMR builds. These are not necessary when building the repo-only build as we don't
need them in downstream source-only PR legs. We could include them, but it may bump us over the package size limit. -->
<IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\*.msi" />
<IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\*.deb" />
<IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\*.rpm" />
<IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\*.pkg" />
</ItemGroup>
</Target>
</Project>

15
eng/SourceBuildPrebuiltBaseline.xml
View file

@ -1,15 +0,0 @@
<!-- Whenever altering this or other Source Build files, please include @dotnet/source-build-internal as a reviewer. -->
<!-- See aka.ms/dotnet/prebuilts for guidance on what pre-builts are and how to eliminate them. -->
<UsageData>
<IgnorePatterns>
<!--
Temporary exclusion for NuGet packages, since NuGet is not producing source-build intermediate package,
see: https://github.com/NuGet/Home/issues/11059
-->
<UsagePattern IdentityGlob="NuGet.*/*" />
<!-- These are coming in via runtime but the source-build infra isn't able to automatically pick up the right intermediate. -->
<UsagePattern IdentityGlob="Microsoft.NETCore.App.Crossgen2.linux-x64/*9.0.*" />
</IgnorePatterns>
</UsageData>

100
eng/Version.Details.xml
View file

@ -1,9 +1,5 @@
<Dependencies>
<ProductDependencies>
<!--
Source-build uses transitive dependency resolution to determine correct build SHA of all product contributing repos.
The order of dependencies is important and should not be modified without approval from dotnet/source-build-internal.
-->
<Dependency Name="Microsoft.WindowsDesktop.App.Ref" Version="9.0.0-preview.5.24256.1" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/windowsdesktop</Uri>
<Sha>436e9a8e566984a5ffe5f022de266f18d9aa9c4b</Sha>
@ -50,12 +46,6 @@
<Uri>https://github.com/dotnet/runtime</Uri>
<Sha>84b33395057737db3ea342a5151feb6b90c1b6f6</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.runtime.linux-x64" Version="9.0.0-preview.5.24256.1" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/runtime</Uri>
<Sha>84b33395057737db3ea342a5151feb6b90c1b6f6</Sha>
<SourceBuild RepoName="runtime" ManagedOnly="false" />
</Dependency>
<Dependency Name="Microsoft.AspNetCore.App.Ref" Version="9.0.0-preview.5.24256.2" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/aspnetcore</Uri>
<Sha>da3aa27233a2cec2f6780884f71934b2f5e686ce</Sha>
@ -84,12 +74,6 @@
<Uri>https://github.com/dotnet/aspnetcore</Uri>
<Sha>da3aa27233a2cec2f6780884f71934b2f5e686ce</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.aspnetcore" Version="9.0.0-preview.5.24256.2" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/aspnetcore</Uri>
<Sha>da3aa27233a2cec2f6780884f71934b2f5e686ce</Sha>
<SourceBuild RepoName="aspnetcore" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.DotNet.Common.ItemTemplates" Version="9.0.100-preview.5.24256.14">
<Uri>https://github.com/dotnet/sdk</Uri>
<Sha>a1863f126667b4fdcd37b451b4cd3c10abf19e10</Sha>
@ -106,12 +90,6 @@
<Uri>https://github.com/dotnet/sdk</Uri>
<Sha>a1863f126667b4fdcd37b451b4cd3c10abf19e10</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.sdk" Version="9.0.100-preview.5.24256.14">
<Uri>https://github.com/dotnet/sdk</Uri>
<Sha>a1863f126667b4fdcd37b451b4cd3c10abf19e10</Sha>
<SourceBuild RepoName="sdk" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.DotNet.Test.ProjectTemplates.2.1" Version="1.0.2-beta4.22406.1" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/test-templates</Uri>
<Sha>0385265f4d0b6413d64aea0223172366a9b9858c</Sha>
@ -136,12 +114,6 @@
<Uri>https://github.com/dotnet/test-templates</Uri>
<Sha>36e4339a33f9bdf3680591e2a3fcbc421aabc22c</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.test-templates" Version="1.1.0-rc.24252.2" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/test-templates</Uri>
<Sha>36e4339a33f9bdf3680591e2a3fcbc421aabc22c</Sha>
<SourceBuild RepoName="test-templates" ManagedOnly="true" />
</Dependency>
<!-- For coherency purposes, these versions should be gated by the versions of winforms and wpf routed via windowsdesktop -->
<Dependency Name="Microsoft.Dotnet.WinForms.ProjectTemplates" Version="9.0.0-preview.5.24253.4" CoherentParentDependency="Microsoft.WindowsDesktop.App.Runtime.win-x64">
<Uri>https://github.com/dotnet/winforms</Uri>
@ -155,22 +127,10 @@
<Uri>https://github.com/dotnet/fsharp</Uri>
<Sha>3ef1cb25ffb292b5c87f9604d1a09b032277bf76</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.fsharp" Version="8.0.400-beta.24229.4" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/fsharp</Uri>
<Sha>3ef1cb25ffb292b5c87f9604d1a09b032277bf76</Sha>
<SourceBuild RepoName="fsharp" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.NET.Test.Sdk" Version="17.11.0-preview-24253-02" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/microsoft/vstest</Uri>
<Sha>b521aa2c9c981f53b85af7c923175a850986173a</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.vstest" Version="17.11.0-preview-24253-02" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/microsoft/vstest</Uri>
<Sha>b521aa2c9c981f53b85af7c923175a850986173a</Sha>
<SourceBuild RepoName="vstest" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.NET.ILLink.Tasks" Version="9.0.0-preview.5.24256.1" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/runtime</Uri>
<Sha>84b33395057737db3ea342a5151feb6b90c1b6f6</Sha>
@ -179,63 +139,26 @@
<Uri>https://github.com/dotnet/roslyn</Uri>
<Sha>062ad3db597a8096b5da2b188dbbbcc7f6137275</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.roslyn" Version="4.11.0-2.24255.2" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/roslyn</Uri>
<Sha>062ad3db597a8096b5da2b188dbbbcc7f6137275</Sha>
<SourceBuild RepoName="roslyn" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.Build" Version="17.11.0-preview-24256-01" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/msbuild</Uri>
<Sha>66dcc32c3344eb76a422917dcb8c9e8f621e18fd</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.msbuild" Version="17.11.0-preview-24256-01" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/msbuild</Uri>
<Sha>66dcc32c3344eb76a422917dcb8c9e8f621e18fd</Sha>
<SourceBuild RepoName="msbuild" ManagedOnly="true" />
</Dependency>
<Dependency Name="NuGet.Build.Tasks" Version="6.11.0-preview.1.35" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/nuget/nuget.client</Uri>
<Sha>12d5c661b77d4933f82a293008c0d56d1d6ce32b</Sha>
<SourceBuildTarball RepoName="nuget-client" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.NET.Workload.Emscripten.Current.Manifest-9.0.100.Transport" Version="9.0.0-preview.5.24223.2" CoherentParentDependency="Microsoft.NETCore.App.Runtime.win-x64">
<Uri>https://github.com/dotnet/emsdk</Uri>
<Sha>53288f87c588907e8ff01f129786820fe998573c</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.emsdk" Version="9.0.0-preview.5.24223.2" CoherentParentDependency="Microsoft.NETCore.App.Runtime.win-x64">
<Uri>https://github.com/dotnet/emsdk</Uri>
<Sha>53288f87c588907e8ff01f129786820fe998573c</Sha>
<SourceBuild RepoName="emsdk" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.Deployment.DotNet.Releases" Version="2.0.0-preview.1.24176.3" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/deployment-tools</Uri>
<Sha>6b80783f6743ee9f18940eb6acb7135e5c111d4b</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.deployment-tools" Version="9.0.0-preview.1.24176.3" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/deployment-tools</Uri>
<Sha>6b80783f6743ee9f18940eb6acb7135e5c111d4b</Sha>
<SourceBuild RepoName="deployment-tools" ManagedOnly="true" />
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-externals" Version="9.0.0-alpha.1.24229.1">
<Uri>https://github.com/dotnet/source-build-externals</Uri>
<Sha>b02769661c9a51985877819e8bdebfbcbee65710</Sha>
<SourceBuild RepoName="source-build-externals" ManagedOnly="true" />
</Dependency>
<Dependency Name="System.CommandLine" Version="2.0.0-beta4.24209.3" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/command-line-api</Uri>
<Sha>963d34b1fb712c673bfb198133d7e988182c9ef4</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.command-line-api" Version="0.1.520903" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/command-line-api</Uri>
<Sha>963d34b1fb712c673bfb198133d7e988182c9ef4</Sha>
<SourceBuild RepoName="command-line-api" ManagedOnly="true" />
</Dependency>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24253.1">
@ -262,31 +185,14 @@
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>020255bcf7d0b8beed7de05338d97396982ae527</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24253.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>020255bcf7d0b8beed7de05338d97396982ae527</Sha>
<SourceBuild RepoName="arcade" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.Extensions.Logging.Console" Version="9.0.0-alpha.1.23612.13">
<Uri>https://github.com/dotnet/runtime</Uri>
<Sha>ab1a8224cdf115b65e0db5dc88d11f205068f444</Sha>
</Dependency>
<Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-reference-packages" Version="9.0.0-alpha.1.24252.1">
<Uri>https://github.com/dotnet/source-build-reference-packages</Uri>
<Sha>9c1cc994f8123ec2a923c5179c238c13da1b4ab7</Sha>
<SourceBuild RepoName="source-build-reference-packages" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.DotNet.ScenarioTests.SdkTemplateTests" Version="9.0.0-preview.24222.1" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/scenario-tests</Uri>
<Sha>4ab07002cb46cf169c85a09a546709a20642c65b</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.scenario-tests" Version="9.0.0-preview.24222.1" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/scenario-tests</Uri>
<Sha>4ab07002cb46cf169c85a09a546709a20642c65b</Sha>
<SourceBuild RepoName="scenario-tests" ManagedOnly="true" />
</Dependency>
<!-- Aspire isn't really a toolset dependency. However, it only inserts a baseline manifest in installer,
and if you squint at it, this means we can say that its specific dependency versions don't matter to installer.
Avoiding this as a product dependency avoids a long coherency path (aspnetcore->extensions->aspire->installer).
@ -298,11 +204,5 @@
<Uri>https://github.com/dotnet/aspire</Uri>
<Sha>0514ea9e12ece4dd764824ce925ae0eae6fcbd86</Sha>
</Dependency>
<!-- Intermediate is necessary for source build. -->
<Dependency Name="Microsoft.SourceBuild.Intermediate.aspire" Version="9.0.0-preview.4.24229.9" CoherentParentDependency="Microsoft.NET.Sdk">
<Uri>https://github.com/dotnet/aspire</Uri>
<Sha>0514ea9e12ece4dd764824ce925ae0eae6fcbd86</Sha>
<SourceBuild RepoName="aspire" ManagedOnly="true" />
</Dependency>
</ToolsetDependencies>
</Dependencies>

25
eng/install-scancode.sh
View file

@ -1,25 +0,0 @@
#!/bin/bash
set -euo pipefail
# Install instructions: https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html#installation-as-a-library-via-pip
# See latest release at https://github.com/nexB/scancode-toolkit/releases
SCANCODE_VERSION="32.1.0"
pyEnvPath="/tmp/scancode-env"
python3 -m venv $pyEnvPath
source $pyEnvPath/bin/activate
pip install scancode-toolkit==$SCANCODE_VERSION
deactivate
# Setup a script which executes scancode in the virtual environment
cat > /usr/local/bin/scancode << EOF
#!/bin/bash
set -euo pipefail
source $pyEnvPath/bin/activate
scancode "\$@"
deactivate
EOF
chmod +x /usr/local/bin/scancode

542
eng/pipelines/templates/jobs/vmr-build.yml
View file

@ -1,542 +0,0 @@
### This job builds https://github.com/dotnet/dotnet with given parameters
### If run in an installer PR, new changes are applied to a local copy of the VMR, then it is built and tested
parameters:
- name: architecture
type: string
- name: artifactsRid
type: string
default: ''
- name: buildName
type: string
- name: container
type: string
default: ''
- name: crossRootFs
type: string
default: ''
- name: pool
type: object
- name: targetOS
type: string
default: ''
- name: targetArchitecture
type: string
default: ''
- name: useMonoRuntime
displayName: True when build output uses the mono runtime
type: boolean
default: false
- name: useDevVersions
displayName: True when build output uses dev/CI versioning instead of official build versioning
type: boolean
default: false
#### SOURCE-ONLY parameters ####
# Instead of building the VMR directly, exports the sources into a tarball and builds from that
- name: buildFromArchive
type: boolean
default: false
# Enable for source-building the VMR
- name: buildSourceOnly
type: boolean
default: false
- name: enablePoison
type: boolean
default: false
- name: excludeOmniSharpTests
type: boolean
default: false
# Name of a previous job (from the same template as this) whose output will be used to build this job
# The SDK from its artifacts is copied to $(sourcesPath)/.dotnet
- name: reuseBuildArtifactsFrom
type: string
default: ''
# Allow downloading artifacts from the internet during the build
- name: runOnline
type: boolean
default: true
- name: runTests
type: boolean
default: true
# Freeform field for extra values to pass to build.sh for special build modes
- name: extraProperties
type: string
default: ''
# Use the previous version's SDK to build the current one
- name: withPreviousSDK
type: boolean
default: false
#### INSTALLER parameters ####
- name: isBuiltFromVmr
displayName: True when build is running from dotnet/dotnet directly
type: boolean
- name: vmrBranch
displayName: dotnet/dotnet branch to use
type: string
jobs:
- job: ${{ parameters.buildName }}_${{ parameters.architecture }}
pool: ${{ parameters.pool }}
# Currently, CodeQL slows the build down too much
# https://github.com/dotnet/source-build/issues/4276
${{ if and(parameters.isBuiltFromVmr, startswith(parameters.buildName, 'Windows'), eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
timeoutInMinutes: 720
${{ else }}:
timeoutInMinutes: 240
${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}:
# Always attempt to run the bootstrap leg (e.g. even when stage 1 tests fail) in order to get a complete accessment of the build status.
# The build shortcuts when stage 1 build fails and doesn't produce the SDK.
condition: succeededOrFailed()
dependsOn: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}
variables:
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
- group: AzureDevOps-Artifact-Feeds-Pats
- ${{ if and(not(parameters.isBuiltFromVmr), eq(variables['System.TeamProject'], 'internal'), not(startswith(parameters.vmrBranch, 'internal/release/')), ne(variables['Build.Reason'], 'PullRequest')) }}:
- group: DotNetBot-GitHub
- ${{ else }}:
- name: BotAccount-dotnet-bot-repo-PAT
value: N/A
- name: additionalBuildArgs
value: ''
- ${{ if parameters.isBuiltFromVmr }}:
- name: vmrPath
value: $(Build.SourcesDirectory)
- ${{ else }}:
- name: vmrPath
value: $(Agent.BuildDirectory)/vmr
# Location of the VMR sources
# We either build the repo directly, or we extract them outside (which is what partners do)
- ${{ if parameters.buildFromArchive }}:
- name: sourcesPath
value: $(Build.StagingDirectory)/dotnet-sources/
- ${{ else }}:
- name: sourcesPath
value: $(vmrPath)
templateContext:
outputs:
- output: pipelineArtifact
displayName: Publish BuildLogs
condition: succeededOrFailed()
targetPath: $(Build.StagingDirectory)/BuildLogs
artifactName: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
- output: pipelineArtifact
path: $(Build.ArtifactStagingDirectory)/publishing
artifact: $(Agent.JobName)_Artifacts
displayName: Publish Artifacts
condition: always()
- ${{ if not(parameters.isBuiltFromVmr) }}:
- output: pipelineArtifact
displayName: Upload failed patches
condition: failed()
targetPath: $(Agent.TempDirectory)
artifactName: $(System.JobDisplayName)_FailedPatches
steps:
- ${{ if not(parameters.isBuiltFromVmr) }}:
- template: ../steps/vmr-prepare.yml@self
parameters:
${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
vmrBranch: $(System.PullRequest.TargetBranch)
${{ else }}:
vmrBranch: ${{ parameters.vmrBranch }}
# Synchronize new content in the VMR during PRs (we expect this to come
- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
- template: ../steps/vmr-pull-updates.yml@self
parameters:
vmrPath: $(vmrPath)
vmrBranch: ${{ parameters.vmrBranch }}
targetRef: $(Build.SourceVersion) # Synchronize the current installer commit
- ${{ if parameters.buildFromArchive }}:
- script: |
set -ex
cp -r "$(vmrPath)" "$(sourcesPath)"
rm -rf "$(sourcesPath)/.git"
displayName: Export VMR sources
workingDirectory: $(Build.StagingDirectory)
- ${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}:
- download: current
artifact: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts
patterns: |
**/Private.SourceBuilt.Artifacts.*.tar.gz
**/dotnet-sdk-*.tar.gz
displayName: Download Previous Build
- task: CopyFiles@2
displayName: Copy Previous Build
inputs:
SourceFolder: $(Pipeline.Workspace)/${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts/assets/Release
Contents: '*.tar.gz'
TargetFolder: $(sourcesPath)/prereqs/packages/archive/
- ${{ if eq(parameters.withPreviousSDK, 'true') }}:
- script: |
set -euo pipefail
if [[ '${{ parameters.artifactsRid }}' == '' ]]; then
echo "'artifactsRid' is not specified. Cannot download source-built SDK."
exit 1
fi
packageVersionsPath="$(sourcesPath)/eng/Versions.props"
notFoundMessage="No source-built SDK found to download..."
echo "Looking for source-built SDK to download..."
archiveVersionLine=$(grep -m 1 "<PrivateSourceBuiltSdkVersion>" "$packageVersionsPath" || :)
versionPattern="<PrivateSourceBuiltSdkVersion>(.*)</PrivateSourceBuiltSdkVersion>"
if [[ $archiveVersionLine =~ $versionPattern ]]; then
archiveVersion="${BASH_REMATCH[1]}"
archiveUrl="https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-$archiveVersion-${{ parameters.artifactsRid }}.tar.gz"
downloadDir="$(sourcesPath)/prereqs/packages/archive/"
echo "Downloading source-built SDK from $archiveUrl..."
(cd "$downloadDir" && curl --retry 5 -O "$archiveUrl")
else
echo "$notFoundMessage"
exit 1
fi
displayName: Setup Previously Source-Built SDK
- ${{ if eq(parameters.targetOS, 'windows') }}:
# Node 20.x is a toolset dependency to build aspnetcore
# Keep in sync with aspnetcore: https://github.com/dotnet/aspnetcore/blob/7d5309210d8f7bae8fa074da495e9d009d67f1b4/.azure/pipelines/ci.yml#L719-L722
- task: NodeTool@0
displayName: Install Node 20.x
inputs:
versionSpec: 20.x
- script: |
call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
displayName: Build
env:
${{ if eq(parameters.useDevVersions, 'True') }}:
devArgument: -dev
${{ else }}:
devArgument: ''
- ${{ if eq(parameters.runTests, 'True') }}:
- script: |
call $(sourcesPath)\build.cmd -ci -prepareMachine -test /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
displayName: Run Tests
- ${{ else }}:
- ${{ if eq(parameters.buildSourceOnly, 'true') }}:
- script: |
set -ex
customPrepArgs=""
prepSdk=true
if [[ -n '${{ parameters.artifactsRid }}' ]]; then
customPrepArgs="${customPrepArgs} --artifacts-rid ${{ parameters.artifactsRid }}"
fi
if [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then
# Source-built artifacts are from CentOS 8 Stream or Alpine 3.19. We want to download them without
# downloading portable versions from the internet.
customPrepArgs="${customPrepArgs} --no-sdk --no-bootstrap"
prepSdk=false
elif [[ -n '${{ parameters.reuseBuildArtifactsFrom }}' ]]; then
customPrepArgs="${customPrepArgs} --no-sdk --no-artifacts"
prepSdk=false
fi
if [[ "$prepSdk" == "false" ]]; then
mkdir $(sourcesPath)/.dotnet
previousSdkPath="$(sourcesPath)/prereqs/packages/archive/dotnet-sdk-*.tar.gz"
eval tar -ozxf "$previousSdkPath" -C "$(sourcesPath)/.dotnet"
eval rm -f "$previousSdkPath"
echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /vmr/.dotnet"
fi
docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep-source-build.sh $customPrepArgs
displayName: Prep the Build
- script: |
set -ex
df -h
customEnvVars=""
customBuildArgs="--ci --clean-while-building --prepareMachine"
if [[ '${{ parameters.runOnline }}' == 'True' ]]; then
customBuildArgs="$customBuildArgs --online"
fi
if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
customBuildArgs="$customBuildArgs --poison"
fi
if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then
customBuildArgs="$customBuildArgs --source-repository https://github.com/dotnet/dotnet"
customBuildArgs="$customBuildArgs --source-version $(git -C "$(vmrPath)" rev-parse HEAD)"
fi
if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
customBuildArgs="$customBuildArgs --source-only"
fi
if [[ '${{ parameters.useMonoRuntime }}' == 'True' ]]; then
customBuildArgs="$customBuildArgs --use-mono-runtime"
fi
if [[ '${{ parameters.useDevVersions }}' == 'True' ]]; then
customBuildArgs="$customBuildArgs --dev"
fi
if [[ -n "${{ parameters.crossRootFs }}" ]]; then
customEnvVars="$customEnvVars ROOTFS_DIR=${{ parameters.crossRootFs}}"
if [[ '${{ parameters.targetArchitecture }}' != 'wasm' ]]; then
extraBuildProperties="$extraBuildProperties /p:CrossBuild=true"
fi
fi
if [[ ! -z '${{ parameters.targetOS }}' ]]; then
extraBuildProperties="$extraBuildProperties /p:TargetOS=${{ parameters.targetOS }}"
fi
if [[ ! -z '${{ parameters.targetArchitecture }}' ]]; then
extraBuildProperties="$extraBuildProperties /p:TargetArchitecture=${{ parameters.targetArchitecture }}"
fi
if [[ -n "${{ parameters.extraProperties }}" ]]; then
extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}"
fi
buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties"
# Only use Docker when a container is specified
if [[ -n "${{ parameters.container }}" ]]; then
# Allows Arcade to have access to the commit for the build, pass it through to the container
customEnvVars="$customEnvVars BUILD_SOURCEVERSION=$BUILD_SOURCEVERSION"
customEnvVars="$customEnvVars BUILD_BUILDNUMBER=$(Build.BuildNumber)"
customDockerRunArgs=""
for envVar in $customEnvVars; do
customDockerRunArgs="$customDockerRunArgs -e $envVar"
done
if [[ '${{ parameters.runOnline }}' == 'False' ]]; then
customDockerRunArgs="$customDockerRunArgs --network none"
fi
docker run --rm -v "$(sourcesPath):/vmr" -w /vmr $customDockerRunArgs ${{ parameters.container }} ./build.sh $buildArgs
else
for envVar in $customEnvVars; do
customEnvVarsWithBashSyntax="$customEnvVarsWithBashSyntax export $envVar;"
done
cd $(sourcesPath)
eval $customEnvVarsWithBashSyntax
./build.sh $buildArgs
fi
displayName: Build
# Only run tests if enabled
# TODO: Remove the PullRequest condition after the next re-bootstrap: https://github.com/dotnet/source-build/issues/4362
- ${{ if eq(parameters.runTests, 'True') }}:
- script: |
set -ex
dockerVolumeArgs="-v $(sourcesPath):/vmr"
customBuildArgs=''
extraBuildProperties=''
if [[ ! -z '${{ parameters.targetOS }}' ]]; then
extraBuildProperties="$extraBuildProperties /p:TargetOS=${{ parameters.targetOS }}"
fi
if [[ ! -z '${{ parameters.targetArchitecture }}' ]]; then
extraBuildProperties="$extraBuildProperties /p:TargetArchitecture=${{ parameters.targetArchitecture }}"
fi
if [[ '${{ parameters.useDevVersions }}' == 'True' ]]; then
customBuildArgs="$customBuildArgs --dev"
fi
if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
customBuildArgs="$customBuildArgs --poison"
fi
customBuildArgs="$customBuildArgs --source-only /p:SmokeTestsWarnOnSdkContentDiffs=true /p:SmokeTestsExcludeOmniSharpTests=${{ parameters.excludeOmniSharpTests }}"
fi
if [[ -n "${{ parameters.extraProperties }}" ]]; then
extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}"
fi
# Only use Docker when a container is specified
if [[ -n "${{ parameters.container }}" ]]; then
docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh /bl:artifacts/log/Release/Test.binlog --test $customBuildArgs $extraBuildProperties $(additionalBuildArgs)
else
cd $(sourcesPath)
./build.sh /bl:artifacts/log/Release/Test.binlog --test $customBuildArgs $extraBuildProperties $(additionalBuildArgs)
fi
displayName: Run Tests
- ${{ if eq(parameters.targetOS, 'windows') }}:
# Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph.
- powershell: |
function CopyWithRelativeFolders($sourcePath, $targetFolder, $filter) {
Get-ChildItem -Path $sourcePath -Filter $filter -Recurse | ForEach-Object {
$targetPath = Join-Path $targetFolder (Resolve-Path -Relative $_.FullName)
New-Item -ItemType Directory -Path (Split-Path -Parent $targetPath) -Force | Out-Null
Copy-Item $_.FullName -Destination $targetPath -Force
}
}
$targetFolder = "$(Build.StagingDirectory)/BuildLogs/"
New-Item -ItemType Directory -Path $targetFolder -Force | Out-Null
cd "$(sourcesPath)"
CopyWithRelativeFolders "artifacts/log/" $targetFolder "*.binlog"
CopyWithRelativeFolders "artifacts/log/" $targetFolder "*.log"
CopyWithRelativeFolders "src/" $targetFolder "*.binlog"
CopyWithRelativeFolders "src/" $targetFolder "*.log"
if (Test-Path "artifacts/scenario-tests/") {
CopyWithRelativeFolders "artifacts/scenario-tests/" $targetFolder "*.binlog"
}
if (Test-Path "artifacts/TestResults/*") {
CopyWithRelativeFolders "artifacts/TestResults/" $targetFolder "*.binlog"
CopyWithRelativeFolders "artifacts/TestResults/" $targetFolder "*.diff"
CopyWithRelativeFolders "artifacts/TestResults/" $targetFolder "Updated*.txt"
CopyWithRelativeFolders "artifacts/TestResults/" $targetFolder "*.trx"
}
# check if we have assets to publish
if (Test-Path "artifacts/assets/Release/*") {
echo "##vso[task.setvariable variable=hasAssets]true"
}
displayName: Prepare BuildLogs staging directory and check assets
continueOnError: true
condition: succeededOrFailed()
- ${{ else }}:
# Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph.
- script: |
set -x
targetFolder=$(Build.StagingDirectory)/BuildLogs/
mkdir -p ${targetFolder}
cd "$(sourcesPath)"
find artifacts/log/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
find artifacts/log/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
[ -d "artifacts/scenario-tests/" ] && find artifacts/scenario-tests/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
find artifacts/TestResults/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
find artifacts/TestResults/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
find artifacts/TestResults/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \;
find artifacts/TestResults/ -type f -name "*.trx" -exec rsync -R {} -t ${targetFolder} \;
if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then
find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
find artifacts/log/binary-report/ -exec rsync -R {} -t ${targetFolder} \;
fi
find src/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
find src/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
# check if we have assets to publish
if [ -n "$(ls -A 'artifacts/assets/Release/')" ]; then
echo "##vso[task.setvariable variable=hasAssets]true"
fi
displayName: Prepare BuildLogs staging directory and check assets
continueOnError: true
condition: succeededOrFailed()
- ${{ if or(ne(variables['System.TeamProject'], 'internal'), eq(variables['Build.Reason'], 'PullRequest')) }}:
- publish: $(Build.StagingDirectory)/BuildLogs
artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
displayName: Publish BuildLogs
continueOnError: true
condition: always()
# Only upload test results if enabled
- ${{ if eq(parameters.runTests, 'True') }}:
- task: PublishTestResults@2
displayName: Publish Test Results
condition: succeededOrFailed()
continueOnError: true
inputs:
testRunner: VSTest
testResultsFiles: 'artifacts/TestResults/Release/*.trx'
searchFolder: $(sourcesPath)
mergeTestResults: true
publishRunAttachments: true
testRunTitle: Tests_$(Agent.JobName)
- task: PublishTestResults@2
displayName: Publish Scenario Test Results
condition: succeededOrFailed()
continueOnError: true
inputs:
testRunner: xUnit
testResultsFiles: 'artifacts/TestResults/**/scenario-tests/*.xml'
searchFolder: $(sourcesPath)
mergeTestResults: true
publishRunAttachments: true
testRunTitle: ScenarioTests_$(Agent.JobName)
- task: CopyFiles@2
inputs:
SourceFolder: $(sourcesPath)/artifacts
Contents: |
VerticalManifest.xml
assets/**
TargetFolder: $(Build.ArtifactStagingDirectory)/publishing
displayName: Copy artifacts to Artifact Staging Directory
condition: succeededOrFailed()
# When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages
- ${{ if ne(parameters.buildSourceOnly, 'true') }}:
- task: CopyFiles@2
inputs:
SourceFolder: $(sourcesPath)/artifacts/packages
TargetFolder: $(Build.ArtifactStagingDirectory)/publishing/packages
displayName: Copy packages to Artifact Staging Directory
condition: succeededOrFailed()
- ${{ if or(ne(variables['System.TeamProject'], 'internal'), eq(variables['Build.Reason'], 'PullRequest')) }}:
- publish: $(Build.ArtifactStagingDirectory)/publishing
artifact: $(Agent.JobName)_Artifacts
displayName: Publish Artifacts
condition: succeededOrFailed()
continueOnError: true

89
eng/pipelines/templates/jobs/vmr-synchronization.yml
View file

@ -1,89 +0,0 @@
### This job synchronizes code from product repositories into the VMR (https://github.com/dotnet/dotnet)
### It synchronizes the content of the VMR to this new commit and pushes the changes
parameters:
- name: targetRef
displayName: Target revision of dotnet/installer to synchronize
type: string
default: $(Build.SourceVersion)
- name: vmrBranch
displayName: dotnet/dotnet branch to use
type: string
default: $(Build.SourceBranch)
- name: noPush
displayName: Don't push changes to dotnet/dotnet
type: boolean
default: false
jobs:
- job: Synchronize_VMR
displayName: Synchronize VMR's ${{ parameters.vmrBranch }}
timeoutInMinutes: 120
variables:
- ${{ if eq(variables['System.TeamProject'], 'public') }}:
- template: /eng/common/templates/variables/pool-providers.yml
- ${{ else }}:
- template: /eng/common/templates-official/variables/pool-providers.yml
- name: vmrPath
value: $(Agent.BuildDirectory)/vmr
- ${{ if not(parameters.noPush) }}:
- ${{ if and( eq(variables['System.TeamProject'], 'internal'), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
- group: DotNetBot-GitHub
- group: DotNetBot-GitHub-No-Scopes
- name: vmrPublicUrl
value: https://github.com/dotnet/dotnet
- ${{ if and( eq(variables['System.TeamProject'], 'internal'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/')) }}:
# https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=172&path=DotNetBot-AzDO-PAT
- group: DotNetBot-AzDO-PAT
- name: vmrInternalUrl
value: https://dnceng@dev.azure.com/dnceng/internal/_git/dotnet-dotnet
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $(DncEngPublicBuildPool)
image: 1es-ubuntu-2004-open
os: linux
${{ else }}:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals 1es-ubuntu-2004
os: linux
steps:
- template: ../steps/vmr-prepare.yml
parameters:
vmrBranch: ${{ parameters.vmrBranch }}
- template: ../steps/vmr-pull-updates.yml
parameters:
vmrPath: $(vmrPath)
vmrBranch: ${{ parameters.vmrBranch }}
targetRef: ${{ parameters.targetRef }}
- ${{ if and(not(parameters.noPush), not(in(variables['Build.Reason'], 'PullRequest')), eq(variables['System.TeamProject'], 'internal')) }}:
# Push main and release branches to the public VMR
- ${{ if or(eq(parameters.vmrBranch, 'main'), startsWith(parameters.vmrBranch, 'release/')) }}:
- script: >
./.dotnet/dotnet darc vmr push
--vmr '$(vmrPath)'
--commit-verification-pat '$(BotAccount-dotnet-maestro-bot-no-scopes-PAT)'
--branch '${{ parameters.vmrBranch }}'
--remote-url '$(vmrPublicUrl)'
--github-pat '$(BotAccount-dotnet-bot-repo-PAT)'
--verbose
displayName: Push changes to dotnet/dotnet (public)
workingDirectory: $(Agent.BuildDirectory)/installer
# Push internal/release branches to the internal VMR
- ${{ if startsWith(parameters.vmrBranch, 'internal/release/') }}:
- script: >
./.dotnet/dotnet darc vmr push
--vmr '$(vmrPath)'
--skip-commit-verification
--branch '${{ parameters.vmrBranch }}'
--remote-url '$(vmrInternalUrl)'
--azdev-pat '$(dn-bot-dnceng-build-rw-code-rw)'
--verbose
displayName: Push changes to dotnet-dotnet (internal)
workingDirectory: $(Agent.BuildDirectory)/installer

1037
eng/pipelines/templates/stages/vmr-build.yml
View file

File diff suppressed because it is too large Load diff

18
eng/pipelines/templates/steps/vmr-prepare.yml
View file

@ -1,18 +0,0 @@
### These steps clone the VMR (https://github.com/dotnet/dotnet) into $(Agent.BuildDirectory)/vmr for installer
parameters:
- name: vmrBranch
displayName: dotnet/dotnet branch to use
type: string
steps:
- checkout: vmr
displayName: Clone dotnet/dotnet
path: vmr
clean: true
- script: |
git checkout --track origin/${{ parameters.vmrBranch }}
echo "##vso[task.setvariable variable=vmrBranch]${{ parameters.vmrBranch }}"
displayName: Check out ${{ parameters.vmrBranch }}
workingDirectory: $(Agent.BuildDirectory)/vmr

99
eng/pipelines/templates/steps/vmr-pull-updates.yml
View file

@ -1,99 +0,0 @@
### These steps synchronize new code from product repositories into the VMR (https://github.com/dotnet/dotnet).
### They initialize the darc CLI and pull the new updates.
### Changes are applied locally onto the already cloned VMR (located in $vmrPath).
parameters:
- name: vmrBranch
displayName: dotnet/dotnet branch to use
type: string
- name: targetRef
displayName: Target revision in dotnet/installer to synchronize
type: string
default: $(Build.SourceVersion)
- name: vmrPath
displayName: Path where the dotnet/dotnet is checked out to
type: string
default: $(Agent.BuildDirectory)/vmr
steps:
- checkout: self
displayName: Clone dotnet/installer
path: installer
# This step is needed so that when we get a detached HEAD / shallow clone,
# we still pull the commit into the temporary installer clone to use it during the sync.
- script: |
git branch installer-head
git rev-parse HEAD
displayName: Label PR commit
workingDirectory: $(Agent.BuildDirectory)/installer
- script: |
git checkout -B ${{ parameters.vmrBranch }}
echo "##vso[task.setvariable variable=vmrBranch]${{ parameters.vmrBranch }}"
displayName: Prepare branch ${{ parameters.vmrBranch }}
workingDirectory: ${{ parameters.vmrPath }}
- script: |
git config --global user.name "dotnet-maestro[bot]"
git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com"
displayName: Set git author to dotnet-maestro[bot]
workingDirectory: ${{ parameters.vmrPath }}
- script: |
echo '**/*' > .artifactignore
echo '!./*.patch' >> .artifactignore
displayName: Prepare .artifactignore
workingDirectory: $(Agent.TempDirectory)
- script: >
./eng/vmr-sync.sh
--vmr ${{ parameters.vmrPath }}
--tmp $(Agent.TempDirectory)
--azdev-pat '$(dn-bot-all-orgs-code-r)'
--branch ${{ parameters.vmrBranch }}
--repository "installer:${{ parameters.targetRef }}"
--recursive
--remote "installer:$(Agent.BuildDirectory)/installer"
--component-template $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/Component.template.md
--tpn-template $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt
--debug
||
(echo "##vso[task.logissue type=error]Failed to synchronize the VMR" && exit 1)
displayName: Synchronize dotnet/dotnet (Unix)
condition: ne(variables['Agent.OS'], 'Windows_NT')
workingDirectory: $(Agent.BuildDirectory)/installer
- script: git config --global diff.astextplain.textconv echo
displayName: Disable astextplain in git diff (Windows)
condition: eq(variables['Agent.OS'], 'Windows_NT')
- powershell: >
./eng/vmr-sync.ps1 `
-vmr ${{ parameters.vmrPath }} `
-tmp $(Agent.TempDirectory) `
-azdevPat '$(dn-bot-all-orgs-code-r)' `
-branch ${{ parameters.vmrBranch }} `
-repository "installer:${{ parameters.targetRef }}" `
-recursive `
# passing remote fails for some reason, but it is the default anyway
# -remote "installer:$(Agent.BuildDirectory)/installer"
-componentTemplate $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/Component.template.md `
-tpnTemplate $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt `
-debugOutput
if ($LASTEXITCODE -ne 0) {
echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
exit 1
}
displayName: Synchronize dotnet/dotnet (Windows)
condition: eq(variables['Agent.OS'], 'Windows_NT')
workingDirectory: $(Agent.BuildDirectory)/installer
- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
- publish: $(Agent.TempDirectory)
artifact: $(System.JobDisplayName)_FailedPatches
displayName: Upload failed patches
condition: failed()

71
eng/pipelines/templates/variables/vmr-build.yml
View file

@ -1,71 +0,0 @@
parameters:
- name: vmrBranch
type: string
variables:
- ${{ if ne(parameters.vmrBranch, '') }}:
- name: VmrBranch
value: ${{ parameters.vmrBranch }}
- ${{ else }}:
- name: VmrBranch
value: ${{ replace(replace(variables['Build.SourceBranch'], 'refs/heads/', ''), 'refs/pull/', '') }}
- name: alpine319Container
value: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode
- name: centOSStream9Container
value: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9
- name: fedora39Container
value: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39
- name: ubuntu2204Container
value: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04
- name: ubuntu2204ArmContainer
value: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-arm64
- name: marinerX64CrossContainer
value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64
- name: marinerArmCrossContainer
value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm
- name: marinerArm64CrossContainer
value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64
- name: marinerX64AlpineCrossContainer
value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine
- name: marinerArmAlpineCrossContainer
value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine
- name: marinerArm64AlpineCrossContainer
value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine
- name: androidCrossContainer
value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64
- name: browserCrossContainer
value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230917141449-2aaa02c
- name: wasiCrossContainer
value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly
- ${{ if eq(variables['System.TeamProject'], 'public') }}:
- name: defaultPoolName
value: NetCore-Public-XL
- name: poolImage_Linux
value: 1es-ubuntu-2004-open
- name: poolImage_LinuxArm64
value: Mariner-2-Docker-ARM64
- name: poolName_LinuxArm64
value: Docker-Linux-Arm-Public
- name: poolImage_Mac
value: macos-12
- name: poolImage_Windows
value: windows.vs2022preview.amd64.open
- ${{ else }}:
- ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
- name: defaultPoolName
value: NetCore1ESPool-Internal-XL
- ${{ else }}:
- name: defaultPoolName
value: $(DncEngInternalBuildPool)
- name: poolImage_Linux
value: 1es-ubuntu-2204
- name: poolImage_LinuxArm64
value: Mariner-2-Docker-ARM64
- name: poolName_LinuxArm64
value: Docker-Linux-Arm-Internal
- name: poolImage_Mac
value: macos-13-arm64
- name: poolImage_Windows
value: windows.vs2022preview.amd64

79
eng/pipelines/vmr-build-pr.yml
View file

@ -1,79 +0,0 @@
# This YAML is used by these PR pipelines:
#
# - installer-source-build
# https://dev.azure.com/dnceng-public/public/_build?definitionId=233
# - installer-unified-build
# https://dev.azure.com/dnceng-public/public/_build?definitionId=277
# - installer-unified-build-full
# https://dev.azure.com/dnceng-public/public/_build?definitionId=279
trigger: none
pr:
branches:
include:
- main
- release/*
exclude:
- release/*.0.2xx
- release/*.0.3xx
- release/*.0.4xx
parameters:
- name: vmrBranch
displayName: dotnet/dotnet branch to push to
type: string
default: ' '
- name: disableBuild
displayName: Skip the VMR Build stage
type: boolean
default: false
variables:
- template: /eng/common/templates/variables/pool-providers.yml@self
- ${{ if ne(parameters.vmrBranch, ' ') }}:
- name: VmrBranch
value: ${{ replace(parameters.vmrBranch, ' ', '') }}
- ${{ else }}:
- name: VmrBranch
value: ${{ replace(replace(variables['System.PullRequest.TargetBranch'], 'refs/heads/', ''), 'refs/pull/', '') }}
# enable source-only build for pipelines with the -source-build suffix
- name: isSourceOnlyBuild
value: ${{ contains(variables['Build.DefinitionName'], '-source-build') }}
resources:
repositories:
- repository: vmr
type: github
name: dotnet/dotnet
endpoint: dotnet
ref: ${{ variables.VmrBranch }}
stages:
# You can temporarily disable the VMR Build stage by setting the disableBuild variable
- ${{ if not(parameters.disableBuild) }}:
- template: templates/stages/vmr-build.yml
parameters:
vmrBranch: ${{ variables.VmrBranch }}
isBuiltFromVmr: false
isSourceOnlyBuild: ${{ variables.isSourceOnlyBuild }}
${{ if contains(variables['Build.DefinitionName'], '-full') }}:
scope: full
${{ elseif eq(variables.isSourceOnlyBuild, 'true') }}:
scope: ultralite
${{ else }}:
scope: lite
# In case the VMR Build stage is temporarily disabled, the VMR synchronization step is run to validate
# that the PR can be merged and later synchronized into the VMR without problems.
- ${{ else }}:
- stage: Synchronize_VMR
displayName: Synchronize VMR
dependsOn: []
jobs:
- template: templates/jobs/vmr-synchronization.yml
parameters:
vmrBranch: ${{ variables.VmrBranch }}
noPush: true

64
eng/pipelines/vmr-sync-internal.yml
View file

@ -1,64 +0,0 @@
pr: none
trigger:
batch: true
branches:
include:
- internal/release/*
exclude:
- internal/release/*.0.2xx
- internal/release/*.0.3xx
- internal/release/*.0.4xx
resources:
repositories:
- repository: vmr
type: git
name: dotnet-dotnet
ref: $(Build.SourceBranch)
- repository: 1ESPipelineTemplates
type: git
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release
parameters:
- name: vmrBranch
displayName: dotnet-dotnet branch to push to
type: string
default: ' '
variables:
- template: /eng/common/templates-official/variables/pool-providers.yml@self
- ${{ if ne(parameters.vmrBranch, ' ') }}:
- name: VmrBranch
value: ${{ replace(parameters.vmrBranch, ' ', '') }}
- ${{ else }}:
- name: VmrBranch
value: ${{ replace(replace(variables['Build.SourceBranch'], 'refs/heads/', ''), 'refs/pull/', '') }}
extends:
template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
parameters:
sdl:
sourceAnalysisPool:
name: $(DncEngInternalBuildPool)
image: 1es-windows-2022
os: windows
sourceRepositoriesToScan:
exclude:
- repository: vmr
componentgovernance:
sourceScanPath: $(Agent.BuildDirectory)/vmr
ignoreDirectories: $(Agent.BuildDirectory)/vmr/src
stages:
- stage: VMRSynchronization
displayName: VMR Synchronization
jobs:
- template: /eng/pipelines/templates/jobs/vmr-synchronization.yml@self
parameters:
vmrBranch: ${{ variables.VmrBranch }}

66
eng/pipelines/vmr-sync.yml
View file

@ -1,66 +0,0 @@
pr: none
trigger:
batch: true
branches:
include:
- main
- release/*
exclude:
- release/*.0.2xx
- release/*.0.3xx
- release/*.0.4xx
resources:
repositories:
- repository: vmr
type: github
name: dotnet/dotnet
endpoint: dotnet
ref: $(Build.SourceBranch)
- repository: 1ESPipelineTemplates
type: git
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release
parameters:
- name: vmrBranch
displayName: dotnet/dotnet branch to push to
type: string
default: ' '
variables:
- template: /eng/common/templates-official/variables/pool-providers.yml@self
- ${{ if ne(parameters.vmrBranch, ' ') }}:
- name: VmrBranch
value: ${{ replace(parameters.vmrBranch, ' ', '') }}
- ${{ else }}:
- name: VmrBranch
value: ${{ replace(replace(variables['Build.SourceBranch'], 'refs/heads/', ''), 'refs/pull/', '') }}
extends:
template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
parameters:
sdl:
sourceAnalysisPool:
name: $(DncEngInternalBuildPool)
image: 1es-windows-2022
os: windows
sourceRepositoriesToScan:
exclude:
- repository: vmr
componentgovernance:
sourceScanPath: $(Agent.BuildDirectory)/vmr
ignoreDirectories: $(Agent.BuildDirectory)/vmr/src
stages:
- stage: VMRSynchronization
displayName: VMR Synchronization
jobs:
- template: /eng/pipelines/templates/jobs/vmr-synchronization.yml@self
parameters:
vmrBranch: ${{ variables.VmrBranch }}

6
eng/restore-toolset.ps1
View file

@ -3,12 +3,6 @@ function InitializeCustomSDKToolset {
return
}
# The following frameworks and tools are used only for testing.
# Do not attempt to install them in source build.
if ($productBuild -or $properties -like "*DotNetBuildRepo=true*") {
return
}
# InstallDotNetSharedFramework "1.0.5"
# InstallDotNetSharedFramework "1.1.2"
# InstallDotNetSharedFramework "2.1.0"

6
eng/restore-toolset.sh
View file

@ -3,12 +3,6 @@ function InitializeCustomSDKToolset {
return
fi
# The following frameworks and tools are used only for testing.
# Do not attempt to install them in source build.
if [[ $product_build == true || $properties == *"DotNetBuildRepo=true"* ]]; then
return
fi
#InstallDotNetSharedFramework "1.0.5"
#InstallDotNetSharedFramework "1.1.2"

216
eng/vmr-sync.ps1
View file

@ -1,216 +0,0 @@
<#
.SYNOPSIS
This script is used for synchronizing the dotnet/dotnet VMR locally. This means pulling new
code from various repositories into the 'dotnet/dotnet' repository.
.DESCRIPTION
The script is used during CI to ingest new code based on dotnet/installer but it can also help
for reproducing potential failures during installer's PRs, namely to fix the Source-Build.
Another usecase is to try manually synchronizing a given commit of some repo into the VMR and
trying to Source-Build the VMR. This can help when fixing the Source-Build but using a commit
from a not-yet merged branch (or fork) to test the fix will help.
The tooling that synchronizes the VMR will need to clone the various repositories into a temporary
folder. These clones can be re-used in future synchronizations so it is advised you dedicate a
folder to this to speed up your re-runs.
.EXAMPLE
Synchronize current installer and all dependencies into a local VMR:
./vmr-sync.ps1 -vmrDir "$HOME/repos/dotnet" -tmpDir "$HOME/repos/tmp"
Synchronize the VMR to a specific commit of dotnet/runtime using custom fork:
./vmr-sync.ps1 `
-repository runtime:e7e71da303af8dc97df99b098f21f526398c3943 `
-remote runtime:https://github.com/yourfork/runtime `
-tmpDir "$HOME/repos/tmp"
.PARAMETER tmpDir
Required. Path to the temporary folder where repositories will be cloned
.PARAMETER vmrBranch
Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch
.PARAMETER component-template
Optional. Template for VMRs Component.md used for regenerating the file to list the newest versions of
components.
Defaults to src/VirtualMonoRepo/Component.template.md
.PARAMETER recursive
Optional. Recursively synchronize all the source build dependencies (declared in Version.Details.xml)
This is used when performing the full synchronization during installer's CI and the final VMR sync.
Defaults to false unless no repository is supplied in which case a recursive sync of installer is performed.
.PARAMETER remote
Optional. Additional remote to use during the synchronization
This can be used to synchronize to a commit from a fork of the repository
Example: 'runtime:https://github.com/yourfork/runtime'
.PARAMETER repository
Optional. Repository + git ref separated by colon to synchronize to.
This can be a specific commit, branch, tag.
If not supplied, the revision of the parent installer repository of this script will be used (recursively).
Example: 'runtime:my-branch-name'
.PARAMETER tpnTemplate
Optional. Template for the header of VMRs THIRD-PARTY-NOTICES file.
Defaults to src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt
.PARAMETER azdevPat
Optional. Azure DevOps PAT to use for cloning private repositories.
.PARAMETER vmrDir
Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder
.PARAMETER debugOutput
Optional. Enables debug logging in the darc vmr command.
#>
param (
[Parameter(Mandatory=$true, HelpMessage="Path to the temporary folder where repositories will be cloned")]
[string][Alias('t', 'tmp')]$tmpDir,
[string][Alias('b', 'branch')]$vmrBranch,
[string]$componentTemplate = "src/VirtualMonoRepo/Component.template.md",
[switch]$recursive,
[string]$remote,
[string][Alias('r')]$repository,
[string]$tpnTemplate = "src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt",
[string]$azdevPat,
[string][Alias('v', 'vmr')]$vmrDir,
[switch]$debugOutput
)
$scriptRoot = $PSScriptRoot
function Fail {
Write-Host "> $($args[0])" -ForegroundColor 'Red'
}
function Highlight {
Write-Host "> $($args[0])" -ForegroundColor 'Cyan'
}
$installerDir = (Split-Path -Parent $scriptRoot)
# If installer is a repo, we're in an installer and not in the dotnet/dotnet repo
if (Test-Path -Path "$installerDir/.git" -PathType Container) {
$additionalRemotes = "installer:$installerDir"
}
if ($remote) {
$additionalRemotes = "$additionalRemotes $remote"
}
$verbosity = 'verbose'
if ($debugOutput) {
$verbosity = 'debug'
}
# Validation
if (-not (Test-Path -Path $installerDir -PathType Container)) {
Fail "Directory '$installerDir' does not exist. Please specify the path to the dotnet/installer repo"
exit 1
}
if (-not $tmpDir) {
Fail "Missing -tmpDir argument. Please specify the path to the temporary folder where the repositories will be cloned"
exit 1
}
if (-not (Test-Path -Path $componentTemplate -PathType Leaf)) {
Fail "File '$componentTemplate' does not exist. Please specify a valid path to the component template"
exit 1
}
if (-not (Test-Path -Path $tpnTemplate -PathType Leaf)) {
Fail "File '$tpnTemplate' does not exist. Please specify a valid path to the THIRD-PARTY-NOTICES template"
exit 1
}
# Sanitize the input
# Default when no repository is provided
if (-not $repository) {
$repository = "installer:$(git -C $installerDir rev-parse HEAD)"
$recursive = $true
}
if (-not $vmrDir) {
$vmrDir = Join-Path $tmpDir 'dotnet'
}
if (-not (Test-Path -Path $tmpDir -PathType Container)) {
New-Item -ItemType Directory -Path $tmpDir | Out-Null
}
# Prepare the VMR
if (-not (Test-Path -Path $vmrDir -PathType Container)) {
Highlight "Cloning 'dotnet/dotnet' into $vmrDir.."
git clone https://github.com/dotnet/dotnet $vmrDir
if ($vmrBranch) {
git -C $vmrDir switch -c $vmrBranch
}
}
else {
if ((git -C $vmrDir diff --quiet) -eq $false) {
Fail "There are changes in the working tree of $vmrDir. Please commit or stash your changes"
exit 1
}
if ($vmrBranch) {
Highlight "Preparing $vmrDir"
git -C $vmrDir checkout $vmrBranch
git -C $vmrDir pull
}
}
Set-StrictMode -Version Latest
# Prepare darc
Highlight 'Installing .NET, preparing the tooling..'
. $scriptRoot\common\tools.ps1
$dotnetRoot = InitializeDotNetCli -install:$true
$dotnet = "$dotnetRoot\dotnet.exe"
& "$dotnet" tool restore
Highlight "Starting the synchronization of '$repository'.."
# Synchronize the VMR
$darcArgs = (
"darc", "vmr", "update",
"--vmr", $vmrDir,
"--tmp", $tmpDir,
"--$verbosity",
"--component-template", $componentTemplate,
"--tpn-template", $tpnTemplate,
"--discard-patches",
$repository
)
if ($recursive) {
$darcArgs += ("--recursive")
}
if ($additionalRemotes) {
$darcArgs += ("--additional-remotes", $additionalRemotes)
}
if ($azdevPat) {
$darcArgs += ("--azdev-pat", $azdevPat)
}
& "$dotnet" $darcArgs
if ($LASTEXITCODE -eq 0) {
Highlight "Synchronization succeeded"
}
else {
Fail "Synchronization of dotnet/dotnet to '$repository' failed!"
Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)."
Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)."
Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script."
exit 1
}

284
eng/vmr-sync.sh
View file

@ -1,284 +0,0 @@
#!/bin/bash
### This script is used for synchronizing the dotnet/dotnet VMR locally. This means pulling new
### code from various repositories into the 'dotnet/dotnet' repository.
###
### The script is used during CI to ingest new code based on dotnet/installer but it can also help
### for reproducing potential failures during installer's PRs, namely to fix the Source-Build.
### Another usecase is to try manually synchronizing a given commit of some repo into the VMR and
### trying to Source-Build the VMR. This can help when fixing the Source-Build but using a commit
### from a not-yet merged branch (or fork) to test the fix will help.
###
### The tooling that synchronizes the VMR will need to clone the various repositories into a temporary
### folder. These clones can be re-used in future synchronizations so it is advised you dedicate a
### folder to this to speed up your re-runs.
###
### USAGE:
### Synchronize current installer and all dependencies into a local VMR:
### ./vmr-sync.sh --vmr "$HOME/repos/dotnet" --tmp "$HOME/repos/tmp"
###
### Synchronize the VMR to a specific commit of dotnet/runtime using custom fork:
### ./vmr-sync.sh \
### --repository runtime:e7e71da303af8dc97df99b098f21f526398c3943 \
### --remote runtime:https://github.com/yourfork/runtime \
### --tmp "$HOME/repos/tmp"
###
### Options:
### -t, --tmp, --tmp-dir PATH
### Required. Path to the temporary folder where repositories will be cloned
###
### -b, --branch, --vmr-branch BRANCH_NAME
### Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch
###
### --debug
### Optional. Turns on the most verbose logging for the VMR tooling
###
### --component-template
### Optional. Template for VMRs Component.md used for regenerating the file to list the newest versions of
### components.
### Defaults to src/VirtualMonoRepo/Component.template.md
###
### --recursive
### Optional. Recursively synchronize all the source build dependencies (declared in Version.Details.xml)
### This is used when performing the full synchronization during installer's CI and the final VMR sync.
### Defaults to false unless no repository is supplied in which case a recursive sync of installer is performed.
###
### --remote name:URI
### Optional. Additional remote to use during the synchronization
### This can be used to synchronize to a commit from a fork of the repository
### Example: 'runtime:https://github.com/yourfork/runtime'
###
### -r, --repository name:GIT_REF
### Optional. Repository + git ref separated by colon to synchronize to.
### This can be a specific commit, branch, tag.
### If not supplied, the revision of the parent installer repository of this script will be used (recursively).
### Example: 'runtime:my-branch-name'
###
### --tpn-template
### Optional. Template for the header of VMRs THIRD-PARTY-NOTICES file.
### Defaults to src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt
###
### --azdev-pat
### Optional. Azure DevOps PAT to use for cloning private repositories.
###
### -v, --vmr, --vmr-dir PATH
### Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder
source="${BASH_SOURCE[0]}"
# resolve $source until the file is no longer a symlink
while [[ -h "$source" ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
function print_help () {
sed -n '/^### /,/^$/p' "$source" | cut -b 5-
}
COLOR_RED=$(tput setaf 1 2>/dev/null || true)
COLOR_CYAN=$(tput setaf 6 2>/dev/null || true)
COLOR_CLEAR=$(tput sgr0 2>/dev/null || true)
COLOR_RESET=uniquesearchablestring
FAILURE_PREFIX='> '
function fail () {
echo "${COLOR_RED}$FAILURE_PREFIX${1//${COLOR_RESET}/${COLOR_RED}}${COLOR_CLEAR}" >&2
}
function highlight () {
echo "${COLOR_CYAN}$FAILURE_PREFIX${1//${COLOR_RESET}/${COLOR_CYAN}}${COLOR_CLEAR}"
}
# realpath is not available in macOS 12, try horrible-but-portable workaround
installer_dir=$(cd "$scriptroot/../"; pwd -P)
tmp_dir=''
vmr_dir=''
vmr_branch=''
repository=''
additional_remotes=''
recursive=false
verbosity=verbose
component_template="$installer_dir/src/VirtualMonoRepo/Component.template.md"
tpn_template="$installer_dir/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt"
azdev_pat=''
# If installer is a repo, we're in an installer and not in the dotnet/dotnet repo
if [[ -d "$installer_dir/.git" ]]; then
additional_remotes="installer:$installer_dir"
fi
while [[ $# -gt 0 ]]; do
opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-t|--tmp|--tmp-dir)
tmp_dir=$2
shift
;;
-v|--vmr|--vmr-dir)
vmr_dir=$2
shift
;;
-b|--branch|--vmr-branch)
vmr_branch=$2
shift
;;
-r|--repository)
repository=$2
shift
;;
--recursive)
recursive=true
;;
--remote)
additional_remotes="$additional_remotes $2"
shift
;;
--component-template)
component_template=$2
shift
;;
--tpn-template)
tpn_template=$2
shift
;;
--azdev-pat)
azdev_pat=$2
shift
;;
-d|--debug)
verbosity=debug
;;
-h|--help)
print_help
exit 0
;;
*)
fail "Invalid argument: $1"
print_help
exit 1
;;
esac
shift
done
# Validation
if [[ ! -d "$installer_dir" ]]; then
fail "Directory '$installer_dir' does not exist. Please specify the path to the dotnet/installer repo"
exit 1
fi
if [[ -z "$tmp_dir" ]]; then
fail "Missing --tmp-dir argument. Please specify the path to the temporary folder where the repositories will be cloned"
exit 1
fi
if [[ ! -f "$component_template" ]]; then
fail "File '$component_template' does not exist. Please specify a valid path to the Component.md template"
exit 1
fi
if [[ ! -f "$tpn_template" ]]; then
fail "File '$tpn_template' does not exist. Please specify a valid path to the THIRD-PARTY-NOTICES template"
exit 1
fi
# Sanitize the input
# Default when no repository is provided
if [[ -z "$repository" ]]; then
repository="installer:$(git -C "$installer_dir" rev-parse HEAD)"
recursive=true
fi
if [[ -z "$vmr_dir" ]]; then
vmr_dir="$tmp_dir/dotnet"
fi
if [[ ! -d "$tmp_dir" ]]; then
mkdir -p "$tmp_dir"
fi
if [[ "$verbosity" == "debug" ]]; then
set -x
fi
# Prepare the VMR
if [[ ! -d "$vmr_dir" ]]; then
highlight "Cloning 'dotnet/dotnet' into $vmr_dir.."
git clone https://github.com/dotnet/dotnet "$vmr_dir"
if [[ -n "$vmr_branch" ]]; then
git -C "$vmr_dir" switch -c "$vmr_branch"
fi
else
if ! git -C "$vmr_dir" diff --quiet; then
fail "There are changes in the working tree of $vmr_dir. Please commit or stash your changes"
exit 1
fi
if [[ -n "$vmr_branch" ]]; then
highlight "Preparing $vmr_dir"
git -C "$vmr_dir" checkout "$vmr_branch"
git -C "$vmr_dir" pull
fi
fi
set -e
# Prepare darc
highlight 'Installing .NET, preparing the tooling..'
source "$scriptroot/common/tools.sh"
InitializeDotNetCli true
dotnetDir=$( cd $scriptroot/../.dotnet/; pwd -P )
dotnet=$dotnetDir/dotnet
"$dotnet" tool restore
highlight "Starting the synchronization of '$repository'.."
set +e
recursive_arg=''
if [[ "$recursive" == "true" ]]; then
recursive_arg="--recursive"
fi
if [[ -n "$additional_remotes" ]]; then
additional_remotes="--additional-remotes $additional_remotes"
fi
if [[ -n "$azdev_pat" ]]; then
azdev_pat="--azdev-pat $azdev_pat"
fi
# Synchronize the VMR
"$dotnet" darc vmr update \
--vmr "$vmr_dir" \
--tmp "$tmp_dir" \
$azdev_pat \
--$verbosity \
$recursive_arg \
$additional_remotes \
--component-template "$component_template" \
--tpn-template "$tpn_template" \
--discard-patches \
"$repository"
if [[ $? == 0 ]]; then
highlight "Synchronization succeeded"
else
fail "Synchronization of dotnet/dotnet to '$repository' failed!"
fail "'$vmr_dir' is left in its last state (re-run of this script will reset it)."
fail "Please inspect the logs which contain path to the failing patch file (use --debug to get all the details)."
fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script."
exit 1
fi

10
source-build.slnf
View file

@ -1,10 +0,0 @@
{
"solution": {
"path": "Microsoft.DotNet.Cli.sln",
"projects": [
"src\\Microsoft.Dotnet.Sdk.Internal\\Microsoft.Dotnet.Sdk.Internal.csproj",
"src\\core-sdk-tasks\\core-sdk-tasks.csproj",
"src\\redist\\redist.csproj",
]
}
}

18
src/SourceBuild/README.md
View file

@ -1,18 +0,0 @@
# Source-Build
This directory contains the .NET source build infrastructure.
_content_ - source build infrastructure mirrored to [dotnet/dotnet](https://github.com/dotnet/dotnet)
[VMR](https://github.com/dotnet/arcade/blob/main/Documentation/UnifiedBuild/VMR-Design-And-Operation.md).
_patches_ - repo patches needed for .NET source build. Typically these are ephemeral to workaround integration
issues. For more information, see the [Patch Guidelines](https://github.com/dotnet/source-build/blob/main/Documentation/patching-guidelines.md).
For more information, see [dotnet/source-build](https://github.com/dotnet/source-build).
## Local development workflow
When making changes to the source build infrastructure, devs would typically make and test the
changes in a local clone of [dotnet/dotnet](https://github.com/dotnet/dotnet). Once complete
you would copy the changed files here and make a PR. To validate the end to end experience, you
can synchronize the VMR with any changes made here by running [eng/vmr-sync.sh](https://github.com/dotnet/installer/blob/main/eng/vmr-sync.sh).

21
src/SourceBuild/content/.config/1espt/PipelineAutobaseliningConfig.yml
View file

@ -1,21 +0,0 @@
## DO NOT MODIFY THIS FILE MANUALLY. This is part of auto-baselining from 1ES Pipeline Templates. Go to [https://aka.ms/1espt-autobaselining] for more details.
pipelines:
1330:
retail:
source:
credscan:
lastModifiedDate: 2024-03-25
eslint:
lastModifiedDate: 2024-03-25
psscriptanalyzer:
lastModifiedDate: 2024-03-25
armory:
lastModifiedDate: 2024-03-25
binary:
credscan:
lastModifiedDate: 2024-03-25
binskim:
lastModifiedDate: 2024-03-25
spotbugs:
lastModifiedDate: 2024-03-25

1479
src/SourceBuild/content/.config/guardian/.gdnbaselines
View file

File diff suppressed because it is too large Load diff

59
src/SourceBuild/content/.devcontainer/README.md
View file

@ -1,59 +0,0 @@
<!--
######## ######## ### ######## ######## ## ## #### ######
## ## ## ## ## ## ## ## ## ## ## ## ##
## ## ## ## ## ## ## ## ## ## ## ##
######## ###### ## ## ## ## ## ######### ## ######
## ## ## ######### ## ## ## ## ## ## ##
## ## ## ## ## ## ## ## ## ## ## ## ##
## ## ######## ## ## ######## ## ## ## #### ######
-->
This Codespace can help you debug the source build of .NET. This build takes about
45 minutes and, after completion, produces an archived .NET SDK located in
`/workspaces/dotnet/artifacts/assets/Release`. In case you selected the `prebuilt-sdk`
Codespace, the built-from-source SDK will already be there.
## Build the SDK
To build the repository, run one of the following:
```bash
# Microsoft based build
./build.sh
```
or
```bash
# Building from source only
./prep-source-build.sh && ./build.sh -sb
```
> Please note that, at this time, the build modifies some of the checked-in sources so it might
be preferential to rebuild the Codespace between attempts (or reset the working tree changes).
For more details, see the instructions at https://github.com/dotnet/dotnet.
## Synchronize your changes in locally
When debugging the build, you have two options how to test your changes in this environment.
### Making changes to the VMR directly
You can make the changes directly to the local checkout of the VMR at `/workspaces/dotnet`. You
can then try to build the VMR and see if the change works for you.
### Pull changes into the Codespace from your fork
You can also make a fix in the individual source repository (e.g. `dotnet/runtime`) and push the
fix into a branch; can be in your fork too. Once you have the commit pushed, you can pull this
version of the repository into the Codespace by running:
```
/workspaces/synchronize-vmr.sh \
--repository <repo>:<commit, tag or branch> \
--remote <repo>:<fork URI>
```
You can now proceed building the VMR in the Codespace using instructions above. You can repeat
this process and sync a new commit from your fork. Only note that, at this time, Source-Build
modifies some of the checked-in sources so you'll need to revert the working tree changes
between attempts.

22
src/SourceBuild/content/.devcontainer/devcontainer.json
View file

@ -1,22 +0,0 @@
// Container contains checked-out source code only
{
"name": "Default",
"image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39",
"hostRequirements": {
// A completely source built .NET is >64 GB with all the repos/artifacts
"storage": "128gb"
},
"customizations": {
"vscode": {
"extensions": [
"ms-dotnettools.csharp"
]
},
"codespaces": {
"openFiles": [
".devcontainer/README.md"
]
}
},
"onCreateCommand": ".devcontainer/init.sh"
}

14
src/SourceBuild/content/.devcontainer/init.sh
View file

@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -eux
source="${BASH_SOURCE[0]}"
script_root="$( cd -P "$( dirname "$source" )" && pwd )"
workspace_dir=$(realpath "$script_root/../../")
tmp_dir=$(realpath "$workspace_dir/tmp")
vmr_dir=$(realpath "$workspace_dir/dotnet")
cp "$vmr_dir/.devcontainer/synchronize-vmr.sh" "$workspace_dir"
mkdir -p "$tmp_dir"

22
src/SourceBuild/content/.devcontainer/prebuilt-sdk/devcontainer.json
View file

@ -1,22 +0,0 @@
// Container contains a pre-built SDK
{
"name": "Pre-built .NET SDK",
"image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39",
"hostRequirements": {
// A completely source built .NET is >64 GB with all the repos/artifacts
"storage": "128gb"
},
"customizations": {
"vscode": {
"extensions": [
"ms-dotnettools.csharp"
]
},
"codespaces": {
"openFiles": [
".devcontainer/README.md"
]
}
},
"onCreateCommand": ".devcontainer/prebuilt-sdk/init.sh"
}

9
src/SourceBuild/content/.devcontainer/prebuilt-sdk/init.sh
View file

@ -1,9 +0,0 @@
#!/usr/bin/env bash
source="${BASH_SOURCE[0]}"
script_root="$( cd -P "$( dirname "$source" )" && pwd )"
"$script_root"/../../prep-source-build.sh
cp "$script_root/../synchronize-vmr.sh" "/workspaces/"
"$script_root"/../../build.sh --online --clean-while-building || exit 0

4
src/SourceBuild/content/.devcontainer/synchronize-vmr.sh
View file

@ -1,4 +0,0 @@
#!/bin/bash
(cd /workspaces/dotnet/src/installer \
&& ./eng/vmr-sync.sh --vmr /workspaces/dotnet --tmp /workspaces/tmp $*)

6
src/SourceBuild/content/.gitignore vendored
View file

@ -1,6 +0,0 @@
/.dotnet
/.packages
/artifacts
/prereqs/packages
/src/nuget-client/NuGet.config
*.binlog

6
src/SourceBuild/content/.sscignore
View file

@ -1,6 +0,0 @@
{
"cfs" : [
"CFS0001",
"CFS0013"
]
}

6
src/SourceBuild/content/CODE_OF_CONDUCT.md
View file

@ -1,6 +0,0 @@
# Code of Conduct
This project has adopted the code of conduct defined by the Contributor Covenant
to clarify expected behavior in our community.
For more information, see the [.NET Foundation Code of Conduct](https://dotnetfoundation.org/code-of-conduct).

7
src/SourceBuild/content/CONTRIBUTING.md
View file

@ -1,7 +0,0 @@
Contributing
============
See [dotnet/runtime](./src/runtime/CONTRIBUTING.md) for general contribution guidelines such as allowed licenses.
At this time, the VMR will not accept any changes and is a read-only mirror of the development repositories only.
Please, make the changes in the respective development repositories (e.g., [dotnet/runtime](https://github.com/dotnet/runtime) or [dotnet/sdk](https://github.com/dotnet/sdk)) and they will get synchronized into the VMR automatically.

199
src/SourceBuild/content/Directory.Build.props
View file

@ -1,199 +0,0 @@
<Project>
<PropertyGroup>
<Configuration Condition="$(Configuration) == ''">Release</Configuration>
<DotNetBuildOrchestrator>true</DotNetBuildOrchestrator>
<RootRepo>dotnet</RootRepo>
</PropertyGroup>
<PropertyGroup Label="CalculateTargetOS">
<BuildOS>linux</BuildOS>
<BuildOS Condition="$([MSBuild]::IsOSPlatform('OSX'))">osx</BuildOS>
<BuildOS Condition="$([MSBuild]::IsOSPlatform('FREEBSD'))">freebsd</BuildOS>
<BuildOS Condition="$([MSBuild]::IsOSPlatform('NETBSD'))">netbsd</BuildOS>
<BuildOS Condition="$([MSBuild]::IsOSPlatform('ILLUMOS'))">illumos</BuildOS>
<BuildOS Condition="$([MSBuild]::IsOSPlatform('SOLARIS'))">solaris</BuildOS>
<BuildOS Condition="$([MSBuild]::IsOSPlatform('HAIKU'))">haiku</BuildOS>
<BuildOS Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">windows</BuildOS>
<TargetOS Condition="'$(TargetOS)' == ''">$(BuildOS)</TargetOS>
<HostOS Condition="'$(HostOS)' == ''">$(TargetOS)</HostOS>
</PropertyGroup>
<!-- This is a list of cases where we aren't producing a whole SDK, just a runtime. This list should be kept in sync with https://github.com/dotnet/runtime/blob/main/eng/SourceBuild.props#L26 -->
<PropertyGroup Label="ShortStacks">
<ShortStack Condition="'$(TargetOS)' == 'wasi'">true</ShortStack>
<ShortStack Condition="'$(TargetOS)' == 'browser'">true</ShortStack>
<ShortStack Condition="'$(TargetOS)' == 'ios'">true</ShortStack>
<ShortStack Condition="'$(TargetOS)' == 'iossimulator'">true</ShortStack>
<ShortStack Condition="'$(TargetOS)' == 'tvos'">true</ShortStack>
<ShortStack Condition="'$(TargetOS)' == 'tvossimulator'">true</ShortStack>
<ShortStack Condition="'$(TargetOS)' == 'maccatalyst'">true</ShortStack>
<ShortStack Condition="'$(TargetOS)' == 'android'">true</ShortStack>
<ShortStack Condition="'$(TargetOS)' == 'linux-bionic'">true</ShortStack>
<!-- NativeAOT Mac builds are short -->
<ShortStack Condition="'$(TargetOS)' == 'osx' and '$(DotNetBuildRuntimeNativeAOTRuntimePack)' == 'true'">true</ShortStack>
<!-- Mono LLVM builds are short -->
<ShortStack Condition="'$(MonoEnableLLVM)' == 'true' or '$(MonoAOTEnableLLVM)' == 'true'">true</ShortStack>
<!-- Short stack builds stop at runtime, not the whole SDK -->
<RootRepo Condition="'$(ShortStack)' == 'true'">runtime</RootRepo>
</PropertyGroup>
<!-- See https://github.com/dotnet/arcade/blob/main/Documentation/UnifiedBuild/Unified-Build-Controls.md#output-controls for
control set definition. -->
<PropertyGroup Label="CalculateArch">
<!-- Build architecture is what we are building on. -->
<BuildArchitecture>$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant)</BuildArchitecture>
<!-- The target architecture is the what the customer is targeting their outputs to run on. -->
<TargetArchitecture Condition="'$(TargetArchitecture)' == ''">$(BuildArchitecture)</TargetArchitecture>
<!-- The host architecture is the what the customer will build on. Much of the time, Host==Target. -->
<HostArchitecture Condition="'$(HostArchitecture)' == ''">$(TargetArchitecture)</HostArchitecture>
<Platform Condition="'$(Platform)' == ''">AnyCPU</Platform>
</PropertyGroup>
<PropertyGroup Label="CalculateRID">
<!-- Use current machine distro RID if set. Otherwise, fall back to RuntimeInformation.RuntimeIdentifier -->
<BuildRid>$(__DistroRid)</BuildRid>
<BuildRid Condition="'$(BuildRid)' == '' and '$(MSBuildRuntimeType)' == 'core'">$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier)</BuildRid>
<BuildRid Condition="'$(BuildRid)' == '' and '$(MSBuildRuntimeType)' != 'core'">win-$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant)</BuildRid>
<TargetRid Condition="'$(TargetRid)' == ''">$(BuildRid.Substring(0, $(BuildRid.LastIndexOf('-'))))-$(TargetArchitecture)</TargetRid>
<HostRid Condition="'$(HostRid)' == ''">$(TargetRid)</HostRid>
<!-- Source-only builds are non portable, except for cross-builds.
Source-only cross-builds default to the portable configuration so the resulting SDK works on a wider range of distros. -->
<PortableBuild Condition="'$(PortableBuild)' == '' and '$(DotNetBuildSourceOnly)' == 'true' and '$(BuildArchitecture)' == '$(TargetArchitecture)'">false</PortableBuild>
<PortableBuild Condition="'$(PortableBuild)' == ''">true</PortableBuild>
<PortableRid Condition="'$(__PortableTargetOS)' != ''">$(__PortableTargetOS)-$(TargetArchitecture)</PortableRid>
<PortableRid Condition="'$(PortableRid)' == '' and '$(TargetOS)' == 'freebsd'">freebsd-$(TargetArchitecture)</PortableRid>
<PortableRid Condition="'$(PortableRid)' == '' and '$(TargetOS)' == 'osx'">osx-$(TargetArchitecture)</PortableRid>
<PortableRid Condition="'$(PortableRid)' == '' and '$(TargetOS)' == 'linux'">linux-$(TargetArchitecture)</PortableRid>
<PortableRid Condition="$(TargetRid.StartsWith('linux-musl')) or $(TargetRid.StartsWith('alpine'))">linux-musl-$(TargetArchitecture)</PortableRid>
<PortableRid Condition="'$(PortableRid)' == '' and '$(TargetOS)' == 'windows'">win-$(TargetArchitecture)</PortableRid>
<TargetRid Condition="'$(PortableBuild)' == 'true' and '$(PortableRid)' != ''">$(PortableRid)</TargetRid>
<TargetRid Condition="'$(ShortStack)' == 'true' and '$(TargetOS)' != 'windows'">$(TargetOS)-$(TargetArchitecture)</TargetRid>
<TargetRid Condition="'$(ShortStack)' == 'true' and '$(TargetOS)' == 'windows'">win-$(TargetArchitecture)</TargetRid>
</PropertyGroup>
<!-- Set NuGetPackageRoot before Arcade SDK sets it. -->
<PropertyGroup>
<!-- Set RestorePackagesPath so that we don't accidentally pull some packages from the global location. -->
<RestorePackagesPath Condition="'$(RestorePackagesPath)' == ''">$([MSBuild]::NormalizeDirectory('$(MSBuildThisFileDirectory)', '.packages'))</RestorePackagesPath>
<NuGetPackageRoot Condition="'$(NuGetPackageRoot)' == ''">$(RestorePackagesPath)</NuGetPackageRoot>
</PropertyGroup>
<Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" Condition="'$(SkipArcadeSdkImport)' != 'true'" />
<!-- Init basic Arcade props, if the project importing this file doesn't use Arcade.
Keep in sync with props/targets in the Arcade.Sdk. -->
<PropertyGroup Condition="'$(SkipArcadeSdkImport)' == 'true'">
<!-- RepoLayout.props -->
<RepoRoot Condition="'$(RepoRoot)' == ''">$([MSBuild]::NormalizeDirectory('$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildProjectDirectory), 'global.json'))'))</RepoRoot>
<ArtifactsDir Condition="'$(ArtifactsDir)' == ''">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts'))</ArtifactsDir>
<ArtifactsObjDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'obj'))</ArtifactsObjDir>
<ArtifactsBinDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'bin'))</ArtifactsBinDir>
<RepositoryEngineeringDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'eng'))</RepositoryEngineeringDir>
<!-- ProjectLayout.props -->
<PlatformName Condition="'$(PlatformName)' == ''">$(Platform)</PlatformName>
<OutDirName Condition="'$(OutDirName)' == ''">$(MSBuildProjectName)</OutDirName>
<BaseOutputPath Condition="'$(BaseOutputPath)' == ''">$([System.IO.Path]::GetFullPath('$(ArtifactsBinDir)$(OutDirName)\'))</BaseOutputPath>
<OutputPath Condition="'$(PlatformName)' == 'AnyCPU'">$(BaseOutputPath)$(Configuration)\</OutputPath>
<OutputPath Condition="'$(PlatformName)' != 'AnyCPU'">$(BaseOutputPath)$(PlatformName)\$(Configuration)\</OutputPath>
<BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)' == ''">$([System.IO.Path]::GetFullPath('$(ArtifactsObjDir)$(OutDirName)\'))</BaseIntermediateOutputPath>
<IntermediateOutputPath Condition="'$(PlatformName)' == 'AnyCPU'">$(BaseIntermediateOutputPath)$(Configuration)\</IntermediateOutputPath>
<IntermediateOutputPath Condition="'$(PlatformName)' != 'AnyCPU'">$(BaseIntermediateOutputPath)$(PlatformName)\$(Configuration)\</IntermediateOutputPath>
<!-- TargetFrameworkDefaults.props -->
<NetCurrent>net9.0</NetCurrent>
</PropertyGroup>
<!-- Manually import the Versions.props file when the Arcade SDK isn't used. -->
<Import Project="$(RepositoryEngineeringDir)Versions.props" Condition="'$(SkipArcadeSdkImport)' == 'true'" />
<PropertyGroup>
<!-- We have no projects targeting multiple frameworks, so don't include in output path. -->
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
<!-- This repo's projects are entirely infrastructure and do not ship. -->
<IsShipping>false</IsShipping>
<!-- It's not unusual to build with a preview SDK -->
<SuppressNETCoreSdkPreviewMessage>true</SuppressNETCoreSdkPreviewMessage>
<!-- By default, the VMR builds with online sources when not building source-only. -->
<DotNetBuildWithOnlineFeeds Condition="'$(DotNetBuildWithOnlineFeeds)' == '' and '$(DotNetBuildSourceOnly)' != 'true'">true</DotNetBuildWithOnlineFeeds>
<!-- Don't use Arcade's ExcludeFrom* build infra in the VMR orchestrator. -->
<DisableArcadeExcludeFromBuildSupport>true</DisableArcadeExcludeFromBuildSupport>
</PropertyGroup>
<PropertyGroup>
<LogVerbosity Condition="'$(LogVerbosity)'==''">minimal</LogVerbosity>
<ShellExtension Condition="'$(BuildOS)' == 'windows'">.cmd</ShellExtension>
<ShellExtension Condition="'$(BuildOS)' != 'windows'">.sh</ShellExtension>
<!-- While the tarball infrastructure currently only supports non Windows, these properties
are already defined for more flexibility in the future. -->
<ArchiveExtension Condition="'$(BuildOS)' == 'windows'">.zip</ArchiveExtension>
<ArchiveExtension Condition="'$(BuildOS)' != 'windows'">.tar.gz</ArchiveExtension>
</PropertyGroup>
<PropertyGroup>
<SrcDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src'))</SrcDir>
<ToolsDir>$([MSBuild]::NormalizeDirectory('$(RepositoryEngineeringDir)', 'tools'))</ToolsDir>
<TasksDir>$([MSBuild]::NormalizeDirectory('$(ToolsDir)', 'tasks'))</TasksDir>
<RepoProjectsDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'repo-projects'))</RepoProjectsDir>
<PrereqsDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'prereqs'))</PrereqsDir>
<KeysDir>$([MSBuild]::NormalizeDirectory('$(PrereqsDir)', 'keys'))</KeysDir>
<PrereqsPackagesDir>$([MSBuild]::NormalizeDirectory('$(PrereqsDir)', 'packages'))</PrereqsPackagesDir>
<!-- When SB mode, the initial toolsets are unzipped to the "BootstrapPackages" dir.
When running in non-source-build, the bootstrap package is the one restored at the beginning of the build. -->
<BootstrapPackagesDir Condition="'$(DotNetBuildSourceOnly)' == 'true'">$([MSBuild]::NormalizeDirectory('$(NuGetPackageRoot)', 'BootstrapPackages'))</BootstrapPackagesDir>
<BootstrapPackagesDir Condition="'$(DotNetBuildSourceOnly)' != 'true'">$(NuGetPackageRoot)</BootstrapPackagesDir>
<VSMSBuildSdkResolversDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'toolset', 'VSSdkResolvers'))</VSMSBuildSdkResolversDir>
<IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
<AssetManifestsIntermediateDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'AssetManifests'))</AssetManifestsIntermediateDir>
<ArtifactsAssetsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)'))</ArtifactsAssetsDir>
<DotNetSdkExtractDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'extracted-dotnet-sdk'))</DotNetSdkExtractDir>
<PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
<PreviouslyRestoredPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored'))</PreviouslyRestoredPackagesPath>
<PrebuiltSourceBuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previously-source-built'))</PrebuiltSourceBuiltPackagesPath>
<PrebuiltSourceBuiltPackagesPath Condition="'$(CustomPrebuiltSourceBuiltPackagesPath)' != ''">$([MSBuild]::EnsureTrailingSlash('$(CustomPrebuiltSourceBuiltPackagesPath)'))</PrebuiltSourceBuiltPackagesPath>
<GitInfoDir>$([MSBuild]::NormalizeDirectory('$(PrereqsDir)', 'git-info'))</GitInfoDir>
<PackageReportDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'prebuilt-report'))</PackageReportDir>
<ResultingPrebuiltPackagesDir>$([MSBuild]::NormalizeDirectory('$(PackageReportDir)', 'prebuilt-packages'))</ResultingPrebuiltPackagesDir>
<ReferencePackagesDir>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'reference'))</ReferencePackagesDir>
<SourceBuiltArtifactsTarballName>Private.SourceBuilt.Artifacts</SourceBuiltArtifactsTarballName>
<SourceBuiltPrebuiltsTarballName>Private.SourceBuilt.Prebuilts</SourceBuiltPrebuiltsTarballName>
<BaselineDataFile>$(ToolsDir)prebuilt-baseline.xml</BaselineDataFile>
</PropertyGroup>
<!-- Build task assembly paths -->
<PropertyGroup>
<MicrosoftDotNetUnifiedBuildTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.UnifiedBuild.Tasks', '$(Configuration)', 'Microsoft.DotNet.UnifiedBuild.Tasks.dll'))</MicrosoftDotNetUnifiedBuildTasksAssembly>
<MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll'))</MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly>
</PropertyGroup>
<PropertyGroup Condition="'$(EnablePoison)' == 'true'">
<PoisonMarkerFile>.prebuilt.xml</PoisonMarkerFile>
<PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>
<PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
<PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
</PropertyGroup>
<PropertyGroup>
<MergedAssetManifestOutputPath>$(ArtifactsDir)VerticalManifest.xml</MergedAssetManifestOutputPath>
</PropertyGroup>
</Project>

1
src/SourceBuild/content/Directory.Build.rsp
View file

@ -1 +0,0 @@
-tl:false

29
src/SourceBuild/content/Directory.Build.targets
View file

@ -1,29 +0,0 @@
<Project>
<Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" Condition="'$(SkipArcadeSdkImport)' != 'true'" />
<Target Name="DetermineSourceBuiltSdkVersion">
<PropertyGroup>
<SdkFilenamePrefix>dotnet-sdk-</SdkFilenamePrefix>
</PropertyGroup>
<ItemGroup>
<SdkTarballItem Include="$(ArtifactsAssetsDir)Sdk/**/$(SdkFilenamePrefix)*$(ArchiveExtension)" />
</ItemGroup>
<!--
Extract SDK version from SDK tarball filename.
Keep in sync with dotnet-sdk's archive location and filename.
Example:
artifacts\assets\<config>\Sdk\9.0.100-alpha.1.24057.1\dotnet-sdk-9.0.100-alpha.1.24057.1-fedora.38-x64.tar.gz
artifacts\assets\<config>\Sdk\<SdkVersion>\dotnet-sdk-<SdkVersion>-<TargetRid><ArchiveExtension>
-->
<PropertyGroup>
<SdkFilename>%(SdkTarballItem.Filename)%(SdkTarballItem.Extension)</SdkFilename>
<SdkTarballPath Condition="'$(SdkTarballPath)' == ''">%(SdkTarballItem.Identity)</SdkTarballPath>
<SourceBuiltSdkVersion>$(SdkFilename.Replace('$(SdkFilenamePrefix)','').Replace('-$(TargetRid)$(ArchiveExtension)',''))</SourceBuiltSdkVersion>
</PropertyGroup>
</Target>
</Project>

26
src/SourceBuild/content/Directory.Packages.props
View file

@ -1,26 +0,0 @@
<Project>
<PropertyGroup>
<CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
<!-- Using multiple feeds isn't supported by Maestro: https://github.com/dotnet/arcade/issues/14155. -->
<NoWarn>$(NoWarn);NU1507</NoWarn>
</PropertyGroup>
<ItemGroup>
<!-- Command-line-api dependencies -->
<PackageVersion Include="System.CommandLine" Version="$(SystemCommandLineVersion)" />
<!-- MSBuild dependencies -->
<PackageVersion Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildVersion)" />
<PackageVersion Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildVersion)" />
<!-- Runtime dependencies -->
<PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="$(MicrosoftExtensionsFileSystemGlobbingVersion)" />
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingVersion)" />
<!-- External dependencies -->
<PackageVersion Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
<PackageVersion Include="xunit.extensibility.core" Version="$(XUnitVersion)" />
<PackageVersion Include="xunit.extensibility.execution" Version="$(XUnitVersion)" />
<PackageVersion Include="NuGet.Protocol" Version="$(NuGetProtocolVersion)" />
</ItemGroup>
</Project>

23
src/SourceBuild/content/LICENSE.TXT
View file

@ -1,23 +0,0 @@
The MIT License (MIT)
Copyright (c) .NET Foundation and Contributors
All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

15
src/SourceBuild/content/NuGet.config
View file

@ -1,15 +0,0 @@
<!--
This root NuGet.config ensures that a global/default one is not used by accident.
The individual repo NuGet.config files are utilized during the actual build.
This one is used by the smoke tests to restore packages such as xunit.
-->
<configuration>
<packageSources>
<clear />
<add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
<add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
</packageSources>
<disabledPackageSources>
<clear />
</disabledPackageSources>
</configuration>

205
src/SourceBuild/content/README.md
View file

@ -1,205 +0,0 @@
# dotnet/dotnet - Home of the .NET VMR
This repository is a **Virtual Monolithic Repository (VMR)** which includes all the source code and infrastructure needed to build the .NET SDK.
What this means:
- **Monolithic** - a join of multiple repositories that make up the whole product, such as [dotnet/runtime](https://github.com/dotnet/runtime) or [dotnet/sdk](https://github.com/dotnet/sdk).
- **Virtual** - a mirror (not replacement) of product repos where sources from those repositories are synchronized into.
In the VMR, you can find:
- source files of [each product repository](#list-of-components) which are mirrored inside of their respective directories under [`src/`](https://github.com/dotnet/dotnet/tree/main/src),
- tooling that enables [building the whole .NET product from source](https://github.com/dotnet/source-build) on Linux platforms,
- small customizations, in the form of [patches](https://github.com/dotnet/dotnet/tree/main/src/installer/src/SourceBuild/patches), applied on top of the original code to make the build possible,
- *[in future]* E2E tests for the whole .NET product.
Just like the development repositories, the VMR will have a release branch for every feature band (e.g. `release/8.0.1xx`).
Similarly, VMR's `main` branch will follow `main` branches of product repositories (see [Synchronization Based on Declared Dependencies](src/arcade/Documentation/UnifiedBuild/VMR-Design-And-Operation.md#synchronization-based-on-declared-dependencies)).
More in-depth documentation about the VMR can be found in [VMR Design And Operation](src/arcade/Documentation/UnifiedBuild/VMR-Design-And-Operation.md#layout).
See also [dotnet/source-build](https://github.com/dotnet/source-build) for more information about our whole-product source-build.
## Goals
- The main purpose of the [dotnet/dotnet](https://github.com/dotnet/dotnet) repository is to have all source code necessary to build the .NET product available in one repository and identified by a single commit.
- The VMR also aims to become the place from which we release and service future versions of .NET to reduce the complexity of the product construction process. This should allow our partners and and 3rd parties to easily build, test and modify .NET using their custom infrastructure as well as make the process available to the community.
- Lastly, we hope to solve other problems that the current multi-repo setup brings:
- Enable the standard [down-/up-stream open-source model](src/arcade/Documentation/UnifiedBuild/VMR-Upstream-Downstream.md).
- Fulfill requirements of .NET distro builders such as RedHat or Canonical to natively include .NET in their distribution repositories.
- Simplify scenarios such as client-run testing of bug fixes and improvements. The build should work in an offline environment too for certain platforms.
- Enable developers to make and test changes spanning multiple repositories.
- More efficient pipeline for security fixes during the CVE pre-disclosure process.
We will achieve these goals while keeping active coding work in the separate repos where it happens today. For example: ASP.NET features will continue to be developed in `dotnet/aspnetcore` and CLR features will be continue to be developed in `dotnet/runtime`. Each of these repos have their own distinct communities and processes, and aggregating development into a true mono-repo would work against that. Hence, the "virtual" monolithic repo: the VMR gives us the simplicity of a mono-repo for building and servicing the product, while active development of components of that product stays in its various existing repos. The day to day experience for typical contributors will not change.
## Limitations
**This is a work-in-progress.**
There are considerable limitations to what is possible at the moment. For an extensive list of current limitations, please see [Temporary Mechanics](src/arcade/Documentation/UnifiedBuild/VMR-Design-And-Operation.md#temporary-mechanics).
See the [Unified Build roadmap](src/arcade/Documentation/UnifiedBuild/Roadmap.md) for more details.
### Supported platforms
* 8.0
* source-build configuration on Linux
* 9.0+ (WIP)
* source-build configuration on Linux
* non-source-build configuration on Linux, Mac, and Windows
For the latest information about Source-Build support for new .NET versions, please check our [GitHub Discussions page](https://github.com/dotnet/source-build/discussions) for announcements.
### Code flow
For the time being, the source code only flows one way - from the development repos into the VMR.
More details on this process:
- [Source Synchronization Process](src/arcade/Documentation/UnifiedBuild/VMR-Design-And-Operation.md#source-synchronization-process)
- [Synchronization Based on Declared Dependencies](src/arcade/Documentation/UnifiedBuild/VMR-Design-And-Operation.md#synchronization-based-on-declared-dependencies)
- [Moving Code and Dependencies between the VMR and Development Repos](src/arcade/Documentation/UnifiedBuild/VMR-Design-And-Operation.md#moving-code-and-dependencies-between-the-vmr-and-development-repos)
We expect the code flow to start working both ways in the .NET 9 timeframe.
See the [Unified Build roadmap](src/arcade/Documentation/UnifiedBuild/Roadmap.md) for more details.
### Contribution
At this time, the VMR will not accept any changes and is a read-only mirror of the development repositories only.
Please, make the changes in the respective development repositories (e.g., [dotnet/runtime](https://github.com/dotnet/runtime) or [dotnet/sdk](https://github.com/dotnet/sdk)) and they will get synchronized into the VMR automatically.
## Dev instructions
Please note that **this repository is a work-in-progress** and there are some usability issues connected to this.
These can be nuisances such as some checked-in files getting modified by the build itself and similar.
For the latest information about Source-Build support, please watch for announcements posted on our [GitHub Discussions page](https://github.com/dotnet/source-build/discussions).
### Prerequisites
The dependencies for building can be found [here](https://github.com/dotnet/runtime/blob/main/docs/workflow/requirements/).
In case you don't want to / cannot prepare your environment per the requirements, consider [using Docker](#building-using-docker).
### Building
1. **Clone the repository**
```bash
git clone https://github.com/dotnet/dotnet dotnet-dotnet
cd dotnet-dotnet
```
2. **Build the .NET SDK**
Choose one of the following build modes:
- **Microsoft based build**
For Unix:
```bash
./build.sh --clean-while-building
```
For Windows:
```cmd
.\build.cmd -cleanWhileBuilding
```
- **Building from source**
```bash
# Prep the source to build on your distro.
# This downloads a .NET SDK and a number of .NET packages needed to build .NET from source.
./prep-source-build.sh
# Build the .NET SDK
./build.sh -sb --clean-while-building
```
The resulting SDK is placed at `artifacts/assets/Release/dotnet-sdk-9.0.100-[your-RID].tar.gz` (for Unix) or `artifacts/assets/Release/dotnet-sdk-9.0.100-[your-RID].zip` (for Windows).
4. *(Optional)* **Unpack and install the .NET SDK**
For Unix:
```bash
mkdir -p $HOME/dotnet
tar zxf artifacts/assets/Release/dotnet-sdk-9.0.100-[your-RID].tar.gz -C $HOME/dotnet
ln -s $HOME/dotnet/dotnet /usr/bin/dotnet
```
For Windows:
```cmd
mkdir %userprofile%\dotnet
tar -xf artifacts/assets/Release/dotnet-sdk-9.0.100-[your RID].zip -C %userprofile%\dotnet
set "PATH=%userprofile%\dotnet;%PATH%"
```
To test your built SDK, run the following:
```bash
dotnet --info
```
> [!NOTE]
> Run `./build.sh --help` (for Unix) or `.\build.cmd -help` (for Windows) to see more information about supported build options.
### Building using Docker
You can also build the repository using a Docker image which has the required prerequisites inside.
The example below creates a Docker volume named `vmr` and clones and builds the VMR there.
```sh
docker run --rm -it -v vmr:/vmr -w /vmr mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8
git clone https://github.com/dotnet/dotnet .
# - Microsoft based build
./build.sh --clean-while-building
# - Building from source
./prep-source-build.sh && ./build.sh -sb --clean-while-building
mkdir -p $HOME/.dotnet
tar -zxf artifacts/assets/Release/dotnet-sdk-9.0.100-centos.8-x64.tar.gz -C $HOME/.dotnet
ln -s $HOME/.dotnet/dotnet /usr/bin/dotnet
```
### Codespaces
You can also utilize [GitHub Codespaces](https://github.com/features/codespaces) where you can find preset containers in this repository.
### Building from released sources
You can also build from sources (and not from a context of a git repository), such as the ones you can acquire from a [dotnet/dotnet release](https://github.com/dotnet/dotnet/releases).
In this case, you need to provide additional information which includes the original repository and commit hash the code was built from so that the SDK can provide a better debugging experience (think the `Step into..` functionality).
Usually, this means the [dotnet/dotnet repository](https://github.com/dotnet/dotnet) together with the commit the release tag is connected to.
In practice, this means that when calling the main build script, you need to provide additional arguments when building outside of a context of a git repository.
Alternatively, you can also provide a manifest file where this information can be read from. This file (`release.json`) can be found attached with the [dotnet/dotnet release](https://github.com/dotnet/dotnet/releases).
### Synchronizing code into the VMR
Sometimes you want to make a change in a repository and test that change in the VMR. You could of course make the change in the VMR directly (locally, as the VMR is read-only for now) but in case it's already available in your repository, you can synchronize it into the VMR (again locally).
To do this, you can either start a [dotnet/dotnet](https://github.com/dotnet/dotnet) Codespace - you will see instructions right after it starts. Alternatively, you can clone the repository locally and use the [vmr-sync.sh](src/installer/eng/vmr-sync.sh) or [vmr-sync.ps1](src/installer/eng/vmr-sync.ps1) script to pull your changes in. Please refer to the documentation in the script for more details.
## List of components
The full list of components synchronized into the VMR is [here (Components.md)](./Components.md).
The repository also contains a [JSON manifest](https://github.com/dotnet/dotnet/blob/main/src/source-manifest.json) listing all components in a machine-readable format.
## Filing Issues
This repo does not accept issues as of now. Please file issues to the appropriate development repos.
For issues with the VMR itself, please use the [source-build repository](https://github.com/dotnet/source-build).
## Useful Links
- Design documentation for the VMR - a set of documents describing the high-level design and the why's and how's
- [Design and Operation](src/arcade/Documentation/UnifiedBuild/VMR-Design-And-Operation.md)
- [Upstream/Downstream Relationships](src/arcade/Documentation/UnifiedBuild/VMR-Upstream-Downstream.md)
- [Code and Build Workflow](src/arcade/Documentation/UnifiedBuild/VMR-Code-And-Build-Workflow.md)
- [Strategy for Managing External Source Dependencies](src/arcade/Documentation/UnifiedBuild/VMR-Strategy-For-External-Source.md)
- [.NET Source-Build](https://github.com/dotnet/source-build)
- [What is .NET](https://dotnet.microsoft.com)
## .NET Foundation
.NET Runtime is a [.NET Foundation](https://www.dotnetfoundation.org/projects) project.
## License
.NET is licensed under the [MIT](LICENSE.TXT) license.

41
src/SourceBuild/content/SECURITY.md
View file

@ -1,41 +0,0 @@
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.3 BLOCK -->
## Security
Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
If you believe you have found a security vulnerability in any Microsoft-owned repository that meets Microsoft's [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)) of a security vulnerability, please report it to us as described below.
## Reporting Security Issues
**Please do not report security vulnerabilities through public GitHub issues.**
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report).
If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc).
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).
Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
* Full paths of source file(s) related to the manifestation of the issue
* The location of the affected source code (tag/branch/commit or direct URL)
* Any special configuration required to reproduce the issue
* Step-by-step instructions to reproduce the issue
* Proof-of-concept or exploit code (if possible)
* Impact of the issue, including how an attacker might exploit the issue
This information will help us triage your report more quickly.
If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs.
## Preferred Languages
We prefer all communications to be in English.
## Policy
Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd).
<!-- END MICROSOFT SECURITY.MD BLOCK -->

9
src/SourceBuild/content/build.cmd
View file

@ -1,9 +0,0 @@
@echo off
setlocal
set _args=%*
if "%~1"=="-?" set _args=-help
if "%~1"=="/?" set _args=-help
powershell -ExecutionPolicy ByPass -NoProfile -Command "& '%~dp0eng\build.ps1'" %_args%
exit /b %ERRORLEVEL%

45
src/SourceBuild/content/build.proj
View file

@ -1,45 +0,0 @@
<Project Sdk="Microsoft.Build.Traversal" InitialTargets="PrintInfo">
<!-- Default targets and parallelization -->
<ItemDefinitionGroup>
<ProjectReference>
<Test>false</Test>
<Publish>false</Publish>
<BuildInParallel>false</BuildInParallel>
</ProjectReference>
</ItemDefinitionGroup>
<ItemGroup>
<!-- Pre-build: Init tools-->
<ProjectReference Include="$(RepositoryEngineeringDir)init-cross-build.proj"
Condition="'$(BuildOS)' != 'windows' and '$(CrossBuild)' == 'true' and '$(ROOTFS_DIR)' == ''"
BuildInParallel="true" />
<ProjectReference Include="$(RepositoryEngineeringDir)init-poison.proj"
Condition="'$(EnablePoison)' == 'true'"
BuildInParallel="true" />
<ProjectReference Include="$(ToolsDir)tools.proj" BuildInParallel="true" />
<ProjectReference Include="$(RepoProjectsDir)$(RootRepo).proj" />
<!-- Post-build: Source-only validation, packaging and publishing -->
<ProjectReference Include="$(RepositoryEngineeringDir)merge-asset-manifests.proj" />
<ProjectReference Include="$(RepositoryEngineeringDir)finish-source-only.proj" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
</ItemGroup>
<Target Name="PrintInfo">
<PropertyGroup>
<BuildModeInfoText Condition="'$(DotNetBuildSourceOnly)' == 'true'">source-build</BuildModeInfoText>
<BuildModeInfoText Condition="'$(DotNetBuildSourceOnly)' != 'true'">non-source-build</BuildModeInfoText>
</PropertyGroup>
<Message Text="Build Mode: $(BuildModeInfoText)" Importance="high" />
<Message Text="Build Environment: $(TargetArchitecture) $(Configuration) $(TargetOS) $(TargetRid)" Importance="high" />
</Target>
<Target Name="LogBuildOutputFolders"
AfterTargets="Build">
<Message Importance="high" Text="Shipping packages are located in '$(ArtifactsShippingPackagesDir)'." />
<Message Importance="high" Text="Shipping assets are located in '$(ArtifactsAssetsDir)'." />
</Target>
</Project>

426
src/SourceBuild/content/build.sh
View file

@ -1,426 +0,0 @@
#!/usr/bin/env bash
# Stop script if unbound variable found (use ${var:-} if intentional)
set -u
# Stop script if command returns non-zero exit code.
# Prevents hidden errors caused by missing error code propagation.
set -e
usage()
{
echo "Common settings:"
echo " --binaryLog Create MSBuild binary log (short: -bl)"
echo " --configuration <value> Build configuration: 'Debug' or 'Release' (short: -c)"
echo " --verbosity <value> Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
echo ""
echo "Actions:"
echo " --clean Clean the solution"
echo " --help Print help and exit (short: -h)"
echo " --test Run tests (short: -t)"
echo ""
echo "Source-only settings:"
echo " --source-only, --source-build Source-build the solution (short: -so, -sb)"
echo " --online Build using online sources"
echo " --poison Build with poisoning checks"
echo " --release-manifest <FILE> A JSON file, an alternative source of Source Link metadata"
echo " --source-repository <URL> Source Link repository URL, required when building from tarball"
echo " --source-version <SHA> Source Link revision, required when building from tarball"
echo " --with-packages <DIR> Use the specified directory of previously-built packages"
echo " --with-sdk <DIR> Use the SDK in the specified directory for bootstrapping"
echo ""
echo "Non-source-only settings:"
echo " --build-repo-tests Build repository tests"
echo " --dev Use -dev or -ci versioning instead of .NET official build versions"
echo "Advanced settings:"
echo " --ci Set when running on CI server"
echo " --clean-while-building Cleans each repo after building (reduces disk space usage, short: -cwb)"
echo " --excludeCIBinarylog Don't output binary log (short: -nobl)"
echo " --prepareMachine Prepare machine for CI run, clean up processes after build"
echo " --use-mono-runtime Output uses the mono runtime"
echo ""
echo "Command line arguments not listed above are passed thru to msbuild."
echo "Arguments can also be passed in with a single hyphen."
}
source="${BASH_SOURCE[0]}"
# resolve $source until the file is no longer a symlink
while [[ -h "$source" ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
packagesRestoredDir="$scriptroot/.packages/"
# Common settings
binary_log=false
configuration='Release'
verbosity='minimal'
# Actions
clean=false
test=false
# Source-only settings
sourceOnly=false
releaseManifest=''
sourceRepository=''
sourceVersion=''
CUSTOM_PACKAGES_DIR=''
CUSTOM_SDK_DIR=''
packagesDir="$scriptroot/prereqs/packages/"
packagesArchiveDir="${packagesDir}archive/"
packagesPreviouslySourceBuiltDir="${packagesDir}previously-source-built/"
# Advanced settings
ci=false
exclude_ci_binary_log=false
prepare_machine=false
use_dev_versioning=false
properties=()
while [[ $# > 0 ]]; do
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
# Common settings
-binarylog|-bl)
binary_log=true
;;
-configuration|-c)
configuration=$2
shift
;;
-verbosity|-v)
verbosity=$2
shift
;;
# Actions
-clean)
clean=true
;;
-help|-h|-\?|/?)
usage
exit 0
;;
-test|-t)
test=true
;;
# Source-only settings
-source-only|-source-build|-so|-sb)
sourceOnly=true
properties+=( "/p:DotNetBuildSourceOnly=true" )
;;
-online)
properties+=( "/p:DotNetBuildWithOnlineFeeds=true" )
;;
-poison)
properties+=( "/p:EnablePoison=true" )
;;
-release-manifest)
releaseManifest="$2"
shift
;;
-source-repository)
sourceRepository="$2"
shift
;;
-source-version)
sourceVersion="$2"
shift
;;
-with-packages)
CUSTOM_PACKAGES_DIR="$(cd -P "$2" && pwd)"
if [ ! -d "$CUSTOM_PACKAGES_DIR" ]; then
echo "Custom prviously built packages directory '$CUSTOM_PACKAGES_DIR' does not exist"
exit 1
fi
shift
;;
-with-sdk)
CUSTOM_SDK_DIR="$(cd -P "$2" && pwd)"
if [ ! -d "$CUSTOM_SDK_DIR" ]; then
echo "Custom SDK directory '$CUSTOM_SDK_DIR' does not exist"
exit 1
fi
if [ ! -x "$CUSTOM_SDK_DIR/dotnet" ]; then
echo "Custom SDK '$CUSTOM_SDK_DIR/dotnet' does not exist or is not executable"
exit 1
fi
shift
;;
# Advanced settings
-build-repo-tests)
properties+=( "/p:DotNetBuildTests=true" )
;;
-ci)
ci=true
;;
-clean-while-building|-cwb)
properties+=( "/p:CleanWhileBuilding=true" )
;;
-excludecibinarylog|-nobl)
exclude_ci_binary_log=true
;;
-preparemachine)
prepare_machine=true
;;
-use-mono-runtime)
properties+=( "/p:SourceBuildUseMonoRuntime=true" )
;;
-dev)
use_dev_versioning=true
;;
*)
properties+=( "$1" )
;;
esac
shift
done
if [[ "$ci" == true ]]; then
if [[ "$exclude_ci_binary_log" == false ]]; then
binary_log=true
fi
fi
if [[ "$use_dev_versioning" == true && "$sourceOnly" != true ]]; then
properties+=( "/p:UseOfficialBuildVersioning=false" )
fi
# Never use the global nuget cache folder
use_global_nuget_cache=false
. "$scriptroot/eng/common/tools.sh"
project="$scriptroot/build.proj"
targets="/t:Build"
# This repo uses the VSTest integration instead of the Arcade Test target
if [[ "$test" == true ]]; then
project="$scriptroot/test/tests.proj"
targets="$targets;VSTest"
fi
function Build {
if [[ "$sourceOnly" != "true" ]]; then
InitializeToolset
# Manually unset NUGET_PACKAGES as InitializeToolset sets it unconditionally.
# The env var shouldn't be set so that the RestorePackagesPath msbuild property is respected.
unset NUGET_PACKAGES
local bl=""
if [[ "$binary_log" == true ]]; then
bl="/bl:\"$log_dir/Build.binlog\""
fi
MSBuild --restore \
$project \
$targets \
$bl \
/p:Configuration=$configuration \
"${properties[@]}"
ExitWithExitCode 0
else
if [ "$ci" == "true" ]; then
properties+=( "/p:ContinuousIntegrationBuild=true" )
fi
if [ "$test" != "true" ]; then
initSourceOnlyBinaryLog=""
if [[ "$binary_log" == true ]]; then
initSourceOnlyBinaryLog="/bl:\"$log_dir/init-source-only.binlog\""
fi
"$CLI_ROOT/dotnet" build-server shutdown
"$CLI_ROOT/dotnet" msbuild "$scriptroot/eng/init-source-only.proj" $initSourceOnlyBinaryLog "${properties[@]}"
# kill off the MSBuild server so that on future invocations we pick up our custom SDK Resolver
"$CLI_ROOT/dotnet" build-server shutdown
fi
# Point MSBuild to the custom SDK resolvers folder, so it will pick up our custom SDK Resolver
export MSBUILDADDITIONALSDKRESOLVERSFOLDER="$scriptroot/artifacts/toolset/VSSdkResolvers/"
local bl=""
if [[ "$binary_log" == true ]]; then
bl="/bl:\"$log_dir/Build.binlog\""
fi
"$CLI_ROOT/dotnet" msbuild --restore "$project" $bl $targets "${properties[@]}"
fi
}
if [[ "$clean" == true ]]; then
if [ -d "$artifacts_dir" ]; then
rm -rf $artifacts_dir
echo "Artifacts directory deleted."
fi
exit 0
fi
# Initialize __DistroRid and __PortableTargetOS
source $scriptroot/eng/common/native/init-os-and-arch.sh
source $scriptroot/eng/common/native/init-distro-rid.sh
initDistroRidGlobal "$os" "$arch" ""
# Source-only settings
if [[ "$sourceOnly" == "true" ]]; then
# For build purposes, we need to make sure we have all the SourceLink information
if [ "$test" != "true" ]; then
GIT_DIR="$scriptroot/.git"
if [ -f "$GIT_DIR/index" ]; then # We check for index because if outside of git, we create config and HEAD manually
if [ -n "$sourceRepository" ] || [ -n "$sourceVersion" ] || [ -n "$releaseManifest" ]; then
echo "ERROR: Source Link arguments cannot be used in a git repository"
exit 1
fi
else
if [ -z "$releaseManifest" ]; then
if [ -z "$sourceRepository" ] || [ -z "$sourceVersion" ]; then
echo "ERROR: $scriptroot is not a git repository, either --release-manifest or --source-repository and --source-version must be specified"
exit 1
fi
else
if [ -n "$sourceRepository" ] || [ -n "$sourceVersion" ]; then
echo "ERROR: --release-manifest cannot be specified together with --source-repository and --source-version"
exit 1
fi
get_property() {
local json_file_path="$1"
local property_name="$2"
grep -oP '(?<="'$property_name'": ")[^"]*' "$json_file_path"
}
sourceRepository=$(get_property "$releaseManifest" sourceRepository) \
|| (echo "ERROR: Failed to find sourceRepository in $releaseManifest" && exit 1)
sourceVersion=$(get_property "$releaseManifest" sourceVersion) \
|| (echo "ERROR: Failed to find sourceVersion in $releaseManifest" && exit 1)
if [ -z "$sourceRepository" ] || [ -z "$sourceVersion" ]; then
echo "ERROR: sourceRepository and sourceVersion must be specified in $releaseManifest"
exit 1
fi
fi
# We need to add "fake" .git/ files when not building from a git repository
mkdir -p "$GIT_DIR"
echo '[remote "origin"]' > "$GIT_DIR/config"
echo "url=\"$sourceRepository\"" >> "$GIT_DIR/config"
echo "$sourceVersion" > "$GIT_DIR/HEAD"
fi
fi
# Support custom source built package locations
if [ "$CUSTOM_PACKAGES_DIR" != "" ]; then
if [ "$test" == "true" ]; then
properties+=( "/p:CustomSourceBuiltPackagesPath=$CUSTOM_PACKAGES_DIR" )
else
properties+=( "/p:CustomPrebuiltSourceBuiltPackagesPath=$CUSTOM_PACKAGES_DIR" )
fi
fi
if [ ! -d "$scriptroot/.git" ]; then
echo "ERROR: $scriptroot is not a git repository."
exit 1
fi
# Allow a custom SDK directory to be specified
if [ -d "$CUSTOM_SDK_DIR" ]; then
export SDK_VERSION=$("$CUSTOM_SDK_DIR/dotnet" --version)
export CLI_ROOT="$CUSTOM_SDK_DIR"
export _InitializeDotNetCli="$CLI_ROOT/dotnet"
export DOTNET_INSTALL_DIR="$CLI_ROOT"
echo "Using custom bootstrap SDK from '$CLI_ROOT', version '$SDK_VERSION'"
else
sdkLine=$(grep -m 1 'dotnet' "$scriptroot/global.json")
sdkPattern="\"dotnet\" *: *\"(.*)\""
if [[ $sdkLine =~ $sdkPattern ]]; then
export SDK_VERSION=${BASH_REMATCH[1]}
export CLI_ROOT="$scriptroot/.dotnet"
fi
fi
# Find the Arcade SDK version and set env vars for the msbuild sdk resolver
packageVersionsPath=''
if [[ "$CUSTOM_PACKAGES_DIR" != "" && -f "$CUSTOM_PACKAGES_DIR/PackageVersions.props" ]]; then
packageVersionsPath="$CUSTOM_PACKAGES_DIR/PackageVersions.props"
elif [ -d "$packagesArchiveDir" ]; then
sourceBuiltArchive=$(find "$packagesArchiveDir" -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz')
if [ -f "${packagesPreviouslySourceBuiltDir}PackageVersions.props" ]; then
packageVersionsPath=${packagesPreviouslySourceBuiltDir}PackageVersions.props
elif [ -f "$sourceBuiltArchive" ]; then
tar -xzf "$sourceBuiltArchive" -C /tmp PackageVersions.props
packageVersionsPath=/tmp/PackageVersions.props
fi
fi
if [ ! -f "$packageVersionsPath" ]; then
echo "Cannot find PackagesVersions.props. Debugging info:"
echo " Attempted custom PVP path: $CUSTOM_PACKAGES_DIR/PackageVersions.props"
echo " Attempted previously-source-built path: ${packagesPreviouslySourceBuiltDir}PackageVersions.props"
echo " Attempted archive path: $packagesArchiveDir"
exit 1
fi
# Extract toolset packages
# Ensure that by default, the bootstrap version of the toolset SDK is used. Source-build infra
# projects use bootstrap toolset SDKs, and would fail to find it in the build. The repo
# projects overwrite this so that they use the source-built toolset SDK instad.
# 1. Microsoft.DotNet.Arcade.Sdk
arcadeSdkLine=$(grep -m 1 'MicrosoftDotNetArcadeSdkVersion' "$packageVersionsPath")
arcadeSdkPattern="<MicrosoftDotNetArcadeSdkVersion>(.*)</MicrosoftDotNetArcadeSdkVersion>"
if [[ $arcadeSdkLine =~ $arcadeSdkPattern ]]; then
export ARCADE_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
export SOURCE_BUILT_SDK_ID_ARCADE=Microsoft.DotNet.Arcade.Sdk
export SOURCE_BUILT_SDK_VERSION_ARCADE=$ARCADE_BOOTSTRAP_VERSION
export SOURCE_BUILT_SDK_DIR_ARCADE=$packagesRestoredDir/BootstrapPackages/microsoft.dotnet.arcade.sdk/$ARCADE_BOOTSTRAP_VERSION
fi
# 2. Microsoft.Build.NoTargets
notargetsSdkLine=$(grep -m 1 'Microsoft.Build.NoTargets' "$scriptroot/global.json")
notargetsSdkPattern="\"Microsoft\.Build\.NoTargets\" *: *\"(.*)\""
if [[ $notargetsSdkLine =~ $notargetsSdkPattern ]]; then
export NOTARGETS_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
export SOURCE_BUILT_SDK_ID_NOTARGETS=Microsoft.Build.NoTargets
export SOURCE_BUILT_SDK_VERSION_NOTARGETS=$NOTARGETS_BOOTSTRAP_VERSION
export SOURCE_BUILT_SDK_DIR_NOTARGETS=$packagesRestoredDir/BootstrapPackages/microsoft.build.notargets/$NOTARGETS_BOOTSTRAP_VERSION
fi
# 3. Microsoft.Build.Traversal
traversalSdkLine=$(grep -m 1 'Microsoft.Build.Traversal' "$scriptroot/global.json")
traversalSdkPattern="\"Microsoft\.Build\.Traversal\" *: *\"(.*)\""
if [[ $traversalSdkLine =~ $traversalSdkPattern ]]; then
export TRAVERSAL_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
export SOURCE_BUILT_SDK_ID_TRAVERSAL=Microsoft.Build.Traversal
export SOURCE_BUILT_SDK_VERSION_TRAVERSAL=$TRAVERSAL_BOOTSTRAP_VERSION
export SOURCE_BUILT_SDK_DIR_TRAVERSAL=$packagesRestoredDir/BootstrapPackages/microsoft.build.traversal/$TRAVERSAL_BOOTSTRAP_VERSION
fi
echo "Found bootstrap versions: SDK $SDK_VERSION, Arcade $ARCADE_BOOTSTRAP_VERSION, NoTargets $NOTARGETS_BOOTSTRAP_VERSION and Traversal $TRAVERSAL_BOOTSTRAP_VERSION"
fi
Build

24
src/SourceBuild/content/docs/license-scanning.md
View file

@ -1,24 +0,0 @@
# License Scanning
The VMR is regularly scanned for license references to ensure that only open-source license are used where relevant.
License scanning pipline: https://dev.azure.com/dnceng/internal/_build?definitionId=1301 (internal only)
License scanning test: https://github.com/dotnet/dotnet/blob/main/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs
By default, running the pipeline will scan all repos within the VMR which takes several hours to run.
The pipeline can be triggered manually to target a specific repo within the VMR by setting the `specificRepoName` parameter.
This value should be the name of the repo within the VMR (i.e. a name of a directory within https://github.com/dotnet/dotnet/tree/main/src).
To test source modifications intended to resolve a license issue, apply the change in an internal branch of the VMR.
Run this pipeline, targeting your branch, and set the `specificRepoName` parameter to the name of the repo containing the change.
The output of the pipeline is a set of test results and logs.
The logs are published as an artifact and can be found at test/Microsoft.DotNet/SourceBuild.SmokeTests/bin/Release/netX.0/logs.
It consists of the following:
* `UpdatedLicenses.<repo-name>.json`: This is the output of that gets compared to the stored baseline.
If they're the same, the test passes; if not, it fails. By comparing this file to the baseline, one can determine which new license
references have been introduced.
If everything is deemed to be acceptable, the developer can either update the allowed licenses, update the exclusions file, update the
baseline, or any combination.
* `scancode-results.json`: This is the raw output that comes from scancode. This file is useful for diagnostic purposes because it tells you
the exact line number of where a license has been detected in a file.

7
src/SourceBuild/content/eng/Build.props
View file

@ -1,7 +0,0 @@
<Project>
<ItemGroup>
<ProjectToBuild Include="$(RepoRoot)build.proj" />
</ItemGroup>
</Project>

10
src/SourceBuild/content/eng/Version.Details.xml
View file

@ -1,10 +0,0 @@
<Dependencies>
<ProductDependencies>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24253.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>020255bcf7d0b8beed7de05338d97396982ae527</Sha>
</Dependency>
</ToolsetDependencies>
</Dependencies>

41
src/SourceBuild/content/eng/Versions.props
View file

@ -1,41 +0,0 @@
<Project>
<!-- Repo Version Information -->
<PropertyGroup>
<VersionPrefix>0.1.0</VersionPrefix>
<PreReleaseVersionLabel>alpha.1</PreReleaseVersionLabel>
<UseVSTestRunner>true</UseVSTestRunner>
</PropertyGroup>
<PropertyGroup>
<!--
Building .NET from source depends on several archives, depending on the branch's current
source-buildability status.
PrivateSourceBuiltSdkVersion is the version number of a tar.gz of the source build .NET SDK
from the previous release for a particular OS. It is used by CI to validate that the current
release can be built with it.
PrivateSourceBuiltArtifactsVersion is the version number of a tar.gz of .NET build outputs
from a previous build needed to build the current version of .NET. This is always defined
because .NET needs to be bootstrappable at any point in time.
PrivateSourceBuiltPrebuiltsVersion is the version number of a tar.gz of assets downloaded
from the internet that are needed to build the current version of .NET. Early in the lifecycle
of a .NET major or minor release, prebuilts may be needed. When the release is mature, prebuilts
are not necessary, and this property is removed from the file.
-->
<PrivateSourceBuiltSdkVersion>9.0.100-preview.5.24256.1</PrivateSourceBuiltSdkVersion>
<PrivateSourceBuiltArtifactsVersion>9.0.100-preview.5.24256.1</PrivateSourceBuiltArtifactsVersion>
<PrivateSourceBuiltPrebuiltsVersion>9.0.100-preview.5.24256.1</PrivateSourceBuiltPrebuiltsVersion>
<!-- command-line-api dependencies -->
<SystemCommandLineVersion>2.0.0-beta4.24126.1</SystemCommandLineVersion>
<!-- msbuild dependencies -->
<MicrosoftBuildVersion>17.8.3</MicrosoftBuildVersion>
<!-- runtime dependencies -->
<MicrosoftExtensionsFileSystemGlobbingVersion>8.0.0</MicrosoftExtensionsFileSystemGlobbingVersion>
<MicrosoftExtensionsLoggingConsoleVersion>8.0.0</MicrosoftExtensionsLoggingConsoleVersion>
<MicrosoftExtensionsLoggingVersion>8.0.0</MicrosoftExtensionsLoggingVersion>
<!-- external dependencies -->
<NewtonsoftJsonVersion>13.0.3</NewtonsoftJsonVersion>
<NuGetProtocolVersion>6.9.1</NuGetProtocolVersion>
</PropertyGroup>
</Project>

59
src/SourceBuild/content/eng/allowed-sb-binaries.txt
View file

@ -1,59 +0,0 @@
# Contains the binaries that are allowed for source build
# This file is used by the Binary Tool to remove binaries from the VMR
# If importing a file, include the relative path to the file
**/*.bmp
**/*.doc
**/*.docx
**/*.gif
**/*.ico
**/*.jpg
**/*.JPG
**/*.pdf
**/*.png
**/*.PNG
**/*.rtf
**/*.snk
**/*.vsd
**/*.vsdx
**/*.xlsx
**/*.ttf
**/*.cur
**/*.icm
**/*.reg
# aspnetcore
src/aspnetcore/src/**/samples/**/*.eot
src/aspnetcore/src/**/samples/**/*.pfx
src/aspnetcore/src/**/samples/**/*.woff*
src/aspnetcore/src/**/Samples/**/*.woff*
src/aspnetcore/src/Components/benchmarkapps/BlazingPizza.Server/wwwroot/css/font/quicksand-v8-latin-*.woff*
src/aspnetcore/src/Components/Web.JS/dist/Release/blazor.*.js # JavaScript files with a null bytes
src/aspnetcore/src/ProjectTemplates/Web.ProjectTemplates/**/app.db
src/aspnetcore/src/submodules/Node-Externals/cache/**/* # https://github.com/dotnet/source-build/issues/4161
# fsharp
src/fsharp/src/fsi/fsi.res # Icon
# razor
src/razor/**/SampleApp/**/fonts/**/*.eot
src/razor/**/SampleApp/**/fonts/**/*.otf
src/razor/**/SampleApp/**/fonts/**/*.woff
# roslyn
src/roslyn/src/ExpressionEvaluator/Core/Source/ExpressionCompiler/Resources/WindowsProxy.winmd
# runtime
src/runtime/src/libraries/System.Diagnostics.EventLog/src/Messages/EventLogMessages.res # Icon
src/runtime/src/libraries/System.Speech/src/**/*.upsmap # https://github.com/dotnet/runtime/issues/81692
src/runtime/src/libraries/System.Text.Encoding.CodePages/src/Data/codepages.nlp # https://github.com/dotnet/runtime/issues/81693
src/runtime/src/native/external/brotli/common/dictionary.bin.br
# source-build-externals
src/source-build-externals/src/humanizer/src/Humanizer.Tests**/*.pfx
src/source-build-externals/src/newtonsoft-json/Src/Newtonsoft.Json.Tests/SpaceShipV2.bson
src/source-build-externals/src/azure-activedirectory-identitymodel-extensions-for-dotnet/build/strongNameBypass.reg # UTF16-LE text file
src/source-build-externals/src/azure-activedirectory-identitymodel-extensions-for-dotnet/build/strongNameBypass2.reg # UTF16-LE text file
src/source-build-externals/src/azure-activedirectory-identitymodel-extensions-for-dotnet/test/Certs/*.pfx
src/source-build-externals/src/azure-activedirectory-identitymodel-extensions-for-dotnet/test/Certs/*.cer

154
src/SourceBuild/content/eng/allowed-vmr-binaries.txt
View file

@ -1,154 +0,0 @@
# Contains the binaries that are allowed in the VMR
# This file is used by the Binary Tool to detect new binaries that get added to the VMR
# Import the allowed souce-build binaries (a stricter set).
import:allowed-sb-binaries.txt
**/testCert*.pfx
**/TestCert*.pfx
# arcade
src/arcade/src/Microsoft.DotNet.*.Tests/**/*
src/arcade/src/Microsoft.DotNet.NuGetRepack/tests/Resources/*.nupkg
src/arcade/src/Microsoft.DotNet.NuGetRepack/tests/Resources/.signature.p7s
# aspire
src/aspire/tests/Shared/TestCertificates/*.pfx
# aspnetcore
src/aspnetcore/src/submodules/MessagePack-CSharp/**/*.dll
src/aspnetcore/src/SignalR/clients/java/signalr/gradle/wrapper/gradle-wrapper.jar
src/aspnetcore/src/Components/**/testassets/**/*.woff*
src/aspnetcore/src/Security/Authentication/Negotiate/test/Negotiate.FunctionalTest/negotiateAuthCert.pfx
src/aspnetcore/src/**/test/**/*.cer
src/aspnetcore/src/Shared/TestCertificates/*.pfx
src/aspnetcore/src/Shared/TestCertificates/*.crt
# cecil
src/cecil/Test/Resources/assemblies/*.netmodule
src/cecil/Test/Resources/assemblies/*.winmd
src/cecil/Test/Resources/assemblies/*.exe
src/cecil/Test/Resources/assemblies/*.dll
src/cecil/Test/Resources/assemblies/*.pdb
src/cecil/Test/Resources/assemblies/*.mdb
src/cecil/rocks/Test/Resources/assemblies/*.dll
src/cecil/symbols/**/Test/Resources/assemblies/*.exe
src/cecil/symbols/**/Test/Resources/assemblies/*.pdb
src/cecil/symbols/**/Test/Resources/assemblies/*.dll
src/cecil/symbols/**/Test/Resources/assemblies/*.mdb
# diagnostics
src/diagnostics/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/**/*
src/diagnostics/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/**/*
# efcore
src/efcore/test/EFCore.Sqlite.FunctionalTests/northwind.db # https://github.com/dotnet/source-build/issues/4326
src/efcore/benchmark/EFCore.Sqlite.Benchmarks/AdventureWorks2014.db # https://github.com/dotnet/source-build/issues/4326
# fsharp
src/fsharp/tests/**/*.resources
src/fsharp/tests/**/*.dll
src/fsharp/tests/**/*.exe
src/fsharp/tests/fsharp/core/resources/chimes.wav
# msbuild
src/msbuild/src/Tasks.UnitTests/TestResources/*.pfx
src/msbuild/src/Tasks.UnitTests/AssemblyDependency/CacheFileSamples/Microsoft.VisualStudio.LanguageServices.Implementation.csprojAssemblyReference.cache
# nuget-client
src/nuget-client/test/EndToEnd/Packages/**/*.nupkg
src/nuget-client/test/EndToEnd/Packages/**/*.zip
src/nuget-client/test/EndToEnd/Packages/**/*.dll
src/nuget-client/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/compiler/resources/*.nupkg
src/nuget-client/test/NuGet.Clients.Tests/NuGet.PackageManagement.UI.Test/Resources/customMetadata.jpeg
src/nuget-client/test/NuGet.Core.Tests/NuGet.Commands.Test/compiler/resources/EmptyCertificateStore.p7b
src/nuget-client/test/NuGet.Core.Tests/NuGet.Protocol.Tests/compiler/resources/*.dll
src/nuget-client/test/NuGet.Core.FuncTests/NuGet.Packaging.FuncTest/compiler/resources/*.nupkg
src/nuget-client/test/NuGet.Core.FuncTests/NuGet.Packaging.FuncTest/compiler/resources/*.zip
src/nuget-client/test/NuGet.Core.FuncTests/Dotnet.Integration.Test/compiler/resources/*.nupkg
src/nuget-client/test/TestUtilities/Test.Utility/compiler/resources/*.crt
src/nuget-client/test/TestUtilities/Test.Utility/compiler/resources/.signature.p7s
src/nuget-client/test/TestUtilities/Test.Utility/compiler/resources/*.nupkg
src/nuget-client/test/TestUtilities/Test.Utility/compiler/resources/*.zip
# razor
src/razor/src/Razor/test/Microsoft.VisualStudio.Razor.IntegrationTests/TestFiles/BlazorProject.zip
src/razor/src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/Resources/project.razor.bin
# roslyn
src/roslyn/src/Compilers/Test/Resources/Core/**/*.metadata
src/roslyn/src/Compilers/Test/Resources/Core/**/*.winmd
src/roslyn/src/Compilers/Test/Resources/Core/**/*.mod
src/roslyn/src/Compilers/Test/Resources/Core/**/*.netmodule
src/roslyn/src/Compilers/Test/Resources/Core/**/*.obj
src/roslyn/src/Compilers/Test/Resources/Core/**/*.dll
src/roslyn/src/Compilers/Test/Resources/Core/**/*.exe
src/roslyn/src/Compilers/Test/Resources/Core/**/*.Dll
src/roslyn/src/Workspaces/MSBuildTest/Resources/Dlls/*.dll
src/roslyn/**/CodeAnalysisTest/**/*.res
src/roslyn/**/CodeAnalysisTest/**/*.blah
src/roslyn/**/CodeAnalysisTest/**/*.RES
# runtime
src/runtime/src/mono/mono/eglib/test/*.txt
src/runtime/src/mono/mono/tests/exiting/*.out
src/runtime/src/**/tests/**/*.res
src/runtime/src/**/tests/**/*.resources
src/runtime/src/tests/FunctionalTests/Android/Device_Emulator/gRPC/grpc-dotnet/testassets/Certs/InteropTests/server1.pfx
src/runtime/src/tests/FunctionalTests/Android/Device_Emulator/AOT_PROFILED/*.mibc
src/runtime/src/tests/FunctionalTests/Android/Device_Emulator/AOT_PROFILED/*.nettrace
src/runtime/src/libraries/System.Runtime.Serialization.Xml/tests/Canonicalization/baselines/ReaderWriter_C14N_BaselineXML_Binary.xml
src/runtime/src/libraries/System.Reflection.Metadata/tests/Resources/NetModule/*.mod
src/runtime/src/libraries/System.Reflection.Metadata/tests/Resources/WinRT/Lib.winmd
src/runtime/src/libraries/System.Reflection.Metadata/tests/Resources/Misc/CPPClassLibrary2.obj
src/runtime/src/libraries/System.Console/tests/TestData/ncursesFormats/s/screen-256color
src/runtime/src/libraries/System.Console/tests/TestData/ncursesFormats/x/xterm
src/runtime/src/mono/wasm/testassets/**/*.dat
src/runtime/src/mono/wasm/testassets/**/*.o
src/runtime/src/libraries/**/tests/**/*.dll
src/runtime/src/libraries/**/tests/**/*.exe
src/runtime/src/libraries/**/tests/**/*.pdb
# sdk
src/sdk/src/Assets/TestProjects/**/*.dat
src/sdk/src/Assets/TestProjects/**/*.cache
src/sdk/src/Assets/TestProjects/**/*.tlb
src/sdk/src/Assets/TestPackages/dotnet-new/nupkg_templates/*
src/sdk/test/Microsoft.DotNet.ShellShim.Tests/WpfBinaryTestAsssets/*.dll
src/sdk/test/Microsoft.NET.Sdk.Publish.Tasks.Tests/Resources/*.zip
# source-build-externals
src/source-build-externals/src/application-insights*/WEB/Src/WindowsServer/WindowsServer.Tests/**/*.dll
# symreader
src/symreader/src/PdbTestResources/Resources/*
# templating
src/templating/test/Microsoft.TemplateEngine.TestTemplates/nupkg_templates/*.nupkg
# test-templates
src/test-templates/Templates/**/*.nupkg
# vstest
src/vstest/samples/Microsoft.TestPlatform.*/Adapter/Microsoft.VisualStudio.TestPlatform.*.dll
src/vstest/temp/cpp/**/*.dll
src/vstest/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/TestAssets/*.exe
src/vstest/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/TestAssets/dotnetMac*
src/vstest/test/Microsoft.TestPlatform.Utilities.UnitTests/TestFiles/fullcovered.coverage
src/vstest/test/TestAssets/LegacySettingsUnitTestProject/DependencyAssembly/DependencyAssemblyForTest.dll
# winforms
src/winforms/src/System.Windows.Forms/tests/IntegrationTests/MauiTests/lib/*.dll
src/winforms/src/System.Windows.Forms/tests/IntegrationTests/WinformsControlsTest/Resources/media.mpg
src/winforms/src/System.Windows.Forms/tests/UnitTests/bitmaps/milkmateya01.emf
src/winforms/src/System.Windows.Forms/tests/UnitTests/TestResources/VB6/SimpleControl.vb6
src/winforms/src/System.Windows.Forms*/**/*.wmf
src/winforms/src/System.Windows.Forms.Design/src/Resources/colordlg.data
# wpf
src/wpf/src/Microsoft.DotNet.Wpf/src/ReachFramework/Resources/generated/*.resources
src/wpf/src/Microsoft.DotNet.Wpf/src/PresentationFramework/Resources/Hyphenation/Hyphen_en.lex
src/wpf/src/Microsoft.DotNet.Wpf/src/PresentationFramework/Resources/Hyphenation/Hyphen_en.hdict
src/wpf/src/Microsoft.DotNet.Wpf/src/Shared/Tracing/resources/*.BIN
src/wpf/src/Microsoft.DotNet.Wpf/src/Shared/Tracing/resources/*.bin

9
src/SourceBuild/content/eng/bootstrap/OverrideBootstrapVersions.props
View file

@ -1,9 +0,0 @@
<Project>
<PropertyGroup>
<!-- 7.0.0 produced improperly versioned runtime assets because the OfficialBuildId
was not set correctly. This is the actual shipping version that it should have been -->
<NonshippingRuntimeVersionFor700>7.0.4-servicing.23107.6</NonshippingRuntimeVersionFor700>
<MicrosoftNETCoreTestHostVersion>$(NonshippingRuntimeVersionFor700)</MicrosoftNETCoreTestHostVersion>
</PropertyGroup>
</Project>

193
src/SourceBuild/content/eng/bootstrap/buildBootstrapPreviouslySB.csproj
View file

@ -1,193 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="$(MSBuildProjectDirectory)/PackageVersions.props" />
<Import Project="$(BootstrapOverrideVersionsProps)" Condition="Exists('$(BootstrapOverrideVersionsProps)')" />
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<BaseOutputPath>$(MSBuildProjectDirectory)/artifacts/</BaseOutputPath>
<RestorePackagesPath>$(MSBuildProjectDirectory)/artifacts/restoredPkgs/</RestorePackagesPath>
<UnpackedTarPath>$(MSBuildProjectDirectory)/artifacts/unpacked/</UnpackedTarPath>
<NewTarballName>$(ArchiveDir)Private.SourceBuilt.Artifacts.Bootstrap.tar.gz</NewTarballName>
</PropertyGroup>
<ItemDefinitionGroup>
<PortablePackage>
<IsNative>false</IsNative>
</PortablePackage>
</ItemDefinitionGroup>
<ItemGroup>
<UnixRid Include="linux-x64" />
<UnixRid Include="linux-musl-x64" />
<UnixRid Include="linux-arm64" />
<UnixRid Include="linux-musl-arm64" />
<UnixRid Include="osx-x64" />
<UnixRid Include="osx-arm64" />
</ItemGroup>
<!-- These packages will be replaced with ms-built packages downloaded from official package feeds-->
<ItemGroup>
<RuntimePack Include="Microsoft.Aspnetcore.App.Runtime" Version="[$(MicrosoftAspNetCoreAppRuntimeVersion)]" />
<RuntimePack Include="Microsoft.NETCore.App.Crossgen2" Version="[$(MicrosoftNETCoreAppCrossgen2Version)]" />
<RuntimePack Include="Microsoft.NETCore.App.Host" Version="[$(MicrosoftNETCoreAppHostPackageVersion)]" />
<RuntimePack Include="Microsoft.NETCore.App.Runtime" Version="[$(MicrosoftNETCoreAppRuntimeVersion)]" />
<PortablePackage Include="Microsoft.DotNet.ILCompiler" Version="[$(MicrosoftDotNetILCompilerVersion)]" />
<PortablePackage Include="Microsoft.NETCore.DotNetAppHost" Version="[$(MicrosoftNETCoreDotNetAppHostVersion)]" />
<PortablePackage Include="Microsoft.NETCore.ILAsm" Version="[$(MicrosoftNETCoreILAsmVersion)]" />
<PortablePackage Include="Microsoft.NETCore.ILDAsm" Version="[$(MicrosoftNETCoreILDAsmVersion)]" />
<PortablePackage Include="Microsoft.NETCore.TestHost" Version="[$(MicrosoftNETCoreTestHostVersion)]" />
<PortablePackage Include="System.IO.Ports" Version="[$(RuntimeNativeSystemIOPortsVersion)]" IsNative="true" />
<!-- These packages don't actually exist -->
<ExcludedPackage Include="runtime.linux-musl-x64.runtime.native.System.IO.Ports" />
<ExcludedPackage Include="runtime.linux-musl-arm64.runtime.native.System.IO.Ports" />
</ItemGroup>
<Target Name="GetPackagesToDownload"
AfterTargets="CollectPackageDownloads"
Returns="@(PackageDownload)">
<ItemGroup>
<!-- Generate a cross-product between runtime packs and Unix RIDs -->
<RuntimePackWithUnixRid Include="@(RuntimePack)">
<UnixRid>%(UnixRid.Identity)</UnixRid>
</RuntimePackWithUnixRid>
<!-- Generate a cross-product between portable packages and Unix RIDs -->
<PortablePackageWithUnixRid Include="@(PortablePackage)">
<UnixRid>%(UnixRid.Identity)</UnixRid>
</PortablePackageWithUnixRid>
</ItemGroup>
<ItemGroup>
<!--
Generate package names for runtime packs by concatenating the base name with the Unix RID
(e.g. Microsoft.Aspnetcore.App.Runtime.linux-x64)
-->
<PackageWithName Include="@(RuntimePackWithUnixRid)">
<PackageName>%(RuntimePackWithUnixRid.Identity).%(RuntimePackWithUnixRid.UnixRid)</PackageName>
</PackageWithName>
<!--
Include the base name of each portable package (e.g. Microsoft.NETCore.ILAsm)
Exclude any that are native packages.
-->
<PackageWithName Include="@(PortablePackageWithUnixRid)" Condition=" '%(PortablePackageWithUnixRid.IsNative)' != 'true' ">
<PackageName>%(PortablePackageWithUnixRid.Identity)</PackageName>
</PackageWithName>
<!--
Generate Unix RID package names for portable packages by concatenating the base name with the Unix RID
(e.g. runtime.linux-x64.Microsoft.NETCore.ILAsm)
Do this for two groups: native and non-native packages. They have different naming conventions.
-->
<PackageWithName Include="@(PortablePackageWithUnixRid)" Condition=" '%(PortablePackageWithUnixRid.IsNative)' != 'true' ">
<PackageName>runtime.%(PortablePackageWithUnixRid.UnixRid).%(PortablePackageWithUnixRid.Identity)</PackageName>
</PackageWithName>
<PackageWithName Include="@(PortablePackageWithUnixRid)" Condition=" '%(PortablePackageWithUnixRid.IsNative)' == 'true' ">
<PackageName>runtime.%(PortablePackageWithUnixRid.UnixRid).runtime.native.%(PortablePackageWithUnixRid.Identity)</PackageName>
</PackageWithName>
</ItemGroup>
<ItemGroup>
<PackageDownload Include="@(PackageWithName -> '%(PackageName)')" />
<PackageDownload Remove="@(ExcludedPackage)" />
</ItemGroup>
</Target>
<Target Name="BuildBoostrapPreviouslySourceBuilt"
AfterTargets="Restore"
DependsOnTargets="GetPackagesToDownload">
<ItemGroup>
<RestoredNupkgs Include="$(RestorePackagesPath)**/*.nupkg" />
<PrevSBArchive Include="$(ArchiveDir)Private.SourceBuilt.Artifacts.*.tar.gz" />
</ItemGroup>
<!-- Copy restored nupkg files to root of restored packages dir so they're all in one place-->
<Copy SourceFiles="@(RestoredNupkgs)" DestinationFolder="$(RestorePackagesPath)" />
<!-- Check to ensure there is only one previously source-built archive -->
<Error Text="Multiple Private.SourceBuilt.Artifacts.*.tar.gz tarballs exists at $(ArchiveDir). Expecting only one."
Condition="'@(PrevSBArchive->Count())' != '1'" />
<!-- Unpack existing archive -->
<Message Text=" Unpacking existing tarball from %(PrevSBArchive.Identity)" Importance="High" />
<MakeDir Directories="$(UnpackedTarPath)" />
<Exec Command="tar -xzf %(PrevSBArchive.Identity) -C $(UnpackedTarPath)" />
<!-- Delete existing archive -->
<Message Text=" Deleting existing tarball: %(PrevSBArchive.Identity)" Importance="High" />
<Delete Files="%(PrevSBArchive.Identity)" />
<!-- Copy files specified in package references above from restored package dir to unpacked archive dir -->
<Message Text=" Replacing restored files in $(UnpackedTarPath)" Importance="High" />
<MSBuild Projects="$(MSBuildProjectFile)"
Targets="CopyDownloadedPackage"
Properties="SourcePath=$(RestorePackagesPath);DestinationPath=$(UnpackedTarPath);PackageName=%(PackageDownload.Identity);PackageVersion=%(PackageDownload.Version)" />
<!-- override PVP with bootstrap-override package versions -->
<Message Text=" Overriding previously-source-built package versions with $(BootstrapOverrideVersionsProps)" Importance="High" Condition="Exists('$(BootstrapOverrideVersionsProps)')" />
<ReadLinesFromFile File="$(UnpackedTarPath)/PackageVersions.props">
<Output TaskParameter="Lines" ItemName="OriginalPackageVersionLines" />
</ReadLinesFromFile>
<ReadLinesFromFile File="$(BootstrapOverrideVersionsProps)">
<Output TaskParameter="Lines" ItemName="BootstrapPackageVersionLines" />
</ReadLinesFromFile>
<ItemGroup>
<OriginalPackageVersionLines Remove="&lt;/Project&gt;" />
<BootstrapPackageVersionLines Remove="&lt;Project&gt;" />
</ItemGroup>
<WriteLinesToFile File="$(UnpackedTarPath)/PackageVersions.props"
Lines="@(OriginalPackageVersionLines)"
Overwrite="true"
Condition="Exists('$(BootstrapOverrideVersionsProps)')"
/>
<WriteLinesToFile File="$(UnpackedTarPath)/PackageVersions.props"
Lines="@(BootstrapPackageVersionLines)"
Overwrite="false"
Condition="Exists('$(BootstrapOverrideVersionsProps)')"
/>
<!-- Repack tarball with new bootstrap name -->
<Message Text=" Repacking tarball to $(NewTarballName)" Importance="High" />
<Exec Command="tar --numeric-owner -czf $(NewTarballName) *.nupkg *.props SourceBuildReferencePackages/" WorkingDirectory="$(UnpackedTarPath)" />
</Target>
<Target Name="CopyDownloadedPackage">
<!--
Copy downloaded package to the output path.
Note: The package version may be different than the version specified
since the source-build build number can be different than the official
package build number.
-->
<ItemGroup>
<SourceFileName Include="$(SourcePath)$(PackageName.ToLower()).*.nupkg" />
</ItemGroup>
<PropertyGroup>
<DestinationFileName>@(SourceFileName->'%(Filename)')</DestinationFileName>
<NewVersion>$(DestinationFileName.Replace('$(PackageName.ToLower()).',''))</NewVersion>
</PropertyGroup>
<Copy
SourceFiles="@(SourceFileName)"
DestinationFiles="$(DestinationPath)$(PackageName).$(NewVersion).nupkg" />
<!--
Update the PackageVersions.props if restored version is
different than the specified version.
-->
<PropertyGroup>
<VersionTag>$([System.String]::concat('%3C','$(PackageName)','Version','%3E').Replace('.',''))</VersionTag>
<PackageVersionTag>$([System.String]::concat('%3C','$(PackageName)','PackageVersion','%3E').Replace('.',''))</PackageVersionTag>
<FilePath>$(DestinationPath)PackageVersions.props</FilePath>
</PropertyGroup>
<WriteLinesToFile
File="$(FilePath)"
Lines="$([System.IO.File]::ReadAllText($(FilePath)).Replace('$(VersionTag)$(PackageVersion)','$(VersionTag)$(NewVersion)').Replace('$(PackageVersionTag)$(PackageVersion)','$(PackageVersionTag)$(NewVersion)'))"
Overwrite="true"
Condition=" '$(PackageVersion)' != '$(NewVersion)' " />
</Target>
</Project>

118
src/SourceBuild/content/eng/build.ps1
View file

@ -1,118 +0,0 @@
[CmdletBinding(PositionalBinding=$false)]
Param(
# Common settings
[switch][Alias('bl')]$binaryLog,
[string][Alias('c')]$configuration = "Release",
[string][Alias('v')]$verbosity = "minimal",
# Actions
[switch]$clean,
[switch][Alias('h')]$help,
[switch][Alias('t')]$test,
# Advanced settings
[switch]$buildRepoTests,
[switch]$ci,
[switch][Alias('cwb')]$cleanWhileBuilding,
[switch][Alias('nobl')]$excludeCIBinarylog,
[switch] $prepareMachine,
[switch] $dev,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
)
function Get-Usage() {
Write-Host "Common settings:"
Write-Host " -binaryLog Output binary log (short: -bl)"
Write-Host " -configuration <value> Build configuration: 'Debug' or 'Release' (short: -c). [Default: Release]"
Write-Host " -verbosity <value> Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
Write-Host ""
Write-Host "Actions:"
Write-Host " -clean Clean the solution"
Write-Host " -help Print help and exit (short: -h)"
Write-Host " -test Run tests (repo tests omitted by default) (short: -t)"
Write-Host ""
Write-Host "Advanced settings:"
Write-Host " -buildRepoTests Build repository tests"
Write-Host " -ci Set when running on CI server"
Write-Host " -cleanWhileBuilding Cleans each repo after building (reduces disk space usage, short: -cwb)"
Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)"
Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build"
Write-Host " -dev Use -dev or -ci versioning instead of .NET official build versions"
Write-Host ""
}
$useGlobalNuGetCache=$false
. $PSScriptRoot\common\tools.ps1
if ($help) {
Get-Usage
exit 0
}
$project = Join-Path $RepoRoot "build.proj"
$arguments = @()
$targets = "/t:Build"
# This repo uses the VSTest integration instead of the Arcade Test target
if ($test) {
$project = Join-Path (Join-Path $RepoRoot "test") "tests.proj"
$targets += ";VSTest"
}
if ($buildRepoTests) {
$arguments += "/p:DotNetBuildTests=true"
}
if ($cleanWhileBuilding) {
$arguments += "/p:CleanWhileBuilding=true"
}
if ($dev) {
$arguments += "/p:UseOfficialBuildVersioning=false"
}
function Build {
InitializeToolset
# Manually unset NUGET_PACKAGES as InitializeToolset sets it unconditionally.
# The env var shouldn't be set so that the RestorePackagesPath msbuild property is respected.
$env:NUGET_PACKAGES=''
$bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
MSBuild -restore `
$project `
$bl `
$targets `
/p:Configuration=$configuration `
@properties `
@arguments
}
try {
if ($clean) {
if (Test-Path $ArtifactsDir) {
Remove-Item -Recurse -Force $ArtifactsDir
Write-Host 'Artifacts directory deleted.'
}
exit 0
}
if ($ci) {
if (-not $excludeCIBinarylog) {
$binaryLog = $true
}
}
Build
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Build' -Message $_
ExitWithExitCode 1
}
ExitWithExitCode 0

139
src/SourceBuild/content/eng/detect-binaries.sh
View file

@ -1,139 +0,0 @@
#!/usr/bin/env bash
### Usage: $0
###
### Prepares and runs the binary tooling to detect binaries in the VMR. Default behavior is to report any binaries
### not found in the allowed-binaries file. To remove binaries not specified in the allowed-binaries file, pass --clean.
###
### Options:
### --clean Clean the VMR of binaries not in the specified allowed-binaries file.
### --allowed-binaries-file Path to the file containing the list of binaries to be
### ignored for either cleaning or validating.
### Defaults to eng/allowed-vmr-binaries.txt.
### --log-level <level> Set the log level for the binary tooling. Defaults to Debug.
### --with-packages Use the specified directory as the packages source feed.
### Defaults to online dotnet-public and dotnet-libraries feeds.
### --with-sdk Use the specified directory as the dotnet SDK.
### Defaults to .dotnet.
set -euo pipefail
IFS=$'\n\t'
source="${BASH_SOURCE[0]}"
REPO_ROOT="$( cd -P "$( dirname "$0" )/../" && pwd )"
BINARY_TOOL="$REPO_ROOT/eng/tools/BinaryToolKit"
function print_help () {
sed -n '/^### /,/^$/p' "$source" | cut -b 5-
}
defaultDotnetSdk="$REPO_ROOT/.dotnet"
defaultAllowedBinariesFile="$REPO_ROOT/eng/allowed-vmr-binaries.txt"
# Set default values
allowedBinariesFile=$defaultAllowedBinariesFile
mode='validate'
logLevel='Debug'
propsDir=''
packagesDir=''
restoreSources=''
dotnetSdk=$defaultDotnetSdk
positional_args=()
while :; do
if [ $# -le 0 ]; then
break
fi
lowerI="$(echo "$1" | awk '{print tolower($0)}')"
case $lowerI in
"-?"|-h|--help)
print_help
exit 0
;;
--clean)
mode="clean"
;;
--allowed-binaries-file)
allowedBinariesFile=$2
shift
;;
--log-level)
logLevel=$2
shift
;;
--with-packages)
packagesDir=$2
if [ ! -d "$packagesDir" ]; then
echo "ERROR: The specified packages directory does not exist."
exit 1
elif [ ! -f "$packagesDir/PackageVersions.props" ]; then
echo "ERROR: The specified packages directory does not contain PackageVersions.props."
exit 1
fi
shift
;;
--with-sdk)
dotnetSdk=$2
if [ ! -d "$dotnetSdk" ]; then
echo "Custom SDK directory '$dotnetSdk' does not exist"
exit 1
fi
if [ ! -x "$dotnetSdk/dotnet" ]; then
echo "Custom SDK '$dotnetSdk/dotnet' does not exist or is not executable"
exit 1
fi
shift
;;
*)
positional_args+=("$1")
;;
esac
shift
done
function ParseBinaryArgs
{
# Check allowed binaries file
if [ ! -f "$allowedBinariesFile" ]; then
echo "ERROR: The specified allowed-binaries file does not exist."
exit 1
fi
# Check dotnet sdk
if [ "$dotnetSdk" == "$defaultDotnetSdk" ]; then
if [ ! -d "$dotnetSdk" ]; then
. "$REPO_ROOT/eng/common/tools.sh"
InitializeDotNetCli true
fi
else if [ ! -x "$dotnetSdk/dotnet" ]; then
echo "'$dotnetSdk/dotnet' does not exist or is not executable"
exit 1
fi
fi
# Check the packages directory
if [ -z "$packagesDir" ]; then
# Use dotnet-public and dotnet-libraries feeds as the default packages source feeds
restoreSources="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries/nuget/v3/index.json"
else
restoreSources=$(realpath ${packagesDir})
fi
}
function RunBinaryTool
{
targetDir="$REPO_ROOT"
outputDir="$REPO_ROOT/artifacts/log/binary-report"
BinaryToolCommand=""$dotnetSdk/dotnet" run --project "$BINARY_TOOL" -c Release --property:RestoreSources="$restoreSources" "$mode" "$targetDir" -o "$outputDir" -ab "$allowedBinariesFile" -l "$logLevel""
if [ -n "$packagesDir" ]; then
BinaryToolCommand=""$BinaryToolCommand" -p CustomPackageVersionsProps="$packagesDir/PackageVersions.props""
fi
# Run the Binary Tool
eval "$BinaryToolCommand"
}
ParseBinaryArgs
RunBinaryTool

20
src/SourceBuild/content/eng/extract-sdk-archive.proj
View file

@ -1,20 +0,0 @@
<Project Sdk="Microsoft.Build.NoTargets">
<PropertyGroup>
<TargetFramework>$(NetCurrent)</TargetFramework>
</PropertyGroup>
<Target Name="ExtractSdkArchive"
BeforeTargets="AfterBuild"
DependsOnTargets="DetermineSourceBuiltSdkVersion"
Inputs="$(SdkTarballPath)"
Outputs="$(DotNetSdkExtractDir)">
<MakeDir Directories="$(DotNetSdkExtractDir)" />
<Exec Condition="'$(ArchiveExtension)' == '.tar.gz'"
Command="tar -xzf $(SdkTarballPath) -C $(DotNetSdkExtractDir)" />
<Unzip Condition="'$(ArchiveExtension)' == '.zip'"
SourceFiles="$(SdkTarballPath)"
DestinationFolder="$(DotNetSdkExtractDir)" />
</Target>
</Project>

249
src/SourceBuild/content/eng/finish-source-only.proj
View file

@ -1,249 +0,0 @@
<Project Sdk="Microsoft.Build.NoTargets">
<PropertyGroup>
<!-- Need to set to false to calculate RepositoryCommit. -->
<EnableSourceControlManagerQueries>false</EnableSourceControlManagerQueries>
<TargetFramework>$(NetCurrent)</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" />
<ProjectReference Include="$(TasksDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection\Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj" />
<ProjectReference Include="$(RepositoryEngineeringDir)extract-sdk-archive.proj" />
<ProjectReference Include="$(RepositoryEngineeringDir)merge-asset-manifests.proj" />
</ItemGroup>
<!-- After building, generate a prebuilt usage report. -->
<Target Name="ReportPrebuiltUsage"
AfterTargets="Build"
Condition="'$(SkipReportPrebuiltUsage)' != 'true'">
<MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
</Target>
<!--
Determine symbols tarball names and discover all intermediate symbols,
to be used as inputs and outputs of symbols repackaging targets.
-->
<Target Name="DetermineSymbolsTargetsInputsAndOutputs"
AfterTargets="Build"
DependsOnTargets="DetermineSourceBuiltSdkVersion">
<PropertyGroup>
<UnifiedSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
<SdkSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
</PropertyGroup>
<ItemGroup>
<IntermediateSymbol Include="$(IntermediateSymbolsRootDir)**/*" />
</ItemGroup>
</Target>
<!-- After building, repackage symbols into a single tarball. -->
<Target Name="RepackageSymbols"
AfterTargets="Build"
DependsOnTargets="DetermineSymbolsTargetsInputsAndOutputs"
Inputs="@(IntermediateSymbol)"
Outputs="$(UnifiedSymbolsTarball)">
<MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(UnifiedSymbolsTarball)'))" />
<Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
WorkingDirectory="$(IntermediateSymbolsRootDir)" />
<Message Importance="High" Text="Packaged all symbols in '$(UnifiedSymbolsTarball)'" />
</Target>
<!-- After building, create the sdk symbols tarball. -->
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.CreateSdkSymbolsLayout" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
<Target Name="CreateSdkSymbolsTarball"
AfterTargets="Build"
DependsOnTargets="RepackageSymbols"
Inputs="@(IntermediateSymbol);$(SdkTarballPath)"
Outputs="$(SdkSymbolsTarball)">
<PropertyGroup>
<IntermediateSdkSymbolsLayout>$(BaseIntermediateOutputPath)SdkSymbols</IntermediateSdkSymbolsLayout>
</PropertyGroup>
<CreateSdkSymbolsLayout SdkLayoutPath="$(DotNetSdkExtractDir)"
AllSymbolsPath="$(IntermediateSymbolsRootDir)"
SdkSymbolsLayoutPath="$(IntermediateSdkSymbolsLayout)"
FailOnMissingPDBs="false" />
<Exec Command="tar --numeric-owner -czf $(SdkSymbolsTarball) *"
WorkingDirectory="$(IntermediateSdkSymbolsLayout)" />
<Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
<RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
</Target>
<!--
Dev scenario: rewrite a prebuilt-report. This makes it easy to add data to an existing
prebuilt report without performing another full build. This doesn't reevalutate which packages
are prebuilts or search the projects: it uses the existing usage.xml file to generate report
XMLs based on the info put in artifacts/.
-->
<Target Name="RewritePrebuiltUsageReport">
<MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj"
Targets="ReportPrebuiltUsage" />
</Target>
<UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.CheckForPoison" AssemblyFile="$(MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly)" TaskFactory="TaskHostFactory" Condition="'$(EnablePoison)' == 'true'" />
<Target Name="ReportPoisonUsage"
AfterTargets="Build"
Condition="'$(EnablePoison)' == 'true'"
Inputs="$(MSBuildProjectFullPath)"
Outputs="$(BaseIntermediateOutputPath)ReportPoisonUsage.complete" >
<ItemGroup>
<!-- Exclude the Private.SourceBuilt.Artifacts archive from poison usage scan. -->
<PoisonFileToCheck Include="$(ArtifactsAssetsDir)*$(ArchiveExtension)" />
<PoisonFileToCheck Remove="$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName)*" />
<!-- Include shipping nuget packages. -->
<PoisonFileToCheck Include="$(ArtifactsShippingPackagesDir)*.nupkg" />
<!-- Add and mark SBRP packages to validate that they have the correct poison attribute. -->
<PoisonFileToCheck Include="$(ReferencePackagesDir)**\*.nupkg" IsSourceBuildReferencePackage="true" />
</ItemGroup>
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Checking @(PoisonFileToCheck) for poisoned files." />
<CheckForPoison FilesToCheck="@(PoisonFileToCheck)"
ProjectDirPath="$(RepoRoot)"
HashCatalogFilePath="$(PoisonReportDataFile)"
MarkerFileName="$(PoisonMarkerFile)"
PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done checking for poison." />
<MakeDir Directories="$(BaseIntermediateOutputPath)" />
<Touch Files="$(BaseIntermediateOutputPath)ReportPoisonUsage.complete" AlwaysCreate="true">
<Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
</Touch>
</Target>
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.WriteUsageBurndownData" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
<Target Name="GeneratePrebuiltBurndownData"
Inputs="$(MSBuildProjectFullPath)"
Outputs="$(BaseIntermediateOutputPath)GeneratePrebuiltBurndownData.complete" >
<PropertyGroup>
<PrebuiltBurndownDataFile>$(PackageReportDir)PrebuiltBurndownData.csv</PrebuiltBurndownDataFile>
</PropertyGroup>
<WriteUsageBurndownData RootDirectory="$(RepoRoot)"
PrebuiltBaselineFile="$(BaselineDataFile)"
OutputFilePath="$(PrebuiltBurndownDataFile)" />
<MakeDir Directories="$(BaseIntermediateOutputPath)" />
<Touch Files="$(BaseIntermediateOutputPath)GeneratePrebuiltBurndownData.complete" AlwaysCreate="true">
<Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
</Touch>
</Target>
<Target Name="CreatePrebuiltsTarball"
AfterTargets="Build"
DependsOnTargets="
CheckIfPrebuiltsExistToPack;
CreatePrebuiltsTarballIfPrebuiltsExist;
ErrorOnPrebuilts"/>
<Target Name="CheckIfPrebuiltsExistToPack">
<!-- Directory existence doesn't mean there are files inside. Use a pattern to find files. -->
<ItemGroup>
<PrebuiltFile Include="$(ResultingPrebuiltPackagesDir)**" />
</ItemGroup>
<Message Text="Found @(PrebuiltFile->Count()) files in prebuilt packages dir." Importance="High" />
</Target>
<Target Name="CreatePrebuiltsTarballIfPrebuiltsExist"
DependsOnTargets="DetermineSourceBuiltSdkVersion"
Condition="'@(PrebuiltFile)' != ''">
<PropertyGroup>
<PrebuiltsTarball>$(ArtifactsAssetsDir)$(SourceBuiltPrebuiltsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</PrebuiltsTarball>
<PrebuiltsTarballWorkingDir>$(ResultingPrebuiltPackagesDir)</PrebuiltsTarballWorkingDir>
</PropertyGroup>
<MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(PrebuiltsTarball)'))" />
<Exec Command="tar --numeric-owner -zcf $(PrebuiltsTarball) -C $(PrebuiltsTarballWorkingDir) ." />
<Message Text="Tarball '$(PrebuiltsTarball)' was successfully created from '$(PrebuiltsTarballWorkingDir)'" Importance="High" />
</Target>
<Target Name="ErrorOnPrebuilts"
Condition="'@(PrebuiltFile)' != '' and '$(SkipErrorOnPrebuilts)' != 'true'">
<Error Text="@(PrebuiltFile->Count()) Prebuilts Exist" />
</Target>
<Target Name="GetInputsOutputForCreatePrivateSourceBuiltArtifactsArchive"
DependsOnTargets="DetermineSourceBuiltSdkVersion;ResolveProjectReferences">
<!-- Inputs: Packages to include in the tarball -->
<ItemGroup>
<ArtifactsPackageToBundle Include="$(ArtifactsShippingPackagesDir)**;
$(ArtifactsNonShippingPackagesDir)**"
Condition="!$([System.String]::Copy('%(Identity)').EndsWith('.symbols.nupkg'))" />
<ReferencePackageToBundle Include="$(ReferencePackagesDir)**"
Condition="!$([System.String]::Copy('%(Identity)').EndsWith('.symbols.nupkg'))" />
<MergedAssetManifest Include="$(MergedAssetManifestOutputPath)" />
</ItemGroup>
<PropertyGroup>
<!-- Create a layout directory for the files that are to be included in the artifacts tarball. -->
<SourceBuiltLayoutDir>$([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', 'artifacts-layout'))</SourceBuiltLayoutDir>
<!-- Outputs -->
<SourceBuiltTarballName>$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SourceBuiltTarballName>
<SourceBuiltVersionName>$(SourceBuiltLayoutDir).version</SourceBuiltVersionName>
<AllPackageVersionsPropsName>$(SourceBuiltLayoutDir)PackageVersions.props</AllPackageVersionsPropsName>
<SourceBuiltMergedAssetManifestName>$(SourceBuiltLayoutDir)%(MergedAssetManifest.Filename)%(MergedAssetManifest.Extension)</SourceBuiltMergedAssetManifestName>
</PropertyGroup>
</Target>
<!-- Create the SourceBuilt.Private.Artifacts archive when building source-only. -->
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WritePackageVersionsProps" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
<Target Name="CreatePrivateSourceBuiltArtifactsArchive"
AfterTargets="Build"
DependsOnTargets="GetInputsOutputForCreatePrivateSourceBuiltArtifactsArchive"
Inputs="@(ArtifactsPackageToBundle);@(ReferencePackageToBundle);@(MergedAssetManifest)"
Outputs="$(SourceBuiltTarballName);$(SourceBuiltVersionName);$(AllPackageVersionsPropsName);$(SourceBuiltMergedAssetManifestName)">
<!-- Copy packages to layout directory. Since there are a large number of files,
this will use symlinks instead of copying files to make this execute quickly. -->
<Copy SourceFiles="@(ArtifactsPackageToBundle)"
DestinationFolder="$(SourceBuiltLayoutDir)"
UseSymbolicLinksIfPossible="true" />
<Copy SourceFiles="@(ReferencePackageToBundle)"
DestinationFolder="$(SourceBuiltLayoutDir)SourceBuildReferencePackages"
UseSymbolicLinksIfPossible="true" />
<!-- Content of the .version file to include in the tarball -->
<ItemGroup>
<VersionFileContent Include="$(RepositoryCommit);$(SourceBuiltSdkVersion)" />
</ItemGroup>
<WriteLinesToFile File="$(SourceBuiltVersionName)"
Lines="@(VersionFileContent)"
Overwrite="true" />
<!-- Copy the merged asset manifest into the tarball -->
<Copy SourceFiles="$(MergedAssetManifestOutputPath)"
DestinationFolder="$(SourceBuiltLayoutDir)"
UseSymbolicLinksIfPossible="true" />
<!-- non-rid-specific versions of RID-specific version variables to use for bootstrapping -->
<ItemGroup>
<ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppRuntimeVersion" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
<ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppHostPackageVersion" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
<ExtraPackageVersionPropsPackageInfo Include="MicrosoftAspNetCoreAppRuntimeVersion" Version="%24(MicrosoftAspNetCoreAppRefPackageVersion)" />
<ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppCrossgen2Version" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
</ItemGroup>
<!-- Create a PackageVersions.props file that includes entries for all packages. -->
<WritePackageVersionsProps NuGetPackages="@(ArtifactsPackageToBundle)"
ExtraProperties="@(ExtraPackageVersionPropsPackageInfo)"
VersionPropsFlowType="AllPackages"
OutputPath="$(AllPackageVersionsPropsName)" />
<Exec Command="tar --numeric-owner -czhf $(SourceBuiltTarballName) $([System.IO.Path]::GetFileName('$(SourceBuiltVersionName)')) *"
WorkingDirectory="$(SourceBuiltLayoutDir)" />
<Message Importance="High" Text="Packaged source-built artifacts to $(SourceBuiltTarballName)" />
</Target>
</Project>

22
src/SourceBuild/content/eng/init-cross-build.proj
View file

@ -1,22 +0,0 @@
<Project Sdk="Microsoft.Build.NoTargets">
<PropertyGroup>
<TargetFramework>$(NetCurrent)</TargetFramework>
</PropertyGroup>
<Target Name="GenerateRootFs"
AfterTargets="Build">
<PropertyGroup>
<ArmEnvironmentVariables Condition="'$(ArmEnvironmentVariables)' == ''">ROOTFS_DIR=$(ArtifactsObjDir)crossrootfs/arm</ArmEnvironmentVariables>
<ArmEnvironmentVariables Condition="'$(Platform)' == 'armel'">ROOTFS_DIR=$(ArtifactsObjDir)crossrootfs/armel</ArmEnvironmentVariables>
</PropertyGroup>
<Exec Command="$(RepositoryEngineeringDir)common/cross/build-rootfs.sh"
EnvironmentVariables="$(ArmEnvironmentVariables)"
Condition="'$(TargetArchitecture)' != 'armel' and '$(BuildArchitecture)' != '$(TargetArchitecture)'" />
<Exec Command="$(RepositoryEngineeringDir)common/cross/armel/tizen-build-rootfs.sh"
EnvironmentVariables="$(ArmEnvironmentVariables)"
Condition="'$(TargetArchitecture)' == 'armel'" />
</Target>
</Project>

39
src/SourceBuild/content/eng/init-poison.proj
View file

@ -1,39 +0,0 @@
<Project Sdk="Microsoft.Build.NoTargets">
<PropertyGroup>
<TargetFramework>$(NetCurrent)</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="$(TasksDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection\Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj" />
</ItemGroup>
<UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.MarkAndCatalogPackages" AssemblyFile="$(MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly)" TaskFactory="TaskHostFactory" />
<Target Name="PoisonPrebuiltPackages"
AfterTargets="Build"
Inputs="$(MSBuildProjectFullPath)"
Outputs="$(BaseIntermediateOutputPath)PoisonPrebuiltPackages.complete">
<PropertyGroup>
<SourceBuiltPoisonMarkerFile>.source-built.xml</SourceBuiltPoisonMarkerFile>
<SourceBuiltPoisonReportDataFile>$(PackageReportDir)poison-source-built-catalog.xml</SourceBuiltPoisonReportDataFile>
</PropertyGroup>
<ItemGroup>
<PrebuiltPackages Include="$(PrebuiltPackagesPath)**/*.nupkg" />
<PrebuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)**/*.nupkg" />
</ItemGroup>
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Poisoning existing packages for leak detection." />
<MarkAndCatalogPackages PackagesToMark="@(PrebuiltPackages)" CatalogOutputFilePath="$(PoisonReportDataFile)" MarkerFileName="$(PoisonMarkerFile)" />
<MarkAndCatalogPackages PackagesToMark="@(PrebuiltSourceBuiltPackages)" CatalogOutputFilePath="$(SourceBuiltPoisonReportDataFile)" MarkerFileName="$(SourceBuiltPoisonMarkerFile)" />
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done poisoning." />
<MakeDir Directories="$(BaseIntermediateOutputPath)" />
<Touch Files="$(BaseIntermediateOutputPath)PoisonPrebuiltPackages.complete" AlwaysCreate="true">
<Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
</Touch>
</Target>
</Project>

97
src/SourceBuild/content/eng/init-source-only.proj
View file

@ -1,97 +0,0 @@
<Project>
<!-- This project must not use any prebuilts -->
<PropertyGroup>
<SkipArcadeSdkImport>true</SkipArcadeSdkImport>
<!-- Fake, to satisfy the SDK. -->
<TargetFramework>netstandard2.0</TargetFramework>
<DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
<EnableDefaultItems>false</EnableDefaultItems>
</PropertyGroup>
<Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
<Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
<Target Name="Build"
DependsOnTargets="
UnpackTarballs;
BuildMSBuildSdkResolver;
ExtractToolsetPackages" />
<Target Name="UnpackTarballs"
Inputs="$(MSBuildProjectFullPath)"
Outputs="$(BaseIntermediateOutputPath)UnpackTarballs.complete" >
<PropertyGroup>
<ExternalTarballsDir>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'archive'))</ExternalTarballsDir>
</PropertyGroup>
<MakeDir Directories="$(PrebuiltSourceBuiltPackagesPath)" Condition="'$(CustomPrebuiltSourceBuiltPackagesPath)' == ''" />
<Exec Command="tar -xzf $(ExternalTarballsDir)$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)"
WorkingDirectory="$(PrebuiltSourceBuiltPackagesPath)"
Condition="'$(CustomPrebuiltSourceBuiltPackagesPath)' == ''" />
<!--
Check for a prebuilt dependency tarball and extract if exists. If there isn't one, we expect
the build to be working without prebuilts.
-->
<ItemGroup>
<SourceBuiltPrebuiltsTarballFile Include="$(ExternalTarballsDir)$(SourceBuiltPrebuiltsTarballName).*$(ArchiveExtension)" />
</ItemGroup>
<Exec Command="tar -xzf %(SourceBuiltPrebuiltsTarballFile.FullPath)"
WorkingDirectory="$(PrebuiltPackagesPath)"
Condition="'@(SourceBuiltPrebuiltsTarballFile)' != ''" />
<!-- Copy SBRP packages to reference packages location -->
<ItemGroup>
<UnpackedSourceBuildReferencePackages Include="$(PrebuiltSourceBuiltPackagesPath)SourceBuildReferencePackages/*"/>
</ItemGroup>
<Move SourceFiles="@(UnpackedSourceBuildReferencePackages)" DestinationFiles="$(ReferencePackagesDir)%(Filename)%(Extension)" />
<MakeDir Directories="$(BaseIntermediateOutputPath)" />
<Touch Files="$(BaseIntermediateOutputPath)UnpackTarballs.complete" AlwaysCreate="true">
<Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
</Touch>
</Target>
<!-- Build the custom msbuild sdk resolver. -->
<Target Name="BuildMSBuildSdkResolver"
DependsOnTargets="UnpackTarballs"
Inputs="$(MSBuildProjectFullPath)"
Outputs="$(BaseIntermediateOutputPath)BuildMSBuildSdkResolver.complete">
<MSBuild Projects="$(TasksDir)Microsoft.DotNet.UnifiedBuild.MSBuildSdkResolver\Microsoft.DotNet.UnifiedBuild.MSBuildSdkResolver.csproj"
Targets="Restore"
Properties="MSBuildRestoreSessionId=$([System.Guid]::NewGuid())" />
<MSBuild Projects="$(TasksDir)Microsoft.DotNet.UnifiedBuild.MSBuildSdkResolver\Microsoft.DotNet.UnifiedBuild.MSBuildSdkResolver.csproj"
Targets="Build" />
<MakeDir Directories="$(BaseIntermediateOutputPath)" />
<Touch Files="$(BaseIntermediateOutputPath)BuildMSBuildSdkResolver.complete" AlwaysCreate="true">
<Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
</Touch>
</Target>
<!-- Extract toolset packages into the bootstrap folder -->
<Target Name="ExtractToolsetPackages"
DependsOnTargets="UnpackTarballs"
Inputs="$(MSBuildProjectFullPath)"
Outputs="$(BaseIntermediateOutputPath)ExtractToolsetPackages.complete">
<ItemGroup>
<ToolsetPackage Include="Microsoft.DotNet.Arcade.Sdk" SourceFolder="$(PrebuiltSourceBuiltPackagesPath)" Version="$(ARCADE_BOOTSTRAP_VERSION)" />
<ToolsetPackage Include="Microsoft.Build.NoTargets" SourceFolder="$(ReferencePackagesDir)" Version="$(NOTARGETS_BOOTSTRAP_VERSION)" />
<ToolsetPackage Include="Microsoft.Build.Traversal" SourceFolder="$(ReferencePackagesDir)" Version="$(TRAVERSAL_BOOTSTRAP_VERSION)" />
</ItemGroup>
<Unzip SourceFiles="%(ToolsetPackage.SourceFolder)%(ToolsetPackage.Identity).%(ToolsetPackage.Version).nupkg"
DestinationFolder="$(BootstrapPackagesDir)$([System.String]::Copy('%(ToolsetPackage.Identity)').ToLowerInvariant())/%(ToolsetPackage.Version)"
SkipUnchangedFiles="true" />
<MakeDir Directories="$(BaseIntermediateOutputPath)" />
<Touch Files="$(BaseIntermediateOutputPath)ExtractToolsetPackages.complete" AlwaysCreate="true">
<Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
</Touch>
</Target>
</Project>

27
src/SourceBuild/content/eng/merge-asset-manifests.proj
View file

@ -1,27 +0,0 @@
<Project Sdk="Microsoft.Build.NoTargets">
<PropertyGroup>
<TargetFramework>$(NetCurrent)</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" />
</ItemGroup>
<!-- Create a merge manifest from the individual repository manifest files. -->
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
<Target Name="MergeAssetManifests" AfterTargets="Build">
<ItemGroup>
<RepoAssetManifest Include="$(AssetManifestsIntermediateDir)\**\*.xml" />
</ItemGroup>
<Error Text="Couldn't find any repository asset manifest file. Make sure to build the repositories before invoking this target." Condition="'@(RepoAssetManifest)' == ''" />
<!-- It's OK for the VmrBuildNumber to be empty -->
<Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests
AssetManifest="@(RepoAssetManifest)"
MergedAssetManifestOutputPath="$(MergedAssetManifestOutputPath)"
VmrBuildNumber="$(BUILD_BUILDNUMBER)" />
</Target>
</Project>

91
src/SourceBuild/content/eng/pipelines/ci.yml
View file

@ -1,91 +0,0 @@
# This yml is used by these pipelines and triggers:
# NOTE: the triggers are defined in the Azure DevOps UI as they are too complex
#
# - dotnet-source-build (public)
# https://dev.azure.com/dnceng-public/public/_build?definitionId=240
# - PR: ultralite build
# - CI: release/* only, every batched commit, full build
# - Schedule: main only, full build
#
# - dotnet-unified-build (public)
# https://dev.azure.com/dnceng-public/public/_build?definitionId=278
# - PR: lite build
# - CI: release/* only, every batched commit, full build
# - Schedule: main only, full build
#
# - dotnet-source-build (internal)
# https://dev.azure.com/dnceng/internal/_build?definitionId=1219
# - PR: ultralite build
# - CI: release/* and internal/release/* only, every batched commit, full build
# - Schedule: main only, full build
#
# - dotnet-source-build-lite (internal)
# https://dev.azure.com/dnceng/internal/_build?definitionId=1299
# - PR: release/* and main, lite build, on-demand trigger
# - CI: main only, every batched commit, lite build
#
# - dotnet-unified-build (internal)
# https://dev.azure.com/dnceng/internal/_build?definitionId=1330
# - PR: lite build
# - CI: release/*, internal/release/* and main, every batched commit, full build
variables:
# enable source-only build for pipelines that contain the -source-build string
- name: isSourceOnlyBuild
value: ${{ contains(variables['Build.DefinitionName'], '-source-build') }}
- name: isSourceOnlyBuildLite
value: ${{ contains(variables['Build.DefinitionName'], '-source-build-lite') }}
- name: isScheduleTrigger
value: ${{ eq(variables['Build.Reason'], 'Schedule') }}
- name: isPRTrigger
value: ${{ eq(variables['Build.Reason'], 'PullRequest') }}
- template: /eng/common/templates-official/variables/pool-providers.yml@self
resources:
repositories:
- repository: 1ESPipelineTemplates
type: git
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release
extends:
template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
parameters:
sdl:
sourceAnalysisPool:
name: $(DncEngInternalBuildPool)
image: 1es-windows-2022
os: windows
codeql:
compiled:
enabled: true
# Runs analysis in the SDL stage and not every job
# https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sdlanalysis/codeql#improving-codeql-performance
runSourceLanguagesInSourceAnalysis: true
baseline:
baselineFile: $(Build.SourcesDirectory)\.config\guardian\.gdnbaselines
stages:
- ${{ if and(ne(variables.isPRTrigger, 'true'), eq(variables['System.TeamProject'], 'internal')) }}:
- template: /eng/pipelines/templates/stages/vmr-scan.yml@self
- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml@self
parameters:
isBuiltFromVmr: true
isSourceOnlyBuild: ${{ variables.isSourceOnlyBuild }}
${{ if eq(variables.isScheduleTrigger, 'true') }}:
scope: full
${{ elseif eq(variables.isSourceOnlyBuildLite, 'true') }}:
scope: lite
${{ elseif and(eq(variables.isPRTrigger, 'true'), eq(variables.isSourceOnlyBuild, 'true')) }}:
scope: ultralite
${{ elseif and(eq(variables.isPRTrigger, 'true'), ne(variables.isSourceOnlyBuild, 'true')) }}:
scope: lite
${{ else }}:
scope: full

50
src/SourceBuild/content/eng/pipelines/pr.yml
View file

@ -1,50 +0,0 @@
# This yml is used by these PR pipelines and triggers:
# NOTE: the triggers are defined in the Azure DevOps UI as they are too complex
#
# - dotnet-source-build (public)
# https://dev.azure.com/dnceng-public/public/_build?definitionId=240
# - PR: ultralite build
# - CI: release/* only, every batched commit, full build
# - Schedule: main only, full build
#
# - dotnet-unified-build (public)
# https://dev.azure.com/dnceng-public/public/_build?definitionId=278
# - PR: lite build
# - CI: release/* only, every batched commit, full build
# - Schedule: main only, full build
#
variables:
# enable source-only build for pipelines that contain the -source-build string
- name: isSourceOnlyBuild
value: ${{ contains(variables['Build.DefinitionName'], '-source-build') }}
- name: isSourceOnlyBuildLite
value: ${{ contains(variables['Build.DefinitionName'], '-source-build-lite') }}
- name: isScheduleTrigger
value: ${{ eq(variables['Build.Reason'], 'Schedule') }}
- name: isPRTrigger
value: ${{ eq(variables['Build.Reason'], 'PullRequest') }}
- name: Codeql.Enabled # we run CodeQL on internal builds only
value: false
- template: /eng/common/templates/variables/pool-providers.yml@self
stages:
- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml
parameters:
isBuiltFromVmr: true
isSourceOnlyBuild: ${{ variables.isSourceOnlyBuild }}
${{ if eq(variables.isScheduleTrigger, 'true') }}:
scope: full
${{ elseif eq(variables.isSourceOnlyBuildLite, 'true') }}:
scope: lite
${{ elseif and(eq(variables.isPRTrigger, 'true'), eq(variables.isSourceOnlyBuild, 'true')) }}:
scope: ultralite
${{ elseif and(eq(variables.isPRTrigger, 'true'), ne(variables.isSourceOnlyBuild, 'true')) }}:
scope: lite
${{ else }}:
scope: full

68
src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml
View file

@ -1,68 +0,0 @@
schedules:
- cron: "0 11 * * 1-5"
displayName: Run on weekdays at 11am UTC
branches:
include:
- main
# Relies on dotnet-source-build being in the same repo as this pipeline
# https://learn.microsoft.com/en-us/azure/devops/pipelines/process/pipeline-triggers?view=azure-devops#branch-considerations
resources:
pipelines:
- pipeline: dotnet-source-build
source: dotnet-source-build
trigger:
branches:
include:
- release/*.0.1xx*
- internal/release/*.0.1xx*
pr: none
trigger: none
pool:
name: NetCore1ESPool-Svc-Internal
demands: ImageOverride -equals 1es-ubuntu-2004
parameters:
- name: dotnetDotnetRunId
displayName: 'Specific dotnet-dotnet run ID number (e.g `2108850`)'
type: string
default: ' '
jobs:
- template: templates/jobs/sdk-diff-tests.yml
parameters:
buildName: CentOSStream9_Offline_MsftSdk
targetRid: centos.9-x64
architecture: x64
dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}
includeArtifactsSize: true
- template: templates/jobs/sdk-diff-tests.yml
parameters:
buildName: Alpine319_Online_MsftSdk
targetRid: alpine.3.19-x64
architecture: x64
dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}
- template: templates/jobs/sdk-diff-tests.yml
parameters:
buildName: Fedora39_Offline_MsftSdk
targetRid: fedora.39-x64
architecture: x64
dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}
- template: templates/jobs/sdk-diff-tests.yml
parameters:
buildName: Ubuntu2204_Offline_MsftSdk
targetRid: ubuntu.22.04-x64
architecture: x64
dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}
- template: templates/jobs/sdk-diff-tests.yml
parameters:
buildName: Ubuntu2204Arm64_Offline_MsftSdk
targetRid: ubuntu.22.04-arm64
architecture: arm64
dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}

167
src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml
View file

@ -1,167 +0,0 @@
parameters:
- name: buildName
type: string
- name: targetRid
type: string
- name: architecture
type: string
- name: dotnetDotnetRunId
type: string
- name: includeArtifactsSize
type: boolean
default: false
jobs:
- job: ${{ parameters.buildName }}_${{ parameters.architecture }}
timeoutInMinutes: 150
pool:
name: NetCore1ESPool-Svc-Internal
demands: ImageOverride -equals 1es-ubuntu-2004
variables:
- template: ../variables/pipelines.yml
steps:
- script: |
dotnet_dotnet_build='${{ replace(parameters.dotnetDotnetRunId, ' ', '') }}'
if [[ -z "$dotnet_dotnet_build" ]]; then
dotnet_dotnet_build=$(az pipelines runs list --branch '$(Build.SourceBranch)' --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --pipeline-ids '$(DOTNET_DOTNET_CI_PIPELINE_ID)' --status completed --top 1 --query "[].id" --output tsv)
fi
if [[ -z "$dotnet_dotnet_build" ]]; then
echo "Could not find a completed dotnet-dotnet build for branch '$(Build.SourceBranch)'"
exit 1
fi
echo "Dotnet-dotnet build: https://dev.azure.com/dnceng/internal/_build/results?buildId=$dotnet_dotnet_build&view=results"
installer_sha=$(az pipelines build tag list --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --build-id $dotnet_dotnet_build --query "[?contains(@, 'installer')]" --output tsv | sed "s,installer ,,g")
installer_build=$(az pipelines runs list --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --pipeline-ids '$(INSTALLER_OFFICIAL_CI_PIPELINE_ID)' --query "[?sourceVersion == '$installer_sha'].id" --output tsv)
if [[ -z "$installer_build" ]]; then
echo "Could not find a build of installer for commit '$installer_sha'"
exit 1
fi
echo "Installer build: https://dev.azure.com/dnceng/internal/_build/results?buildId=$installer_build&view=results"
echo "##vso[build.addbuildtag]installer $installer_sha"
echo "##vso[task.setvariable variable=InstallerBuildId]$installer_build"
echo "##vso[task.setvariable variable=DotnetDotnetBuildId]$dotnet_dotnet_build"
displayName: Find associated builds
name: Get_Build_Ids
env:
AZURE_DEVOPS_EXT_PAT: $(System.AccessToken)
- template: ../steps/download-pipeline-artifact.yml
parameters:
patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux*-${{ parameters.architecture }}.tar.gz'
displayName: Download MSFT SDK
- template: ../steps/download-vmr-artifact.yml
parameters:
buildName: ${{ parameters.buildName }}
architecture: ${{ parameters.architecture }}
patterns: '**/assets/Release/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-${{ parameters.targetRid }}.tar.gz'
displayName: Download Source Build SDK
- template: ../steps/download-vmr-artifact.yml
parameters:
buildName: ${{ parameters.buildName }}
architecture: ${{ parameters.architecture }}
patterns: '**/assets/Release/Private.SourceBuilt.Artifacts.+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*).${{ parameters.targetRid }}.tar.gz'
displayName: Download Source Built Artifacts
- script: |
find $(Pipeline.Workspace)/Artifacts -type f -exec mv {} $(Pipeline.Workspace)/Artifacts \;
displayName: Move Artifacts to root
- script: |
platform="linux"
if [[ ${{ parameters.targetRid }} =~ "alpine" ]]; then
platform="$platform-musl"
fi
msft_sdk_tarball_name=$(find "$(Pipeline.Workspace)/Artifacts" -name "dotnet-sdk-*-$platform-${{ parameters.architecture }}.tar.gz" -exec basename {} \;)
if [[ -z "$msft_sdk_tarball_name" ]]; then
echo "Microsoft SDK tarball does not exist in '$(Pipeline.Workspace)/Artifacts'. The associated build https://dev.azure.com/dnceng/internal/_build/results?buildId=$(InstallerBuildId) might have failed."
exit 1
fi
sdk_tarball_name=$(find "$(Pipeline.Workspace)/Artifacts" -name "dotnet-sdk-*-${{ parameters.targetRid }}.tar.gz" -exec basename {} \;)
if [[ -z "$sdk_tarball_name" ]]; then
echo "Source-build SDK tarball does not exist in '$(Pipeline.Workspace)/Artifacts'. The associated build https://dev.azure.com/dnceng/internal/_build/results?buildId=$(DotnetDotnetBuildId) might have failed"
exit 1
fi
artifacts_path=$(find "$(Pipeline.Workspace)/Artifacts" -name "Private.SourceBuilt.Artifacts.*.${{ parameters.targetRid }}.tar.gz" -exec basename {} \;)
if [[ -z "$artifacts_path" ]]; then
echo "Source-build artifacts path does not exist in '$(Pipeline.Workspace)/Artifacts'. The associated build https://dev.azure.com/dnceng/internal/_build/results?buildId=$(DotnetDotnetBuildId) might have failed"
exit 1
fi
eng/common/build.sh -bl --projects $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj --restore
echo "##vso[task.setvariable variable=Platform]$platform"
echo "##vso[task.setvariable variable=MsftSdkTarballPath]$(Pipeline.Workspace)/Artifacts/$msft_sdk_tarball_name"
echo "##vso[task.setvariable variable=SdkTarballPath]$(Pipeline.Workspace)/Artifacts/$sdk_tarball_name"
echo "##vso[task.setvariable variable=SourceBuiltArtifactsPath]$(Pipeline.Workspace)/Artifacts/$artifacts_path"
displayName: Prepare Tests
workingDirectory: $(Build.SourcesDirectory)
- script: >
.dotnet/dotnet test
$(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj
--filter "Category=SdkContent"
--logger:'trx;LogFileName=$(Agent.JobName)_SDKDiffTests.trx'
--logger:'console;verbosity=detailed'
-c Release
-bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog
-flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log
-clp:v=m
/p:MsftSdkTarballPath=$(MsftSdkTarballPath)
/p:SdkTarballPath=$(SdkTarballPath)
/p:SourceBuiltArtifactsPath=$(SourceBuiltArtifactsPath)
/p:SmokeTestsWarnOnSdkContentDiffs=false
/p:SmokeTestsIncludeArtifactsSizeTests=${{ parameters.includeArtifactsSize }}
/p:TargetRid=${{ parameters.targetRid }}
/p:PortableRid=$(Platform)-${{ parameters.architecture }}
displayName: Run Tests
workingDirectory: $(Build.SourcesDirectory)
- script: |
set -x
targetFolder=$(Build.StagingDirectory)/BuildLogs/
mkdir -p ${targetFolder}
cd "$(Build.SourcesDirectory)"
find artifacts/log/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \;
find artifacts/log/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \;
find artifacts/TestResults/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \;
find artifacts/TestResults/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \;
find artifacts/TestResults/ -type f -name "*.diff" -exec cp {} --parents -t ${targetFolder} \;
find artifacts/TestResults/ -type f -name "Updated*.txt" -exec cp {} --parents -t ${targetFolder} \;
displayName: Prepare BuildLogs staging directory
continueOnError: true
condition: succeededOrFailed()
- publish: '$(Build.StagingDirectory)/BuildLogs'
artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
displayName: Publish BuildLogs
continueOnError: true
condition: succeededOrFailed()
- task: PublishTestResults@2
displayName: Publish Test Results
condition: succeededOrFailed()
continueOnError: true
inputs:
testRunner: VSTest
testResultsFiles: '**/*.trx'
searchFolder: $(Build.SourcesDirectory)/artifacts/TestResults
mergeTestResults: true
publishRunAttachments: true
testRunTitle: $(Agent.JobName)

43
src/SourceBuild/content/eng/pipelines/templates/stages/vmr-scan.yml
View file

@ -1,43 +0,0 @@
stages:
- stage: Tag_n_Scan
displayName: Tag & Scan
dependsOn: []
jobs:
- job: Tag_n_Scan
displayName: Tag & Scan
pool:
name: $(DncEngInternalBuildPool)
image: 1es-ubuntu-2004
os: linux
steps:
- checkout: self
- script: |
source ./eng/common/tools.sh
InitializeDotNetCli true
./.dotnet/dotnet tool restore
displayName: Initialize tooling
workingDirectory: $(Build.SourcesDirectory)/src/installer
- script: |
set -e
sha=`./.dotnet/dotnet darc vmr get-version --vmr "$(Build.SourcesDirectory)" installer`
echo "##vso[build.addbuildtag]$sha"
displayName: Tag the build
workingDirectory: $(Build.SourcesDirectory)/src/installer
- script: |
./eng/detect-binaries.sh
displayName: Scan for binaries
workingDirectory: $(Build.SourcesDirectory)
continueOnError: true
- script: >
./.dotnet/dotnet darc vmr scan-cloaked-files
--vmr "$(Build.SourcesDirectory)"
--tmp "$(Agent.TempDirectory)"
|| (echo '##[error]Found cloaked files in the VMR' && exit 1)
displayName: Scan for cloaked files
workingDirectory: $(Build.SourcesDirectory)/src/installer
continueOnError: true

35
src/SourceBuild/content/eng/pipelines/templates/steps/download-pipeline-artifact.yml
View file

@ -1,35 +0,0 @@
parameters:
- name: pipeline
type: string
default: $(INSTALLER_OFFICIAL_CI_PIPELINE_ID)
- name: buildId
type: string
default: $(InstallerBuildId)
- name: artifact
type: string
default: BlobArtifacts
- name: patterns
type: string
- name: displayName
type: string
default: Download Pipeline Artifact
steps:
- task: DownloadPipelineArtifact@2
inputs:
buildType: specific
buildVersionToDownload: specific
project: internal
pipeline: ${{ parameters.pipeline }}
buildId: ${{ parameters.buildId }}
artifact: ${{ parameters.artifact }}
patterns: ${{ parameters.patterns }}
allowPartiallySucceededBuilds: true
allowFailedBuilds: true
downloadPath: $(Pipeline.Workspace)/Artifacts
checkDownloadedFiles: true
displayName: ${{ parameters.displayName }}

22
src/SourceBuild/content/eng/pipelines/templates/steps/download-vmr-artifact.yml
View file

@ -1,22 +0,0 @@
parameters:
- name: buildName
type: string
- name: architecture
type: string
- name: patterns
type: string
- name: displayName
type: string
default: Download VMR Artifact
steps:
- template: ../steps/download-pipeline-artifact.yml
parameters:
pipeline: $(DOTNET_DOTNET_CI_PIPELINE_ID)
buildId: $(DotnetDotnetBuildId)
artifact: ${{ parameters.buildName }}_${{ parameters.architecture }}_Artifacts
patterns: ${{ parameters.patterns }}
displayName: ${{ parameters.displayName }}

11
src/SourceBuild/content/eng/pipelines/templates/variables/pipelines.yml
View file

@ -1,11 +0,0 @@
variables:
- name: AZDO_PROJECT
value: internal
- name: AZDO_ORG
value: https://dev.azure.com/dnceng/
- name: INSTALLER_OFFICIAL_CI_PIPELINE_ID
value: 286
- name: INSTALLER_TARBALL_BUILD_CI_PIPELINE_ID
value: 1011
- name: DOTNET_DOTNET_CI_PIPELINE_ID
value: 1219

143
src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml
View file

@ -1,143 +0,0 @@
# Pipeline documentation at https://github.com/dotnet/dotnet/blob/main/docs/license-scanning.md
schedules:
- cron: "0 7 * * 1"
displayName: Run on Mondays at 7am UTC
branches:
include:
- main
- release/*.0.1xx*
- internal/release/*.0.1xx*
pr: none
trigger: none
parameters:
# Provides a way to scan a specific repo. If not provided, all repos of the VMR will be scanned.
- name: specificRepoName
type: string
displayName: "Specific repo name to scan (e.g. runtime, sdk). If empty, scans all repos of the VMR."
default: " " # Set it to an empty string to allow it be an optional parameter
variables:
installerRoot: '$(Build.SourcesDirectory)/src/installer'
jobs:
- job: Setup
pool:
name: NetCore1ESPool-Svc-Internal
demands: ImageOverride -equals 1es-ubuntu-2004
steps:
- script: |
vmrSrcDir="$(Build.SourcesDirectory)/src"
# Builds an Azure DevOps matrix definition. Each entry in the matrix is a path,
# allowing a job to be run for each src repo.
matrix=""
# Trim leading/trailing spaces from the repo name
specificRepoName=$(echo "${{ parameters.specificRepoName }}" | awk '{$1=$1};1')
# If the repo name is provided, only scan that repo.
if [ ! -z "$specificRepoName" ]; then
matrix="\"$specificRepoName\": { \"repoPath\": \"$vmrSrcDir/$specificRepoName\" }"
else
for dir in $vmrSrcDir/*/
do
if [ ! -z "$matrix" ]; then
matrix="$matrix,"
fi
repoName=$(basename $dir)
matrix="$matrix \"$repoName\": { \"repoPath\": \"$dir\" }"
done
fi
matrix="{ $matrix }"
echo "##vso[task.setvariable variable=matrix;isOutput=true]$matrix"
name: GetMatrix
displayName: Get Matrix
- job: LicenseScan
dependsOn: Setup
pool:
name: NetCore1ESPool-Svc-Internal
demands: ImageOverride -equals 1es-ubuntu-2004
timeoutInMinutes: 420
strategy:
matrix: $[ dependencies.Setup.outputs['GetMatrix.matrix'] ]
steps:
- script: |
source ./eng/common/tools.sh
InitializeDotNetCli true
displayName: Install .NET SDK
workingDirectory: $(Build.SourcesDirectory)
- task: PipAuthenticate@1
displayName: 'Pip Authenticate'
inputs:
artifactFeeds: public/dotnet-public-pypi
onlyAddExtraIndex: false
- script: $(installerRoot)/eng/install-scancode.sh
displayName: Install Scancode
- script: >
$(Build.SourcesDirectory)/.dotnet/dotnet test
$(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj
--filter "FullyQualifiedName=Microsoft.DotNet.SourceBuild.SmokeTests.LicenseScanTests.ScanForLicenses"
--logger:'trx;LogFileName=$(Agent.JobName)_LicenseScan.trx'
--logger:'console;verbosity=detailed'
-c Release
-bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog
-flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log
-clp:v=m
/p:SmokeTestsLicenseScanPath=$(repoPath)
/p:SmokeTestsWarnOnLicenseScanDiffs=false
/p:TargetRid=linux-x64
/p:PortableRid=linux-x64
displayName: Run Tests
workingDirectory: $(Build.SourcesDirectory)
- script: |
set -x
targetFolder=$(Build.StagingDirectory)/BuildLogs/
mkdir -p ${targetFolder}
cd "$(Build.SourcesDirectory)"
find artifacts/log/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \;
find artifacts/log/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \;
find artifacts/TestResults/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \;
find artifacts/TestResults/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \;
echo "Updated:"
find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt"
find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt" -exec cp {} --parents -t ${targetFolder} \;
find artifacts/TestResults/ -type f -name "Updated*.json"
find artifacts/TestResults/ -type f -name "Updated*.json" -exec cp {} --parents -t ${targetFolder} \;
echo "Results:"
find artifacts/TestResults/ -type f -name "scancode-results*.json" -exec cp {} --parents -t ${targetFolder} \;
echo "All:"
ls -R artifacts/TestResults/
echo "BuildLogs:"
ls -R ${targetFolder}
displayName: Prepare BuildLogs staging directory
continueOnError: true
condition: succeededOrFailed()
- publish: '$(Build.StagingDirectory)/BuildLogs'
artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
displayName: Publish BuildLogs
continueOnError: true
condition: succeededOrFailed()
- task: PublishTestResults@2
displayName: Publish Test Results
condition: succeededOrFailed()
continueOnError: true
inputs:
testRunner: vSTest
testResultsFiles: '*.trx'
searchFolder: $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults
mergeTestResults: true
publishRunAttachments: true
testRunTitle: $(Agent.JobName)

45
src/SourceBuild/content/eng/restore-build-edited-files.ps1
View file

@ -1,45 +0,0 @@
<#
.SYNOPSIS
Script to perform git restore for a set of commonly edited paths when building the VMR.
.DESCRIPTION
This script restores the specified paths using git restore command. It provides options for logging, confirmation, and parameterized restore.
.PARAMETER PathsToRestore
Specifies the paths to be restored. Default paths are:
- src/*/eng/common/*
- src/*global.json
.PARAMETER LogPath
Specifies the path to save the log file. Default is 'restore.log' in the script directory.
.PARAMETER NoPrompt
Indicates whether to skip the confirmation prompt. If specified, the script will restore the paths without confirmation.
#>
param (
[string[]]$PathsToRestore = @(
"src/*/eng/common/*",
"src/*global.json"
),
[Alias("y")]
[switch]$NoPrompt = $false
)
# Confirmation prompt
if (-not $NoPrompt) {
Write-Host "Will restore changes in the following paths:"
foreach ($path in $PathsToRestore) {
Write-Host " $path"
}
$choice = Read-Host "Do you want to proceed with restoring the paths? (Y/N)"
if (-not $($choice -ieq "Y")) {
exit 0
}
}
# Perform git restore for each path
foreach ($path in $PathsToRestore) {
git -C (Split-Path -Path $PSScriptRoot -Parent) restore $path
}

80
src/SourceBuild/content/eng/restore-build-edited-files.sh
View file

@ -1,80 +0,0 @@
#!/bin/bash
# Help message
show_help() {
echo "Script to perform git restore for a set of commonly edited paths when building the VMR."
echo ""
echo "Usage: $0 [-h] [-p <path>] [-n]"
echo ""
echo "Options:"
echo " -h, --help Show this help message and exit"
echo " -p, --path <path> Specify the paths to be restored (default: src/*/eng/common/*, src/*global.json)"
echo " -y, --noprompt Skip the confirmation prompt"
echo ""
echo "Example:"
echo " $0 -p \"src/*/eng/common/*\" -p \"src/*global.json\""
echo ""
exit 0
}
source="${BASH_SOURCE[0]}"
# resolve $source until the file is no longer a symlink
while [[ -h "$source" ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
# Default paths to restore
DefaultPathsToRestore=(
"src/*/eng/common/*"
"src/*global.json"
)
PathsToRestore=()
NoPrompt=false
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case $1 in
-h|--help)
show_help
;;
-p|--path)
shift
PathsToRestore+=("$1")
;;
-y|--noprompt)
NoPrompt=true
;;
*)
echo "Invalid option: $1"
show_help
;;
esac
shift
done
# Use default paths if no paths are passed
if [ ${#PathsToRestore[@]} -eq 0 ]; then
PathsToRestore=("${DefaultPathsToRestore[@]}")
fi
# Confirmation prompt
if [ "$NoPrompt" = false ]; then
echo "Will restore changes in the following paths:"
for path in "${PathsToRestore[@]}"; do
echo " $path"
done
read -p "Do you want to proceed with restoring the paths? (Y/N)" choice
if [[ ! "$choice" =~ ^[Yy]$ ]]; then
exit 0
fi
fi
# Perform git restore for each path
for path in "${PathsToRestore[@]}"; do
git -C "$(dirname "$scriptroot")" restore "$path"
done

123
src/SourceBuild/content/eng/tools/BinaryToolKit/BinaryTool.cs
View file

@ -1,123 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
namespace BinaryToolKit;
public class BinaryTool
{
public async Task<int> ExecuteAsync(
string targetDirectory,
string outputReportDirectory,
string? allowedBinariesFile,
Modes mode)
{
DateTime startTime = DateTime.Now;
Log.LogInformation($"Starting binary tool at {startTime} in {mode} mode");
// Parse args
targetDirectory = GetAndValidateFullPath(
"TargetDirectory",
targetDirectory,
isDirectory: true,
createIfNotExist: false,
isRequired: true)!;
outputReportDirectory = GetAndValidateFullPath(
"OutputReportDirectory",
outputReportDirectory,
isDirectory: true,
createIfNotExist: true,
isRequired: true)!;
allowedBinariesFile = GetAndValidateFullPath(
"AllowedBinariesFile",
allowedBinariesFile,
isDirectory: false,
createIfNotExist: false,
isRequired: false);
// Run the tooling
var detectedBinaries = await DetectBinaries.ExecuteAsync(targetDirectory, outputReportDirectory, allowedBinariesFile);
if (mode == Modes.Validate)
{
ValidateBinaries(detectedBinaries, outputReportDirectory);
}
else if (mode == Modes.Clean)
{
RemoveBinaries(detectedBinaries, targetDirectory);
}
Log.LogInformation("Finished all binary tasks. Took " + (DateTime.Now - startTime).TotalSeconds + " seconds.");
return Log.GetExitCode();
}
private string? GetAndValidateFullPath(
string parameterName,
string? path,
bool isDirectory,
bool createIfNotExist,
bool isRequired)
{
if (string.IsNullOrWhiteSpace(path))
{
if (isRequired)
{
Log.LogError($"Required path for '{parameterName}' is empty or contains whitespace.");
Environment.Exit(1);
}
return null;
}
string fullPath = Path.GetFullPath(path);
bool exists = isDirectory ? Directory.Exists(fullPath) : File.Exists(fullPath);
if (!exists)
{
if (createIfNotExist && isDirectory)
{
Log.LogInformation($"Creating directory '{fullPath}' for '{parameterName}'.");
Directory.CreateDirectory(fullPath);
}
else
{
Log.LogError($"{(isDirectory ? "Directory" : "File")} '{fullPath}' for '{parameterName}' does not exist.");
Environment.Exit(1);
}
}
return fullPath;
}
private static void ValidateBinaries(IEnumerable<string> newBinaries, string outputReportDirectory)
{
if (newBinaries.Any())
{
string newBinariesFile = Path.Combine(outputReportDirectory, "NewBinaries.txt");
Log.LogDebug("New binaries:");
File.WriteAllLines(newBinariesFile, newBinaries);
foreach (var binary in newBinaries)
{
Log.LogDebug($" {binary}");
}
Log.LogError($"ERROR: {newBinaries.Count()} new binaries. Check '{newBinariesFile}' for details.");
}
}
private static void RemoveBinaries(IEnumerable<string> binariesToRemove, string targetDirectory)
{
Log.LogInformation($"Removing binaries from '{targetDirectory}'...");
foreach (var binary in binariesToRemove)
{
File.Delete(Path.Combine(targetDirectory, binary));
Log.LogDebug($" {binary}");
}
Log.LogInformation($"Finished binary removal. Removed {binariesToRemove.Count()} binaries.");
}
}

26
src/SourceBuild/content/eng/tools/BinaryToolKit/BinaryToolKit.csproj
View file

@ -1,26 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk" InitialTargets="CheckCustomPackageVersionsProps">
<PropertyGroup>
<TargetFramework>$(NetCurrent)</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<CustomPackageVersionsProps></CustomPackageVersionsProps>
<OutputType>Exe</OutputType>
</PropertyGroup>
<Target Name="CheckCustomPackageVersionsProps">
<Error Condition="'$(CustomPackageVersionsProps)' != '' And !Exists('$(CustomPackageVersionsProps)')" Text="CustomPackageVersionsProps file '$(CustomPackageVersionsProps)' does not exist." />
</Target>
<!-- Need to condition this import because msbuild will complain about the project not being valid otherwise. -->
<!-- With the condition, the CheckCustomPackageVersionsProps will run as expected and show the respective errors. -->
<Import Project="$(CustomPackageVersionsProps)" Condition="'$(CustomPackageVersionsProps)' != ''" />
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" />
<PackageReference Include="Microsoft.Extensions.Logging" />
<PackageReference Include="System.CommandLine" />
</ItemGroup>
</Project>

245
src/SourceBuild/content/eng/tools/BinaryToolKit/DetectBinaries.cs
View file

@ -1,245 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System.Diagnostics;
using Microsoft.Extensions.FileSystemGlobbing;
using System.Text.RegularExpressions;
namespace BinaryToolKit;
public static class DetectBinaries
{
private const string Utf16Marker = "UTF-16";
private const int ChunkSize = 4096;
private static readonly Regex GitCleanRegex = new Regex(@"Would (remove|skip)( repository)? (.*)");
public static async Task<List<string>> ExecuteAsync(
string targetDirectory,
string outputReportDirectory,
string? allowedBinariesFile)
{
Log.LogInformation($"Detecting binaries in '{targetDirectory}' not listed in '{allowedBinariesFile}'...");
var matcher = new Matcher(StringComparison.Ordinal);
matcher.AddInclude("**/*");
matcher.AddExcludePatterns(await GetIgnoredPatternsAsync(targetDirectory));
IEnumerable<string> matchingFiles = matcher.GetResultsInFullPath(targetDirectory);
var tasks = matchingFiles
.Select(async file =>
{
return await IsBinaryAsync(file) ? file.Substring(targetDirectory.Length + 1) : null;
});
var binaryFiles = (await Task.WhenAll(tasks)).OfType<string>();
var unmatchedBinaryFiles = GetUnmatchedBinaries(
binaryFiles,
allowedBinariesFile,
outputReportDirectory,
targetDirectory).ToList();
Log.LogInformation($"Finished binary detection.");
return unmatchedBinaryFiles;
}
private static async Task<List<string>> GetIgnoredPatternsAsync(string targetDirectory)
{
string gitDirectory = Path.Combine(targetDirectory, ".git");
bool isGitRepo = Directory.Exists(gitDirectory);
try
{
if (!isGitRepo)
{
// Configure a fake git repo to use so that we can run git clean -ndx
await ExecuteProcessAsync("git", $"-C {targetDirectory} init -q");
}
await ExecuteProcessAsync("git", $"-C {targetDirectory} config --global safe.directory {targetDirectory}");
string output = await ExecuteProcessAsync("git", $"-C {targetDirectory} clean -ndx");
List<string> ignoredPaths = output.Split(Environment.NewLine)
.Select(line => GitCleanRegex.Match(line))
.Where(match => match.Success)
.Select(match => match.Groups[3].Value)
.ToList();
if (isGitRepo)
{
ignoredPaths.Add(".git");
}
return ignoredPaths;
}
finally
{
// Ensure .git directory is deleted if it wasn't originally a git repo
if (!isGitRepo && Directory.Exists(gitDirectory))
{
Directory.Delete(gitDirectory, true);
}
}
}
private static async Task<bool> IsBinaryAsync(string filePath)
{
// Using the GNU diff heuristic to determine if a file is binary or not.
// For more details, refer to the GNU diff manual:
// https://www.gnu.org/software/diffutils/manual/html_node/Binary.html
using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read))
using (BinaryReader br = new BinaryReader(fs))
{
byte[] buffer = new byte[ChunkSize];
int bytesRead = br.Read(buffer, 0, ChunkSize);
for (int i = 0; i < bytesRead; i++)
{
if (buffer[i] == 0)
{
// Need to check that the file is not UTF-16 encoded
// because heuristic can return false positives
return await IsNotUTF16Async(filePath);
}
}
}
return false;
}
private static async Task<bool> IsNotUTF16Async(string file)
{
if (Environment.OSVersion.Platform == PlatformID.Unix)
{
string output = await ExecuteProcessAsync("file", $"\"{file}\"");
output = output.Split(":")[1].Trim();
if (output.Contains(Utf16Marker))
{
return false;
}
}
return true;
}
private static async Task<string> ExecuteProcessAsync(string executable, string arguments)
{
ProcessStartInfo psi = new ()
{
FileName = executable,
Arguments = arguments,
CreateNoWindow = true,
RedirectStandardOutput = true,
RedirectStandardError = true
};
var proc = Process.Start(psi)!;
string output = await proc.StandardOutput.ReadToEndAsync();
string error = await proc.StandardError.ReadToEndAsync();
await proc.WaitForExitAsync();
if (!string.IsNullOrEmpty(error))
{
Log.LogError(error);
}
return output;
}
private static IEnumerable<string> GetUnmatchedBinaries(
IEnumerable<string> searchFiles,
string? allowedBinariesFile,
string outputReportDirectory,
string targetDirectory)
{
HashSet<string> unmatchedFiles = new HashSet<string>(searchFiles);
var filesToPatterns = new Dictionary<string, HashSet<string>>();
ParseAllowedBinariesFile(allowedBinariesFile, ref filesToPatterns);
foreach (var fileToPatterns in filesToPatterns)
{
var patterns = fileToPatterns.Value;
HashSet<string> unusedPatterns = new HashSet<string>(patterns);
foreach (string pattern in patterns)
{
Matcher matcher = new Matcher(StringComparison.Ordinal);
matcher.AddInclude(pattern);
var matches = matcher.Match(targetDirectory, searchFiles);
if (matches.HasMatches)
{
unusedPatterns.Remove(pattern);
unmatchedFiles.ExceptWith(matches.Files.Select(file => file.Path));
}
}
UpdateAllowedBinariesFile(fileToPatterns.Key, outputReportDirectory, unusedPatterns);
}
return unmatchedFiles;
}
private static void ParseAllowedBinariesFile(string? file, ref Dictionary<string, HashSet<string>> result)
{
if (!File.Exists(file))
{
return;
}
if (!result.ContainsKey(file))
{
result[file] = new HashSet<string>();
}
foreach (var line in File.ReadLines(file))
{
var trimmedLine = line.Trim();
if (string.IsNullOrWhiteSpace(trimmedLine) || trimmedLine.StartsWith("#"))
{
continue;
}
if (trimmedLine.StartsWith("import:"))
{
var importFile = trimmedLine.Substring("import:".Length).Trim();
if (!Path.IsPathFullyQualified(importFile))
{
var currentDirectory = Path.GetDirectoryName(file)!;
importFile = Path.Combine(currentDirectory, importFile);
}
if (result.ContainsKey(importFile))
{
Log.LogWarning($" Duplicate import {importFile}. Skipping.");
continue;
}
ParseAllowedBinariesFile(importFile, ref result);
}
else
{
result[file].Add(trimmedLine.Split('#')[0].Trim());
}
}
}
private static void UpdateAllowedBinariesFile(string? file, string outputReportDirectory, HashSet<string> unusedPatterns)
{
if(File.Exists(file) && unusedPatterns.Any())
{
var lines = File.ReadAllLines(file);
var newLines = lines.Where(line => !unusedPatterns.Contains(line)).ToList();
string updatedFile = Path.Combine(outputReportDirectory, "Updated" + Path.GetFileName(file));
File.WriteAllLines(updatedFile, newLines);
Log.LogInformation($" Updated allowed binaries file '{Path.GetFileName(file)}' written to '{updatedFile}'");
}
}
}

13
src/SourceBuild/content/eng/tools/BinaryToolKit/Directory.Build.props
View file

@ -1,13 +0,0 @@
<Project>
<!--
Do not import the Arcade SDK as this would introduce a prebuilt as this project builds
before the custom sdk resolver is available in source-only mode.
-->
<PropertyGroup>
<SkipArcadeSdkImport>true</SkipArcadeSdkImport>
</PropertyGroup>
<Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.props, $(MSBuildThisFileDirectory)..))" />
</Project>

68
src/SourceBuild/content/eng/tools/BinaryToolKit/Log.cs
View file

@ -1,68 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Extensions.Logging;
namespace BinaryToolKit;
public static class Log
{
public static LogLevel Level = LogLevel.Information;
private static bool WarningLogged = false;
private static bool ErrorLogged = false;
private static readonly Lazy<ILogger> _logger = new Lazy<ILogger>(ConfigureLogger);
public static void LogDebug(string message)
{
_logger.Value.LogDebug(message);
}
public static void LogInformation(string message)
{
_logger.Value.LogInformation(message);
}
public static void LogWarning(string message)
{
_logger.Value.LogWarning(message);
WarningLogged = true;
}
public static void LogError(string message)
{
_logger.Value.LogError(message);
ErrorLogged = true;
}
private static ILogger ConfigureLogger()
{
using ILoggerFactory loggerFactory =
LoggerFactory.Create(builder =>
builder.AddSimpleConsole(options =>
{
options.SingleLine = true;
options.TimestampFormat = "HH:mm:ss ";
options.UseUtcTimestamp = true;
})
.SetMinimumLevel(Level));
return loggerFactory.CreateLogger("BinaryTool");
}
public static int GetExitCode()
{
if (ErrorLogged)
{
return 1;
}
if (WarningLogged)
{
return 2;
}
return 0;
}
}

10
src/SourceBuild/content/eng/tools/BinaryToolKit/Modes.cs
View file

@ -1,10 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
namespace BinaryToolKit;
public enum Modes
{
Validate,
Clean
}

86
src/SourceBuild/content/eng/tools/BinaryToolKit/Program.cs
View file

@ -1,86 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System;
using System.CommandLine;
using Microsoft.Extensions.Logging;
namespace BinaryToolKit;
public class Program
{
public static readonly CliArgument<string> TargetDirectory = new("target-directory")
{
Description = "The directory to run the binary tooling on.",
Arity = ArgumentArity.ExactlyOne
};
public static readonly CliOption<string> OutputReportDirectory = new("--output-directory", "-o")
{
Description = "The directory to output the report to.",
Arity = ArgumentArity.ZeroOrOne,
DefaultValueFactory = _ => Path.Combine(Directory.GetCurrentDirectory(), "binary-report")
};
public static readonly CliOption<LogLevel> Level = new("--log-level", "-l")
{
Description = "The log level to run the tool in.",
Arity = ArgumentArity.ZeroOrOne,
DefaultValueFactory = _ => LogLevel.Information,
Recursive = true
};
public static readonly CliOption<string> AllowedBinariesFile = new("--allowed-binaries-file", "-ab")
{
Description = "The file containing the list of allowed binaries that are ignored for cleaning or validating.\n",
Arity = ArgumentArity.ZeroOrOne
};
public static int ExitCode = 0;
public static async Task<int> Main(string[] args)
{
var cleanCommand = CreateCommand("clean", "Clean the binaries in the target directory.");
var validateCommand = CreateCommand("validate", "Detect new binaries in the target directory.");
var rootCommand = new CliRootCommand("Tool for detecting, validating, and cleaning binaries in the target directory.")
{
Level,
cleanCommand,
validateCommand
};
SetCommandAction(cleanCommand, Modes.Clean);
SetCommandAction(validateCommand, Modes.Validate);
await rootCommand.Parse(args).InvokeAsync();
return ExitCode;
}
private static CliCommand CreateCommand(string name, string description)
{
return new CliCommand(name, description)
{
TargetDirectory,
OutputReportDirectory,
AllowedBinariesFile
};
}
private static void SetCommandAction(CliCommand command, Modes mode)
{
command.SetAction(async (result, CancellationToken) =>
{
Log.Level = result.GetValue(Level);
var binaryTool = new BinaryTool();
ExitCode = await binaryTool.ExecuteAsync(
result.GetValue(TargetDirectory)!,
result.GetValue(OutputReportDirectory)!,
result.GetValue(AllowedBinariesFile),
mode);
});
}
}

21
src/SourceBuild/content/eng/tools/Directory.Build.props
View file

@ -1,21 +0,0 @@
<Project>
<Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.props, $(MSBuildThisFileDirectory)..))" />
<PropertyGroup>
<RestoreSources Condition="'$(DotNetBuildSourceOnly)' == 'true'">$(ReferencePackagesDir);$(PrebuiltPackagesPath);$(PrebuiltSourceBuiltPackagesPath)</RestoreSources>
</PropertyGroup>
<!--
Use some assemblies from the SDK, instead of package references. This ensures they match what's
found when the task is loaded by the SDK's MSBuild.
Reference NuGet assemblies, except a command line assembly that causes warnings such as:
MSB3277: Found conflicts between different versions of "System.Collections" that could not be resolved.
-->
<ItemGroup>
<SdkAssembly Include="$([MSBuild]::NormalizePath('$(NetCoreRoot)', 'sdk', '$(NETCoreSdkVersion)', 'Newtonsoft.Json.dll'));
$([MSBuild]::NormalizeDirectory('$(NetCoreRoot)', 'sdk', '$(NETCoreSdkVersion)'))NuGet.*.dll"
Exclude="$([MSBuild]::NormalizePath('$(NetCoreRoot)', 'sdk', '$(NETCoreSdkVersion)', 'NuGet.CommandLine.XPlat.dll'))" />
</ItemGroup>
</Project>

14
src/SourceBuild/content/eng/tools/EmptySdk/Sdk/Sdk.props
View file

@ -1,14 +0,0 @@
<Project>
<PropertyGroup>
<RepoRoot Condition="'$(RepoRoot)' == ''">$([MSBuild]::NormalizeDirectory('$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildProjectDirectory), 'global.json'))'))</RepoRoot>
<RepositoryEngineeringDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'eng'))</RepositoryEngineeringDir>
</PropertyGroup>
<PropertyGroup>
<!-- Skip importing NuGet Pack targets which are imported from the Microsoft.NET.Sdk targets -->
<ImportNuGetBuildTasksPackTargetsFromSdk>false</ImportNuGetBuildTasksPackTargetsFromSdk>
</PropertyGroup>
</Project>

9
src/SourceBuild/content/eng/tools/EmptySdk/Sdk/Sdk.targets
View file

@ -1,9 +0,0 @@
<Project>
<Target Name="_IsProjectRestoreSupported"/>
<Target Name="Restore"/>
<Target Name="Build"/>
<Target Name="Test"/>
<Target Name="Pack"/>
<Target Name="Publish"/>
</Project>

26
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogFileEntry.cs
View file

@ -1,26 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Collections.Generic;
using System.Text;
using System.Xml.Linq;
namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
{
internal class CatalogFileEntry
{
const string ElementName = "File";
internal string Path { get; set; }
internal byte[] OriginalHash { get; set; }
internal byte[] PoisonedHash { get; set; }
public XElement ToXml() => new XElement(ElementName,
new XAttribute(nameof(Path), Path),
new XAttribute(nameof(OriginalHash), OriginalHash.ToHexString()),
PoisonedHash == null ? null : new XAttribute(nameof(PoisonedHash), PoisonedHash.ToHexString())
);
}
}

38
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogPackageEntry.cs
View file

@ -1,38 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml.Linq;
namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
{
internal class CatalogPackageEntry
{
const string ElementName = "Package";
internal string Path { get; set; }
internal string Id { get; set; }
internal string Version { get; set; }
internal byte[] OriginalHash { get; set; }
internal byte[] PoisonedHash { get; set; }
internal List<CatalogFileEntry> Files { get; }
public CatalogPackageEntry()
{
this.Files = new List<CatalogFileEntry>();
}
public XElement ToXml() => new XElement(ElementName,
new XAttribute(nameof(Path), Path),
new XAttribute(nameof(Id), Id),
new XAttribute(nameof(Version), Version),
new XAttribute(nameof(OriginalHash), OriginalHash.ToHexString()),
PoisonedHash == null ? null : new XAttribute(nameof(PoisonedHash), PoisonedHash.ToHexString()),
Files.Select(f => f.ToXml())
);
}
}

479
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs
View file

@ -1,479 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reflection;
using System.Reflection.Metadata;
using System.Reflection.PortableExecutable;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.Linq;
namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
{
public class CheckForPoison : Task
{
/// <summary>
/// The files to check for poison and/or hash matches. Zips and
/// nupkgs will be extracted and checked recursively.
/// %(Identity): Path to the initial set of files.
/// Add SourceBuildReferencePackage metadata and set to true to
/// indicate that the file comes from SBRP.
/// </summary>
[Required]
public ITaskItem[] FilesToCheck { get; set; }
/// <summary>
/// The path of the project directory to the FilesToCheck.
/// </summary>
[Required]
public string ProjectDirPath { get; set; }
/// <summary>
/// The output path for an XML poison report, if desired.
/// </summary>
public string PoisonReportOutputFilePath { get; set; }
/// <summary>
/// The path of a previously-generated file hash catalog, if
/// hash checked is desired. If not, only assembly attributes
/// and package marker file checked will be done.
/// </summary>
public string HashCatalogFilePath { get; set; }
/// <summary>
/// The marker file name to check for in poisoned nupkg files.
/// </summary>
public string MarkerFileName { get; set; }
/// <summary>
/// If true, fails the build if any poisoned files are found.
/// </summary>
public bool FailOnPoisonFound { get; set; }
/// <summary>
/// Use this directory instead of the system temp directory for staging.
/// Intended for Linux systems with limited /tmp space, like Azure VMs.
/// </summary>
public string OverrideTempPath { get; set; }
private static readonly string[] ZipFileExtensions =
{
".zip",
".nupkg",
};
private static readonly string[] TarFileExtensions =
{
".tar",
};
private static readonly string[] TarGzFileExtensions =
{
".tgz",
".tar.gz",
};
private static readonly string[] FileNamesToSkip =
{
"_._",
"-.-",
".bowerrc",
".gitignore",
".gitkeep",
".rels",
"LICENSE",
"prefercliruntime",
"RunCsc",
"RunVbc",
};
private static readonly string[] FileExtensionsToSkip =
{
".config",
".cs",
".cshtml",
".csproj",
".css",
".db",
".editorconfig",
".eot",
".fs",
".fsproj",
".h",
".html",
".ico",
".js",
".json",
".map",
".md",
".nuspec",
".otf",
".png",
".props",
".proto",
".proj",
".psmdcp",
".pubxml",
".razor",
".rtf",
".scss",
".sln",
".svg",
".targets",
".transform",
".ts",
".ttf",
".txt",
".vb",
".vbproj",
".win32manifest",
".woff",
".woff2",
".xaml",
".xml",
};
private const string PoisonMarker = "POISONED";
private const string SbrpAttributeType = "System.Reflection.AssemblyMetadataAttribute";
private const string SbrpMetadataName = "IsSourceBuildReferencePackage";
private record CandidateFileEntry(string ExtractedPath, string DisplayPath, bool IsSourceBuildReferencePackage);
public override bool Execute()
{
IEnumerable<PoisonedFileEntry> poisons = GetPoisonedFiles(FilesToCheck, HashCatalogFilePath, MarkerFileName);
// if we should write out the poison report, do that
if (!string.IsNullOrWhiteSpace(PoisonReportOutputFilePath))
{
File.WriteAllText(PoisonReportOutputFilePath, (new XElement("PrebuiltLeakReport", poisons.OrderBy(p => p.Path).Select(p => p.ToXml()))).ToString());
}
if (FailOnPoisonFound && poisons.Count() > 0)
{
Log.LogError($"Forced build error: {poisons.Count()} marked files leaked to output. See complete report '{PoisonReportOutputFilePath}' for details.");
return false;
}
else if (poisons.Count() > 0)
{
Log.LogWarning($"{poisons.Count()} marked files leaked to output. See complete report '{PoisonReportOutputFilePath}' for details.");
}
return !Log.HasLoggedErrors;
}
/// <summary>
/// Internal helper to allow other tasks to check for poisoned files.
/// </summary>
/// <param name="initialCandidates">Initial queue of candidate files (will be cleared when done)</param>
/// <param name="catalogedPackagesFilePath">File path to the file hash catalog</param>
/// <param name="markerFileName">Marker file name to check for in poisoned nupkgs</param>
/// <returns>List of poisoned packages and files found and reasons for each</returns>
internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<ITaskItem> initialCandidates, string catalogedPackagesFilePath, string markerFileName)
{
IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
var poisons = new List<PoisonedFileEntry>();
var candidateQueue = new Queue<CandidateFileEntry>(initialCandidates.Select(candidate =>
new CandidateFileEntry(candidate.ItemSpec,
Utility.MakeRelativePath(candidate.ItemSpec, ProjectDirPath),
candidate.GetMetadata(SbrpMetadataName) == "true")));
if (!string.IsNullOrWhiteSpace(OverrideTempPath))
{
Directory.CreateDirectory(OverrideTempPath);
}
var tempDirName = Path.GetRandomFileName();
var tempDir = Directory.CreateDirectory(Path.Combine(OverrideTempPath ?? Path.GetTempPath(), tempDirName));
while (candidateQueue.Any())
{
var candidate = candidateQueue.Dequeue();
// if this is a zip or NuPkg, extract it, check for the poison marker, and
// add its contents to the list to be checked.
if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => candidate.ExtractedPath.ToLowerInvariant().EndsWith(e)))
{
Log.LogMessage($"Zip or NuPkg file to check: {candidate.ExtractedPath}");
var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(candidate.ExtractedPath));
PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, candidate, markerFileName, tempCheckingDir, candidateQueue);
if (result != null)
{
poisons.Add(result);
}
}
else
{
PoisonedFileEntry result = CheckSingleFile(catalogedPackages, candidate);
if (result != null)
{
poisons.Add(result);
}
}
}
tempDir.Delete(true);
return poisons;
}
private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate)
{
// skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as
// file types that we never care about - text files, .gitconfig, etc.
var fileToCheck = candidate.ExtractedPath;
if (FileNamesToSkip.Any(f => Path.GetFileName(fileToCheck).ToLowerInvariant() == f.ToLowerInvariant()) ||
FileExtensionsToSkip.Any(e => Path.GetExtension(fileToCheck).ToLowerInvariant() == e.ToLowerInvariant()) ||
(new FileInfo(fileToCheck).Length == 0))
{
return null;
}
var poisonEntry = new PoisonedFileEntry();
poisonEntry.Path = candidate.DisplayPath;
// There seems to be some weird issues with using file streams both for hashing and assembly loading.
// Copy everything into a memory stream to avoid these problems.
var memStream = new MemoryStream();
using (var stream = File.OpenRead(fileToCheck))
{
stream.CopyTo(memStream);
}
memStream.Seek(0, SeekOrigin.Begin);
using (var sha = SHA256.Create())
{
poisonEntry.Hash = sha.ComputeHash(memStream);
}
foreach (var p in catalogedPackages)
{
// This hash can match either the original hash (we couldn't poison the file, or redownloaded it) or
// the poisoned hash (the obvious failure case of a poisoned file leaked).
foreach (var matchingCatalogedFile in p.Files.Where(f => f.OriginalHash.SequenceEqual(poisonEntry.Hash) || (f.PoisonedHash?.SequenceEqual(poisonEntry.Hash) ?? false)))
{
poisonEntry.Type |= PoisonType.Hash;
var match = new PoisonMatch
{
File = matchingCatalogedFile.Path,
Package = p.Path,
PackageId = p.Id,
PackageVersion = p.Version,
};
poisonEntry.Matches.Add(match);
}
}
try
{
AssemblyName asm = AssemblyName.GetAssemblyName(fileToCheck);
if (!candidate.IsSourceBuildReferencePackage && IsAssemblyFromSbrp(fileToCheck))
{
poisonEntry.Type |= PoisonType.SourceBuildReferenceAssembly;
}
else if (IsAssemblyPoisoned(fileToCheck))
{
poisonEntry.Type |= PoisonType.AssemblyAttribute;
}
}
catch
{
// this is fine, it's just not an assembly.
}
return poisonEntry.Type != PoisonType.None ? poisonEntry : null;
}
private static bool IsAssemblyPoisoned(string path)
{
byte[] buffer = File.ReadAllBytes(path);
byte[] marker = Encoding.UTF8.GetBytes(PoisonMarker);
// Start at end of file and go backwards
// Marker is likely at the end and this saves time when
// we encounter a poisoned file.
for (int j = buffer.Length - marker.Length; j >= 0; j--)
{
int i;
for (i = 0; i < marker.Length && buffer[j + i] == marker[i]; i++) ;
if (i == marker.Length)
{
return true;
}
}
return false;
}
private static bool IsAssemblyFromSbrp(string assemblyPath)
{
using var stream = new FileStream(assemblyPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
using var peReader = new PEReader(stream);
MetadataReader reader = peReader.GetMetadataReader();
return reader.CustomAttributes
.Select(attrHandle => reader.GetCustomAttribute(attrHandle))
.Any(attr => IsAttributeSbrp(reader, attr));
}
private static bool IsAttributeSbrp(MetadataReader reader, CustomAttribute attr)
{
string attributeType = string.Empty;
if (attr.Constructor.Kind == HandleKind.MemberReference)
{
var mref = reader.GetMemberReference((MemberReferenceHandle)attr.Constructor);
if (mref.Parent.Kind == HandleKind.TypeReference)
{
var tref = reader.GetTypeReference((TypeReferenceHandle)mref.Parent);
attributeType = $"{reader.GetString(tref.Namespace)}.{reader.GetString(tref.Name)}";
}
else if (mref.Parent.Kind == HandleKind.TypeDefinition)
{
var tdef = reader.GetTypeDefinition((TypeDefinitionHandle)mref.Parent);
attributeType = $"{reader.GetString(tdef.Namespace)}.{reader.GetString(tdef.Name)}";
}
}
else if (attr.Constructor.Kind == HandleKind.MethodDefinition)
{
var mdef = reader.GetMethodDefinition((MethodDefinitionHandle)attr.Constructor);
var tdef = reader.GetTypeDefinition(mdef.GetDeclaringType());
attributeType = $"{reader.GetString(tdef.Namespace)}.{reader.GetString(tdef.Name)}";
}
if (attributeType == SbrpAttributeType)
{
var decodedValue = attr.DecodeValue(DummyAttributeTypeProvider.Instance);
try
{
return decodedValue.FixedArguments[0].Value?.ToString() == "source" && decodedValue.FixedArguments[1].Value?.ToString() == "source-build-reference-packages";
}
catch
{
throw new InvalidOperationException($"{SbrpAttributeType} is not formatted properly with a key, value pair.");
}
}
return false;
}
private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue<CandidateFileEntry> futureFilesToCheck)
{
var poisonEntry = new PoisonedFileEntry();
var zipToCheck = candidate.ExtractedPath;
poisonEntry.Path = zipToCheck;
using (var sha = SHA256.Create())
using (var stream = File.OpenRead(zipToCheck))
{
poisonEntry.Hash = sha.ComputeHash(stream);
}
// first check for a matching poisoned or non-poisoned hash match:
// - non-poisoned is a potential error where the package was redownloaded.
// - poisoned is a use of a local package we were not expecting.
foreach (var matchingCatalogedPackage in catalogedPackages.Where(c => c.OriginalHash.SequenceEqual(poisonEntry.Hash) || (c.PoisonedHash?.SequenceEqual(poisonEntry.Hash) ?? false)))
{
poisonEntry.Type |= PoisonType.Hash;
var match = new PoisonMatch
{
Package = matchingCatalogedPackage.Path,
PackageId = matchingCatalogedPackage.Id,
PackageVersion = matchingCatalogedPackage.Version,
};
poisonEntry.Matches.Add(match);
}
// now extract and look for the marker file
if (ZipFileExtensions.Any(e => zipToCheck.ToLowerInvariant().EndsWith(e)))
{
ZipFile.ExtractToDirectory(zipToCheck, tempDir, true);
}
else if (TarFileExtensions.Any(e => zipToCheck.ToLowerInvariant().EndsWith(e)))
{
Directory.CreateDirectory(tempDir);
var psi = new ProcessStartInfo("tar", $"xf {zipToCheck} -C {tempDir}");
Process.Start(psi).WaitForExit();
}
else if (TarGzFileExtensions.Any(e => zipToCheck.ToLowerInvariant().EndsWith(e)))
{
Directory.CreateDirectory(tempDir);
var psi = new ProcessStartInfo("tar", $"xzf {zipToCheck} -C {tempDir}");
Process.Start(psi).WaitForExit();
}
else
{
throw new ArgumentOutOfRangeException($"Don't know how to decompress {zipToCheck}");
}
if (!string.IsNullOrWhiteSpace(markerFileName) && File.Exists(Path.Combine(tempDir, markerFileName)))
{
poisonEntry.Type |= PoisonType.NupkgFile;
}
foreach (var child in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories))
{
string displayPath = $"{candidate.DisplayPath}/{child.Replace(tempDir, string.Empty).TrimStart(Path.DirectorySeparatorChar)}";
futureFilesToCheck.Enqueue(new CandidateFileEntry(child, displayPath, candidate.IsSourceBuildReferencePackage));
}
return poisonEntry.Type != PoisonType.None ? poisonEntry : null;
}
private static IEnumerable<CatalogPackageEntry> ReadCatalog(string hashCatalogFilePath)
{
// catalog is optional, we can also just check assembly properties or nupkg marker files
if (string.IsNullOrWhiteSpace(hashCatalogFilePath))
{
return Enumerable.Empty<CatalogPackageEntry>();
}
var doc = new XmlDocument();
using (var stream = File.OpenRead(hashCatalogFilePath))
{
doc.Load(stream);
}
var packages = new List<CatalogPackageEntry>();
var catalog = doc.FirstChild;
foreach (XmlElement p in catalog.ChildNodes)
{
var package = new CatalogPackageEntry
{
Id = p.Attributes["Id"].Value,
Version = p.Attributes["Version"].Value,
OriginalHash = p.Attributes["OriginalHash"].Value.ToBytes(),
PoisonedHash = p.Attributes["PoisonedHash"]?.Value?.ToBytes(),
Path = p.Attributes["Path"].Value,
};
packages.Add(package);
foreach (XmlNode f in p.ChildNodes)
{
var fEntry = new CatalogFileEntry
{
OriginalHash = f.Attributes["OriginalHash"].Value.ToBytes(),
PoisonedHash = f.Attributes["PoisonedHash"]?.Value?.ToBytes(),
Path = f.Attributes["Path"].Value,
};
package.Files.Add(fEntry);
}
}
return packages;
}
}
}

37
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/DummyAttributeTypeProvider.cs
View file

@ -1,37 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Reflection;
using System.Reflection.Metadata;
using System.Reflection.Metadata.Ecma335;
#nullable enable
namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
{
// An empty ICustomAttributeTypeProvider implementation is necessary to read metadata attribute values.
internal class DummyAttributeTypeProvider : ICustomAttributeTypeProvider<Type?>
{
public static readonly DummyAttributeTypeProvider Instance = new();
public Type? GetPrimitiveType(PrimitiveTypeCode typeCode) => default(Type);
public Type? GetSystemType() => default(Type);
public Type? GetSZArrayType(Type? elementType) => default(Type);
public Type? GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => default(Type);
public Type? GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => default(Type);
public Type? GetTypeFromSerializedName(string name) => default(Type);
public PrimitiveTypeCode GetUnderlyingEnumType(Type? type) => default(PrimitiveTypeCode);
public bool IsSystemType(Type? type) => default(bool);
}
}

181
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs
View file

@ -1,181 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using Microsoft.DotNet.UnifiedBuild.Tasks;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Text;
using System.Xml.Linq;
namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
{
public class MarkAndCatalogPackages : Task
{
private const string CatalogElementName = "HashCatalog";
private const string PoisonMarker = "POISONED by DotNetSourceBuild - Should not ship";
private readonly Type[] AssemblyPropertiesToReplace = new Type[] {
typeof(AssemblyProductAttribute),
typeof(AssemblyInformationalVersionAttribute),
typeof(AssemblyDescriptionAttribute),
typeof(AssemblyTitleAttribute)
};
/// <summary>
/// The name of the XML file to write the hash catalog out to,
/// for later checking build output against. This is optional -
/// if not used, assemblies will still be poisoned in their attributes.
/// </summary>
public string CatalogOutputFilePath { get; set; }
/// <summary>
/// The name of the marker file to drop in the nupkgs. This can vary
/// with the packages, so you can use ".prebuilt" for one set of packages
/// and ".source-built" for another if you would like to tell the difference
/// between two sets of poisoned packages.
/// </summary>
public string MarkerFileName { get; set; }
/// <summary>
/// The packages to poison and/or catalog:
/// %(Identity): Path to the nupkg.
/// </summary>
[Required]
public ITaskItem[] PackagesToMark { get; set; }
/// <summary>
/// Use this directory instead of the system temp directory for staging.
/// Intended for Linux systems with limited /tmp space, like Azure VMs.
/// </summary>
public string OverrideTempPath { get; set; }
public override bool Execute()
{
var tempDirName = Path.GetRandomFileName();
if (!string.IsNullOrWhiteSpace(OverrideTempPath))
{
Directory.CreateDirectory(OverrideTempPath);
}
var tempDir = Directory.CreateDirectory(Path.Combine(OverrideTempPath ?? Path.GetTempPath(), tempDirName));
var packageEntries = new List<CatalogPackageEntry>();
using (var sha = SHA256.Create())
{
foreach (var p in PackagesToMark)
{
var packageEntry = new CatalogPackageEntry();
packageEntries.Add(packageEntry);
packageEntry.Path = p.ItemSpec;
using (var stream = File.OpenRead(p.ItemSpec))
{
packageEntry.OriginalHash = sha.ComputeHash(stream);
}
var packageIdentity = ReadNuGetPackageInfos.ReadIdentity(p.ItemSpec);
packageEntry.Id = packageIdentity.Id;
packageEntry.Version = packageIdentity.Version.OriginalVersion;
var packageTempPath = Path.Combine(tempDir.FullName, Path.GetFileName(p.ItemSpec));
ZipFile.ExtractToDirectory(p.ItemSpec, packageTempPath, true);
foreach (string file in Directory.EnumerateFiles(packageTempPath, "*", SearchOption.AllDirectories))
{
// remove signatures so we don't later fail validation
if (Path.GetFileName(file) == ".signature.p7s")
{
File.Delete(file);
continue;
}
var catalogFileEntry = new CatalogFileEntry();
packageEntry.Files.Add(catalogFileEntry);
catalogFileEntry.Path = Utility.MakeRelativePath(file, packageTempPath);
// There seem to be some weird issues with using a file stream both for hashing and
// assembly loading, even closing it in between. Use a MemoryStream to avoid issues.
var memStream = new MemoryStream();
using (var stream = File.OpenRead(file))
{
stream.CopyTo(memStream);
}
// First get the original hash of the file
memStream.Seek(0, SeekOrigin.Begin);
catalogFileEntry.OriginalHash = sha.ComputeHash(memStream);
// Add poison marker to assemblies
try
{
AssemblyName asm = AssemblyName.GetAssemblyName(file);
Poison(file);
// then get the hash of the now-poisoned file
using (var stream = File.OpenRead(file))
{
catalogFileEntry.PoisonedHash = sha.ComputeHash(stream);
}
}
catch
{
// this is okay, it's not an assembly
}
}
if (!string.IsNullOrWhiteSpace(MarkerFileName))
{
var markerFilePath = Path.Combine(packageTempPath, MarkerFileName);
if (File.Exists(markerFilePath))
{
throw new ArgumentException($"Marker file name '{MarkerFileName}' is not sufficiently unique! Exists in '{p.ItemSpec}'.", nameof(MarkerFileName));
}
// mostly we just need to write something unique to this so it's not hashed as a matching file when we check it later.
// but it's also convenient to have the package catalog handy.
File.WriteAllText(markerFilePath, packageEntry.ToXml().ToString());
}
// create a temp file for this so if something goes wrong in the process we're not in too weird of a state
var poisonedPackageName = Path.GetFileName(p.ItemSpec) + ".poisoned";
var poisonedPackagePath = Path.Combine(tempDir.FullName, poisonedPackageName);
ZipFile.CreateFromDirectory(packageTempPath, poisonedPackagePath);
// Get the hash of the poisoned package (with poisoned marker file and poisoned assemblies inside)
using (var stream = File.OpenRead(poisonedPackagePath))
{
packageEntry.PoisonedHash = sha.ComputeHash(stream);
}
File.Delete(p.ItemSpec);
File.Move(poisonedPackagePath, p.ItemSpec);
Directory.Delete(packageTempPath, true);
}
}
// if we should write out the catalog, do that
if (!string.IsNullOrWhiteSpace(CatalogOutputFilePath))
{
var outputFileDir = Path.GetDirectoryName(CatalogOutputFilePath);
if (!Directory.Exists(outputFileDir))
{
Directory.CreateDirectory(outputFileDir);
}
File.WriteAllText(CatalogOutputFilePath, (new XElement("HashCatalog",
packageEntries.Select(p => p.ToXml()))).ToString());
}
tempDir.Delete(true);
return !Log.HasLoggedErrors;
}
private void Poison(string path) => File.AppendAllText(path, PoisonMarker);
}
}

20
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj
View file

@ -1,20 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(NetCurrent)</TargetFramework>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Build.Utilities.Core" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../Microsoft.DotNet.UnifiedBuild.Tasks/Microsoft.DotNet.UnifiedBuild.Tasks.csproj" />
</ItemGroup>
<ItemGroup>
<ReferencePath Include="@(SdkAssembly)" />
</ItemGroup>
</Project>

6
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/NuGet.Config
View file

@ -1,6 +0,0 @@
<configuration>
<packageSources>
<!--To inherit the global NuGet package sources remove the <clear/> line below -->
<clear />
</packageSources>
</configuration>

28
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonMatch.cs
View file

@ -1,28 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Collections.Generic;
using System.Text;
using System.Xml.Linq;
namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
{
internal class PoisonMatch
{
const string ElementName = "Match";
internal string Package { get; set; }
internal string File { get; set; }
internal string PackageId { get; set; }
internal string PackageVersion { get; set; }
public XElement ToXml() => new XElement(ElementName,
Package == null ? null : new XAttribute(nameof(Package), Package),
PackageId == null ? null : new XAttribute(nameof(PackageId), PackageId),
PackageVersion == null ? null : new XAttribute(nameof(PackageVersion), PackageVersion),
File == null ? null: new XAttribute(nameof(File), File)
);
}
}

16
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs
View file

@ -1,16 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
{
[Flags]
internal enum PoisonType
{
None = 0,
Hash = 1,
AssemblyAttribute = 2,
NupkgFile = 4,
SourceBuildReferenceAssembly = 8,
}
}

36
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonedFileEntry.cs
View file

@ -1,36 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml.Linq;
namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
{
internal class PoisonedFileEntry
{
const string ElementName = "File";
internal byte[] Hash { get; set; }
internal string Path { get; set; }
internal PoisonType Type { get; set; }
internal List<PoisonMatch> Matches { get; }
internal PoisonedFileEntry()
{
this.Matches = new List<PoisonMatch>();
}
public XElement ToXml() => this.ToXml(ElementName);
protected XElement ToXml(string myElementName) => new XElement(myElementName,
new XAttribute(nameof(Path), Path),
new XElement(nameof(Hash), Hash.ToHexString()),
new XElement(nameof(Type), Type.ToString()),
Matches.Select(m => m.ToXml())
);
}
}

43
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Utility.cs
View file

@ -1,43 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
{
internal static class Utility
{
internal static string ToHexString(this byte[] bytes)
{
var sb = new StringBuilder();
foreach (var b in bytes)
{
sb.Append(b.ToString("x2"));
}
return sb.ToString();
}
internal static byte[] ToBytes(this string hex)
{
var bytes = new List<byte>();
for (var i = 0; i < hex.Length; i += 2)
{
bytes.Add(Convert.ToByte(hex.Substring(i, 2), 16));
}
return bytes.ToArray();
}
internal static string MakeRelativePath(string filePath, string relativeTo)
{
// Uri.MakeRelativeUri requires the last slash
if (!relativeTo.EndsWith("/") && !relativeTo.EndsWith("\\"))
{
relativeTo += Path.DirectorySeparatorChar;
}
var uri = new Uri(filePath);
var relativeToUri = new Uri(relativeTo);
return relativeToUri.MakeRelativeUri(uri).ToString();
}
}
}

13
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.MSBuildSdkResolver/Directory.Build.props
View file

@ -1,13 +0,0 @@
<Project>
<!--
Do not import the Arcade SDK as this would introduce a prebuilt as this project builds
the custom sdk resolver that makes the bootstrap toolsets available.
-->
<PropertyGroup>
<SkipArcadeSdkImport>true</SkipArcadeSdkImport>
</PropertyGroup>
<Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.props, $(MSBuildThisFileDirectory)..))" />
</Project>

Some files were not shown because too many files have changed in this diff Show more