Add VMR sync functionality into dotnet/dotnet's Codespaces (#15750)

This commit is contained in:
Přemek Vysoký 2023-03-13 14:44:42 +01:00 committed by GitHub
parent 359d6cd0db
commit d4db076b20
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 184 additions and 95 deletions

View file

@ -12,14 +12,14 @@ This Codespace can help you debug the source build of .NET. In case you have run
`dotnet/installer` PR branch, it will contain the VMR (`dotnet/dotnet`) checked out into
`/workspaces/dotnet` with the PR changes pulled into it. You can then attempt to source-build
the VMR which is what the VMR leg in the installer PR build doing. This build takes about 45
minutes and, after completion, produces an archived .NET SDK in `/workspaces/artifacts/x64/Release`.
minutes and, after completion, produces an archived .NET SDK located in
`/workspaces/dotnet/artifacts/x64/Release`.
## Build the SDK
To build the VMR, run following:
```bash
cd /workspaces/dotnet
unset RepositoryName
./build.sh --online
```
@ -37,22 +37,16 @@ When debugging the build, you have two options how to test your changes in this
You can make the changes directly to the local checkout of the VMR at `/workspaces/dotnet`. You
can then try to build the VMR and see if the change works for you.
### Pull changes into the local VMR
### Pull changes into the Codespace from your fork
You can also make a fix in the individual source repository (e.g. `dotnet/runtime`) and push the
fix into a branch (can be in your fork too). Once you have the commit pushed, you can pull this
version of the repository into the VMR locally.
fix into a branch; can be in your fork too. Once you have the commit pushed, you can pull this
version of the repository into the Codespace by running:
Let's consider you pushed a commit with SHA `abcdef` into your fork at `github.com/yourfork/runtime`.
You can now bring this version of runtime into the local VMR in this Codespace by running:
```bash
cd /workspaces/installer
./eng/vmr-sync.sh \
--vmr /workspaces/dotnet \
--tmp /workspaces/tmp \
--repository runtime:abcdef \
--remote runtime:https://github.com/yourfork/runtime
```
/workspaces/synchronize-vmr.sh \
--repository <repo>:<commit, tag or branch> \
--remote <repo>:<fork URI>
```
You can now proceed building the VMR in the Codespace using instructions above. You can repeat

View file

@ -10,6 +10,8 @@ workspace_dir=$(realpath "$installer_dir/../")
tmp_dir=$(realpath "$workspace_dir/tmp")
vmr_dir=$(realpath "$workspace_dir/dotnet")
cp "$installer_dir/.devcontainer/vmr-source-build/synchronize-vmr.sh" "$workspace_dir"
mkdir -p "$tmp_dir"
# Codespaces performs a shallow fetch only
@ -18,21 +20,16 @@ git -C "$installer_dir" fetch --all --unshallow
# We will try to figure out, which branch is the current (PR) branch based off of
# We need this to figure out, which VMR branch to use
vmr_branch=$(git -C "$installer_dir" log --pretty=format:'%D' HEAD^ \
| grep 'origin/' \
| head -n1 \
| grep 'origin/' \
| head -n1 \
| sed 's@origin/@@' \
| sed 's@,.*@@')
pushd "$installer_dir"
"./eng/vmr-sync.sh" \
--vmr "$vmr_dir" \
--tmp "$tmp_dir" \
--branch "$vmr_branch" \
--debug
popd
"$workspace_dir/synchronize-vmr.sh" \
--repository "installer:$(git -C "$installer_dir" rev-parse HEAD)" \
--recursive \
--branch "$vmr_branch" \
--remote "installer:$installer_dir" \
--debug
# Run prep.sh
unset RepositoryName
pushd "$vmr_dir"
./prep.sh
popd
(cd "$vmr_dir" && ./prep.sh)

View file

@ -0,0 +1,4 @@
#!/bin/bash
(cd /workspaces/installer \
&& ./eng/vmr-sync.sh --vmr /workspaces/dotnet --tmp /workspaces/tmp $*)

View file

@ -175,9 +175,9 @@
<Sha>c3ad00ae84489071080a606f6a8e43c9a91a5cc2</Sha>
<SourceBuildTarball RepoName="deployment-tools" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-externals" Version="8.0.0-alpha.1.23157.2">
<Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-externals" Version="8.0.0-alpha.1.23160.1">
<Uri>https://github.com/dotnet/source-build-externals</Uri>
<Sha>7bf6cff81d0dd42b12f6d1a9b29f00607bfb47b4</Sha>
<Sha>57af468ca0fed21a073bd42d46fa6a2181fc7f18</Sha>
<SourceBuild RepoName="source-build-externals" ManagedOnly="true" />
</Dependency>
<Dependency Name="Microsoft.SourceBuild.Intermediate.symreader" Version="1.4.0-beta2-21475-02">

View file

@ -35,6 +35,7 @@ steps:
--branch ${{ parameters.vmrBranch }}
--repository "installer:${{ parameters.targetRef }}"
--recursive
--remote "installer:$(pwd)"
--readme-template $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/README.template.md
--tpn-template $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt
--debug

View file

@ -14,45 +14,47 @@
### folder to this to speed up your re-runs.
###
### USAGE:
### Synchronize the VMR to the commit of dotnet/installer that is currently being built:
### ./vmr-sync.sh --tmp-dir "$HOME/repos/tmp"
###
### Synchronize the VMR to a specific commit of dotnet/runtime using custom fork:
### ./vmr-sync.sh \
### --repository runtime:e7e71da303af8dc97df99b098f21f526398c3943 \
### --remote runtime:https://github.com/yourfork/runtime \
### --tmp-dir "$HOME/repos/tmp"\
### --remote runtime:https://github.com/yourfork/runtime \
### --tmp-dir "$HOME/repos/tmp"
###
### Options:
### -r, --repository name:GIT_REF
### Required. Repository + git ref separated by colon to synchronize to.
### This can be a specific commit, branch, tag..
###
### -t, --tmp, --tmp-dir PATH
### Required. Path to the temporary folder where repositories will be cloned
### --repository name:GIT_REF
### Optional. Repository + git ref separated by colon to synchronize to.
### This can be a specific commit, branch, tag..
### When omitted, the script will synchronize the installer commit based on the version of the parent
### where this script is stored.
### -v, --vmr, --vmr-dir PATH
### Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder
###
### -b, --branch, --vmr-branch BRANCH_NAME
### Optional. Branch of the 'dotnet/dotnet' repo to synchronize to
### This should match the target branch of the PR; defaults to 'main'
### --remote name:URI
### Optional. Additional remote to use during the synchronization
### This can be used to synchronize to a commit from a fork of the repository
### Example: 'runtime:https://github.com/yourfork/runtime'
### --recursive
### Optional. Recursively synchronize all the source build dependencies (declared in Version.Details.xml)
### This is used when performing the full synchronization during installer's CI and the final VMR sync.
### Defaults to false unless no repository is supplied in which case a recursive sync of installer is performed.
### Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch
###
### --debug
### Optional. Turns on the most verbose logging for the VMR tooling
###
### --readme-template
### Optional. Template for VMRs README.md used for regenerating the file to list the newest versions of
### components.
### Defaults to src/VirtualMonoRepo/README.template.md
###
### --recursive
### Optional. Recursively synchronize all the source build dependencies (declared in Version.Details.xml)
### This is used when performing the full synchronization during installer's CI and the final VMR sync.
### Defaults to false unless no repository is supplied in which case a recursive sync of installer is performed.
###
### --remote name:URI
### Optional. Additional remote to use during the synchronization
### This can be used to synchronize to a commit from a fork of the repository
### Example: 'runtime:https://github.com/yourfork/runtime'
###
### --tpn-template
### Optional. Template for the header of VMRs THIRD-PARTY-NOTICES file.
### Defaults to src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt
### --debug
### Optional. Turns on the most verbose logging for the VMR tooling
###
### -v, --vmr, --vmr-dir PATH
### Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder
source="${BASH_SOURCE[0]}"
@ -85,16 +87,22 @@ function highlight () {
}
installer_dir=$(realpath "$scriptroot/../")
tmp_dir=''
vmr_dir=''
vmr_branch='main'
vmr_branch=''
repository=''
additional_remotes=''
recursive=false
verbosity=verbose
additional_remotes="installer:$installer_dir"
readme_template="$installer_dir/src/VirtualMonoRepo/README.template.md"
tpn_template="$installer_dir/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt"
# If installer is a repo, we're in an installer and not in the dotnet/dotnet repo
if [[ -d "$installer_dir/.git" ]]; then
additional_remotes="installer:$installer_dir"
fi
while [[ $# -gt 0 ]]; do
opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
@ -110,7 +118,7 @@ while [[ $# -gt 0 ]]; do
vmr_branch=$2
shift
;;
--repository)
-r|--repository)
repository=$2
shift
;;
@ -138,7 +146,7 @@ while [[ $# -gt 0 ]]; do
;;
*)
fail "Invalid argument: $1"
usage
print_help
exit 1
;;
esac
@ -171,10 +179,8 @@ fi
# Sanitize the input
if [[ -z "$repository" ]]; then
current_sha=$(git -C "$installer_dir" rev-parse HEAD)
echo "No repository specified, will synchronize installer:$current_sha"
repository="installer:$current_sha"
recursive=true
fail "No repository to synchronize specified"
exit 1
fi
if [[ -z "$vmr_dir" ]]; then
@ -194,16 +200,21 @@ fi
if [[ ! -d "$vmr_dir" ]]; then
highlight "Cloning 'dotnet/dotnet' into $vmr_dir.."
git clone https://github.com/dotnet/dotnet "$vmr_dir"
git switch -c "$vmr_branch"
if [[ -n "$vmr_branch" ]]; then
git -C "$vmr_dir" switch -c "$vmr_branch"
fi
else
if ! git -C "$vmr_dir" diff --quiet; then
fail "There are changes in the working tree of $vmr_dir. Please commit or stash your changes"
exit 1
fi
highlight "Preparing $vmr_dir"
git -C "$vmr_dir" checkout "$vmr_branch"
git -C "$vmr_dir" pull
if [[ -n "$vmr_branch" ]]; then
highlight "Preparing $vmr_dir"
git -C "$vmr_dir" checkout "$vmr_branch"
git -C "$vmr_dir" pull
fi
fi
set -e
@ -213,10 +224,10 @@ set -e
highlight 'Installing .NET, preparing the tooling..'
source "$scriptroot/common/tools.sh"
InitializeDotNetCli true
dotnet="$scriptroot/../.dotnet/dotnet"
dotnet=$(realpath "$scriptroot/../.dotnet/dotnet")
"$dotnet" tool restore
highlight "Starting the synchronization to '$repository'.."
highlight "Starting the synchronization of '$repository'.."
set +e
recursive_arg=''
@ -224,6 +235,10 @@ if [[ "$recursive" == "true" ]]; then
recursive_arg="--recursive"
fi
if [[ -n "$additional_remotes" ]]; then
additional_remotes="--additional-remotes $additional_remotes"
fi
# Synchronize the VMR
"$dotnet" darc vmr update \
@ -233,7 +248,7 @@ fi
$recursive_arg \
--readme-template "$readme_template" \
--tpn-template "$tpn_template" \
--additional-remotes "$additional_remotes" \
$additional_remotes \
"$repository"
if [[ $? == 0 ]]; then

View file

@ -0,0 +1,52 @@
<!--
######## ######## ### ######## ######## ## ## #### ######
## ## ## ## ## ## ## ## ## ## ## ## ##
## ## ## ## ## ## ## ## ## ## ## ##
######## ###### ## ## ## ## ## ######### ## ######
## ## ## ######### ## ## ## ## ## ## ##
## ## ## ## ## ## ## ## ## ## ## ## ##
## ## ######## ## ## ######## ## ## ## #### ######
-->
This Codespace can help you debug the source build of .NET. This build takes about
45 minutes and, after completion, produces an archived .NET SDK located in
`/workspaces/dotnet/artifacts/x64/Release`. In case you selected the `prebuilt-sdk`
Codespace, the SDK will already be there.
## Build the SDK
To build the VMR, run following:
```bash
./prep.sh && ./build.sh --online
```
> Please note that, at this time, the build modifies some of the checked-in sources so it might
be preferential to rebuild the Codespace between attempts (or reset the working tree changes).
For more details, see the instructions at https://github.com/dotnet/dotnet.
## Synchronize your changes in locally
When debugging the build, you have two options how to test your changes in this environment.
### Making changes to the VMR directly
You can make the changes directly to the local checkout of the VMR at `/workspaces/dotnet`. You
can then try to build the VMR and see if the change works for you.
### Pull changes into the Codespace from your fork
You can also make a fix in the individual source repository (e.g. `dotnet/runtime`) and push the
fix into a branch; can be in your fork too. Once you have the commit pushed, you can pull this
version of the repository into the Codespace by running:
```
/workspaces/synchronize-vmr.sh \
--repository <repo>:<commit, tag or branch> \
--remote <repo>:<fork URI>
```
You can now proceed building the VMR in the Codespace using instructions above. You can repeat
this process and sync a new commit from your fork. Only note that, at this time, Source-Build
modifies some of the checked-in sources so you'll need to revert the working tree changes
between attempts.

View file

@ -11,6 +11,12 @@
"extensions": [
"ms-dotnettools.csharp"
]
},
"codespaces": {
"openFiles": [
".devcontainer/README.md"
]
}
}
},
"onCreateCommand": ".devcontainer/init.sh"
}

View file

@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -eux
source="${BASH_SOURCE[0]}"
script_root="$( cd -P "$( dirname "$source" )" && pwd )"
workspace_dir=$(realpath "$script_root/../../")
tmp_dir=$(realpath "$workspace_dir/tmp")
vmr_dir=$(realpath "$workspace_dir/dotnet")
cp "$vmr_dir/.devcontainer/synchronize-vmr.sh" "$workspace_dir"
mkdir -p "$tmp_dir"

View file

@ -1,11 +0,0 @@
#!/usr/bin/env bash
source="${BASH_SOURCE[0]}"
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
"$scriptroot"/../../prep.sh
# GitHub Codespaces sets this and it conflicts with source-build scripts.
unset RepositoryName
"$scriptroot"/../../build.sh --online --clean-while-building || exit 0

View file

@ -11,7 +11,12 @@
"extensions": [
"ms-dotnettools.csharp"
]
},
"codespaces": {
"openFiles": [
".devcontainer/README.md"
]
}
},
"onCreateCommand": "${containerWorkspaceFolder}/.devcontainer/prebuilt-sdk/build.sh"
"onCreateCommand": ".devcontainer/prebuilt-sdk/init.sh"
}

View file

@ -0,0 +1,9 @@
#!/usr/bin/env bash
source="${BASH_SOURCE[0]}"
script_root="$( cd -P "$( dirname "$source" )" && pwd )"
"$script_root"/../../prep.sh
cp "$script_root/../synchronize-vmr.sh" "/workspaces/"
"$script_root"/../../build.sh --online --clean-while-building || exit 0

View file

@ -0,0 +1,4 @@
#!/bin/bash
(cd /workspaces/dotnet/src/installer \
&& ./eng/vmr-sync.sh --vmr /workspaces/dotnet --tmp /workspaces/tmp --no-vmr-prepare $*)

View file

@ -19,7 +19,6 @@ usage() {
SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)"
MSBUILD_ARGUMENTS=("-flp:v=detailed")
CUSTOM_REF_PACKAGES_DIR=''
CUSTOM_PACKAGES_DIR=''
alternateTarget=false
runningSmokeTests=false
@ -34,7 +33,7 @@ while :; do
break
fi
lowerI="$(echo $1 | awk '{print tolower($0)}')"
lowerI="$(echo "$1" | awk '{print tolower($0)}')"
case $lowerI in
--clean-while-building)
MSBUILD_ARGUMENTS+=( "-p:CleanWhileBuilding=true")
@ -78,7 +77,7 @@ while :; do
echo "Detected '--': passing remaining parameters '$@' as build.sh arguments."
break
;;
-?|-h|--help)
'-?'|-h|--help)
usage
exit 0
;;
@ -115,13 +114,13 @@ if [ -f "${packagesArchiveDir}archiveArtifacts.txt" ]; then
fi
if [ -d "$CUSTOM_SDK_DIR" ]; then
export SDK_VERSION=`"$CUSTOM_SDK_DIR/dotnet" --version`
export SDK_VERSION=$("$CUSTOM_SDK_DIR/dotnet" --version)
export CLI_ROOT="$CUSTOM_SDK_DIR"
export _InitializeDotNetCli="$CLI_ROOT/dotnet"
export CustomDotNetSdkDir="$CLI_ROOT"
echo "Using custom bootstrap SDK from '$CLI_ROOT', version '$SDK_VERSION'"
else
sdkLine=`grep -m 1 'dotnet' "$SCRIPT_ROOT/global.json"`
sdkLine=$(grep -m 1 'dotnet' "$SCRIPT_ROOT/global.json")
sdkPattern="\"dotnet\" *: *\"(.*)\""
if [[ $sdkLine =~ $sdkPattern ]]; then
export SDK_VERSION=${BASH_REMATCH[1]}
@ -134,7 +133,7 @@ packageVersionsPath=''
if [[ "$CUSTOM_PACKAGES_DIR" != "" && -f "$CUSTOM_PACKAGES_DIR/PackageVersions.props" ]]; then
packageVersionsPath="$CUSTOM_PACKAGES_DIR/PackageVersions.props"
elif [ -d "$packagesArchiveDir" ]; then
sourceBuiltArchive=`find $packagesArchiveDir -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'`
sourceBuiltArchive=$(find "$packagesArchiveDir" -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz')
if [ -f "${packagesPreviouslySourceBuiltDir}}PackageVersions.props" ]; then
packageVersionsPath=${packagesPreviouslySourceBuiltDir}PackageVersions.props
elif [ -f "$sourceBuiltArchive" ]; then
@ -150,7 +149,7 @@ if [ ! -f "$packageVersionsPath" ]; then
exit 1
fi
arcadeSdkLine=`grep -m 1 'MicrosoftDotNetArcadeSdkVersion' "$packageVersionsPath"`
arcadeSdkLine=$(grep -m 1 'MicrosoftDotNetArcadeSdkVersion' "$packageVersionsPath")
versionPattern="<MicrosoftDotNetArcadeSdkVersion>(.*)</MicrosoftDotNetArcadeSdkVersion>"
if [[ $arcadeSdkLine =~ $versionPattern ]]; then
export ARCADE_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
@ -163,7 +162,7 @@ if [[ $arcadeSdkLine =~ $versionPattern ]]; then
export SOURCE_BUILT_SDK_DIR_ARCADE=$packagesRestoredDir/ArcadeBootstrapPackage/microsoft.dotnet.arcade.sdk/$ARCADE_BOOTSTRAP_VERSION
fi
sourceLinkLine=`grep -m 1 'MicrosoftSourceLinkCommonVersion' "$packageVersionsPath"`
sourceLinkLine=$(grep -m 1 'MicrosoftSourceLinkCommonVersion' "$packageVersionsPath")
versionPattern="<MicrosoftSourceLinkCommonVersion>(.*)</MicrosoftSourceLinkCommonVersion>"
if [[ $sourceLinkLine =~ $versionPattern ]]; then
export SOURCE_LINK_BOOTSTRAP_VERSION=${BASH_REMATCH[1]}
@ -179,11 +178,11 @@ LogDateStamp=$(date +"%m%d%H%M%S")
"$CLI_ROOT/dotnet" build-server shutdown
if [ "$alternateTarget" == "true" ]; then
"$CLI_ROOT/dotnet" msbuild "$SCRIPT_ROOT/build.proj" -bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildTests_$LogDateStamp.binlog -flp:LogFile=$SCRIPT_ROOT/artifacts/logs/BuildTests_$LogDateStamp.log -clp:v=m ${MSBUILD_ARGUMENTS[@]} "$@"
"$CLI_ROOT/dotnet" msbuild "$SCRIPT_ROOT/build.proj" -bl:"$SCRIPT_ROOT/artifacts/log/Debug/BuildTests_$LogDateStamp.binlog" -flp:"LogFile=$SCRIPT_ROOT/artifacts/logs/BuildTests_$LogDateStamp.log" -clp:v=m ${MSBUILD_ARGUMENTS[@]} "$@"
else
"$CLI_ROOT/dotnet" msbuild "$SCRIPT_ROOT/eng/tools/init-build.proj" -bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildXPlatTasks_$LogDateStamp.binlog -flp:LogFile=$SCRIPT_ROOT/artifacts/logs/BuildXPlatTasks_$LogDateStamp.log -t:PrepareOfflineLocalTools ${MSBUILD_ARGUMENTS[@]} "$@"
"$CLI_ROOT/dotnet" msbuild "$SCRIPT_ROOT/eng/tools/init-build.proj" -bl:"$SCRIPT_ROOT/artifacts/log/Debug/BuildXPlatTasks_$LogDateStamp.binlog" -flp:LogFile="$SCRIPT_ROOT/artifacts/logs/BuildXPlatTasks_$LogDateStamp.log" -t:PrepareOfflineLocalTools ${MSBUILD_ARGUMENTS[@]} "$@"
# kill off the MSBuild server so that on future invocations we pick up our custom SDK Resolver
"$CLI_ROOT/dotnet" build-server shutdown
"$CLI_ROOT/dotnet" msbuild "$SCRIPT_ROOT/build.proj" -bl:$SCRIPT_ROOT/artifacts/log/Debug/Build_$LogDateStamp.binlog -flp:LogFile=$SCRIPT_ROOT/artifacts/logs/Build_$LogDateStamp.log ${MSBUILD_ARGUMENTS[@]} "$@"
"$CLI_ROOT/dotnet" msbuild "$SCRIPT_ROOT/build.proj" -bl:"$SCRIPT_ROOT/artifacts/log/Debug/Build_$LogDateStamp.binlog" -flp:"LogFile=$SCRIPT_ROOT/artifacts/logs/Build_$LogDateStamp.log" ${MSBUILD_ARGUMENTS[@]} "$@"
fi

View file

@ -1,7 +1,7 @@
<Project>
<PropertyGroup>
<RepositoryName Condition="'$(RepositoryName)' == ''">$(MSBuildProjectName)</RepositoryName>
<RepositoryName>$(MSBuildProjectName)</RepositoryName>
</PropertyGroup>
<Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.props, $(MSBuildThisFileDirectory)..))" />