Move Source Link args to build.sh, support release manifest (#16184)

Co-authored-by: Michael Simons <msimons@microsoft.com>
This commit is contained in:
Přemek Vysoký 2023-04-24 16:23:43 +02:00 committed by GitHub
parent 18d53e5710
commit f308bf657f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 254 additions and 211 deletions

View file

@ -183,11 +183,6 @@ jobs:
echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /vmr/.dotnet"
fi
if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then
customPrepArgs="$customPrepArgs --source-repository https://github.com/dotnet/dotnet"
customPrepArgs="$customPrepArgs --source-version $(git -C "${{ parameters.vmrPath }}" rev-parse HEAD)"
fi
docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh $customPrepArgs
displayName: Prep the Build
@ -213,6 +208,11 @@ jobs:
customBuildArgs="$customBuildArgs --poison"
fi
if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then
customBuildArgs="$customBuildArgs --source-repository https://github.com/dotnet/dotnet"
customBuildArgs="$customBuildArgs --source-version $(git -C "${{ parameters.vmrPath }}" rev-parse HEAD)"
fi
docker run --rm -v "$(sourcesPath):/vmr" -w /vmr $customRunArgs ${{ parameters.container }} ./build.sh --clean-while-building $(additionalBuildArgs) $customBuildArgs
displayName: Build

View file

@ -1,23 +1,31 @@
#!/usr/bin/env bash
### Usage: $0 [options]
###
### Options:
### --clean-while-building Cleans each repo after building (reduces disk space usage)
### --online Build using online sources
### --poison Build with poisoning checks
### --run-smoke-test Don't build; run smoke tests
### --source-repository <URL> Source Link repository URL, required when building from tarball
### --source-version <SHA> Source Link revision, required when building from tarball
### --release-manifest <FILE> A JSON file, an alternative source of Source Link metadata
### --use-mono-runtime Output uses the mono runtime
### --with-packages <DIR> Use the specified directory of previously-built packages
### --with-sdk <DIR> Use the SDK in the specified directory for bootstrapping
###
### Use -- to send the remaining arguments to MSBuild
set -euo pipefail
IFS=$'\n\t'
usage() {
echo "usage: $0 [options]"
echo "options:"
echo " --clean-while-building cleans each repo after building (reduces disk space usage)"
echo " --online build using online sources"
echo " --poison build with poisoning checks"
echo " --run-smoke-test don't build; run smoke tests"
echo " --use-mono-runtime output uses the mono runtime"
echo " --with-packages <dir> use the specified directory of previously-built packages"
echo " --with-sdk <dir> use the SDK in the specified directory for bootstrapping"
echo "use -- to send the remaining arguments to MSBuild"
echo ""
}
source="${BASH_SOURCE[0]}"
SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)"
function print_help () {
sed -n '/^### /,/^$/p' "$source" | cut -b 5-
}
MSBUILD_ARGUMENTS=("-flp:v=detailed")
CUSTOM_PACKAGES_DIR=''
alternateTarget=false
@ -28,67 +36,125 @@ packagesRestoredDir="${packagesDir}restored/"
packagesPreviouslySourceBuiltDir="${packagesDir}previously-source-built/"
CUSTOM_SDK_DIR=''
sourceRepository=''
sourceVersion=''
releaseManifest=''
while :; do
if [ $# -le 0 ]; then
break
if [ $# -le 0 ]; then
break
fi
lowerI="$(echo "$1" | awk '{print tolower($0)}')"
case $lowerI in
--clean-while-building)
MSBUILD_ARGUMENTS+=( "-p:CleanWhileBuilding=true")
;;
--online)
MSBUILD_ARGUMENTS+=( "-p:BuildWithOnlineSources=true")
;;
--poison)
MSBUILD_ARGUMENTS+=( "-p:EnablePoison=true")
;;
--run-smoke-test)
alternateTarget=true
runningSmokeTests=true
MSBUILD_ARGUMENTS+=( "-t:RunSmokeTest" )
;;
--source-repository)
sourceRepository="$2"
shift
;;
--source-version)
sourceVersion="$2"
shift
;;
--release-manifest)
releaseManifest="$2"
shift
;;
--use-mono-runtime)
MSBUILD_ARGUMENTS+=( "/p:SourceBuildUseMonoRuntime=true" )
;;
--with-packages)
CUSTOM_PACKAGES_DIR="$(cd -P "$2" && pwd)"
if [ ! -d "$CUSTOM_PACKAGES_DIR" ]; then
echo "Custom prviously built packages directory '$CUSTOM_PACKAGES_DIR' does not exist"
exit 1
fi
shift
;;
--with-sdk)
CUSTOM_SDK_DIR="$(cd -P "$2" && pwd)"
if [ ! -d "$CUSTOM_SDK_DIR" ]; then
echo "Custom SDK directory '$CUSTOM_SDK_DIR' does not exist"
exit 1
fi
if [ ! -x "$CUSTOM_SDK_DIR/dotnet" ]; then
echo "Custom SDK '$CUSTOM_SDK_DIR/dotnet' does not exist or is not executable"
exit 1
fi
shift
;;
--)
shift
echo "Detected '--': passing remaining parameters '$@' as build.sh arguments."
break
;;
'-?'|-h|--help)
print_help
exit 0
;;
*)
echo "Unrecognized argument '$1'"
print_help
exit 1
;;
esac
shift
done
GIT_DIR="$SCRIPT_ROOT/.git"
if [ -f "$GIT_DIR/index" ]; then # We check for index because if outside of git, we create config and HEAD manually
if [ -n "$sourceRepository" ] || [ -n "$sourceVersion" ] || [ -n "$releaseManifest" ]; then
echo "ERROR: Source Link arguments cannot be used in a git repository"
exit 1
fi
else
if [ -z "$releaseManifest" ]; then
if [ -z "$sourceRepository" ] || [ -z "$sourceVersion" ]; then
echo "ERROR: $SCRIPT_ROOT is not a git repository, either --release-manifest or --source-repository and --source-version must be specified"
exit 1
fi
else
if [ -n "$sourceRepository" ] || [ -n "$sourceVersion" ]; then
echo "ERROR: --release-manifest cannot be specified together with --source-repository and --source-version"
exit 1
fi
lowerI="$(echo "$1" | awk '{print tolower($0)}')"
case $lowerI in
--clean-while-building)
MSBUILD_ARGUMENTS+=( "-p:CleanWhileBuilding=true")
;;
--online)
MSBUILD_ARGUMENTS+=( "-p:BuildWithOnlineSources=true")
;;
--poison)
MSBUILD_ARGUMENTS+=( "-p:EnablePoison=true")
;;
--run-smoke-test)
alternateTarget=true
runningSmokeTests=true
MSBUILD_ARGUMENTS+=( "-t:RunSmokeTest" )
;;
--use-mono-runtime)
MSBUILD_ARGUMENTS+=( "/p:SourceBuildUseMonoRuntime=true" )
;;
--with-packages)
CUSTOM_PACKAGES_DIR="$(cd -P "$2" && pwd)"
if [ ! -d "$CUSTOM_PACKAGES_DIR" ]; then
echo "Custom prviously built packages directory '$CUSTOM_PACKAGES_DIR' does not exist"
exit 1
fi
shift
;;
--with-sdk)
CUSTOM_SDK_DIR="$(cd -P "$2" && pwd)"
if [ ! -d "$CUSTOM_SDK_DIR" ]; then
echo "Custom SDK directory '$CUSTOM_SDK_DIR' does not exist"
exit 1
fi
if [ ! -x "$CUSTOM_SDK_DIR/dotnet" ]; then
echo "Custom SDK '$CUSTOM_SDK_DIR/dotnet' does not exist or is not executable"
exit 1
fi
shift
;;
--)
shift
echo "Detected '--': passing remaining parameters '$@' as build.sh arguments."
break
;;
'-?'|-h|--help)
usage
exit 0
;;
*)
echo "Unrecognized argument '$1'"
usage
exit 1
;;
esac
shift
done
get_property() {
local json_file_path="$1"
local property_name="$2"
grep -oP '(?<="'$property_name'": ")[^"]*' "$json_file_path"
}
sourceRepository=$(get_property "$releaseManifest" sourceRepository) \
|| (echo "ERROR: Failed to find sourceRepository in $releaseManifest" && exit 1)
sourceVersion=$(get_property "$releaseManifest" sourceVersion) \
|| (echo "ERROR: Failed to find sourceVersion in $releaseManifest" && exit 1)
if [ -z "$sourceRepository" ] || [ -z "$sourceVersion" ]; then
echo "ERROR: sourceRepository and sourceVersion must be specified in $releaseManifest"
exit 1
fi
fi
# We need to add "fake" .git/ files when not building from a git repository
mkdir -p "$GIT_DIR"
echo '[remote "origin"]' > "$GIT_DIR/config"
echo "url=\"$sourceRepository\"" >> "$GIT_DIR/config"
echo "$sourceVersion" > "$GIT_DIR/HEAD"
fi
if [ "$CUSTOM_PACKAGES_DIR" != "" ]; then
if [ "$runningSmokeTests" == "true" ]; then

View file

@ -1,199 +1,172 @@
#!/usr/bin/env bash
### Usage: $0
###
### Prepares the environment to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and
### installing the version of dotnet referenced in global.json
###
### Options:
### --no-artifacts Exclude the download of the previously source-built artifacts archive
### --no-bootstrap Don't replace portable packages in the download source-built artifacts
### --no-prebuilts Exclude the download of the prebuilts archive
### --no-sdk Exclude the download of the .NET SDK
### --runtime-source-feed URL of a remote server or a local directory, from which SDKs and
### runtimes can be downloaded
### --runtime-source-feed-key Key for accessing the above server, if necessary
set -euo pipefail
IFS=$'\n\t'
source="${BASH_SOURCE[0]}"
SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)"
usage() {
echo "usage: $0"
echo ""
echo " Prepares the environment to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and"
echo " installing the version of dotnet referenced in global.json"
echo "options:"
echo " --no-artifacts Exclude the download of the previously source-built artifacts archive"
echo " --no-bootstrap Don't replace portable packages in the download source-built artifacts"
echo " --no-prebuilts Exclude the download of the prebuilts archive"
echo " --no-sdk Exclude the download of the .NET SDK"
echo " --source-repository <url> Source Link repository URL, required when building from tarball"
echo " --source-version <sha> Source Link revision, required when building from tarball"
echo " --runtime-source-feed URL of a remote server or a local directory, from which SDKs and"
echo " runtimes can be downloaded"
echo " --runtime-source-feed-key Key for accessing the above server, if necessary"
echo ""
function print_help () {
sed -n '/^### /,/^$/p' "$source" | cut -b 5-
}
buildBootstrap=true
downloadArtifacts=true
downloadPrebuilts=true
installDotnet=true
sourceUrl=''
sourceVersion=''
runtime_source_feed='' # IBM requested these to support s390x scenarios
runtime_source_feed_key='' # IBM requested these to support s390x scenarios
positional_args=()
while :; do
if [ $# -le 0 ]; then
break
fi
lowerI="$(echo "$1" | awk '{print tolower($0)}')"
case $lowerI in
"-?"|-h|--help)
usage
exit 0
;;
--no-bootstrap)
buildBootstrap=false
;;
--no-artifacts)
downloadArtifacts=false
;;
--no-prebuilts)
downloadPrebuilts=false
;;
--no-sdk)
installDotnet=false
;;
--source-repository)
sourceUrl="$2"
shift
;;
--source-version)
sourceVersion="$2"
shift
;;
--runtime-source-feed)
runtime_source_feed=$2
shift
;;
--runtime-source-feed-key)
runtime_source_feed_key=$2
shift
;;
*)
positional_args+=("$1")
;;
esac
if [ $# -le 0 ]; then
break
fi
lowerI="$(echo "$1" | awk '{print tolower($0)}')"
case $lowerI in
"-?"|-h|--help)
print_help
exit 0
;;
--no-bootstrap)
buildBootstrap=false
;;
--no-artifacts)
downloadArtifacts=false
;;
--no-prebuilts)
downloadPrebuilts=false
;;
--no-sdk)
installDotnet=false
;;
--runtime-source-feed)
runtime_source_feed=$2
shift
;;
--runtime-source-feed-key)
runtime_source_feed_key=$2
shift
;;
*)
positional_args+=("$1")
;;
esac
shift
shift
done
# Attempting to bootstrap without an SDK will fail. So either the --no-sdk flag must be passed
# or a pre-existing .dotnet SDK directory must exist.
if [ "$buildBootstrap" == true ] && [ "$installDotnet" == false ] && [ ! -d "$SCRIPT_ROOT/.dotnet" ]; then
echo " ERROR: --no-sdk requires --no-bootstrap or a pre-existing .dotnet SDK directory. Exiting..."
exit 1
echo " ERROR: --no-sdk requires --no-bootstrap or a pre-existing .dotnet SDK directory. Exiting..."
exit 1
fi
# Check to make sure curl exists to download the archive files
if ! command -v curl &> /dev/null
then
echo " ERROR: curl not found. Exiting..."
exit 1
fi
GIT_DIR="$SCRIPT_ROOT/.git"
if [ -f "$GIT_DIR/index" ]; then # We check for index because if outside of git, we create config and HEAD manually
if [ -n "$sourceUrl" ] || [ -n "$sourceVersion" ]; then
echo "ERROR: $SCRIPT_ROOT is a git repository, --source-repository and --source-version cannot be used."
exit 1
fi
else
if [ -z "$sourceUrl" ] || [ -z "$sourceVersion" ]; then
echo "ERROR: $SCRIPT_ROOT is not a git repository, --source-repository and --source-version must be specified."
exit 1
fi
# We need to add "fake" .git/ files when not building from a git repository
mkdir -p "$GIT_DIR"
echo '[remote "origin"]' > "$GIT_DIR/config"
echo "url=\"$sourceUrl\"" >> "$GIT_DIR/config"
echo "$sourceVersion" > "$GIT_DIR/HEAD"
echo " ERROR: curl not found. Exiting..."
exit 1
fi
# Check if Private.SourceBuilt artifacts archive exists
artifactsBaseFileName="Private.SourceBuilt.Artifacts"
packagesArchiveDir="$SCRIPT_ROOT/prereqs/packages/archive/"
if [ "$downloadArtifacts" == true ] && [ -f ${packagesArchiveDir}${artifactsBaseFileName}.*.tar.gz ]; then
echo " Private.SourceBuilt.Artifacts.*.tar.gz exists...it will not be downloaded"
downloadArtifacts=false
echo " Private.SourceBuilt.Artifacts.*.tar.gz exists...it will not be downloaded"
downloadArtifacts=false
fi
# Check if Private.SourceBuilt prebuilts archive exists
prebuiltsBaseFileName="Private.SourceBuilt.Prebuilts"
if [ "$downloadPrebuilts" == true ] && [ -f ${packagesArchiveDir}${prebuiltsBaseFileName}.*.tar.gz ]; then
echo " Private.SourceBuilt.Prebuilts.*.tar.gz exists...it will not be downloaded"
downloadPrebuilts=false
echo " Private.SourceBuilt.Prebuilts.*.tar.gz exists...it will not be downloaded"
downloadPrebuilts=false
fi
# Check if dotnet is installed
if [ "$installDotnet" == true ] && [ -d "$SCRIPT_ROOT/.dotnet" ]; then
echo " ./.dotnet SDK directory exists...it will not be installed"
installDotnet=false;
echo " ./.dotnet SDK directory exists...it will not be installed"
installDotnet=false;
fi
function DownloadArchive {
archiveType="$1"
isRequired="$2"
archiveType="$1"
isRequired="$2"
packageVersionsPath="$SCRIPT_ROOT/eng/Versions.props"
notFoundMessage="No source-built $archiveType found to download..."
packageVersionsPath="$SCRIPT_ROOT/eng/Versions.props"
notFoundMessage="No source-built $archiveType found to download..."
echo " Looking for source-built $archiveType to download..."
archiveVersionLine=$(grep -m 1 "<PrivateSourceBuilt${archiveType}Url>" "$packageVersionsPath" || :)
versionPattern="<PrivateSourceBuilt${archiveType}Url>(.*)</PrivateSourceBuilt${archiveType}Url>"
if [[ $archiveVersionLine =~ $versionPattern ]]; then
archiveUrl="${BASH_REMATCH[1]}"
echo " Downloading source-built $archiveType from $archiveUrl..."
(cd "$packagesArchiveDir" && curl --retry 5 -O "$archiveUrl")
elif [ "$isRequired" == true ]; then
echo " ERROR: $notFoundMessage"
exit 1
else
echo " $notFoundMessage"
fi
echo " Looking for source-built $archiveType to download..."
archiveVersionLine=$(grep -m 1 "<PrivateSourceBuilt${archiveType}Url>" "$packageVersionsPath" || :)
versionPattern="<PrivateSourceBuilt${archiveType}Url>(.*)</PrivateSourceBuilt${archiveType}Url>"
if [[ $archiveVersionLine =~ $versionPattern ]]; then
archiveUrl="${BASH_REMATCH[1]}"
echo " Downloading source-built $archiveType from $archiveUrl..."
(cd "$packagesArchiveDir" && curl --retry 5 -O "$archiveUrl")
elif [ "$isRequired" == true ]; then
echo " ERROR: $notFoundMessage"
exit 1
else
echo " $notFoundMessage"
fi
}
function BootstrapArtifacts {
DOTNET_SDK_PATH="$SCRIPT_ROOT/.dotnet"
DOTNET_SDK_PATH="$SCRIPT_ROOT/.dotnet"
# Create working directory for running bootstrap project
workingDir=$(mktemp -d)
echo " Building bootstrap previously source-built in $workingDir"
# Create working directory for running bootstrap project
workingDir=$(mktemp -d)
echo " Building bootstrap previously source-built in $workingDir"
# Copy bootstrap project to working dir
cp "$SCRIPT_ROOT/eng/bootstrap/buildBootstrapPreviouslySB.csproj" "$workingDir"
# Copy bootstrap project to working dir
cp "$SCRIPT_ROOT/eng/bootstrap/buildBootstrapPreviouslySB.csproj" "$workingDir"
# Copy NuGet.config from the installer repo to have the right feeds
cp "$SCRIPT_ROOT/src/installer/NuGet.config" "$workingDir"
# Copy NuGet.config from the installer repo to have the right feeds
cp "$SCRIPT_ROOT/src/installer/NuGet.config" "$workingDir"
# Get PackageVersions.props from existing prev-sb archive
echo " Retrieving PackageVersions.props from existing archive"
sourceBuiltArchive=$(find "$packagesArchiveDir" -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz')
if [ -f "$sourceBuiltArchive" ]; then
tar -xzf "$sourceBuiltArchive" -C "$workingDir" PackageVersions.props
fi
# Get PackageVersions.props from existing prev-sb archive
echo " Retrieving PackageVersions.props from existing archive"
sourceBuiltArchive=$(find "$packagesArchiveDir" -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz')
if [ -f "$sourceBuiltArchive" ]; then
tar -xzf "$sourceBuiltArchive" -C "$workingDir" PackageVersions.props
fi
# Run restore on project to initiate download of bootstrap packages
"$DOTNET_SDK_PATH/dotnet" restore "$workingDir/buildBootstrapPreviouslySB.csproj" /bl:artifacts/prep/bootstrap.binlog /fileLoggerParameters:LogFile=artifacts/prep/bootstrap.log /p:ArchiveDir="$packagesArchiveDir" /p:BootstrapOverrideVersionsProps="$SCRIPT_ROOT/eng/bootstrap/OverrideBootstrapVersions.props"
# Run restore on project to initiate download of bootstrap packages
"$DOTNET_SDK_PATH/dotnet" restore "$workingDir/buildBootstrapPreviouslySB.csproj" /bl:artifacts/prep/bootstrap.binlog /fileLoggerParameters:LogFile=artifacts/prep/bootstrap.log /p:ArchiveDir="$packagesArchiveDir" /p:BootstrapOverrideVersionsProps="$SCRIPT_ROOT/eng/bootstrap/OverrideBootstrapVersions.props"
# Remove working directory
rm -rf "$workingDir"
# Remove working directory
rm -rf "$workingDir"
}
# Check for the version of dotnet to install
if [ "$installDotnet" == true ]; then
echo " Installing dotnet..."
(source ./eng/common/tools.sh && InitializeDotNetCli true)
echo " Installing dotnet..."
(source ./eng/common/tools.sh && InitializeDotNetCli true)
fi
# Read the eng/Versions.props to get the archives to download and download them
if [ "$downloadArtifacts" == true ]; then
DownloadArchive Artifacts true
if [ "$buildBootstrap" == true ]; then
BootstrapArtifacts
fi
DownloadArchive Artifacts true
if [ "$buildBootstrap" == true ]; then
BootstrapArtifacts
fi
fi
if [ "$downloadPrebuilts" == true ]; then
DownloadArchive Prebuilts false
DownloadArchive Prebuilts false
fi

View file

@ -139,10 +139,14 @@ ln -s $HOME/.dotnet/dotnet /usr/bin/dotnet
You can also utilize [GitHub Codespaces](https://github.com/features/codespaces) where you can find preset containers in this repository.
### Building outside of git
### Building from released sources
.NET uses git metadata so that it can link assemblies to their original source code when debugging (think "Step into.." functionality) and for that it needs information about the original place the code comes from.
When you're building source code only, taken outside of context of a git repository (e.g. you download it from the release page), you will need to specify the source repository to the `prep.sh` script via the `--source-repository` and `--source-version` arguments. This can be your fork of the repository and should match the origin where your SDK was built from.
You can also build from sources (and not from a context of a git repository), such as the ones you can acquire from a [dotnet/dotnet release](https://github.com/dotnet/dotnet/releases).
In this case, you need to provide additional information which includes the original repository and commit hash the code was built from so that the SDK can provide a better debugging experience (think the `Step into..` functionality).
Usually, this means the [dotnet/dotnet repository](https://github.com/dotnet/dotnet) together with the commit the release tag is connected to.
In practice, this means that when calling the main build script, you need to provide additional arguments when building outside of a context of a git repository.
Alternatively, you can also provide a manifest file where this information can be read from. This file (`release.json`) can be found attached with the [dotnet/dotnet release](https://github.com/dotnet/dotnet/releases).
## List of components