diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 77b477c25..975c5e107 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -183,11 +183,6 @@ jobs: echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /vmr/.dotnet" fi - if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then - customPrepArgs="$customPrepArgs --source-repository https://github.com/dotnet/dotnet" - customPrepArgs="$customPrepArgs --source-version $(git -C "${{ parameters.vmrPath }}" rev-parse HEAD)" - fi - docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh $customPrepArgs displayName: Prep the Build @@ -213,6 +208,11 @@ jobs: customBuildArgs="$customBuildArgs --poison" fi + if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then + customBuildArgs="$customBuildArgs --source-repository https://github.com/dotnet/dotnet" + customBuildArgs="$customBuildArgs --source-version $(git -C "${{ parameters.vmrPath }}" rev-parse HEAD)" + fi + docker run --rm -v "$(sourcesPath):/vmr" -w /vmr $customRunArgs ${{ parameters.container }} ./build.sh --clean-while-building $(additionalBuildArgs) $customBuildArgs displayName: Build diff --git a/src/SourceBuild/content/build.sh b/src/SourceBuild/content/build.sh index f797bab62..cd873f9d2 100755 --- a/src/SourceBuild/content/build.sh +++ b/src/SourceBuild/content/build.sh @@ -1,23 +1,31 @@ #!/usr/bin/env bash + +### Usage: $0 [options] +### +### Options: +### --clean-while-building Cleans each repo after building (reduces disk space usage) +### --online Build using online sources +### --poison Build with poisoning checks +### --run-smoke-test Don't build; run smoke tests +### --source-repository Source Link repository URL, required when building from tarball +### --source-version Source Link revision, required when building from tarball +### --release-manifest A JSON file, an alternative source of Source Link metadata +### --use-mono-runtime Output uses the mono runtime +### --with-packages Use the specified directory of previously-built packages +### --with-sdk Use the SDK in the specified directory for bootstrapping +### +### Use -- to send the remaining arguments to MSBuild + set -euo pipefail IFS=$'\n\t' -usage() { - echo "usage: $0 [options]" - echo "options:" - echo " --clean-while-building cleans each repo after building (reduces disk space usage)" - echo " --online build using online sources" - echo " --poison build with poisoning checks" - echo " --run-smoke-test don't build; run smoke tests" - echo " --use-mono-runtime output uses the mono runtime" - echo " --with-packages use the specified directory of previously-built packages" - echo " --with-sdk use the SDK in the specified directory for bootstrapping" - echo "use -- to send the remaining arguments to MSBuild" - echo "" -} - +source="${BASH_SOURCE[0]}" SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" +function print_help () { + sed -n '/^### /,/^$/p' "$source" | cut -b 5- +} + MSBUILD_ARGUMENTS=("-flp:v=detailed") CUSTOM_PACKAGES_DIR='' alternateTarget=false @@ -28,67 +36,125 @@ packagesRestoredDir="${packagesDir}restored/" packagesPreviouslySourceBuiltDir="${packagesDir}previously-source-built/" CUSTOM_SDK_DIR='' +sourceRepository='' +sourceVersion='' +releaseManifest='' + while :; do - if [ $# -le 0 ]; then - break + if [ $# -le 0 ]; then + break + fi + + lowerI="$(echo "$1" | awk '{print tolower($0)}')" + case $lowerI in + --clean-while-building) + MSBUILD_ARGUMENTS+=( "-p:CleanWhileBuilding=true") + ;; + --online) + MSBUILD_ARGUMENTS+=( "-p:BuildWithOnlineSources=true") + ;; + --poison) + MSBUILD_ARGUMENTS+=( "-p:EnablePoison=true") + ;; + --run-smoke-test) + alternateTarget=true + runningSmokeTests=true + MSBUILD_ARGUMENTS+=( "-t:RunSmokeTest" ) + ;; + --source-repository) + sourceRepository="$2" + shift + ;; + --source-version) + sourceVersion="$2" + shift + ;; + --release-manifest) + releaseManifest="$2" + shift + ;; + --use-mono-runtime) + MSBUILD_ARGUMENTS+=( "/p:SourceBuildUseMonoRuntime=true" ) + ;; + --with-packages) + CUSTOM_PACKAGES_DIR="$(cd -P "$2" && pwd)" + if [ ! -d "$CUSTOM_PACKAGES_DIR" ]; then + echo "Custom prviously built packages directory '$CUSTOM_PACKAGES_DIR' does not exist" + exit 1 + fi + shift + ;; + --with-sdk) + CUSTOM_SDK_DIR="$(cd -P "$2" && pwd)" + if [ ! -d "$CUSTOM_SDK_DIR" ]; then + echo "Custom SDK directory '$CUSTOM_SDK_DIR' does not exist" + exit 1 + fi + if [ ! -x "$CUSTOM_SDK_DIR/dotnet" ]; then + echo "Custom SDK '$CUSTOM_SDK_DIR/dotnet' does not exist or is not executable" + exit 1 + fi + shift + ;; + --) + shift + echo "Detected '--': passing remaining parameters '$@' as build.sh arguments." + break + ;; + '-?'|-h|--help) + print_help + exit 0 + ;; + *) + echo "Unrecognized argument '$1'" + print_help + exit 1 + ;; + esac + shift +done + +GIT_DIR="$SCRIPT_ROOT/.git" +if [ -f "$GIT_DIR/index" ]; then # We check for index because if outside of git, we create config and HEAD manually + if [ -n "$sourceRepository" ] || [ -n "$sourceVersion" ] || [ -n "$releaseManifest" ]; then + echo "ERROR: Source Link arguments cannot be used in a git repository" + exit 1 + fi +else + if [ -z "$releaseManifest" ]; then + if [ -z "$sourceRepository" ] || [ -z "$sourceVersion" ]; then + echo "ERROR: $SCRIPT_ROOT is not a git repository, either --release-manifest or --source-repository and --source-version must be specified" + exit 1 + fi + else + if [ -n "$sourceRepository" ] || [ -n "$sourceVersion" ]; then + echo "ERROR: --release-manifest cannot be specified together with --source-repository and --source-version" + exit 1 fi - lowerI="$(echo "$1" | awk '{print tolower($0)}')" - case $lowerI in - --clean-while-building) - MSBUILD_ARGUMENTS+=( "-p:CleanWhileBuilding=true") - ;; - --online) - MSBUILD_ARGUMENTS+=( "-p:BuildWithOnlineSources=true") - ;; - --poison) - MSBUILD_ARGUMENTS+=( "-p:EnablePoison=true") - ;; - --run-smoke-test) - alternateTarget=true - runningSmokeTests=true - MSBUILD_ARGUMENTS+=( "-t:RunSmokeTest" ) - ;; - --use-mono-runtime) - MSBUILD_ARGUMENTS+=( "/p:SourceBuildUseMonoRuntime=true" ) - ;; - --with-packages) - CUSTOM_PACKAGES_DIR="$(cd -P "$2" && pwd)" - if [ ! -d "$CUSTOM_PACKAGES_DIR" ]; then - echo "Custom prviously built packages directory '$CUSTOM_PACKAGES_DIR' does not exist" - exit 1 - fi - shift - ;; - --with-sdk) - CUSTOM_SDK_DIR="$(cd -P "$2" && pwd)" - if [ ! -d "$CUSTOM_SDK_DIR" ]; then - echo "Custom SDK directory '$CUSTOM_SDK_DIR' does not exist" - exit 1 - fi - if [ ! -x "$CUSTOM_SDK_DIR/dotnet" ]; then - echo "Custom SDK '$CUSTOM_SDK_DIR/dotnet' does not exist or is not executable" - exit 1 - fi - shift - ;; - --) - shift - echo "Detected '--': passing remaining parameters '$@' as build.sh arguments." - break - ;; - '-?'|-h|--help) - usage - exit 0 - ;; - *) - echo "Unrecognized argument '$1'" - usage - exit 1 - ;; - esac - shift -done + get_property() { + local json_file_path="$1" + local property_name="$2" + grep -oP '(?<="'$property_name'": ")[^"]*' "$json_file_path" + } + + sourceRepository=$(get_property "$releaseManifest" sourceRepository) \ + || (echo "ERROR: Failed to find sourceRepository in $releaseManifest" && exit 1) + sourceVersion=$(get_property "$releaseManifest" sourceVersion) \ + || (echo "ERROR: Failed to find sourceVersion in $releaseManifest" && exit 1) + + if [ -z "$sourceRepository" ] || [ -z "$sourceVersion" ]; then + echo "ERROR: sourceRepository and sourceVersion must be specified in $releaseManifest" + exit 1 + fi + fi + + # We need to add "fake" .git/ files when not building from a git repository + mkdir -p "$GIT_DIR" + echo '[remote "origin"]' > "$GIT_DIR/config" + echo "url=\"$sourceRepository\"" >> "$GIT_DIR/config" + echo "$sourceVersion" > "$GIT_DIR/HEAD" +fi if [ "$CUSTOM_PACKAGES_DIR" != "" ]; then if [ "$runningSmokeTests" == "true" ]; then diff --git a/src/SourceBuild/content/prep.sh b/src/SourceBuild/content/prep.sh index df8f88cc0..472c348ae 100755 --- a/src/SourceBuild/content/prep.sh +++ b/src/SourceBuild/content/prep.sh @@ -1,199 +1,172 @@ #!/usr/bin/env bash + +### Usage: $0 +### +### Prepares the environment to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and +### installing the version of dotnet referenced in global.json +### +### Options: +### --no-artifacts Exclude the download of the previously source-built artifacts archive +### --no-bootstrap Don't replace portable packages in the download source-built artifacts +### --no-prebuilts Exclude the download of the prebuilts archive +### --no-sdk Exclude the download of the .NET SDK +### --runtime-source-feed URL of a remote server or a local directory, from which SDKs and +### runtimes can be downloaded +### --runtime-source-feed-key Key for accessing the above server, if necessary + set -euo pipefail IFS=$'\n\t' +source="${BASH_SOURCE[0]}" SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" -usage() { - echo "usage: $0" - echo "" - echo " Prepares the environment to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and" - echo " installing the version of dotnet referenced in global.json" - echo "options:" - echo " --no-artifacts Exclude the download of the previously source-built artifacts archive" - echo " --no-bootstrap Don't replace portable packages in the download source-built artifacts" - echo " --no-prebuilts Exclude the download of the prebuilts archive" - echo " --no-sdk Exclude the download of the .NET SDK" - echo " --source-repository Source Link repository URL, required when building from tarball" - echo " --source-version Source Link revision, required when building from tarball" - echo " --runtime-source-feed URL of a remote server or a local directory, from which SDKs and" - echo " runtimes can be downloaded" - echo " --runtime-source-feed-key Key for accessing the above server, if necessary" - echo "" +function print_help () { + sed -n '/^### /,/^$/p' "$source" | cut -b 5- } buildBootstrap=true downloadArtifacts=true downloadPrebuilts=true installDotnet=true -sourceUrl='' -sourceVersion='' runtime_source_feed='' # IBM requested these to support s390x scenarios runtime_source_feed_key='' # IBM requested these to support s390x scenarios positional_args=() while :; do - if [ $# -le 0 ]; then - break - fi - lowerI="$(echo "$1" | awk '{print tolower($0)}')" - case $lowerI in - "-?"|-h|--help) - usage - exit 0 - ;; - --no-bootstrap) - buildBootstrap=false - ;; - --no-artifacts) - downloadArtifacts=false - ;; - --no-prebuilts) - downloadPrebuilts=false - ;; - --no-sdk) - installDotnet=false - ;; - --source-repository) - sourceUrl="$2" - shift - ;; - --source-version) - sourceVersion="$2" - shift - ;; - --runtime-source-feed) - runtime_source_feed=$2 - shift - ;; - --runtime-source-feed-key) - runtime_source_feed_key=$2 - shift - ;; - *) - positional_args+=("$1") - ;; - esac + if [ $# -le 0 ]; then + break + fi + lowerI="$(echo "$1" | awk '{print tolower($0)}')" + case $lowerI in + "-?"|-h|--help) + print_help + exit 0 + ;; + --no-bootstrap) + buildBootstrap=false + ;; + --no-artifacts) + downloadArtifacts=false + ;; + --no-prebuilts) + downloadPrebuilts=false + ;; + --no-sdk) + installDotnet=false + ;; + --runtime-source-feed) + runtime_source_feed=$2 + shift + ;; + --runtime-source-feed-key) + runtime_source_feed_key=$2 + shift + ;; + *) + positional_args+=("$1") + ;; + esac - shift + shift done # Attempting to bootstrap without an SDK will fail. So either the --no-sdk flag must be passed # or a pre-existing .dotnet SDK directory must exist. if [ "$buildBootstrap" == true ] && [ "$installDotnet" == false ] && [ ! -d "$SCRIPT_ROOT/.dotnet" ]; then - echo " ERROR: --no-sdk requires --no-bootstrap or a pre-existing .dotnet SDK directory. Exiting..." - exit 1 + echo " ERROR: --no-sdk requires --no-bootstrap or a pre-existing .dotnet SDK directory. Exiting..." + exit 1 fi # Check to make sure curl exists to download the archive files if ! command -v curl &> /dev/null then - echo " ERROR: curl not found. Exiting..." - exit 1 -fi - -GIT_DIR="$SCRIPT_ROOT/.git" -if [ -f "$GIT_DIR/index" ]; then # We check for index because if outside of git, we create config and HEAD manually - if [ -n "$sourceUrl" ] || [ -n "$sourceVersion" ]; then - echo "ERROR: $SCRIPT_ROOT is a git repository, --source-repository and --source-version cannot be used." - exit 1 - fi -else - if [ -z "$sourceUrl" ] || [ -z "$sourceVersion" ]; then - echo "ERROR: $SCRIPT_ROOT is not a git repository, --source-repository and --source-version must be specified." - exit 1 - fi - - # We need to add "fake" .git/ files when not building from a git repository - mkdir -p "$GIT_DIR" - echo '[remote "origin"]' > "$GIT_DIR/config" - echo "url=\"$sourceUrl\"" >> "$GIT_DIR/config" - echo "$sourceVersion" > "$GIT_DIR/HEAD" + echo " ERROR: curl not found. Exiting..." + exit 1 fi # Check if Private.SourceBuilt artifacts archive exists artifactsBaseFileName="Private.SourceBuilt.Artifacts" packagesArchiveDir="$SCRIPT_ROOT/prereqs/packages/archive/" if [ "$downloadArtifacts" == true ] && [ -f ${packagesArchiveDir}${artifactsBaseFileName}.*.tar.gz ]; then - echo " Private.SourceBuilt.Artifacts.*.tar.gz exists...it will not be downloaded" - downloadArtifacts=false + echo " Private.SourceBuilt.Artifacts.*.tar.gz exists...it will not be downloaded" + downloadArtifacts=false fi # Check if Private.SourceBuilt prebuilts archive exists prebuiltsBaseFileName="Private.SourceBuilt.Prebuilts" if [ "$downloadPrebuilts" == true ] && [ -f ${packagesArchiveDir}${prebuiltsBaseFileName}.*.tar.gz ]; then - echo " Private.SourceBuilt.Prebuilts.*.tar.gz exists...it will not be downloaded" - downloadPrebuilts=false + echo " Private.SourceBuilt.Prebuilts.*.tar.gz exists...it will not be downloaded" + downloadPrebuilts=false fi # Check if dotnet is installed if [ "$installDotnet" == true ] && [ -d "$SCRIPT_ROOT/.dotnet" ]; then - echo " ./.dotnet SDK directory exists...it will not be installed" - installDotnet=false; + echo " ./.dotnet SDK directory exists...it will not be installed" + installDotnet=false; fi function DownloadArchive { - archiveType="$1" - isRequired="$2" + archiveType="$1" + isRequired="$2" - packageVersionsPath="$SCRIPT_ROOT/eng/Versions.props" - notFoundMessage="No source-built $archiveType found to download..." + packageVersionsPath="$SCRIPT_ROOT/eng/Versions.props" + notFoundMessage="No source-built $archiveType found to download..." - echo " Looking for source-built $archiveType to download..." - archiveVersionLine=$(grep -m 1 "" "$packageVersionsPath" || :) - versionPattern="(.*)" - if [[ $archiveVersionLine =~ $versionPattern ]]; then - archiveUrl="${BASH_REMATCH[1]}" - echo " Downloading source-built $archiveType from $archiveUrl..." - (cd "$packagesArchiveDir" && curl --retry 5 -O "$archiveUrl") - elif [ "$isRequired" == true ]; then - echo " ERROR: $notFoundMessage" - exit 1 - else - echo " $notFoundMessage" - fi + echo " Looking for source-built $archiveType to download..." + archiveVersionLine=$(grep -m 1 "" "$packageVersionsPath" || :) + versionPattern="(.*)" + if [[ $archiveVersionLine =~ $versionPattern ]]; then + archiveUrl="${BASH_REMATCH[1]}" + echo " Downloading source-built $archiveType from $archiveUrl..." + (cd "$packagesArchiveDir" && curl --retry 5 -O "$archiveUrl") + elif [ "$isRequired" == true ]; then + echo " ERROR: $notFoundMessage" + exit 1 + else + echo " $notFoundMessage" + fi } function BootstrapArtifacts { - DOTNET_SDK_PATH="$SCRIPT_ROOT/.dotnet" + DOTNET_SDK_PATH="$SCRIPT_ROOT/.dotnet" - # Create working directory for running bootstrap project - workingDir=$(mktemp -d) - echo " Building bootstrap previously source-built in $workingDir" + # Create working directory for running bootstrap project + workingDir=$(mktemp -d) + echo " Building bootstrap previously source-built in $workingDir" - # Copy bootstrap project to working dir - cp "$SCRIPT_ROOT/eng/bootstrap/buildBootstrapPreviouslySB.csproj" "$workingDir" + # Copy bootstrap project to working dir + cp "$SCRIPT_ROOT/eng/bootstrap/buildBootstrapPreviouslySB.csproj" "$workingDir" - # Copy NuGet.config from the installer repo to have the right feeds - cp "$SCRIPT_ROOT/src/installer/NuGet.config" "$workingDir" + # Copy NuGet.config from the installer repo to have the right feeds + cp "$SCRIPT_ROOT/src/installer/NuGet.config" "$workingDir" - # Get PackageVersions.props from existing prev-sb archive - echo " Retrieving PackageVersions.props from existing archive" - sourceBuiltArchive=$(find "$packagesArchiveDir" -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz') - if [ -f "$sourceBuiltArchive" ]; then - tar -xzf "$sourceBuiltArchive" -C "$workingDir" PackageVersions.props - fi + # Get PackageVersions.props from existing prev-sb archive + echo " Retrieving PackageVersions.props from existing archive" + sourceBuiltArchive=$(find "$packagesArchiveDir" -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz') + if [ -f "$sourceBuiltArchive" ]; then + tar -xzf "$sourceBuiltArchive" -C "$workingDir" PackageVersions.props + fi - # Run restore on project to initiate download of bootstrap packages - "$DOTNET_SDK_PATH/dotnet" restore "$workingDir/buildBootstrapPreviouslySB.csproj" /bl:artifacts/prep/bootstrap.binlog /fileLoggerParameters:LogFile=artifacts/prep/bootstrap.log /p:ArchiveDir="$packagesArchiveDir" /p:BootstrapOverrideVersionsProps="$SCRIPT_ROOT/eng/bootstrap/OverrideBootstrapVersions.props" + # Run restore on project to initiate download of bootstrap packages + "$DOTNET_SDK_PATH/dotnet" restore "$workingDir/buildBootstrapPreviouslySB.csproj" /bl:artifacts/prep/bootstrap.binlog /fileLoggerParameters:LogFile=artifacts/prep/bootstrap.log /p:ArchiveDir="$packagesArchiveDir" /p:BootstrapOverrideVersionsProps="$SCRIPT_ROOT/eng/bootstrap/OverrideBootstrapVersions.props" - # Remove working directory - rm -rf "$workingDir" + # Remove working directory + rm -rf "$workingDir" } # Check for the version of dotnet to install if [ "$installDotnet" == true ]; then - echo " Installing dotnet..." - (source ./eng/common/tools.sh && InitializeDotNetCli true) + echo " Installing dotnet..." + (source ./eng/common/tools.sh && InitializeDotNetCli true) fi # Read the eng/Versions.props to get the archives to download and download them if [ "$downloadArtifacts" == true ]; then - DownloadArchive Artifacts true - if [ "$buildBootstrap" == true ]; then - BootstrapArtifacts - fi + DownloadArchive Artifacts true + if [ "$buildBootstrap" == true ]; then + BootstrapArtifacts + fi fi if [ "$downloadPrebuilts" == true ]; then - DownloadArchive Prebuilts false + DownloadArchive Prebuilts false fi diff --git a/src/VirtualMonoRepo/README.template.md b/src/VirtualMonoRepo/README.template.md index 5c8bdd06e..650bd6da4 100644 --- a/src/VirtualMonoRepo/README.template.md +++ b/src/VirtualMonoRepo/README.template.md @@ -139,10 +139,14 @@ ln -s $HOME/.dotnet/dotnet /usr/bin/dotnet You can also utilize [GitHub Codespaces](https://github.com/features/codespaces) where you can find preset containers in this repository. -### Building outside of git +### Building from released sources -.NET uses git metadata so that it can link assemblies to their original source code when debugging (think "Step into.." functionality) and for that it needs information about the original place the code comes from. -When you're building source code only, taken outside of context of a git repository (e.g. you download it from the release page), you will need to specify the source repository to the `prep.sh` script via the `--source-repository` and `--source-version` arguments. This can be your fork of the repository and should match the origin where your SDK was built from. +You can also build from sources (and not from a context of a git repository), such as the ones you can acquire from a [dotnet/dotnet release](https://github.com/dotnet/dotnet/releases). +In this case, you need to provide additional information which includes the original repository and commit hash the code was built from so that the SDK can provide a better debugging experience (think the `Step into..` functionality). +Usually, this means the [dotnet/dotnet repository](https://github.com/dotnet/dotnet) together with the commit the release tag is connected to. + +In practice, this means that when calling the main build script, you need to provide additional arguments when building outside of a context of a git repository. +Alternatively, you can also provide a manifest file where this information can be read from. This file (`release.json`) can be found attached with the [dotnet/dotnet release](https://github.com/dotnet/dotnet/releases). ## List of components