Make the VMR build work outside of git repository (#15963)

eng/pipelines/templates/jobs/vmr-build.yml

@@ -106,9 +106,8 @@ jobs:
   - ${{ if parameters.buildFromArchive }}:
     - script: |
         set -ex
-        "${{ parameters.vmrPath }}/eng/pack-sources.sh" -o "$(Build.StagingDirectory)/dotnet-sources.tar.gz"
-        tar -xf dotnet-sources.tar.gz
-        mv "dotnet-$(Build.SourceVersion)" "$(sourcesPath)"
+        cp -r "${{ parameters.vmrPath }}" "$(sourcesPath)"
+        rm -rf "$(sourcesPath)/.git"
       displayName: Export VMR sources
       workingDirectory: $(Build.StagingDirectory)
 
@@ -182,7 +181,12 @@ jobs:
         echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /vmr/.dotnet"
       fi
 
-      docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh ${customPrepArgs}
+      if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then
+        customPrepArgs="$customPrepArgs --source-repository https://github.com/dotnet/dotnet"
+        customPrepArgs="$customPrepArgs --source-version $(git -C "${{ parameters.vmrPath }}" rev-parse HEAD)"
+      fi
+
+      docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh $customPrepArgs
     displayName: Prep the Build
 
   - script: |
@@ -207,7 +211,7 @@ jobs:
         customBuildArgs="$customBuildArgs --poison"
       fi
 
-      docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${customRunArgs} ${{ parameters.container }} ./build.sh --clean-while-building $(additionalBuildArgs) ${customBuildArgs}
+      docker run --rm -v "$(sourcesPath):/vmr" -w /vmr $customRunArgs ${{ parameters.container }} ./build.sh --clean-while-building $(additionalBuildArgs) $customBuildArgs
     displayName: Build
 
   - script: |

src/SourceBuild/content/build.sh

@@ -113,6 +113,11 @@ if [ -f "${packagesArchiveDir}archiveArtifacts.txt" ]; then
   fi
 fi
 
+if [ ! -d "$SCRIPT_ROOT/.git" ]; then
+  echo "ERROR: $SCRIPT_ROOT is not a git repository. Please run prep.sh add initialize Source Link metadata."
+  exit 1
+fi
+
 if [ -d "$CUSTOM_SDK_DIR" ]; then
   export SDK_VERSION=$("$CUSTOM_SDK_DIR/dotnet" --version)
   export CLI_ROOT="$CUSTOM_SDK_DIR"

src/SourceBuild/content/eng/pack-sources.sh

@@ -1,96 +0,0 @@
-#!/usr/bin/env bash
-
-### This script exports the sources of VMR (dotnet/dotnet) as an archive that can be used
-### to build the .NET SDK.
-### It expects you clone the dotnet/dotnet repo locally and check out the desired revision.
-###
-### USAGE:
-###   ./pack-sources.sh -o dotnet.tar.gz
-### Options:
-###   -o, --output PATH
-###       Optional. Path or dir where the archive is created.
-###       Defaults to artifacts/packages/dotnet-[SHA].tar.gz
-
-source="${BASH_SOURCE[0]}"
-
-# resolve $source until the file is no longer a symlink
-while [[ -h "$source" ]]; do
-  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-  source="$(readlink "$source")"
-  # if $source was a relative symlink, we need to resolve it relative to the path where the
-  # symlink file was located
-  [[ $source != /* ]] && source="$scriptroot/$source"
-done
-scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-
-function print_help () {
-    sed -n '/^### /,/^$/p' "$source" | cut -b 5-
-}
-
-GIT_ROOT=$(realpath "$scriptroot/../")
-
-output=''
-
-while [[ $# -gt 0 ]]; do
-  opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
-  case "$opt" in
-    -o|--output)
-      output=$2
-      shift
-      ;;
-    -h|--help)
-      print_help
-      exit 0
-      ;;
-    *)
-      fail "Invalid argument: $1"
-      usage
-      exit 1
-      ;;
-  esac
-
-  shift
-done
-
-revision=$(git -C "$GIT_ROOT" rev-parse HEAD)
-filename="dotnet-$revision"
-
-if [[ -z "$output" ]]; then
-  output="$GIT_ROOT/artifacts/packages/$filename.tar.gz"
-fi
-
-# If output is directory, use the default filename
-if [[ -d "$output" ]]; then
-  output="$output/$filename.tar.gz"
-fi
-
-set -e
-
-echo "Packing sources of $revision to $output.."
-mkdir -p "$(dirname "$output")"
-rm -f "$output"
-
-tmp="$GIT_ROOT/artifacts/tmp"
-mkdir -p "$tmp"
-config_file="$tmp/config"
-HEAD_file="$tmp/HEAD"
-
-start_time=$(date +%s)
-
-# We need to had `.git/HEAD` and `.git/config` to the archive as the build expects those
-echo $'[remote "origin"]\nurl="http://github.com/dotnet/dotnet"' > "$config_file"
-echo "$revision" > "$HEAD_file"
-
-git -C "$GIT_ROOT" archive   \
-  -o "$output"               \
-  --prefix "$filename/.git/" \
-  --add-file "$config_file"  \
-  --add-file "$HEAD_file"    \
-  --prefix "$filename/"      \
-  "$revision" "$GIT_ROOT"
-
-end_time=$(date +%s)
-elapsed=$(( end_time - start_time ))
-
-duration=$(date -u +%H:%M:%S "-d@$elapsed")
-echo "Archive created in $duration"

src/SourceBuild/content/prep.sh

@@ -10,10 +10,12 @@ usage() {
     echo "  Prepares the environment to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and"
     echo "  installing the version of dotnet referenced in global.json"
     echo "options:"
-    echo "  --no-artifacts              Exclude the download of the previously source-built artifacts archive."
+    echo "  --no-artifacts              Exclude the download of the previously source-built artifacts archive"
     echo "  --no-bootstrap              Don't replace portable packages in the download source-built artifacts"
-    echo "  --no-prebuilts              Exclude the download of the prebuilts archive."
-    echo "  --no-sdk                    Exclude the download of the .NET SDK."
+    echo "  --no-prebuilts              Exclude the download of the prebuilts archive"
+    echo "  --no-sdk                    Exclude the download of the .NET SDK"
+    echo "  --source-repository <url>   Source Link repository URL, required when building from tarball"
+    echo "  --source-version <sha>      Source Link revision, required when building from tarball"
     echo "  --runtime-source-feed       URL of a remote server or a local directory, from which SDKs and"
     echo "                              runtimes can be downloaded"
     echo "  --runtime-source-feed-key   Key for accessing the above server, if necessary"
@@ -24,8 +26,10 @@ buildBootstrap=true
 downloadArtifacts=true
 downloadPrebuilts=true
 installDotnet=true
-runtime_source_feed=''
-runtime_source_feed_key=''
+sourceUrl=''
+sourceVersion=''
+runtime_source_feed='' # IBM requested these to support s390x scenarios
+runtime_source_feed_key='' # IBM requested these to support s390x scenarios
 positional_args=()
 while :; do
     if [ $# -le 0 ]; then
@@ -49,6 +53,14 @@ while :; do
         --no-sdk)
             installDotnet=false
             ;;
+        --source-repository)
+            sourceUrl="$2"
+            shift
+            ;;
+        --source-version)
+            sourceVersion="$2"
+            shift
+            ;;
         --runtime-source-feed)
             runtime_source_feed=$2
             shift
@@ -67,57 +79,75 @@ done
 
 # Attempting to bootstrap without an SDK will fail. So either the --no-sdk flag must be passed
 # or a pre-existing .dotnet SDK directory must exist.
-if [[ "$buildBootstrap" == "true" && "$installDotnet" == "false" && ! -d $SCRIPT_ROOT/.dotnet ]]; then
+if [ "$buildBootstrap" == true ] && [ "$installDotnet" == false ] && [ ! -d "$SCRIPT_ROOT/.dotnet" ]; then
     echo "  ERROR: --no-sdk requires --no-bootstrap or a pre-existing .dotnet SDK directory.  Exiting..."
-    exit -1
+    exit 1
 fi
 
 # Check to make sure curl exists to download the archive files
 if ! command -v curl &> /dev/null
 then
     echo "  ERROR: curl not found.  Exiting..."
-    exit -1
+    exit 1
+fi
+
+GIT_DIR="$SCRIPT_ROOT/.git"
+if [ -f "$GIT_DIR/index" ]; then # We check for index because if outside of git, we create config and HEAD manually
+    if [ -n "$sourceUrl" ] || [ -n "$sourceVersion" ]; then
+        echo "ERROR: $SCRIPT_ROOT is a git repository, --source-repository and --source-version cannot be used."
+        exit 1
+    fi
+else
+    if [ -z "$sourceUrl" ] || [ -z "$sourceVersion" ]; then
+      echo "ERROR: $SCRIPT_ROOT is not a git repository, --source-repository and --source-version must be specified."
+        exit 1
+    fi
+
+    # We need to add "fake" .git/ files when not building from a git repository
+    mkdir -p "$GIT_DIR"
+    echo '[remote "origin"]' > "$GIT_DIR/config"
+    echo "url=\"$sourceUrl\"" >> "$GIT_DIR/config"
+    echo "$sourceVersion" > "$GIT_DIR/HEAD"
 fi
 
 # Check if Private.SourceBuilt artifacts archive exists
 artifactsBaseFileName="Private.SourceBuilt.Artifacts"
 packagesArchiveDir="$SCRIPT_ROOT/prereqs/packages/archive/"
-if [ "$downloadArtifacts" == "true" ] && [ -f ${packagesArchiveDir}${artifactsBaseFileName}.*.tar.gz ]; then
+if [ "$downloadArtifacts" == true ] && [ -f ${packagesArchiveDir}${artifactsBaseFileName}.*.tar.gz ]; then
     echo "  Private.SourceBuilt.Artifacts.*.tar.gz exists...it will not be downloaded"
     downloadArtifacts=false
 fi
 
 # Check if Private.SourceBuilt prebuilts archive exists
 prebuiltsBaseFileName="Private.SourceBuilt.Prebuilts"
-if [ "$downloadPrebuilts" == "true" ] && [ -f ${packagesArchiveDir}${prebuiltsBaseFileName}.*.tar.gz ]; then
+if [ "$downloadPrebuilts" == true ] && [ -f ${packagesArchiveDir}${prebuiltsBaseFileName}.*.tar.gz ]; then
     echo "  Private.SourceBuilt.Prebuilts.*.tar.gz exists...it will not be downloaded"
     downloadPrebuilts=false
 fi
 
 # Check if dotnet is installed
-if [[ "$installDotnet" == "true" && -d $SCRIPT_ROOT/.dotnet ]]; then
+if [ "$installDotnet" == true ] && [ -d "$SCRIPT_ROOT/.dotnet" ]; then
     echo "  ./.dotnet SDK directory exists...it will not be installed"
     installDotnet=false;
 fi
 
 function DownloadArchive {
     archiveType="$1"
-    baseFileName="$2"
-    isRequired="$3"
+    isRequired="$2"
 
     packageVersionsPath="$SCRIPT_ROOT/eng/Versions.props"
     notFoundMessage="No source-built $archiveType found to download..."
 
     echo "  Looking for source-built $archiveType to download..."
-    archiveVersionLine=`grep -m 1 "<PrivateSourceBuilt${archiveType}Url>" "$packageVersionsPath" || :`
+    archiveVersionLine=$(grep -m 1 "<PrivateSourceBuilt${archiveType}Url>" "$packageVersionsPath" || :)
     versionPattern="<PrivateSourceBuilt${archiveType}Url>(.*)</PrivateSourceBuilt${archiveType}Url>"
     if [[ $archiveVersionLine =~ $versionPattern ]]; then
         archiveUrl="${BASH_REMATCH[1]}"
         echo "  Downloading source-built $archiveType from $archiveUrl..."
-        (cd $packagesArchiveDir && curl --retry 5 -O $archiveUrl)
-    elif [ "$isRequired" == "true" ]; then
+        (cd "$packagesArchiveDir" && curl --retry 5 -O "$archiveUrl")
+    elif [ "$isRequired" == true ]; then
       echo "  ERROR: $notFoundMessage"
-      exit -1
+      exit 1
     else
       echo "  $notFoundMessage"
     fi
@@ -131,39 +161,39 @@ function BootstrapArtifacts {
     echo "  Building bootstrap previously source-built in $workingDir"
 
     # Copy bootstrap project to working dir
-    cp $SCRIPT_ROOT/eng/bootstrap/buildBootstrapPreviouslySB.csproj $workingDir
+    cp "$SCRIPT_ROOT/eng/bootstrap/buildBootstrapPreviouslySB.csproj" "$workingDir"
 
     # Copy NuGet.config from the installer repo to have the right feeds
-    cp $SCRIPT_ROOT/src/installer/NuGet.config $workingDir
+    cp "$SCRIPT_ROOT/src/installer/NuGet.config" "$workingDir"
 
     # Get PackageVersions.props from existing prev-sb archive
     echo "  Retrieving PackageVersions.props from existing archive"
-    sourceBuiltArchive=`find $packagesArchiveDir -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'`
+    sourceBuiltArchive=$(find "$packagesArchiveDir" -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz')
     if [ -f "$sourceBuiltArchive" ]; then
-        tar -xzf "$sourceBuiltArchive" -C $workingDir PackageVersions.props
+        tar -xzf "$sourceBuiltArchive" -C "$workingDir" PackageVersions.props
     fi
 
     # Run restore on project to initiate download of bootstrap packages
-    $DOTNET_SDK_PATH/dotnet restore $workingDir/buildBootstrapPreviouslySB.csproj /bl:artifacts/prep/bootstrap.binlog /fileLoggerParameters:LogFile=artifacts/prep/bootstrap.log /p:ArchiveDir="$packagesArchiveDir" /p:BootstrapOverrideVersionsProps="$SCRIPT_ROOT/eng/bootstrap/OverrideBootstrapVersions.props"
+    "$DOTNET_SDK_PATH/dotnet" restore "$workingDir/buildBootstrapPreviouslySB.csproj" /bl:artifacts/prep/bootstrap.binlog /fileLoggerParameters:LogFile=artifacts/prep/bootstrap.log /p:ArchiveDir="$packagesArchiveDir" /p:BootstrapOverrideVersionsProps="$SCRIPT_ROOT/eng/bootstrap/OverrideBootstrapVersions.props"
 
     # Remove working directory
-    rm -rf $workingDir
+    rm -rf "$workingDir"
 }
 
 # Check for the version of dotnet to install
-if [ "$installDotnet" == "true" ]; then
+if [ "$installDotnet" == true ]; then
     echo "  Installing dotnet..."
     (source ./eng/common/tools.sh && InitializeDotNetCli true)
 fi
 
 # Read the eng/Versions.props to get the archives to download and download them
-if [ "$downloadArtifacts" == "true" ]; then
-    DownloadArchive "Artifacts" $artifactsBaseFileName "true"
-    if [ "$buildBootstrap" == "true" ]; then
+if [ "$downloadArtifacts" == true ]; then
+    DownloadArchive Artifacts true
+    if [ "$buildBootstrap" == true ]; then
         BootstrapArtifacts
     fi
 fi
 
-if [ "$downloadPrebuilts" == "true" ]; then
-    DownloadArchive "Prebuilts" $prebuiltsBaseFileName "false"
+if [ "$downloadPrebuilts" == true ]; then
+    DownloadArchive Prebuilts false
 fi

src/VirtualMonoRepo/README.template.md

@@ -139,11 +139,10 @@ ln -s $HOME/.dotnet/dotnet /usr/bin/dotnet
 
 You can also utilize [GitHub Codespaces](https://github.com/features/codespaces) where you can find preset containers in this repository.
 
-### Exporting a source archive
+### Building outside of git
 
-In case you'd like to export a more lightweight archive of sources that can be built outside of this git repository, a simple copy of the working tree won't do.
-The build is using some git metadata (information from the `.git` directory) that are needed to be kept with the sources.
-To export a `tar.gz` archive of the sources, you need to use the `eng/pack-sources.sh` script from within a clone of the VMR checked out at the revision that you're interested in.
+.NET uses git metadata so that it can link assemblies to their original source code when debugging (think "Step into.." functionality) and for that it needs information about the original place the code comes from.
+When you're building source code only, taken outside of context of a git repository (e.g. you download it from the release page), you will need to specify the source repository to the `prep.sh` script via the `--source-repository` and `--source-version` arguments. This can be your fork of the repository and should match the origin where your SDK was built from.
 
 ## List of components