[ArPow] Add offline tarball build infra and CI (#11403)

* Add offline build to CI

* Add sourcelink patches to unblock offline aspnetcore build

.vsts-ci.yml

@@ -344,6 +344,11 @@ stages:
 
   - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml
 
+  - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
+    parameters:
+      dependsOn: Source_Build_Create_Tarball
+      condition: eq(dependencies.Source_Build_Create_Tarball.outputs['Tarball_Build_Check._includeTarballBuild'], 'true')
+
   # https://github.com/dotnet/core-sdk/issues/248
   # - template: /eng/build.yml
   #   parameters:

eng/Versions.props

@@ -159,7 +159,7 @@
     <MicrosoftBuildFrameworkVersion>15.7.179</MicrosoftBuildFrameworkVersion>
     <MicrosoftBuildUtilitiesCoreVersion>15.7.179</MicrosoftBuildUtilitiesCoreVersion>
     <PrivateSourceBuiltArtifactsPackageVersion>0.1.0-6.0.100-bootstrap.11</PrivateSourceBuiltArtifactsPackageVersion>
-    <PrivateSourceBuiltPrebuiltsPackageVersion>0.1.0-6.0.100-1</PrivateSourceBuiltPrebuiltsPackageVersion>
+    <PrivateSourceBuiltPrebuiltsPackageVersion>0.1.0-6.0.100-2</PrivateSourceBuiltPrebuiltsPackageVersion>
   </PropertyGroup>
   <!-- Workload manifest package versions -->
   <PropertyGroup>

eng/source-build-tarball-build.yml

@@ -9,32 +9,6 @@ resources:
 stages:
 - stage: build
   jobs:
-  - job: build
+  - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
-    displayName: Build
+    parameters:
-    pool:
+      tarballResourceId: installer-build-resource
-      vmImage: ubuntu-20.04
-    container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-32-20210222183556-031e7d2
-    workspace:
-      clean: all
-
-    variables:
-      _TarballDir: $(Build.StagingDirectory)/tarball
-    steps:
-    - checkout: none
-
-    - download: installer-build-resource
-      patterns: 'BlobArtifacts/dotnet-sdk-source*.tar.gz'
-
-    - script: |
-        set -x
-
-        mkdir -p "$(_TarballDir)"
-        tarballFilePath="$(PIPELINE.WORKSPACE)/installer-build-resource/BlobArtifacts/dotnet-sdk-source*.tar.gz"
-        eval tar -ozxf "$tarballFilePath" -C "$(_TarballDir)"
-
-        df -h
-      displayName: Extract Tarball
-
-    - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-run-tarball-build.yml
-      parameters:
-        sourceFolder: $(_TarballDir)

src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml

@@ -2,10 +2,12 @@
 
 jobs:
 - job: Source_Build_Create_Tarball
+  container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-33-20210222183538-031e7d2
   displayName: Source-Build Create Tarball
   pool:
     vmImage: ubuntu-20.04
-  container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-32-20210222183556-031e7d2
+  variables:
+    _BuildConfig: Release
   workspace:
     clean: all
 
@@ -14,40 +16,31 @@ jobs:
       set -x
       df -h
 
-      buildConfig=Release
-      # Check if AzDO substitutes in a build config from a variable, and use it if so.
-      if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
-        buildConfig='$(_BuildConfig)'
-      fi
-
       officialBuildArgs=
       if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
-        officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+        officialBuildArgs='/p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
       fi
 
       ./build.sh \
         --ci \
-        --configuration $buildConfig \
+        --configuration $(_BuildConfig) \
         --publish \
         -bl \
         $officialBuildArgs \
+        /p:DotNetPublishUsingPipelines=true \
         /p:ArcadeBuildTarball=true
     displayName: Create Tarball
 
   - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
-    parameters:
-      artifactQualifier: TarballCreation
 
   - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
-    # Only build the tarball if the PR touches source-build source
+    # Check if the tarball should be built now so the result can be used in the build tarball job condition.
+    # This prevents allocation of additional agents if the tarball build legs should be skipped.
+    # Only build the tarball if the PR touches source-build source.
     - script: |
         if curl "https://api.github.com/repos/dotnet/installer/pulls/$(System.PullRequest.PullRequestNumber)/files" | grep '"filename": "src/SourceBuild/*'
         then
-          echo "##vso[task.setvariable variable=_includeTarballBuild]true"
+          echo "##vso[task.setvariable variable=_includeTarballBuild;isoutput=true]true"
         fi
-      displayName: Include Tarball Build Check
+      displayName: Tarball Build Check
-
+      name: Tarball_Build_Check
-    - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-run-tarball-build.yml
-      parameters:
-        sourceFolder: $(Build.SourcesDirectory)/artifacts/tarball
-        condition: eq(variables['_includeTarballBuild'], 'true')

src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml

@@ -0,0 +1,94 @@
+# Builds a source-build tarball
+
+parameters:
+  # Custom condition to apply to the job
+  condition: true
+
+  # Dependent jobs that must be completed before this job will run
+  dependsOn:
+
+  # The resource id of the tarball to download and build
+  tarballResourceId: current
+
+jobs:
+- job: Source_Build_Run_Tarball_Build
+  condition: ${{ parameters.condition }}
+  displayName: Source-Build Build Tarball
+  dependsOn: ${{ parameters.dependsOn }}
+  pool:
+    vmImage: ubuntu-20.04
+  strategy:
+    matrix:
+      Fedora33-Online:
+        _runOnline: true
+        _Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-33-20210222183538-031e7d2
+      Fedora33-Offline:
+        _runOnline: false
+        _Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-33-20210222183538-031e7d2
+  timeoutInMinutes: 120
+  variables:
+    _TarballDir: $(Build.StagingDirectory)/tarball
+  workspace:
+    clean: all
+
+  steps:
+  - checkout: none
+
+  - download: ${{ parameters.tarballResourceId }}
+    artifact: BlobArtifacts
+    patterns: '**/dotnet-sdk-source*.tar.gz'
+    displayName: Download Tarball
+
+  - script: |
+      set -x
+
+      resourceIdPathSegment=
+      if [ '${{ parameters.tarballResourceId }}' != 'current' ]; then
+        resourceIdPathSegment='${{ parameters.tarballResourceId }}/'
+      fi
+
+      mkdir -p "$(_TarballDir)"
+      tarballFilePath="$(PIPELINE.WORKSPACE)/${resourceIdPathSegment}BlobArtifacts/dotnet-sdk-source*.tar.gz"
+      eval tar -ozxf "$tarballFilePath" -C "$(_TarballDir)"
+    displayName: Extract Tarball
+
+  - script: |
+      set -x
+
+      docker run --rm -v $(_TarballDir):/tarball -w /tarball $(_Container) ./prep.sh
+    displayName: Prep Tarball
+
+  - script: |
+      set -x
+      df -h
+
+      networkArgs=
+      customBuildArgs=
+      if [ '$(_runOnline)' = 'true' ]; then
+        customBuildArgs='--online'
+      else
+        networkArgs='--network none'
+      fi
+
+      docker run --rm -v $(_TarballDir):/tarball -w /tarball ${networkArgs} $(_Container) ./build.sh ${customBuildArgs} -- /p:CleanWhileBuilding=true
+    displayName: Build Tarball
+
+  - task: CopyFiles@2
+    displayName: Gather Source Build Artifacts
+    inputs:
+      SourceFolder: $(_TarballDir)
+      Contents: |
+        artifacts/**/Private.SourceBuilt.Artifacts*.tar.gz
+        artifacts/prebuilt-report/Private.SourceBuilt.Prebuilts.*.tar.gz
+      TargetFolder: '$(Build.StagingDirectory)/artifacts'
+      CleanTargetFolder: true
+
+  - task: PublishPipelineArtifact@1
+    displayName: Publish Source Build Artifacts
+    inputs:
+      targetPath: '$(Build.StagingDirectory)/artifacts'
+      artifactName: $(Agent.JobName)_Artifacts_Attempt$(System.JobAttempt)
+
+  - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
+    parameters:
+      sourceFolder: $(_TarballDir)

src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml

@@ -4,12 +4,6 @@ parameters:
   # Folder to scan for source-build logs from
   sourceFolder: $(Build.SourcesDirectory)
 
-  # The unique qualifier to append to the artifactName
-  artifactQualifier:
-
-  # Custom condition to apply to each step
-  condition: true
-
 steps:
 - task: CopyFiles@2
   displayName: Prepare BuildLogs staging directory
@@ -22,12 +16,12 @@ steps:
     TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
     CleanTargetFolder: true
   continueOnError: true
-  condition: and(${{ parameters.condition }}, succeededOrFailed())
+  condition: succeededOrFailed()
 
 - task: PublishPipelineArtifact@1
   displayName: Publish BuildLogs
   inputs:
     targetPath: '$(Build.StagingDirectory)/BuildLogs'
-    artifactName: BuildLogs_SourceBuild_${{ parameters.artifactQualifier }}_Attempt$(System.JobAttempt)
+    artifactName: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
   continueOnError: true
-  condition: and(${{ parameters.condition }}, succeededOrFailed())
+  condition: succeededOrFailed()

src/SourceBuild/Arcade/eng/common/templates/steps/source-build-run-tarball-build.yml

@@ -1,51 +0,0 @@
-# Builds the specified source-build tarball
-
-parameters:
-  # Folder containing the tarball source to build
-  sourceFolder: $(Build.SourcesDirectory)
-
-  # Custom condition to apply to each step
-  condition: true
-
-steps:
-- script: |
-    set -x
-
-    ./prep.sh
-  workingDirectory: ${{ parameters.sourceFolder }}
-  displayName: Prep Tarball
-  condition: and(${{ parameters.condition }}, succeeded())
-
-- script: |
-    set -x
-
-    ./build.sh \
-      --online \
-      -- /p:CleanWhileBuilding=true
-  workingDirectory: ${{ parameters.sourceFolder }}
-  displayName: Build Tarball
-  condition: and(${{ parameters.condition }}, succeeded())
-
-- task: CopyFiles@2
-  displayName: Gather Source Build Artifacts
-  inputs:
-    SourceFolder: '${{ parameters.sourceFolder }}'
-    Contents: |
-      artifacts/**/Private.SourceBuilt.Artifacts*.tar.gz
-      artifacts/prebuilt-report/Private.SourceBuilt.Prebuilts.*.tar.gz
-    TargetFolder: '$(Build.StagingDirectory)/artifacts'
-    CleanTargetFolder: true
-  condition: and(${{ parameters.condition }}, succeeded())
-
-- task: PublishPipelineArtifact@1
-  displayName: Publish Source Build Artifacts
-  inputs:
-    targetPath: '$(Build.StagingDirectory)/artifacts'
-    artifactName: Artifacts_SourceBuild_TarballBuild_Attempt$(System.JobAttempt)
-  condition: and(${{ parameters.condition }}, succeeded())
-
-- template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
-  parameters:
-    sourceFolder: ${{ parameters.sourceFolder }}
-    artifactQualifier: TarballBuild
-    condition: and(${{ parameters.condition }}, succeededOrFailed())

src/SourceBuild/tarball/content/eng/Versions.props

@@ -23,6 +23,6 @@
   <PropertyGroup>
     <PrivateSourceBuiltArtifactsPackageVersion>0.1.0-6.0.100-bootstrap.11</PrivateSourceBuiltArtifactsPackageVersion>
     <PrivateSourceBuiltPrebuiltsPackageVersionPrefix>0.1.0-6.0.100-</PrivateSourceBuiltPrebuiltsPackageVersionPrefix>
-    <PrivateSourceBuiltPrebuiltsPackageVersionSuffix>1</PrivateSourceBuiltPrebuiltsPackageVersionSuffix>
+    <PrivateSourceBuiltPrebuiltsPackageVersionSuffix>2</PrivateSourceBuiltPrebuiltsPackageVersionSuffix>
   </PropertyGroup>
 </Project>

src/SourceBuild/tarball/content/patches/aspnetcore/0004-Revert-sourcelink-submodule-workaround.patch

@@ -0,0 +1,46 @@
+From 9b529c2f911e9a7ad25586d0d4a8db8bc5a7b2e5 Mon Sep 17 00:00:00 2001
+From: Michael Simons <msimons@microsoft.com>
+Date: Thu, 5 Aug 2021 22:44:48 +0000
+Subject: [PATCH] Revert sourcelink submodule workaround
+
+aspnetcore will fail the offline tarball build because of the sourcelink workaround being removed here.
+To fix this, a sourcelink patch was added to address the issue.
+---
+ eng/SourceBuild.props | 14 --------------
+ 1 file changed, 14 deletions(-)
+
+diff --git a/eng/SourceBuild.props b/eng/SourceBuild.props
+index 3dbfe2ce7e..403b91f976 100644
+--- a/eng/SourceBuild.props
++++ b/eng/SourceBuild.props
+@@ -2,7 +2,6 @@
+   <PropertyGroup>
+     <GitHubRepositoryName>aspnetcore</GitHubRepositoryName>
+     <SourceBuildManagedOnly>true</SourceBuildManagedOnly>
+-    <CloneSubmodulesToInnerSourceBuildRepo>false</CloneSubmodulesToInnerSourceBuildRepo>
+   </PropertyGroup>
+ 
+   <Target Name="PrepareGlobalJsonForSourceBuild"
+@@ -13,19 +12,6 @@
+       WorkingDirectory="$(InnerSourceBuildRepoRoot)" />
+   </Target>
+ 
+-  <!--
+-    Init submodules - temporarary workaround for https://github.com/dotnet/sourcelink/pull/653
+-  -->
+-  <Target Name="InitSubmodules"
+-          DependsOnTargets="PrepareInnerSourceBuildRepoRoot"
+-          BeforeTargets="RunInnerSourceBuildCommand">
+-
+-    <Exec
+-      Command="git submodule update --init --recursive"
+-      WorkingDirectory="$(InnerSourceBuildRepoRoot)"
+-      EnvironmentVariables="@(InnerBuildEnv)" />
+-  </Target>
+-
+   <!--
+     Remove inner source .globalconfig file as both the inner and outer config files get loaded and cause a conflict.
+     Leaving the inner will cause all conflicting settings to be ignored.
+-- 
+2.29.2
+

src/SourceBuild/tarball/content/patches/sourcelink/0003-Allow-using-.git-directory-instead-of-gitdir-redirec.patch

@@ -0,0 +1,36 @@
+From cfb26098630fe3cd9648ebcbbe9ac59278b63242 Mon Sep 17 00:00:00 2001
+From: Michael Simons <msimons@microsoft.com>
+Date: Thu, 5 Aug 2021 19:51:19 +0000
+Subject: [PATCH] Allow using .git directory instead of gitdir redirect in
+ submodules
+
+---
+ .../GitDataReader/GitRepository.cs                   | 12 ++++++++++++
+ 1 file changed, 12 insertions(+)
+
+diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs
+index 9f1648f..c079197 100644
+--- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs
++++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs
+@@ -209,6 +209,18 @@ public static GitRepository OpenRepository(GitRepositoryLocation location, GitEn
+         /// <returns>Null if the HEAD tip reference can't be resolved.</returns>
+         internal string? ReadSubmoduleHeadCommitSha(string submoduleWorkingDirectoryFullPath)
+         {
++            // Submodules don't usually have their own .git directories but this is still legal.
++            // This can occur with older versions of Git or other tools, or when a user clones one
++            // repo into another's source tree (but it was not yet registered as a submodule).
++            // See https://git-scm.com/docs/gitsubmodules#_forms for more details.
++            // Handle this case first since the other case throws.
++            var dotGitPath = Path.Combine(submoduleWorkingDirectoryFullPath, GitDirName);
++            if (IsGitDirectory(dotGitPath, out var directSubmoduleGitDirectory))
++            {
++                var submoduleGitDirResolver = new GitReferenceResolver(directSubmoduleGitDirectory, directSubmoduleGitDirectory);
++                return submoduleGitDirResolver.ResolveHeadReference();
++            }
++
+             var gitDirectory = ReadDotGitFile(Path.Combine(submoduleWorkingDirectoryFullPath, GitDirName));
+             if (!IsGitDirectory(gitDirectory, out var commonDirectory))
+             {
+-- 
+2.29.2
+

src/SourceBuild/tarball/content/prep.sh

@@ -70,13 +70,13 @@ fi
 while read -r line; do
     if [[ $line == *"Private.SourceBuilt.Artifacts"* ]]; then
         if [ "$downloadArtifacts" == "true" ]; then
-            echo "  Downloading source-built artifacts..."
+            echo "  Downloading source-built artifacts from $line..."
             (cd $SCRIPT_ROOT/packages/archive/ && curl -O $line)
         fi
     fi
     if [[ $line == *"Private.SourceBuilt.Prebuilts"* ]]; then
         if [ "$downloadPrebuilts" == "true" ]; then
-            echo "  Downloading source-built prebuilts..."
+            echo "  Downloading source-built prebuilts from $line..."
             (cd $SCRIPT_ROOT/packages/archive/ && curl -O $line)
         fi
     fi