Add bootstrap build to CI (#12651)

src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml

@@ -10,87 +10,103 @@ parameters:
   # The resource id of the tarball to download and build
   tarballResourceId: current
 
+  # The following parameters aren't expected to be passed in rather they are used for encapsulation
+  # -----------------------------------------------------------------------------------------------
+  fedora33Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-33-20210222183538-031e7d2
+  poolInternal:
+    name: NetCore1ESPool-Svc-Internal
+    demands: ImageOverride -equals Build.Ubuntu.1604.Amd64
+  poolPublic:
+    name: NetCore1ESPool-Svc-Public
+    demands: ImageOverride -equals Build.Ubuntu.1604.Amd64.Open
+  tarballDir: $(Build.StagingDirectory)/tarball
+
 jobs:
-- job: Source_Build_Run_Tarball_Build
+- job: Build_Tarball
   condition: ${{ parameters.condition }}
-  displayName: Source-Build Build Tarball
+  displayName: Build Tarball
   dependsOn: ${{ parameters.dependsOn }}
   pool:
     ${{ if eq(variables['System.TeamProject'], 'public') }}:
-      name: NetCore1ESPool-Svc-Public
+      ${{ parameters.poolPublic }}
-      demands: ImageOverride -equals Build.Ubuntu.1604.Amd64.Open
     ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-      name: NetCore1ESPool-Svc-Internal
+      ${{ parameters.poolInternal }}
-      demands: ImageOverride -equals Build.Ubuntu.1604.Amd64
   strategy:
     matrix:
       Fedora33-Online:
-        _runOnline: true
-        _Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-33-20210222183538-031e7d2
-      Fedora33-Offline:
-        _runOnline: false
-        _Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-33-20210222183538-031e7d2
-  timeoutInMinutes: 240
-  variables:
-    _TarballDir: $(Build.StagingDirectory)/tarball
-    _BuildConfig: Release
         _BuildArch: x64
+        _Container: ${{ parameters.fedora33Container }}
+        _RunOnline: true
+      Fedora33-Offline:
+        _BuildArch: x64
+        _Container: ${{ parameters.fedora33Container }}
+        _RunOnline: false
+  timeoutInMinutes: 240
   workspace:
     clean: all
 
   steps:
   - checkout: none
 
-  - download: ${{ parameters.tarballResourceId }}
+  - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml
-    artifact: BlobArtifacts
-    patterns: '**/dotnet-sdk-source*.tar.gz'
-    displayName: Download Tarball
-
-  - script: |
-      set -x
-
-      resourceIdPathSegment=
-      if [ '${{ parameters.tarballResourceId }}' != 'current' ]; then
-        resourceIdPathSegment='${{ parameters.tarballResourceId }}/'
-      fi
-
-      mkdir -p "$(_TarballDir)"
-      tarballFilePath="$(PIPELINE.WORKSPACE)/${resourceIdPathSegment}BlobArtifacts/dotnet-sdk-source*.tar.gz"
-      eval tar -ozxf "$tarballFilePath" -C "$(_TarballDir)"
-      eval rm -f "$tarballFilePath"
-    displayName: Extract Tarball
-
-  - script: |
-      set -x
-
-      docker run --rm -v $(_TarballDir):/tarball -w /tarball $(_Container) ./prep.sh
-    displayName: Prep Tarball
-
-  - script: |
-      set -x
-      df -h
-
-      networkArgs=
-      customBuildArgs=
-      if [ '$(_runOnline)' = 'true' ]; then
-        customBuildArgs='--online'
-      else
-        networkArgs='--network none'
-      fi
-
-      docker run --rm -v $(_TarballDir):/tarball -w /tarball ${networkArgs} $(_Container) ./build.sh ${customBuildArgs} -- /p:CleanWhileBuilding=true
-    displayName: Build Tarball
-
-  - publish: '$(_TarballDir)/artifacts/$(_BuildArch)/$(_BuildConfig)/'
-    artifact: $(Agent.JobName)_Artifacts_Attempt$(System.JobAttempt)
-    displayName: Publish Source Build Artifacts
-
-  - script: |
-      set -x
-
-      docker run --rm -v $(_TarballDir):/tarball -w /tarball $(_Container) ./build.sh --run-smoke-test
-    displayName: Run Tests
-
-  - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
     parameters:
-      sourceFolder: $(_TarballDir)
+      buildArch: $(_BuildArch)
+      container: $(_Container)
+      prepScript: |
+        set -x
+
+        docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball $(_Container) ./prep.sh
+      runOnline: $(_RunOnline)
+      tarballDir: ${{ parameters.tarballDir }}
+      tarballResourceId: ${{ parameters.tarballResourceId }}
+
+- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+  - job: Rebuild_Tarball
+    displayName: Rebuild Tarball With Previous
+    dependsOn: Build_Tarball
+    pool:
+      ${{ if eq(variables['System.TeamProject'], 'public') }}:
+        ${{ parameters.poolPublic }}
+      ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+        ${{ parameters.poolInternal }}
+    strategy:
+      matrix:
+        Fedora33-Offline:
+          _PreviousSourceBuildArtifact: Build Tarball Fedora33-Offline_Artifacts
+          _BuildArch: x64
+          _Container: ${{ parameters.fedora33Container }}
+          _RunOnline: false
+    timeoutInMinutes: 180
+    workspace:
+      clean: all
+
+    steps:
+    - checkout: none
+
+    - download: current
+      artifact: $(_PreviousSourceBuildArtifact)
+      patterns: '*.tar.gz'
+      displayName: Download Previous Source Build Artifacts
+
+    - task: CopyFiles@2
+      displayName: Copy Previous Source Build Artifacts
+      inputs:
+        SourceFolder: $(PIPELINE.WORKSPACE)/$(_PreviousSourceBuildArtifact)
+        Contents: '*.tar.gz'
+        TargetFolder: ${{ parameters.tarballDir }}/packages/archive/
+
+    - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml
+      parameters:
+        additionalBuildArgs: --with-sdk /tarball/.dotnet
+        buildArch: $(_BuildArch)
+        container: $(_Container)
+        prepScript: |
+          set -x
+
+          mkdir ${{ parameters.tarballDir }}/.dotnet
+          tarballFilePath="${{ parameters.tarballDir }}/packages/archive/dotnet-sdk-*.tar.gz"
+          eval tar -ozxf "$tarballFilePath" -C "${{ parameters.tarballDir }}/.dotnet"
+          eval rm -f "$tarballFilePath"
+        runOnline: $(_RunOnline)
+        tarballDir: ${{ parameters.tarballDir }}
+        tarballResourceId: ${{ parameters.tarballResourceId }}

src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml

@@ -0,0 +1,75 @@
+# Steps used to build a source-build tarball
+
+parameters:
+  # Additional custom build args
+  additionalBuildArgs: ""
+
+  # Architecture to build
+  buildArch: null
+
+  # Container image to build within
+  container: null
+
+  # Script logic to prep the tarball for building
+  prepScript: null
+
+  # Whether or not to build online
+  runOnline: null
+
+  # Directory to build the tarball in
+  tarballDir:
+
+  # Resource id of the tarball to download and build
+  tarballResourceId: null
+
+steps:
+  - download: ${{ parameters.tarballResourceId }}
+    artifact: BlobArtifacts
+    patterns: '**/dotnet-sdk-source*.tar.gz'
+    displayName: Download Tarball
+
+  - script: |
+      set -x
+
+      resourceIdPathSegment=
+      if [ '${{ parameters.tarballResourceId }}' != 'current' ]; then
+        resourceIdPathSegment='${{ parameters.tarballResourceId }}/'
+      fi
+
+      mkdir -p "${{ parameters.tarballDir }}"
+      tarballFilePath="$(PIPELINE.WORKSPACE)/${resourceIdPathSegment}BlobArtifacts/dotnet-sdk-source*.tar.gz"
+      eval tar -ozxf "$tarballFilePath" -C "${{ parameters.tarballDir }}"
+      eval rm -f "$tarballFilePath"
+    displayName: Extract Tarball
+
+  - script: ${{ parameters.prepScript }}
+    displayName: Prep Tarball
+
+  - script: |
+      set -x
+      df -h
+
+      networkArgs=
+      customBuildArgs=
+      if [ '${{ parameters.runOnline}}' = 'true' ]; then
+        customBuildArgs='--online'
+      else
+        networkArgs='--network none'
+      fi
+
+      docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball ${networkArgs} ${{ parameters.container }} ./build.sh ${customBuildArgs} ${{ parameters.additionalBuildArgs }} -- /p:CleanWhileBuilding=true
+    displayName: Build Tarball
+
+  - script: |
+      set -x
+
+      docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }}
+    displayName: Run Tests
+
+  - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
+    parameters:
+      sourceFolder: ${{ parameters.tarballDir }}
+
+  - publish: '${{ parameters.tarballDir }}/artifacts/${{ parameters.buildArch}}/Release/'
+    artifact: $(Agent.JobName)_Artifacts
+    displayName: Publish Artifacts