Add source-build tarball build ci (#10995)

.vsts-ci.yml

@@ -337,7 +337,7 @@ stages:
 
   - template: /eng/common/templates/jobs/source-build.yml
 
-  - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-tarball-creation.yml
+  - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml
 
   # https://github.com/dotnet/core-sdk/issues/248
   # - template: /eng/build.yml
@@ -360,7 +360,7 @@ stages:
           - Linux
           - Darwin
           - Source_Build_Managed
-          - Source_Build_Tarball_Creation
+          - Source_Build_Create_Tarball
         publishUsingPipelines: true
         pool:
           vmImage: vs2017-win2016

eng/source-build-tarball-build.yml

@@ -0,0 +1,40 @@
+trigger: none
+pr: none
+resources:
+  pipelines:
+  - pipeline: installer-build-resource
+    source: dotnet-installer-official-ci
+    trigger: true # Run pipeline when any run of dotnet-installer-official-ci completes
+
+stages:
+- stage: build
+  jobs:
+  - job: build
+    displayName: Build
+    pool:
+      vmImage: ubuntu-20.04
+    container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-32-20210222183556-031e7d2
+    workspace:
+      clean: all
+
+    variables:
+      _TarballDir: $(Build.StagingDirectory)/tarball
+    steps:
+    - checkout: none
+
+    - download: installer-build-resource
+      patterns: 'BlobArtifacts/dotnet-sdk-source*.tar.gz'
+
+    - script: |
+        set -x
+
+        mkdir -p "$(_TarballDir)"
+        tarballFilePath="$(PIPELINE.WORKSPACE)/installer-build-resource/BlobArtifacts/dotnet-sdk-source*.tar.gz"
+        eval tar -ozxf "$tarballFilePath" -C "$(_TarballDir)"
+
+        df -h
+      displayName: Extract Tarball
+
+    - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-run-tarball-build.yml
+      parameters:
+        sourceFolder: $(_TarballDir)

src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml

@@ -0,0 +1,53 @@
+# Creates a source-build tarball
+
+jobs:
+- job: Source_Build_Create_Tarball
+  displayName: Source-Build Create Tarball
+  pool:
+    vmImage: ubuntu-20.04
+  container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-32-20210222183556-031e7d2
+  workspace:
+    clean: all
+
+  steps:
+  - script: |
+      set -x
+      df -h
+
+      buildConfig=Release
+      # Check if AzDO substitutes in a build config from a variable, and use it if so.
+      if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
+        buildConfig='$(_BuildConfig)'
+      fi
+
+      officialBuildArgs=
+      if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
+        officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+      fi
+
+      ./build.sh \
+        --ci \
+        --configuration $buildConfig \
+        --publish \
+        -bl \
+        $officialBuildArgs \
+        /p:ArcadeBuildTarball=true
+    displayName: Create Tarball
+
+  - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
+    parameters:
+      artifactQualifier: TarballCreation
+
+  - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
+    # Only build the tarball if the PR touches source-build source
+    - script: |
+        if curl "https://api.github.com/repos/dotnet/installer/pulls/$(System.PullRequest.PullRequestNumber)/files" | grep '"filename": "src/SourceBuild/*'
+        then
+          echo "##vso[task.setvariable variable=_includeTarballBuild]true"
+        fi
+      displayName: Include Tarball Build Check
+
+    - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-run-tarball-build.yml
+      parameters:
+        sourceFolder: $(Build.SourcesDirectory)/artifacts/tarball
+        condition: eq(variables['_includeTarballBuild'], 'true')

src/SourceBuild/Arcade/eng/common/templates/job/source-build-tarball-creation.yml

@@ -1,53 +0,0 @@
-jobs:
-  - job: Source_Build_Tarball_Creation
-    displayName: Source-Build Tarball Creation
-    pool:
-      vmImage: ubuntu-20.04
-    container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-32-20210222183556-031e7d2
-    workspace:
-      clean: all
-
-    steps:
-    - script: |
-        set -x
-        df -h
-
-        buildConfig=Release
-        # Check if AzDO substitutes in a build config from a variable, and use it if so.
-        if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
-          buildConfig='$(_BuildConfig)'
-        fi
-
-        officialBuildArgs=
-        if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
-          officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
-        fi
-
-        ./build.sh \
-          --ci \
-          --configuration $buildConfig \
-          --publish \
-          -bl \
-          $officialBuildArgs \
-          /p:ArcadeBuildTarball=true
-      displayName: Create Tarball
-
-    - task: CopyFiles@2
-      displayName: Prepare BuildLogs staging directory
-      inputs:
-        SourceFolder: '$(Build.SourcesDirectory)'
-        Contents: |
-          **/*.log
-          **/*.binlog
-        TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
-        CleanTargetFolder: true
-      continueOnError: true
-      condition: succeededOrFailed()
-
-    - task: PublishPipelineArtifact@1
-      displayName: Publish BuildLogs
-      inputs:
-        targetPath: '$(Build.StagingDirectory)/BuildLogs'
-        artifactName: BuildLogs_SourceBuild_TarballCreation_Attempt$(System.JobAttempt)
-      continueOnError: true
-      condition: succeededOrFailed()

src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml

@@ -0,0 +1,33 @@
+# Gathers and publishes the build logs for a source-build
+
+parameters:
+  # Folder to scan for source-build logs from
+  sourceFolder: $(Build.SourcesDirectory)
+
+  # The unique qualifier to append to the artifactName
+  artifactQualifier:
+
+  # Custom condition to apply to each step
+  condition: true
+
+steps:
+- task: CopyFiles@2
+  displayName: Prepare BuildLogs staging directory
+  inputs:
+    SourceFolder: '${{ parameters.sourceFolder }}'
+    Contents: |
+      **/*.log
+      **/*.binlog
+      artifacts/source-build/self/prebuilt-report/**
+    TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+    CleanTargetFolder: true
+  continueOnError: true
+  condition: and(${{ parameters.condition }}, succeededOrFailed())
+
+- task: PublishPipelineArtifact@1
+  displayName: Publish BuildLogs
+  inputs:
+    targetPath: '$(Build.StagingDirectory)/BuildLogs'
+    artifactName: BuildLogs_SourceBuild_${{ parameters.artifactQualifier }}_Attempt$(System.JobAttempt)
+  continueOnError: true
+  condition: and(${{ parameters.condition }}, succeededOrFailed())

src/SourceBuild/Arcade/eng/common/templates/steps/source-build-run-tarball-build.yml

@@ -0,0 +1,31 @@
+# Builds the specified source-build tarball
+
+parameters:
+  # Folder containing the tarball source to build
+  sourceFolder: $(Build.SourcesDirectory)
+
+  # Custom condition to apply to each step
+  condition: true
+
+steps:
+- script: |
+    set -x
+
+    ./prep.sh
+  workingDirectory: ${{ parameters.sourceFolder }}
+  displayName: Prep Tarball
+  condition: and(${{ parameters.condition }}, succeeded())
+
+- script: |
+    set -x
+
+    ./build.sh
+  workingDirectory: ${{ parameters.sourceFolder }}
+  displayName: Build Tarball
+  condition: and(${{ parameters.condition }}, succeeded())
+
+- template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
+  parameters:
+    sourceFolder: ${{ parameters.sourceFolder }}
+    artifactQualifier: TarballBuild
+    condition: and(${{ parameters.condition }}, succeeded())