[release/6.0.1xx] Add some additional distros to the CI matrix (#12141)

src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml

@@ -53,7 +53,22 @@ jobs:
         /p:ArcadeBuildTarball=true
     displayName: Create Tarball
 
-  - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
+  - task: CopyFiles@2
+    displayName: Prepare BuildLogs staging directory
+    inputs:
+      Contents: |
+        **/*.log
+        **/*.binlog
+      TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+      CleanTargetFolder: true
+    continueOnError: true
+    condition: succeededOrFailed()
+
+  - publish: '$(Build.StagingDirectory)/BuildLogs'
+    artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
+    displayName: Publish BuildLogs
+    continueOnError: true
+    condition: succeededOrFailed()
 
   - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
     # Check if the tarball should be built now so the result can be used in the build tarball job condition.

src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml

@@ -12,7 +12,11 @@ parameters:
 
   # The following parameters aren't expected to be passed in rather they are used for encapsulation
   # -----------------------------------------------------------------------------------------------
+  centOS7Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-source-build-20210714125450-5d87b80
+  centOS8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-source-build-20211118190102-9355e7b
+  debian9Container: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-20211001171226-047508b
   fedora33Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-33-20210222183538-031e7d2
+  ubuntu1804Container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20210924170306-047508b
   poolInternal:
     name: NetCore1ESPool-Svc-Internal
     demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
@@ -33,15 +37,33 @@ jobs:
       ${{ parameters.poolInternal }}
   strategy:
     matrix:
-      Fedora33-Online:
+      CentOS7-Online:
+        _BootstrapPrep: true
         _BuildArch: x64
-        _Container: ${{ parameters.fedora33Container }}
+        _Container: ${{ parameters.centOS7Container }}
         _RunOnline: true
-      Fedora33-Offline:
+      CentOS7-Offline:
+        _BootstrapPrep: true
         _BuildArch: x64
-        _Container: ${{ parameters.fedora33Container }}
+        _Container: ${{ parameters.centOS7Container }}
         _RunOnline: false
-  timeoutInMinutes: 240
+      ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+        CentOS8-Offline:
+          _BootstrapPrep: true
+          _BuildArch: x64
+          _Container: ${{ parameters.centOS8Container }}
+          _RunOnline: false
+        Fedora33-Offline:
+          _BootstrapPrep: false
+          _BuildArch: x64
+          _Container: ${{ parameters.fedora33Container }}
+          _RunOnline: false
+        Ubuntu1804-Offline:
+          _BootstrapPrep: true
+          _BuildArch: x64
+          _Container: ${{ parameters.ubuntu1804Container }}
+          _RunOnline: false
+  timeoutInMinutes: 300
   workspace:
     clean: all
 
@@ -55,7 +77,12 @@ jobs:
       prepScript: |
         set -x
 
-        docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball $(_Container) ./prep.sh
+        customPrepArgs=
+        if [ '$(_BootstrapPrep)' = 'true' ]; then
+          customPrepArgs='--bootstrap'
+        fi
+
+        docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball $(_Container) ./prep.sh  ${customPrepArgs}
       runOnline: $(_RunOnline)
       tarballDir: ${{ parameters.tarballDir }}
       tarballResourceId: ${{ parameters.tarballResourceId }}

src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml

@@ -66,9 +66,29 @@ steps:
       docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }}
     displayName: Run Tests
 
-  - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
-    parameters:
-      sourceFolder: ${{ parameters.tarballDir }}
+  # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph.
+  - script: |
+      set -x
+
+      targetFolder=$(Build.StagingDirectory)/BuildLogs/
+      mkdir -p ${targetFolder}
+
+      cd "${{ parameters.tarballDir }}"
+      find artifacts/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/prebuilt-report/ -exec cp {} --parents -t ${targetFolder} \;
+      find src/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \;
+      find src/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \;
+      find testing-smoke/logs -exec cp {} --parents -t ${targetFolder} \;
+    displayName: Prepare BuildLogs staging directory
+    continueOnError: true
+    condition: succeededOrFailed()
+
+  - publish: '$(Build.StagingDirectory)/BuildLogs'
+    artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
+    displayName: Publish BuildLogs
+    continueOnError: true
+    condition: succeededOrFailed()
 
   - publish: '${{ parameters.tarballDir }}/artifacts/${{ parameters.buildArch}}/Release/'
     artifact: $(Agent.JobName)_Artifacts

src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml

@@ -1,25 +0,0 @@
-# Gathers and publishes the build logs for a source-build
-
-parameters:
-  # Folder to scan for source-build logs from
-  sourceFolder: $(Build.SourcesDirectory)
-
-steps:
-- task: CopyFiles@2
-  displayName: Prepare BuildLogs staging directory
-  inputs:
-    SourceFolder: '${{ parameters.sourceFolder }}'
-    Contents: |
-      **/*.log
-      **/*.binlog
-      artifacts/prebuilt-report/*
-    TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
-    CleanTargetFolder: true
-  continueOnError: true
-  condition: succeededOrFailed()
-
-- publish: '$(Build.StagingDirectory)/BuildLogs'
-  artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
-  displayName: Publish BuildLogs
-  continueOnError: true
-  condition: succeededOrFailed()

src/SourceBuild/tarball/content/build.sh

@@ -156,11 +156,12 @@ export DOTNET_CLI_TELEMETRY_OPTOUT=1
 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 export NUGET_PACKAGES=$restoredPackagesDir/
 
-if [ "$alternateTarget" == "true" ]; then
-  "$CLI_ROOT/dotnet" $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll "$SCRIPT_ROOT/build.proj" /bl:source-build-test.binlog /clp:v=m ${MSBUILD_ARGUMENTS[@]} "$@"
-else
-  LogDateStamp=$(date +"%m%d%H%M%S")
-  $CLI_ROOT/dotnet $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll /bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildXPlatTasks_$LogDateStamp.binlog $SCRIPT_ROOT/tools-local/init-build.proj /t:PrepareOfflineLocalTools ${MSBUILD_ARGUMENTS[@]} "$@"
+LogDateStamp=$(date +"%m%d%H%M%S")
 
-  $CLI_ROOT/dotnet $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll /bl:$SCRIPT_ROOT/artifacts/log/Debug/Build_$LogDateStamp.binlog $SCRIPT_ROOT/build.proj ${MSBUILD_ARGUMENTS[@]} "$@"
+if [ "$alternateTarget" == "true" ]; then
+  "$CLI_ROOT/dotnet" $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll "$SCRIPT_ROOT/build.proj" /bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildTests_$LogDateStamp.binlog /fileLoggerParameters:LogFile=$SCRIPT_ROOT/artifacts/logs/BuildTests_$LogDateStamp.log /clp:v=m ${MSBUILD_ARGUMENTS[@]} "$@"
+else
+  $CLI_ROOT/dotnet $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll /bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildXPlatTasks_$LogDateStamp.binlog /fileLoggerParameters:LogFile=$SCRIPT_ROOT/artifacts/logs/BuildXPlatTasks_$LogDateStamp.log $SCRIPT_ROOT/tools-local/init-build.proj /t:PrepareOfflineLocalTools ${MSBUILD_ARGUMENTS[@]} "$@"
+
+  $CLI_ROOT/dotnet $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll /bl:$SCRIPT_ROOT/artifacts/log/Debug/Build_$LogDateStamp.binlog /fileLoggerParameters:LogFile=$SCRIPT_ROOT/artifacts/logs/Build_$LogDateStamp.log $SCRIPT_ROOT/build.proj ${MSBUILD_ARGUMENTS[@]} "$@"
 fi

src/SourceBuild/tarball/content/prep.sh

@@ -118,7 +118,7 @@ if [ "$buildBootstrap" == "true" ]; then
     fi
 
     # Run restore on project to initiate download of bootstrap packages
-    $DOTNET_SDK_PATH/dotnet restore $workingDir/buildBootstrapPreviouslySB.csproj /bl /p:ArchiveDir="$SCRIPT_ROOT/packages/archive/"
+    $DOTNET_SDK_PATH/dotnet restore $workingDir/buildBootstrapPreviouslySB.csproj /bl:artifacts/prep/bootstrap.binlog /fileLoggerParameters:LogFile=artifacts/prep/bootstrap.log /p:ArchiveDir="$SCRIPT_ROOT/packages/archive/"
 
     # Remove working directory
     rm -rf $workingDir

src/SourceBuild/tarball/content/smoke-test.sh

@@ -55,7 +55,9 @@ excludeOnlineTests=false
 devCertsVersion="$DEV_CERTS_VERSION_DEFAULT"
 testingDir="$SCRIPT_ROOT/testing-smoke"
 cliDir="$testingDir/builtCli"
-logFile="$testingDir/smoke-test.log"
+logsDir="$testingDir/logs"
+logFile="$logsDir/smoke-test.log"
+omnisharpLogFile="$logsDir/omnisharp.log"
 restoredPackagesDir="$testingDir/packages"
 testingHome="$testingDir/home"
 archiveRestoredPackages=false
@@ -206,7 +208,7 @@ function doCommand() {
             binlogHttpsPart="https"
         fi
 
-        binlogPrefix="$testingDir/${projectDir}_${binlogOnlinePart}_${binlogHttpsPart}_"
+        binlogPrefix="$logsDir/${projectDir}_${binlogOnlinePart}_${binlogHttpsPart}_"
         binlog="${binlogPrefix}$1.binlog"
         echo "    running $1" | tee -a "$logFile"
 
@@ -619,7 +621,7 @@ function runOmniSharpTests() {
         "${dotnetCmd}" new $project
         popd
 
-        ./omnisharp/run -s "$(readlink -f hello-$project)" > omnisharp.log &
+        ./omnisharp/run -s "$(readlink -f hello-$project)" > "$omnisharpLogFile" &
 
         sleep 5
 
@@ -632,9 +634,9 @@ function runOmniSharpTests() {
 
         kill "$(pgrep -f "$(pwd)")"
 
-        cat omnisharp.log
+        cat "$omnisharpLogFile"
 
-        if grep ERROR omnisharp.log; then
+        if grep ERROR "$omnisharpLogFile"; then
             echo "test failed"
             exit 1
         else
@@ -702,6 +704,7 @@ if [ -e "$testingDir"  ]; then
 fi
 
 mkdir -p "$testingDir"
+mkdir -p "$logsDir"
 cd "$testingDir"
 
 # Create blank Directory.Build files to avoid traversing to source-build infra.