dotnet-installer/eng/pipelines/templates/jobs/vmr-build.yml

### This job builds https://github.com/dotnet/dotnet with given parameters
### If run in an installer PR, new changes are applied to a local copy of the VMR, then it is built and tested

parameters:
- name: architecture
  type: string

- name: artifactsRid
  type: string
  default: ''

- name: buildName
  type: string

- name: container
  type: string

- name: crossRootFs
  type: string
  default: ''

- name: pool
  type: object

- name: targetOS
  type: string
  default: ''

- name: targetArchitecture
  type: string
  default: ''

#### SOURCE-ONLY parameters ####

# Instead of building the VMR directly, exports the sources into a tarball and builds from that
- name: buildFromArchive
  type: boolean
  default: false

# Enable for source-building the VMR
- name: buildSourceOnly
  type: boolean
  default: false

- name: enablePoison
  type: boolean
  default: false

- name: excludeOmniSharpTests
  type: boolean
  default: false

# Name of a previous job (from the same template as this) whose output will be used to build this job
# The SDK from its artifacts is copied to $(sourcesPath)/.dotnet
- name: reuseBuildArtifactsFrom
  type: string
  default: ''

# Allow downloading artifacts from the internet during the build
- name: runOnline
  type: boolean
  default: true

- name: runTests
  type: boolean
  default: false

- name: useMonoRuntime
  displayName: True when build output uses the mono runtime
  type: boolean
  default: false

# Use the previous version's SDK to build the current one
- name: withPreviousSDK
  type: boolean
  default: false

#### INSTALLER parameters ####

- name: isBuiltFromVmr
  displayName: True when build is running from dotnet/dotnet directly
  type: boolean

- name: vmrBranch
  displayName: dotnet/dotnet branch to use
  type: string

jobs:
- job: ${{ parameters.buildName }}_${{ parameters.architecture }}
  timeoutInMinutes: 150
  pool: ${{ parameters.pool }}
  ${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}:
    # Always attempt to run the bootstrap leg (e.g. even when stage 1 tests fail) in order to get a complete accessment of the build status.
    # The build shortcuts when stage 1 build fails and doesn't produce the SDK.
    condition: succeededOrFailed()
    dependsOn: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}
  variables:
  - template: /eng/common/templates/variables/pool-providers.yml
  - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
    - group: AzureDevOps-Artifact-Feeds-Pats
  - ${{ if and(not(parameters.isBuiltFromVmr), eq(variables['System.TeamProject'], 'internal'), not(startswith(parameters.vmrBranch, 'internal/release/')), not(eq(variables['Build.Reason'], 'PullRequest'))) }}:
    - group: DotNetBot-GitHub
  - ${{ else }}:
    - name: BotAccount-dotnet-bot-repo-PAT
      value: N/A
  - name: additionalBuildArgs
    value: ''

  - ${{ if parameters.isBuiltFromVmr }}:
    - name: vmrPath
      value: $(Build.SourcesDirectory)
  - ${{ else }}:
    - name: vmrPath
      value: $(Agent.BuildDirectory)/vmr

  # Location of the VMR sources
  # We either build the repo directly, or we extract them outside (which is what partners do)
  - ${{ if parameters.buildFromArchive }}:
    - name: sourcesPath
      value: $(Build.StagingDirectory)/dotnet-sources/
  - ${{ else }}:
    - name: sourcesPath
      value: $(vmrPath)

  steps:
  - ${{ if not(parameters.isBuiltFromVmr) }}:
    - template: ../steps/vmr-prepare.yml
      parameters:
        ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
          vmrBranch: $(System.PullRequest.TargetBranch)
        ${{ else }}:
          vmrBranch: ${{ parameters.vmrBranch }}

    # Synchronize new content in the VMR during PRs (we expect this to come
    - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
      - template: ../steps/vmr-pull-updates.yml
        parameters:
          vmrPath: $(vmrPath)
          vmrBranch: ${{ parameters.vmrBranch }}
          targetRef: $(Build.SourceVersion) # Synchronize the current installer commit

  - ${{ if parameters.buildFromArchive }}:
    - script: |
        set -ex
        cp -r "$(vmrPath)" "$(sourcesPath)"
        rm -rf "$(sourcesPath)/.git"        
      displayName: Export VMR sources
      workingDirectory: $(Build.StagingDirectory)

  - ${{ if and(ne(variables['System.TeamProject'], 'public'), eq(parameters.runTests, 'True')) }}:
    - script: cp "$(sourcesPath)/src/installer/NuGet.config" "$(sourcesPath)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config"
      displayName: Copy Test NuGet Config

    - task: Bash@3
      displayName: Setup Private Feeds Credentials
      inputs:
        filePath: $(sourcesPath)/src/installer/eng/common/SetupNugetSources.sh
        arguments: $(sourcesPath)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config $Token
      env:
        Token: $(dn-bot-dnceng-artifact-feeds-rw)

  - ${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}:
    - download: current
      artifact: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts
      patterns: |
        **/Private.SourceBuilt.Artifacts.*.tar.gz
        **/dotnet-sdk-*.tar.gz        
      displayName: Download Previous Build

    - task: CopyFiles@2
      displayName: Copy Previous Build
      inputs:
        SourceFolder: $(Pipeline.Workspace)/${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts
        Contents: '*.tar.gz'
        TargetFolder: $(sourcesPath)/prereqs/packages/archive/

  - ${{ if eq(parameters.withPreviousSDK, 'true') }}:
    - script: |
        set -euo pipefail

        if [[ '${{ parameters.artifactsRid }}' == '' ]]; then
           echo "'artifactsRid' is not specified. Cannot download source-built SDK."
           exit 1
        fi

        packageVersionsPath="$(sourcesPath)/eng/Versions.props"
        notFoundMessage="No source-built SDK found to download..."

        echo "Looking for source-built SDK to download..."
        archiveVersionLine=$(grep -m 1 "<PrivateSourceBuiltSdkVersion>" "$packageVersionsPath" || :)
        versionPattern="<PrivateSourceBuiltSdkVersion>(.*)</PrivateSourceBuiltSdkVersion>"

        if [[ $archiveVersionLine =~ $versionPattern ]]; then
          archiveVersion="${BASH_REMATCH[1]}"
          archiveUrl="https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-$archiveVersion-${{ parameters.artifactsRid }}.tar.gz"
          downloadDir="$(sourcesPath)/prereqs/packages/archive/"

          echo "Downloading source-built SDK from $archiveUrl..."
          (cd "$downloadDir" && curl --retry 5 -O "$archiveUrl")
        else
          echo "$notFoundMessage"
          exit 1
        fi        
      displayName: Setup Previously Source-Built SDK
  
  - ${{ if eq(parameters.targetOS, 'windows') }}:
    - script: |
        call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine        
      displayName: Build

  - ${{ else }}:
    - ${{ if eq(parameters.buildSourceOnly, 'true') }}:
      - script: |
          set -x

          customPrepArgs=""
          prepSdk=true

          if [[ -n '${{ parameters.artifactsRid }}' ]]; then
            customPrepArgs="${customPrepArgs} --artifacts-rid ${{ parameters.artifactsRid }}"
          fi

          if [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then
            # Source-built artifacts are from CentOS 8 Stream or Alpine 3.19. We want to download them without
            # downloading portable versions from the internet.
            customPrepArgs="${customPrepArgs} --no-sdk --no-bootstrap"
            prepSdk=false
          elif [[ -n '${{ parameters.reuseBuildArtifactsFrom }}' ]]; then
            customPrepArgs="${customPrepArgs} --no-sdk --no-artifacts"
            prepSdk=false
          fi

          if [[ "$prepSdk" == "false" ]]; then
            mkdir $(sourcesPath)/.dotnet
            previousSdkPath="$(sourcesPath)/prereqs/packages/archive/dotnet-sdk-*.tar.gz"
            eval tar -ozxf "$previousSdkPath" -C "$(sourcesPath)/.dotnet"
            eval rm -f "$previousSdkPath"

            echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /vmr/.dotnet"
          fi

          docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh $customPrepArgs          
        displayName: Prep the Build

    - script: |
        set -x
        df -h

        customEnvVars=""
        customBuildArgs="--ci --clean-while-building --prepareMachine"
        if [[ '${{ parameters.runOnline }}' == 'True' ]]; then
          customBuildArgs="$customBuildArgs --online"
        fi

        if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
          customBuildArgs="$customBuildArgs --poison"
        fi

        if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then
          customBuildArgs="$customBuildArgs --source-repository https://github.com/dotnet/dotnet"
          customBuildArgs="$customBuildArgs --source-version $(git -C "$(vmrPath)" rev-parse HEAD)"
        fi

        if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
          customBuildArgs="$customBuildArgs --source-only"
        fi

        if [[ '${{ parameters.useMonoRuntime }}' == 'True' ]]; then
          customBuildArgs="$customBuildArgs --use-mono-runtime"
        fi

        if [[ -n "${{ parameters.crossRootFs }}" ]]; then
          customEnvVars="$customEnvVars ROOTFS_DIR=${{ parameters.crossRootFs}} CROSSCOMPILE=1"
        fi

        if [[ ! -z '${{ parameters.targetOS }}' ]]; then
          extraBuildProperties="$extraBuildProperties /p:TargetOS=${{ parameters.targetOS }}"
        fi

        if [[ ! -z '${{ parameters.targetArchitecture }}' ]]; then
          extraBuildProperties="$extraBuildProperties /p:TargetArchitecture=${{ parameters.targetArchitecture }}"
        fi

        buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties"

        # Only use Docker when a container is specified
        if [[ -n "${{ parameters.container }}" ]]; then
          # Allows Arcade to have access to the commit for the build, pass it through to the container
          customEnvVars="$customEnvVars BUILD_SOURCEVERSION=$BUILD_SOURCEVERSION"

          customDockerRunArgs=""
          for envVar in $customEnvVars; do
            customDockerRunArgs="$customDockerRunArgs -e $envVar"
          done
          
          if [[ '${{ parameters.runOnline }}' == 'False' ]]; then
            customDockerRunArgs="$customDockerRunArgs --network none"
          fi

          docker run --rm -v "$(sourcesPath):/vmr" -w /vmr $customDockerRunArgs ${{ parameters.container }} ./build.sh $buildArgs
        else
          for envVar in $customEnvVars; do
            customEnvVarsWithBashSyntax="$customEnvVarsWithBashSyntax export $envVar;"
          done
          cd $(sourcesPath)
          eval $customEnvVarsWithBashSyntax
          ./build.sh $buildArgs
        fi        
      displayName: Build

  # Only run tests if enabled
  - ${{ if eq(parameters.runTests, 'True') }}:
    - script: |
        set -x

        dockerVolumeArgs="-v $(sourcesPath):/vmr"
        dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=${{ parameters.excludeOmniSharpTests }} -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true -e SMOKE_TESTS_RUNNING_IN_CI=true"
        poisonArg=''

        if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
          poisonArg='--poison'
        fi

        docker run --rm $dockerVolumeArgs -w /vmr $dockerEnvArgs ${{ parameters.container }} ./build.sh --source-only $poisonArg --test $(additionalBuildArgs) /p:SmokeTestConsoleVerbosity=detailed        
      displayName: Run Tests

  - ${{ if eq(parameters.targetOS, 'windows') }}:

    # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph.
    - powershell: |
        function CopyWithRelativeFolders($sourcePath, $targetFolder, $filter) {
          Get-ChildItem -Path $sourcePath -Filter $filter -Recurse | ForEach-Object { 
            $targetPath = Join-Path $targetFolder (Resolve-Path -Relative $_.FullName)
            New-Item -ItemType Directory -Path (Split-Path -Parent $targetPath) -Force | Out-Null
            Copy-Item $_.FullName -Destination $targetPath -Force 
          }
        }

        $targetFolder = "$(Build.StagingDirectory)/BuildLogs/"
        New-Item -ItemType Directory -Path $targetFolder -Force | Out-Null

        cd "$(sourcesPath)"

        CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.binlog"
        CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.log"
        CopyWithRelativeFolders "artifacts/prebuilt-report/" $targetFolder
        CopyWithRelativeFolders "src/"                       $targetFolder "*.binlog"
        CopyWithRelativeFolders "src/"                       $targetFolder "*.log"
        CopyWithRelativeFolders "test/"                      $targetFolder "*.binlog"
        CopyWithRelativeFolders "test/"                      $targetFolder "Updated*.diff"
        CopyWithRelativeFolders "test/"                      $targetFolder "Updated*.txt"        
      displayName: Prepare BuildLogs staging directory
      continueOnError: true
      condition: succeededOrFailed()

  - ${{ else }}:

    # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph.
    - script: |
        set -x

        targetFolder=$(Build.StagingDirectory)/BuildLogs/
        mkdir -p ${targetFolder}

        cd "$(sourcesPath)"
        find artifacts/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
        find artifacts/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
        find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
        find src/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
        find src/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
        find test/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
        find test/ -type f -name "Updated*.diff" -exec rsync -R {} -t ${targetFolder} \;
        find test/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \;        
      displayName: Prepare BuildLogs staging directory
      continueOnError: true
      condition: succeededOrFailed()

  - publish: '$(Build.StagingDirectory)/BuildLogs'
    artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
    displayName: Publish BuildLogs
    continueOnError: true
    condition: succeededOrFailed()

  # Only upload test results if enabled
  - ${{ if eq(parameters.runTests, 'True') }}:
    - task: PublishTestResults@2
      displayName: Publish Test Results
      condition: succeededOrFailed()
      continueOnError: true
      inputs:
        testRunner: vSTest
        testResultsFiles: 'test/**/*.trx'
        searchFolder: $(sourcesPath)
        mergeTestResults: true
        publishRunAttachments: true
        testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName)

  - publish: '$(sourcesPath)/artifacts/${{ parameters.architecture }}/Release/'
    artifact: $(Agent.JobName)_Artifacts
    displayName: Publish Artifacts
    condition: succeededOrFailed()
    continueOnError: true