electron/vsts.yml

# The config expects the following environment variables to be set:
#  - "Custom.UseArtifacts.BuildId" aka "CUSTOM_USEARTIFACTS_BUILDID"
#      Set it to a build number, e.g. 15912, to download artifacts
#      from that build and only run the tests, instead of building from scratch.
#  - "ELECTRON_RELEASE" Set it to '1' upload binaries on success.
#  - "GN_CONFIG" Full path to a build config, e.g. '//electron/build/args/testing.gn'.
#  - "GN_EXTRA_ARGS" Additional gn args, e.g. 'is_mas_build=true'.
#  - "NOTIFY_SLACK" Set it to '1' to enable Slack notifications.
#  - "RUN_TESTS" Set it to '1' to run Electron's tests.
#  - "UPLOAD_TO_S3" Set it to '1' upload a release to the S3 bucket.
#      Otherwise the release will be uploaded to the Github Releases.
#      (The value is only checked if "ELECTRON_RELEASE" is defined.)
#
# The publishing scripts expect access tokens to be defined as env vars,
# but those are not covered here.
#
# VSTS docs on variables:
# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables

jobs:
- job: build
  displayName: Build
  # Build Electron only if we are NOT going to use artifacts produced by a different build.
  condition: eq(variables['Custom.UseArtifacts.BuildId'], '')
  timeoutInMinutes: 120
  variables:
    CI: true
  steps:

  - bash: |
      git clean -fdx
    displayName: Clean unneeded git directories
    timeoutInMinutes: 2

  - task: CopyFiles@2
    displayName: 'Copy Files to: src/electron'
    inputs:
      TargetFolder: src/electron
    timeoutInMinutes: 2

  - bash: |
      export PATH="$PATH:/Users/electron/depot_tools"
      echo "##vso[task.setvariable variable=PATH]$PATH"
      export GIT_CACHE_PATH="/Users/electron/libcc_cache"
      if [ "$ELECTRON_RELEASE" == "1" ]; then
        GCLIENT_EXTRA_ARGS='--custom-var=checkout_boto=True --custom-var=checkout_requests=True'
      fi

      set -ex
      gclient config \
        --name "src/electron" \
        --unmanaged \
        $GCLIENT_EXTRA_ARGS \
        "https://github.com/electron/electron"
      gclient sync --with_branch_heads --with_tags
      cd src
      export CHROMIUM_BUILDTOOLS_PATH=`pwd`/buildtools
      echo "##vso[task.setvariable variable=CHROMIUM_BUILDTOOLS_PATH]`pwd`/buildtools"
    displayName: gclient sync
    timeoutInMinutes: 25  # Usually takes about 10-15 minutes.

  - bash: |
      cd src
      export SCCACHE_BINARY="`pwd`/electron/external_binaries/sccache"
      "$SCCACHE_BINARY" --start-server
      echo "##vso[task.setvariable variable=SCCACHE_BINARY]$SCCACHE_BINARY"
      "$SCCACHE_BINARY" -s
      echo "GN gen for: $GN_CONFIG"
      gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_BINARY"'"'" $GN_EXTRA_ARGS"
    env:
      AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY)
      AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET)
    displayName: GN gen with sccache
    condition: and(succeeded(), ne(variables['ELECTRON_RELEASE'], '1'))
    timeoutInMinutes: 1  # Shouldn't take more than 10 seconds.

  - bash: |
      cd src
      echo "GN gen for: $GN_CONFIG"
      gn gen out/Default --args='import("'$GN_CONFIG'")'" $GN_EXTRA_ARGS"
    displayName: GN gen without sccache
    condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
    timeoutInMinutes: 1  # Shouldn't take more than 10 seconds.

  - bash: |
      cd src
      ninja -C out/Default electron:electron_app
    displayName: Ninja build app
    timeoutInMinutes: 100  # Can take up to 80 minutes without compiler cache.

  - bash: |
      cd src
      ninja -C out/Default third_party/electron_node:headers
    displayName: Build Node.js headers for testing
    timeoutInMinutes: 1  # Usually takes less than 10 seconds.

  - bash: |
      cd src
      gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
      ninja -C out/ffmpeg electron:electron_ffmpeg_zip
    displayName: Non proprietary ffmpeg build
    condition: and(succeeded(), or(eq(variables['RUN_TESTS'], '1'), eq(variables['ELECTRON_RELEASE'], '1')))
    timeoutInMinutes: 2  # Usually takes less than 30 seconds.

  - bash: |
      cd src
      # Build needed dump_syms executable
      ninja -C out/Default third_party/breakpad:dump_syms

      # TODO(alexeykuzmin): Explicitly pass an out folder path to the scripts.
      export ELECTRON_OUT_DIR=Default

      electron/script/dump-symbols.py -d "$PWD/out/Default/breakpad_symbols"
      electron/script/zip-symbols.py
    displayName: Collect symbols
    condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
    timeoutInMinutes: 5  # Usually takes less than two.

  - bash: |
      "$SCCACHE_BINARY" --stop-server
    displayName: Check sccache stats after build
    condition: and(succeeded(), ne(variables['ELECTRON_RELEASE'], '1'))
    timeoutInMinutes: 1

  - bash: |
      cd src
      ninja -C out/Default electron:electron_dist_zip
    displayName: Build dist zip
    timeoutInMinutes: 2  # Usually takes less than 20 seconds.

  - bash: |
      cd src
      ninja -C out/Default electron:electron_chromedriver_zip
    displayName: Build chromedriver and zip
    timeoutInMinutes: 3  # Normally less than one minute.

  - bash: |
      cd src
      ninja -C out/Default electron:electron_mksnapshot_zip
    displayName: Build mksnapshot and zip
    timeoutInMinutes: 1  # Normally less than 10 seconds.

  - bash: |
      cd src/electron
      npm run create-typescript-definitions
    displayName: Generate type declarations
    condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
    timeoutInMinutes: 2  # Normally takes about 10-15 seconds.

  - bash: |
      cd src/electron

      export ELECTRON_OUT_DIR=Default
      export ELECTRON_S3_BUCKET="$(s3_bucket)"
      export ELECTRON_S3_ACCESS_KEY="$(s3_access_key)"
      export ELECTRON_S3_SECRET_KEY="$(s3_secret_key)"
      export ELECTRON_GITHUB_TOKEN="$(github_token)"

      if [ "$UPLOAD_TO_S3" == "1" ]; then
        echo 'Uploading Electron release distribution to s3'
        script/upload.py --upload_to_s3
      else
        echo 'Uploading Electron release distribution to Github releases'
        script/upload.py
      fi
    displayName: Upload distribution
    condition: and(succeeded(), eq(variables['ELECTRON_RELEASE'], '1'))
    timeoutInMinutes: 5  # Normally takes about two minutes.

  - task: PublishBuildArtifacts@1
    displayName: Publish Build Artifacts (application zip)
    inputs:
      PathtoPublish: '$(System.DefaultWorkingDirectory)/src/out/Default/dist.zip'
      ArtifactName: Default
    timeoutInMinutes: 2  # The file is relatively big, so it takes about 20 seconds.

  - task: PublishBuildArtifacts@1
    displayName: Publish Build Artifacts (Node.js headers)
    inputs:
      PathtoPublish: '$(System.DefaultWorkingDirectory)/src/out/Default/gen/node_headers.tar.gz'
      ArtifactName: Default
    timeoutInMinutes: 1

  - task: PublishBuildArtifacts@1
    displayName: Publish Build Artifacts (chromedriver.zip)
    inputs:
      PathtoPublish: '$(System.DefaultWorkingDirectory)/src/out/Default/chromedriver.zip'
      ArtifactName: Default
    timeoutInMinutes: 1

  - task: PublishBuildArtifacts@1
    displayName: Publish Build Artifacts (mksnapshot.zip)
    inputs:
      PathtoPublish: '$(System.DefaultWorkingDirectory)/src/out/Default/mksnapshot.zip'
      ArtifactName: Default
    timeoutInMinutes: 1

  - task: PublishBuildArtifacts@1
    displayName: Publish Build Artifacts (ffmpeg.zip)
    inputs:
      PathtoPublish: '$(System.DefaultWorkingDirectory)/src/out/ffmpeg/ffmpeg.zip'
      ArtifactName: Default
    timeoutInMinutes: 1

  - bash: |
      echo $BUILD_SOURCEVERSION > revision
    displayName: Save exact revision
    timeoutInMinutes: 1

  - task: PublishBuildArtifacts@1
    displayName: Publish Build Artifacts (revision)
    inputs:
      PathtoPublish: '$(System.DefaultWorkingDirectory)/revision'
      ArtifactName: Default
    timeoutInMinutes: 1

  # Notify about success only if we are supposed to, and we don't have to run tests first.
  - bash: |
      export BUILD_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${SYSTEM_TEAMPROJECT}/_build/results?buildId=${BUILD_BUILDID}"
      export MESSAGE="Build succeeded for *<$BUILD_URL|$BUILD_DEFINITIONNAME>* nightly build from *$BUILD_SOURCEBRANCHNAME*."
      curl -g -H "Content-Type: application/json" -X POST \
      -d "{\"text\": \"$MESSAGE\", \"attachments\": [{\"color\": \"good\",\"title\": \"$BUILD_DEFINITIONNAME nightly build results\",\"title_link\": \"$BUILD_URL\"}]}" $(slack_webhook)
    displayName: Post Slack Notification on success
    condition: and(succeeded(), and(eq(variables['NOTIFY_SLACK'], '1'), ne(variables['RUN_TESTS'], '1')))
    timeoutInMinutes: 1

  - bash: |
      export BUILD_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${SYSTEM_TEAMPROJECT}/_build/results?buildId=${BUILD_BUILDID}"
      export MESSAGE="Build failed for *<$BUILD_URL|$BUILD_DEFINITIONNAME>* nightly build from *$BUILD_SOURCEBRANCHNAME*."
      curl -g -H "Content-Type: application/json" -X POST \
      -d "{\"text\": \"$MESSAGE\", \"attachments\": [{\"color\": \"#FC5C3C\",\"title\": \"$BUILD_DEFINITIONNAME nightly build results\",\"title_link\": \"$BUILD_URL\"}]}" $(slack_webhook)
    displayName: 'Post Slack Notification on Failure'
    condition: and(failed(), eq(variables['NOTIFY_SLACK'], '1'))
    timeoutInMinutes: 1

- job: run_tests
  displayName: Run Tests
  dependsOn: build
  # Run this job only if we are supposed to run the tests,
  # and we have an Electron built either in the previous job or in a different job, defined by a user.
  condition: and(eq(variables['RUN_TESTS'], '1'), or(succeeded(), ne(variables['Custom.UseArtifacts.BuildId'], '')))
  pool:
    vmImage: 'macOS-10.13'
  timeoutInMinutes: 20
  variables:
    CI: true
  steps:

  - task: CopyFiles@2
    displayName: Copy Electron sources to "src/electron"
    inputs:
      TargetFolder: src/electron
    timeoutInMinutes: 1

  # Use Electron built in the current job, if a user didn't specify a build id.
  - task: DownloadBuildArtifacts@0
    displayName: Download build artifacts from the current build
    inputs:
      buildType: 'current'
      artifactName: Default
    condition: and(succeeded(), eq(variables['Custom.UseArtifacts.BuildId'], ''))
    timeoutInMinutes: 1

  # Use Electron built in a different job.
  - task: DownloadBuildArtifacts@0
    displayName: Download build artifacts from a build specified by user
    inputs:
      buildType: 'specific'
      project: $(System.TeamProjectId)
      pipeline: $(Build.DefinitionName)
      buildVersionToDownload: 'specific'
      buildId: $(Custom.UseArtifacts.BuildId)
      artifactName: Default
    condition: and(succeeded(), ne(variables['Custom.UseArtifacts.BuildId'], ''))
    timeoutInMinutes: 1

  - bash: |
      TESTS_REVISION=$BUILD_SOURCEVERSION
      SOURCES_REVISION=`cat ${SYSTEM_ARTIFACTSDIRECTORY}/Default/revision`
      if [ $TESTS_REVISION != $SOURCES_REVISION ]; then
        echo "$TESTS_REVISION != $SOURCES_REVISION"
        exit 1
      fi
    displayName: Check if revisions of build sources and test sources match
    timeoutInMinutes: 1

  - task: ExtractFiles@1
    displayName: Extract Electron app
    inputs:
      archiveFilePatterns: $(System.ArtifactsDirectory)/Default/dist.zip
      destinationFolder: src/out/Default
    timeoutInMinutes: 1

  - task: ExtractFiles@1
    displayName: Extract ffmpeg
    inputs:
      archiveFilePatterns: $(System.ArtifactsDirectory)/Default/ffmpeg.zip
      destinationFolder: src/out/ffmpeg
    timeoutInMinutes: 1

  - task: ExtractFiles@1
    displayName: Extract mksnapshot
    inputs:
      archiveFilePatterns: $(System.ArtifactsDirectory)/Default/mksnapshot.zip
      destinationFolder: src/out/mksnapshot
    timeoutInMinutes: 1

  - task: CopyFiles@2
    displayName: 'Copy Files to: src/out/Default'
    inputs:
      SourceFolder: src/out/mksnapshot
      TargetFolder: src/out/Default
      OverWrite: true
    timeoutInMinutes: 1

  - task: ExtractFiles@1
    displayName: Extract Node.js headers
    inputs:
      archiveFilePatterns: $(System.ArtifactsDirectory)/Default/node_headers.tar.gz
      destinationFolder: src/out/Default/gen
    timeoutInMinutes: 1

  - task: NodeTool@0
    displayName: Install Node.js 10.x
    inputs:
      versionSpec: '10.x'
    timeoutInMinutes: 2  # Should take less than a minute.

  - bash: |
      cd src/electron
      npm install
    displayName: Install Node.js modules
    timeoutInMinutes: 4  # Should take about 30 seconds.

  - bash: |
      cd src
      python electron/script/verify-ffmpeg.py --source-root "$PWD" --build-dir out/Default --ffmpeg-path out/ffmpeg
    displayName: Verify non proprietary ffmpeg
    timeoutInMinutes: 5

  - bash: |
      cd src
      python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default
    displayName: Verify mksnapshot
    timeoutInMinutes: 5

  - bash: |
      if pgrep Electron; then
        killall Electron
      fi
      rm -rf ~/Library/Saved\ Application\ State/com.github.electron.savedState
      rm -rf ~/Library/Application\ Support/Electron
    displayName: Make sure Electron isn't running from previous tests
    timeoutInMinutes: 1

  - bash: |
      export ELECTRON_OUT_DIR=Default
      (cd src/electron && npm run test -- --ci)
    displayName: Run Electron test suite
    timeoutInMinutes: 10

  - bash: |
      cd src
      if [ ! -s "$MOCHA_FILE" ]; then
        exit 1
      fi
    displayName: Check test results existence
    timeoutInMinutes: 1

  - task: PublishTestResults@2
    displayName: Publish Test Results
    inputs:
      testResultsFiles: '*.xml'
      searchFolder: '$(System.DefaultWorkingDirectory)/src/junit/'
    condition: and(always(), eq(variables['MOCHA_FILE'], 'junit/test-results.xml'))
    timeoutInMinutes: 1

  - bash: |
      export BUILD_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${SYSTEM_TEAMPROJECT}/_build/results?buildId=${BUILD_BUILDID}"
      export MESSAGE="Build failed for *<$BUILD_URL|$BUILD_DEFINITIONNAME>* nightly build from *$BUILD_SOURCEBRANCHNAME*."
      curl -g -H "Content-Type: application/json" -X POST \
      -d "{\"text\": \"$MESSAGE\", \"attachments\": [{\"color\": \"#FC5C3C\",\"title\": \"$BUILD_DEFINITIONNAME nightly build results\",\"title_link\": \"$BUILD_URL\"}]}" $(slack_webhook)
    displayName: Post Slack Notification on failure
    condition: and(failed(), eq(variables['NOTIFY_SLACK'], '1'))
    timeoutInMinutes: 1

  - bash: |
      export BUILD_URL="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}${SYSTEM_TEAMPROJECT}/_build/results?buildId=${BUILD_BUILDID}"
      export MESSAGE="Build succeeded for *<$BUILD_URL|$BUILD_DEFINITIONNAME>* nightly build from *$BUILD_SOURCEBRANCHNAME*."
      curl -g -H "Content-Type: application/json" -X POST \
      -d "{\"text\": \"$MESSAGE\", \"attachments\": [{\"color\": \"good\",\"title\": \"$BUILD_DEFINITIONNAME nightly build results\",\"title_link\": \"$BUILD_URL\"}]}" $(slack_webhook)
    displayName: Post Slack Notification on success
    condition: and(succeeded(), eq(variables['NOTIFY_SLACK'], '1'))
    timeoutInMinutes: 1