electron/.circleci/config.yml
Samuel Attard e9114b3c00
build: optimize the happy path when syncing on CI (#17827)
* build: optimize the happy path when syncing on CI

This adds a new cache for the "src" directory that is only ever used if
the cache key matches exactly.  If there is no exact match we fall back
to the old strategy of using the git cache.

On the happy path this can make the checkout on linux/macOS take around
5-6 minutes which is **significantly** faster than the original 15-18
minutes.

* build: sort readdir result to ensure stability

* build: increment cache key

* Update config.yml

* build: ensure that the cleanly checked out Electron has had hooks run on it

* build: do not remove deps/v8

* build: ensure clean git directory when generating deps hash

* chore: add comments to caching logic

* Update .circleci/config.yml

Co-Authored-By: MarshallOfSound <samuel.r.attard@gmail.com>
2019-04-22 15:36:59 -07:00

1686 lines
47 KiB
YAML

# The config expects the following environment variables to be set:
# - "SLACK_WEBHOOK" Slack hook URL to send notifications.
#
# The publishing scripts expect access tokens to be defined as env vars,
# but those are not covered here.
#
# CircleCI docs on variables:
# https://circleci.com/docs/2.0/env-vars/
# Build machines configs.
docker-image: &docker-image
docker:
- image: electronbuilds/electron:0.0.9
machine-linux-medium: &machine-linux-medium
<<: *docker-image
resource_class: medium
machine-linux-2xlarge: &machine-linux-2xlarge
<<: *docker-image
resource_class: 2xlarge
machine-mac: &machine-mac
macos:
xcode: "9.4.1"
machine-mac-large: &machine-mac-large
resource_class: large
macos:
xcode: "9.4.1"
# Build configurations options.
env-debug-build: &env-debug-build
GN_CONFIG: //electron/build/args/debug.gn
env-testing-build: &env-testing-build
GN_CONFIG: //electron/build/args/testing.gn
env-release-build: &env-release-build
GN_CONFIG: //electron/build/args/release.gn
STRIP_BINARIES: true
GENERATE_SYMBOLS: true
env-headless-testing: &env-headless-testing
DISPLAY: ':99.0'
env-stack-dumping: &env-stack-dumping
ELECTRON_ENABLE_STACK_DUMPING: '1'
env-browsertests: &env-browsertests
GN_CONFIG: //electron/build/args/native_tests.gn
BUILD_TARGET: electron/spec:chromium_browsertests
TESTS_CONFIG: src/electron/spec/configs/browsertests.yml
env-unittests: &env-unittests
GN_CONFIG: //electron/build/args/native_tests.gn
BUILD_TARGET: electron/spec:chromium_unittests
TESTS_CONFIG: src/electron/spec/configs/unittests.yml
# Build targets options.
env-ia32: &env-ia32
GN_EXTRA_ARGS: 'target_cpu = "x86"'
NPM_CONFIG_ARCH: ia32
TARGET_ARCH: ia32
env-arm: &env-arm
GN_EXTRA_ARGS: 'target_cpu = "arm"'
MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm
BUILD_NATIVE_MKSNAPSHOT: 1
TARGET_ARCH: arm
env-arm64: &env-arm64
GN_EXTRA_ARGS: 'target_cpu = "arm64" fatal_linker_warnings = false enable_linux_installer = false'
MKSNAPSHOT_TOOLCHAIN: //build/toolchain/linux:clang_arm64
BUILD_NATIVE_MKSNAPSHOT: 1
TARGET_ARCH: arm64
env-mas: &env-mas
GN_EXTRA_ARGS: 'is_mas_build = true'
MAS_BUILD: 'true'
# Misc build configuration options.
env-enable-sccache: &env-enable-sccache
USE_SCCACHE: true
env-send-slack-notifications: &env-send-slack-notifications
NOTIFY_SLACK: true
env-linux-medium: &env-linux-medium
NUMBER_OF_NINJA_PROCESSES: 3
env-linux-2xlarge: &env-linux-2xlarge
NUMBER_OF_NINJA_PROCESSES: 18
env-machine-mac: &env-machine-mac
NUMBER_OF_NINJA_PROCESSES: 6
env-mac-large: &env-mac-large
NUMBER_OF_NINJA_PROCESSES: 10
env-disable-crash-reporter-tests: &env-disable-crash-reporter-tests
DISABLE_CRASH_REPORTER_TESTS: true
# Individual (shared) steps.
step-maybe-notify-slack-failure: &step-maybe-notify-slack-failure
run:
name: Send a Slack notification on failure
command: |
if [ "$NOTIFY_SLACK" == "true" ]; then
export MESSAGE="Build failed for *<$CIRCLE_BUILD_URL|$CIRCLE_JOB>* nightly build from *$CIRCLE_BRANCH*."
curl -g -H "Content-Type: application/json" -X POST \
-d "{\"text\": \"$MESSAGE\", \"attachments\": [{\"color\": \"#FC5C3C\",\"title\": \"$CIRCLE_JOB nightly build results\",\"title_link\": \"$CIRCLE_BUILD_URL\"}]}" $SLACK_WEBHOOK
fi
when: on_fail
step-maybe-notify-slack-success: &step-maybe-notify-slack-success
run:
name: Send a Slack notification on success
command: |
if [ "$NOTIFY_SLACK" == "true" ]; then
export MESSAGE="Build succeeded for *<$CIRCLE_BUILD_URL|$CIRCLE_JOB>* nightly build from *$CIRCLE_BRANCH*."
curl -g -H "Content-Type: application/json" -X POST \
-d "{\"text\": \"$MESSAGE\", \"attachments\": [{\"color\": \"good\",\"title\": \"$CIRCLE_JOB nightly build results\",\"title_link\": \"$CIRCLE_BUILD_URL\"}]}" $SLACK_WEBHOOK
fi
when: on_success
step-checkout-electron: &step-checkout-electron
checkout:
path: src/electron
step-depot-tools-get: &step-depot-tools-get
run:
name: Get depot tools
command: |
git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
step-depot-tools-add-to-path: &step-depot-tools-add-to-path
run:
name: Add depot tools to PATH
command: echo 'export PATH="$PATH:'"$PWD"'/depot_tools"' >> $BASH_ENV
step-gclient-sync: &step-gclient-sync
run:
name: Gclient sync
command: |
# If we did not restore a complete sync then we need to sync for realz
if [ ! -s "src/electron/.circle-sync-done" ]; then
gclient config \
--name "src/electron" \
--unmanaged \
$GCLIENT_EXTRA_ARGS \
"$CIRCLE_REPOSITORY_URL"
gclient sync --with_branch_heads --with_tags
fi
step-setup-env-for-build: &step-setup-env-for-build
run:
name: Setup Environment Variables
command: |
# To find `gn` executable.
echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV
if [ "$USE_SCCACHE" == "true" ]; then
# https://github.com/mozilla/sccache
SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache"
echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV
if [ "$CIRCLE_PR_NUMBER" != "" ]; then
#if building a fork set readonly access to sccache
echo 'export SCCACHE_BUCKET="electronjs-sccache"' >> $BASH_ENV
echo 'export SCCACHE_TWO_TIER=true' >> $BASH_ENV
fi
fi
step-restore-brew-cache: &step-restore-brew-cache
restore_cache:
paths:
- /usr/local/Homebrew
keys:
- v1-brew-cache-{{ arch }}
# On macOS the npm install command during gclient sync was run on a linux
# machine and therefore installed a slightly different set of dependencies
# Notably "fsevents" is a macOS only dependency, we rerun npm install once
# we are on a macOS machine to get the correct state
step-install-npm-deps-on-mac: &step-install-npm-deps-on-mac
run:
name: Install NPM Dependencies on MacOS
command: |
if [ "`uname`" == "Darwin" ]; then
cd src/electron
npm install
fi
# This step handles the differences between the linux "gclient sync"
# and the expected state on macOS
step-fix-sync-on-mac: &step-fix-sync-on-mac
run:
name: Fix Sync on macOS
command: |
if [ "`uname`" == "Darwin" ]; then
# Fix Clang Install (wrong binary)
rm -rf src/third_party/llvm-build
python src/tools/clang/scripts/update.py
# Fix Framework Header Installs (symlinks not retained)
rm -rf src/electron/external_binaries
python src/electron/script/update-external-binaries.py
fi
step-install-signing-cert-on-mac: &step-install-signing-cert-on-mac
run:
name: Import and trust self-signed codesigning cert on MacOS
command: |
if [ "`uname`" == "Darwin" ]; then
cd src/electron
./script/codesign/import-testing-cert-ci.sh
fi
step-install-gnutar-on-mac: &step-install-gnutar-on-mac
run:
name: Install gnu-tar on macos
command: |
if [ "`uname`" == "Darwin" ]; then
brew update
brew install gnu-tar
ln -fs /usr/local/bin/gtar /usr/local/bin/tar
fi
step-gn-gen-default: &step-gn-gen-default
run:
name: Default GN gen
command: |
cd src
gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
step-gn-check: &step-gn-check
run:
name: GN check
command: |
cd src
gn check out/Default //electron:electron_lib
gn check out/Default //electron:electron_app
gn check out/Default //electron:manifests
gn check out/Default //electron/atom/common/api:mojo
step-electron-build: &step-electron-build
run:
name: Electron build
no_output_timeout: 30m
command: |
cd src
ninja -C out/Default electron -j $NUMBER_OF_NINJA_PROCESSES
step-maybe-electron-dist-strip: &step-maybe-electron-dist-strip
run:
name: Strip electron binaries
command: |
if [ "$STRIP_BINARIES" == "true" ] && [ "`uname`" != "Darwin" ]; then
cd src
electron/script/strip-binaries.py --target-cpu="$TARGET_ARCH"
fi
step-electron-dist-build: &step-electron-dist-build
run:
name: Build dist.zip
command: |
cd src
ninja -C out/Default electron:electron_dist_zip
step-electron-dist-store: &step-electron-dist-store
store_artifacts:
path: src/out/Default/dist.zip
destination: dist.zip
step-electron-chromedriver-build: &step-electron-chromedriver-build
run:
name: Build chromedriver.zip
command: |
cd src
ninja -C out/Default chrome/test/chromedriver -j $NUMBER_OF_NINJA_PROCESSES
electron/script/strip-binaries.py --target-cpu="$TARGET_ARCH" --file $PWD/out/Default/chromedriver
ninja -C out/Default electron:electron_chromedriver_zip
step-electron-chromedriver-store: &step-electron-chromedriver-store
store_artifacts:
path: src/out/Default/chromedriver.zip
destination: chromedriver.zip
step-nodejs-headers-build: &step-nodejs-headers-build
run:
name: Build Node.js headers
command: |
cd src
ninja -C out/Default third_party/electron_node:headers
step-nodejs-headers-store: &step-nodejs-headers-store
store_artifacts:
path: src/out/Default/gen/node_headers.tar.gz
destination: node_headers.tar.gz
step-electron-publish: &step-electron-publish
run:
name: Publish Electron Dist
command: |
cd src/electron
if [ "$UPLOAD_TO_S3" == "1" ]; then
echo 'Uploading Electron release distribution to S3'
script/upload.py --upload_to_s3
else
echo 'Uploading Electron release distribution to Github releases'
script/upload.py
fi
step-persist-data-for-tests: &step-persist-data-for-tests
persist_to_workspace:
root: .
paths:
# Build artifacts
- src/out/Default/dist.zip
- src/out/Default/mksnapshot.zip
- src/out/Default/gen/node_headers
- src/out/ffmpeg/ffmpeg.zip
step-electron-dist-unzip: &step-electron-dist-unzip
run:
name: Unzip dist.zip
command: |
cd src/out/Default
# -o overwrite files WITHOUT prompting
# TODO(alexeykuzmin): Remove '-o' when it's no longer needed.
unzip -o dist.zip
step-ffmpeg-unzip: &step-ffmpeg-unzip
run:
name: Unzip ffmpeg.zip
command: |
cd src/out/ffmpeg
unzip -o ffmpeg.zip
step-mksnapshot-unzip: &step-mksnapshot-unzip
run:
name: Unzip mksnapshot.zip
command: |
cd src/out/Default
unzip -o mksnapshot.zip
step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen
run:
name: ffmpeg GN gen
command: |
cd src
gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
step-ffmpeg-build: &step-ffmpeg-build
run:
name: Non proprietary ffmpeg build
command: |
cd src
ninja -C out/ffmpeg electron:electron_ffmpeg_zip -j $NUMBER_OF_NINJA_PROCESSES
step-verify-ffmpeg: &step-verify-ffmpeg
run:
name: Verify ffmpeg
command: |
cd src
python electron/script/verify-ffmpeg.py --source-root "$PWD" --build-dir out/Default --ffmpeg-path out/ffmpeg
step-ffmpeg-store: &step-ffmpeg-store
store_artifacts:
path: src/out/ffmpeg/ffmpeg.zip
destination: ffmpeg.zip
step-verify-mksnapshot: &step-verify-mksnapshot
run:
name: Verify mksnapshot
command: |
cd src
python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default
step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing
run:
name: Setup for headless testing
command: |
if [ "`uname`" != "Darwin" ]; then
sh -e /etc/init.d/xvfb start
fi
step-show-sccache-stats: &step-show-sccache-stats
run:
name: Check sccache stats after build
command: |
if [ "$SCCACHE_PATH" != "" ]; then
$SCCACHE_PATH -s
fi
step-mksnapshot-build: &step-mksnapshot-build
run:
name: mksnapshot build
command: |
cd src
if [ "`uname`" != "Darwin" ]; then
if [ "$TARGET_ARCH" == "arm" ]; then
electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/mksnapshot
elif [ "$TARGET_ARCH" == "arm64" ]; then
electron/script/strip-binaries.py --file $PWD/out/Default/clang_x64_v8_arm64/mksnapshot
else
electron/script/strip-binaries.py --file $PWD/out/Default/mksnapshot
fi
fi
ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES
step-mksnapshot-store: &step-mksnapshot-store
store_artifacts:
path: src/out/Default/mksnapshot.zip
destination: mksnapshot.zip
step-maybe-build-dump-syms: &step-maybe-build-dump-syms
run:
name: Build dump_syms binary
command: |
if [ "$GENERATE_SYMBOLS" == "true" ]; then
cd src
# Build needed dump_syms executable
ninja -C out/Default third_party/breakpad:dump_syms
fi
step-maybe-generate-breakpad-symbols: &step-maybe-generate-breakpad-symbols
run:
name: Generate breakpad symbols
command: |
if [ "$GENERATE_SYMBOLS" == "true" ]; then
cd src
export BUILD_PATH="$PWD/out/Default"
export DEST_PATH="$BUILD_PATH/breakpad_symbols"
electron/script/dump-symbols.py -b $BUILD_PATH -d $DEST_PATH -v
fi
step-maybe-zip-symbols: &step-maybe-zip-symbols
run:
name: Zip symbols
command: |
cd src
export BUILD_PATH="$PWD/out/Default"
electron/script/zip-symbols.py -b $BUILD_PATH
step-maybe-cross-arch-snapshot: &step-maybe-cross-arch-snapshot
run:
name: Generate cross arch snapshot (arm/arm64)
command: |
if [ "$TRIGGER_ARM_TEST" == "true" ] && [ -z "$CIRCLE_PR_NUMBER" ]; then
cd src
if [ "$TARGET_ARCH" == "arm" ]; then
export MKSNAPSHOT_PATH="clang_x86_v8_arm"
elif [ "$TARGET_ARCH" == "arm64" ]; then
export MKSNAPSHOT_PATH="clang_x64_v8_arm64"
fi
cp "out/Default/$MKSNAPSHOT_PATH/mksnapshot" out/Default
cp "out/Default/$MKSNAPSHOT_PATH/libffmpeg.so" out/Default
cp "out/Default/$MKSNAPSHOT_PATH/v8_context_snapshot_generator" out/Default
python electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --create-snapshot-only
mkdir cross-arch-snapshots
cp out/Default-mksnapshot-test/*.bin cross-arch-snapshots
fi
step-maybe-cross-arch-snapshot-store: &step-maybe-cross-arch-snapshot-store
store_artifacts:
path: src/cross-arch-snapshots
destination: cross-arch-snapshots
step-maybe-trigger-arm-test: &step-maybe-trigger-arm-test
run:
name: Trigger an arm test on VSTS if applicable
command: |
cd src
# Only run for non-fork prs
if [ "$TRIGGER_ARM_TEST" == "true" ] && [ -z "$CIRCLE_PR_NUMBER" ]; then
#Trigger VSTS job, passing along CircleCI job number and branch to build
echo "Triggering electron-$TARGET_ARCH-testing build on VSTS"
node electron/script/ci-release-build.js --job=electron-$TARGET_ARCH-testing --ci=VSTS --armTest --circleBuildNum=$CIRCLE_BUILD_NUM $CIRCLE_BRANCH
fi
step-maybe-generate-typescript-defs: &step-maybe-generate-typescript-defs
run:
name: Generate type declarations
command: |
if [ "`uname`" == "Darwin" ]; then
cd src/electron
npm run create-typescript-definitions
fi
step-fix-known-hosts-linux: &step-fix-known-hosts-linux
run:
name: Fix Known Hosts on Linux
command: |
if [ "`uname`" == "Linux" ]; then
./src/electron/.circleci/fix-known-hosts.sh
fi
# Lists of steps.
steps-lint: &steps-lint
steps:
- *step-checkout-electron
- run:
name: Setup third_party Depot Tools
command: |
# "depot_tools" has to be checkout into "//third_party/depot_tools" so pylint.py can a "pylintrc" file.
git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git src/third_party/depot_tools
echo 'export PATH="$PATH:'"$PWD"'/src/third_party/depot_tools"' >> $BASH_ENV
- run:
name: Download GN Binary
command: |
chromium_revision="$(grep -A1 chromium_version src/electron/DEPS | tr -d '\n' | cut -d\' -f4)"
gn_version="$(curl -sL "https://chromium.googlesource.com/chromium/src/+/${chromium_revision}/DEPS?format=TEXT" | base64 -d | grep gn_version | head -n1 | cut -d\' -f4)"
cipd ensure -ensure-file - -root . <<-CIPD
\$ServiceURL https://chrome-infra-packages.appspot.com/
@Subdir buildtools/linux64
gn/gn/linux-amd64 $gn_version
CIPD
echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/buildtools"' >> $BASH_ENV
- run:
name: Run Lint
command: |
# gn.py tries to find a gclient root folder starting from the current dir.
# When it fails and returns "None" path, the whole script fails. Let's "fix" it.
touch .gclient
# Another option would be to checkout "buildtools" inside the Electron checkout,
# but then we would lint its contents (at least gn format), and it doesn't pass it.
cd src/electron
npm install
npm run lint
steps-checkout: &steps-checkout
steps:
- *step-checkout-electron
- *step-depot-tools-get
- *step-depot-tools-add-to-path
- *step-restore-brew-cache
- *step-install-gnutar-on-mac
- run:
name: Generate DEPS Hash
command: node src/electron/script/generate-deps-hash.js
- run:
name: Touch Sync Done
command: touch src/electron/.circle-sync-done
# Restore exact src cache based on the hash of DEPS and patches/*
# If no cache is matched EXACTLY then the .circle-sync-done file is empty
# If a cache is matched EXACTLY then the .circle-sync-done file contains "done"
- restore_cache:
paths:
- ./src
keys:
- v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
name: Restoring src cache
# Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done
# If the src cache was restored above then this will match an empty cache
# If the src cache was not restored above then this will match a close git cache
- restore_cache:
paths:
- ~/.gclient-cache
keys:
- v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
- v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}
name: Conditionally restoring git cache
- run:
name: Set GIT_CACHE_PATH to make gclient to use the cache
command: |
# CircleCI does not support interpolation when setting environment variables.
# https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command
echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV
# This sync call only runs if .circle-sync-done is an EMPTY file
- *step-gclient-sync
# Persist the git cache based on the hash of DEPS and .circle-sync-done
# If the src cache was restored above then this will persist an empty cache
- save_cache:
paths:
- ~/.gclient-cache
key: v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
name: Persisting git cache
# These next few steps reset Electron to the correct commit regardless of which cache was restored
- run:
name: Wipe Electron
command: rm -rf src/electron
- *step-checkout-electron
- run:
name: Run Electron Only Hooks
command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
- run:
name: Generate DEPS Hash
command: (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js
# Mark the sync as done for future cache saving
- run:
name: Mark Sync Done
command: echo DONE > src/electron/.circle-sync-done
# Minimize the size of the cache
- run:
name: Remove some unused data to avoid storing it in the workspace/cache
command: |
rm -rf src/android_webview
rm -rf src/ios
rm -rf src/third_party/blink/web_tests
rm -rf src/third_party/blink/perf_tests
rm -rf src/third_party/hunspell_dictionaries
rm -rf src/third_party/WebKit/LayoutTests
# Save the src cache based on the deps hash
- save_cache:
paths:
- ./src
key: v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
name: Persisting src cache
- save_cache:
paths:
- /usr/local/Homebrew
key: v1-brew-cache-{{ arch }}
name: Persisting brew cache
- persist_to_workspace:
root: .
paths:
- depot_tools
- src
steps-electron-gn-check: &steps-electron-gn-check
steps:
- attach_workspace:
at: .
- *step-depot-tools-add-to-path
- *step-setup-env-for-build
- *step-gn-gen-default
- *step-gn-check
steps-electron-build: &steps-electron-build
steps:
- attach_workspace:
at: .
- *step-depot-tools-add-to-path
- *step-setup-env-for-build
- *step-gn-gen-default
# Electron app
- *step-electron-build
- *step-electron-dist-build
- *step-electron-dist-store
# Node.js headers
- *step-nodejs-headers-build
- *step-nodejs-headers-store
- *step-show-sccache-stats
steps-electron-build-for-tests: &steps-electron-build-for-tests
steps:
- attach_workspace:
at: .
- *step-depot-tools-add-to-path
- *step-setup-env-for-build
- *step-restore-brew-cache
- *step-install-npm-deps-on-mac
- *step-fix-sync-on-mac
- *step-gn-gen-default
# Electron app
- *step-electron-build
- *step-maybe-electron-dist-strip
- *step-electron-dist-build
- *step-electron-dist-store
# Node.js headers
- *step-nodejs-headers-build
- *step-nodejs-headers-store
- *step-show-sccache-stats
# mksnapshot
- *step-mksnapshot-build
- *step-mksnapshot-store
- *step-maybe-cross-arch-snapshot
- *step-maybe-cross-arch-snapshot-store
# ffmpeg
- *step-ffmpeg-gn-gen
- *step-ffmpeg-build
- *step-ffmpeg-store
# Save all data needed for a further tests run.
- *step-persist-data-for-tests
- *step-maybe-build-dump-syms
- *step-maybe-generate-breakpad-symbols
- *step-maybe-zip-symbols
# Trigger tests on arm hardware if needed
- *step-maybe-trigger-arm-test
- *step-maybe-notify-slack-failure
steps-electron-build-for-publish: &steps-electron-build-for-publish
steps:
- *step-checkout-electron
- *step-depot-tools-get
- *step-depot-tools-add-to-path
- *step-restore-brew-cache
- *step-gclient-sync
- *step-setup-env-for-build
- *step-gn-gen-default
# Electron app
- *step-electron-build
- *step-maybe-electron-dist-strip
- *step-electron-dist-build
- *step-electron-dist-store
- *step-maybe-build-dump-syms
- *step-maybe-generate-breakpad-symbols
- *step-maybe-zip-symbols
# mksnapshot
- *step-mksnapshot-build
- *step-mksnapshot-store
# chromedriver
- *step-electron-chromedriver-build
- *step-electron-chromedriver-store
# Node.js headers
- *step-nodejs-headers-build
- *step-nodejs-headers-store
# ffmpeg
- *step-ffmpeg-gn-gen
- *step-ffmpeg-build
- *step-ffmpeg-store
# typescript defs
- *step-maybe-generate-typescript-defs
# Publish
- *step-electron-publish
steps-chromedriver-build: &steps-chromedriver-build
steps:
- attach_workspace:
at: .
- *step-depot-tools-add-to-path
- *step-setup-env-for-build
- *step-fix-sync-on-mac
- *step-gn-gen-default
- *step-electron-chromedriver-build
- *step-electron-chromedriver-store
- *step-maybe-notify-slack-failure
steps-native-tests: &steps-native-tests
steps:
- attach_workspace:
at: .
- *step-depot-tools-add-to-path
- *step-setup-env-for-build
- *step-gn-gen-default
- run:
name: Build tests
command: |
cd src
ninja -C out/Default $BUILD_TARGET
- *step-show-sccache-stats
- *step-setup-linux-for-headless-testing
- run:
name: Run tests
command: |
mkdir test_results
python src/electron/script/native-tests.py run \
--config $TESTS_CONFIG \
--tests-dir src/out/Default \
--output-dir test_results \
$TESTS_ARGS
- store_artifacts:
path: test_results
destination: test_results # Put it in the root folder.
- store_test_results:
path: test_results
steps-verify-ffmpeg: &steps-verify-ffmpeg
steps:
- attach_workspace:
at: .
- *step-depot-tools-add-to-path
- *step-electron-dist-unzip
- *step-ffmpeg-unzip
- *step-setup-linux-for-headless-testing
- *step-verify-ffmpeg
- *step-maybe-notify-slack-failure
steps-verify-mksnapshot: &steps-verify-mksnapshot
steps:
- attach_workspace:
at: .
- *step-depot-tools-add-to-path
- *step-electron-dist-unzip
- *step-mksnapshot-unzip
- *step-setup-linux-for-headless-testing
- *step-verify-mksnapshot
- *step-maybe-notify-slack-failure
steps-tests: &steps-tests
steps:
- attach_workspace:
at: .
- *step-depot-tools-add-to-path
- *step-electron-dist-unzip
- *step-mksnapshot-unzip
- *step-setup-linux-for-headless-testing
- *step-restore-brew-cache
- *step-fix-known-hosts-linux
- *step-install-signing-cert-on-mac
- run:
name: Run Electron tests
environment:
MOCHA_REPORTER: mocha-multi-reporters
MOCHA_FILE: junit/test-results.xml
MOCHA_MULTI_REPORTERS: mocha-junit-reporter, tap
ELECTRON_DISABLE_SECURITY_WARNINGS: 1
command: |
cd src
export ELECTRON_OUT_DIR=Default
(cd electron && npm run test -- --ci --enable-logging)
- run:
name: Check test results existence
command: |
MOCHA_FILE='src/junit/test-results-remote.xml'
# Check if it exists and not empty.
if [ ! -s "$MOCHA_FILE" ]; then
exit 1
fi
MOCHA_FILE='src/junit/test-results-main.xml'
# Check if it exists and not empty.
if [ ! -s "$MOCHA_FILE" ]; then
exit 1
fi
- store_test_results:
path: src/junit
- *step-verify-mksnapshot
- *step-maybe-notify-slack-failure
chromium-upgrade-branches: &chromium-upgrade-branches
/chromium\-upgrade\/[0-9]+/
# List of all jobs.
version: 2
jobs:
# Layer 0: Lint. Standalone.
lint:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *steps-lint
# Layer 1: Checkout.
linux-checkout:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
<<: *steps-checkout
linux-checkout-for-native-tests:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_pyyaml=True'
<<: *steps-checkout
linux-checkout-for-native-tests-with-no-patches:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
GCLIENT_EXTRA_ARGS: '--custom-var=apply_patches=False --custom-var=checkout_pyyaml=True'
<<: *steps-checkout
mac-checkout:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
<<: *steps-checkout
# Layer 2: Builds.
linux-x64-debug:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-debug-build
<<: *env-enable-sccache
<<: *steps-electron-build
linux-x64-debug-gn-check:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-debug-build
<<: *steps-electron-gn-check
linux-x64-testing:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-testing-build
<<: *env-enable-sccache
<<: *steps-electron-build-for-tests
linux-x64-testing-gn-check:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-testing-build
<<: *steps-electron-gn-check
linux-x64-chromedriver:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
linux-x64-release:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-electron-build-for-tests
linux-x64-publish:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True'
<<: *env-release-build
<<: *steps-electron-build-for-publish
linux-ia32-debug:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-ia32
<<: *env-debug-build
<<: *env-enable-sccache
<<: *steps-electron-build
linux-ia32-testing:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-ia32
<<: *env-testing-build
<<: *env-enable-sccache
<<: *steps-electron-build-for-tests
linux-ia32-chromedriver:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-ia32
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
linux-ia32-release:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-ia32
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-electron-build-for-tests
linux-ia32-publish:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True'
<<: *env-ia32
<<: *env-release-build
<<: *steps-electron-build-for-publish
linux-arm-debug:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-arm
<<: *env-debug-build
<<: *env-enable-sccache
<<: *steps-electron-build
linux-arm-testing:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-arm
<<: *env-testing-build
<<: *env-enable-sccache
TRIGGER_ARM_TEST: true
<<: *steps-electron-build-for-tests
linux-arm-chromedriver:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-arm
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
linux-arm-release:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-arm
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-electron-build-for-tests
linux-arm-publish:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-arm
<<: *env-release-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_boto=True --custom-var=checkout_requests=True'
<<: *steps-electron-build-for-publish
linux-arm64-debug:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-arm64
<<: *env-debug-build
<<: *env-enable-sccache
<<: *steps-electron-build
linux-arm64-debug-gn-check:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-arm64
<<: *env-debug-build
<<: *steps-electron-gn-check
linux-arm64-testing:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-arm64
<<: *env-testing-build
<<: *env-enable-sccache
TRIGGER_ARM_TEST: true
<<: *steps-electron-build-for-tests
linux-arm64-testing-gn-check:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-arm64
<<: *env-testing-build
<<: *steps-electron-gn-check
linux-arm64-chromedriver:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-arm64
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
linux-arm64-release:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-arm64
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-electron-build-for-tests
linux-arm64-publish:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-arm64
<<: *env-release-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True --custom-var=checkout_boto=True --custom-var=checkout_requests=True'
<<: *steps-electron-build-for-publish
osx-testing:
<<: *machine-mac-large
environment:
<<: *env-mac-large
<<: *env-testing-build
<<: *env-enable-sccache
<<: *steps-electron-build-for-tests
osx-debug-gn-check:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-debug-build
<<: *steps-electron-gn-check
osx-testing-gn-check:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-testing-build
<<: *steps-electron-gn-check
osx-chromedriver:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
osx-release:
<<: *machine-mac-large
environment:
<<: *env-mac-large
<<: *env-release-build
<<: *env-enable-sccache
<<: *steps-electron-build-for-tests
osx-publish:
<<: *machine-mac-large
environment:
<<: *env-mac-large
<<: *env-release-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True'
<<: *steps-electron-build-for-publish
mas-testing:
<<: *machine-mac-large
environment:
<<: *env-mac-large
<<: *env-mas
<<: *env-testing-build
<<: *env-enable-sccache
<<: *steps-electron-build-for-tests
mas-debug-gn-check:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-mas
<<: *env-debug-build
<<: *steps-electron-gn-check
mas-testing-gn-check:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-mas
<<: *env-testing-build
<<: *steps-electron-gn-check
mas-chromedriver:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
mas-release:
<<: *machine-mac-large
environment:
<<: *env-mac-large
<<: *env-mas
<<: *env-release-build
<<: *env-enable-sccache
<<: *steps-electron-build-for-tests
mas-publish:
<<: *machine-mac-large
environment:
<<: *env-mac-large
<<: *env-mas
<<: *env-release-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_boto=True --custom-var=checkout_requests=True'
<<: *steps-electron-build-for-publish
# Layer 3: Tests.
linux-x64-unittests:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-unittests
<<: *env-enable-sccache
<<: *env-headless-testing
<<: *steps-native-tests
linux-x64-disabled-unittests:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-unittests
<<: *env-enable-sccache
<<: *env-headless-testing
TESTS_ARGS: '--only-disabled-tests'
<<: *steps-native-tests
linux-x64-chromium-unittests:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-unittests
<<: *env-enable-sccache
<<: *env-headless-testing
TESTS_ARGS: '--include-disabled-tests'
<<: *steps-native-tests
linux-x64-browsertests:
<<: *machine-linux-2xlarge
environment:
<<: *env-linux-2xlarge
<<: *env-browsertests
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-headless-testing
<<: *steps-native-tests
linux-x64-testing-tests:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-headless-testing
<<: *env-stack-dumping
<<: *steps-tests
linux-x64-release-tests:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-headless-testing
<<: *env-send-slack-notifications
<<: *steps-tests
linux-x64-verify-ffmpeg:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-headless-testing
<<: *env-send-slack-notifications
<<: *steps-verify-ffmpeg
linux-x64-verify-mksnapshot:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-headless-testing
<<: *env-send-slack-notifications
<<: *steps-verify-mksnapshot
linux-ia32-testing-tests:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-ia32
<<: *env-headless-testing
<<: *env-stack-dumping
<<: *steps-tests
linux-ia32-release-tests:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-ia32
<<: *env-headless-testing
<<: *env-send-slack-notifications
<<: *steps-tests
linux-ia32-verify-ffmpeg:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-ia32
<<: *env-headless-testing
<<: *env-send-slack-notifications
<<: *steps-verify-ffmpeg
linux-ia32-verify-mksnapshot:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-ia32
<<: *env-headless-testing
<<: *env-send-slack-notifications
<<: *steps-verify-mksnapshot
osx-testing-tests:
<<: *machine-mac-large
environment:
<<: *env-mac-large
<<: *env-stack-dumping
<<: *env-disable-crash-reporter-tests
<<: *steps-tests
osx-release-tests:
<<: *machine-mac-large
environment:
<<: *env-mac-large
<<: *env-stack-dumping
<<: *env-send-slack-notifications
<<: *env-disable-crash-reporter-tests
<<: *steps-tests
osx-verify-ffmpeg:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-send-slack-notifications
<<: *steps-verify-ffmpeg
osx-verify-mksnapshot:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-send-slack-notifications
<<: *steps-verify-mksnapshot
mas-testing-tests:
<<: *machine-mac-large
environment:
<<: *env-mac-large
<<: *env-stack-dumping
<<: *steps-tests
mas-release-tests:
<<: *machine-mac-large
environment:
<<: *env-mac-large
<<: *env-stack-dumping
<<: *env-send-slack-notifications
<<: *steps-tests
mas-verify-ffmpeg:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-send-slack-notifications
<<: *steps-verify-ffmpeg
mas-verify-mksnapshot:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-send-slack-notifications
<<: *steps-verify-mksnapshot
# Layer 4: Summary.
linux-x64-release-summary:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-send-slack-notifications
steps:
- *step-maybe-notify-slack-success
linux-ia32-release-summary:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-send-slack-notifications
steps:
- *step-maybe-notify-slack-success
linux-arm-release-summary:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-send-slack-notifications
steps:
- *step-maybe-notify-slack-success
linux-arm64-release-summary:
<<: *machine-linux-medium
environment:
<<: *env-linux-medium
<<: *env-send-slack-notifications
steps:
- *step-maybe-notify-slack-success
mas-release-summary:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-send-slack-notifications
steps:
- *step-maybe-notify-slack-success
osx-release-summary:
<<: *machine-mac
environment:
<<: *env-machine-mac
<<: *env-send-slack-notifications
steps:
- *step-maybe-notify-slack-success
workflows:
version: 2
lint:
jobs:
- lint
build-linux:
jobs:
- linux-checkout
- linux-x64-debug:
requires:
- linux-checkout
- linux-x64-debug-gn-check:
requires:
- linux-checkout
- linux-x64-testing:
requires:
- linux-checkout
- linux-x64-testing-gn-check:
requires:
- linux-checkout
- linux-x64-testing-tests:
requires:
- linux-x64-testing
- linux-ia32-debug:
requires:
- linux-checkout
- linux-ia32-testing:
requires:
- linux-checkout
- linux-ia32-testing-tests:
requires:
- linux-ia32-testing
- linux-arm-debug:
requires:
- linux-checkout
- linux-arm-testing:
requires:
- linux-checkout
- linux-arm64-debug:
requires:
- linux-checkout
- linux-arm64-debug-gn-check:
requires:
- linux-checkout
- linux-arm64-testing:
requires:
- linux-checkout
- linux-arm64-testing-gn-check:
requires:
- linux-checkout
build-mac:
jobs:
- mac-checkout
- osx-testing:
requires:
- mac-checkout
- osx-debug-gn-check:
requires:
- mac-checkout
- osx-testing-gn-check:
requires:
- mac-checkout
- osx-testing-tests:
requires:
- osx-testing
- mas-testing:
requires:
- mac-checkout
- mas-debug-gn-check:
requires:
- mac-checkout
- mas-testing-gn-check:
requires:
- mac-checkout
- mas-testing-tests:
requires:
- mas-testing
nightly-linux-release-test:
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- master
- *chromium-upgrade-branches
jobs:
- linux-checkout
- linux-x64-release:
requires:
- linux-checkout
- linux-x64-release-tests:
requires:
- linux-x64-release
- linux-x64-verify-ffmpeg:
requires:
- linux-x64-release
- linux-x64-verify-mksnapshot:
requires:
- linux-x64-release
- linux-x64-chromedriver:
requires:
- linux-checkout
- linux-x64-release-summary:
requires:
- linux-x64-release
- linux-x64-release-tests
- linux-x64-verify-ffmpeg
- linux-x64-chromedriver
- linux-ia32-release:
requires:
- linux-checkout
- linux-ia32-release-tests:
requires:
- linux-ia32-release
- linux-ia32-verify-ffmpeg:
requires:
- linux-ia32-release
- linux-ia32-verify-mksnapshot:
requires:
- linux-ia32-release
- linux-ia32-chromedriver:
requires:
- linux-checkout
- linux-ia32-release-summary:
requires:
- linux-ia32-release
- linux-ia32-release-tests
- linux-ia32-verify-ffmpeg
- linux-ia32-chromedriver
- linux-arm-release:
requires:
- linux-checkout
- linux-arm-chromedriver:
requires:
- linux-checkout
- linux-arm-release-summary:
requires:
- linux-arm-release
- linux-arm-chromedriver
- linux-arm64-release:
requires:
- linux-checkout
- linux-arm64-chromedriver:
requires:
- linux-checkout
- linux-arm64-release-summary:
requires:
- linux-arm64-release
- linux-arm64-chromedriver
nightly-mac-release-test:
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- master
- *chromium-upgrade-branches
jobs:
- mac-checkout
- osx-release:
requires:
- mac-checkout
- osx-release-tests:
requires:
- osx-release
- osx-verify-ffmpeg:
requires:
- osx-release
- osx-verify-mksnapshot:
requires:
- osx-release
- osx-chromedriver:
requires:
- mac-checkout
- osx-release-summary:
requires:
- osx-release
- osx-release-tests
- osx-verify-ffmpeg
- osx-chromedriver
- mas-release:
requires:
- mac-checkout
- mas-release-tests:
requires:
- mas-release
- mas-verify-ffmpeg:
requires:
- mas-release
- mas-verify-mksnapshot:
requires:
- mas-release
- mas-chromedriver:
requires:
- mac-checkout
- mas-release-summary:
requires:
- mas-release
- mas-release-tests
- mas-verify-ffmpeg
- mas-chromedriver
# Various slow and non-essential checks we run only nightly.
# Sanitizer jobs should be added here.
linux-checks-nightly:
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- master
- *chromium-upgrade-branches
jobs:
- linux-checkout-for-native-tests
# TODO(alexeykuzmin): Enable it back.
# Tons of crashes right now, see
# https://circleci.com/gh/electron/electron/67463
# - linux-x64-browsertests:
# requires:
# - linux-checkout-for-native-tests
- linux-x64-unittests:
requires:
- linux-checkout-for-native-tests
- linux-x64-disabled-unittests:
requires:
- linux-checkout-for-native-tests
- linux-checkout-for-native-tests-with-no-patches
- linux-x64-chromium-unittests:
requires:
- linux-checkout-for-native-tests-with-no-patches