From 999f7df158f8eba7445745c9c6a89aa5b127707f Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Mon, 16 Sep 2019 09:31:08 -0700 Subject: [PATCH] build: speed up linux CI even more (#20241) * build: yolo a theoretical speed improvement * chore: persist src/electron and friends for testing purposes * build: do not generate dist.zip on debug builds * chore: ensure licenses and version file exist for zip-symbols script --- .circleci/config.yml | 187 +++++++++++++++++++++++++++---------------- 1 file changed, 119 insertions(+), 68 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e7c31eadd19c..bf296f5e7252 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,6 +32,7 @@ machine-mac-large: &machine-mac-large # Build configurations options. env-debug-build: &env-debug-build GN_CONFIG: //electron/build/args/debug.gn + SKIP_DIST_ZIP: '1' env-testing-build: &env-testing-build GN_CONFIG: //electron/build/args/testing.gn @@ -301,28 +302,30 @@ step-electron-dist-build: &step-electron-dist-build name: Build dist.zip command: | cd src - ninja -C out/Default electron:electron_dist_zip - if [ "$CHECK_DIST_MANIFEST" == "1" ]; then - if [ "`uname`" == "Darwin" ]; then - target_os=mac - target_cpu=x64 - if [ x"$MAS_BUILD" == x"true" ]; then - target_os=mac_mas - fi - elif [ "`uname`" == "Linux" ]; then - target_os=linux - if [ x"$TARGET_ARCH" == x ]; then + if [ "$SKIP_DIST_ZIP" != "1" ]; then + ninja -C out/Default electron:electron_dist_zip + if [ "$CHECK_DIST_MANIFEST" == "1" ]; then + if [ "`uname`" == "Darwin" ]; then + target_os=mac target_cpu=x64 - elif [ "$TARGET_ARCH" == "ia32" ]; then - target_cpu=x86 + if [ x"$MAS_BUILD" == x"true" ]; then + target_os=mac_mas + fi + elif [ "`uname`" == "Linux" ]; then + target_os=linux + if [ x"$TARGET_ARCH" == x ]; then + target_cpu=x64 + elif [ "$TARGET_ARCH" == "ia32" ]; then + target_cpu=x86 + else + target_cpu="$TARGET_ARCH" + fi else - target_cpu="$TARGET_ARCH" + echo "Unknown system: `uname`" + exit 1 fi - else - echo "Unknown system: `uname`" - exit 1 + electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.$target_cpu.manifest fi - electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.$target_cpu.manifest fi step-electron-dist-store: &step-electron-dist-store @@ -378,6 +381,9 @@ step-persist-data-for-tests: &step-persist-data-for-tests - src/out/Default/mksnapshot.zip - src/out/Default/gen/node_headers - src/out/ffmpeg/ffmpeg.zip + - src/electron + - src/third_party/electron_node + - src/third_party/nan step-electron-dist-unzip: &step-electron-dist-unzip run: @@ -465,7 +471,9 @@ step-mksnapshot-build: &step-mksnapshot-build electron/script/strip-binaries.py --file $PWD/out/Default/mksnapshot fi fi - ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES + if [ "$SKIP_DIST_ZIP" != "1" ]; then + ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES + fi step-mksnapshot-store: &step-mksnapshot-store store_artifacts: @@ -487,6 +495,8 @@ step-maybe-zip-symbols: &step-maybe-zip-symbols command: | cd src export BUILD_PATH="$PWD/out/Default" + ninja -C out/Default electron:licenses + ninja -C out/Default electron:electron_version electron/script/zip-symbols.py -b $BUILD_PATH step-maybe-cross-arch-snapshot: &step-maybe-cross-arch-snapshot @@ -755,10 +765,16 @@ steps-electron-build: &steps-electron-build at: . - *step-depot-tools-add-to-path - *step-setup-env-for-build + - *step-restore-brew-cache + - *step-get-more-space-on-mac + - *step-install-npm-deps-on-mac + - *step-fix-sync-on-mac - *step-gn-gen-default + - *step-delete-git-directories # Electron app - *step-electron-build + - *step-maybe-electron-dist-strip - *step-electron-dist-build - *step-electron-dist-store - *step-ninja-summary @@ -769,10 +785,54 @@ steps-electron-build: &steps-electron-build - *step-show-sccache-stats -steps-electron-build-for-tests: &steps-electron-build-for-tests + # mksnapshot + - *step-mksnapshot-build + - *step-mksnapshot-store + - *step-maybe-cross-arch-snapshot + - *step-maybe-cross-arch-snapshot-store + + # ffmpeg + - *step-ffmpeg-gn-gen + - *step-ffmpeg-build + - *step-ffmpeg-store + + # Save all data needed for a further tests run. + - *step-persist-data-for-tests + + - *step-maybe-generate-breakpad-symbols + - *step-maybe-zip-symbols + + # Trigger tests on arm hardware if needed + - *step-maybe-trigger-arm-test + + - *step-maybe-notify-slack-failure + +steps-electron-build-with-inline-checkout-for-tests: &steps-electron-build-with-inline-checkout-for-tests steps: - - attach_workspace: - at: . + # Checkout - Copied ffrom steps-checkout + - *step-checkout-electron + - *step-depot-tools-get + - *step-depot-tools-add-to-path + - *step-restore-brew-cache + - *step-get-more-space-on-mac + - *step-install-gnutar-on-mac + - *step-generate-deps-hash + - *step-touch-sync-done + - *step-maybe-restore-src-cache + - *step-maybe-restore-git-cache + - *step-set-git-cache-path + # This sync call only runs if .circle-sync-done is an EMPTY file + - *step-gclient-sync + # These next few steps reset Electron to the correct commit regardless of which cache was restored + - run: + name: Wipe Electron + command: rm -rf src/electron + - *step-checkout-electron + - *step-run-electron-only-hooks + - *step-generate-deps-hash-cleanly + - *step-mark-sync-done + - *step-minimize-workspace-size-from-checkout + - *step-depot-tools-add-to-path - *step-setup-env-for-build - *step-restore-brew-cache @@ -1068,7 +1128,8 @@ jobs: <<: *env-debug-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + <<: *steps-electron-build-with-inline-checkout-for-tests linux-x64-debug-gn-check: <<: *machine-linux-medium @@ -1084,7 +1145,8 @@ jobs: <<: *env-testing-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build-for-tests + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + <<: *steps-electron-build-with-inline-checkout-for-tests linux-x64-testing-no-run-as-node: <<: *machine-linux-2xlarge @@ -1094,7 +1156,8 @@ jobs: <<: *env-enable-sccache <<: *env-ninja-status <<: *env-disable-run-as-node - <<: *steps-electron-build-for-tests + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + <<: *steps-electron-build-with-inline-checkout-for-tests linux-x64-testing-gn-check: <<: *machine-linux-medium @@ -1120,7 +1183,7 @@ jobs: <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *env-ninja-status - <<: *steps-electron-build-for-tests + <<: *steps-electron-build linux-x64-publish: <<: *machine-linux-2xlarge @@ -1138,7 +1201,8 @@ jobs: <<: *env-debug-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + <<: *steps-electron-build-with-inline-checkout-for-tests linux-ia32-testing: <<: *machine-linux-2xlarge @@ -1148,7 +1212,8 @@ jobs: <<: *env-testing-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build-for-tests + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + <<: *steps-electron-build-with-inline-checkout-for-tests linux-ia32-chromedriver: <<: *machine-linux-medium @@ -1169,7 +1234,7 @@ jobs: <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *env-ninja-status - <<: *steps-electron-build-for-tests + <<: *steps-electron-build linux-ia32-publish: <<: *machine-linux-2xlarge @@ -1188,7 +1253,8 @@ jobs: <<: *env-debug-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + <<: *steps-electron-build-with-inline-checkout-for-tests linux-arm-testing: <<: *machine-linux-2xlarge @@ -1199,7 +1265,8 @@ jobs: <<: *env-enable-sccache <<: *env-ninja-status TRIGGER_ARM_TEST: true - <<: *steps-electron-build-for-tests + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + <<: *steps-electron-build-with-inline-checkout-for-tests linux-arm-chromedriver: <<: *machine-linux-medium @@ -1220,7 +1287,7 @@ jobs: <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *env-ninja-status - <<: *steps-electron-build-for-tests + <<: *steps-electron-build linux-arm-publish: <<: *machine-linux-2xlarge @@ -1239,7 +1306,8 @@ jobs: <<: *env-debug-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + <<: *steps-electron-build-with-inline-checkout-for-tests linux-arm64-debug-gn-check: <<: *machine-linux-medium @@ -1258,7 +1326,8 @@ jobs: <<: *env-enable-sccache <<: *env-ninja-status TRIGGER_ARM_TEST: true - <<: *steps-electron-build-for-tests + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + <<: *steps-electron-build-with-inline-checkout-for-tests linux-arm64-testing-gn-check: <<: *machine-linux-medium @@ -1287,7 +1356,7 @@ jobs: <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *env-ninja-status - <<: *steps-electron-build-for-tests + <<: *steps-electron-build linux-arm64-publish: <<: *machine-linux-2xlarge @@ -1305,7 +1374,7 @@ jobs: <<: *env-testing-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build-for-tests + <<: *steps-electron-build osx-debug: <<: *machine-mac-large @@ -1314,7 +1383,7 @@ jobs: <<: *env-debug-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build-for-tests + <<: *steps-electron-build osx-debug-gn-check: <<: *machine-mac @@ -1346,7 +1415,7 @@ jobs: <<: *env-release-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build-for-tests + <<: *steps-electron-build osx-publish: <<: *machine-mac-large @@ -1364,7 +1433,7 @@ jobs: <<: *env-testing-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build-for-tests + <<: *steps-electron-build mas-debug: <<: *machine-mac-large @@ -1374,7 +1443,7 @@ jobs: <<: *env-debug-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build-for-tests + <<: *steps-electron-build mas-debug-gn-check: <<: *machine-mac @@ -1409,7 +1478,7 @@ jobs: <<: *env-release-build <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build-for-tests + <<: *steps-electron-build mas-publish: <<: *machine-mac-large @@ -1682,18 +1751,12 @@ workflows: - linux-checkout-fast - linux-checkout-and-save-cache - - linux-x64-debug: - requires: - - linux-checkout-fast + - linux-x64-debug - linux-x64-debug-gn-check: requires: - linux-checkout-fast - - linux-x64-testing: - requires: - - linux-checkout-fast - - linux-x64-testing-no-run-as-node: - requires: - - linux-checkout-fast + - linux-x64-testing + - linux-x64-testing-no-run-as-node - linux-x64-testing-gn-check: requires: - linux-checkout-fast @@ -1707,12 +1770,8 @@ workflows: requires: - linux-x64-testing - - linux-ia32-debug: - requires: - - linux-checkout-fast - - linux-ia32-testing: - requires: - - linux-checkout-fast + - linux-ia32-debug + - linux-ia32-testing - linux-ia32-testing-tests: requires: - linux-ia32-testing @@ -1723,22 +1782,14 @@ workflows: requires: - linux-ia32-testing - - linux-arm-debug: - requires: - - linux-checkout-fast - - linux-arm-testing: - requires: - - linux-checkout-fast + - linux-arm-debug + - linux-arm-testing - - linux-arm64-debug: - requires: - - linux-checkout-fast + - linux-arm64-debug - linux-arm64-debug-gn-check: requires: - linux-checkout-fast - - linux-arm64-testing: - requires: - - linux-checkout-fast + - linux-arm64-testing - linux-arm64-testing-gn-check: requires: - linux-checkout-fast