From f65db1df79a75841f552199ac4e4fcf6d0f08632 Mon Sep 17 00:00:00 2001 From: John Kleinschmidt Date: Mon, 16 Nov 2020 14:23:23 -0500 Subject: [PATCH] build: Use goma for all releases (#26476) * build: use goma for all release builds * Make sure goma is setup everywhere it is needed * Show ninja stats on release builds --- .circleci/config.yml | 155 ++++++++++--------------------------------- 1 file changed, 36 insertions(+), 119 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 92f89d4f281d..03e3979de4f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -158,10 +158,6 @@ env-mas-apple-silicon: &env-mas-apple-silicon TARGET_ARCH: arm64 USE_PREBUILT_V8_CONTEXT_SNAPSHOT: 1 -# Misc build configuration options. -env-enable-sccache: &env-enable-sccache - USE_SCCACHE: true - env-send-slack-notifications: &env-send-slack-notifications NOTIFY_SLACK: true @@ -295,22 +291,10 @@ step-setup-env-for-build: &step-setup-env-for-build # To find `gn` executable. echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV - if [ "$USE_SCCACHE" == "true" ]; then - # https://github.com/mozilla/sccache - SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache" - echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV - if [ "$CIRCLE_PR_NUMBER" != "" ]; then - #if building a fork set readonly access to sccache - echo 'export SCCACHE_BUCKET="electronjs-sccache-ci"' >> $BASH_ENV - echo 'export SCCACHE_TWO_TIER=true' >> $BASH_ENV - fi - fi - step-setup-goma-for-build: &step-setup-goma-for-build run: name: Setup Goma command: | - echo 'export USE_GOMA=true' >> $BASH_ENV if [ "`uname`" == "Linux" ]; then echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV else @@ -490,11 +474,7 @@ step-gn-gen-default: &step-gn-gen-default name: Default GN gen command: | cd src - if [ "$USE_GOMA" == "true" ]; then - gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" - else - gn gen out/Default --args="import(\"$GN_CONFIG\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" - fi + gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" step-gn-check: &step-gn-check run: @@ -534,12 +514,8 @@ step-electron-build: &step-electron-build gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args rm -rf out/Default/clang_x64_v8_arm64/obj - # Regenerate because we just deleted some ninja files - if [ "$USE_GOMA" == "true" ]; then - gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" - else - gn gen out/Default --args="import(\"$GN_CONFIG\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" - fi + # Regenerate because we just deleted some ninja files + gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" fi ninja -C out/Default electron -j $NUMBER_OF_NINJA_PROCESSES node electron/script/check-symlinks.js @@ -615,11 +591,7 @@ step-electron-maybe-chromedriver-gn-gen: &step-electron-maybe-chromedriver-gn-ge command: | cd src if [ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]; then - if [ "$USE_GOMA" == "true" ]; then - gn gen out/chromedriver --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") is_component_ffmpeg=false proprietary_codecs=false $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" - else - gn gen out/chromedriver --args="import(\"$GN_CONFIG\") cc_wrapper=\"$SCCACHE_PATH\" is_component_ffmpeg=false proprietary_codecs=false $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" - fi + gn gen out/chromedriver --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") is_component_ffmpeg=false proprietary_codecs=false $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" fi step-electron-chromedriver-build: &step-electron-chromedriver-build @@ -730,11 +702,7 @@ step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen name: ffmpeg GN gen command: | cd src - if [ "$USE_GOMA" == "true" ]; then - gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS" - else - gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS" - fi + gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS" step-ffmpeg-build: &step-ffmpeg-build run: @@ -777,21 +745,16 @@ step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing sh -e /etc/init.d/xvfb start fi -step-show-sccache-stats: &step-show-sccache-stats +step-show-goma-stats: &step-show-goma-stats run: shell: /bin/bash - name: Check sccache/goma stats after build - command: | - if [ "$SCCACHE_PATH" != "" ]; then - $SCCACHE_PATH -s - fi - if [ "$USE_GOMA" == "true" ]; then - set +e - set +o pipefail - $LOCAL_GOMA_DIR/goma_ctl.py stat - $LOCAL_GOMA_DIR/diagnose_goma_log.py - true - fi + name: Check goma stats after build + command: | + set +e + set +o pipefail + $LOCAL_GOMA_DIR/goma_ctl.py stat + $LOCAL_GOMA_DIR/diagnose_goma_log.py + true when: always step-mksnapshot-build: &step-mksnapshot-build @@ -1198,6 +1161,7 @@ steps-electron-gn-check: &steps-electron-gn-check - *step-maybe-early-exit-doc-only-change - *step-depot-tools-add-to-path - *step-setup-env-for-build + - *step-setup-goma-for-build - *step-gn-gen-default - *step-gn-check @@ -1231,6 +1195,7 @@ steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-cha - *step-depot-tools-add-to-path - *step-setup-env-for-build + - *step-setup-goma-for-build - *step-restore-brew-cache - *step-get-more-space-on-mac - *step-install-npm-deps-on-mac @@ -1240,26 +1205,13 @@ steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-cha #Compile ts/js to verify doc change didn't break anything - *step-ts-compile -steps-chromedriver-build: &steps-chromedriver-build - steps: - - attach_workspace: - at: . - - *step-depot-tools-add-to-path - - *step-setup-env-for-build - - *step-fix-sync-on-mac - - - *step-electron-maybe-chromedriver-gn-gen - - *step-electron-chromedriver-build - - *step-electron-chromedriver-store - - - *step-maybe-notify-slack-failure - steps-native-tests: &steps-native-tests steps: - attach_workspace: at: . - *step-depot-tools-add-to-path - *step-setup-env-for-build + - *step-setup-goma-for-build - *step-gn-gen-default - run: @@ -1267,7 +1219,7 @@ steps-native-tests: &steps-native-tests command: | cd src ninja -C out/Default $BUILD_TARGET - - *step-show-sccache-stats + - *step-show-goma-stats - *step-setup-linux-for-headless-testing - run: @@ -1572,7 +1524,7 @@ commands: - *step-nodejs-headers-build - *step-nodejs-headers-store - - *step-show-sccache-stats + - *step-show-goma-stats # mksnapshot - *step-mksnapshot-build @@ -1660,13 +1612,14 @@ commands: - *step-gclient-sync - *step-delete-git-directories - *step-minimize-workspace-size-from-checkout - - *step-fix-sync-on-mac + - *step-fix-sync-on-mac - *step-setup-env-for-build + - *step-setup-goma-for-build - *step-gn-gen-default # Electron app - *step-electron-build - - *step-show-sccache-stats + - *step-show-goma-stats - *step-maybe-generate-breakpad-symbols - *step-maybe-electron-dist-strip - *step-electron-dist-build @@ -1850,14 +1803,6 @@ jobs: <<: *env-testing-build <<: *steps-electron-gn-check - linux-x64-chromedriver: - <<: *machine-linux-medium - environment: - <<: *env-linux-medium - <<: *env-release-build - <<: *env-send-slack-notifications - <<: *steps-chromedriver-build - linux-x64-release: <<: *machine-linux-2xlarge environment: @@ -1877,8 +1822,8 @@ jobs: <<: *env-linux-2xlarge-release GCLIENT_EXTRA_ARGS: '--custom-var=checkout_requests=True' <<: *env-release-build - <<: *env-enable-sccache UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: false @@ -1889,8 +1834,8 @@ jobs: environment: <<: *env-linux-2xlarge-release <<: *env-release-build - <<: *env-enable-sccache UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: true @@ -1910,15 +1855,6 @@ jobs: checkout: true use-out-cache: false - linux-ia32-chromedriver: - <<: *machine-linux-medium - environment: - <<: *env-linux-medium - <<: *env-ia32 - <<: *env-release-build - <<: *env-send-slack-notifications - <<: *steps-chromedriver-build - linux-ia32-release: <<: *machine-linux-2xlarge environment: @@ -1940,9 +1876,9 @@ jobs: GCLIENT_EXTRA_ARGS: '--custom-var=checkout_requests=True' <<: *env-ia32 <<: *env-release-build - <<: *env-enable-sccache <<: *env-32bit-release UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: false @@ -1954,9 +1890,9 @@ jobs: <<: *env-linux-2xlarge-release <<: *env-ia32 <<: *env-release-build - <<: *env-enable-sccache <<: *env-32bit-release UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: true @@ -1977,15 +1913,6 @@ jobs: checkout: true use-out-cache: false - linux-arm-chromedriver: - <<: *machine-linux-medium - environment: - <<: *env-linux-medium - <<: *env-arm - <<: *env-release-build - <<: *env-send-slack-notifications - <<: *steps-chromedriver-build - linux-arm-release: <<: *machine-linux-2xlarge environment: @@ -2006,10 +1933,10 @@ jobs: <<: *env-linux-2xlarge-release <<: *env-arm <<: *env-release-build - <<: *env-enable-sccache <<: *env-32bit-release GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_requests=True' UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: false @@ -2021,9 +1948,9 @@ jobs: <<: *env-linux-2xlarge-release <<: *env-arm <<: *env-release-build - <<: *env-enable-sccache <<: *env-32bit-release UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: true @@ -2052,15 +1979,6 @@ jobs: <<: *env-testing-build <<: *steps-electron-gn-check - linux-arm64-chromedriver: - <<: *machine-linux-medium - environment: - <<: *env-linux-medium - <<: *env-arm64 - <<: *env-release-build - <<: *env-send-slack-notifications - <<: *steps-chromedriver-build - linux-arm64-release: <<: *machine-linux-2xlarge environment: @@ -2081,9 +1999,9 @@ jobs: <<: *env-linux-2xlarge-release <<: *env-arm64 <<: *env-release-build - <<: *env-enable-sccache GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm64=True --custom-var=checkout_requests=True' UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: false @@ -2095,8 +2013,8 @@ jobs: <<: *env-linux-2xlarge-release <<: *env-arm64 <<: *env-release-build - <<: *env-enable-sccache UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: true @@ -2143,9 +2061,9 @@ jobs: environment: <<: *env-mac-large-release <<: *env-release-build - <<: *env-enable-sccache GCLIENT_EXTRA_ARGS: '--custom-var=checkout_requests=True' UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: false @@ -2156,10 +2074,10 @@ jobs: environment: <<: *env-mac-large-release <<: *env-release-build - <<: *env-enable-sccache <<: *env-apple-silicon GCLIENT_EXTRA_ARGS: '--custom-var=checkout_requests=True' UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: false @@ -2170,8 +2088,8 @@ jobs: environment: <<: *env-mac-large-release <<: *env-release-build - <<: *env-enable-sccache UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: true @@ -2182,9 +2100,9 @@ jobs: environment: <<: *env-mac-large-release <<: *env-release-build - <<: *env-enable-sccache <<: *env-apple-silicon UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: true @@ -2251,9 +2169,9 @@ jobs: <<: *env-mac-large-release <<: *env-mas <<: *env-release-build - <<: *env-enable-sccache GCLIENT_EXTRA_ARGS: '--custom-var=checkout_requests=True' UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: false @@ -2265,9 +2183,9 @@ jobs: <<: *env-mac-large-release <<: *env-mas-apple-silicon <<: *env-release-build - <<: *env-enable-sccache GCLIENT_EXTRA_ARGS: '--custom-var=checkout_requests=True' UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: false @@ -2279,7 +2197,6 @@ jobs: <<: *env-mac-large-release <<: *env-mas <<: *env-release-build - <<: *env-enable-sccache UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> steps: - electron-publish: @@ -2292,8 +2209,8 @@ jobs: <<: *env-mac-large-release <<: *env-mas-apple-silicon <<: *env-release-build - <<: *env-enable-sccache UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >> + <<: *env-ninja-status steps: - electron-publish: attach: true