diff --git a/.circleci/config.yml b/.circleci/config.yml index 6470314b452b..6bbfadf3e2b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -228,15 +228,6 @@ step-setup-env-for-build: &step-setup-env-for-build command: | # To find `gn` executable. echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV - # Setup Goma if applicable - if [ ! -z "$RAW_GOMA_AUTH" ] && [ "`uname`" == "Linux" ]; then - export USE_SCCACHE="false" - echo 'export USE_SCCACHE=false' >> $BASH_ENV - echo 'export USE_GOMA=true' >> $BASH_ENV - echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV - echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config - src/electron/external_binaries/goma/goma_ctl.py ensure_start - fi if [ "$USE_SCCACHE" == "true" ]; then # https://github.com/mozilla/sccache @@ -249,6 +240,29 @@ step-setup-env-for-build: &step-setup-env-for-build fi fi +step-setup-goma-for-build: &step-setup-goma-for-build + run: + name: Setup Goma + command: | + echo 'export USE_GOMA=true' >> $BASH_ENV + if [ "`uname`" == "Linux" ]; then + echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV + else + echo 'export NUMBER_OF_NINJA_PROCESSES=25' >> $BASH_ENV + fi + if [ ! -z "$RAW_GOMA_AUTH" ]; then + echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config + fi + git clone https://github.com/electron/build-tools.git + cd build-tools + npm install + mkdir third_party + node -e "require('./src/utils/goma.js').downloadAndPrepare()" + node -e "require('./src/utils/goma.js').ensure()" + echo 'export GN_GOMA_FILE='`node -e "console.log(require('./src/utils/goma.js').gnFilePath)"` >> $BASH_ENV + echo 'export LOCAL_GOMA_DIR='`node -e "console.log(require('./src/utils/goma.js').dir)"` >> $BASH_ENV + cd .. + step-restore-brew-cache: &step-restore-brew-cache restore_cache: paths: @@ -339,7 +353,7 @@ step-gn-gen-default: &step-gn-gen-default command: | cd src if [ "$USE_GOMA" == "true" ]; then - gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"//electron/build/args/goma.gn\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" + gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" else gn gen out/Default --args="import(\"$GN_CONFIG\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" fi @@ -595,7 +609,7 @@ step-show-sccache-stats: &step-show-sccache-stats $SCCACHE_PATH -s fi if [ "$USE_GOMA" == "true" ]; then - src/electron/external_binaries/goma/goma_ctl.py stat + $LOCAL_GOMA_DIR/goma_ctl.py stat fi step-mksnapshot-build: &step-mksnapshot-build @@ -964,69 +978,6 @@ steps-electron-gn-check: &steps-electron-gn-check - *step-gn-gen-default - *step-gn-check -steps-electron-build: &steps-electron-build - steps: - - attach_workspace: - at: . - - *step-maybe-early-exit-doc-only-change - - *step-depot-tools-add-to-path - - *step-setup-env-for-build - - *step-restore-brew-cache - - *step-get-more-space-on-mac - - *step-install-npm-deps-on-mac - - *step-fix-sync-on-mac - - *step-delete-git-directories - - *step-gn-gen-default - - # Electron app - - *step-electron-build - - *step-maybe-electron-dist-strip - - *step-electron-dist-build - - *step-electron-dist-store - - *step-ninja-summary - - # Native test targets - - *step-native-unittests-build - - *step-native-unittests-store - - # Node.js headers - - *step-nodejs-headers-build - - *step-nodejs-headers-store - - - *step-show-sccache-stats - - # mksnapshot - - *step-mksnapshot-build - - *step-mksnapshot-store - - *step-maybe-cross-arch-snapshot - - *step-maybe-cross-arch-snapshot-store - - # chromedriver - - *step-electron-chromedriver-gn-gen - - *step-electron-chromedriver-build - - *step-electron-chromedriver-store - - # ffmpeg - - *step-ffmpeg-gn-gen - - *step-ffmpeg-build - - *step-ffmpeg-store - - # hunspell - - *step-hunspell-build - - *step-hunspell-store - - # Save all data needed for a further tests run. - - *step-persist-data-for-tests - - - *step-maybe-generate-breakpad-symbols - - *step-maybe-zip-symbols - - *step-symbols-store - - # Trigger tests on arm hardware if needed - - *step-maybe-trigger-arm-test - - - *step-maybe-notify-slack-failure - steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-change steps: # Checkout - Copied ffrom steps-checkout @@ -1394,10 +1345,10 @@ commands: steps: - *step-depot-tools-add-to-path - *step-setup-env-for-build + - *step-setup-goma-for-build - *step-get-more-space-on-mac - *step-fix-sync-on-mac - *step-delete-git-directories - - *step-gn-gen-default # Electron app - when: @@ -1569,7 +1520,6 @@ jobs: environment: <<: *env-linux-2xlarge <<: *env-testing-build - <<: *env-enable-sccache <<: *env-ninja-status GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' steps: @@ -1583,7 +1533,6 @@ jobs: environment: <<: *env-linux-2xlarge <<: *env-testing-build - <<: *env-enable-sccache <<: *env-ninja-status <<: *env-disable-run-as-node GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' @@ -1605,7 +1554,6 @@ jobs: environment: <<: *env-linux-medium <<: *env-release-build - <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *steps-chromedriver-build @@ -1614,10 +1562,13 @@ jobs: environment: <<: *env-linux-2xlarge-release <<: *env-release-build - <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *env-ninja-status - <<: *steps-electron-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: true + checkout: true linux-x64-publish: <<: *machine-linux-2xlarge @@ -1635,7 +1586,6 @@ jobs: <<: *env-linux-2xlarge <<: *env-ia32 <<: *env-testing-build - <<: *env-enable-sccache <<: *env-ninja-status GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' steps: @@ -1650,7 +1600,6 @@ jobs: <<: *env-linux-medium <<: *env-ia32 <<: *env-release-build - <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *steps-chromedriver-build @@ -1660,10 +1609,13 @@ jobs: <<: *env-linux-2xlarge-release <<: *env-ia32 <<: *env-release-build - <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *env-ninja-status - <<: *steps-electron-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: true + checkout: true linux-ia32-publish: <<: *machine-linux-2xlarge @@ -1683,7 +1635,6 @@ jobs: <<: *env-linux-2xlarge <<: *env-arm <<: *env-testing-build - <<: *env-enable-sccache <<: *env-ninja-status TRIGGER_ARM_TEST: true GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' @@ -1699,7 +1650,6 @@ jobs: <<: *env-linux-medium <<: *env-arm <<: *env-release-build - <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *steps-chromedriver-build @@ -1709,10 +1659,13 @@ jobs: <<: *env-linux-2xlarge-release <<: *env-arm <<: *env-release-build - <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *env-ninja-status - <<: *steps-electron-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: false + checkout: true linux-arm-publish: <<: *machine-linux-2xlarge @@ -1732,7 +1685,6 @@ jobs: <<: *env-linux-2xlarge <<: *env-arm64 <<: *env-testing-build - <<: *env-enable-sccache <<: *env-ninja-status TRIGGER_ARM_TEST: true GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' @@ -1756,7 +1708,6 @@ jobs: <<: *env-linux-medium <<: *env-arm64 <<: *env-release-build - <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *steps-chromedriver-build @@ -1766,10 +1717,13 @@ jobs: <<: *env-linux-2xlarge-release <<: *env-arm64 <<: *env-release-build - <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *env-ninja-status - <<: *steps-electron-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' + steps: + - electron-build: + persist: false + checkout: true linux-arm64-publish: <<: *machine-linux-2xlarge @@ -1787,7 +1741,6 @@ jobs: environment: <<: *env-mac-large <<: *env-testing-build - <<: *env-enable-sccache <<: *env-ninja-status <<: *env-macos-build GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' @@ -1810,7 +1763,6 @@ jobs: environment: <<: *env-machine-mac <<: *env-release-build - <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *steps-chromedriver-build @@ -1819,9 +1771,14 @@ jobs: environment: <<: *env-mac-large <<: *env-release-build - <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' + steps: + - electron-build: + persist: true + checkout: false + checkout-and-assume-cache: true + attach: false osx-publish: <<: *machine-mac-large @@ -1839,7 +1796,6 @@ jobs: <<: *env-mac-large <<: *env-mas <<: *env-testing-build - <<: *env-enable-sccache <<: *env-ninja-status <<: *env-macos-build GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' @@ -1863,7 +1819,6 @@ jobs: environment: <<: *env-machine-mac <<: *env-release-build - <<: *env-enable-sccache <<: *env-send-slack-notifications <<: *steps-chromedriver-build @@ -1873,10 +1828,15 @@ jobs: <<: *env-mac-large <<: *env-mas <<: *env-release-build - <<: *env-enable-sccache <<: *env-ninja-status - <<: *steps-electron-build - + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' + steps: + - electron-build: + persist: true + checkout: false + checkout-and-assume-cache: true + attach: false + mas-publish: <<: *machine-mac-large environment: @@ -1894,7 +1854,6 @@ jobs: environment: <<: *env-linux-2xlarge <<: *env-unittests - <<: *env-enable-sccache <<: *env-headless-testing <<: *steps-native-tests @@ -1903,7 +1862,6 @@ jobs: environment: <<: *env-linux-2xlarge <<: *env-unittests - <<: *env-enable-sccache <<: *env-headless-testing TESTS_ARGS: '--only-disabled-tests' <<: *steps-native-tests @@ -1913,7 +1871,6 @@ jobs: environment: <<: *env-linux-2xlarge <<: *env-unittests - <<: *env-enable-sccache <<: *env-headless-testing TESTS_ARGS: '--include-disabled-tests' <<: *steps-native-tests @@ -1924,7 +1881,6 @@ jobs: <<: *env-linux-2xlarge <<: *env-browsertests <<: *env-testing-build - <<: *env-enable-sccache <<: *env-headless-testing <<: *steps-native-tests @@ -2302,10 +2258,9 @@ workflows: - *chromium-upgrade-branches jobs: - linux-checkout-fast + - linux-checkout-and-save-cache - - linux-x64-release: - requires: - - linux-checkout-fast + - linux-x64-release - linux-x64-release-tests: requires: - linux-x64-release @@ -2329,9 +2284,7 @@ workflows: - linux-x64-verify-ffmpeg - linux-x64-chromedriver - - linux-ia32-release: - requires: - - linux-checkout-fast + - linux-ia32-release - linux-ia32-release-tests: requires: - linux-ia32-release @@ -2355,9 +2308,7 @@ workflows: - linux-ia32-verify-ffmpeg - linux-ia32-chromedriver - - linux-arm-release: - requires: - - linux-checkout-fast + - linux-arm-release - linux-arm-chromedriver: requires: - linux-checkout-fast @@ -2366,10 +2317,7 @@ workflows: - linux-arm-release - linux-arm-chromedriver - - - linux-arm64-release: - requires: - - linux-checkout-fast + - linux-arm64-release - linux-arm64-chromedriver: requires: - linux-checkout-fast @@ -2389,10 +2337,11 @@ workflows: - *chromium-upgrade-branches jobs: - mac-checkout-fast + - mac-checkout-and-save-cache - osx-release: requires: - - mac-checkout-fast + - mac-checkout-and-save-cache - osx-release-tests: requires: - osx-release @@ -2418,7 +2367,7 @@ workflows: - mas-release: requires: - - mac-checkout-fast + - mac-checkout-and-save-cache - mas-release-tests: requires: - mas-release diff --git a/appveyor.yml b/appveyor.yml index dd8a0d6114df..205620e4564b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -118,19 +118,29 @@ build_script: } } - ps: >- - if (Test-Path 'env:RAW_GOMA_AUTH') { - $env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config" - $env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE - .\src\electron\script\start-goma.ps1 -gomaDir "$pwd\src\electron\external_binaries\goma" + if ($env:GN_CONFIG -ne 'release') { + if (Test-Path 'env:RAW_GOMA_AUTH') { + $env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config" + $env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE + } + git clone https://github.com/electron/build-tools.git + cd build-tools + npm install + mkdir third_party + node -e "require('./src/utils/goma.js').downloadAndPrepare()" + $env:GN_GOMA_FILE = node -e "console.log(require('./src/utils/goma.js').gnFilePath)" + $env:LOCAL_GOMA_DIR = node -e "console.log(require('./src/utils/goma.js').dir)" + cd .. + .\src\electron\script\start-goma.ps1 -gomaDir $env:LOCAL_GOMA_DIR } - cd src - set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn - - if DEFINED RAW_GOMA_AUTH (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"//electron/build/args/goma.gn\") %GN_EXTRA_ARGS% ") else (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\"") + - if DEFINED GN_GOMA_FILE (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"%GN_GOMA_FILE%\") %GN_EXTRA_ARGS% ") else (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\"") - gn check out/Default //electron:electron_lib - gn check out/Default //electron:electron_app - gn check out/Default //electron:manifests - gn check out/Default //electron/shell/common/api:mojo - - if DEFINED RAW_GOMA_AUTH (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app) + - if DEFINED GN_GOMA_FILE (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app) - if "%GN_CONFIG%"=="testing" ( python C:\depot_tools\post_build_ninja_summary.py -C out\Default ) - gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%" - ninja -C out/ffmpeg electron:electron_ffmpeg_zip @@ -140,7 +150,7 @@ build_script: - ninja -C out/Default electron:hunspell_dictionaries_zip - ninja -C out/Default electron:electron_chromedriver_zip - ninja -C out/Default third_party/electron_node:headers - - if DEFINED RAW_GOMA_AUTH (python electron\external_binaries\goma\goma_ctl.py stat) + - if "%GN_CONFIG%"=="testing" ( python %LOCAL_GOMA_DIR%\goma_ctl.py stat ) - python electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json - appveyor PushArtifact out/Default/windows_toolchain_profile.json - appveyor PushArtifact out/Default/dist.zip diff --git a/script/start-goma.ps1 b/script/start-goma.ps1 index c8bf40c84b15..d53fc2965cd7 100755 --- a/script/start-goma.ps1 +++ b/script/start-goma.ps1 @@ -1,7 +1,6 @@ param([string]$gomaDir=$PWD) $cmdPath = Join-Path -Path $gomaDir -ChildPath "goma_ctl.py" Start-Process -FilePath cmd -ArgumentList "/C", "python", "$cmdPath", "ensure_start" -$env:GLOG_logtostderr = "true" $timedOut = $false; $waitTime = 0; $waitIncrement = 5; $timeout=120; Do { sleep $waitIncrement; $timedOut = (($waitTime+=$waitIncrement) -gt $timeout); iex "$gomaDir\gomacc.exe port 2" > $null; } Until(($LASTEXITCODE -eq 0) -or $timedOut) if ($timedOut) {