ci: enable goma for all testing builds (#21992)

* ci: enable goma for all testing builds

* Use powershell script to start goma on Windows

* Only run ensure_start from PS script.

* add missing percent

* Run stats via command instead of powershell

* use command env var

* Use out cache for faster mac builds

* Disable unneeded logging

* .git directories need to be delete before gn gen

Even though gn gen shows and error about a missing .git directory, the .git delete needs to happen before gn gen.

.circleci/config.yml

@@ -228,15 +228,6 @@ step-setup-env-for-build: &step-setup-env-for-build
     command: |
       # To find `gn` executable.
       echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV
-      # Setup Goma if applicable
-      if [ ! -z "$RAW_GOMA_AUTH" ] && [ "`uname`" == "Linux" ]; then
-        export USE_SCCACHE="false"
-        echo 'export USE_SCCACHE=false' >> $BASH_ENV
-        echo 'export USE_GOMA=true' >> $BASH_ENV
-        echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV
-        echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config
-        src/electron/external_binaries/goma/goma_ctl.py ensure_start
-      fi
 
       if [ "$USE_SCCACHE" == "true" ]; then
         # https://github.com/mozilla/sccache
@@ -249,6 +240,29 @@ step-setup-env-for-build: &step-setup-env-for-build
         fi
       fi
 
+step-setup-goma-for-build: &step-setup-goma-for-build
+  run:
+    name: Setup Goma
+    command: |
+      echo 'export USE_GOMA=true' >> $BASH_ENV
+      if [ "`uname`" == "Linux" ]; then
+        echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV
+      else
+        echo 'export NUMBER_OF_NINJA_PROCESSES=25' >> $BASH_ENV
+      fi
+      if [ ! -z "$RAW_GOMA_AUTH" ]; then
+        echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config
+      fi
+      git clone https://github.com/electron/build-tools.git
+      cd build-tools
+      npm install
+      mkdir third_party
+      node -e "require('./src/utils/goma.js').downloadAndPrepare()"
+      node -e "require('./src/utils/goma.js').ensure()"
+      echo 'export GN_GOMA_FILE='`node -e "console.log(require('./src/utils/goma.js').gnFilePath)"` >> $BASH_ENV
+      echo 'export LOCAL_GOMA_DIR='`node -e "console.log(require('./src/utils/goma.js').dir)"` >> $BASH_ENV
+      cd ..
+
 step-restore-brew-cache: &step-restore-brew-cache
   restore_cache:
     paths:
@@ -339,7 +353,7 @@ step-gn-gen-default: &step-gn-gen-default
     command: |
       cd src
       if [ "$USE_GOMA" == "true" ]; then
-        gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"//electron/build/args/goma.gn\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
+        gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
       else
         gn gen out/Default --args="import(\"$GN_CONFIG\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
       fi
@@ -595,7 +609,7 @@ step-show-sccache-stats: &step-show-sccache-stats
         $SCCACHE_PATH -s
       fi
       if [ "$USE_GOMA" == "true" ]; then
-        src/electron/external_binaries/goma/goma_ctl.py stat
+        $LOCAL_GOMA_DIR/goma_ctl.py stat
       fi
 
 step-mksnapshot-build: &step-mksnapshot-build
@@ -964,69 +978,6 @@ steps-electron-gn-check: &steps-electron-gn-check
     - *step-gn-gen-default
     - *step-gn-check
 
-steps-electron-build: &steps-electron-build
-  steps:
-    - attach_workspace:
-        at: .
-    - *step-maybe-early-exit-doc-only-change
-    - *step-depot-tools-add-to-path
-    - *step-setup-env-for-build
-    - *step-restore-brew-cache
-    - *step-get-more-space-on-mac
-    - *step-install-npm-deps-on-mac
-    - *step-fix-sync-on-mac
-    - *step-delete-git-directories
-    - *step-gn-gen-default
-
-    # Electron app
-    - *step-electron-build
-    - *step-maybe-electron-dist-strip
-    - *step-electron-dist-build
-    - *step-electron-dist-store
-    - *step-ninja-summary
-
-    # Native test targets
-    - *step-native-unittests-build
-    - *step-native-unittests-store
-
-    # Node.js headers
-    - *step-nodejs-headers-build
-    - *step-nodejs-headers-store
-
-    - *step-show-sccache-stats
-
-    # mksnapshot
-    - *step-mksnapshot-build
-    - *step-mksnapshot-store
-    - *step-maybe-cross-arch-snapshot
-    - *step-maybe-cross-arch-snapshot-store
-
-    # chromedriver
-    - *step-electron-chromedriver-gn-gen
-    - *step-electron-chromedriver-build
-    - *step-electron-chromedriver-store
-
-    # ffmpeg
-    - *step-ffmpeg-gn-gen
-    - *step-ffmpeg-build
-    - *step-ffmpeg-store
-
-    # hunspell
-    - *step-hunspell-build
-    - *step-hunspell-store
-
-    # Save all data needed for a further tests run.
-    - *step-persist-data-for-tests
-
-    - *step-maybe-generate-breakpad-symbols
-    - *step-maybe-zip-symbols
-    - *step-symbols-store
-
-    # Trigger tests on arm hardware if needed
-    - *step-maybe-trigger-arm-test
-
-    - *step-maybe-notify-slack-failure
-
 steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-change
   steps:
     # Checkout - Copied ffrom steps-checkout
@@ -1394,10 +1345,10 @@ commands:
           steps:
             - *step-depot-tools-add-to-path
             - *step-setup-env-for-build
+            - *step-setup-goma-for-build
             - *step-get-more-space-on-mac
             - *step-fix-sync-on-mac
             - *step-delete-git-directories
-            - *step-gn-gen-default
 
             # Electron app
             - when:
@@ -1569,7 +1520,6 @@ jobs:
     environment:
       <<: *env-linux-2xlarge
       <<: *env-testing-build
-      <<: *env-enable-sccache
       <<: *env-ninja-status
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
     steps:
@@ -1583,7 +1533,6 @@ jobs:
     environment:
       <<: *env-linux-2xlarge
       <<: *env-testing-build
-      <<: *env-enable-sccache
       <<: *env-ninja-status
       <<: *env-disable-run-as-node
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
@@ -1605,7 +1554,6 @@ jobs:
     environment:
       <<: *env-linux-medium
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-send-slack-notifications
     <<: *steps-chromedriver-build
 
@@ -1614,10 +1562,13 @@ jobs:
     environment:
       <<: *env-linux-2xlarge-release
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-send-slack-notifications
       <<: *env-ninja-status
-    <<: *steps-electron-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    steps:
+      - electron-build:
+          persist: true
+          checkout: true
 
   linux-x64-publish:
     <<: *machine-linux-2xlarge
@@ -1635,7 +1586,6 @@ jobs:
       <<: *env-linux-2xlarge
       <<: *env-ia32
       <<: *env-testing-build
-      <<: *env-enable-sccache
       <<: *env-ninja-status
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
     steps:
@@ -1650,7 +1600,6 @@ jobs:
       <<: *env-linux-medium
       <<: *env-ia32
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-send-slack-notifications
     <<: *steps-chromedriver-build
 
@@ -1660,10 +1609,13 @@ jobs:
       <<: *env-linux-2xlarge-release
       <<: *env-ia32
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-send-slack-notifications
       <<: *env-ninja-status
-    <<: *steps-electron-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    steps:
+      - electron-build:
+          persist: true
+          checkout: true
 
   linux-ia32-publish:
     <<: *machine-linux-2xlarge
@@ -1683,7 +1635,6 @@ jobs:
       <<: *env-linux-2xlarge
       <<: *env-arm
       <<: *env-testing-build
-      <<: *env-enable-sccache
       <<: *env-ninja-status
       TRIGGER_ARM_TEST: true
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
@@ -1699,7 +1650,6 @@ jobs:
       <<: *env-linux-medium
       <<: *env-arm
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-send-slack-notifications
     <<: *steps-chromedriver-build
 
@@ -1709,10 +1659,13 @@ jobs:
       <<: *env-linux-2xlarge-release
       <<: *env-arm
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-send-slack-notifications
       <<: *env-ninja-status
-    <<: *steps-electron-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    steps:
+      - electron-build:
+          persist: false
+          checkout: true
 
   linux-arm-publish:
     <<: *machine-linux-2xlarge
@@ -1732,7 +1685,6 @@ jobs:
       <<: *env-linux-2xlarge
       <<: *env-arm64
       <<: *env-testing-build
-      <<: *env-enable-sccache
       <<: *env-ninja-status
       TRIGGER_ARM_TEST: true
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
@@ -1756,7 +1708,6 @@ jobs:
       <<: *env-linux-medium
       <<: *env-arm64
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-send-slack-notifications
     <<: *steps-chromedriver-build
 
@@ -1766,10 +1717,13 @@ jobs:
       <<: *env-linux-2xlarge-release
       <<: *env-arm64
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-send-slack-notifications
       <<: *env-ninja-status
-    <<: *steps-electron-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    steps:
+      - electron-build:
+          persist: false
+          checkout: true
 
   linux-arm64-publish:
     <<: *machine-linux-2xlarge
@@ -1787,7 +1741,6 @@ jobs:
     environment:
       <<: *env-mac-large
       <<: *env-testing-build
-      <<: *env-enable-sccache
       <<: *env-ninja-status
       <<: *env-macos-build
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
@@ -1810,7 +1763,6 @@ jobs:
     environment:
       <<: *env-machine-mac
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-send-slack-notifications
     <<: *steps-chromedriver-build
 
@@ -1819,9 +1771,14 @@ jobs:
     environment:
       <<: *env-mac-large
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
+    steps:
+      - electron-build:
+          persist: true
+          checkout: false
+          checkout-and-assume-cache: true
+          attach: false
 
   osx-publish:
     <<: *machine-mac-large
@@ -1839,7 +1796,6 @@ jobs:
       <<: *env-mac-large
       <<: *env-mas
       <<: *env-testing-build
-      <<: *env-enable-sccache
       <<: *env-ninja-status
       <<: *env-macos-build
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
@@ -1863,7 +1819,6 @@ jobs:
     environment:
       <<: *env-machine-mac
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-send-slack-notifications
     <<: *steps-chromedriver-build
 
@@ -1873,9 +1828,14 @@ jobs:
       <<: *env-mac-large
       <<: *env-mas
       <<: *env-release-build
-      <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
+    steps:
+      - electron-build:
+          persist: true
+          checkout: false
+          checkout-and-assume-cache: true
+          attach: false
   
   mas-publish:
     <<: *machine-mac-large
@@ -1894,7 +1854,6 @@ jobs:
     environment:
       <<: *env-linux-2xlarge
       <<: *env-unittests
-      <<: *env-enable-sccache
       <<: *env-headless-testing
     <<: *steps-native-tests
 
@@ -1903,7 +1862,6 @@ jobs:
     environment:
       <<: *env-linux-2xlarge
       <<: *env-unittests
-      <<: *env-enable-sccache
       <<: *env-headless-testing
       TESTS_ARGS: '--only-disabled-tests'
     <<: *steps-native-tests
@@ -1913,7 +1871,6 @@ jobs:
     environment:
       <<: *env-linux-2xlarge
       <<: *env-unittests
-      <<: *env-enable-sccache
       <<: *env-headless-testing
       TESTS_ARGS: '--include-disabled-tests'
     <<: *steps-native-tests
@@ -1924,7 +1881,6 @@ jobs:
       <<: *env-linux-2xlarge
       <<: *env-browsertests
       <<: *env-testing-build
-      <<: *env-enable-sccache
       <<: *env-headless-testing
     <<: *steps-native-tests
 
@@ -2302,10 +2258,9 @@ workflows:
                 - *chromium-upgrade-branches
     jobs:
       - linux-checkout-fast
+      - linux-checkout-and-save-cache
 
-      - linux-x64-release:
+      - linux-x64-release
-          requires:
-            - linux-checkout-fast
       - linux-x64-release-tests:
           requires:
             - linux-x64-release
@@ -2329,9 +2284,7 @@ workflows:
             - linux-x64-verify-ffmpeg
             - linux-x64-chromedriver
 
-      - linux-ia32-release:
+      - linux-ia32-release
-          requires:
-            - linux-checkout-fast
       - linux-ia32-release-tests:
           requires:
             - linux-ia32-release
@@ -2355,9 +2308,7 @@ workflows:
             - linux-ia32-verify-ffmpeg
             - linux-ia32-chromedriver
 
-      - linux-arm-release:
+      - linux-arm-release
-          requires:
-            - linux-checkout-fast
       - linux-arm-chromedriver:
           requires:
             - linux-checkout-fast
@@ -2366,10 +2317,7 @@ workflows:
             - linux-arm-release
             - linux-arm-chromedriver
 
-
+      - linux-arm64-release
-      - linux-arm64-release:
-          requires:
-            - linux-checkout-fast
       - linux-arm64-chromedriver:
           requires:
             - linux-checkout-fast
@@ -2389,10 +2337,11 @@ workflows:
                 - *chromium-upgrade-branches
     jobs:
       - mac-checkout-fast
+      - mac-checkout-and-save-cache
 
       - osx-release:
           requires:
-            - mac-checkout-fast
+            - mac-checkout-and-save-cache
       - osx-release-tests:
           requires:
             - osx-release
@@ -2418,7 +2367,7 @@ workflows:
 
       - mas-release:
           requires:
-            - mac-checkout-fast
+            - mac-checkout-and-save-cache
       - mas-release-tests:
           requires:
             - mas-release

appveyor.yml

@@ -118,19 +118,29 @@ build_script:
         }
       }
   - ps: >-
+      if ($env:GN_CONFIG -ne 'release') {
         if (Test-Path 'env:RAW_GOMA_AUTH') {
           $env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config"
           $env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE      
-        .\src\electron\script\start-goma.ps1 -gomaDir "$pwd\src\electron\external_binaries\goma"
+        }
+        git clone https://github.com/electron/build-tools.git
+        cd build-tools
+        npm install
+        mkdir third_party
+        node -e "require('./src/utils/goma.js').downloadAndPrepare()"
+        $env:GN_GOMA_FILE = node -e "console.log(require('./src/utils/goma.js').gnFilePath)"
+        $env:LOCAL_GOMA_DIR = node -e "console.log(require('./src/utils/goma.js').dir)"
+        cd ..
+        .\src\electron\script\start-goma.ps1 -gomaDir $env:LOCAL_GOMA_DIR
       }
   - cd src
   - set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn 
-  - if DEFINED RAW_GOMA_AUTH (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"//electron/build/args/goma.gn\") %GN_EXTRA_ARGS% ") else (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\"")
+  - if DEFINED GN_GOMA_FILE (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"%GN_GOMA_FILE%\") %GN_EXTRA_ARGS% ") else (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\"")
   - gn check out/Default //electron:electron_lib
   - gn check out/Default //electron:electron_app
   - gn check out/Default //electron:manifests
   - gn check out/Default //electron/shell/common/api:mojo
-  - if DEFINED RAW_GOMA_AUTH (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app)
+  - if DEFINED GN_GOMA_FILE (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app)
   - if "%GN_CONFIG%"=="testing" ( python C:\depot_tools\post_build_ninja_summary.py -C out\Default )
   - gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%"
   - ninja -C out/ffmpeg electron:electron_ffmpeg_zip
@@ -140,7 +150,7 @@ build_script:
   - ninja -C out/Default electron:hunspell_dictionaries_zip
   - ninja -C out/Default electron:electron_chromedriver_zip
   - ninja -C out/Default third_party/electron_node:headers
-  - if DEFINED RAW_GOMA_AUTH (python electron\external_binaries\goma\goma_ctl.py stat)
+  - if "%GN_CONFIG%"=="testing" ( python %LOCAL_GOMA_DIR%\goma_ctl.py stat )
   - python electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json
   - appveyor PushArtifact out/Default/windows_toolchain_profile.json
   - appveyor PushArtifact out/Default/dist.zip

script/start-goma.ps1

@@ -1,7 +1,6 @@
 param([string]$gomaDir=$PWD)
 $cmdPath = Join-Path -Path $gomaDir -ChildPath "goma_ctl.py" 
 Start-Process -FilePath cmd -ArgumentList "/C", "python", "$cmdPath", "ensure_start"
-$env:GLOG_logtostderr = "true"
 $timedOut = $false; $waitTime = 0; $waitIncrement = 5; $timeout=120;
 Do { sleep $waitIncrement; $timedOut = (($waitTime+=$waitIncrement) -gt $timeout); iex "$gomaDir\gomacc.exe port 2" > $null; } Until(($LASTEXITCODE -eq 0) -or $timedOut) 
 if ($timedOut) {