build: speed up linux CI even more (#20241)

* build: yolo a theoretical speed improvement

* chore: persist src/electron and friends for testing purposes

* build: do not generate dist.zip on debug builds

* chore: ensure licenses and version file exist for zip-symbols script

.circleci/config.yml

@@ -32,6 +32,7 @@ machine-mac-large: &machine-mac-large
 # Build configurations options.
 env-debug-build: &env-debug-build
   GN_CONFIG: //electron/build/args/debug.gn
+  SKIP_DIST_ZIP: '1'
 
 env-testing-build: &env-testing-build
   GN_CONFIG: //electron/build/args/testing.gn
@@ -301,6 +302,7 @@ step-electron-dist-build: &step-electron-dist-build
     name: Build dist.zip
     command: |
       cd src
+      if [ "$SKIP_DIST_ZIP" != "1" ]; then
         ninja -C out/Default electron:electron_dist_zip
         if [ "$CHECK_DIST_MANIFEST" == "1" ]; then
           if [ "`uname`" == "Darwin" ]; then
@@ -324,6 +326,7 @@ step-electron-dist-build: &step-electron-dist-build
           fi
           electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.$target_cpu.manifest
         fi
+      fi
 
 step-electron-dist-store: &step-electron-dist-store
   store_artifacts:
@@ -378,6 +381,9 @@ step-persist-data-for-tests: &step-persist-data-for-tests
       - src/out/Default/mksnapshot.zip
       - src/out/Default/gen/node_headers
       - src/out/ffmpeg/ffmpeg.zip
+      - src/electron
+      - src/third_party/electron_node
+      - src/third_party/nan
 
 step-electron-dist-unzip: &step-electron-dist-unzip
   run:
@@ -465,7 +471,9 @@ step-mksnapshot-build: &step-mksnapshot-build
           electron/script/strip-binaries.py --file $PWD/out/Default/mksnapshot
         fi
       fi
+      if [ "$SKIP_DIST_ZIP" != "1" ]; then
         ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES
+      fi
 
 step-mksnapshot-store: &step-mksnapshot-store
   store_artifacts:
@@ -487,6 +495,8 @@ step-maybe-zip-symbols: &step-maybe-zip-symbols
     command: |
       cd src
       export BUILD_PATH="$PWD/out/Default"
+      ninja -C out/Default electron:licenses
+      ninja -C out/Default electron:electron_version
       electron/script/zip-symbols.py -b $BUILD_PATH
 
 step-maybe-cross-arch-snapshot: &step-maybe-cross-arch-snapshot
@@ -755,10 +765,16 @@ steps-electron-build: &steps-electron-build
         at: .
     - *step-depot-tools-add-to-path
     - *step-setup-env-for-build
+    - *step-restore-brew-cache
+    - *step-get-more-space-on-mac
+    - *step-install-npm-deps-on-mac
+    - *step-fix-sync-on-mac
     - *step-gn-gen-default
+    - *step-delete-git-directories
 
     # Electron app
     - *step-electron-build
+    - *step-maybe-electron-dist-strip
     - *step-electron-dist-build
     - *step-electron-dist-store
     - *step-ninja-summary
@@ -769,10 +785,54 @@ steps-electron-build: &steps-electron-build
 
     - *step-show-sccache-stats
 
-steps-electron-build-for-tests: &steps-electron-build-for-tests
+    # mksnapshot
+    - *step-mksnapshot-build
+    - *step-mksnapshot-store
+    - *step-maybe-cross-arch-snapshot
+    - *step-maybe-cross-arch-snapshot-store
+
+    # ffmpeg
+    - *step-ffmpeg-gn-gen
+    - *step-ffmpeg-build
+    - *step-ffmpeg-store
+
+    # Save all data needed for a further tests run.
+    - *step-persist-data-for-tests
+
+    - *step-maybe-generate-breakpad-symbols
+    - *step-maybe-zip-symbols
+
+    # Trigger tests on arm hardware if needed
+    - *step-maybe-trigger-arm-test
+
+    - *step-maybe-notify-slack-failure
+
+steps-electron-build-with-inline-checkout-for-tests: &steps-electron-build-with-inline-checkout-for-tests
   steps:
-    - attach_workspace:
+    # Checkout - Copied ffrom steps-checkout
-        at: .
+    - *step-checkout-electron
+    - *step-depot-tools-get
+    - *step-depot-tools-add-to-path
+    - *step-restore-brew-cache
+    - *step-get-more-space-on-mac
+    - *step-install-gnutar-on-mac
+    - *step-generate-deps-hash
+    - *step-touch-sync-done
+    - *step-maybe-restore-src-cache
+    - *step-maybe-restore-git-cache
+    - *step-set-git-cache-path
+    # This sync call only runs if .circle-sync-done is an EMPTY file
+    - *step-gclient-sync
+    # These next few steps reset Electron to the correct commit regardless of which cache was restored
+    - run:
+        name: Wipe Electron
+        command: rm -rf src/electron
+    - *step-checkout-electron
+    - *step-run-electron-only-hooks
+    - *step-generate-deps-hash-cleanly
+    - *step-mark-sync-done
+    - *step-minimize-workspace-size-from-checkout
+    
     - *step-depot-tools-add-to-path
     - *step-setup-env-for-build
     - *step-restore-brew-cache
@@ -1068,7 +1128,8 @@ jobs:
       <<: *env-debug-build
       <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    <<: *steps-electron-build-with-inline-checkout-for-tests
 
   linux-x64-debug-gn-check:
     <<: *machine-linux-medium
@@ -1084,7 +1145,8 @@ jobs:
       <<: *env-testing-build
       <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build-for-tests
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    <<: *steps-electron-build-with-inline-checkout-for-tests
 
   linux-x64-testing-no-run-as-node:
     <<: *machine-linux-2xlarge
@@ -1094,7 +1156,8 @@ jobs:
       <<: *env-enable-sccache
       <<: *env-ninja-status
       <<: *env-disable-run-as-node
-    <<: *steps-electron-build-for-tests
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    <<: *steps-electron-build-with-inline-checkout-for-tests
 
   linux-x64-testing-gn-check:
     <<: *machine-linux-medium
@@ -1120,7 +1183,7 @@ jobs:
       <<: *env-enable-sccache
       <<: *env-send-slack-notifications
       <<: *env-ninja-status
-    <<: *steps-electron-build-for-tests
+    <<: *steps-electron-build
 
   linux-x64-publish:
     <<: *machine-linux-2xlarge
@@ -1138,7 +1201,8 @@ jobs:
       <<: *env-debug-build
       <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    <<: *steps-electron-build-with-inline-checkout-for-tests
 
   linux-ia32-testing:
     <<: *machine-linux-2xlarge
@@ -1148,7 +1212,8 @@ jobs:
       <<: *env-testing-build
       <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build-for-tests
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    <<: *steps-electron-build-with-inline-checkout-for-tests
 
   linux-ia32-chromedriver:
     <<: *machine-linux-medium
@@ -1169,7 +1234,7 @@ jobs:
       <<: *env-enable-sccache
       <<: *env-send-slack-notifications
       <<: *env-ninja-status
-    <<: *steps-electron-build-for-tests
+    <<: *steps-electron-build
 
   linux-ia32-publish:
     <<: *machine-linux-2xlarge
@@ -1188,7 +1253,8 @@ jobs:
       <<: *env-debug-build
       <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    <<: *steps-electron-build-with-inline-checkout-for-tests
 
   linux-arm-testing:
     <<: *machine-linux-2xlarge
@@ -1199,7 +1265,8 @@ jobs:
       <<: *env-enable-sccache
       <<: *env-ninja-status
       TRIGGER_ARM_TEST: true
-    <<: *steps-electron-build-for-tests
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    <<: *steps-electron-build-with-inline-checkout-for-tests
 
   linux-arm-chromedriver:
     <<: *machine-linux-medium
@@ -1220,7 +1287,7 @@ jobs:
       <<: *env-enable-sccache
       <<: *env-send-slack-notifications
       <<: *env-ninja-status
-    <<: *steps-electron-build-for-tests
+    <<: *steps-electron-build
 
   linux-arm-publish:
     <<: *machine-linux-2xlarge
@@ -1239,7 +1306,8 @@ jobs:
       <<: *env-debug-build
       <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    <<: *steps-electron-build-with-inline-checkout-for-tests
 
   linux-arm64-debug-gn-check:
     <<: *machine-linux-medium
@@ -1258,7 +1326,8 @@ jobs:
       <<: *env-enable-sccache
       <<: *env-ninja-status
       TRIGGER_ARM_TEST: true
-    <<: *steps-electron-build-for-tests
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+    <<: *steps-electron-build-with-inline-checkout-for-tests
 
   linux-arm64-testing-gn-check:
     <<: *machine-linux-medium
@@ -1287,7 +1356,7 @@ jobs:
       <<: *env-enable-sccache
       <<: *env-send-slack-notifications
       <<: *env-ninja-status
-    <<: *steps-electron-build-for-tests
+    <<: *steps-electron-build
 
   linux-arm64-publish:
     <<: *machine-linux-2xlarge
@@ -1305,7 +1374,7 @@ jobs:
       <<: *env-testing-build
       <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build-for-tests
+    <<: *steps-electron-build
 
   osx-debug:
     <<: *machine-mac-large
@@ -1314,7 +1383,7 @@ jobs:
       <<: *env-debug-build
       <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build-for-tests    
+    <<: *steps-electron-build
 
   osx-debug-gn-check:
     <<: *machine-mac
@@ -1346,7 +1415,7 @@ jobs:
       <<: *env-release-build
       <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build-for-tests
+    <<: *steps-electron-build
 
   osx-publish:
     <<: *machine-mac-large
@@ -1364,7 +1433,7 @@ jobs:
       <<: *env-testing-build
       <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build-for-tests
+    <<: *steps-electron-build
 
   mas-debug:
     <<: *machine-mac-large
@@ -1374,7 +1443,7 @@ jobs:
       <<: *env-debug-build
       <<: *env-enable-sccache
       <<: *env-ninja-status
-    <<: *steps-electron-build-for-tests
+    <<: *steps-electron-build
 
   mas-debug-gn-check:
     <<: *machine-mac
@@ -1409,7 +1478,7 @@ jobs:
       <<: *env-release-build
       <<: *env-enable-sccache
       <<: *env-ninja-status      
-    <<: *steps-electron-build-for-tests
+    <<: *steps-electron-build
 
   mas-publish:
     <<: *machine-mac-large
@@ -1682,18 +1751,12 @@ workflows:
       - linux-checkout-fast
       - linux-checkout-and-save-cache
 
-      - linux-x64-debug:
+      - linux-x64-debug
-          requires:
-            - linux-checkout-fast
       - linux-x64-debug-gn-check:
           requires:
             - linux-checkout-fast
-      - linux-x64-testing:
+      - linux-x64-testing
-          requires:
+      - linux-x64-testing-no-run-as-node
-            - linux-checkout-fast
-      - linux-x64-testing-no-run-as-node:
-          requires:
-            - linux-checkout-fast
       - linux-x64-testing-gn-check:
           requires:
             - linux-checkout-fast
@@ -1707,12 +1770,8 @@ workflows:
           requires:
             - linux-x64-testing
 
-      - linux-ia32-debug:
+      - linux-ia32-debug
-          requires:
+      - linux-ia32-testing
-            - linux-checkout-fast
-      - linux-ia32-testing:
-          requires:
-            - linux-checkout-fast
       - linux-ia32-testing-tests:
           requires:
             - linux-ia32-testing
@@ -1723,22 +1782,14 @@ workflows:
           requires:
             - linux-ia32-testing
 
-      - linux-arm-debug:
+      - linux-arm-debug
-          requires:
+      - linux-arm-testing
-            - linux-checkout-fast
-      - linux-arm-testing:
-          requires:
-            - linux-checkout-fast
 
-      - linux-arm64-debug:
+      - linux-arm64-debug
-          requires:
-            - linux-checkout-fast
       - linux-arm64-debug-gn-check:
           requires:
             - linux-checkout-fast
-      - linux-arm64-testing:
+      - linux-arm64-testing
-          requires:
-            - linux-checkout-fast
       - linux-arm64-testing-gn-check:
           requires:
             - linux-checkout-fast