ci: refactor the CircleCI config (#14891)

- explicitly mark jobs that should use sccache
 - explicitly mark jobs that should send Slack notifications
 - remove unnecessary env vars and checks
 - build and verify ffmpeg, and build native mksnapshot nightly
 - add "native" to the native mksnapshot jobs names
 - minor changes

.circleci/config.yml

@@ -24,11 +24,6 @@ env-testing-build: &env-testing-build
 
 env-release-build: &env-release-build
   GN_CONFIG: //electron/build/args/release.gn
-  NOTIFY_SLACK: true
-
-env-publish-build: &env-publish-build
-  GN_CONFIG: //electron/build/args/release.gn
-  ELECTRON_RELEASE: 1
 
 env-browsertests: &env-browsertests
   BUILD_TARGET: electron:chromium_browsertests
@@ -61,6 +56,13 @@ env-arm64: &env-arm64
 env-mas: &env-mas
   GN_EXTRA_ARGS: 'is_mas_build = true'
 
+# Misc build configuration options.
+env-enable-sccache: &env-enable-sccache
+  USE_SCCACHE: true
+
+env-send-slack-notifications: &env-send-slack-notifications
+  NOTIFY_SLACK: true
+
 # Individual (shared) steps.
 step-maybe-notify-slack-failure: &step-maybe-notify-slack-failure
   run:
@@ -118,7 +120,7 @@ step-setup-env-for-build: &step-setup-env-for-build
       # To find `gn` executable.
       echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV
 
-      if [ "$ELECTRON_RELEASE" != "1" ]; then
+      if [ "$USE_SCCACHE" == "true" ]; then
         # https://github.com/mozilla/sccache
         SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache"
         echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV
@@ -138,11 +140,7 @@ step-electron-gn-gen: &step-electron-gn-gen
     name: Electron GN gen
     command: |
       cd src
-      if [ "$ELECTRON_RELEASE" == "1" ]; then
+      gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
-        gn gen out/Default --args='import("'$GN_CONFIG'")'" $GN_EXTRA_ARGS"
-      else
-        gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
-      fi
 
 step-electron-build: &step-electron-build
   run:
@@ -187,14 +185,12 @@ step-electron-publish: &step-electron-publish
     name: Publish Electron Dist
     command: |
       cd src/electron
-      if [ "$ELECTRON_RELEASE" == "1" ]; then
+      if [ "$UPLOAD_TO_S3" == "1" ]; then
-        if [ "$UPLOAD_TO_S3" != "1" ]; then
+        echo 'Uploading Electron release distribution to S3'
-          echo 'Uploading Electron release distribution to github releases'
+        script/upload.py --upload_to_s3
-          script/upload.py
+      else
-        else
+        echo 'Uploading Electron release distribution to Github releases'
-          echo 'Uploading Electron release distribution to s3'
+        script/upload.py
-          script/upload.py --upload_to_s3
-        fi
       fi
 
 step-persist-data-for-tests: &step-persist-data-for-tests
@@ -223,11 +219,7 @@ step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen
     name: ffmpeg GN gen
     command: |
       cd src
-      if [ "$ELECTRON_RELEASE" == "1" ]; then
+      gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
-        gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn")'" $GN_EXTRA_ARGS"
-      else
-        gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS"
-      fi
 
 step-ffmpeg-build: &step-ffmpeg-build
   run:
@@ -260,7 +252,10 @@ step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing
 step-show-sccache-stats: &step-show-sccache-stats
   run:
     name: Check sccache stats after build
-    command: $SCCACHE_PATH -s
+    command: |
+      if [ "$SCCACHE_PATH" != "" ]; then
+        $SCCACHE_PATH -s
+      fi
 
 step-mksnapshot-build: &step-mksnapshot-build
   run:
@@ -284,36 +279,31 @@ step-generate-breakpad_symbols: &step-generate-breakpad_symbols
       electron/script/dump-symbols.py -d "$PWD/out/Default/electron.breakpad.syms"
       electron/script/zip-symbols.py
 
-step-native-mksnapshot-gn-gen: &step-native-mksnapshot-gn-gen
+step-maybe-native-mksnapshot-gn-gen: &step-maybe-native-mksnapshot-gn-gen
   run:
-    name: native mksnapshot GN gen (if needed)
+    name: Native mksnapshot GN gen (arm/arm64)
     command: |
       if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then
         cd src
-        if [ "$ELECTRON_RELEASE" == "1" ]; then
+        gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") cc_wrapper="'"$SCCACHE_PATH"'" v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS"
-          gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS"
-        else
-          gn gen out/native_mksnapshot --args='import("//electron/build/args/native_mksnapshot.gn") cc_wrapper="'"$SCCACHE_PATH"'" v8_snapshot_toolchain="'"$MKSNAPSHOT_TOOLCHAIN"'"'" $GN_EXTRA_ARGS"
-        fi
       else
-        echo 'skipping native mksnapshot GN gen for non arm build'
+        echo 'Skipping native mksnapshot GN gen for non arm build'
       fi
 
-step-native-mksnapshot-build: &step-native-mksnapshot-build
+step-maybe-native-mksnapshot-build: &step-maybe-native-mksnapshot-build
   run:
-    name: native mksnapshot (arm/arm64) build
+    name: Native mksnapshot build (arm/arm64)
     command: |
       if [ "$BUILD_NATIVE_MKSNAPSHOT" == "1" ]; then
         cd src
         ninja -C out/native_mksnapshot electron:electron_mksnapshot_zip
-        cp out/native_mksnapshot/mksnapshot.zip out/native_mksnapshot/native_mksnapshot.zip
       else
-        echo 'skipping native mksnapshot build for non arm build'
+        echo 'Skipping native mksnapshot build for non arm build'
       fi
 
-step-native-mksnapshot-store: &step-native-mksnapshot-store
+step-maybe-native-mksnapshot-store: &step-maybe-native-mksnapshot-store
   store_artifacts:
-    path: src/out/native_mksnapshot/native_mksnapshot.zip
+    path: src/out/native_mksnapshot/mksnapshot.zip
     destination: native_mksnapshot.zip
 
 # Lists of steps.
@@ -418,9 +408,9 @@ steps-electron-build-for-publish: &steps-electron-build-for-publish
     - *step-mksnapshot-store
 
     # native_mksnapshot
-    - *step-native-mksnapshot-gn-gen
+    - *step-maybe-native-mksnapshot-gn-gen
-    - *step-native-mksnapshot-build
+    - *step-maybe-native-mksnapshot-build
-    - *step-native-mksnapshot-store
+    - *step-maybe-native-mksnapshot-store
 
     # chromedriver
     - *step-electron-chromedriver-build
@@ -432,20 +422,20 @@ steps-electron-build-for-publish: &steps-electron-build-for-publish
     # ffmpeg
     - *step-ffmpeg-gn-gen
     - *step-ffmpeg-build
+    - *step-ffmpeg-store
 
     # Publish
     - *step-electron-publish
 
-
 steps-native-mksnapshot-build: &steps-native-mksnapshot-build
   steps:
     - attach_workspace:
         at: .
     - *step-depot-tools-add-to-path
     - *step-setup-env-for-build
-    - *step-native-mksnapshot-gn-gen
+    - *step-maybe-native-mksnapshot-gn-gen
-    - *step-native-mksnapshot-build
+    - *step-maybe-native-mksnapshot-build
-    - *step-native-mksnapshot-store
+    - *step-maybe-native-mksnapshot-store
 
 steps-ffmpeg-build: &steps-ffmpeg-build
   steps:
@@ -464,6 +454,8 @@ steps-ffmpeg-build: &steps-ffmpeg-build
           - src/out/ffmpeg/libffmpeg.so
           - src/out/ffmpeg/ffmpeg.zip
 
+    - *step-show-sccache-stats
+
 steps-native-tests: &steps-native-tests
   steps:
     - attach_workspace:
@@ -472,14 +464,6 @@ steps-native-tests: &steps-native-tests
     - *step-setup-env-for-build
     - *step-electron-gn-gen
 
-    # This list is used to split tests across executors.
-    - run:
-        name: List tests
-        command: |
-          python src/electron/script/native-tests.py list \
-            --config $TESTS_CONFIG > tests_list.txt
-
-    # TODO(alexeykuzmin): Build only a subset of all tests.
     - run:
         name: Build tests
         command: |
@@ -495,8 +479,7 @@ steps-native-tests: &steps-native-tests
           python src/electron/script/native-tests.py run \
             --config $TESTS_CONFIG \
             --tests-dir src/out/Default \
-            --output-dir test_results \
+            --output-dir test_results
-            --binary $(circleci tests split tests_list.txt)
 
     - store_artifacts:
         path: test_results
@@ -626,29 +609,34 @@ jobs:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-debug-build
+      <<: *env-enable-sccache
     <<: *steps-electron-build
 
   linux-x64-testing:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-testing-build
+      <<: *env-enable-sccache
     <<: *steps-electron-build-for-tests
 
   linux-x64-ffmpeg:
     <<: *machine-linux-medium
     environment:
+      <<: *env-enable-sccache
     <<: *steps-ffmpeg-build
 
   linux-x64-release:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-release-build
+      <<: *env-enable-sccache
+      <<: *env-send-slack-notifications
     <<: *steps-electron-build-for-tests
 
   linux-x64-publish:
     <<: *machine-linux-2xlarge
     environment:
-      <<: *env-publish-build
+      <<: *env-release-build
     <<: *steps-electron-build-for-publish
 
   linux-ia32-debug:
@@ -656,6 +644,7 @@ jobs:
     environment:
       <<: *env-ia32
       <<: *env-debug-build
+      <<: *env-enable-sccache
     <<: *steps-electron-build
 
   linux-ia32-testing:
@@ -663,12 +652,14 @@ jobs:
     environment:
       <<: *env-ia32
       <<: *env-testing-build
+      <<: *env-enable-sccache
     <<: *steps-electron-build-for-tests
 
   linux-ia32-ffmpeg:
     <<: *machine-linux-medium
     environment:
       <<: *env-ia32
+      <<: *env-enable-sccache
     <<: *steps-ffmpeg-build
 
   linux-ia32-release:
@@ -676,13 +667,15 @@ jobs:
     environment:
       <<: *env-ia32
       <<: *env-release-build
+      <<: *env-enable-sccache
+      <<: *env-send-slack-notifications
     <<: *steps-electron-build-for-tests
 
   linux-ia32-publish:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-ia32
-      <<: *env-publish-build
+      <<: *env-release-build
     <<: *steps-electron-build-for-publish
 
   linux-arm-debug:
@@ -690,6 +683,7 @@ jobs:
     environment:
       <<: *env-arm
       <<: *env-debug-build
+      <<: *env-enable-sccache
     <<: *steps-electron-build
 
   linux-arm-testing:
@@ -697,12 +691,14 @@ jobs:
     environment:
       <<: *env-arm
       <<: *env-testing-build
+      <<: *env-enable-sccache
     <<: *steps-electron-build-for-tests
 
   linux-arm-ffmpeg:
     <<: *machine-linux-medium
     environment:
       <<: *env-arm
+      <<: *env-enable-sccache
     <<: *steps-ffmpeg-build
 
   linux-arm-release:
@@ -710,19 +706,22 @@ jobs:
     environment:
       <<: *env-arm
       <<: *env-release-build
+      <<: *env-enable-sccache
+      <<: *env-send-slack-notifications
     <<: *steps-electron-build-for-tests
 
   linux-arm-publish:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-arm
-      <<: *env-publish-build
+      <<: *env-release-build
     <<: *steps-electron-build-for-publish
 
-  linux-arm-mksnapshot:
+  linux-arm-native-mksnapshot:
     <<: *machine-linux-medium
     environment:
       <<: *env-arm
+      <<: *env-enable-sccache
     <<: *steps-native-mksnapshot-build
 
   linux-arm64-debug:
@@ -730,6 +729,7 @@ jobs:
     environment:
       <<: *env-arm64
       <<: *env-debug-build
+      <<: *env-enable-sccache
     <<: *steps-electron-build
 
   linux-arm64-testing:
@@ -737,12 +737,14 @@ jobs:
     environment:
       <<: *env-arm64
       <<: *env-testing-build
+      <<: *env-enable-sccache
     <<: *steps-electron-build-for-tests
 
   linux-arm64-ffmpeg:
     <<: *machine-linux-medium
     environment:
       <<: *env-arm64
+      <<: *env-enable-sccache
     <<: *steps-ffmpeg-build
 
   linux-arm64-release:
@@ -750,25 +752,29 @@ jobs:
     environment:
       <<: *env-arm64
       <<: *env-release-build
+      <<: *env-enable-sccache
+      <<: *env-send-slack-notifications
     <<: *steps-electron-build-for-tests
 
   linux-arm64-publish:
     <<: *machine-linux-2xlarge
     environment:
       <<: *env-arm64
-      <<: *env-publish-build
+      <<: *env-release-build
     <<: *steps-electron-build-for-publish
 
-  linux-arm64-mksnapshot:
+  linux-arm64-native-mksnapshot:
     <<: *machine-linux-medium
     environment:
       <<: *env-arm64
+      <<: *env-enable-sccache
     <<: *steps-native-mksnapshot-build
 
   osx-testing:
     <<: *machine-mac
     environment:
       <<: *env-testing-build
+      <<: *env-enable-sccache
     <<: *steps-build-mac
 
   mas-testing:
@@ -776,23 +782,24 @@ jobs:
     environment:
       <<: *env-mas
       <<: *env-testing-build
+      <<: *env-enable-sccache
     <<: *steps-build-mac
 
   # Layer 3: Tests.
   linux-x64-unittests:
     <<: *machine-linux-2xlarge
     environment:
-      <<: *env-testing-build
       <<: *env-unittests
+      <<: *env-testing-build
+      <<: *env-enable-sccache
     <<: *steps-native-tests
-#    TODO(alexeykuzmin): Use parallelism.
-#    parallelism: 4  # https://xkcd.com/221/
 
   linux-x64-browsertests:
     <<: *machine-linux-2xlarge
     environment:
-      <<: *env-testing-build
       <<: *env-browsertests
+      <<: *env-testing-build
+      <<: *env-enable-sccache
     <<: *steps-native-tests
 
   linux-x64-testing-tests:
@@ -807,6 +814,10 @@ jobs:
     <<: *machine-linux-medium
     <<: *steps-tests
 
+  linux-x64-release-verify-ffmpeg:
+    <<: *machine-linux-medium
+    <<: *steps-verify-ffmpeg
+
   linux-ia32-testing-tests:
     <<: *machine-linux-medium
     environment:
@@ -825,6 +836,12 @@ jobs:
       <<: *env-ia32
     <<: *steps-tests
 
+  linux-ia32-release-verify-ffmpeg:
+    <<: *machine-linux-medium
+    environment:
+      <<: *env-ia32
+    <<: *steps-verify-ffmpeg
+
   osx-testing-tests:
     <<: *machine-mac
     <<: *steps-tests
@@ -884,7 +901,7 @@ workflows:
       - linux-arm-ffmpeg:
            requires:
              - linux-arm-checkout
-      - linux-arm-mksnapshot:
+      - linux-arm-native-mksnapshot:
           requires:
             - linux-arm-checkout
 
@@ -897,7 +914,7 @@ workflows:
       - linux-arm64-ffmpeg:
            requires:
              - linux-arm64-checkout
-      - linux-arm64-mksnapshot:
+      - linux-arm64-native-mksnapshot:
           requires:
             - linux-arm64-checkout
 
@@ -946,6 +963,13 @@ workflows:
       - linux-x64-release-tests:
           requires:
             - linux-x64-release
+      - linux-x64-ffmpeg:
+          requires:
+            - linux-checkout
+      - linux-x64-release-verify-ffmpeg:
+          requires:
+            - linux-x64-release
+            - linux-x64-ffmpeg
 
       - linux-ia32-release:
           requires:
@@ -953,17 +977,30 @@ workflows:
       - linux-ia32-release-tests:
           requires:
             - linux-ia32-release
+      - linux-ia32-ffmpeg:
+          requires:
+            - linux-checkout
+      - linux-ia32-release-verify-ffmpeg:
+          requires:
+            - linux-ia32-release
+            - linux-ia32-ffmpeg
 
       - linux-arm-release:
           requires:
             - linux-arm-checkout
-      - linux-arm-mksnapshot:
+      - linux-arm-ffmpeg:
+           requires:
+             - linux-arm-checkout
+      - linux-arm-native-mksnapshot:
           requires:
             - linux-arm-checkout
 
       - linux-arm64-release:
           requires:
             - linux-arm64-checkout
-      - linux-arm64-mksnapshot:
+      - linux-arm64-ffmpeg:
+           requires:
+             - linux-arm64-checkout
+      - linux-arm64-native-mksnapshot:
           requires:
             - linux-arm64-checkout