build: use siso instead of reclient (36-x-y) (#48228)

build: use siso instead of reclient

.github/actions/build-electron/action.yml

@@ -60,14 +60,14 @@ runs:
           sudo launchctl limit maxfiles 65536 200000
         fi
 
-        NINJA_SUMMARIZE_BUILD=1 e build -j $NUMBER_OF_NINJA_PROCESSES
+        NINJA_SUMMARIZE_BUILD=1 e build
         cp out/Default/.ninja_log out/electron_ninja_log
         node electron/script/check-symlinks.js
     - name: Build Electron dist.zip ${{ inputs.step-suffix }}
       shell: bash
       run: |
         cd src
-        e build --target electron:electron_dist_zip -j $NUMBER_OF_NINJA_PROCESSES -d explain
+        e build --target electron:electron_dist_zip
         if [ "${{ inputs.is-asan }}" != "true" ]; then
           target_os=${{ inputs.target-platform == 'macos' && 'mac' || inputs.target-platform }}
           if [ "${{ inputs.artifact-platform }}" = "mas" ]; then
@@ -79,7 +79,7 @@ runs:
       shell: bash
       run: |
         cd src
-        e build --target electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES
+        e build --target electron:electron_mksnapshot
         ELECTRON_DEPOT_TOOLS_DISABLE_LOG=1 e d gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args
         # Remove unused args from mksnapshot_args
         SEDOPTION="-i"
@@ -89,7 +89,7 @@ runs:
         sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args
         sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args
 
-        e build --target electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES
+        e build --target electron:electron_mksnapshot_zip
         if [ "${{ inputs.target-platform }}" = "win" ]; then
           cd out/Default
           powershell Compress-Archive -update mksnapshot_args mksnapshot.zip
@@ -123,7 +123,7 @@ runs:
       shell: bash
       run: |
         cd src
-        e build --target electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES
+        e build --target electron:electron_chromedriver
         e build --target electron:electron_chromedriver_zip
     - name: Build Node.js headers ${{ inputs.step-suffix }}
       shell: bash
@@ -166,24 +166,24 @@ runs:
           electron/script/zip-symbols.py -b $BUILD_PATH
         fi
     - name: Generate FFMpeg ${{ inputs.step-suffix }}
-      if: ${{ inputs.is-release == 'true' }}
       shell: bash
+      if: ${{ inputs.is-release == 'true' }}
       run: |
         cd src
-        gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true $GN_EXTRA_ARGS"
-        e build --target electron:electron_ffmpeg_zip -C ../../out/ffmpeg -j $NUMBER_OF_NINJA_PROCESSES
+        gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true use_siso=true $GN_EXTRA_ARGS"
+        e build --target electron:electron_ffmpeg_zip -C ../../out/ffmpeg
     - name: Generate Hunspell Dictionaries ${{ inputs.step-suffix }}
       shell: bash
       if: ${{ inputs.is-release == 'true' && inputs.target-platform == 'linux' }}
       run: |
-        e build --target electron:hunspell_dictionaries_zip -j $NUMBER_OF_NINJA_PROCESSES
+        e build --target electron:hunspell_dictionaries_zip
     - name: Generate Libcxx ${{ inputs.step-suffix }}
       shell: bash
       if: ${{ inputs.is-release == 'true' && inputs.target-platform == 'linux' }}
       run: |
-        e build --target electron:libcxx_headers_zip -j $NUMBER_OF_NINJA_PROCESSES
-        e build --target electron:libcxxabi_headers_zip -j $NUMBER_OF_NINJA_PROCESSES
-        e build --target electron:libcxx_objects_zip -j $NUMBER_OF_NINJA_PROCESSES
+        e build --target electron:libcxx_headers_zip
+        e build --target electron:libcxxabi_headers_zip
+        e build --target electron:libcxx_objects_zip
     - name: Remove Clang problem matcher
       shell: bash
       run: echo "::remove-matcher owner=clang::"
@@ -206,7 +206,26 @@ runs:
           echo 'Uploading Electron release distribution to GitHub releases'
           script/release/uploaders/upload.py --verbose
         fi
+    - name: Generate siso report
+      if: ${{ inputs.target-platform != 'win' && !cancelled() }}
+      shell: bash
+      run: |
+        cd src
+        e d siso report -C out/Default > siso_report.txt
+        SISO_REPORT_PATH=$(grep -o '/.*siso-report-[^ ]*' siso_report.txt)
+        echo "SISO_REPORT_PATH=$SISO_REPORT_PATH" >> $GITHUB_ENV
+    - name: Generate siso report (Windows)
+      if: ${{ inputs.target-platform == 'win' && !cancelled() }}
+      shell: powershell
+      run: |
+        cd src
+        e d siso report -C out\Default > siso_report.txt
+        $SISO_REPORT_PATH = Get-Content "siso_report.txt" | Select-String "report file:\s*(.+)" | ForEach-Object { 
+          $_.Matches.Groups[1].Value.Trim() 
+        }
+        echo "SISO_REPORT_PATH=$SISO_REPORT_PATH" >> $env:GITHUB_ENV
     - name: Generate Artifact Key
+      if: always() && !cancelled()
       shell: bash
       run: |
         if [ "${{ inputs.is-asan }}" = "true" ]; then 
@@ -218,9 +237,11 @@ runs:
     # The current generated_artifacts_<< artifact.key >> name was taken from CircleCI
     # to ensure we don't break anything, but we may be able to improve that.
     - name: Move all Generated Artifacts to Upload Folder ${{ inputs.step-suffix }}
+      if: always() && !cancelled()
       shell: bash
       run: ./src/electron/script/actions/move-artifacts.sh
     - name: Upload Generated Artifacts ${{ inputs.step-suffix }}
+      if: always() && !cancelled()
       uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
       with:
         name: generated_artifacts_${{ env.ARTIFACT_KEY }}

.github/actions/cipd-install/action.yml

@@ -14,6 +14,9 @@ inputs:
     description: 'Target platform, should be linux, win, macos'
   package:
     description: 'Package to install'
+  dependency-version:
+    description: 'Version of the dependency to install'
+    default: ''
 runs:
   using: "composite"
   steps:
@@ -22,15 +25,23 @@ runs:
       run : |
         rm -rf ${{ inputs.cipd-root-prefix-path }}${{ inputs.installation-dir }}
     - name: Create ensure file for ${{ inputs.dependency }}
+      if: ${{ inputs.dependency-version == '' }}
       shell: bash
       run: |
         echo '${{ inputs.package }}' `e d gclient getdep --deps-file=${{ inputs.deps-file }} -r '${{ inputs.installation-dir }}:${{ inputs.package }}'` > ${{ inputs.dependency }}_ensure_file
         cat ${{ inputs.dependency }}_ensure_file
-    - name: CIPD installation of ${{ inputs.dependency }} (macOS)
-      if: ${{ inputs.target-platform == 'macos' }}
+
+    - name: Create ensure file for ${{ inputs.dependency }} from dependency-version
+      if: ${{ inputs.dependency-version != '' }}
       shell: bash
       run: |
-        echo "ensuring ${{ inputs.dependency }} on macOS"
+        echo '${{ inputs.package }} ${{ inputs.dependency-version }}'  > ${{ inputs.dependency }}_ensure_file
+        cat ${{ inputs.dependency }}_ensure_file
+    - name: CIPD installation of ${{ inputs.dependency }} (macOS)
+      if: ${{ inputs.target-platform != 'win' }}
+      shell: bash
+      run: |
+        echo "ensuring ${{ inputs.dependency }}"
         e d cipd ensure --root ${{ inputs.cipd-root-prefix-path }}${{ inputs.installation-dir }} -ensure-file ${{ inputs.dependency }}_ensure_file
     - name: CIPD installation of ${{ inputs.dependency }} (Windows)
       if: ${{ inputs.target-platform == 'win' }}

.github/actions/fix-sync/action.yml

@@ -20,11 +20,13 @@ runs:
   using: "composite"
   steps:
     - name: Fix clang
+      if: ${{ inputs.target-platform != 'linux' }}
       shell: bash
       run : |
         rm -rf src/third_party/llvm-build
         python3 src/tools/clang/scripts/update.py
     - name: Fix esbuild
+      if: ${{ inputs.target-platform != 'linux' }}
       uses: ./src/electron/.github/actions/cipd-install
       with:
         cipd-root-prefix-path: src/third_party/devtools-frontend/src/
@@ -34,6 +36,7 @@ runs:
         target-platform: ${{ inputs.target-platform }}
         package: infra/3pp/tools/esbuild/${platform}
     - name: Fix rustc
+      if: ${{ inputs.target-platform != 'linux' }}
       shell: bash
       run : |
         rm -rf src/third_party/rust-toolchain
@@ -57,6 +60,7 @@ runs:
         target-platform: ${{ inputs.target-platform }}
         package: gn/gn/windows-amd64
     - name: Fix reclient
+      if: ${{ inputs.target-platform != 'linux' }}
       uses: ./src/electron/.github/actions/cipd-install
       with:
         dependency: reclient
@@ -65,6 +69,7 @@ runs:
         target-platform: ${{ inputs.target-platform }}
         package: infra/rbe/client/${platform}
     - name: Configure reclient configs
+      if: ${{ inputs.target-platform != 'linux' }}
       shell: bash
       run : |
         python3 src/buildtools/reclient_cfgs/configure_reclient_cfgs.py --rbe_instance "projects/rbe-chrome-untrusted/instances/default_instance" --reproxy_cfg_template reproxy.cfg.template --rewrapper_cfg_project "" --skip_remoteexec_cfg_fetch
@@ -82,6 +87,7 @@ runs:
           python3 src/third_party/depot_tools/download_from_google_storage.py --no_resume --no_auth --bucket chromium-browser-clang -s $DSYM_SHA_FILE -o src/tools/clang/dsymutil/bin/dsymutil
         fi
     - name: Fix ninja
+      if: ${{ inputs.target-platform != 'linux' }}
       uses: ./src/electron/.github/actions/cipd-install
       with:
         dependency: ninja
@@ -90,10 +96,20 @@ runs:
         target-platform: ${{ inputs.target-platform }}
         package: infra/3pp/tools/ninja/${platform}
     - name: Set ninja in path
+      if: ${{ inputs.target-platform != 'linux' }}
       shell: bash
       run : |
         echo "$(pwd)/src/third_party/ninja" >> $GITHUB_PATH
+    - name: Fix siso
+      uses: ./src/electron/.github/actions/cipd-install
+      with:
+        dependency: siso
+        deps-file: src/DEPS
+        installation-dir: src/third_party/siso/cipd
+        target-platform: ${{ inputs.target-platform }}
+        package: infra/build/siso/${platform}
     - name: Fixup angle git
+      if: ${{ inputs.target-platform != 'linux' }}
       shell: bash
       run : |
         cd src/third_party/angle

.github/actions/install-build-tools/action.yml

@@ -13,8 +13,9 @@ runs:
         git config --global core.fscache true
         git config --global core.longpaths true
         git config --global core.preloadindex true
+        git config --global core.longpaths true
       fi
-      export BUILD_TOOLS_SHA=6e8526315ea3b4828882497e532b8340e64e053c
+      export BUILD_TOOLS_SHA=8559e5d325d61f195a255f41077ffc9e5b70b0e5
       npm i -g @electron/build-tools
       # Update depot_tools to ensure python
       e d update_depot_tools

.github/workflows/pipeline-segment-electron-build.yml

@@ -171,7 +171,7 @@ jobs:
         ELECTRON_DEPOT_TOOLS_DISABLE_LOG: true
     - name: Init Build Tools
       run: |
-        e init -f --root=$(pwd) --out=Default ${{ inputs.gn-build-type }} --import ${{ inputs.gn-build-type }} --target-cpu ${{ inputs.target-arch }}
+        e init -f --root=$(pwd) --out=Default ${{ inputs.gn-build-type }} --import ${{ inputs.gn-build-type }} --target-cpu ${{ inputs.target-arch }} --remote-build siso
     - name: Run Electron Only Hooks
       run: |
         e d gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
@@ -181,9 +181,6 @@ jobs:
         echo "DEPSHASH=$(cat src/electron/.depshash)" >> $GITHUB_ENV
     - name: Add CHROMIUM_BUILDTOOLS_PATH to env
       run: echo "CHROMIUM_BUILDTOOLS_PATH=$(pwd)/src/buildtools" >> $GITHUB_ENV
-    - name: Setup Number of Ninja Processes
-      run: |
-        echo "NUMBER_OF_NINJA_PROCESSES=${{ inputs.target-platform != 'macos' && '300' || '200' }}" >> $GITHUB_ENV
     - name: Free up space (macOS)
       if: ${{ inputs.target-platform == 'macos' }}
       uses: ./src/electron/.github/actions/free-space-macos

.github/workflows/pipeline-segment-electron-test.yml

@@ -141,6 +141,7 @@ jobs:
         git config --global core.fscache true
         git config --global core.longpaths true
         git config --global core.preloadindex true
+        git config --global core.longpaths true
         git clone --filter=tree:0 https://chromium.googlesource.com/chromium/tools/depot_tools.git
         # Ensure depot_tools does not update.
         test -d depot_tools && cd depot_tools

build/siso/backend.star

@@ -0,0 +1,21 @@
+# -*- bazel-starlark -*-
+
+load("@builtin//struct.star", "module")
+
+def __platform_properties(ctx):
+    container_image = "docker://gcr.io/chops-public-images-prod/rbe/siso-chromium/linux@sha256:d7cb1ab14a0f20aa669c23f22c15a9dead761dcac19f43985bf9dd5f41fbef3a"
+    return {
+        "default": {
+            "OSFamily": "Linux",
+            "container-image": container_image,
+        },
+        "large": {
+            "OSFamily": "Linux",
+            "container-image": container_image,
+        },
+    }
+
+backend = module(
+    "backend",
+    platform_properties = __platform_properties,
+)

build/siso/main.star

@@ -0,0 +1,66 @@
+load("@builtin//encoding.star", "json")
+load("@builtin//path.star", "path")
+load("@builtin//runtime.star", "runtime")
+load("@builtin//struct.star", "module")
+load("@config//main.star", upstream_init = "init")
+load("@config//win_sdk.star", "win_sdk")
+load("@config//gn_logs.star", "gn_logs")
+
+def init(ctx):
+    mod = upstream_init(ctx)
+    step_config = json.decode(mod.step_config)
+
+    # Buildbarn doesn't support input_root_absolute_path so disable that
+    for rule in step_config["rules"]:    
+      input_root_absolute_path = rule.get("input_root_absolute_path", False)
+      if input_root_absolute_path:
+        rule.pop("input_root_absolute_path", None)
+
+    # Only wrap clang rules with a remote wrapper if not on Linux. These are currently only
+    # needed for X-Compile builds, which run on Windows and Mac.
+    if runtime.os != "linux":
+      for rule in step_config["rules"]:
+        if rule["name"].startswith("clang/") or rule["name"].startswith("clang-cl/"):
+          rule["remote_wrapper"] = "../../buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper"
+          if "inputs" not in rule:
+            rule["inputs"] = []
+          rule["inputs"].append("buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper")
+          rule["inputs"].append("third_party/llvm-build/Release+Asserts_linux/bin/clang")
+
+      if "executables" not in step_config:
+        step_config["executables"] = []
+      step_config["executables"].append("buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper")
+      step_config["executables"].append("third_party/llvm-build/Release+Asserts_linux/bin/clang")
+
+    if runtime.os == "darwin":
+      # Update platforms to match our default siso config instead of reclient configs.
+      step_config["platforms"].update({
+          "clang": step_config["platforms"]["default"],
+          "clang_large": step_config["platforms"]["default"],          
+      })      
+
+    if runtime.os == "windows":
+      # Add additional Windows SDK headers needed by Electron      
+      win_toolchain_dir = win_sdk.toolchain_dir(ctx)
+      if win_toolchain_dir:
+        sdk_version = gn_logs.read(ctx).get("windows_sdk_version")
+        step_config["input_deps"][win_toolchain_dir + ":headers"].extend([
+          # third_party/electron_node/deps/uv/include/uv/win.h includes mswsock.h
+          path.join(win_toolchain_dir, "Windows Kits/10/Include", sdk_version, "um/mswsock.h"),
+          # third_party/electron_node/src/debug_utils.cc includes lm.h
+          path.join(win_toolchain_dir, "Windows Kits/10/Include", sdk_version, "um/Lm.h"),          
+        ])
+      
+      # Update platforms to match our default siso config instead of reclient configs.
+      step_config["platforms"].update({
+          "clang-cl": step_config["platforms"]["default"],
+          "clang-cl_large": step_config["platforms"]["default"],
+          "lld-link": step_config["platforms"]["default"],
+      })
+
+    return module(
+      "config",
+      step_config = json.encode(step_config),
+      filegroups = mod.filegroups,
+      handlers = mod.handlers,
+    )

script/actions/move-artifacts.sh

@@ -113,5 +113,8 @@ mv_if_exist src/out/Default/hunspell_dictionaries.zip
 mv_if_exist src/cross-arch-snapshots
 cp_if_exist src/out/electron_ninja_log
 cp_if_exist src/out/Default/.ninja_log
+if [ -n "$SISO_REPORT_PATH" ]; then
+  cp_if_exist "$SISO_REPORT_PATH"
+fi
 
 move_src_dirs_if_exist

script/node/generate_original_fs.py

@@ -16,10 +16,6 @@ for fs_file in fs_files:
                 'lib/original-fs.js').replace('lib/fs/',
                 'lib/original-fs/')
 
-        with open(os.path.join(out_dir, fs_file), 'w', encoding='utf-8'
-                  ) as original_f:
-            original_f.write(contents)
-
         with open(os.path.join(out_dir, original_fs_file), 'w',
                   encoding='utf-8') as transformed_f:
             transformed_contents = contents.replace('internal/fs/',