build: add Linux GHA build step (#42432)

Co-authored-by: Keeley Hammond <khammond@slack-corp.com>

.github/workflows/linux-pipeline.yml

@@ -43,28 +43,26 @@ env:
   GN_CONFIG: ${{ inputs.gn-config }}
   # Disable pre-compiled headers to reduce out size - only useful for rebuilds
   GN_BUILDFLAG_ARGS: 'enable_precompiled_headers = false'
-  GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
-  CHECK_DIST_MANIFEST: '1'
+  GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
+  CHECK_DIST_MANIFEST: true
   IS_GHA_RELEASE: true
   ELECTRON_OUT_DIR: Default
 
 jobs:
   checkout:
-    runs-on: LargeLinuxRunner
+    runs-on: aks-linux-large
+    container:
+      image: ghcr.io/electron/build:latest
+      options: --user root
     steps:
     - name: Checkout Electron
       uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
       with:
         path: src/electron
+        fetch-depth: 0
     - name: Set GIT_CACHE_PATH to make gclient to use the cache
       run: |
         echo "GIT_CACHE_PATH=$(pwd)/git-cache" >> $GITHUB_ENV
-    - name: Setup Node.js/npm
-      uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8
-      with:
-        node-version: 20.11.x
-        cache: yarn
-        cache-dependency-path: src/electron/yarn.lock
     - name: Install Dependencies
       run: |
         cd src/electron
@@ -73,15 +71,11 @@ jobs:
       timeout-minutes: 5
       run: |
         git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
-        if [ "`uname`" == "Darwin" ]; then
-          # remove ninjalog_uploader_wrapper.py from autoninja since we don't use it and it causes problems
-          sed -i '' '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
-        else
-          sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
-          # Remove swift-format dep from cipd on macOS until we send a patch upstream.
-          cd depot_tools
-          git apply --3way ../src/electron/.github/workflows/config/gclient.diff
-        fi
+
+        sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
+        cd depot_tools
+        git apply --3way ../src/electron/.github/workflows/config/gclient.diff
+
         # Ensure depot_tools does not update.
         test -d depot_tools && cd depot_tools
         touch .disable_auto_update
@@ -118,7 +112,7 @@ jobs:
           "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY"
 
         ELECTRON_USE_THREE_WAY_MERGE_FOR_PATCHES=1 gclient sync --with_branch_heads --with_tags -vvvvv
-        if [ ${{ inputs.is-release != true }} ]; then
+        if [ "${{ inputs.is-release }}" != "true" ]; then
           # Re-export all the patches to check if there were changes.
           python3 src/electron/script/export_all_patches.py src/electron/patches/config.json
           cd src/electron
@@ -172,6 +166,8 @@ jobs:
         rm -rf src/third_party/swift-toolchain
         rm -rf src/third_party/swiftshader/tests/regres/testlists
         rm -rf src/electron
+        rm -rf src/third_party/electron_node/deps/openssl
+        rm -rf src/third_party/electron_node/deps/v8
     - name: Compress Src Directory
       if: steps.check-cache.outputs.cache_exists == 'false'
       run: |
@@ -186,5 +182,246 @@ jobs:
           --account-key $AZURE_STORAGE_KEY \
           --container-name $AZURE_STORAGE_CONTAINER_NAME \
           --file $DEPSHASH.tar \
-          --name $DEPSHASH \
-          --debug
+          --name $DEPSHASH
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        build-arch: [ arm64 ] # x64, arm 
+    env: 
+      TARGET_ARCH: ${{ matrix.build-arch }}
+    runs-on: aks-linux-large
+    container:
+      image: ghcr.io/electron/build:latest
+      options: --user root
+    needs: checkout
+    steps:
+    - name: Load Build Tools
+      run: |
+        export BUILD_TOOLS_SHA=ef894bc3cfa99d84a3b731252da0f83f500e4032
+        npm i -g @electron/build-tools
+        e auto-update disable
+        e init --root=$(pwd) --out=Default ${{ inputs.gn-build-type }} --import ${{ inputs.gn-build-type }} --target-cpu ${{ matrix.build-arch }}
+    - name: Checkout Electron
+      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+      with:
+        path: src/electron
+        fetch-depth: 0
+    - name: Install Dependencies
+      run: |
+        cd src/electron
+        node script/yarn install
+    - name: Set GN_EXTRA_ARGS
+      run: |
+        if [ "${{ matrix.build-arch  }}" = "arm" ]; then
+          GN_EXTRA_ARGS='target_cpu="arm" build_tflite_with_xnnpack=false'
+        elif [ "${{ matrix.build-arch }}" = "arm64" ]; then
+          GN_EXTRA_ARGS='target_cpu="arm64" fatal_linker_warnings=false enable_linux_installer=false'
+        fi
+        echo "GN_EXTRA_ARGS=$GN_EXTRA_ARGS" >> $GITHUB_ENV
+    - name: Get Depot Tools
+      timeout-minutes: 5
+      run: |
+        git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
+
+        sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
+        cd depot_tools
+        git apply --3way ../src/electron/.github/workflows/config/gclient.diff
+
+        # Ensure depot_tools does not update.
+        test -d depot_tools && cd depot_tools
+        touch .disable_auto_update
+    - name: Add Depot Tools to PATH
+      run: echo "$(pwd)/depot_tools" >> $GITHUB_PATH
+    - name: Generate DEPS Hash
+      run: |
+        node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target
+        echo "DEPSHASH=v1-src-cache-$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV
+    - name: Download Src Cache
+      # The cache will always exist here as a result of the checkout job
+      # Either it was uploaded to Azure in the checkout job for this commit
+      # or it was uploaded in the checkout job for a previous commit.
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
+      with:
+        timeout_minutes: 20
+        max_attempts: 3
+        retry_on: error
+        command: |
+          az storage blob download \
+            --account-name $AZURE_STORAGE_ACCOUNT \
+            --account-key $AZURE_STORAGE_KEY \
+            --container-name $AZURE_STORAGE_CONTAINER_NAME \
+            --name $DEPSHASH \
+            --file $DEPSHASH.tar
+    - name: Unzip and Ensure Src Cache
+      run: |
+        echo "Downloaded cache is $(du -sh $DEPSHASH.tar | cut -f1)"
+        mkdir temp-cache
+        tar -xf $DEPSHASH.tar -C temp-cache
+        echo "Unzipped cache is $(du -sh temp-cache/src | cut -f1)"
+
+        if [ -d "temp-cache/src" ]; then
+          echo "Relocating Cache"
+          rm -rf src
+          mv temp-cache/src src
+
+          echo "Deleting zip file"
+          rm -rf $DEPSHASH.tar
+        fi
+
+        if [ ! -d "src/third_party/blink" ]; then
+          echo "Cache was not correctly restored - exiting"
+          exit 1
+        fi
+
+        echo "Wiping Electron Directory"
+        rm -rf src/electron
+    - name: Checkout Electron
+      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+      with:
+        path: src/electron
+        fetch-depth: 0
+    - name: Run Electron Only Hooks
+      run: |
+        echo "Running Electron Only Hooks"
+        gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
+    - name: Regenerate DEPS Hash
+      run: |
+        (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target
+        echo "DEPSHASH=$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV
+    - name: Add CHROMIUM_BUILDTOOLS_PATH to env
+      run: echo "CHROMIUM_BUILDTOOLS_PATH=$(pwd)/src/buildtools" >> $GITHUB_ENV
+    - name: Fix Known Hosts on Linux
+      run: src/electron/.circleci/fix-known-hosts.sh
+    - name: Install build-tools & Setup RBE
+      run: |
+        echo "NUMBER_OF_NINJA_PROCESSES=300" >> $GITHUB_ENV
+        cd ~/.electron_build_tools
+        npx yarn --ignore-engines
+
+        # Pull down credential helper and print status
+        node -e "require('./src/utils/reclient.js').downloadAndPrepare({})"
+        HELPER=$(node -p "require('./src/utils/reclient.js').helperPath({})")
+        $HELPER login
+
+        echo 'RBE_service='`node -e "console.log(require('./src/utils/reclient.js').serviceAddress)"` >> $GITHUB_ENV
+        echo 'RBE_experimental_credentials_helper='`node -e "console.log(require('./src/utils/reclient.js').helperPath({}))"` >> $GITHUB_ENV
+        echo 'RBE_experimental_credentials_helper_args=print' >> $GITHUB_ENV
+    - name: Build Electron
+      run: |
+        cd src/electron
+        # TODO(codebytere): remove this once we figure out why .git/packed-refs is initially missing
+        git pack-refs
+        cd ..
+
+        NINJA_SUMMARIZE_BUILD=1 e build -j $NUMBER_OF_NINJA_PROCESSES
+        cp out/Default/.ninja_log out/electron_ninja_log
+        node electron/script/check-symlinks.js
+    - name: Build Electron dist.zip
+      run: |
+        cd src
+        e build electron:electron_dist_zip -j $NUMBER_OF_NINJA_PROCESSES
+        if [ "${{ env.CHECK_DIST_MANIFEST }}" = "true" ]; then
+          target_os=linux
+          target_cpu=${{ matrix.build-arch }}
+          electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.${{ matrix.build-arch }}.manifest
+        fi
+    - name: Build Mksnapshot
+      run: |
+        cd src
+        e build electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES
+        gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args
+
+        # Remove unused args from mksnapshot_args
+        SEDOPTION="-i"
+        sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args
+        sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args
+        sed $SEDOPTION '/The gn arg use_goma=true .*/d' out/Default/mksnapshot_args
+
+        if [ "${{ matrix.build-arch }}" = "arm" ]; then
+          electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/mksnapshot
+          electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/v8_context_snapshot_generator
+        elif [ "${{ matrix.build-arch }}" = "arm64" ]; then
+          electron/script/strip-binaries.py --file $PWD/out/Default/clang_x64_v8_arm64/mksnapshot
+          electron/script/strip-binaries.py --file $PWD/out/Default/clang_x64_v8_arm64/v8_context_snapshot_generator
+        else
+          electron/script/strip-binaries.py --file $PWD/out/Default/mksnapshot
+          electron/script/strip-binaries.py --file $PWD/out/Default/v8_context_snapshot_generator
+        fi
+      
+        e build electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES
+        (cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S)
+    - name: Generate Cross-Arch Snapshot (arm/arm64)
+      if: ${{ matrix.build-arch == 'arm' }} || ${{ matrix.build-arch == 'arm64' }}
+      run: |
+        cd src
+        if [ "${{ matrix.build-arch }}" = "arm" ]; then
+          export MKSNAPSHOT_PATH="clang_x86_v8_arm"
+        elif [ "${{ matrix.build-arch }}" = "arm64" ]; then
+          export MKSNAPSHOT_PATH="clang_x64_v8_arm64"
+        fi
+
+        cp "out/Default/$MKSNAPSHOT_PATH/mksnapshot" out/Default
+        cp "out/Default/$MKSNAPSHOT_PATH/v8_context_snapshot_generator" out/Default
+        cp "out/Default/$MKSNAPSHOT_PATH/libffmpeg.so" out/Default
+
+        python3 electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --create-snapshot-only
+        mkdir cross-arch-snapshots
+        cp out/Default-mksnapshot-test/*.bin cross-arch-snapshots
+        # Clean up so that ninja does not get confused
+        rm -f out/Default/libffmpeg.so
+    - name: Build Chromedriver
+      run: |
+        cd src
+        e build electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES
+        e build electron:electron_chromedriver_zip
+    - name: Generate & Zip Symbols
+      run: |
+        # Generate breakpad symbols on release builds
+        if [ "${{ inputs.generate-symbols }}" = "true" ]; then
+          e build electron:electron_symbols
+        fi
+        cd src
+        export BUILD_PATH="$(pwd)/out/Default"
+        e build electron:licenses
+        e build electron:electron_version_file
+        if [ "${{ inputs.is-release }}" = "true" ]; then
+          DELETE_DSYMS_AFTER_ZIP=1 electron/script/zip-symbols.py -b $BUILD_PATH
+        else
+          electron/script/zip-symbols.py -b $BUILD_PATH
+        fi
+    - name: Generate FFMpeg
+      if: ${{ inputs.is-release }}
+      run: |
+        cd src
+        gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true $GN_EXTRA_ARGS"
+        autoninja -C out/ffmpeg electron:electron_ffmpeg_zip -j $NUMBER_OF_NINJA_PROCESSES
+    - name: Generate Hunspell Dictionaries
+      if: ${{ inputs.is-release }}
+      run: |
+        cd src
+        autoninja -C out/Default electron:hunspell_dictionaries_zip -j $NUMBER_OF_NINJA_PROCESSES
+    - name: Generate TypeScript Definitions
+      if: ${{ inputs.is-release }}
+      run: |
+        cd src/electron
+        node script/yarn create-typescript-definitions
+    - name: Publish Electron Dist
+      if: ${{ inputs.is-release }}
+      run: |
+        rm -rf src/out/Default/obj
+        cd src/electron
+        if [ "${{ inputs.upload-to-storage }}" = "1" ]; then
+          echo 'Uploading Electron release distribution to Azure'
+          script/release/uploaders/upload.py --verbose --upload_to_storage
+        else
+          echo 'Uploading Electron release distribution to GitHub releases'
+          script/release/uploaders/upload.py --verbose
+        fi
+    - name: Move all Generated Artifacts to Upload Folder
+      run: ./src/electron/script/actions/move-artifacts.sh
+    - name: Upload Generated Artifacts
+      uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
+      with:
+        name: generated_artifacts_linux_${{ matrix.build-arch }}
+        path: ./generated_artifacts_linux_${{ matrix.build-arch }}

.github/workflows/macos-pipeline.yml

@@ -201,7 +201,7 @@ jobs:
     steps:
     - name: Load Build Tools
       run: |
-        export BUILD_TOOLS_SHA=2bb63e2e7877491b52f972532b52adc979a6ec2f
+        export BUILD_TOOLS_SHA=ef894bc3cfa99d84a3b731252da0f83f500e4032
         npm i -g @electron/build-tools
         e auto-update disable
         e init --root=$(pwd) --out=Default ${{ inputs.gn-build-type }} --import ${{ inputs.gn-build-type }} --target-cpu ${{ matrix.build-arch }}
@@ -689,7 +689,7 @@ jobs:
     steps:
     - name: Load Build Tools
       run: |
-        export BUILD_TOOLS_SHA=2bb63e2e7877491b52f972532b52adc979a6ec2f
+        export BUILD_TOOLS_SHA=ef894bc3cfa99d84a3b731252da0f83f500e4032
         npm i -g @electron/build-tools
         e auto-update disable
         e init --root=$(pwd) --out=Default ${{ inputs.gn-build-type }}

script/actions/move-artifacts.sh

@@ -1,11 +1,13 @@
-#!/bin/sh
+#!/bin/bash
 
-set -eo pipefail
-
-if [ -z "$MAS_BUILD" ]; then
-  BUILD_TYPE="darwin"
-else
-  BUILD_TYPE="mas"
+if [ "`uname`" == "Darwin" ]; then
+  if [ -z "$MAS_BUILD" ]; then
+    BUILD_TYPE="darwin"
+  else
+    BUILD_TYPE="mas"
+  fi
+elif [ "`uname`" == "Linux" ]; then
+  BUILD_TYPE="linux"
 fi
 
 echo Creating generated_artifacts_${BUILD_TYPE}_${TARGET_ARCH}...
@@ -20,6 +22,7 @@ mv_if_exist() {
     echo Skipping $1 - It is not present on disk
   fi
 }
+
 cp_if_exist() {
   if [ -f "$1" ] || [ -d "$1" ]; then
     echo Storing $1
@@ -29,6 +32,40 @@ cp_if_exist() {
   fi
 }
 
+tar_src_dirs_if_exist() {
+  mkdir build_artifacts
+
+  for dir in \
+    src/out/Default/gen/node_headers \
+    src/out/Default/overlapped-checker \
+    src/out/Default/ffmpeg \
+    src/out/Default/hunspell_dictionaries \
+    src/electron \
+    src/third_party/electron_node \
+    src/third_party/nan \
+    src/cross-arch-snapshots \
+    src/third_party/llvm-build \
+    src/build/linux \
+    src/buildtools/mac \
+    src/buildtools/third_party/libc++ \
+    src/buildtools/third_party/libc++abi \
+    src/third_party/libc++ \
+    src/third_party/libc++abi \
+    src/out/Default/obj/buildtools/third_party \
+    src/v8/tools/builtins-pgo
+  do
+    if [ -d "$dir" ]; then
+      mkdir -p build_artifacts/$dir
+      cp -r $dir build_artifacts/$dir
+    fi      
+  done
+
+  tar -cf build_artifacts.tarbuild_artifacts
+
+  mv_if_exist build_artifacts.tar
+}
+
+# Generated Artifacts
 mv_if_exist src/out/Default/dist.zip
 mv_if_exist src/out/Default/gen/node_headers.tar.gz
 mv_if_exist src/out/Default/symbols.zip
@@ -39,3 +76,6 @@ mv_if_exist src/out/Default/hunspell_dictionaries.zip
 mv_if_exist src/cross-arch-snapshots
 cp_if_exist src/out/electron_ninja_log
 cp_if_exist src/out/Default/.ninja_log
+
+tar_src_dirs_if_exist
+

script/actions/restore-artifacts.sh

@@ -1,17 +1,25 @@
 #!/bin/bash
 
-set -eo pipefail
-
 mv_if_exist() {
   if [ -f "generated_artifacts_${BUILD_TYPE}_${TARGET_ARCH}/$1" ] || [ -d "generated_artifacts_${BUILD_TYPE}_${TARGET_ARCH}/$1" ]; then
     echo Restoring $1 to $2
     mkdir -p $2
-    mv generated_artifacts_${BUILD_TYPE_${TARGET_ARCH}}/$1 $2
+    mv generated_artifacts_${BUILD_TYPE}_${TARGET_ARCH}/$1 $2
   else
     echo Skipping $1 - It is not present on disk
   fi
 }
 
+untar_if_exist() {
+  if [ -f "generated_artifacts_${BUILD_TYPE}_${TARGET_ARCH}/$1" ] || [ -d "generated_artifacts_${BUILD_TYPE}_${TARGET_ARCH}/$1" ]; then
+    echo Restoring $1 to $2
+    tar -xf generated_artifacts_${BUILD_TYPE}_${TARGET_ARCH}/$1 $2
+  else
+    echo Skipping $1 - It is not present on disk
+  fi
+}
+
+# Restore generated artifacts
 mv_if_exist dist.zip src/out/Default
 mv_if_exist node_headers.tar.gz src/out/Default/gen
 mv_if_exist symbols.zip src/out/Default
@@ -19,4 +27,7 @@ mv_if_exist mksnapshot.zip src/out/Default
 mv_if_exist chromedriver.zip src/out/Default
 mv_if_exist ffmpeg.zip src/out/ffmpeg
 mv_if_exist hunspell_dictionaries.zip src/out/Default
-mv_if_exist cross-arch-snapshots src
+mv_if_exist cross-arch-snapshots src
+
+# Restore build artifacts
+untar_if_exist build_artifacts.tar ./