electron/.github/workflows/linux-pipeline.yml

name: Pipeline Linux

on:
  workflow_call:
    inputs:
      is-release:
        description: 'Whether this build job is a release job'
        required: true
        type: boolean
        default: false
      gn-config:
        description: 'The gn arg configuration to use'
        required: true
        type: string
        default: //electron/build/args/testing.gn
      gn-build-type:
        description: 'The gn build type - testing or release'
        required: true
        type: string
        default: testing
      generate-symbols: 
        description: 'Whether or not to generate symbols'
        required: true
        type: boolean
        default: false
      upload-to-storage: 
        description: 'Whether or not to upload build artifacts to external storage'
        required: true
        type: string
        default: '0'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  ELECTRON_ARTIFACTS_BLOB_STORAGE: ${{ secrets.ELECTRON_ARTIFACTS_BLOB_STORAGE }}
  ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }}
  ELECTRON_GITHUB_TOKEN: ${{ secrets.ELECTRON_GITHUB_TOKEN }}
  GN_CONFIG: ${{ inputs.gn-config }}
  # Disable pre-compiled headers to reduce out size - only useful for rebuilds
  GN_BUILDFLAG_ARGS: 'enable_precompiled_headers = false'
  GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
  # Only disable this in the Asan build
  CHECK_DIST_MANIFEST: true
  IS_GHA_RELEASE: true
  ELECTRON_OUT_DIR: Default

jobs:
  checkout:
    runs-on: aks-linux-large
    container:
      image: ghcr.io/electron/build:latest
      options: --user root
      volumes:
        - /mnt/cross-instance-cache:/mnt/cross-instance-cache
    steps:
    - name: Checkout Electron
      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      with:
        path: src/electron
        fetch-depth: 0
    - name: Set GIT_CACHE_PATH to make gclient to use the cache
      run: |
        echo "GIT_CACHE_PATH=$(pwd)/git-cache" >> $GITHUB_ENV        
    - name: Install Dependencies
      run: |
        cd src/electron
        node script/yarn install        
    - name: Get Depot Tools
      timeout-minutes: 5
      run: |
        git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
        sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
        cd depot_tools
        git apply --3way ../src/electron/.github/workflows/config/gclient.diff
        # Ensure depot_tools does not update.
        test -d depot_tools && cd depot_tools
        touch .disable_auto_update        
    - name: Add Depot Tools to PATH
      run: echo "$(pwd)/depot_tools" >> $GITHUB_PATH
    - name: Generate DEPS Hash
      run: |
        node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target
        echo "DEPSHASH=v1-src-cache-$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV        
    - name: Check If Cache Exists
      id: check-cache
      run: |
        cache_path=/mnt/cross-instance-cache/$DEPSHASH.tar
        echo "Using cache key: $DEPSHASH"
        echo "Checking for cache in: $cache_path"
        if [ ! -f "$cache_path" ]; then
          echo "cache_exists=false" >> $GITHUB_OUTPUT
          echo "Cache Does Not Exist for $DEPSHASH"
        else
          echo "cache_exists=true" >> $GITHUB_OUTPUT
          echo "Cache Already Exists for $DEPSHASH, Skipping.."
        fi        
    - name: Gclient Sync
      if: steps.check-cache.outputs.cache_exists == 'false'
      run: |
        gclient config \
          --name "src/electron" \
          --unmanaged \
          ${GCLIENT_EXTRA_ARGS} \
          "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY"

        ELECTRON_USE_THREE_WAY_MERGE_FOR_PATCHES=1 gclient sync --with_branch_heads --with_tags -vvvvv
        if [ "${{ inputs.is-release }}" != "true" ]; then
          # Re-export all the patches to check if there were changes.
          python3 src/electron/script/export_all_patches.py src/electron/patches/config.json
          cd src/electron
          git update-index --refresh || true
          if ! git diff-index --quiet HEAD --; then
            # There are changes to the patches. Make a git commit with the updated patches
            git add patches
            GIT_COMMITTER_NAME="PatchUp" GIT_COMMITTER_EMAIL="73610968+patchup[bot]@users.noreply.github.com" git commit -m "chore: update patches" --author="PatchUp <73610968+patchup[bot]@users.noreply.github.com>"
            # Export it
            mkdir -p ../../patches
            git format-patch -1 --stdout --keep-subject --no-stat --full-index > ../../patches/update-patches.patch
            if (node ./script/push-patch.js 2> /dev/null > /dev/null); then
              echo
              echo "======================================================================"
              echo "Changes to the patches when applying, we have auto-pushed the diff to the current branch"
              echo "A new CI job will kick off shortly"
              echo "======================================================================"
              exit 1
            else
              echo
              echo "======================================================================"
              echo "There were changes to the patches when applying."
              echo "Check the CI artifacts for a patch you can apply to fix it."
              echo "======================================================================"
              exit 1
            fi
          fi
        fi        

    # delete all .git directories under src/ except for
    # third_party/angle/ and third_party/dawn/ because of build time generation of files
    # gen/angle/commit.h depends on third_party/angle/.git/HEAD
    # https://chromium-review.googlesource.com/c/angle/angle/+/2074924
    # and dawn/common/Version_autogen.h depends on  third_party/dawn/.git/HEAD
    # https://dawn-review.googlesource.com/c/dawn/+/83901
    # TODO: maybe better to always leave out */.git/HEAD file for all targets ?
    - name: Delete .git directories under src to free space
      if: steps.check-cache.outputs.cache_exists == 'false'
      run: |
        cd src
        ( find . -type d -name ".git" -not -path "./third_party/angle/*" -not -path "./third_party/dawn/*" -not -path "./electron/*" ) | xargs rm -rf        
    - name: Minimize Cache Size for Upload
      if: steps.check-cache.outputs.cache_exists == 'false'
      run: |
        rm -rf src/android_webview
        rm -rf src/ios/chrome
        rm -rf src/third_party/blink/web_tests
        rm -rf src/third_party/blink/perf_tests
        rm -rf src/chrome/test/data/xr/webvr_info
        rm -rf src/third_party/angle/third_party/VK-GL-CTS/src
        rm -rf src/third_party/swift-toolchain
        rm -rf src/third_party/swiftshader/tests/regres/testlists
        rm -rf src/electron        
    - name: Compress Src Directory
      if: steps.check-cache.outputs.cache_exists == 'false'
      run: |
        echo "Uncompressed src size: $(du -sh src | cut -f1 -d' ')"
        tar -cf $DEPSHASH.tar src
        echo "Compressed src to $(du -sh $DEPSHASH.tar | cut -f1 -d' ')"
        cp ./$DEPSHASH.tar /mnt/cross-instance-cache/        
    - name: Persist Src Cache
      if: steps.check-cache.outputs.cache_exists == 'false'
      run: |
        final_cache_path=/mnt/cross-instance-cache/$DEPSHASH.tar
        echo "Using cache key: $DEPSHASH"
        echo "Checking path: $final_cache_path"
        if [ ! -f "$final_cache_path" ]; then
          echo "Cache key not found"
          exit 1
        else
          echo "Cache key persisted in $final_cache_path"
        fi        
  build:
    strategy:
      fail-fast: false
      matrix:
        build-arch: [ arm64 ] # x64, arm 
    env: 
      TARGET_ARCH: ${{ matrix.build-arch }}
    runs-on: aks-linux-large
    container:
      image: ghcr.io/electron/build:latest
      options: --user root
      volumes:
        - /mnt/cross-instance-cache:/mnt/cross-instance-cache
    needs: checkout
    steps:
    - name: Load Build Tools
      run: |
        export BUILD_TOOLS_SHA=ef894bc3cfa99d84a3b731252da0f83f500e4032
        npm i -g @electron/build-tools
        e auto-update disable
        e init --root=$(pwd) --out=Default ${{ inputs.gn-build-type }} --import ${{ inputs.gn-build-type }} --target-cpu ${{ matrix.build-arch }}        
    - name: Checkout Electron
      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      with:
        path: src/electron
        fetch-depth: 0
    - name: Install Dependencies
      run: |
        cd src/electron
        node script/yarn install        
    - name: Set GN_EXTRA_ARGS
      run: |
        if [ "${{ matrix.build-arch  }}" = "arm" ]; then
          GN_EXTRA_ARGS='target_cpu="arm" build_tflite_with_xnnpack=false'
        elif [ "${{ matrix.build-arch }}" = "arm64" ]; then
          GN_EXTRA_ARGS='target_cpu="arm64" fatal_linker_warnings=false enable_linux_installer=false'
        fi
        echo "GN_EXTRA_ARGS=$GN_EXTRA_ARGS" >> $GITHUB_ENV        
    - name: Get Depot Tools
      timeout-minutes: 5
      run: |
        git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git

        sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
        cd depot_tools
        git apply --3way ../src/electron/.github/workflows/config/gclient.diff

        # Ensure depot_tools does not update.
        test -d depot_tools && cd depot_tools
        touch .disable_auto_update        
    - name: Add Depot Tools to PATH
      run: echo "$(pwd)/depot_tools" >> $GITHUB_PATH
    - name: Generate DEPS Hash
      run: |
        node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target
        echo "DEPSHASH=v1-src-cache-$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV        
    - name: Restore and Ensure Src Cache
      run: |
        cache_path=/mnt/cross-instance-cache/$DEPSHASH.tar
        echo "Using cache key: $DEPSHASH"
        echo "Checking for cache in: $cache_path"
        if [ ! -f "$cache_path" ]; then
          echo "Cache Does Not Exist for $DEPSHASH - exiting"
          exit 1
        else
          echo "Found Cache for $DEPSHASH at $cache_path"
        fi

        echo "Persisted cache is $(du -sh $cache_path | cut -f1)"
        mkdir temp-cache
        tar -xf $cache_path -C temp-cache
        echo "Unzipped cache is $(du -sh temp-cache/src | cut -f1)"

        if [ -d "temp-cache/src" ]; then
          echo "Relocating Cache"
          rm -rf src
          mv temp-cache/src src
        fi

        if [ ! -d "src/third_party/blink" ]; then
          echo "Cache was not correctly restored - exiting"
          exit 1
        fi

        echo "Wiping Electron Directory"
        rm -rf src/electron        
    - name: Checkout Electron
      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
      with:
        path: src/electron
        fetch-depth: 0
    - name: Run Electron Only Hooks
      run: |
        echo "Running Electron Only Hooks"
        gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"        
    - name: Regenerate DEPS Hash
      run: |
        (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target
        echo "DEPSHASH=$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV        
    - name: Add CHROMIUM_BUILDTOOLS_PATH to env
      run: echo "CHROMIUM_BUILDTOOLS_PATH=$(pwd)/src/buildtools" >> $GITHUB_ENV
    - name: Fix Known Hosts on Linux
      run: src/electron/.circleci/fix-known-hosts.sh
    - name: Install build-tools & Setup RBE
      run: |
        echo "NUMBER_OF_NINJA_PROCESSES=300" >> $GITHUB_ENV
        cd ~/.electron_build_tools
        npx yarn --ignore-engines

        # Pull down credential helper and print status
        node -e "require('./src/utils/reclient.js').downloadAndPrepare({})"
        HELPER=$(node -p "require('./src/utils/reclient.js').helperPath({})")
        $HELPER login

        echo 'RBE_service='`node -e "console.log(require('./src/utils/reclient.js').serviceAddress)"` >> $GITHUB_ENV
        echo 'RBE_experimental_credentials_helper='`node -e "console.log(require('./src/utils/reclient.js').helperPath({}))"` >> $GITHUB_ENV
        echo 'RBE_experimental_credentials_helper_args=print' >> $GITHUB_ENV        
    - name: Build Electron
      run: |
        cd src/electron
        # TODO(codebytere): remove this once we figure out why .git/packed-refs is initially missing
        git pack-refs
        cd ..

        NINJA_SUMMARIZE_BUILD=1 e build -j $NUMBER_OF_NINJA_PROCESSES
        cp out/Default/.ninja_log out/electron_ninja_log
        node electron/script/check-symlinks.js        
    - name: Build Electron dist.zip
      run: |
        cd src
        e build electron:electron_dist_zip -j $NUMBER_OF_NINJA_PROCESSES
        if [ "${{ env.CHECK_DIST_MANIFEST }}" = "true" ]; then
          target_os=linux
          target_cpu=${{ matrix.build-arch }}
          electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.${{ matrix.build-arch }}.manifest
        fi        
    - name: Build Mksnapshot
      run: |
        cd src
        e build electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES
        gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args

        # Remove unused args from mksnapshot_args
        SEDOPTION="-i"
        sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args
        sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args
        sed $SEDOPTION '/The gn arg use_goma=true .*/d' out/Default/mksnapshot_args

        if [ "${{ matrix.build-arch }}" = "arm" ]; then
          electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/mksnapshot
          electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/v8_context_snapshot_generator
        elif [ "${{ matrix.build-arch }}" = "arm64" ]; then
          electron/script/strip-binaries.py --file $PWD/out/Default/clang_x64_v8_arm64/mksnapshot
          electron/script/strip-binaries.py --file $PWD/out/Default/clang_x64_v8_arm64/v8_context_snapshot_generator
        else
          electron/script/strip-binaries.py --file $PWD/out/Default/mksnapshot
          electron/script/strip-binaries.py --file $PWD/out/Default/v8_context_snapshot_generator
        fi        
      
        e build electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES
        (cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S)
    - name: Generate Cross-Arch Snapshot (arm/arm64)
      if: ${{ matrix.build-arch == 'arm' }} || ${{ matrix.build-arch == 'arm64' }}
      run: |
        cd src
        if [ "${{ matrix.build-arch }}" = "arm" ]; then
          export MKSNAPSHOT_PATH="clang_x86_v8_arm"
        elif [ "${{ matrix.build-arch }}" = "arm64" ]; then
          export MKSNAPSHOT_PATH="clang_x64_v8_arm64"
        fi

        cp "out/Default/$MKSNAPSHOT_PATH/mksnapshot" out/Default
        cp "out/Default/$MKSNAPSHOT_PATH/v8_context_snapshot_generator" out/Default
        cp "out/Default/$MKSNAPSHOT_PATH/libffmpeg.so" out/Default

        python3 electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --create-snapshot-only
        mkdir cross-arch-snapshots
        cp out/Default-mksnapshot-test/*.bin cross-arch-snapshots
        # Clean up so that ninja does not get confused
        rm -f out/Default/libffmpeg.so        
    - name: Build Chromedriver
      run: |
        cd src
        e build electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES
        e build electron:electron_chromedriver_zip        
    - name: Generate & Zip Symbols
      run: |
        # Generate breakpad symbols on release builds
        if [ "${{ inputs.generate-symbols }}" = "true" ]; then
          e build electron:electron_symbols
        fi
        cd src
        export BUILD_PATH="$(pwd)/out/Default"
        e build electron:licenses
        e build electron:electron_version_file
        if [ "${{ inputs.is-release }}" = "true" ]; then
          DELETE_DSYMS_AFTER_ZIP=1 electron/script/zip-symbols.py -b $BUILD_PATH
        else
          electron/script/zip-symbols.py -b $BUILD_PATH
        fi        
    - name: Generate FFMpeg
      if: ${{ inputs.is-release }}
      run: |
        cd src
        gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true $GN_EXTRA_ARGS"
        autoninja -C out/ffmpeg electron:electron_ffmpeg_zip -j $NUMBER_OF_NINJA_PROCESSES        
    - name: Generate Hunspell Dictionaries
      if: ${{ inputs.is-release }}
      run: |
        cd src
        autoninja -C out/Default electron:hunspell_dictionaries_zip -j $NUMBER_OF_NINJA_PROCESSES        
    - name: Generate TypeScript Definitions
      if: ${{ inputs.is-release }}
      run: |
        cd src/electron
        node script/yarn create-typescript-definitions        
    - name: Publish Electron Dist
      if: ${{ inputs.is-release }}
      run: |
        rm -rf src/out/Default/obj
        cd src/electron
        if [ "${{ inputs.upload-to-storage }}" = "1" ]; then
          echo 'Uploading Electron release distribution to Azure'
          script/release/uploaders/upload.py --verbose --upload_to_storage
        else
          echo 'Uploading Electron release distribution to GitHub releases'
          script/release/uploaders/upload.py --verbose
        fi        
    - name: Move all Generated Artifacts to Upload Folder
      run: ./src/electron/script/actions/move-artifacts.sh
    - name: Upload Generated Artifacts
      uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
      with:
        name: generated_artifacts_linux_${{ matrix.build-arch }}
        path: ./generated_artifacts_linux_${{ matrix.build-arch }}