electron/.github/workflows/linux-pipeline.yml

505 lines
19 KiB
YAML

name: Linux Pipeline
on:
workflow_call:
inputs:
is-release:
description: 'Whether this build job is a release job'
required: true
type: boolean
default: false
gn-config:
description: 'The gn arg configuration to use'
required: true
type: string
default: //electron/build/args/testing.gn
gn-build-type:
description: 'The gn build type - testing or release'
required: true
type: string
default: testing
generate-symbols:
description: 'Whether or not to generate symbols'
required: true
type: boolean
default: false
upload-to-storage:
description: 'Whether or not to upload build artifacts to external storage'
required: true
type: string
default: '0'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
ELECTRON_ARTIFACTS_BLOB_STORAGE: ${{ secrets.ELECTRON_ARTIFACTS_BLOB_STORAGE }}
ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }}
ELECTRON_GITHUB_TOKEN: ${{ secrets.ELECTRON_GITHUB_TOKEN }}
GN_CONFIG: ${{ inputs.gn-config }}
# Disable pre-compiled headers to reduce out size - only useful for rebuilds
GN_BUILDFLAG_ARGS: 'enable_precompiled_headers = false'
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
# Only disable this in the Asan build
CHECK_DIST_MANIFEST: true
IS_GHA_RELEASE: true
ELECTRON_OUT_DIR: Default
jobs:
checkout:
runs-on: aks-linux-large
container:
image: ghcr.io/electron/build:latest
options: --user root
volumes:
- /mnt/cross-instance-cache:/mnt/cross-instance-cache
steps:
- name: Checkout Electron
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
with:
path: src/electron
fetch-depth: 0
- name: Checkout & Sync & Save
uses: ./src/electron/.github/actions/checkout
build:
strategy:
fail-fast: false
matrix:
build-arch: [ x64 ] # arm64, arm
env:
TARGET_ARCH: ${{ matrix.build-arch }}
runs-on: aks-linux-large
container:
image: ghcr.io/electron/build:latest
options: --user root
volumes:
- /mnt/cross-instance-cache:/mnt/cross-instance-cache
needs: checkout
steps:
- name: Load Build Tools
run: |
export BUILD_TOOLS_SHA=ef894bc3cfa99d84a3b731252da0f83f500e4032
npm i -g @electron/build-tools
e auto-update disable
e init --root=$(pwd) --out=Default ${{ inputs.gn-build-type }} --import ${{ inputs.gn-build-type }} --target-cpu ${{ matrix.build-arch }}
- name: Checkout Electron
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
with:
path: src/electron
fetch-depth: 0
- name: Install Dependencies
run: |
cd src/electron
node script/yarn install
- name: Set GN_EXTRA_ARGS
run: |
if [ "${{ matrix.build-arch }}" = "arm" ]; then
GN_EXTRA_ARGS='target_cpu="arm" build_tflite_with_xnnpack=false'
elif [ "${{ matrix.build-arch }}" = "arm64" ]; then
GN_EXTRA_ARGS='target_cpu="arm64" fatal_linker_warnings=false enable_linux_installer=false'
fi
echo "GN_EXTRA_ARGS=$GN_EXTRA_ARGS" >> $GITHUB_ENV
- name: Get Depot Tools
timeout-minutes: 5
run: |
git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
cd depot_tools
git apply --3way ../src/electron/.github/workflows/config/gclient.diff
# Ensure depot_tools does not update.
test -d depot_tools && cd depot_tools
touch .disable_auto_update
- name: Add Depot Tools to PATH
run: echo "$(pwd)/depot_tools" >> $GITHUB_PATH
- name: Generate DEPS Hash
run: |
node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target
echo "DEPSHASH=v1-src-cache-$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV
- name: Restore and Ensure Src Cache
run: |
cache_path=/mnt/cross-instance-cache/$DEPSHASH.tar
echo "Using cache key: $DEPSHASH"
echo "Checking for cache in: $cache_path"
if [ ! -f "$cache_path" ]; then
echo "Cache Does Not Exist for $DEPSHASH - exiting"
exit 1
else
echo "Found Cache for $DEPSHASH at $cache_path"
fi
echo "Persisted cache is $(du -sh $cache_path | cut -f1)"
mkdir temp-cache
tar -xf $cache_path -C temp-cache
echo "Unzipped cache is $(du -sh temp-cache/src | cut -f1)"
if [ -d "temp-cache/src" ]; then
echo "Relocating Cache"
rm -rf src
mv temp-cache/src src
fi
if [ ! -d "src/third_party/blink" ]; then
echo "Cache was not correctly restored - exiting"
exit 1
fi
echo "Wiping Electron Directory"
rm -rf src/electron
- name: Checkout Electron
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
with:
path: src/electron
fetch-depth: 0
- name: Run Electron Only Hooks
run: |
echo "Running Electron Only Hooks"
gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
- name: Regenerate DEPS Hash
run: |
(cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target
echo "DEPSHASH=$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV
- name: Add CHROMIUM_BUILDTOOLS_PATH to env
run: echo "CHROMIUM_BUILDTOOLS_PATH=$(pwd)/src/buildtools" >> $GITHUB_ENV
- name: Install build-tools & Setup RBE
run: |
echo "NUMBER_OF_NINJA_PROCESSES=300" >> $GITHUB_ENV
cd ~/.electron_build_tools
npx yarn --ignore-engines
# Pull down credential helper and print status
node -e "require('./src/utils/reclient.js').downloadAndPrepare({})"
HELPER=$(node -p "require('./src/utils/reclient.js').helperPath({})")
$HELPER login
echo 'RBE_service='`node -e "console.log(require('./src/utils/reclient.js').serviceAddress)"` >> $GITHUB_ENV
echo 'RBE_experimental_credentials_helper='`node -e "console.log(require('./src/utils/reclient.js').helperPath({}))"` >> $GITHUB_ENV
echo 'RBE_experimental_credentials_helper_args=print' >> $GITHUB_ENV
- name: Build Electron
run: |
cd src/electron
# TODO(codebytere): remove this once we figure out why .git/packed-refs is initially missing
git pack-refs
cd ..
NINJA_SUMMARIZE_BUILD=1 e build -j $NUMBER_OF_NINJA_PROCESSES
cp out/Default/.ninja_log out/electron_ninja_log
node electron/script/check-symlinks.js
- name: Build Electron dist.zip
run: |
cd src
e build electron:electron_dist_zip -j $NUMBER_OF_NINJA_PROCESSES
if [ "${{ env.CHECK_DIST_MANIFEST }}" = "true" ]; then
target_os=linux
target_cpu=${{ matrix.build-arch }}
electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.${{ matrix.build-arch }}.manifest
fi
- name: Build Mksnapshot
run: |
cd src
e build electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES
gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args
# Remove unused args from mksnapshot_args
SEDOPTION="-i"
sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args
sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args
sed $SEDOPTION '/The gn arg use_goma=true .*/d' out/Default/mksnapshot_args
if [ "${{ matrix.build-arch }}" = "arm" ]; then
electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/mksnapshot
electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/v8_context_snapshot_generator
elif [ "${{ matrix.build-arch }}" = "arm64" ]; then
electron/script/strip-binaries.py --file $PWD/out/Default/clang_x64_v8_arm64/mksnapshot
electron/script/strip-binaries.py --file $PWD/out/Default/clang_x64_v8_arm64/v8_context_snapshot_generator
else
electron/script/strip-binaries.py --file $PWD/out/Default/mksnapshot
electron/script/strip-binaries.py --file $PWD/out/Default/v8_context_snapshot_generator
fi
e build electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES
(cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S)
- name: Generate Cross-Arch Snapshot (arm/arm64)
if: ${{ matrix.build-arch == 'arm' || matrix.build-arch == 'arm64' }}
run: |
cd src
if [ "${{ matrix.build-arch }}" = "arm" ]; then
MKSNAPSHOT_PATH="clang_x86_v8_arm"
elif [ "${{ matrix.build-arch }}" = "arm64" ]; then
MKSNAPSHOT_PATH="clang_x64_v8_arm64"
fi
cp "out/Default/$MKSNAPSHOT_PATH/mksnapshot" out/Default
cp "out/Default/$MKSNAPSHOT_PATH/v8_context_snapshot_generator" out/Default
cp "out/Default/$MKSNAPSHOT_PATH/libffmpeg.so" out/Default
python3 electron/script/verify-mksnapshot.py --source-root "$PWD" --build-dir out/Default --create-snapshot-only
mkdir cross-arch-snapshots
cp out/Default-mksnapshot-test/*.bin cross-arch-snapshots
# Clean up so that ninja does not get confused
rm -f out/Default/libffmpeg.so
- name: Build Chromedriver
run: |
cd src
e build electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES
e build electron:electron_chromedriver_zip
- name: Build Node.js headers
run: |
cd src
e build electron:node_headers
- name: Generate & Zip Symbols
run: |
# Generate breakpad symbols on release builds
if [ "${{ inputs.generate-symbols }}" = "true" ]; then
e build electron:electron_symbols
fi
cd src
export BUILD_PATH="$(pwd)/out/Default"
e build electron:licenses
e build electron:electron_version_file
if [ "${{ inputs.is-release }}" = "true" ]; then
DELETE_DSYMS_AFTER_ZIP=1 electron/script/zip-symbols.py -b $BUILD_PATH
else
electron/script/zip-symbols.py -b $BUILD_PATH
fi
- name: Generate FFMpeg
if: ${{ inputs.is-release }}
run: |
cd src
gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true $GN_EXTRA_ARGS"
autoninja -C out/ffmpeg electron:electron_ffmpeg_zip -j $NUMBER_OF_NINJA_PROCESSES
- name: Generate Hunspell Dictionaries
if: ${{ inputs.is-release }}
run: |
cd src
autoninja -C out/Default electron:hunspell_dictionaries_zip -j $NUMBER_OF_NINJA_PROCESSES
- name: Generate Libcxx
if: ${{ inputs.is-release }}
run: |
cd src
autoninja -C out/Default electron:libcxx_headers_zip -j $NUMBER_OF_NINJA_PROCESSES
autoninja -C out/Default electron:libcxxabi_headers_zip -j $NUMBER_OF_NINJA_PROCESSES
autoninja -C out/Default electron:libcxx_objects_zip -j $NUMBER_OF_NINJA_PROCESSES
- name: Publish Electron Dist
if: ${{ inputs.is-release }}
run: |
rm -rf src/out/Default/obj
cd src/electron
if [ "${{ inputs.upload-to-storage }}" = "1" ]; then
echo 'Uploading Electron release distribution to Azure'
script/release/uploaders/upload.py --verbose --upload_to_storage
else
echo 'Uploading Electron release distribution to GitHub releases'
script/release/uploaders/upload.py --verbose
fi
- name: Move all Generated Artifacts to Upload Folder
run: ./src/electron/script/actions/move-artifacts.sh
- name: Upload Generated Artifacts
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
with:
name: generated_artifacts_linux_${{ matrix.build-arch }}
path: ./generated_artifacts_linux_${{ matrix.build-arch }}
test:
if: ${{ inputs.is-release == false }}
runs-on: aks-linux-medium
container:
image: ghcr.io/electron/build:latest
options: --user root
needs: build
strategy:
fail-fast: false
matrix:
build-arch: [ arm64 ] # x64, arm
env:
TARGET_ARCH: ${{ matrix.build-arch }}
steps:
- name: Load Build Tools
run: |
export BUILD_TOOLS_SHA=ef894bc3cfa99d84a3b731252da0f83f500e4032
npm i -g @electron/build-tools
e auto-update disable
e init --root=$(pwd) --out=Default ${{ inputs.gn-build-type }}
- name: Checkout Electron
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
with:
path: src/electron
fetch-depth: 0
- name: Install Dependencies
run: |
cd src/electron
node script/yarn install
- name: Get Depot Tools
timeout-minutes: 5
run: |
git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
cd depot_tools
git apply --3way ../src/electron/.github/workflows/config/gclient.diff
# Ensure depot_tools does not update.
test -d depot_tools && cd depot_tools
touch .disable_auto_update
- name: Add Depot Tools to PATH
run: echo "$(pwd)/depot_tools" >> $GITHUB_PATH
- name: Download Generated Artifacts
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
with:
name: generated_artifacts_linux_${{ matrix.build-arch }}
path: ./generated_artifacts_linux_${{ matrix.build-arch }}
- name: Restore Generated Artifacts
run: ./src/electron/script/actions/restore-artifacts.sh
- name: Unzip Dist, Mksnapshot & Chromedriver
run: |
cd src/out/Default
unzip -:o dist.zip
unzip -:o chromedriver.zip
unzip -:o mksnapshot.zip
- name: Setup for headless testing
run: sh -e /etc/init.d/xvfb start
- name: Run Electron Tests
env:
MOCHA_REPORTER: mocha-multi-reporters
ELECTRON_TEST_RESULTS_DIR: junit
MOCHA_MULTI_REPORTERS: mocha-junit-reporter, tap
ELECTRON_DISABLE_SECURITY_WARNINGS: 1
ELECTRON_SKIP_NATIVE_MODULE_TESTS: true
run: |
cd src/electron
node script/yarn test --runners=main --trace-uncaught --enable-logging
- name: Wait for active SSH sessions
if: always() && !cancelled()
run: |
while [ -f /var/.ssh-lock ]
do
sleep 60
done
node-tests:
name: Run Node.js Tests
if: ${{ inputs.is-release == false }}
runs-on: aks-linux-medium
needs: build
timeout-minutes: 20
env:
TARGET_ARCH: x64
container:
image: ghcr.io/electron/build:latest
options: --user root
steps:
- name: Load Build Tools
run: |
export BUILD_TOOLS_SHA=ef894bc3cfa99d84a3b731252da0f83f500e4032
npm i -g @electron/build-tools
e auto-update disable
e init --root=$(pwd) --out=Default ${{ inputs.gn-build-type }}
- name: Checkout Electron
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
with:
path: src/electron
fetch-depth: 0
- name: Install Dependencies
run: |
cd src/electron
node script/yarn install
- name: Get Depot Tools
timeout-minutes: 5
run: |
git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
cd depot_tools
git apply --3way ../src/electron/.github/workflows/config/gclient.diff
# Ensure depot_tools does not update.
test -d depot_tools && cd depot_tools
touch .disable_auto_update
- name: Add Depot Tools to PATH
run: echo "$(pwd)/depot_tools" >> $GITHUB_PATH
- name: Download Generated Artifacts
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
with:
name: generated_artifacts_linux_${{ env.TARGET_ARCH }}
path: ./generated_artifacts_linux_${{ env.TARGET_ARCH }}
- name: Restore Generated Artifacts
run: ./src/electron/script/actions/restore-artifacts.sh
- name: Unzip Dist
run: |
cd src/out/Default
unzip -:o dist.zip
- name: Setup Linux for Headless Testing
run: sh -e /etc/init.d/xvfb start
- name: Run Node.js Tests
run: |
cd src
node electron/script/node-spec-runner.js --default --jUnitDir=junit
- name: Wait for active SSH sessions
if: always() && !cancelled()
run: |
while [ -f /var/.ssh-lock ]
do
sleep 60
done
nan-tests:
name: Run Nan Tests
if: ${{ inputs.is-release == false }}
runs-on: aks-linux-medium
needs: build
timeout-minutes: 20
env:
TARGET_ARCH: x64
container:
image: ghcr.io/electron/build:latest
options: --user root
steps:
- name: Load Build Tools
run: |
export BUILD_TOOLS_SHA=ef894bc3cfa99d84a3b731252da0f83f500e4032
npm i -g @electron/build-tools
e auto-update disable
e init --root=$(pwd) --out=Default ${{ inputs.gn-build-type }}
- name: Checkout Electron
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
with:
path: src/electron
fetch-depth: 0
- name: Install Dependencies
run: |
cd src/electron
node script/yarn install
- name: Get Depot Tools
timeout-minutes: 5
run: |
git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
cd depot_tools
git apply --3way ../src/electron/.github/workflows/config/gclient.diff
# Ensure depot_tools does not update.
test -d depot_tools && cd depot_tools
touch .disable_auto_update
- name: Add Depot Tools to PATH
run: echo "$(pwd)/depot_tools" >> $GITHUB_PATH
- name: Download Generated Artifacts
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
with:
name: generated_artifacts_linux_${{ env.TARGET_ARCH }}
path: ./generated_artifacts_linux_${{ env.TARGET_ARCH }}
- name: Restore Generated Artifacts
run: ./src/electron/script/actions/restore-artifacts.sh
- name: Unzip Dist
run: |
cd src/out/Default
unzip -:o dist.zip
- name: Setup Linux for Headless Testing
run: sh -e /etc/init.d/xvfb start
- name: Run Node.js Tests
run: |
cd src
node electron/script/nan-spec-runner.js
- name: Wait for active SSH sessions
if: always() && !cancelled()
run: |
while [ -f /var/.ssh-lock ]
do
sleep 60
done