mirrors/electron
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 1

build: add infra for reclient support (#40850)

Browse source 
* chore: add patch to always set macos platform for x-build

* build: add infra for reclient support

* build: override reclient version

* build: use RBE in CI

* chore: hardcode reclient fix version

* build: lower process count on macOS

* build: use large macOS instance for testing-arm64

* Revert "build: use large macOS instance for testing-arm64"

This reverts commit 6844adfd00a5230e68234112dfd84caa50d3f621.

* build: login in via helper not writing file

* chore: update patches

* build: use recelint from DEPS

* build: fix windows reproxy cfg

* build: use reclient in appveyor

* build: update WOA job too

* build: force another build

* build: do not checkout reclient
This commit is contained in:
Samuel Attard 2024-01-10 10:19:42 +13:00 • committed by GitHub
parent c184b93fc5
commit 3afb012ad1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 301 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
.circleci/config/base.yml
View file

@ -335,46 +335,26 @@ step-setup-env-for-build: &step-setup-env-for-build
# To find `gn` executable.
echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV
step-setup-goma-for-build: &step-setup-goma-for-build
step-setup-rbe-for-build: &step-setup-rbe-for-build
run:
name: Setup Goma
name: Setup RBE
command: |
echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV
if [ "`uname`" == "Darwin" ]; then
echo 'export NUMBER_OF_NINJA_PROCESSES=200' >> $BASH_ENV
echo 'ulimit -n 10000' >> $BASH_ENV
echo 'sudo launchctl limit maxfiles 65536 200000' >> $BASH_ENV
fi
if [ ! -z "$RAW_GOMA_AUTH" ]; then
echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config
fi
git clone https://github.com/electron/build-tools.git
cd build-tools
npm install
npx yarn --ignore-engines
mkdir third_party
node -e "require('./src/utils/goma.js').downloadAndPrepare({ gomaOneForAll: true })"
export GOMA_FALLBACK_ON_AUTH_FAILURE=true
third_party/goma/goma_ctl.py ensure_start
if [ ! -z "$RAW_GOMA_AUTH" ] && [ "`third_party/goma/goma_auth.py info`" != "Login as Fermi Planck" ]; then
echo "WARNING!!!!!! Goma authentication is incorrect; please update Goma auth token."
exit 1
fi
echo 'export GN_GOMA_FILE='`node -e "console.log(require('./src/utils/goma.js').gnFilePath)"` >> $BASH_ENV
echo 'export GOMA_DIR='`node -e "console.log(require('./src/utils/goma.js').dir)"` >> $BASH_ENV
echo 'export GOMA_FALLBACK_ON_AUTH_FAILURE=true' >> $BASH_ENV
cd ..
touch "${TMPDIR:=/tmp}"/.goma-ready
background: true
step-wait-for-goma: &step-wait-for-goma
run:
name: Wait for Goma
command: |
until [ -f "${TMPDIR:=/tmp}"/.goma-ready ]
do
sleep 5
done
echo "Goma ready"
no_output_timeout: 5m
# Pull down credential helper and print status
node -e "require('./src/utils/reclient.js').downloadAndPrepare({})"
HELPER=$(node -p "require('./src/utils/reclient.js').helperPath")
$HELPER login
echo 'export RBE_service='`node -e "console.log(require('./src/utils/reclient.js').serviceAddress)"` >> $BASH_ENV
echo 'export RBE_experimental_credentials_helper='`node -e "console.log(require('./src/utils/reclient.js').helperPath)"` >> $BASH_ENV
step-restore-brew-cache: &step-restore-brew-cache
restore_cache:
@ -548,6 +528,13 @@ step-fix-sync: &step-fix-sync
sed -i '' "s/Updating depot_tools... //g" gn_ensure_file
cipd ensure --root src/buildtools/mac -ensure-file gn_ensure_file
# Fix reclient (wrong binary)
echo 'infra/rbe/client/${platform}' `gclient getdep --deps-file=src/DEPS -r 'src/buildtools/reclient:infra/rbe/client/${platform}'` > gn_ensure_file
# Remove extra output from calling gclient getdep which always calls update_depot_tools
sed -i '' "s/Updating depot_tools... //g" gn_ensure_file
cipd ensure --root src/buildtools/reclient -ensure-file gn_ensure_file
python3 src/buildtools/reclient_cfgs/configure_reclient_cfgs.py --rbe_instance "projects/rbe-chrome-untrusted/instances/default_instance" --reproxy_cfg_template reproxy.cfg.template --rewrapper_cfg_project "" --skip_remoteexec_cfg_fetch
# Fix dsymutil (wrong binary)
if [ "$TARGET_ARCH" == "arm64" ]; then
export DSYM_SHA_FILE=src/tools/clang/dsymutil/bin/dsymutil.arm64.sha1
@ -600,7 +587,7 @@ step-gn-gen-default: &step-gn-gen-default
name: Default GN gen
command: |
cd src
gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
gn gen out/Default --args="import(\"$GN_CONFIG\") use_remoteexec=true $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
step-gn-check: &step-gn-check
run:
@ -636,16 +623,16 @@ step-electron-chromedriver-build: &step-electron-chromedriver-build
command: |
cd src
if [ "`uname`" != "Darwin" ] && ([ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]); then
gn gen out/chromedriver --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") is_component_ffmpeg=false proprietary_codecs=false $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
gn gen out/chromedriver --args="import(\"$GN_CONFIG\") use_remoteexec=true is_component_ffmpeg=false proprietary_codecs=false $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
export CHROMEDRIVER_DIR="out/chromedriver"
else
export CHROMEDRIVER_DIR="out/Default"
fi
ninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES
autoninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES
if [ "`uname`" == "Linux" ]; then
electron/script/strip-binaries.py --target-cpu="$TARGET_ARCH" --file $PWD/$CHROMEDRIVER_DIR/chromedriver
fi
ninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver_zip
autoninja -C $CHROMEDRIVER_DIR electron:electron_chromedriver_zip
if [ "`uname`" != "Darwin" ] && ([ "$TARGET_ARCH" == "arm" ] || [ "$TARGET_ARCH" == "arm64" ]); then
cp out/chromedriver/chromedriver.zip out/Default
fi
@ -655,7 +642,7 @@ step-nodejs-headers-build: &step-nodejs-headers-build
name: Build Node.js headers
command: |
cd src
ninja -C out/Default electron:node_headers
autoninja -C out/Default electron:node_headers
step-electron-publish: &step-electron-publish
run:
@ -709,14 +696,14 @@ step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen
name: ffmpeg GN gen
command: |
cd src
gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS"
gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true $GN_EXTRA_ARGS"
step-ffmpeg-build: &step-ffmpeg-build
run:
name: Non proprietary ffmpeg build
command: |
cd src
ninja -C out/ffmpeg electron:electron_ffmpeg_zip -j $NUMBER_OF_NINJA_PROCESSES
autoninja -C out/ffmpeg electron:electron_ffmpeg_zip -j $NUMBER_OF_NINJA_PROCESSES
step-verify-mksnapshot: &step-verify-mksnapshot
run:
@ -748,26 +735,13 @@ step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing
sh -e /etc/init.d/xvfb start
fi
step-show-goma-stats: &step-show-goma-stats
run:
shell: /bin/bash
name: Check goma stats after build
command: |
set +e
set +o pipefail
python3 $GOMA_DIR/goma_ctl.py stat
python3 $GOMA_DIR/diagnose_goma_log.py
true
when: always
background: true
step-mksnapshot-build: &step-mksnapshot-build
run:
name: mksnapshot build
no_output_timeout: 30m
command: |
cd src
ninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES
autoninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES
gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args
# Remove unused args from mksnapshot_args
SEDOPTION="-i"
@ -790,7 +764,7 @@ step-mksnapshot-build: &step-mksnapshot-build
fi
fi
if [ "$SKIP_DIST_ZIP" != "1" ]; then
ninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES
autoninja -C out/Default electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES
(cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S)
fi
@ -800,7 +774,7 @@ step-hunspell-build: &step-hunspell-build
command: |
cd src
if [ "$SKIP_DIST_ZIP" != "1" ]; then
ninja -C out/Default electron:hunspell_dictionaries_zip -j $NUMBER_OF_NINJA_PROCESSES
autoninja -C out/Default electron:hunspell_dictionaries_zip -j $NUMBER_OF_NINJA_PROCESSES
fi
step-maybe-generate-libcxx: &step-maybe-generate-libcxx
@ -809,9 +783,9 @@ step-maybe-generate-libcxx: &step-maybe-generate-libcxx
command: |
cd src
if [ "`uname`" == "Linux" ]; then
ninja -C out/Default electron:libcxx_headers_zip -j $NUMBER_OF_NINJA_PROCESSES
ninja -C out/Default electron:libcxxabi_headers_zip -j $NUMBER_OF_NINJA_PROCESSES
ninja -C out/Default electron:libcxx_objects_zip -j $NUMBER_OF_NINJA_PROCESSES
autoninja -C out/Default electron:libcxx_headers_zip -j $NUMBER_OF_NINJA_PROCESSES
autoninja -C out/Default electron:libcxxabi_headers_zip -j $NUMBER_OF_NINJA_PROCESSES
autoninja -C out/Default electron:libcxx_objects_zip -j $NUMBER_OF_NINJA_PROCESSES
fi
step-maybe-generate-breakpad-symbols: &step-maybe-generate-breakpad-symbols
@ -821,7 +795,7 @@ step-maybe-generate-breakpad-symbols: &step-maybe-generate-breakpad-symbols
command: |
if [ "$GENERATE_SYMBOLS" == "true" ]; then
cd src
ninja -C out/Default electron:electron_symbols
autoninja -C out/Default electron:electron_symbols
fi
step-maybe-zip-symbols: &step-maybe-zip-symbols
@ -830,8 +804,8 @@ step-maybe-zip-symbols: &step-maybe-zip-symbols
command: |
cd src
export BUILD_PATH="$PWD/out/Default"
ninja -C out/Default electron:licenses
ninja -C out/Default electron:electron_version_file
autoninja -C out/Default electron:licenses
autoninja -C out/Default electron:electron_version_file
electron/script/zip-symbols.py -b $BUILD_PATH
step-maybe-zip-symbols-and-clean: &step-maybe-zip-symbols-and-clean
@ -840,8 +814,8 @@ step-maybe-zip-symbols-and-clean: &step-maybe-zip-symbols-and-clean
command: |
cd src
export BUILD_PATH="$PWD/out/Default"
ninja -C out/Default electron:licenses
ninja -C out/Default electron:electron_version_file
autoninja -C out/Default electron:licenses
autoninja -C out/Default electron:electron_version_file
DELETE_DSYMS_AFTER_ZIP=1 electron/script/zip-symbols.py -b $BUILD_PATH
step-maybe-cross-arch-snapshot: &step-maybe-cross-arch-snapshot
@ -1188,11 +1162,10 @@ commands:
could-be-aks:
type: boolean
steps:
- *step-setup-goma-for-build
- *step-setup-rbe-for-build
- checkout-from-cache:
could-be-aks: << parameters.could-be-aks >>
- *step-setup-env-for-build
- *step-wait-for-goma
- *step-gn-gen-default
- *step-gn-check
build_and_save_artifacts:
@ -1213,8 +1186,6 @@ commands:
- step-electron-dist-build:
additional-targets: electron:node_headers third_party/electron_node:overlapped-checker electron:hunspell_dictionaries_zip
- *step-show-goma-stats
# mksnapshot
- *step-mksnapshot-build
- *step-maybe-cross-arch-snapshot
@ -1343,7 +1314,7 @@ commands:
command: |
cd src
if [ "$SKIP_DIST_ZIP" != "1" ]; then
ninja -C out/Default electron:electron_dist_zip << parameters.additional-targets >> -j $NUMBER_OF_NINJA_PROCESSES
autoninja -C out/Default electron:electron_dist_zip << parameters.additional-targets >> -j $NUMBER_OF_NINJA_PROCESSES
if [ "$CHECK_DIST_MANIFEST" == "1" ]; then
if [ "`uname`" == "Darwin" ]; then
target_os=mac
@ -1444,7 +1415,7 @@ commands:
- when:
condition: << parameters.build >>
steps:
- *step-setup-goma-for-build
- *step-setup-rbe-for-build
- when:
condition: << parameters.checkout-and-assume-cache >>
steps:
@ -1563,7 +1534,6 @@ commands:
steps:
- *step-depot-tools-add-to-path
- *step-setup-env-for-build
- *step-wait-for-goma
- *step-get-more-space-on-mac
- *step-fix-sync
- *step-delete-git-directories
@ -1751,14 +1721,12 @@ commands:
- *step-fix-sync
- *step-setup-env-for-build
- *step-fix-known-hosts-linux
- *step-setup-goma-for-build
- *step-wait-for-goma
- *step-setup-rbe-for-build
- *step-gn-gen-default
# Electron app
- ninja_build_electron:
build-type: << parameters.build-type >>
- *step-show-goma-stats
- *step-maybe-generate-breakpad-symbols
- *step-maybe-electron-dist-strip
- step-electron-dist-build

7
DEPS
View file

@ -13,6 +13,8 @@ vars = {
'74ab5baccc6f7202c8ac69a8d1e152c29dc1ea76',
'mantle_version':
'78d3966b3c331292ea29ec38661b25df0a245948',
'engflow_reclient_configs_version':
'955335c30a752e9ef7bff375baab5e0819b6c00d',
'pyyaml_version': '3.12',
@ -23,6 +25,7 @@ vars = {
'squirrel_git': 'https://github.com/Squirrel',
'reactiveobjc_git': 'https://github.com/ReactiveCocoa',
'mantle_git': 'https://github.com/Mantle',
'engflow_git': 'https://github.com/EngFlow',
# The path of the sysroots.json file.
'sysroots_json_path': 'electron/script/sysroots.json',
@ -102,6 +105,10 @@ deps = {
'src/third_party/squirrel.mac/vendor/Mantle': {
'url': Var("mantle_git") + '/Mantle.git@' + Var("mantle_version"),
'condition': 'process_deps',
},
'src/third_party/engflow-reclient-configs': {
'url': Var("engflow_git") + '/reclient-configs.git@' + Var("engflow_reclient_configs_version"),
'condition': 'process_deps'
}
}

1
appveyor-bake.yml
View file

@ -13,7 +13,6 @@ environment:
ELECTRON_ENABLE_STACK_DUMPING: 1
MOCHA_REPORTER: mocha-multi-reporters
MOCHA_MULTI_REPORTERS: mocha-appveyor-reporter, tap
GOMA_FALLBACK_ON_AUTH_FAILURE: true
DEPOT_TOOLS_WIN_TOOLCHAIN: 0
PYTHONIOENCODING: UTF-8

51
appveyor-woa.yml
View file

@ -37,7 +37,6 @@ environment:
ELECTRON_ALSO_LOG_TO_STDERR: 1
MOCHA_REPORTER: mocha-multi-reporters
MOCHA_MULTI_REPORTERS: "@marshallofsound/mocha-appveyor-reporter, tap"
GOMA_FALLBACK_ON_AUTH_FAILURE: true
DEPOT_TOOLS_WIN_TOOLCHAIN: 1
DEPOT_TOOLS_WIN_TOOLCHAIN_BASE_URL: "https://dev-cdn.electronjs.org/windows-toolchains/_"
GYP_MSVS_HASH_27370823e7: 28622d16b1
@ -101,31 +100,20 @@ for:
if (Test-Path -Path "$pwd\src\electron") {
Remove-Item -Recurse -Force $pwd\src\electron
}
- ps: >-
if (Test-Path 'env:RAW_GOMA_AUTH') {
$env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config"
$env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE
}
- git clone https://github.com/electron/build-tools.git
- cd build-tools
- npm install
- npx yarn --ignore-engines
- mkdir third_party
- ps: >-
node -e "require('./src/utils/goma.js').downloadAndPrepare({ gomaOneForAll: true })"
- ps: $env:GN_GOMA_FILE = node -e "console.log(require('./src/utils/goma.js').gnFilePath)"
- ps: $env:LOCAL_GOMA_DIR = node -e "console.log(require('./src/utils/goma.js').dir)"
- cd ..\..
- ps: .\src\electron\script\start-goma.ps1 -gomaDir $env:LOCAL_GOMA_DIR
node -e "require('./src/utils/reclient.js').downloadAndPrepare({})"
- ps: $env:RECLIENT_HELPER = node -p "require('./src/utils/reclient.js').helperPath"
- ps: >-
if (Test-Path 'env:RAW_GOMA_AUTH') {
$goma_login = python3 $env:LOCAL_GOMA_DIR\goma_auth.py info
if ($goma_login -eq 'Login as Fermi Planck') {
Write-warning "Goma authentication is correct";
} else {
Write-warning "WARNING!!!!!! Goma authentication is incorrect; please update Goma auth token.";
$host.SetShouldExit(1)
}
}
& $env:RECLIENT_HELPER login
- ps: >-
$env:RBE_service = node -e "console.log(require('./src/utils/reclient.js').serviceAddress)"
- ps: >-
$env:RBE_experimental_credentials_helper = $env:RECLIENT_HELPER
- cd ..\..
- ps: $env:CHROMIUM_BUILDTOOLS_PATH="$pwd\src\buildtools"
- ps: >-
if ($env:GN_CONFIG -ne 'release') {
@ -147,27 +135,26 @@ for:
- cd src
- ps: $env:PATH="$pwd\third_party\ninja;$env:PATH"
- set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn
- gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"%GN_GOMA_FILE%\") %GN_EXTRA_ARGS% "
- gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") use_remoteexec=true %GN_EXTRA_ARGS% "
- gn check out/Default //electron:electron_lib
- gn check out/Default //electron:electron_app
- gn check out/Default //electron/shell/common/api:mojo
- if DEFINED GN_GOMA_FILE (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app)
- if DEFINED ELECTRON_RBE_JWT (autoninja -j 300 -C out/Default electron:electron_app) else (autoninja -C out/Default electron:electron_app)
- if "%GN_CONFIG%"=="testing" ( python C:\depot_tools\post_build_ninja_summary.py -C out\Default )
- gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%"
- ninja -C out/ffmpeg electron:electron_ffmpeg_zip
- ninja -C out/Default electron:electron_dist_zip
- gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true %GN_EXTRA_ARGS%"
- autoninja -C out/ffmpeg electron:electron_ffmpeg_zip
- autoninja -C out/Default electron:electron_dist_zip
- gn desc out/Default v8:run_mksnapshot_default args > out/Default/default_mksnapshot_args
# Remove unused args from mksnapshot_args
- ps: >-
Get-Content out/Default/default_mksnapshot_args | Where-Object { -not $_.Contains('--turbo-profiling-input') -And -not $_.Contains('builtins-pgo') -And -not $_.Contains('The gn arg use_goma=true') } | Set-Content out/Default/mksnapshot_args
- ninja -C out/Default electron:electron_mksnapshot_zip
- autoninja -C out/Default electron:electron_mksnapshot_zip
- cd out\Default
- 7z a mksnapshot.zip mksnapshot_args gen\v8\embedded.S
- cd ..\..
- ninja -C out/Default electron:hunspell_dictionaries_zip
- ninja -C out/Default electron:electron_chromedriver_zip
- ninja -C out/Default electron:node_headers
- python3 %LOCAL_GOMA_DIR%\goma_ctl.py stat
- autoninja -C out/Default electron:hunspell_dictionaries_zip
- autoninja -C out/Default electron:electron_chromedriver_zip
- autoninja -C out/Default electron:node_headers
- ps: >-
Get-CimInstance -Namespace root\cimv2 -Class Win32_product | Select vendor, description, @{l='install_location';e='InstallLocation'}, @{l='install_date';e='InstallDate'}, @{l='install_date_2';e='InstallDate2'}, caption, version, name, @{l='sku_number';e='SKUNumber'} | ConvertTo-Json | Out-File -Encoding utf8 -FilePath .\installed_software.json
- python3 electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json
@ -177,7 +164,7 @@ for:
if ($env:GN_CONFIG -eq 'release') {
# Needed for msdia140.dll on 64-bit windows
$env:Path += ";$pwd\third_party\llvm-build\Release+Asserts\bin"
ninja -C out/Default electron:electron_symbols
autoninja -C out/Default electron:electron_symbols
}
- ps: >-
if ($env:GN_CONFIG -eq 'release') {

51
appveyor.yml
View file

@ -37,7 +37,6 @@ environment:
ELECTRON_ALSO_LOG_TO_STDERR: 1
MOCHA_REPORTER: mocha-multi-reporters
MOCHA_MULTI_REPORTERS: "@marshallofsound/mocha-appveyor-reporter, tap"
GOMA_FALLBACK_ON_AUTH_FAILURE: true
DEPOT_TOOLS_WIN_TOOLCHAIN: 1
DEPOT_TOOLS_WIN_TOOLCHAIN_BASE_URL: "https://dev-cdn.electronjs.org/windows-toolchains/_"
GYP_MSVS_HASH_27370823e7: 28622d16b1
@ -99,31 +98,20 @@ for:
if (Test-Path -Path "$pwd\src\electron") {
Remove-Item -Recurse -Force $pwd\src\electron
}
- ps: >-
if (Test-Path 'env:RAW_GOMA_AUTH') {
$env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config"
$env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE
}
- git clone https://github.com/electron/build-tools.git
- cd build-tools
- npm install
- npx yarn --ignore-engines
- mkdir third_party
- ps: >-
node -e "require('./src/utils/goma.js').downloadAndPrepare({ gomaOneForAll: true })"
- ps: $env:GN_GOMA_FILE = node -e "console.log(require('./src/utils/goma.js').gnFilePath)"
- ps: $env:LOCAL_GOMA_DIR = node -e "console.log(require('./src/utils/goma.js').dir)"
- cd ..\..
- ps: .\src\electron\script\start-goma.ps1 -gomaDir $env:LOCAL_GOMA_DIR
node -e "require('./src/utils/reclient.js').downloadAndPrepare({})"
- ps: $env:RECLIENT_HELPER = node -p "require('./src/utils/reclient.js').helperPath"
- ps: >-
if (Test-Path 'env:RAW_GOMA_AUTH') {
$goma_login = python3 $env:LOCAL_GOMA_DIR\goma_auth.py info
if ($goma_login -eq 'Login as Fermi Planck') {
Write-warning "Goma authentication is correct";
} else {
Write-warning "WARNING!!!!!! Goma authentication is incorrect; please update Goma auth token.";
$host.SetShouldExit(1)
}
}
& $env:RECLIENT_HELPER login
- ps: >-
$env:RBE_service = node -e "console.log(require('./src/utils/reclient.js').serviceAddress)"
- ps: >-
$env:RBE_experimental_credentials_helper = $env:RECLIENT_HELPER
- cd ..\..
- ps: $env:CHROMIUM_BUILDTOOLS_PATH="$pwd\src\buildtools"
- ps: >-
if ($env:GN_CONFIG -ne 'release') {
@ -145,27 +133,26 @@ for:
- cd src
- ps: $env:PATH="$pwd\third_party\ninja;$env:PATH"
- set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn
- gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"%GN_GOMA_FILE%\") %GN_EXTRA_ARGS% "
- gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") use_remoteexec=true %GN_EXTRA_ARGS% "
- gn check out/Default //electron:electron_lib
- gn check out/Default //electron:electron_app
- gn check out/Default //electron/shell/common/api:mojo
- if DEFINED GN_GOMA_FILE (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app)
- if DEFINED ELECTRON_RBE_JWT (autoninja -j 300 -C out/Default electron:electron_app) else (autoninja -C out/Default electron:electron_app)
- if "%GN_CONFIG%"=="testing" ( python C:\depot_tools\post_build_ninja_summary.py -C out\Default )
- gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%"
- ninja -C out/ffmpeg electron:electron_ffmpeg_zip
- ninja -C out/Default electron:electron_dist_zip
- gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") use_remoteexec=true %GN_EXTRA_ARGS%"
- autoninja -C out/ffmpeg electron:electron_ffmpeg_zip
- autoninja -C out/Default electron:electron_dist_zip
- gn desc out/Default v8:run_mksnapshot_default args > out/Default/default_mksnapshot_args
# Remove unused args from mksnapshot_args
- ps: >-
Get-Content out/Default/default_mksnapshot_args | Where-Object { -not $_.Contains('--turbo-profiling-input') -And -not $_.Contains('builtins-pgo') -And -not $_.Contains('The gn arg use_goma=true') } | Set-Content out/Default/mksnapshot_args
- ninja -C out/Default electron:electron_mksnapshot_zip
- autoninja -C out/Default electron:electron_mksnapshot_zip
- cd out\Default
- 7z a mksnapshot.zip mksnapshot_args gen\v8\embedded.S
- cd ..\..
- ninja -C out/Default electron:hunspell_dictionaries_zip
- ninja -C out/Default electron:electron_chromedriver_zip
- ninja -C out/Default electron:node_headers
- python3 %LOCAL_GOMA_DIR%\goma_ctl.py stat
- autoninja -C out/Default electron:hunspell_dictionaries_zip
- autoninja -C out/Default electron:electron_chromedriver_zip
- autoninja -C out/Default electron:node_headers
- ps: >-
Get-CimInstance -Namespace root\cimv2 -Class Win32_product | Select vendor, description, @{l='install_location';e='InstallLocation'}, @{l='install_date';e='InstallDate'}, @{l='install_date_2';e='InstallDate2'}, caption, version, name, @{l='sku_number';e='SKUNumber'} | ConvertTo-Json | Out-File -Encoding utf8 -FilePath .\installed_software.json
- python3 electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json
@ -174,7 +161,7 @@ for:
if ($env:GN_CONFIG -eq 'release') {
# Needed for msdia140.dll on 64-bit windows
$env:Path += ";$pwd\third_party\llvm-build\Release+Asserts\bin"
ninja -C out/Default electron:electron_symbols
autoninja -C out/Default electron:electron_symbols
}
- ps: >-
if ($env:GN_CONFIG -eq 'release') {

1
patches/chromium/.patches
View file

@ -128,3 +128,4 @@ chore_remove_check_is_test_on_script_injection_tracker.patch
fix_restore_original_resize_performance_on_macos.patch
feat_allow_code_cache_in_custom_schemes.patch
enable_partition_alloc_ref_count_size.patch
build_run_reclient_cfg_generator_after_chrome.patch

30
patches/chromium/build_run_reclient_cfg_generator_after_chrome.patch Normal file
View file

@ -0,0 +1,30 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Samuel Attard <samuel.r.attard@gmail.com>
Date: Thu, 28 Dec 2023 17:09:08 +1300
Subject: build: run reclient cfg generator after chrome
We can't run our generator after chromes in gclient hooks so instead we patch
their generator to run ours immediately after.
This can't be upstreamed though may be replaceable later with some upstream refactors
around reclient config generation.
diff --git a/buildtools/reclient_cfgs/configure_reclient_cfgs.py b/buildtools/reclient_cfgs/configure_reclient_cfgs.py
index 35aaf829ae998b52ca2ae21450a66f2c53fd457c..ecd46ccb33c5a8fe1cae526081a3bd1aa997cf81 100755
--- a/buildtools/reclient_cfgs/configure_reclient_cfgs.py
+++ b/buildtools/reclient_cfgs/configure_reclient_cfgs.py
@@ -317,4 +317,13 @@ def main():
if __name__ == "__main__":
- sys.exit(main())
+ r = main()
+ if r != 0:
+ sys.exit(r)
+
+ electron_script = os.path.join(CHROMIUM_SRC, 'third_party', 'engflow-reclient-configs', 'configure_reclient.py')
+ r = subprocess.call([sys.executable, electron_script, '--src_dir=src', '--force'])
+ if r != 0:
+ sys.exit(r)
+
+ print('done')

4
patches/config.json
View file

@ -21,5 +21,7 @@
"src/electron/patches/ReactiveObjC": "src/third_party/squirrel.mac/vendor/ReactiveObjC",
"src/electron/patches/webrtc": "src/third_party/webrtc"
"src/electron/patches/webrtc": "src/third_party/webrtc",
"src/electron/patches/reclient-configs": "src/third_party/engflow-reclient-configs"
}

3
patches/reclient-configs/.patches Normal file
View file

@ -0,0 +1,3 @@
fix_disable_compression_threshold.patch
fix_add_python_remote_wrapper.patch
build_on_windows_provide_clang_wrapper_as_toolchain_input.patch

19
patches/reclient-configs/build_on_windows_provide_clang_wrapper_as_toolchain_input.patch Normal file
View file

@ -0,0 +1,19 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Samuel Attard <marshallofsound@electronjs.org>
Date: Tue, 9 Jan 2024 11:18:28 +1300
Subject: build: on windows provide clang wrapper as toolchain input
Should be upstreamed, fixes the executable bit being missing on win->lin x builds
diff --git a/chromium-browser-clang/rewrapper_windows.cfg b/chromium-browser-clang/rewrapper_windows.cfg
index 543b68bbd736bfb6cb2cf3e939c4cdc34b6a18da..539f37a53d990df39203496bb2d579f2ce38de75 100644
--- a/chromium-browser-clang/rewrapper_windows.cfg
+++ b/chromium-browser-clang/rewrapper_windows.cfg
@@ -15,6 +15,5 @@
# This config is merged with Chromium config. See README.md.
server_address=pipe://reproxy.pipe
-inputs={src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper
-toolchain_inputs={linux_clang_base_path}/bin/clang
+toolchain_inputs={linux_clang_base_path}/bin/clang,{src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper
remote_wrapper={src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/clang_remote_wrapper

144
patches/reclient-configs/fix_add_python_remote_wrapper.patch Normal file
View file

@ -0,0 +1,144 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Samuel Attard <samuel.r.attard@gmail.com>
Date: Thu, 28 Dec 2023 16:47:13 +1300
Subject: fix: add python_remote_wrapper
Our RBE cluster struggles to find python3 but bash finds it when spawned
so we wrap all python remote execs with a helper bash script that just
runs the underlying command.
diff --git a/configure_reclient.py b/configure_reclient.py
index 5948be95b6b19e6af4f9a9503c18ce2bebf63f2d..a157ceeb155987f6ab9b11d3cef8c2c9894e0436 100755
--- a/configure_reclient.py
+++ b/configure_reclient.py
@@ -21,6 +21,7 @@ import os
import re
import runpy
import shutil
+import stat
import string
import subprocess
import sys
@@ -109,6 +110,8 @@ class ReclientConfigurator:
self.download_linux_clang_toolchain()
self.generate_clang_remote_wrapper()
+ self.generate_python_remote_wrapper()
+
# Reproxy config includes auth and network-related parameters.
self.generate_reproxy_cfg()
# Rewrapper configs describe how different tools should be run remotely.
@@ -199,6 +202,32 @@ class ReclientConfigurator:
(f'{Paths.src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/'
'clang_remote_wrapper'), clang_remote_wrapper)
+ FileUtils.chmod_x((f'{Paths.src_dir}/buildtools/reclient_cfgs/chromium-browser-clang/'
+ 'clang_remote_wrapper'))
+
+ @staticmethod
+ def generate_python_remote_wrapper():
+ # Load python remote wrapper template.
+ template_file = (f'{Paths.script_dir}/python/'
+ 'python_remote_wrapper.template')
+ python_remote_wrapper_template = FileUtils.read_text_file(template_file)
+
+ # Variables to set in the template.
+ template_vars = {
+ }
+
+ # Substitute variables into the template.
+ python_remote_wrapper = ShellTemplate(
+ python_remote_wrapper_template).substitute(template_vars)
+
+ # Write the python remote wrapper.
+ FileUtils.write_text_file(
+ (f'{Paths.src_dir}/buildtools/reclient_cfgs/python/'
+ 'python_remote_wrapper'), python_remote_wrapper)
+
+ FileUtils.chmod_x((f'{Paths.src_dir}/buildtools/reclient_cfgs/python/'
+ 'python_remote_wrapper'))
+
def generate_reproxy_cfg(self):
# Load Chromium config template and remove everything starting with $
# symbol on each line.
@@ -536,6 +565,11 @@ class FileUtils:
shutil.move(filepath_new, filepath)
+ @classmethod
+ def chmod_x(cls, filepath):
+ st = os.stat(filepath)
+ os.chmod(filepath, st.st_mode | stat.S_IEXEC)
+
@classmethod
def create_generated_header(cls, source_files):
if not isinstance(source_files, (list, tuple)):
diff --git a/python/python_remote_wrapper.template b/python/python_remote_wrapper.template
new file mode 100644
index 0000000000000000000000000000000000000000..54817e4f6f9e3cb2f1e7ea1317fa8fefdf7a7da5
--- /dev/null
+++ b/python/python_remote_wrapper.template
@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright (c) 2023 Contributors to the reclient-configs project. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# AUTOGENERATED FILE - DO NOT EDIT
+# Generated by:
+# {configurator_dir}/configure_reclient.py
+# To edit update:
+# {configurator_dir}/python/python_remote_wrapper.template
+# And rerun configurator.
+
+# WARNING: This file is a part of reclient action inputs. Any modification will
+# invalidate remote cache.
+
+set -e
+
+# Launch
+"$1" "${@:2}"
diff --git a/python/rewrapper_linux.cfg b/python/rewrapper_linux.cfg
index 951bc66afd28b280fe936141f0b1a8e473bdba05..4ac7ec4b7559ffc133912ee932f9cf8e7450e4db 100644
--- a/python/rewrapper_linux.cfg
+++ b/python/rewrapper_linux.cfg
@@ -13,3 +13,6 @@
# limitations under the License.
# This config is merged with Chromium config. See README.md.
+
+inputs=buildtools/reclient_cfgs/python/python_remote_wrapper
+remote_wrapper=../../buildtools/reclient_cfgs/python/python_remote_wrapper
\ No newline at end of file
diff --git a/python/rewrapper_mac.cfg b/python/rewrapper_mac.cfg
index 951bc66afd28b280fe936141f0b1a8e473bdba05..4ac7ec4b7559ffc133912ee932f9cf8e7450e4db 100644
--- a/python/rewrapper_mac.cfg
+++ b/python/rewrapper_mac.cfg
@@ -13,3 +13,6 @@
# limitations under the License.
# This config is merged with Chromium config. See README.md.
+
+inputs=buildtools/reclient_cfgs/python/python_remote_wrapper
+remote_wrapper=../../buildtools/reclient_cfgs/python/python_remote_wrapper
\ No newline at end of file
diff --git a/python/rewrapper_windows.cfg b/python/rewrapper_windows.cfg
index 7ff80607546455eb7c5f16a410770d18949cd7f5..ce19dc95094d87b494e4b89a2bf1a368ace567a2 100644
--- a/python/rewrapper_windows.cfg
+++ b/python/rewrapper_windows.cfg
@@ -15,3 +15,5 @@
# This config is merged with Chromium config. See README.md.
server_address=pipe://reproxy.pipe
+inputs=buildtools/reclient_cfgs/python/python_remote_wrapper
+remote_wrapper=../../buildtools/reclient_cfgs/python/python_remote_wrapper

17
patches/reclient-configs/fix_disable_compression_threshold.patch Normal file
View file

@ -0,0 +1,17 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Samuel Attard <samuel.r.attard@gmail.com>
Date: Thu, 28 Dec 2023 16:13:18 +1300
Subject: fix: disable compression_threshold
This appears to cause issues with nginx ingress for RBE endpoints, disable
it till we can figure out why.
diff --git a/reproxy.cfg b/reproxy.cfg
index 4325d36cbfcb6d81eb0f8dc7ab7cf8e8857d608c..2390a73f4674a591ef833d10697e4aaf3894c011 100644
--- a/reproxy.cfg
+++ b/reproxy.cfg
@@ -17,3 +17,4 @@
# Unset Chromium variables.
service=
automatic_auth=
+compression_threshold=

1
script/prepare-appveyor.js
View file

@ -110,7 +110,6 @@ async function callAppVeyorBuildJobs (targetBranch, job, options) {
ELECTRON_OUT_DIR: 'Default',
ELECTRON_ENABLE_STACK_DUMPING: 1,
ELECTRON_ALSO_LOG_TO_STDERR: 1,
GOMA_FALLBACK_ON_AUTH_FAILURE: true,
DEPOT_TOOLS_WIN_TOOLCHAIN: 0,
PYTHONIOENCODING: 'UTF-8'
};

10
script/start-goma.ps1
View file

@ -1,10 +0,0 @@
param([string]$gomaDir=$PWD)
$cmdPath = Join-Path -Path $gomaDir -ChildPath "goma_ctl.py"
Start-Process -FilePath cmd -ArgumentList "/C", "python3", "$cmdPath", "ensure_start"
$timedOut = $false; $waitTime = 0; $waitIncrement = 5; $timeout=120;
Do { sleep $waitIncrement; $timedOut = (($waitTime+=$waitIncrement) -gt $timeout); iex "$gomaDir\gomacc.exe port 2" > $null; } Until(($LASTEXITCODE -eq 0) -or $timedOut)
if ($timedOut) {
write-error 'Timed out waiting for goma to start'; exit 1;
} else {
Write-Output "Successfully started goma!"
}