From dc2fcff01c0fe59bb2144f93cf2d89d92670771b Mon Sep 17 00:00:00 2001 From: John Kleinschmidt Date: Thu, 23 Jan 2020 09:47:38 -0800 Subject: [PATCH] ci: use goma for windows and linux builds (#21868) * ci: use goma for windows and linux builds --- .circleci/config.yml | 46 +++++++++++++++++++++++++++++++++++++------ appveyor.yml | 16 ++++++++++----- script/start-goma.ps1 | 11 +++++++++++ 3 files changed, 62 insertions(+), 11 deletions(-) create mode 100755 script/start-goma.ps1 diff --git a/.circleci/config.yml b/.circleci/config.yml index ec05c2ac7bc5..5407112250e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,7 +53,7 @@ parameters: # Build machines configs. docker-image: &docker-image docker: - - image: electronjs/build:697b894f36d127155e020f4e8ad4b2e5f6a09613 + - image: electronjs/build:3bba0fdee0d5650e751c2561f1806f9138dfe56a machine-linux-medium: &machine-linux-medium <<: *docker-image @@ -219,6 +219,15 @@ step-setup-env-for-build: &step-setup-env-for-build command: | # To find `gn` executable. echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV + # Setup Goma if applicable + if [ ! -z "$RAW_GOMA_AUTH" ] && [ "`uname`" == "Linux" ]; then + export USE_SCCACHE="false" + echo 'export USE_SCCACHE=false' >> $BASH_ENV + echo 'export USE_GOMA=true' >> $BASH_ENV + echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV + echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config + src/electron/external_binaries/goma/goma_ctl.py ensure_start + fi if [ "$USE_SCCACHE" == "true" ]; then # https://github.com/mozilla/sccache @@ -320,7 +329,11 @@ step-gn-gen-default: &step-gn-gen-default name: Default GN gen command: | cd src - gn gen out/Default --args='import("'$GN_CONFIG'") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" + if [ "$USE_GOMA" == "true" ]; then + gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"//electron/build/args/goma.gn\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" + else + gn gen out/Default --args="import(\"$GN_CONFIG\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS" + fi step-gn-check: &step-gn-check run: @@ -499,7 +512,11 @@ step-ffmpeg-gn-gen: &step-ffmpeg-gn-gen name: ffmpeg GN gen command: | cd src - gn gen out/ffmpeg --args='import("//electron/build/args/ffmpeg.gn") cc_wrapper="'"$SCCACHE_PATH"'"'" $GN_EXTRA_ARGS" + if [ "$USE_GOMA" == "true" ]; then + gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") import(\"//electron/build/args/goma.gn\") $GN_EXTRA_ARGS" + else + gn gen out/ffmpeg --args="import(\"//electron/build/args/ffmpeg.gn\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS" + fi step-ffmpeg-build: &step-ffmpeg-build run: @@ -537,11 +554,14 @@ step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing step-show-sccache-stats: &step-show-sccache-stats run: - name: Check sccache stats after build + name: Check sccache/goma stats after build command: | if [ "$SCCACHE_PATH" != "" ]; then $SCCACHE_PATH -s fi + if [ "$USE_GOMA" == "true" ]; then + src/electron/external_binaries/goma/goma_ctl.py stat + fi step-mksnapshot-build: &step-mksnapshot-build run: @@ -1264,6 +1284,9 @@ commands: build: type: boolean default: true + use-out-cache: + type: boolean + default: true steps: - when: condition: << parameters.attach >> @@ -1323,7 +1346,10 @@ commands: - *step-gn-gen-default # Electron app - - *step-restore-out-cache + - when: + condition: << parameters.use-out-cache >> + steps: + - *step-restore-out-cache - *step-gn-gen-default - *step-electron-build - *step-ninja-summary @@ -1385,7 +1411,10 @@ commands: # These files do not seem to like being in a cache, let us remove them find . -type f -name '*_pkg_info' -delete fi - - *step-save-out-cache + - when: + condition: << parameters.use-out-cache >> + steps: + - *step-save-out-cache # Trigger tests on arm hardware if needed - *step-maybe-trigger-arm-test @@ -1488,6 +1517,7 @@ jobs: - electron-build: persist: true checkout: true + use-out-cache: false linux-x64-testing-no-run-as-node: <<: *machine-linux-2xlarge @@ -1502,6 +1532,7 @@ jobs: - electron-build: persist: false checkout: true + use-out-cache: false linux-x64-testing-gn-check: <<: *machine-linux-medium @@ -1552,6 +1583,7 @@ jobs: - electron-build: persist: true checkout: true + use-out-cache: false linux-ia32-chromedriver: <<: *machine-linux-medium @@ -1600,6 +1632,7 @@ jobs: - electron-build: persist: false checkout: true + use-out-cache: false linux-arm-chromedriver: <<: *machine-linux-medium @@ -1648,6 +1681,7 @@ jobs: - electron-build: persist: false checkout: true + use-out-cache: false linux-arm64-testing-gn-check: <<: *machine-linux-medium diff --git a/appveyor.yml b/appveyor.yml index 8b545430ee56..1a1e64273508 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -108,15 +108,21 @@ build_script: Write-warning "Could not save source to shared drive; continuing anyway" } } + - ps: >- + if (Test-Path 'env:RAW_GOMA_AUTH') { + $env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config" + $env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE + .\src\electron\script\start-goma.ps1 -gomaDir "$pwd\src\electron\external_binaries\goma" + } - cd src - - ps: $env:BUILD_CONFIG_PATH="//electron/build/args/%GN_CONFIG%.gn" - - gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\"" + - set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn + - if DEFINED RAW_GOMA_AUTH (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"//electron/build/args/goma.gn\") %GN_EXTRA_ARGS% ") else (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\"") - gn check out/Default //electron:electron_lib - gn check out/Default //electron:electron_app - gn check out/Default //electron:manifests - gn check out/Default //electron/shell/common/api:mojo - - ninja -C out/Default electron:electron_app - - if "%GN_CONFIG%"=="testing" ( python C:\Users\electron\depot_tools\post_build_ninja_summary.py -C out\Default ) + - if DEFINED RAW_GOMA_AUTH (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app) + - if "%GN_CONFIG%"=="testing" ( python C:\Users\electron\depot_tools\post_build_ninja_summary.py -C out\Default ) - gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%" - ninja -C out/ffmpeg electron:electron_ffmpeg_zip - ninja -C out/Default electron:electron_dist_zip @@ -125,7 +131,7 @@ build_script: - ninja -C out/Default electron:hunspell_dictionaries_zip - ninja -C out/Default electron:electron_chromedriver_zip - ninja -C out/Default third_party/electron_node:headers - - cmd /C %SCCACHE_PATH% --show-stats + - if DEFINED RAW_GOMA_AUTH (python electron\external_binaries\goma\goma_ctl.py stat) - python electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json - appveyor PushArtifact out/Default/windows_toolchain_profile.json - appveyor PushArtifact out/Default/dist.zip diff --git a/script/start-goma.ps1 b/script/start-goma.ps1 new file mode 100755 index 000000000000..c8bf40c84b15 --- /dev/null +++ b/script/start-goma.ps1 @@ -0,0 +1,11 @@ +param([string]$gomaDir=$PWD) +$cmdPath = Join-Path -Path $gomaDir -ChildPath "goma_ctl.py" +Start-Process -FilePath cmd -ArgumentList "/C", "python", "$cmdPath", "ensure_start" +$env:GLOG_logtostderr = "true" +$timedOut = $false; $waitTime = 0; $waitIncrement = 5; $timeout=120; +Do { sleep $waitIncrement; $timedOut = (($waitTime+=$waitIncrement) -gt $timeout); iex "$gomaDir\gomacc.exe port 2" > $null; } Until(($LASTEXITCODE -eq 0) -or $timedOut) +if ($timedOut) { + write-error 'Timed out waiting for goma to start'; exit 1; +} else { + Write-Output "Successfully started goma!" +}