build: optimize the CI path where we update the patch files (#19851)
Currently the happy checkout takes 7 minutes and the sad checkout takes 30 minutes. This updates our CI to run checkout twice for every job to make the sad checkout take nearer 10 minutes instead.
This commit is contained in:
Samuel Attard
John Kleinschmidt
parent
991a56135d
commit
612e3f7ff7
1 changed files with 179 additions and 99 deletions
|
|
@ -186,6 +186,13 @@ step-restore-brew-cache: &step-restore-brew-cache
|
||||||
keys:
|
keys:
|
||||||
- v1-brew-cache-{{ arch }}
|
- v1-brew-cache-{{ arch }}
|
||||||
|
|
||||||
|
step-save-brew-cache: &step-save-brew-cache
|
||||||
|
save_cache:
|
||||||
|
paths:
|
||||||
|
- /usr/local/Homebrew
|
||||||
|
key: v1-brew-cache-{{ arch }}
|
||||||
|
name: Persisting brew cache
|
||||||
|
|
||||||
step-get-more-space-on-mac: &step-get-more-space-on-mac
|
step-get-more-space-on-mac: &step-get-more-space-on-mac
|
||||||
run:
|
run:
|
||||||
name: Free up space on MacOS
|
name: Free up space on MacOS
|
||||||
|
|
@ -535,6 +542,93 @@ step-ninja-summary: &step-ninja-summary
|
||||||
command: |
|
command: |
|
||||||
python depot_tools/post_build_ninja_summary.py -C src/out/Default
|
python depot_tools/post_build_ninja_summary.py -C src/out/Default
|
||||||
|
|
||||||
|
# Checkout Steps
|
||||||
|
step-generate-deps-hash: &step-generate-deps-hash
|
||||||
|
run:
|
||||||
|
name: Generate DEPS Hash
|
||||||
|
command: node src/electron/script/generate-deps-hash.js
|
||||||
|
|
||||||
|
step-touch-sync-done: &step-touch-sync-done
|
||||||
|
run:
|
||||||
|
name: Touch Sync Done
|
||||||
|
command: touch src/electron/.circle-sync-done
|
||||||
|
|
||||||
|
# Restore exact src cache based on the hash of DEPS and patches/*
|
||||||
|
# If no cache is matched EXACTLY then the .circle-sync-done file is empty
|
||||||
|
# If a cache is matched EXACTLY then the .circle-sync-done file contains "done"
|
||||||
|
step-maybe-restore-src-cache: &step-maybe-restore-src-cache
|
||||||
|
restore_cache:
|
||||||
|
paths:
|
||||||
|
- ./src
|
||||||
|
keys:
|
||||||
|
- v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
|
||||||
|
name: Restoring src cache
|
||||||
|
|
||||||
|
# Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done
|
||||||
|
# If the src cache was restored above then this will match an empty cache
|
||||||
|
# If the src cache was not restored above then this will match a close git cache
|
||||||
|
step-maybe-restore-git-cache: &step-maybe-restore-git-cache
|
||||||
|
restore_cache:
|
||||||
|
paths:
|
||||||
|
- ~/.gclient-cache
|
||||||
|
keys:
|
||||||
|
- v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
|
||||||
|
- v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}
|
||||||
|
name: Conditionally restoring git cache
|
||||||
|
|
||||||
|
step-set-git-cache-path: &step-set-git-cache-path
|
||||||
|
run:
|
||||||
|
name: Set GIT_CACHE_PATH to make gclient to use the cache
|
||||||
|
command: |
|
||||||
|
# CircleCI does not support interpolation when setting environment variables.
|
||||||
|
# https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command
|
||||||
|
echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV
|
||||||
|
|
||||||
|
# Persist the git cache based on the hash of DEPS and .circle-sync-done
|
||||||
|
# If the src cache was restored above then this will persist an empty cache
|
||||||
|
step-save-git-cache: &step-save-git-cache
|
||||||
|
save_cache:
|
||||||
|
paths:
|
||||||
|
- ~/.gclient-cache
|
||||||
|
key: v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
|
||||||
|
name: Persisting git cache
|
||||||
|
|
||||||
|
step-run-electron-only-hooks: &step-run-electron-only-hooks
|
||||||
|
run:
|
||||||
|
name: Run Electron Only Hooks
|
||||||
|
command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
|
||||||
|
|
||||||
|
step-generate-deps-hash-cleanly: &step-generate-deps-hash-cleanly
|
||||||
|
run:
|
||||||
|
name: Generate DEPS Hash
|
||||||
|
command: (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js
|
||||||
|
|
||||||
|
# Mark the sync as done for future cache saving
|
||||||
|
step-mark-sync-done: &step-mark-sync-done
|
||||||
|
run:
|
||||||
|
name: Mark Sync Done
|
||||||
|
command: echo DONE > src/electron/.circle-sync-done
|
||||||
|
|
||||||
|
# Minimize the size of the cache
|
||||||
|
step-minimize-workspace-size-from-checkout: &step-minimize-workspace-size-from-checkout
|
||||||
|
run:
|
||||||
|
name: Remove some unused data to avoid storing it in the workspace/cache
|
||||||
|
command: |
|
||||||
|
rm -rf src/android_webview
|
||||||
|
rm -rf src/ios
|
||||||
|
rm -rf src/third_party/blink/web_tests
|
||||||
|
rm -rf src/third_party/blink/perf_tests
|
||||||
|
rm -rf src/third_party/hunspell_dictionaries
|
||||||
|
rm -rf src/third_party/WebKit/LayoutTests
|
||||||
|
|
||||||
|
# Save the src cache based on the deps hash
|
||||||
|
step-save-src-cache: &step-save-src-cache
|
||||||
|
save_cache:
|
||||||
|
paths:
|
||||||
|
- ./src
|
||||||
|
key: v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
|
||||||
|
name: Persisting src cache
|
||||||
|
|
||||||
# Lists of steps.
|
# Lists of steps.
|
||||||
steps-lint: &steps-lint
|
steps-lint: &steps-lint
|
||||||
steps:
|
steps:
|
||||||
|
|
@ -580,7 +674,7 @@ steps-lint: &steps-lint
|
||||||
node script/yarn install --frozen-lockfile
|
node script/yarn install --frozen-lockfile
|
||||||
node script/yarn lint
|
node script/yarn lint
|
||||||
|
|
||||||
steps-checkout: &steps-checkout
|
steps-checkout-fast: &steps-checkout-fast
|
||||||
steps:
|
steps:
|
||||||
- *step-checkout-electron
|
- *step-checkout-electron
|
||||||
- *step-depot-tools-get
|
- *step-depot-tools-get
|
||||||
|
|
@ -589,88 +683,57 @@ steps-checkout: &steps-checkout
|
||||||
- *step-get-more-space-on-mac
|
- *step-get-more-space-on-mac
|
||||||
- *step-install-gnutar-on-mac
|
- *step-install-gnutar-on-mac
|
||||||
|
|
||||||
- run:
|
- *step-generate-deps-hash
|
||||||
name: Generate DEPS Hash
|
- *step-touch-sync-done
|
||||||
command: node src/electron/script/generate-deps-hash.js
|
- *step-maybe-restore-src-cache
|
||||||
- run:
|
- *step-maybe-restore-git-cache
|
||||||
name: Touch Sync Done
|
- *step-set-git-cache-path
|
||||||
command: touch src/electron/.circle-sync-done
|
|
||||||
# Restore exact src cache based on the hash of DEPS and patches/*
|
|
||||||
# If no cache is matched EXACTLY then the .circle-sync-done file is empty
|
|
||||||
# If a cache is matched EXACTLY then the .circle-sync-done file contains "done"
|
|
||||||
- restore_cache:
|
|
||||||
paths:
|
|
||||||
- ./src
|
|
||||||
keys:
|
|
||||||
- v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
|
|
||||||
name: Restoring src cache
|
|
||||||
# Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done
|
|
||||||
# If the src cache was restored above then this will match an empty cache
|
|
||||||
# If the src cache was not restored above then this will match a close git cache
|
|
||||||
- restore_cache:
|
|
||||||
paths:
|
|
||||||
- ~/.gclient-cache
|
|
||||||
keys:
|
|
||||||
- v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
|
|
||||||
- v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}
|
|
||||||
name: Conditionally restoring git cache
|
|
||||||
- run:
|
|
||||||
name: Set GIT_CACHE_PATH to make gclient to use the cache
|
|
||||||
command: |
|
|
||||||
# CircleCI does not support interpolation when setting environment variables.
|
|
||||||
# https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command
|
|
||||||
echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV
|
|
||||||
# This sync call only runs if .circle-sync-done is an EMPTY file
|
# This sync call only runs if .circle-sync-done is an EMPTY file
|
||||||
- *step-gclient-sync
|
- *step-gclient-sync
|
||||||
# Persist the git cache based on the hash of DEPS and .circle-sync-done
|
|
||||||
# If the src cache was restored above then this will persist an empty cache
|
|
||||||
- save_cache:
|
|
||||||
paths:
|
|
||||||
- ~/.gclient-cache
|
|
||||||
key: v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
|
|
||||||
name: Persisting git cache
|
|
||||||
# These next few steps reset Electron to the correct commit regardless of which cache was restored
|
# These next few steps reset Electron to the correct commit regardless of which cache was restored
|
||||||
- run:
|
- run:
|
||||||
name: Wipe Electron
|
name: Wipe Electron
|
||||||
command: rm -rf src/electron
|
command: rm -rf src/electron
|
||||||
- *step-checkout-electron
|
- *step-checkout-electron
|
||||||
- run:
|
- *step-run-electron-only-hooks
|
||||||
name: Run Electron Only Hooks
|
- *step-generate-deps-hash-cleanly
|
||||||
command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
|
- *step-mark-sync-done
|
||||||
- run:
|
- *step-minimize-workspace-size-from-checkout
|
||||||
name: Generate DEPS Hash
|
|
||||||
command: (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js
|
|
||||||
# Mark the sync as done for future cache saving
|
|
||||||
- run:
|
|
||||||
name: Mark Sync Done
|
|
||||||
command: echo DONE > src/electron/.circle-sync-done
|
|
||||||
# Minimize the size of the cache
|
|
||||||
- run:
|
|
||||||
name: Remove some unused data to avoid storing it in the workspace/cache
|
|
||||||
command: |
|
|
||||||
rm -rf src/android_webview
|
|
||||||
rm -rf src/ios
|
|
||||||
rm -rf src/third_party/blink/web_tests
|
|
||||||
rm -rf src/third_party/blink/perf_tests
|
|
||||||
rm -rf src/third_party/hunspell_dictionaries
|
|
||||||
rm -rf src/third_party/WebKit/LayoutTests
|
|
||||||
# Save the src cache based on the deps hash
|
|
||||||
- save_cache:
|
|
||||||
paths:
|
|
||||||
- ./src
|
|
||||||
key: v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
|
|
||||||
name: Persisting src cache
|
|
||||||
- save_cache:
|
|
||||||
paths:
|
|
||||||
- /usr/local/Homebrew
|
|
||||||
key: v1-brew-cache-{{ arch }}
|
|
||||||
name: Persisting brew cache
|
|
||||||
- persist_to_workspace:
|
- persist_to_workspace:
|
||||||
root: .
|
root: .
|
||||||
paths:
|
paths:
|
||||||
- depot_tools
|
- depot_tools
|
||||||
- src
|
- src
|
||||||
|
|
||||||
|
steps-checkout-and-save-cache: &steps-checkout-and-save-cache
|
||||||
|
steps:
|
||||||
|
- *step-checkout-electron
|
||||||
|
- *step-depot-tools-get
|
||||||
|
- *step-depot-tools-add-to-path
|
||||||
|
- *step-restore-brew-cache
|
||||||
|
- *step-get-more-space-on-mac
|
||||||
|
- *step-install-gnutar-on-mac
|
||||||
|
|
||||||
|
- *step-generate-deps-hash
|
||||||
|
- *step-touch-sync-done
|
||||||
|
- *step-maybe-restore-src-cache
|
||||||
|
- *step-maybe-restore-git-cache
|
||||||
|
- *step-set-git-cache-path
|
||||||
|
# This sync call only runs if .circle-sync-done is an EMPTY file
|
||||||
|
- *step-gclient-sync
|
||||||
|
- *step-save-git-cache
|
||||||
|
# These next few steps reset Electron to the correct commit regardless of which cache was restored
|
||||||
|
- run:
|
||||||
|
name: Wipe Electron
|
||||||
|
command: rm -rf src/electron
|
||||||
|
- *step-checkout-electron
|
||||||
|
- *step-run-electron-only-hooks
|
||||||
|
- *step-generate-deps-hash-cleanly
|
||||||
|
- *step-mark-sync-done
|
||||||
|
- *step-minimize-workspace-size-from-checkout
|
||||||
|
- *step-save-src-cache
|
||||||
|
- *step-save-brew-cache
|
||||||
|
|
||||||
steps-electron-gn-check: &steps-electron-gn-check
|
steps-electron-gn-check: &steps-electron-gn-check
|
||||||
steps:
|
steps:
|
||||||
- attach_workspace:
|
- attach_workspace:
|
||||||
|
|
@ -949,33 +1012,47 @@ jobs:
|
||||||
<<: *steps-lint
|
<<: *steps-lint
|
||||||
|
|
||||||
# Layer 1: Checkout.
|
# Layer 1: Checkout.
|
||||||
linux-checkout:
|
linux-checkout-fast:
|
||||||
<<: *machine-linux-2xlarge
|
<<: *machine-linux-2xlarge
|
||||||
environment:
|
environment:
|
||||||
<<: *env-linux-2xlarge
|
<<: *env-linux-2xlarge
|
||||||
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
|
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
|
||||||
<<: *steps-checkout
|
<<: *steps-checkout-fast
|
||||||
|
|
||||||
|
linux-checkout-and-save-cache:
|
||||||
|
<<: *machine-linux-2xlarge
|
||||||
|
environment:
|
||||||
|
<<: *env-linux-2xlarge
|
||||||
|
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
|
||||||
|
<<: *steps-checkout-and-save-cache
|
||||||
|
|
||||||
linux-checkout-for-native-tests:
|
linux-checkout-for-native-tests:
|
||||||
<<: *machine-linux-2xlarge
|
<<: *machine-linux-2xlarge
|
||||||
environment:
|
environment:
|
||||||
<<: *env-linux-2xlarge
|
<<: *env-linux-2xlarge
|
||||||
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_pyyaml=True'
|
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_pyyaml=True'
|
||||||
<<: *steps-checkout
|
<<: *steps-checkout-fast
|
||||||
|
|
||||||
linux-checkout-for-native-tests-with-no-patches:
|
linux-checkout-for-native-tests-with-no-patches:
|
||||||
<<: *machine-linux-2xlarge
|
<<: *machine-linux-2xlarge
|
||||||
environment:
|
environment:
|
||||||
<<: *env-linux-2xlarge
|
<<: *env-linux-2xlarge
|
||||||
GCLIENT_EXTRA_ARGS: '--custom-var=apply_patches=False --custom-var=checkout_pyyaml=True'
|
GCLIENT_EXTRA_ARGS: '--custom-var=apply_patches=False --custom-var=checkout_pyyaml=True'
|
||||||
<<: *steps-checkout
|
<<: *steps-checkout-fast
|
||||||
|
|
||||||
mac-checkout:
|
mac-checkout-fast:
|
||||||
<<: *machine-linux-2xlarge
|
<<: *machine-linux-2xlarge
|
||||||
environment:
|
environment:
|
||||||
<<: *env-linux-2xlarge
|
<<: *env-linux-2xlarge
|
||||||
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
|
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
|
||||||
<<: *steps-checkout
|
<<: *steps-checkout-fast
|
||||||
|
|
||||||
|
mac-checkout-and-save-cache:
|
||||||
|
<<: *machine-linux-2xlarge
|
||||||
|
environment:
|
||||||
|
<<: *env-linux-2xlarge
|
||||||
|
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
|
||||||
|
<<: *steps-checkout-and-save-cache
|
||||||
|
|
||||||
# Layer 2: Builds.
|
# Layer 2: Builds.
|
||||||
linux-x64-debug:
|
linux-x64-debug:
|
||||||
|
|
@ -1596,23 +1673,24 @@ workflows:
|
||||||
|
|
||||||
build-linux:
|
build-linux:
|
||||||
jobs:
|
jobs:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
|
- linux-checkout-and-save-cache
|
||||||
|
|
||||||
- linux-x64-debug:
|
- linux-x64-debug:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
- linux-x64-debug-gn-check:
|
- linux-x64-debug-gn-check:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
- linux-x64-testing:
|
- linux-x64-testing:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
- linux-x64-testing-no-run-as-node:
|
- linux-x64-testing-no-run-as-node:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
- linux-x64-testing-gn-check:
|
- linux-x64-testing-gn-check:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
- linux-x64-testing-tests:
|
- linux-x64-testing-tests:
|
||||||
requires:
|
requires:
|
||||||
- linux-x64-testing
|
- linux-x64-testing
|
||||||
|
|
@ -1625,10 +1703,10 @@ workflows:
|
||||||
|
|
||||||
- linux-ia32-debug:
|
- linux-ia32-debug:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
- linux-ia32-testing:
|
- linux-ia32-testing:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
- linux-ia32-testing-tests:
|
- linux-ia32-testing-tests:
|
||||||
requires:
|
requires:
|
||||||
- linux-ia32-testing
|
- linux-ia32-testing
|
||||||
|
|
@ -1641,42 +1719,44 @@ workflows:
|
||||||
|
|
||||||
- linux-arm-debug:
|
- linux-arm-debug:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
- linux-arm-testing:
|
- linux-arm-testing:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
|
|
||||||
- linux-arm64-debug:
|
- linux-arm64-debug:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
- linux-arm64-debug-gn-check:
|
- linux-arm64-debug-gn-check:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
- linux-arm64-testing:
|
- linux-arm64-testing:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
- linux-arm64-testing-gn-check:
|
- linux-arm64-testing-gn-check:
|
||||||
requires:
|
requires:
|
||||||
- linux-checkout
|
- linux-checkout-fast
|
||||||
|
|
||||||
build-mac:
|
build-mac:
|
||||||
jobs:
|
jobs:
|
||||||
- mac-checkout
|
- mac-checkout-fast
|
||||||
|
- mac-checkout-and-save-cache
|
||||||
|
|
||||||
- osx-testing:
|
- osx-testing:
|
||||||
requires:
|
requires:
|
||||||
- mac-checkout
|
- mac-checkout-fast
|
||||||
|
|
||||||
- osx-debug:
|
- osx-debug:
|
||||||
requires:
|
requires:
|
||||||
- mac-checkout
|
- mac-checkout-fast
|
||||||
|
|
||||||
- osx-debug-gn-check:
|
- osx-debug-gn-check:
|
||||||
requires:
|
requires:
|
||||||
- mac-checkout
|
- mac-checkout-fast
|
||||||
|
|
||||||
- osx-testing-gn-check:
|
- osx-testing-gn-check:
|
||||||
requires:
|
requires:
|
||||||
- mac-checkout
|
- mac-checkout-fast
|
||||||
|
|
||||||
- osx-testing-tests:
|
- osx-testing-tests:
|
||||||
requires:
|
requires:
|
||||||
|
|
@ -1684,19 +1764,19 @@ workflows:
|
||||||
|
|
||||||
- mas-testing:
|
- mas-testing:
|
||||||
requires:
|
requires:
|
||||||
- mac-checkout
|
- mac-checkout-fast
|
||||||
|
|
||||||
- mas-debug:
|
- mas-debug:
|
||||||
requires:
|
requires:
|
||||||
- mac-checkout
|
- mac-checkout-fast
|
||||||
|
|
||||||
- mas-debug-gn-check:
|
- mas-debug-gn-check:
|
||||||
requires:
|
requires:
|
||||||
- mac-checkout
|
- mac-checkout-fast
|
||||||
|
|
||||||
- mas-testing-gn-check:
|
- mas-testing-gn-check:
|
||||||
requires:
|
requires:
|
||||||
- mac-checkout
|
- mac-checkout-fast
|
||||||
|
|
||||||
- mas-testing-tests:
|
- mas-testing-tests:
|
||||||
requires:
|
requires:
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue