build: optimize the happy path when syncing on CI (#17827)
* build: optimize the happy path when syncing on CI This adds a new cache for the "src" directory that is only ever used if the cache key matches exactly. If there is no exact match we fall back to the old strategy of using the git cache. On the happy path this can make the checkout on linux/macOS take around 5-6 minutes which is **significantly** faster than the original 15-18 minutes. * build: sort readdir result to ensure stability * build: increment cache key * Update config.yml * build: ensure that the cleanly checked out Electron has had hooks run on it * build: do not remove deps/v8 * build: ensure clean git directory when generating deps hash * chore: add comments to caching logic * Update .circleci/config.yml Co-Authored-By: MarshallOfSound <samuel.r.attard@gmail.com>
This commit is contained in:
parent
df269ecb24
commit
e9114b3c00
4 changed files with 119 additions and 24 deletions
|
@ -143,13 +143,16 @@ step-gclient-sync: &step-gclient-sync
|
|||
run:
|
||||
name: Gclient sync
|
||||
command: |
|
||||
gclient config \
|
||||
--name "src/electron" \
|
||||
--unmanaged \
|
||||
$GCLIENT_EXTRA_ARGS \
|
||||
"$CIRCLE_REPOSITORY_URL"
|
||||
# If we did not restore a complete sync then we need to sync for realz
|
||||
if [ ! -s "src/electron/.circle-sync-done" ]; then
|
||||
gclient config \
|
||||
--name "src/electron" \
|
||||
--unmanaged \
|
||||
$GCLIENT_EXTRA_ARGS \
|
||||
"$CIRCLE_REPOSITORY_URL"
|
||||
|
||||
gclient sync --with_branch_heads --with_tags
|
||||
gclient sync --with_branch_heads --with_tags
|
||||
fi
|
||||
|
||||
step-setup-env-for-build: &step-setup-env-for-build
|
||||
run:
|
||||
|
@ -537,34 +540,82 @@ steps-checkout: &steps-checkout
|
|||
- *step-restore-brew-cache
|
||||
- *step-install-gnutar-on-mac
|
||||
|
||||
- run:
|
||||
name: Generate DEPS Hash
|
||||
command: node src/electron/script/generate-deps-hash.js
|
||||
- run:
|
||||
name: Touch Sync Done
|
||||
command: touch src/electron/.circle-sync-done
|
||||
# Restore exact src cache based on the hash of DEPS and patches/*
|
||||
# If no cache is matched EXACTLY then the .circle-sync-done file is empty
|
||||
# If a cache is matched EXACTLY then the .circle-sync-done file contains "done"
|
||||
- restore_cache:
|
||||
paths:
|
||||
- ./src
|
||||
keys:
|
||||
- v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
|
||||
name: Restoring src cache
|
||||
# Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done
|
||||
# If the src cache was restored above then this will match an empty cache
|
||||
# If the src cache was not restored above then this will match a close git cache
|
||||
- restore_cache:
|
||||
paths:
|
||||
- ~/.gclient-cache
|
||||
keys:
|
||||
- v1-gclient-cache-{{ arch }}-{{ checksum "src/electron/DEPS" }}
|
||||
- v1-gclient-cache-{{ arch }}-
|
||||
- v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
|
||||
- v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}
|
||||
name: Conditionally restoring git cache
|
||||
- run:
|
||||
name: Set GIT_CACHE_PATH to make gclient to use the cache
|
||||
command: |
|
||||
# CircleCI does not support interpolation when setting environment variables.
|
||||
# https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command
|
||||
echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV
|
||||
# This sync call only runs if .circle-sync-done is an EMPTY file
|
||||
- *step-gclient-sync
|
||||
# Persist the git cache based on the hash of DEPS and .circle-sync-done
|
||||
# If the src cache was restored above then this will persist an empty cache
|
||||
- save_cache:
|
||||
paths:
|
||||
- ~/.gclient-cache
|
||||
key: v1-gclient-cache-{{ arch }}-{{ checksum "src/electron/DEPS" }}
|
||||
key: v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }}
|
||||
name: Persisting git cache
|
||||
# These next few steps reset Electron to the correct commit regardless of which cache was restored
|
||||
- run:
|
||||
name: Wipe Electron
|
||||
command: rm -rf src/electron
|
||||
- *step-checkout-electron
|
||||
- run:
|
||||
name: Run Electron Only Hooks
|
||||
command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
|
||||
- run:
|
||||
name: Generate DEPS Hash
|
||||
command: (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js
|
||||
# Mark the sync as done for future cache saving
|
||||
- run:
|
||||
name: Mark Sync Done
|
||||
command: echo DONE > src/electron/.circle-sync-done
|
||||
# Minimize the size of the cache
|
||||
- run:
|
||||
name: Remove some unused data to avoid storing it in the workspace/cache
|
||||
command: |
|
||||
rm -rf src/android_webview
|
||||
rm -rf src/ios
|
||||
rm -rf src/third_party/blink/web_tests
|
||||
rm -rf src/third_party/blink/perf_tests
|
||||
rm -rf src/third_party/hunspell_dictionaries
|
||||
rm -rf src/third_party/WebKit/LayoutTests
|
||||
# Save the src cache based on the deps hash
|
||||
- save_cache:
|
||||
paths:
|
||||
- ./src
|
||||
key: v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }}
|
||||
name: Persisting src cache
|
||||
- save_cache:
|
||||
paths:
|
||||
- /usr/local/Homebrew
|
||||
key: v1-brew-cache-{{ arch }}
|
||||
|
||||
- run:
|
||||
name: Remove some unused data to avoid storing it in the workspace
|
||||
command: |
|
||||
rm -rf src/android_webview
|
||||
rm -rf src/ios
|
||||
rm -rf src/third_party/WebKit/LayoutTests
|
||||
name: Persisting brew cache
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
|
|
3
.gitignore
vendored
3
.gitignore
vendored
|
@ -62,3 +62,6 @@ spec/.hash
|
|||
|
||||
# If someone runs tsc this is where stuff will end up
|
||||
ts-gen
|
||||
|
||||
# Used to accelerate CI builds
|
||||
.depshash
|
||||
|
|
19
DEPS
19
DEPS
|
@ -40,6 +40,9 @@ vars = {
|
|||
# Python "requests" module is used for releases only.
|
||||
'checkout_requests': False,
|
||||
|
||||
# To allow running hooks without parsing the DEPS tree
|
||||
'process_deps': True,
|
||||
|
||||
# It is always needed for normal Electron builds,
|
||||
# but might be impossible for custom in-house builds.
|
||||
'download_external_binaries': True,
|
||||
|
@ -61,30 +64,30 @@ vars = {
|
|||
deps = {
|
||||
'src': {
|
||||
'url': (Var("chromium_git")) + '/chromium/src.git@' + (Var("chromium_version")),
|
||||
'condition': 'checkout_chromium',
|
||||
'condition': 'checkout_chromium and process_deps',
|
||||
},
|
||||
'src/third_party/electron_node': {
|
||||
'url': (Var("electron_git")) + '/node.git@' + (Var("node_version")),
|
||||
'condition': 'checkout_node',
|
||||
'condition': 'checkout_node and process_deps',
|
||||
},
|
||||
'src/electron/vendor/pyyaml': {
|
||||
'url': (Var("yaml_git")) + '/pyyaml.git@' + (Var("pyyaml_version")),
|
||||
'condition': 'checkout_pyyaml',
|
||||
'condition': 'checkout_pyyaml and process_deps',
|
||||
},
|
||||
'src/electron/vendor/boto': {
|
||||
'url': Var('boto_git') + '/boto.git' + '@' + Var('boto_version'),
|
||||
'condition': 'checkout_boto',
|
||||
'condition': 'checkout_boto and process_deps',
|
||||
},
|
||||
'src/electron/vendor/requests': {
|
||||
'url': Var('requests_git') + '/requests.git' + '@' + Var('requests_version'),
|
||||
'condition': 'checkout_requests',
|
||||
'condition': 'checkout_requests and process_deps',
|
||||
},
|
||||
}
|
||||
|
||||
hooks = [
|
||||
{
|
||||
'name': 'patch_chromium',
|
||||
'condition': 'checkout_chromium and apply_patches',
|
||||
'condition': '(checkout_chromium and apply_patches) and process_deps',
|
||||
'pattern': 'src/electron',
|
||||
'action': [
|
||||
'python',
|
||||
|
@ -113,7 +116,7 @@ hooks = [
|
|||
{
|
||||
'name': 'setup_boto',
|
||||
'pattern': 'src/electron',
|
||||
'condition': 'checkout_boto',
|
||||
'condition': 'checkout_boto and process_deps',
|
||||
'action': [
|
||||
'python',
|
||||
'-c',
|
||||
|
@ -123,7 +126,7 @@ hooks = [
|
|||
{
|
||||
'name': 'setup_requests',
|
||||
'pattern': 'src/electron',
|
||||
'condition': 'checkout_requests',
|
||||
'condition': 'checkout_requests and process_deps',
|
||||
'action': [
|
||||
'python',
|
||||
'-c',
|
||||
|
|
38
script/generate-deps-hash.js
Normal file
38
script/generate-deps-hash.js
Normal file
|
@ -0,0 +1,38 @@
|
|||
const crypto = require('crypto')
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
|
||||
// Fallback to blow away old cache keys
|
||||
const HASH_VERSION = 1
|
||||
|
||||
// Base files to hash
|
||||
const filesToHash = [
|
||||
path.resolve(__dirname, '../DEPS'),
|
||||
path.resolve(__dirname, '../package-lock.json')
|
||||
]
|
||||
|
||||
const addAllFiles = (dir) => {
|
||||
for (const child of fs.readdirSync(dir).sort()) {
|
||||
const childPath = path.resolve(dir, child)
|
||||
if (fs.statSync(childPath).isDirectory()) {
|
||||
addAllFiles(childPath)
|
||||
} else {
|
||||
filesToHash.push(childPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add all patch files to the hash
|
||||
addAllFiles(path.resolve(__dirname, '../patches'))
|
||||
|
||||
// Create Hash
|
||||
const hasher = crypto.createHash('SHA256')
|
||||
for (const file of filesToHash) {
|
||||
hasher.update(fs.readFileSync(file))
|
||||
}
|
||||
|
||||
// Add the GCLIENT_EXTRA_ARGS variable to the hash
|
||||
hasher.update(process.env.GCLIENT_EXTRA_ARGS || 'no_extra_args')
|
||||
|
||||
// Write the hash to disk
|
||||
fs.writeFileSync(path.resolve(__dirname, '../.depshash'), hasher.digest('hex'))
|
Loading…
Reference in a new issue