diff --git a/.circleci/config.yml b/.circleci/config.yml index d210b7cc7d32..7d963ba47581 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -143,13 +143,16 @@ step-gclient-sync: &step-gclient-sync run: name: Gclient sync command: | - gclient config \ - --name "src/electron" \ - --unmanaged \ - $GCLIENT_EXTRA_ARGS \ - "$CIRCLE_REPOSITORY_URL" + # If we did not restore a complete sync then we need to sync for realz + if [ ! -s "src/electron/.circle-sync-done" ]; then + gclient config \ + --name "src/electron" \ + --unmanaged \ + $GCLIENT_EXTRA_ARGS \ + "$CIRCLE_REPOSITORY_URL" - gclient sync --with_branch_heads --with_tags + gclient sync --with_branch_heads --with_tags + fi step-setup-env-for-build: &step-setup-env-for-build run: @@ -537,34 +540,82 @@ steps-checkout: &steps-checkout - *step-restore-brew-cache - *step-install-gnutar-on-mac + - run: + name: Generate DEPS Hash + command: node src/electron/script/generate-deps-hash.js + - run: + name: Touch Sync Done + command: touch src/electron/.circle-sync-done + # Restore exact src cache based on the hash of DEPS and patches/* + # If no cache is matched EXACTLY then the .circle-sync-done file is empty + # If a cache is matched EXACTLY then the .circle-sync-done file contains "done" + - restore_cache: + paths: + - ./src + keys: + - v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }} + name: Restoring src cache + # Restore exact or closest git cache based on the hash of DEPS and .circle-sync-done + # If the src cache was restored above then this will match an empty cache + # If the src cache was not restored above then this will match a close git cache - restore_cache: paths: - ~/.gclient-cache keys: - - v1-gclient-cache-{{ arch }}-{{ checksum "src/electron/DEPS" }} - - v1-gclient-cache-{{ arch }}- + - v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }} + - v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }} + name: Conditionally restoring git cache - run: name: Set GIT_CACHE_PATH to make gclient to use the cache command: | # CircleCI does not support interpolation when setting environment variables. # https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command echo 'export GIT_CACHE_PATH="$HOME/.gclient-cache"' >> $BASH_ENV + # This sync call only runs if .circle-sync-done is an EMPTY file - *step-gclient-sync + # Persist the git cache based on the hash of DEPS and .circle-sync-done + # If the src cache was restored above then this will persist an empty cache - save_cache: paths: - ~/.gclient-cache - key: v1-gclient-cache-{{ arch }}-{{ checksum "src/electron/DEPS" }} + key: v2-gclient-cache-{{ arch }}-{{ checksum "src/electron/.circle-sync-done" }}-{{ checksum "src/electron/DEPS" }} + name: Persisting git cache + # These next few steps reset Electron to the correct commit regardless of which cache was restored + - run: + name: Wipe Electron + command: rm -rf src/electron + - *step-checkout-electron + - run: + name: Run Electron Only Hooks + command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]" + - run: + name: Generate DEPS Hash + command: (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js + # Mark the sync as done for future cache saving + - run: + name: Mark Sync Done + command: echo DONE > src/electron/.circle-sync-done + # Minimize the size of the cache + - run: + name: Remove some unused data to avoid storing it in the workspace/cache + command: | + rm -rf src/android_webview + rm -rf src/ios + rm -rf src/third_party/blink/web_tests + rm -rf src/third_party/blink/perf_tests + rm -rf src/third_party/hunspell_dictionaries + rm -rf src/third_party/WebKit/LayoutTests + # Save the src cache based on the deps hash + - save_cache: + paths: + - ./src + key: v5-src-cache-{{ arch }}-{{ checksum "src/electron/.depshash" }} + name: Persisting src cache - save_cache: paths: - /usr/local/Homebrew key: v1-brew-cache-{{ arch }} - - - run: - name: Remove some unused data to avoid storing it in the workspace - command: | - rm -rf src/android_webview - rm -rf src/ios - rm -rf src/third_party/WebKit/LayoutTests + name: Persisting brew cache - persist_to_workspace: root: . paths: diff --git a/.gitignore b/.gitignore index c5c143a12449..424394c64f02 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,6 @@ spec/.hash # If someone runs tsc this is where stuff will end up ts-gen + +# Used to accelerate CI builds +.depshash diff --git a/DEPS b/DEPS index 8f9180707be0..4816feed99b4 100644 --- a/DEPS +++ b/DEPS @@ -40,6 +40,9 @@ vars = { # Python "requests" module is used for releases only. 'checkout_requests': False, + # To allow running hooks without parsing the DEPS tree + 'process_deps': True, + # It is always needed for normal Electron builds, # but might be impossible for custom in-house builds. 'download_external_binaries': True, @@ -61,30 +64,30 @@ vars = { deps = { 'src': { 'url': (Var("chromium_git")) + '/chromium/src.git@' + (Var("chromium_version")), - 'condition': 'checkout_chromium', + 'condition': 'checkout_chromium and process_deps', }, 'src/third_party/electron_node': { 'url': (Var("electron_git")) + '/node.git@' + (Var("node_version")), - 'condition': 'checkout_node', + 'condition': 'checkout_node and process_deps', }, 'src/electron/vendor/pyyaml': { 'url': (Var("yaml_git")) + '/pyyaml.git@' + (Var("pyyaml_version")), - 'condition': 'checkout_pyyaml', + 'condition': 'checkout_pyyaml and process_deps', }, 'src/electron/vendor/boto': { 'url': Var('boto_git') + '/boto.git' + '@' + Var('boto_version'), - 'condition': 'checkout_boto', + 'condition': 'checkout_boto and process_deps', }, 'src/electron/vendor/requests': { 'url': Var('requests_git') + '/requests.git' + '@' + Var('requests_version'), - 'condition': 'checkout_requests', + 'condition': 'checkout_requests and process_deps', }, } hooks = [ { 'name': 'patch_chromium', - 'condition': 'checkout_chromium and apply_patches', + 'condition': '(checkout_chromium and apply_patches) and process_deps', 'pattern': 'src/electron', 'action': [ 'python', @@ -113,7 +116,7 @@ hooks = [ { 'name': 'setup_boto', 'pattern': 'src/electron', - 'condition': 'checkout_boto', + 'condition': 'checkout_boto and process_deps', 'action': [ 'python', '-c', @@ -123,7 +126,7 @@ hooks = [ { 'name': 'setup_requests', 'pattern': 'src/electron', - 'condition': 'checkout_requests', + 'condition': 'checkout_requests and process_deps', 'action': [ 'python', '-c', diff --git a/script/generate-deps-hash.js b/script/generate-deps-hash.js new file mode 100644 index 000000000000..5cb3f21d27c0 --- /dev/null +++ b/script/generate-deps-hash.js @@ -0,0 +1,38 @@ +const crypto = require('crypto') +const fs = require('fs') +const path = require('path') + +// Fallback to blow away old cache keys +const HASH_VERSION = 1 + +// Base files to hash +const filesToHash = [ + path.resolve(__dirname, '../DEPS'), + path.resolve(__dirname, '../package-lock.json') +] + +const addAllFiles = (dir) => { + for (const child of fs.readdirSync(dir).sort()) { + const childPath = path.resolve(dir, child) + if (fs.statSync(childPath).isDirectory()) { + addAllFiles(childPath) + } else { + filesToHash.push(childPath) + } + } +} + +// Add all patch files to the hash +addAllFiles(path.resolve(__dirname, '../patches')) + +// Create Hash +const hasher = crypto.createHash('SHA256') +for (const file of filesToHash) { + hasher.update(fs.readFileSync(file)) +} + +// Add the GCLIENT_EXTRA_ARGS variable to the hash +hasher.update(process.env.GCLIENT_EXTRA_ARGS || 'no_extra_args') + +// Write the hash to disk +fs.writeFileSync(path.resolve(__dirname, '../.depshash'), hasher.digest('hex'))