build: use electron-frameworks sccache (#14171)

* build: update-external-binaries fetches sccache

* build: add util.add_exec_bit in scripts/

* build: use util.add_exec_bit in create-dist

* build: use util.add_exec_bit in update-external-binaries

this is needed to work around a bug in python's zipfile module that doesn't preserve the exec bit

https://bugs.python.org/issue18262

* fix: linting errors

* build: vsts, circleci use patched sccache

* build: always look for the x64 sccache

as it's the only arch we have it on

* fix: windows-specific errors in updaste-external-binaries

* fix: tyop

* fix: set SCCACHE_BUCKET, SCCACHE_TWO_TIER on circleci

* fix: syntax error in circleci yaml

* fix: keep churning

* chore: add tracer to file downloader

* docs: add sccache instructions for GN builds

* build: pull down the darwin sccache on mas builds

* build: use gn sync verbosely on circleci and vsts

* docs: copyediting

* build: remove unnecessary cache-dir arg

* docs: fix shell quoting in gn build instructions

* fix: invoke gclient without -verbose in circleci

* refactor: remove debug tracer

* fix: invoke gclient without -verbose in appveyor

* fix: invoke gclient without -verbose in vsts

* fix: pull add_exec_bit from correct source

* fix: remove 'SCCACHE_TWO_TIER' from CI scripts

* refactor: remove SCCACHE_BUCKET from ci scripts

this environment variable will be set via the CI UI instead

* refactor: clarify log message

* fix: set SCCACHE_PATH correctly for Windows CI

script/create-dist.py

@@ -11,10 +11,13 @@ import stat
 if sys.platform == "win32":
   import _winreg
 
-from lib.config import BASE_URL, PLATFORM, enable_verbose_mode, \
-                       get_target_arch, get_zip_name, build_env
-from lib.util import scoped_cwd, rm_rf, get_electron_version, make_zip, \
-                     execute, electron_gyp, electron_features, parse_version
+from lib.config import BASE_URL, PLATFORM, build_env, \
+                       enable_verbose_mode, get_target_arch, get_zip_name
+
+from lib.util import add_exec_bit, electron_features, electron_gyp, \
+                     execute, get_electron_version, make_zip, \
+                     parse_version, rm_rf, scoped_cwd
+
 from lib.env_util import get_vs_location
 
 
@@ -144,7 +147,7 @@ def copy_chrome_binary(binary):
 
   # Copy file and keep the executable bit.
   shutil.copyfile(src, dest)
-  os.chmod(dest, os.stat(dest).st_mode | stat.S_IEXEC)
+  add_exec_bit(dest)
 
 def copy_vcruntime_binaries():
   arch = get_target_arch()
@@ -299,7 +302,7 @@ def create_chrome_binary_zip(binary, version):
     dest = os.path.join(DIST_DIR, binary)
     # Copy file and keep the executable bit.
     shutil.copyfile(src, dest)
-    os.chmod(dest, os.stat(dest).st_mode | stat.S_IEXEC)
+    add_exec_bit(dest)
 
     dist_name = get_zip_name(binary, version)
     zip_file = os.path.join(SOURCE_ROOT, 'dist', dist_name)

script/lib/util.py

@@ -8,6 +8,7 @@ import platform
 import re
 import shutil
 import ssl
+import stat
 import subprocess
 import sys
 import tarfile
@@ -83,6 +84,7 @@ def download(text, url, path):
     if hasattr(ssl, '_create_unverified_context'):
       ssl._create_default_https_context = ssl._create_unverified_context
 
+    print "Downloading %s to %s" % (url, path)
     web_file = urllib2.urlopen(url)
     file_size = int(web_file.info().getheaders("Content-Length")[0])
     downloaded_size = 0
@@ -295,6 +297,9 @@ def update_node_modules(dirname, env=None):
     else:
       execute_stdout(args, env)
 
+def add_exec_bit(filename):
+  os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC)
+
 def clean_parse_version(v):
   return parse_version(v.split("-")[0])        
 

script/update-external-binaries.py

@@ -4,11 +4,11 @@ import errno
 import sys
 import os
 
-from lib.config import get_target_arch
-from lib.util import safe_mkdir, rm_rf, extract_zip, tempdir, download
+from lib.config import PLATFORM, get_target_arch
+from lib.util import add_exec_bit, download, extract_zip, rm_rf, \
+                     safe_mkdir, tempdir
 
-
-VERSION = 'v1.3.0'
+VERSION = 'v1.4.0'
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 FRAMEWORKS_URL = 'http://github.com/electron/electron-frameworks/releases' \
                  '/download/' + VERSION
@@ -31,6 +31,13 @@ def main():
   elif sys.platform in ['cygwin', 'win32']:
     download_and_unzip('directxsdk-' + get_target_arch())
 
+  # get sccache & set exec bit. https://bugs.python.org/issue15795
+  download_and_unzip('sccache-{0}-x64'.format(PLATFORM))
+  appname = 'sccache'
+  if sys.platform == 'win32':
+    appname += '.exe'
+  add_exec_bit(os.path.join('external_binaries', appname))
+
   with open(version_file, 'w') as f:
     f.write(VERSION)