build: move external binaries to S3 (#21829)

script/external-binaries.json

@@ -1,61 +1,48 @@
 {
-  "baseUrl": "https://github.com/electron/electron-frameworks/releases/download",
+  "baseUrl": "https://electron-build-tools.s3-us-west-2.amazonaws.com/build-dependencies",
-  "version": "v1.4.1",
+  "files": [
-  "binaries": [
     {
-      "url": "Mantle.zip",
+      "name": "Mantle.zip",
       "platform": "darwin",
       "sha": "f9865e115c03871b45d3a2d8734220cb147a02dace46c92f766ca5d3059281dd"
     },
     {
-      "url": "ReactiveCocoa.zip",
+      "name": "ReactiveCocoa.zip",
       "platform": "darwin",
       "sha": "8ae85cd226fa4076472bfdfcda4745b5c7edf31fbe695868068eeaf62e7fa962"
     },
     {
-      "url": "Squirrel.zip",
+      "name": "Squirrel.zip",
       "platform": "darwin",
       "sha": "e516fd5c24c0ad267fd854848b04be0552be977aa846fa7f3c65ef4618699511"
     },
     {
-      "url": "directxsdk-ia32.zip",
+      "name": "sccache-darwin-x64.zip",
-      "platform": "win32",
-      "targetArch": "ia32",
-      "sha": "f777bd5ab524bf39c3bfc68ac2b3f95ff2136c92328cf63e857f399e849db037"
-    },
-    {
-      "url": "directxsdk-x64.zip",
-      "platform": "win32",
-      "targetArch": "x64",
-      "sha": "46c1f8afb9180516013c39e8d73182a7f15f0ea89c61dc94f92605b4734d447b"
-    },
-    {
-      "url": "sccache-darwin-x64.zip",
       "platform": "darwin",
       "sha": "3bfe114b49a15e4f15e2e3a9ee6699f1acdb89446badbaa4144869c72a7690ca"
     },
     {
-      "url": "sccache-linux-x64.zip",
+      "name": "sccache-linux-x64.zip",
       "platform": "linux",
       "sha": "dd379b494122f9e85bdae3597b02c67b0a46192f20f4f16cae3f1258a57b39dd"
     },
     {
-      "url": "sccache-win32-x64.zip",
+      "name": "sccache-win32-x64.zip",
       "platform": "win32",
       "sha": "b6a20fd1c2026f3792e7286bc768a7ebc261847b76449b49f55455e1f841fecd"
     },
     {
-      "url": "goma-win.zip",
+      "name": "goma-win.zip",
       "platform": "win32",
       "sha": "f97c88aa5d49395ae20387b6329ad406fd019f5fb4aac4ba639ca928b7151f6b"
     },
     {
-      "url": "goma-linux.tgz",
+      "name": "goma-linux.tgz",
       "platform": "linux",
       "sha": "1cb3099a40f6200ae57216efa26af795c587b6ac7ae97955d1078d0b1e3011a6"
     },
     {
-      "url": "goma-mac.tgz",
+      "name": "goma-mac.tgz",
       "platform": "darwin",
       "sha": "da1e7de82fbf3b99f1a9d0f9bf51b25e75e8778fec180deb72cff873813d717c"
     }

script/update-external-binaries.py

@@ -12,6 +12,7 @@ from lib.util import add_exec_bit, download, extract_zip, rm_rf, \
                      safe_mkdir, tempdir
 
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+CONFIG_PATH = os.path.join(SOURCE_ROOT, 'script', 'external-binaries.json')
 
 def parse_args():
   parser = argparse.ArgumentParser(
@@ -25,8 +26,7 @@ def parse_args():
 
 
 def parse_config():
-  config_path = os.path.join(SOURCE_ROOT, 'script', 'external-binaries.json')
+  with open(CONFIG_PATH, 'r') as config_file:
-  with open(config_path, 'r') as config_file:
     config = json.load(config_file)
     return config
 
@@ -36,21 +36,21 @@ def main():
   config = parse_config()
 
   base_url = args.base_url if args.base_url is not None else config['baseUrl']
-  version = config['version']
-  output_dir = os.path.join(SOURCE_ROOT, 'external_binaries')
-  version_file = os.path.join(output_dir, '.version')
   
-  if (is_updated(version_file, version) and not args.force):
+  output_dir = os.path.join(SOURCE_ROOT, 'external_binaries')
+  config_hash_path = os.path.join(output_dir, '.hash')
+
+  if (not is_updated(config_hash_path) and not args.force):
     return
 
   rm_rf(output_dir)
   safe_mkdir(output_dir)
 
-  for binary in config['binaries']:
+  for binary in config['files']:
     if not binary_should_be_downloaded(binary):
       continue
 
-    temp_path = download_binary(base_url, version, binary['url'], binary['sha'])
+    temp_path = download_binary(base_url, binary['sha'], binary['name'])
 
     if temp_path.endswith('.zip'):
       extract_zip(temp_path, output_dir)
@@ -61,22 +61,22 @@ def main():
 
     # Hack alert. Set exec bit for sccache binaries.
     # https://bugs.python.org/issue15795
-    if 'sccache' in binary['url']:
+    if 'sccache' in binary['name']:
       add_exec_bit_to_sccache_binary(output_dir)
 
-  with open(version_file, 'w') as f:
+  with open(config_hash_path, 'w') as f:
-    f.write(version)
+    f.write(sha256(CONFIG_PATH))
 
 
-def is_updated(version_file, version):
+def is_updated(config_hash_path):
-  existing_version = ''
+  existing_hash = ''
   try:
-    with open(version_file, 'r') as f:
+    with open(config_hash_path, 'r') as f:
-      existing_version = f.readline().strip()
+      existing_hash = f.readline().strip()
   except IOError as e:
     if e.errno != errno.ENOENT:
       raise
-  return existing_version == version
+  return not sha256(CONFIG_PATH) == existing_hash
 
 
 def binary_should_be_downloaded(binary):
@@ -97,9 +97,9 @@ def sha256(file_path):
   return hash_256.hexdigest()
 
 
-def download_binary(base_url, version, binary_url, sha):
+def download_binary(base_url, sha, binary_name):
-  full_url = '{0}/{1}/{2}'.format(base_url, version, binary_url)
+  full_url = '{0}/{1}/{2}'.format(base_url, sha, binary_name)
-  temp_path = download_to_temp_dir(full_url, filename=binary_url, sha=sha)
+  temp_path = download_to_temp_dir(full_url, filename=binary_name, sha=sha)
   return temp_path