Use S3 as an intermediary store & merge shasum files before uploading to GitHub

script/lib/util.py

@@ -16,9 +16,12 @@ import urllib2
 import os
 import zipfile
 
-from config import is_verbose_mode
+from config import is_verbose_mode, s3_config
 from env_util import get_vs_env
 
+BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor',
+                                        'boto'))
+
 
 def get_host_arch():
   """Returns the host architecture with a predictable string."""
@@ -130,10 +133,11 @@ def make_zip(zip_file_path, files, dirs):
         for f in filenames:
           zip_file.write(os.path.join(root, f))
     zip_file.close()
-  make_zip_sha256_checksum(zip_file_path)
+  upload_zip_sha256_checksum(zip_file_path)
 
 
-def make_zip_sha256_checksum(zip_file_path):
+def upload_zip_sha256_checksum(zip_file_path):
+  bucket, access_key, secret_key = s3_config()
   checksum_path = '{}.sha256sum'.format(zip_file_path)
   safe_unlink(checksum_path)
   sha256 = hashlib.sha256()
@@ -143,6 +147,8 @@ def make_zip_sha256_checksum(zip_file_path):
   zip_basename = os.path.basename(zip_file_path)
   with open(checksum_path, 'w') as checksum:
     checksum.write('{} *{}'.format(sha256.hexdigest(), zip_basename))
+  s3put(bucket, access_key, secret_key, os.path.dirname(checksum_path),
+      'atom-shell/tmp', [checksum_path])
 
 
 def rm_rf(path):
@@ -216,28 +222,39 @@ def parse_version(version):
     return vs + ['0'] * (4 - len(vs))
 
 
-def s3put(bucket, access_key, secret_key, prefix, key_prefix, files):
+def boto_path_dirs():
-  env = os.environ.copy()
+  return [
-  BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor',
-                                          'boto'))
-  env['PYTHONPATH'] = os.path.pathsep.join([
-    env.get('PYTHONPATH', ''),
     os.path.join(BOTO_DIR, 'build', 'lib'),
-    os.path.join(BOTO_DIR, 'build', 'lib.linux-x86_64-2.7')])
+    os.path.join(BOTO_DIR, 'build', 'lib.linux-x86_64-2.7')
+  ]
+
+
+def run_boto_script(access_key, secret_key, script_name, *args):
+  env = os.environ.copy()
+  env['AWS_ACCESS_KEY_ID'] = access_key
+  env['AWS_SECRET_ACCESS_KEY'] = secret_key
+  env['PYTHONPATH'] = os.path.pathsep.join(
+      [env.get('PYTHONPATH', '')] + boto_path_dirs())
+
+  boto = os.path.join(BOTO_DIR, 'bin', script_name)
 
-  boto = os.path.join(BOTO_DIR, 'bin', 's3put')
   args = [
     sys.executable,
-    boto,
+    boto
+  ] + args
+
+  execute(args, env)
+
+
+def s3put(bucket, access_key, secret_key, prefix, key_prefix, files):
+  args = [
     '--bucket', bucket,
-    '--access_key', access_key,
-    '--secret_key', secret_key,
     '--prefix', prefix,
     '--key_prefix', key_prefix,
     '--grant', 'public-read'
   ] + files
 
-  execute(args, env)
+  run_boto_script(access_key, secret_key, 's3put', *args)
 
 
 def import_vs_env(target_arch):

script/merge-electron-checksums.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+# Download individual checksum files for Electron zip files from S3,
+# concatenate them, and upload to GitHub.
+
+from __future__ import print_function
+
+import argparse
+import sys
+
+from lib.config import s3_config
+from lib.util import boto_path_dirs
+
+sys.path.extend(boto_path_dirs())
+
+from boto.s3.connection import S3Connection
+
+def main():
+  args = parse_args()
+  bucket_name, access_key, secret_key = s3_config()
+  s3 = S3Connection(access_key, secret_key)
+  bucket = s3.get_bucket(bucket_name)
+  if bucket is None:
+    print('S3 bucket "{}" does not exist!'.format(bucket_name), file=sys.stderr)
+    return 1
+  shasums = [s3_object.get_contents_as_string().strip()
+             for s3_object in bucket.list('atom-shell/tmp/', delimiter='/')
+             if s3_object.key.endswith('.sha256sum') and
+             args.version in s3_object.key]
+  print('\n'.join(shasums))
+  return 0
+
+
+def parse_args():
+  parser = argparse.ArgumentParser(description='Upload SHASUMS files to GitHub')
+  parser.add_argument('-v', '--version', help='Specify the version',
+                      required=True)
+  return parser.parse_args()
+
+if __name__ == '__main__':
+  sys.exit(main())

script/upload.py

@@ -2,6 +2,7 @@
 
 import argparse
 import errno
+from io import StringIO
 import os
 import subprocess
 import sys
@@ -46,8 +47,7 @@ def main():
 
   if not args.publish_release:
     if not dist_newer_than_head():
-      create_dist = os.path.join(SOURCE_ROOT, 'script', 'create-dist.py')
+      run_python_script('create-dist.py')
-      execute([sys.executable, create_dist])
 
     build_version = get_electron_build_version()
     if not ELECTRON_VERSION.startswith(build_version):
@@ -69,14 +69,14 @@ def main():
                                         tag_exists)
 
   if args.publish_release:
-    # Upload the SHASUMS.txt.
+    # Upload the Node SHASUMS*.txt.
-    execute([sys.executable,
+    run_python_script('upload-node-checksums.py', '-v', ELECTRON_VERSION)
-             os.path.join(SOURCE_ROOT, 'script', 'upload-checksums.py'),
-             '-v', ELECTRON_VERSION])
 
     # Upload the index.json.
-    execute([sys.executable,
+    run_python_script('upload-index-json.py')
-             os.path.join(SOURCE_ROOT, 'script', 'upload-index-json.py')])
+
+    # Create and upload the Electron SHASUMS*.txt
+    release_electron_checksums(github, release)
 
     # Press the publish button.
     publish_release(github, release['id'])
@@ -108,13 +108,10 @@ def main():
 
   if PLATFORM == 'win32' and not tag_exists:
     # Upload PDBs to Windows symbol server.
-    execute([sys.executable,
+    run_python_script('upload-windows-pdb.py')
-             os.path.join(SOURCE_ROOT, 'script', 'upload-windows-pdb.py')])
 
     # Upload node headers.
-    execute([sys.executable,
+    run_python_script('upload-node-headers.py', '-v', args.version)
-             os.path.join(SOURCE_ROOT, 'script', 'upload-node-headers.py'),
-             '-v', args.version])
 
 
 def parse_args():
@@ -127,6 +124,11 @@ def parse_args():
   return parser.parse_args()
 
 
+def run_python_script(script, *args):
+  script_path = os.path.join(SOURCE_ROOT, 'script', script),
+  return execute([sys.executable, script_path] + args)
+
+
 def get_electron_build_version():
   if get_target_arch() == 'arm' or os.environ.has_key('CI'):
     # In CI we just build as told.
@@ -202,32 +204,35 @@ def create_release_draft(github, tag):
   return r
 
 
+def release_electron_checksums(github, release):
+  checksums = run_python_script(
+      'merge-electron-checksums.py', '-v', ELECTRON_VERSION)
+  upload_io_to_github(github, release,
+      'SHASUMS256.txt', StringIO(checksums), 'text/plain')
+
+
 def upload_electron(github, release, file_path):
-  checksum_path = '{}.sha256sum'.format(file_path)
+  # Delete the original file before uploading in CI.
-  # Delete the original file & its checksum before uploading in CI.
   filename = os.path.basename(file_path)
-  checksum_filename = os.path.basename(checksum_path)
   if os.environ.has_key('CI'):
     try:
       for asset in release['assets']:
-        if asset['name'] in [filename, checksum_filename]:
+        if asset['name'] == filename:
           github.repos(ELECTRON_REPO).releases.assets(asset['id']).delete()
     except Exception:
       pass
 
   # Upload the file.
-  upload_asset_to_github(github, release, file_path, 'application/zip')
+  name = os.path.dirname(file_path)
-
+  with open(file_path, 'rb') as f:
-  # Upload the file's checksum.
+    upload_io_to_github(github, release, name, f, 'application/zip')
-  upload_asset_to_github(github, release, checksum_path, 'text/plain')
 
 
-def upload_asset_to_github(github, release, asset_path, content_type):
+def upload_io_to_github(github, release, name, io, content_type):
-  params = {'name': os.path.dirname(asset_path)}
+  params = {'name': name}
   headers = {'Content-Type': content_type}
-  with open(asset_path) as f:
   github.repos(ELECTRON_REPO).releases(release['id']).assets.post(
-        params=params, headers=headers, data=f, verify=False)
+      params=params, headers=headers, data=io, verify=False)
 
 
 def publish_release(github, release_id):