From 079a7a1a1cde24ca7af069a2ad55531461410e75 Mon Sep 17 00:00:00 2001 From: Cheng Zhao Date: Mon, 1 Aug 2016 21:03:55 +0900 Subject: [PATCH 1/6] Do not put the upload logic in make_zip --- script/lib/util.py | 19 +------------------ script/merge-electron-checksums.py | 11 +++++++---- script/upload.py | 24 +++++++++++++++++++++--- 3 files changed, 29 insertions(+), 25 deletions(-) mode change 100644 => 100755 script/merge-electron-checksums.py diff --git a/script/lib/util.py b/script/lib/util.py index 933144d95c8..55e596ba098 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -3,7 +3,6 @@ import atexit import contextlib import errno -import hashlib import platform import re import shutil @@ -16,7 +15,7 @@ import urllib2 import os import zipfile -from config import is_verbose_mode, s3_config +from config import is_verbose_mode from env_util import get_vs_env BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', @@ -133,22 +132,6 @@ def make_zip(zip_file_path, files, dirs): for f in filenames: zip_file.write(os.path.join(root, f)) zip_file.close() - upload_zip_sha256_checksum(zip_file_path) - - -def upload_zip_sha256_checksum(zip_file_path): - bucket, access_key, secret_key = s3_config() - checksum_path = '{}.sha256sum'.format(zip_file_path) - safe_unlink(checksum_path) - sha256 = hashlib.sha256() - with open(zip_file_path, 'rb') as f: - sha256.update(f.read()) - - zip_basename = os.path.basename(zip_file_path) - with open(checksum_path, 'w') as checksum: - checksum.write('{} *{}'.format(sha256.hexdigest(), zip_basename)) - s3put(bucket, access_key, secret_key, os.path.dirname(checksum_path), - 'atom-shell/tmp', [checksum_path]) def rm_rf(path): diff --git a/script/merge-electron-checksums.py b/script/merge-electron-checksums.py old mode 100644 new mode 100755 index 7a5be5fe14a..bab9d97a0c4 --- a/script/merge-electron-checksums.py +++ b/script/merge-electron-checksums.py @@ -12,21 +12,23 @@ from lib.config import s3_config from lib.util import boto_path_dirs sys.path.extend(boto_path_dirs()) - from boto.s3.connection import S3Connection + def main(): args = parse_args() + bucket_name, access_key, secret_key = s3_config() s3 = S3Connection(access_key, secret_key) bucket = s3.get_bucket(bucket_name) if bucket is None: print('S3 bucket "{}" does not exist!'.format(bucket_name), file=sys.stderr) return 1 + + prefix = 'atom-shell/tmp/{0}/'.format(args.version) shasums = [s3_object.get_contents_as_string().strip() - for s3_object in bucket.list('atom-shell/tmp/', delimiter='/') - if s3_object.key.endswith('.sha256sum') and - args.version in s3_object.key] + for s3_object in bucket.list(prefix, delimiter='/') + if s3_object.key.endswith('.sha256sum')] print('\n'.join(shasums)) return 0 @@ -37,5 +39,6 @@ def parse_args(): required=True) return parser.parse_args() + if __name__ == '__main__': sys.exit(main()) diff --git a/script/upload.py b/script/upload.py index e41946dffe4..8e04f35cc32 100755 --- a/script/upload.py +++ b/script/upload.py @@ -2,16 +2,17 @@ import argparse import errno -from io import StringIO +import hashlib import os import subprocess import sys import tempfile +from io import StringIO from lib.config import PLATFORM, get_target_arch, get_chromedriver_version, \ - get_platform_key, get_env_var + get_platform_key, get_env_var, s3_config from lib.util import electron_gyp, execute, get_electron_version, \ - parse_version, scoped_cwd + parse_version, scoped_cwd, s3put from lib.github import GitHub @@ -227,6 +228,9 @@ def upload_electron(github, release, file_path): with open(file_path, 'rb') as f: upload_io_to_github(github, release, name, f, 'application/zip') + # Upload the checksum file. + upload_sha256_checksum(release['tag'], file_path) + def upload_io_to_github(github, release, name, io, content_type): params = {'name': name} @@ -235,6 +239,20 @@ def upload_io_to_github(github, release, name, io, content_type): params=params, headers=headers, data=io, verify=False) +def upload_sha256_checksum(version, file_path): + bucket, access_key, secret_key = s3_config() + checksum_path = '{}.sha256sum'.format(file_path) + sha256 = hashlib.sha256() + with open(file_path, 'rb') as f: + sha256.update(f.read()) + + filename = os.path.basename(file_path) + with open(checksum_path, 'w') as checksum: + checksum.write('{} *{}'.format(sha256.hexdigest(), filename)) + s3put(bucket, access_key, secret_key, os.path.dirname(checksum_path), + 'atom-shell/tmp/{0}'.format(version), [checksum_path]) + + def publish_release(github, release_id): data = dict(draft=False) github.repos(ELECTRON_REPO).releases(release_id).patch(data=data) From d00bff4d94d5ddae1bfe05b9c507b97fe410f010 Mon Sep 17 00:00:00 2001 From: Cheng Zhao Date: Mon, 1 Aug 2016 21:41:55 +0900 Subject: [PATCH 2/6] Fix error of run_python_script --- script/upload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/upload.py b/script/upload.py index 8e04f35cc32..a8223afd251 100755 --- a/script/upload.py +++ b/script/upload.py @@ -126,8 +126,8 @@ def parse_args(): def run_python_script(script, *args): - script_path = os.path.join(SOURCE_ROOT, 'script', script), - return execute([sys.executable, script_path] + args) + script_path = os.path.join(SOURCE_ROOT, 'script', script) + return execute([sys.executable, script_path] + list(args)) def get_electron_build_version(): From c943e4a61fb903d7df023f3024895a311c90b589 Mon Sep 17 00:00:00 2001 From: Cheng Zhao Date: Mon, 1 Aug 2016 21:54:03 +0900 Subject: [PATCH 3/6] Fix error in run_boto_script --- script/lib/util.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/script/lib/util.py b/script/lib/util.py index 55e596ba098..cff7e94a042 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -220,13 +220,7 @@ def run_boto_script(access_key, secret_key, script_name, *args): [env.get('PYTHONPATH', '')] + boto_path_dirs()) boto = os.path.join(BOTO_DIR, 'bin', script_name) - - args = [ - sys.executable, - boto - ] + args - - execute(args, env) + execute([sys.executable, boto] + list(args), env) def s3put(bucket, access_key, secret_key, prefix, key_prefix, files): From 217b639597bf32a8231377a7f2870004bcaf447e Mon Sep 17 00:00:00 2001 From: Cheng Zhao Date: Mon, 1 Aug 2016 21:59:25 +0900 Subject: [PATCH 4/6] Fix filename when uploading to github --- script/upload.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/script/upload.py b/script/upload.py index a8223afd251..375b07500e9 100755 --- a/script/upload.py +++ b/script/upload.py @@ -224,12 +224,11 @@ def upload_electron(github, release, file_path): pass # Upload the file. - name = os.path.dirname(file_path) with open(file_path, 'rb') as f: - upload_io_to_github(github, release, name, f, 'application/zip') + upload_io_to_github(github, release, filename, f, 'application/zip') # Upload the checksum file. - upload_sha256_checksum(release['tag'], file_path) + upload_sha256_checksum(release['tag_name'], file_path) def upload_io_to_github(github, release, name, io, content_type): From 94a17bb494f9b61bd4713032757c4e88112af30f Mon Sep 17 00:00:00 2001 From: Cheng Zhao Date: Mon, 1 Aug 2016 22:00:35 +0900 Subject: [PATCH 5/6] Use spaces between checksum and filename --- script/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/upload.py b/script/upload.py index 375b07500e9..8268dd09124 100755 --- a/script/upload.py +++ b/script/upload.py @@ -247,7 +247,7 @@ def upload_sha256_checksum(version, file_path): filename = os.path.basename(file_path) with open(checksum_path, 'w') as checksum: - checksum.write('{} *{}'.format(sha256.hexdigest(), filename)) + checksum.write('{} {}'.format(sha256.hexdigest(), filename)) s3put(bucket, access_key, secret_key, os.path.dirname(checksum_path), 'atom-shell/tmp/{0}'.format(version), [checksum_path]) From 9765599b4984adeb3cc0082282ec4c9ce4e1cdab Mon Sep 17 00:00:00 2001 From: Cheng Zhao Date: Mon, 1 Aug 2016 22:08:29 +0900 Subject: [PATCH 6/6] Must pass unicode to StringIO --- script/upload.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/script/upload.py b/script/upload.py index 8268dd09124..ee97220a82b 100755 --- a/script/upload.py +++ b/script/upload.py @@ -206,10 +206,10 @@ def create_release_draft(github, tag): def release_electron_checksums(github, release): - checksums = run_python_script( - 'merge-electron-checksums.py', '-v', ELECTRON_VERSION) - upload_io_to_github(github, release, - 'SHASUMS256.txt', StringIO(checksums), 'text/plain') + checksums = run_python_script('merge-electron-checksums.py', + '-v', ELECTRON_VERSION) + upload_io_to_github(github, release, 'SHASUMS256.txt', + StringIO(checksums.decode('utf-8')), 'text/plain') def upload_electron(github, release, file_path):