electron/script/release/uploaders/upload.py

#!/usr/bin/env python

from __future__ import print_function
import argparse
import datetime
import hashlib
import json
import mmap
import os
import shutil
import subprocess
from struct import Struct
import sys

sys.path.append(
  os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../.."))

from zipfile import ZipFile
from lib.config import PLATFORM, get_target_arch,  get_env_var, s3_config, \
                       get_zip_name, enable_verbose_mode, get_platform_key
from lib.util import get_electron_branding, execute, get_electron_version, \
                     s3put, get_electron_exec, get_out_dir, \
                     SRC_DIR, ELECTRON_DIR


ELECTRON_REPO = 'electron/electron'
ELECTRON_VERSION = get_electron_version()

PROJECT_NAME = get_electron_branding()['project_name']
PRODUCT_NAME = get_electron_branding()['product_name']

OUT_DIR = get_out_dir()

DIST_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION)
SYMBOLS_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'symbols')
DSYM_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'dsym')
PDB_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'pdb')
DEBUG_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION, 'debug')
TOOLCHAIN_PROFILE_NAME = get_zip_name(PROJECT_NAME, ELECTRON_VERSION,
                                      'toolchain-profile')


def main():
  args = parse_args()
  if args.verbose:
    enable_verbose_mode()
  if args.upload_to_s3:
    utcnow = datetime.datetime.utcnow()
    args.upload_timestamp = utcnow.strftime('%Y%m%d')

  build_version = get_electron_build_version()
  if not ELECTRON_VERSION.startswith(build_version):
    error = 'Tag name ({0}) should match build version ({1})\n'.format(
        ELECTRON_VERSION, build_version)
    sys.stderr.write(error)
    sys.stderr.flush()
    return 1

  tag_exists = False
  release = get_release(args.version)
  if not release['draft']:
    tag_exists = True

  if not args.upload_to_s3:
    assert release['exists'], \
          'Release does not exist; cannot upload to GitHub!'
    assert tag_exists == args.overwrite, \
          'You have to pass --overwrite to overwrite a published release'

  # Upload Electron files.
  # Rename dist.zip to  get_zip_name('electron', version, suffix='')
  electron_zip = os.path.join(OUT_DIR, DIST_NAME)
  shutil.copy2(os.path.join(OUT_DIR, 'dist.zip'), electron_zip)
  upload_electron(release, electron_zip, args)
  if get_target_arch() != 'mips64el':
    symbols_zip = os.path.join(OUT_DIR, SYMBOLS_NAME)
    shutil.copy2(os.path.join(OUT_DIR, 'symbols.zip'), symbols_zip)
    upload_electron(release, symbols_zip, args)
  if get_platform_key() == 'darwin':
    if get_target_arch() == 'x64':
      api_path = os.path.join(ELECTRON_DIR, 'electron-api.json')
      upload_electron(release, api_path, args)

      ts_defs_path = os.path.join(ELECTRON_DIR, 'electron.d.ts')
      upload_electron(release, ts_defs_path, args)

    dsym_zip = os.path.join(OUT_DIR, DSYM_NAME)
    shutil.copy2(os.path.join(OUT_DIR, 'dsym.zip'), dsym_zip)
    upload_electron(release, dsym_zip, args)
  elif PLATFORM == 'win32':
    pdb_zip = os.path.join(OUT_DIR, PDB_NAME)
    shutil.copy2(os.path.join(OUT_DIR, 'pdb.zip'), pdb_zip)
    upload_electron(release, pdb_zip, args)
  elif PLATFORM == 'linux':
    debug_zip = os.path.join(OUT_DIR, DEBUG_NAME)
    shutil.copy2(os.path.join(OUT_DIR, 'debug.zip'), debug_zip)
    upload_electron(release, debug_zip, args)

  # Upload free version of ffmpeg.
  ffmpeg = get_zip_name('ffmpeg', ELECTRON_VERSION)
  ffmpeg_zip = os.path.join(OUT_DIR, ffmpeg)
  ffmpeg_build_path = os.path.join(SRC_DIR, 'out', 'ffmpeg', 'ffmpeg.zip')
  shutil.copy2(ffmpeg_build_path, ffmpeg_zip)
  upload_electron(release, ffmpeg_zip, args)

  chromedriver = get_zip_name('chromedriver', ELECTRON_VERSION)
  chromedriver_zip = os.path.join(OUT_DIR, chromedriver)
  shutil.copy2(os.path.join(OUT_DIR, 'chromedriver.zip'), chromedriver_zip)
  upload_electron(release, chromedriver_zip, args)

  mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION)
  mksnapshot_zip = os.path.join(OUT_DIR, mksnapshot)
  if get_target_arch().startswith('arm') and PLATFORM != 'darwin':
    # Upload the x64 binary for arm/arm64 mksnapshot
    mksnapshot = get_zip_name('mksnapshot', ELECTRON_VERSION, 'x64')
    mksnapshot_zip = os.path.join(OUT_DIR, mksnapshot)

  shutil.copy2(os.path.join(OUT_DIR, 'mksnapshot.zip'), mksnapshot_zip)
  upload_electron(release, mksnapshot_zip, args)

  if PLATFORM == 'linux' and get_target_arch() == 'x64':
    # Upload the hunspell dictionaries only from the linux x64 build
    hunspell_dictionaries_zip = os.path.join(
      OUT_DIR, 'hunspell_dictionaries.zip')
    upload_electron(release, hunspell_dictionaries_zip, args)

  if not tag_exists and not args.upload_to_s3:
    # Upload symbols to symbol server.
    run_python_upload_script('upload-symbols.py')
    if PLATFORM == 'win32':
      run_python_upload_script('upload-node-headers.py', '-v', args.version)

  if PLATFORM == 'win32':
    toolchain_profile_zip = os.path.join(OUT_DIR, TOOLCHAIN_PROFILE_NAME)
    with ZipFile(toolchain_profile_zip, 'w') as myzip:
      myzip.write(
        os.path.join(OUT_DIR, 'windows_toolchain_profile.json'),
        'toolchain_profile.json')
    upload_electron(release, toolchain_profile_zip, args)


def parse_args():
  parser = argparse.ArgumentParser(description='upload distribution file')
  parser.add_argument('-v', '--version', help='Specify the version',
                      default=ELECTRON_VERSION)
  parser.add_argument('-o', '--overwrite',
                      help='Overwrite a published release',
                      action='store_true')
  parser.add_argument('-p', '--publish-release',
                      help='Publish the release',
                      action='store_true')
  parser.add_argument('-s', '--upload_to_s3',
                      help='Upload assets to s3 bucket',
                      dest='upload_to_s3',
                      action='store_true',
                      default=False,
                      required=False)
  parser.add_argument('--verbose', help='Mooooorreee logs')
  return parser.parse_args()


def run_python_upload_script(script, *args):
  script_path = os.path.join(
    ELECTRON_DIR, 'script', 'release', 'uploaders', script)
  print(execute([sys.executable, script_path] + list(args)))


def get_electron_build_version():
  if get_target_arch().startswith('arm') or 'CI' in os.environ:
    # In CI we just build as told.
    return ELECTRON_VERSION
  electron = get_electron_exec()
  return subprocess.check_output([electron, '--version']).strip()


class NonZipFileError(ValueError):
  """Raised when a given file does not appear to be a zip"""


def zero_zip_date_time(fname):
  """ Wrap strip-zip zero_zip_date_time within a file opening operation """
  try:
    with open(fname, 'r+b') as f:
      _zero_zip_date_time(f)
  except:
    raise NonZipFileError(fname)


def _zero_zip_date_time(zip_):
  def purify_extra_data(mm, offset, length, compressed_size=0):
    extra_header_struct = Struct("<HH")
    # 0. id
    # 1. length
    STRIPZIP_OPTION_HEADER = 0xFFFF
    EXTENDED_TIME_DATA = 0x5455
    # Some sort of extended time data, see
    # ftp://ftp.info-zip.org/pub/infozip/src/zip30.zip ./proginfo/extrafld.txt
    # fallthrough
    UNIX_EXTRA_DATA = 0x7875
    # Unix extra data; UID / GID stuff, see
    # ftp://ftp.info-zip.org/pub/infozip/src/zip30.zip ./proginfo/extrafld.txt
    ZIP64_EXTRA_HEADER = 0x0001
    zip64_extra_struct = Struct("<HHQQ")
    # ZIP64.
    # When a ZIP64 extra field is present his 8byte length
    # will override the 4byte length defined in canonical zips.
    # This is in the form:
    # - 0x0001 (header_id)
    # - 0x0010 [16] (header_length)
    # - ... (8byte uncompressed_length)
    # - ... (8byte compressed_length)
    mlen = offset + length

    while offset < mlen:
      values = list(extra_header_struct.unpack_from(mm, offset))
      _, header_length = values
      extra_struct = Struct("<HH" + "B" * header_length)
      values = list(extra_struct.unpack_from(mm, offset))
      header_id, header_length = values[:2]

      if header_id in (EXTENDED_TIME_DATA, UNIX_EXTRA_DATA):
        values[0] = STRIPZIP_OPTION_HEADER
        for i in range(2, len(values)):
          values[i] = 0xff
        extra_struct.pack_into(mm, offset, *values)
      if header_id == ZIP64_EXTRA_HEADER:
        assert header_length == 16
        values = list(zip64_extra_struct.unpack_from(mm, offset))
        header_id, header_length, _, compressed_size = values

      offset += extra_header_struct.size + header_length

    return compressed_size

  FILE_HEADER_SIGNATURE = 0x04034b50
  CENDIR_HEADER_SIGNATURE = 0x02014b50

  archive_size = os.fstat(zip_.fileno()).st_size
  signature_struct = Struct("<L")
  local_file_header_struct = Struct("<LHHHHHLLLHH")
  # 0. L signature
  # 1. H version_needed
  # 2. H gp_bits
  # 3. H compression_method
  # 4. H last_mod_time
  # 5. H last_mod_date
  # 6. L crc32
  # 7. L compressed_size
  # 8. L uncompressed_size
  # 9. H name_length
  # 10. H extra_field_length
  central_directory_header_struct = Struct("<LHHHHHHLLLHHHHHLL")
  # 0. L signature
  # 1. H version_made_by
  # 2. H version_needed
  # 3. H gp_bits
  # 4. H compression_method
  # 5. H last_mod_time
  # 6. H last_mod_date
  # 7. L crc32
  # 8. L compressed_size
  # 9. L uncompressed_size
  # 10. H file_name_length
  # 11. H extra_field_length
  # 12. H file_comment_length
  # 13. H disk_number_start
  # 14. H internal_attr
  # 15. L external_attr
  # 16. L rel_offset_local_header
  offset = 0

  mm = mmap.mmap(zip_.fileno(), 0)
  while offset < archive_size:
    if signature_struct.unpack_from(mm, offset) != (FILE_HEADER_SIGNATURE,):
      break
    values = list(local_file_header_struct.unpack_from(mm, offset))
    compressed_size, _, name_length, extra_field_length = values[7:11]
    # reset last_mod_time
    values[4] = 0
    # reset last_mod_date
    values[5] = 0x21
    local_file_header_struct.pack_into(mm, offset, *values)
    offset += local_file_header_struct.size + name_length
    if extra_field_length != 0:
      compressed_size = purify_extra_data(mm, offset, extra_field_length,
                                          compressed_size)
    offset += compressed_size + extra_field_length

  while offset < archive_size:
    if signature_struct.unpack_from(mm, offset) != (CENDIR_HEADER_SIGNATURE,):
      break
    values = list(central_directory_header_struct.unpack_from(mm, offset))
    file_name_length, extra_field_length, file_comment_length = values[10:13]
    # reset last_mod_time
    values[5] = 0
    # reset last_mod_date
    values[6] = 0x21
    central_directory_header_struct.pack_into(mm, offset, *values)
    offset += central_directory_header_struct.size
    offset += file_name_length + extra_field_length + file_comment_length
    if extra_field_length != 0:
      purify_extra_data(mm, offset - extra_field_length, extra_field_length)

  if offset == 0:
    raise NonZipFileError(zip_.name)


def upload_electron(release, file_path, args):
  filename = os.path.basename(file_path)

  # Strip zip non determinism before upload, in-place operation
  try:
    zero_zip_date_time(file_path)
  except NonZipFileError:
    pass

  # if upload_to_s3 is set, skip github upload.
  if args.upload_to_s3:
    bucket, access_key, secret_key = s3_config()
    key_prefix = 'electron-artifacts/{0}_{1}'.format(args.version,
                                                     args.upload_timestamp)
    s3put(bucket, access_key, secret_key, os.path.dirname(file_path),
          key_prefix, [file_path])
    upload_sha256_checksum(args.version, file_path, key_prefix)
    s3url = 'https://gh-contractor-zcbenz.s3.amazonaws.com'
    print('{0} uploaded to {1}/{2}/{0}'.format(filename, s3url, key_prefix))
    return

  # Upload the file.
  upload_io_to_github(release, filename, file_path, args.version)

  # Upload the checksum file.
  upload_sha256_checksum(args.version, file_path)


def upload_io_to_github(release, filename, filepath, version):
  print('Uploading %s to Github' % \
      (filename))
  script_path = os.path.join(
    ELECTRON_DIR, 'script', 'release', 'uploaders', 'upload-to-github.js')
  execute(['node', script_path, filepath, filename, str(release['id']),
          version])


def upload_sha256_checksum(version, file_path, key_prefix=None):
  bucket, access_key, secret_key = s3_config()
  checksum_path = '{}.sha256sum'.format(file_path)
  if key_prefix is None:
    key_prefix = 'atom-shell/tmp/{0}'.format(version)
  sha256 = hashlib.sha256()
  with open(file_path, 'rb') as f:
    sha256.update(f.read())

  filename = os.path.basename(file_path)
  with open(checksum_path, 'w') as checksum:
    checksum.write('{} *{}'.format(sha256.hexdigest(), filename))
  s3put(bucket, access_key, secret_key, os.path.dirname(checksum_path),
        key_prefix, [checksum_path])


def auth_token():
  token = get_env_var('GITHUB_TOKEN')
  message = ('Error: Please set the $ELECTRON_GITHUB_TOKEN '
             'environment variable, which is your personal token')
  assert token, message
  return token


def get_release(version):
  script_path = os.path.join(
    ELECTRON_DIR, 'script', 'release', 'find-github-release.js')
  release_info = execute(['node', script_path, version])
  release = json.loads(release_info)
  return release

if __name__ == '__main__':
  sys.exit(main())