#!/usr/bin/env python import atexit import contextlib import datetime import errno import platform import re import shutil import ssl import stat import subprocess import sys import tarfile import tempfile import urllib2 import os import zipfile from config import is_verbose_mode, PLATFORM from env_util import get_vs_env BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', 'boto')) NPM = 'npm' if sys.platform in ['win32', 'cygwin']: NPM += '.cmd' def get_host_arch(): """Returns the host architecture with a predictable string.""" host_arch = platform.machine() # Convert machine type to format recognized by gyp. if re.match(r'i.86', host_arch) or host_arch == 'i86pc': host_arch = 'ia32' elif host_arch in ['x86_64', 'amd64']: host_arch = 'x64' elif host_arch.startswith('arm'): host_arch = 'arm' # platform.machine is based on running kernel. It's possible to use 64-bit # kernel with 32-bit userland, e.g. to give linker slightly more memory. # Distinguish between different userland bitness by querying # the python binary. if host_arch == 'x64' and platform.architecture()[0] == '32bit': host_arch = 'ia32' return host_arch def tempdir(prefix=''): directory = tempfile.mkdtemp(prefix=prefix) atexit.register(shutil.rmtree, directory) return directory @contextlib.contextmanager def scoped_cwd(path): cwd = os.getcwd() os.chdir(path) try: yield finally: os.chdir(cwd) @contextlib.contextmanager def scoped_env(key, value): origin = '' if key in os.environ: origin = os.environ[key] os.environ[key] = value try: yield finally: os.environ[key] = origin def download(text, url, path): safe_mkdir(os.path.dirname(path)) with open(path, 'wb') as local_file: if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context print "Downloading %s to %s" % (url, path) web_file = urllib2.urlopen(url) file_size = int(web_file.info().getheaders("Content-Length")[0]) downloaded_size = 0 block_size = 128 ci = os.environ.get('CI') is not None while True: buf = web_file.read(block_size) if not buf: break downloaded_size += len(buf) local_file.write(buf) if not ci: percent = downloaded_size * 100. / file_size status = "\r%s %10d [%3.1f%%]" % (text, downloaded_size, percent) print status, if ci: print "%s done." % (text) else: print return path def extract_tarball(tarball_path, member, destination): with tarfile.open(tarball_path) as tarball: tarball.extract(member, destination) def extract_zip(zip_path, destination): if sys.platform == 'darwin': # Use unzip command on Mac to keep symbol links in zip file work. execute(['unzip', zip_path, '-d', destination]) else: with zipfile.ZipFile(zip_path) as z: z.extractall(destination) def make_zip(zip_file_path, files, dirs): safe_unlink(zip_file_path) if sys.platform == 'darwin': files += dirs execute(['zip', '-r', '-y', zip_file_path] + files) else: zip_file = zipfile.ZipFile(zip_file_path, "w", zipfile.ZIP_DEFLATED) for filename in files: zip_file.write(filename, filename) for dirname in dirs: for root, _, filenames in os.walk(dirname): for f in filenames: zip_file.write(os.path.join(root, f)) zip_file.close() def rm_rf(path): try: shutil.rmtree(path) except OSError: pass def safe_unlink(path): try: os.unlink(path) except OSError as e: if e.errno != errno.ENOENT: raise def safe_mkdir(path): try: os.makedirs(path) except OSError as e: if e.errno != errno.EEXIST: raise def execute(argv, env=os.environ, cwd=None): if is_verbose_mode(): print ' '.join(argv) try: output = subprocess.check_output(argv, stderr=subprocess.STDOUT, env=env, cwd=cwd) if is_verbose_mode(): print output return output except subprocess.CalledProcessError as e: print e.output raise e def execute_stdout(argv, env=os.environ, cwd=None): if is_verbose_mode(): print ' '.join(argv) try: subprocess.check_call(argv, env=env, cwd=cwd) except subprocess.CalledProcessError as e: print e.output raise e else: execute(argv, env, cwd) def electron_gyp(): SOURCE_ROOT = os.path.abspath(os.path.join(__file__, '..', '..', '..')) gyp = os.path.join(SOURCE_ROOT, 'electron.gyp') with open(gyp) as f: obj = eval(f.read()); return obj['variables'] def electron_features(): SOURCE_ROOT = os.path.abspath(os.path.join(__file__, '..', '..', '..')) gyp = os.path.join(SOURCE_ROOT, 'features.gypi') with open(gyp) as f: obj = eval(f.read()); return obj['variables']['variables'] def get_electron_version(): return 'v' + electron_gyp()['version%'] def parse_version(version): if version[0] == 'v': version = version[1:] vs = version.split('.') if len(vs) > 4: return vs[0:4] else: return vs + ['0'] * (4 - len(vs)) def boto_path_dirs(): return [ os.path.join(BOTO_DIR, 'build', 'lib'), os.path.join(BOTO_DIR, 'build', 'lib.linux-x86_64-2.7') ] def run_boto_script(access_key, secret_key, script_name, *args): env = os.environ.copy() env['AWS_ACCESS_KEY_ID'] = access_key env['AWS_SECRET_ACCESS_KEY'] = secret_key env['PYTHONPATH'] = os.path.pathsep.join( [env.get('PYTHONPATH', '')] + boto_path_dirs()) boto = os.path.join(BOTO_DIR, 'bin', script_name) execute([sys.executable, boto] + list(args), env) def s3put(bucket, access_key, secret_key, prefix, key_prefix, files): args = [ '--bucket', bucket, '--prefix', prefix, '--key_prefix', key_prefix, '--grant', 'public-read' ] + files run_boto_script(access_key, secret_key, 's3put', *args) def import_vs_env(target_arch): if sys.platform != 'win32': return if target_arch == 'ia32': vs_arch = 'amd64_x86' else: vs_arch = 'x86_amd64' env = get_vs_env('[15.0,16.0)', vs_arch) os.environ.update(env) def set_clang_env(env): SOURCE_ROOT = os.path.abspath(os.path.join(__file__, '..', '..', '..')) llvm_dir = os.path.join(SOURCE_ROOT, 'vendor', 'llvm-build', 'Release+Asserts', 'bin') env['CC'] = os.path.join(llvm_dir, 'clang') env['CXX'] = os.path.join(llvm_dir, 'clang++') def update_electron_modules(dirname, target_arch, nodedir): env = os.environ.copy() version = get_electron_version() env['npm_config_arch'] = target_arch env['npm_config_target'] = version env['npm_config_nodedir'] = nodedir update_node_modules(dirname, env) execute_stdout([NPM, 'rebuild'], env, dirname) def update_node_modules(dirname, env=None): if env is None: env = os.environ.copy() if PLATFORM == 'linux': # Use prebuilt clang for building native modules. set_clang_env(env) env['npm_config_clang'] = '1' with scoped_cwd(dirname): args = [NPM, 'install'] if is_verbose_mode(): args += ['--verbose'] # Ignore npm install errors when running in CI. if os.environ.has_key('CI'): try: execute_stdout(args, env) except subprocess.CalledProcessError: pass else: execute_stdout(args, env) def add_exec_bit(filename): os.chmod(filename, os.stat(filename).st_mode | stat.S_IEXEC) def clean_parse_version(v): return parse_version(v.split("-")[0]) def is_stable(v): return len(v.split(".")) == 3 def is_beta(v): return 'beta' in v def is_nightly(v): return 'nightly' in v def get_nightly_date(): return datetime.datetime.today().strftime('%Y%m%d') def get_last_major(): return execute(['node', 'script/get-last-major-for-master.js']) def get_next_nightly(v): pv = clean_parse_version(v) major = pv[0]; minor = pv[1]; patch = pv[2] if (is_stable(v)): patch = str(int(pv[2]) + 1) if execute(['git', 'rev-parse', '--abbrev-ref', 'HEAD']) == "master": major = str(get_last_major() + 1) minor = '0' patch = '0' pre = 'nightly.' + get_nightly_date() return make_version(major, minor, patch, pre) def non_empty(thing): return thing.strip() != '' def get_next_beta(v): pv = clean_parse_version(v) tag_pattern = 'v' + pv[0] + '.' + pv[1] + '.' + pv[2] + '-beta.*' tag_list = filter( non_empty, execute(['git', 'tag', '--list', '-l', tag_pattern]).strip().split('\n') ) if len(tag_list) == 0: return make_version(pv[0] , pv[1], pv[2], 'beta.1') lv = parse_version(tag_list[-1]) return make_version(pv[0] , pv[1], pv[2], 'beta.' + str(int(lv[3]) + 1)) def get_next_stable_from_pre(v): pv = clean_parse_version(v) major = pv[0]; minor = pv[1]; patch = pv[2] return make_version(major, minor, patch) def get_next_stable_from_stable(v): pv = clean_parse_version(v) major = pv[0]; minor = pv[1]; patch = pv[2] return make_version(major, minor, str(int(patch) + 1)) def make_version(major, minor, patch, pre = None): if pre is None: return major + '.' + minor + '.' + patch return major + "." + minor + "." + patch + '-' + pre