Merge pull request #2111 from atom/upload-in-ci

Create Release distributions in CI machine

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "vendor/crashpad"]
 	path = vendor/crashpad
 	url = https://github.com/atom/crashpad.git
+[submodule "vendor/requests"]
+	path = vendor/requests
+	url = https://github.com/kennethreitz/requests

script/bootstrap.py

@@ -37,6 +37,7 @@ def main():
     update_clang()
 
   update_submodules()
+  setup_requests()
   update_node_modules('.')
   bootstrap_brightray(args.dev, args.url, args.target_arch)
 
@@ -84,6 +85,11 @@ def update_submodules():
   execute_stdout(['git', 'submodule', 'update', '--init', '--recursive'])
 
 
+def setup_requests():
+  with scoped_cwd(os.path.join(VENDOR_DIR, 'requests')):
+    execute_stdout([sys.executable, 'setup.py', 'build'])
+
+
 def bootstrap_brightray(is_dev, url, target_arch):
   bootstrap = os.path.join(VENDOR_DIR, 'brightray', 'script', 'bootstrap')
   args = [
@@ -109,7 +115,7 @@ def update_node_modules(dirname, env=None):
   with scoped_cwd(dirname):
     args = [NPM, 'install']
     if is_verbose_mode():
-      args += '--verbose'
+      args += ['--verbose']
     # Ignore npm install errors when running in CI.
     if os.environ.has_key('CI'):
       try:

script/cibuild

@@ -43,15 +43,12 @@ def main():
       deps += LINUX_DEPS_ARM
     execute(['sudo', 'apt-get', 'install'] + deps)
 
-    os.environ['DISPLAY'] = ':99.0'
     execute(['sh', '-e', '/etc/init.d/xvfb', 'start'])
 
-  rm_rf(os.path.join(SOURCE_ROOT, 'out'))
+  if PLATFORM == 'linux':
-  rm_rf(os.path.join(SOURCE_ROOT, 'frameworks'))
+    os.environ['DISPLAY'] = ':99.0'
-  rm_rf(os.path.join(SOURCE_ROOT, 'external_binaries'))
+
-  rm_rf(os.path.join(SOURCE_ROOT, 'vendor', 'apm', 'node_modules'))
+  run_script('clean.py')
-  rm_rf(os.path.join(SOURCE_ROOT, 'vendor', 'brightray', 'vendor', 'download',
-                     'libchromiumcontent'))
 
   # CI's npm is not reliable.
   npm = 'npm.cmd' if PLATFORM == 'win32' else 'npm'
@@ -70,6 +67,7 @@ def main():
   if is_release:
     run_script('build.py', ['-c', 'R'])
     run_script('create-dist.py')
+    run_script('upload.py')
   elif PLATFORM == 'win32' or target_arch == 'x64':
     run_script('build.py', ['-c', 'D'])
     if PLATFORM != 'win32':

script/clean.py

@@ -12,8 +12,10 @@ SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 def main():
   os.chdir(SOURCE_ROOT)
   rm_rf('node_modules')
+  rm_rf('dist')
   rm_rf('out')
   rm_rf('spec/node_modules')
+  rm_rf('vendor/brightray/vendor/download/libchromiumcontent')
 
 
 if __name__ == '__main__':

script/lib/github.py

@@ -1,7 +1,14 @@
 #!/usr/bin/env python
 
 import json
+import os
 import re
+import sys
+
+REQUESTS_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..',
+                                            'vendor', 'requests'))
+sys.path.append(os.path.join(REQUESTS_DIR, 'build', 'lib'))
+sys.path.append(os.path.join(REQUESTS_DIR, 'build', 'lib.linux-x86_64-2.7'))
 import requests
 
 GITHUB_URL = 'https://api.github.com'

script/lib/util.py

@@ -133,9 +133,8 @@ def make_zip(zip_file_path, files, dirs):
 def rm_rf(path):
   try:
     shutil.rmtree(path)
-  except OSError as e:
+  except OSError:
-    if e.errno != errno.ENOENT:
+    pass
-      raise
 
 
 def safe_unlink(path):

script/update.py

@@ -51,6 +51,10 @@ def run_gyp(target_arch, component):
     # Force using win32 python on cygwin.
     python = os.path.join('vendor', 'python_26', 'python.exe')
   gyp = os.path.join('vendor', 'brightray', 'vendor', 'gyp', 'gyp_main.py')
+  gyp_pylib = os.path.join(os.path.dirname(gyp), 'pylib')
+  # Avoid using the old gyp lib in system.
+  env['PYTHONPATH'] = os.path.pathsep.join([gyp_pylib,
+                                            env.get('PYTHONPATH', '')])
   defines = [
     '-Dlibchromiumcontent_component={0}'.format(component),
     '-Dtarget_arch={0}'.format(target_arch),

script/upload.py

@@ -51,7 +51,15 @@ def main():
     return 1
 
   github = GitHub(auth_token())
-  release_id = create_or_get_release_draft(github, args.version)
+  releases = github.repos(ATOM_SHELL_REPO).releases.get()
+  tag_exists = False
+  for release in releases:
+    if release['tag_name'] == args.version:
+      tag_exists = True
+      break
+
+  release = create_or_get_release_draft(github, releases, args.version,
+                                        tag_exists)
 
   if args.publish_release:
     # Upload the SHASUMS.txt.
@@ -64,29 +72,27 @@ def main():
              os.path.join(SOURCE_ROOT, 'script', 'upload-index-json.py')])
 
     # Press the publish button.
-    publish_release(github, release_id)
+    publish_release(github, release['id'])
 
     # Do not upload other files when passed "-p".
     return
 
   # Upload atom-shell with GitHub Releases API.
-  upload_atom_shell(github, release_id, os.path.join(DIST_DIR, DIST_NAME))
+  upload_atom_shell(github, release, os.path.join(DIST_DIR, DIST_NAME))
-  upload_atom_shell(github, release_id, os.path.join(DIST_DIR, SYMBOLS_NAME))
+  upload_atom_shell(github, release, os.path.join(DIST_DIR, SYMBOLS_NAME))
 
   # Upload chromedriver and mksnapshot for minor version update.
   if get_target_arch() != 'arm' and parse_version(args.version)[2] == '0':
     chromedriver = 'chromedriver-{0}-{1}-{2}.zip'.format(
         get_chromedriver_version(), PLATFORM, get_target_arch())
-    upload_atom_shell(github, release_id,
+    upload_atom_shell(github, release, os.path.join(DIST_DIR, chromedriver))
-                      os.path.join(DIST_DIR, chromedriver))
+    upload_atom_shell(github, release, os.path.join(DIST_DIR, MKSNAPSHOT_NAME))
-    upload_atom_shell(github, release_id,
-                      os.path.join(DIST_DIR, MKSNAPSHOT_NAME))
 
-  if PLATFORM == 'win32':
+  if PLATFORM == 'win32' and not tag_exists:
     # Upload node headers.
     execute([sys.executable,
              os.path.join(SOURCE_ROOT, 'script', 'upload-node-headers.py'),
-             '-v', ATOM_SHELL_VERSION])
+             '-v', args.version])
 
 
 def parse_args():
@@ -100,6 +106,9 @@ def parse_args():
 
 
 def get_atom_shell_build_version():
+  if os.environ.has_key('CI'):
+    # In CI we just build as told.
+    return ATOM_SHELL_VERSION
   if PLATFORM == 'darwin':
     atom_shell = os.path.join(SOURCE_ROOT, 'out', 'R',
                               '{0}.app'.format(PRODUCT_NAME), 'Contents',
@@ -145,34 +154,49 @@ def get_text_with_editor(name):
   os.unlink(t.name)
   return text
 
-def create_or_get_release_draft(github, tag):
+def create_or_get_release_draft(github, releases, tag, tag_exists):
-  name = '{0} {1}'.format(PROJECT_NAME, tag)
+  # Search for existing draft.
-  releases = github.repos(ATOM_SHELL_REPO).releases.get()
   for release in releases:
-    # The untagged commit doesn't have a matching tag_name, so also check name.
+    if release['draft']:
-    if release['tag_name'] == tag or release['name'] == name:
+      return release
-      return release['id']
 
+  if tag_exists:
+    tag = 'do-not-publish-me'
   return create_release_draft(github, tag)
 
 
 def create_release_draft(github, tag):
-  name = '{0} {1}'.format(PROJECT_NAME, tag)
+  if os.environ.has_key('CI'):
-  body = get_text_with_editor(name)
+    name = '{0} pending draft'.format(PROJECT_NAME)
+    body = '(placeholder)'
+  else:
+    name = '{0} {1}'.format(PROJECT_NAME, tag)
+    body = get_text_with_editor(name)
   if body == '':
     sys.stderr.write('Quit due to empty release note.\n')
     sys.exit(0)
 
   data = dict(tag_name=tag, name=name, body=body, draft=True)
   r = github.repos(ATOM_SHELL_REPO).releases.post(data=data)
-  return r['id']
+  return r
 
 
-def upload_atom_shell(github, release_id, file_path):
+def upload_atom_shell(github, release, file_path):
+  # Delete the original file before uploading in CI.
+  if os.environ.has_key('CI'):
+    try:
+      for asset in release['assets']:
+        if asset['name'] == os.path.basename(file_path):
+          github.repos(ATOM_SHELL_REPO).releases.assets(asset['id']).delete()
+          break
+    except Exception:
+      pass
+
+  # Upload the file.
   params = {'name': os.path.basename(file_path)}
   headers = {'Content-Type': 'application/zip'}
   with open(file_path, 'rb') as f:
-    github.repos(ATOM_SHELL_REPO).releases(release_id).assets.post(
+    github.repos(ATOM_SHELL_REPO).releases(release['id']).assets.post(
         params=params, headers=headers, data=f, verify=False)
 
 

spec/api-browser-window-spec.coffee

@@ -206,7 +206,7 @@ describe 'browser-window module', ->
       w.loadUrl "file://#{fixtures}/pages/target-name.html"
 
   describe 'maximize event', ->
-    return if isCI and process.platform is 'linux'
+    return if isCI
     it 'emits when window is maximized', (done) ->
       @timeout 10000
       w.once 'maximize', -> done()
@@ -214,7 +214,7 @@ describe 'browser-window module', ->
       w.maximize()
 
   describe 'unmaximize event', ->
-    return if isCI and process.platform is 'linux'
+    return if isCI
     it 'emits when window is unmaximized', (done) ->
       @timeout 10000
       w.once 'unmaximize', -> done()
@@ -223,7 +223,7 @@ describe 'browser-window module', ->
       w.unmaximize()
 
   describe 'minimize event', ->
-    return if isCI and process.platform is 'linux'
+    return if isCI
     it 'emits when window is minimized', (done) ->
       @timeout 10000
       w.once 'minimize', -> done()

vendor/requests

@@ -0,0 +1 @@
+Subproject commit e4d59bedfd3c7f4f254f4f5d036587bcd8152458